diff --git a/.gitignore b/.gitignore index 3a82b85..ea17a21 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,11 @@ dist/ testdist/ # Logs -*.logz_bridge_gen.go\nz_entry.c\nbuild/\ndist/ +*.log + +# Generated bridge files z_bridge_gen.go z_entry.c -build/ -dist/ + +# Binary +plugindev diff --git a/example/ai_image/README.md b/example/ai_image/README.md new file mode 100644 index 0000000..dd7f750 --- /dev/null +++ b/example/ai_image/README.md @@ -0,0 +1,13 @@ +# ai_image + +ai_image plugin + +## Build + +```bash +plugindev build +``` + +## Install + +Upload the .hmap file through the Plugin Manager API. diff --git a/example/ai_image/go.mod b/example/ai_image/go.mod new file mode 100644 index 0000000..3349672 --- /dev/null +++ b/example/ai_image/go.mod @@ -0,0 +1,7 @@ +module ai_image + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0 + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone diff --git a/example/ai_image/plg.json b/example/ai_image/plg.json new file mode 100644 index 0000000..678af1e --- /dev/null +++ b/example/ai_image/plg.json @@ -0,0 +1,11 @@ +{ + "name": "ai_image", + "name_zh": "AI绘图", + "name_en": "AI Image", + "version": "1.0.0", + "description": "AI 图像生成插件，支持 OpenAI DALL·E / Stable Diffusion", + "author": "HomeAgent", + "entry": "plugin.so", + "tags": ["ai", "image", "draw", "generate"], + "targets": "linux/amd64" +} diff --git a/example/ai_image/plugin.go b/example/ai_image/plugin.go new file mode 100644 index 0000000..88a0ab0 --- /dev/null +++ b/example/ai_image/plugin.go @@ -0,0 +1,344 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +type Plugin struct { + name string + sdk *sdk.PluginSDK + client *http.Client + apiKey string + provider string + model string + size string +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T { + v, err := s.Get(key) + if err == nil && v != nil { + if sv, ok := v.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + v2, err2 := s.GetCore("plugin." + "ai_image" + "." + key) + if err2 == nil && v2 != nil { + if sv, ok := v2.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + return fallback +} + +func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T { + v, ok := args[key] + if !ok || v == nil { + return fallback + } + switch any(fallback).(type) { + case string: + if s, ok := v.(string); ok { + return any(s).(T) + } + case int64: + switch n := v.(type) { + case float64: + return any(int64(n)).(T) + case int64: + return any(n).(T) + case string: + if i, err := strconv.ParseInt(n, 10, 64); err == nil { + return any(i).(T) + } + } + case float64: + switch n := v.(type) { + case float64: + return any(n).(T) + case int64: + return any(float64(n)).(T) + case string: + if f, err := strconv.ParseFloat(n, 64); err == nil { + return any(f).(T) + } + } + } + return fallback +} + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + p.sdk = s + + p.apiKey = readCfg(s.Settings(), "api_key", "") + p.provider = readCfg(s.Settings(), "provider", "openai") + p.model = readCfg(s.Settings(), "model", "dall-e-3") + p.size = readCfg(s.Settings(), "size", "1024x1024") + + p.client = &http.Client{Timeout: 120 * time.Second} + + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.ai_image.api_key", Default: "", Type: "string", + DisplayName: "API Key", Description: "OpenAI / Stable Diffusion API Key", + Category: "ai_image", Secret: true, + }) + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.ai_image.provider", Default: "openai", Type: "string", + DisplayName: "Provider", Description: "Image generation provider: openai / stability", + Category: "ai_image", + }) + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.ai_image.model", Default: "dall-e-3", Type: "string", + DisplayName: "Model", Description: "Model name (dall-e-3, sd-xl, etc.)", + Category: "ai_image", + }) + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.ai_image.size", Default: "1024x1024", Type: "string", + DisplayName: "Size", Description: "Default image size (1024x1024, 1024x1792, 1792x1024)", + Category: "ai_image", + }) + + tp := p.name + "_" + s.RegisterTool(tp+"generate", sdk.ToolDef{ + Name: tp + "generate", Description: "Generate image from text prompt using AI. Returns image URL.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "prompt": map[string]interface{}{"type": "string", "description": "Text description of the image to generate"}, + "size": map[string]interface{}{"type": "string", "description": "Image size (1024x1024, 1024x1792, 1792x1024), default from config"}, + "model": map[string]interface{}{"type": "string", "description": "Model override (dall-e-3, dall-e-2)"}, + "n": map[string]interface{}{"type": "integer", "description": "Number of images to generate (1-10), default 1"}, + }, + "required": []string{"prompt"}, + }, + }, p.handleGenerate) + + fmt.Printf("[%s] started (provider=%s, model=%s)\n", p.name, p.provider, p.model) + return nil +} + +func (p *Plugin) Stop() error { + fmt.Printf("[%s] stopped\n", p.name) + return nil +} + +type openAIReq struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + N int `json:"n"` + Size string `json:"size"` + ResponseFormat string `json:"response_format"` +} + +type openAIResp struct { + Created int64 `json:"created"` + Data []struct { + RevisedPrompt string `json:"revised_prompt"` + URL string `json:"url"` + } `json:"data"` + Error *struct { + Message string `json:"message"` + Type string `json:"type"` + } `json:"error"` +} + +func (p *Plugin) handleGenerate(args map[string]interface{}) (interface{}, error) { + prompt := readArg(args, "prompt", "") + if prompt == "" { + return map[string]interface{}{"isError": true, "content": "prompt is required"}, nil + } + + p.apiKey = readCfg(p.sdk.Settings(), "api_key", p.apiKey) + if p.apiKey == "" { + return map[string]interface{}{"isError": true, "content": "API key not configured. Set plugin.ai_image.api_key via CLI."}, nil + } + + provider := readCfg(p.sdk.Settings(), "provider", p.provider) + model := readArg(args, "model", readCfg(p.sdk.Settings(), "model", p.model)) + size := readArg(args, "size", readCfg(p.sdk.Settings(), "size", p.size)) + n := readArg(args, "n", int64(1)) + if n < 1 { + n = 1 + } + if n > 10 { + n = 10 + } + + switch provider { + case "openai": + return p.generateOpenAI(prompt, model, size, int(n)) + case "stability": + return p.generateStability(prompt, model, size, int(n)) + default: + return map[string]interface{}{"isError": true, "content": "Unknown provider: " + provider + ". Supported: openai, stability"}, nil + } +} + +func (p *Plugin) generateOpenAI(prompt, model, size string, n int) (interface{}, error) { + body := openAIReq{ + Model: model, + Prompt: prompt, + N: n, + Size: size, + ResponseFormat: "url", + } + + b, _ := json.Marshal(body) + req, _ := http.NewRequest("POST", "https://api.openai.com/v1/images/generations", bytes.NewReader(b)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + + resp, err := p.client.Do(req) + if err != nil { + return map[string]interface{}{"isError": true, "content": "Request failed: " + err.Error()}, nil + } + defer resp.Body.Close() + + respBody, _ := io.ReadAll(resp.Body) + var result openAIResp + if err := json.Unmarshal(respBody, &result); err != nil { + return map[string]interface{}{"isError": true, "content": "Failed to parse response: " + err.Error()}, nil + } + + if result.Error != nil { + return map[string]interface{}{"isError": true, "content": "API error: " + result.Error.Message}, nil + } + + if len(result.Data) == 0 { + return map[string]interface{}{"isError": true, "content": "No images returned"}, nil + } + + urls := make([]string, len(result.Data)) + for i, d := range result.Data { + urls[i] = d.URL + } + + return map[string]interface{}{ + "content": fmt.Sprintf("Generated %d image(s) with model %s:\n%s", len(urls), model, strings.Join(urls, "\n")), + "images": urls, + "prompt": prompt, + "model": model, + }, nil +} + +type stabilityReq struct { + TextPrompts []stabilityPrompt `json:"text_prompts"` + Width int `json:"width"` + Height int `json:"height"` + Samples int `json:"samples"` +} + +type stabilityPrompt struct { + Text string `json:"text"` + Weight float64 `json:"weight,omitempty"` +} + +type stabilityArtifact struct { + Base64 string `json:"base64"` + Seed int `json:"seed"` +} + +type stabilityResp struct { + Artifacts []stabilityArtifact `json:"artifacts"` + Message string `json:"message,omitempty"` +} + +func (p *Plugin) generateStability(prompt, model, size string, n int) (interface{}, error) { + width, height := 1024, 1024 + if parts := strings.Split(size, "x"); len(parts) == 2 { + if w, err := strconv.Atoi(parts[0]); err == nil { + width = w + } + if h, err := strconv.Atoi(parts[1]); err == nil { + height = h + } + } + + body := stabilityReq{ + TextPrompts: []stabilityPrompt{{Text: prompt, Weight: 1.0}}, + Width: width, + Height: height, + Samples: n, + } + + apiURL := "https://api.stability.ai/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image" + + b, _ := json.Marshal(body) + req, _ := http.NewRequest("POST", apiURL, bytes.NewReader(b)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + req.Header.Set("Accept", "application/json") + + resp, err := p.client.Do(req) + if err != nil { + return map[string]interface{}{"isError": true, "content": "Request failed: " + err.Error()}, nil + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + respBody, _ := io.ReadAll(resp.Body) + return map[string]interface{}{"isError": true, "content": fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(respBody))}, nil + } + + respBody, _ := io.ReadAll(resp.Body) + var result stabilityResp + if err := json.Unmarshal(respBody, &result); err != nil { + return map[string]interface{}{"isError": true, "content": "Failed to parse response: " + err.Error()}, nil + } + + if len(result.Artifacts) == 0 { + msg := result.Message + if msg == "" { + msg = "No images returned" + } + return map[string]interface{}{"isError": true, "content": msg}, nil + } + + urls := make([]string, len(result.Artifacts)) + for i, a := range result.Artifacts { + urls[i] = "data:image/png;base64," + a.Base64 + } + + return map[string]interface{}{ + "content": fmt.Sprintf("Generated %d image(s) via Stability AI:\n%s", len(urls), strings.Join(urls, "\n")), + "images": urls, + "prompt": prompt, + "model": model, + }, nil +} diff --git a/example/browser/go.mod b/example/browser/go.mod index 45d9c1a..f56bb87 100644 --- a/example/browser/go.mod +++ b/example/browser/go.mod @@ -1,7 +1,21 @@ -module browser +module browser-plugin go 1.25.0 -require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0-20260708004841-e9bdcf9304b0 +require ( + gitcode.com/JianFeeeee/homeagent-sdk v0.0.0 + github.com/chromedp/chromedp v0.9.5 +) -replace gitcode.com/JianFeeeee/homeagent-sdk => ../../. +require ( + github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect + github.com/chromedp/sysutil v1.0.0 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.3.2 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + golang.org/x/sys v0.16.0 // indirect +) + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone diff --git a/example/browser/go.sum b/example/browser/go.sum new file mode 100644 index 0000000..0744849 --- /dev/null +++ b/example/browser/go.sum @@ -0,0 +1,23 @@ +github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 h1:XYUCaZrW8ckGWlCRJKCSoh/iFwlpX316a8yY9IFEzv8= +github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg= +github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y= +github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q= +github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/example/browser/plg.json b/example/browser/plg.json index 52c862f..0cbb719 100644 --- a/example/browser/plg.json +++ b/example/browser/plg.json @@ -1,11 +1,11 @@ { "name": "browser", "name_zh": "浏览器", - "name_en": "browser", - "version": "1.0.0", - "description": "网络资源搜索与获取：搜索引擎查询（browser_search）、网页抓取（browser_fetch，SSRF防护）、无头浏览器渲染（browser_render）", + "name_en": "Browser", + "version": "2.0.0", + "description": "统一浏览器插件：搜索、HTTP抓取(quick)、无头渲染(normal)、交互式浏览器(interactive/CDP)", "author": "HomeAgent", "entry": "plugin.so", - "tags": ["web", "search", "fetch", "browser"], + "tags": ["web", "search", "fetch", "browser", "cdp"], "targets": "linux/amd64" } diff --git a/example/browser/plugin.go b/example/browser/plugin.go index 4bc1ac5..33e9879 100644 --- a/example/browser/plugin.go +++ b/example/browser/plugin.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -12,21 +14,122 @@ import ( "os" "os/exec" "regexp" + "strconv" "strings" "sync" "time" "unicode" - "gitcode.com/JianFeeeee/homeagent-sdk/sdk" + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" + + "github.com/chromedp/chromedp" ) type Plugin struct { - name string - sdk *sdk.PluginSDK - mu sync.RWMutex - timeout int - proxy string - client *http.Client + name string + sdk *sdk.PluginSDK + mu sync.RWMutex + timeout int + proxy string + dataDir string + client *http.Client + + sessions map[string]*BrowserSession + nextID int + wg sync.WaitGroup + stopCh chan struct{} +} + +type BrowserSession struct { + id string + allocCtx context.Context + cancel context.CancelFunc + ctx context.Context + createdAt time.Time + timeout time.Duration + closed bool + mu sync.Mutex + currentURL string +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name, stopCh: make(chan struct{}), sessions: make(map[string]*BrowserSession)}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T { + v, err := s.Get(key) + if err == nil && v != nil { + if s, ok := v.(string); ok && s != "" { + switch any(fallback).(type) { + case string: + return any(s).(T) + case int64: + if n, err := strconv.ParseInt(s, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(s, 64); err == nil { + return any(n).(T) + } + } + } + } + v2, err2 := s.GetCore("plugin." + "browser" + "." + key) + if err2 == nil && v2 != nil { + if s, ok := v2.(string); ok && s != "" { + switch any(fallback).(type) { + case string: + return any(s).(T) + case int64: + if n, err := strconv.ParseInt(s, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(s, 64); err == nil { + return any(n).(T) + } + } + } + } + return fallback +} + +func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T { + v, ok := args[key] + if !ok || v == nil { + return fallback + } + switch any(fallback).(type) { + case string: + if s, ok := v.(string); ok { + return any(s).(T) + } + case int64: + switch val := v.(type) { + case float64: + return any(int64(val)).(T) + case string: + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + return any(n).(T) + } + } + case float64: + switch val := v.(type) { + case float64: + return any(val).(T) + case string: + if n, err := strconv.ParseFloat(val, 64); err == nil { + return any(n).(T) + } + } + } + return fallback +} + +func errResult(msg string) map[string]interface{} { + return map[string]interface{}{"isError": true, "content": msg} } func newHTTPClient(timeout int, proxyURL string) *http.Client { @@ -39,8 +142,7 @@ func newHTTPClient(timeout int, proxyURL string) *http.Client { ResponseHeaderTimeout: time.Duration(timeout) * time.Second, } if proxyURL != "" { - u, err := url.Parse(proxyURL) - if err == nil { + if u, err := url.Parse(proxyURL); err == nil { transport.Proxy = http.ProxyURL(u) } } @@ -54,106 +156,219 @@ func newHTTPClient(timeout int, proxyURL string) *http.Client { return nil }, } + } -func (p *Plugin) Name() string { return p.name } +// ── Start / Stop ────────────────────────────────────────── func (p *Plugin) Start(s *sdk.PluginSDK) error { - s.SetAutoRestart(true) p.sdk = s + s.SetAutoRestart(true) s.Settings().RegisterDef(sdk.ConfigDef{ - Key: "plugin.browser.timeout", Default: "30", Type: "int", + Key: "timeout", Default: "30", Type: "int", DisplayName: "HTTP 超时(秒)", Description: "HTTP 请求超时时间", Category: "browser", }) s.Settings().RegisterDef(sdk.ConfigDef{ - Key: "plugin.browser.proxy", Default: "", Type: "string", - DisplayName: "HTTP 代理", Description: "HTTP 代理地址，如 http://proxy:port。为空则不使用代理", + Key: "proxy", Default: "", Type: "string", + DisplayName: "HTTP 代理", Description: "HTTP 代理地址，如 http://proxy:port", + Category: "browser", + }) + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "data_dir", Default: "", Type: "string", + DisplayName: "浏览器数据目录", Description: "Chromium 用户数据目录路径（持久化 cookies/登录状态）。留空则每次启动临时目录。", Category: "browser", }) - t := getSetting[float64](s.Settings(), "timeout", 30) - p.timeout = int(t) - if p.timeout < 5 { p.timeout = 5 } - if p.timeout > 120 { p.timeout = 120 } - p.proxy = getSetting[string](s.Settings(), "proxy", "") + t := readCfg(s.Settings(), "timeout", float64(30)) + p.timeout = int(t) + if p.timeout < 5 { + p.timeout = 5 + } + if p.timeout > 120 { + p.timeout = 120 + } + p.proxy = readCfg(s.Settings(), "proxy", "") + p.dataDir = readCfg(s.Settings(), "data_dir", "") + if p.dataDir != "" { + os.MkdirAll(p.dataDir, 0700) + } p.client = newHTTPClient(p.timeout, p.proxy) tp := p.name + "_" s.RegisterTool(tp+"search", sdk.ToolDef{ - Name: tp + "search", - Description: "Search the web for current information using DuckDuckGo. Returns formatted results with titles, URLs, and snippets.", + Name: tp + "search", + Description: "使用 Bing 搜索网页。返回标题、URL 和摘要。", Parameters: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "query": map[string]interface{}{"type": "string", "description": "Search query"}, - "count": map[string]interface{}{"type": "integer", "description": "Number of results (1-20, default 5)"}, + "query": map[string]interface{}{"type": "string", "description": "搜索关键词"}, + "count": map[string]interface{}{"type": "integer", "description": "结果数量(1-20，默认5)"}, }, "required": []string{"query"}, }, }, p.handleSearch) s.RegisterTool(tp+"fetch", sdk.ToolDef{ - Name: tp + "fetch", - Description: "Fetch a URL and extract readable content as markdown-like text. Blocked on private/internal IPs.", + Name: tp + "fetch", + Description: "快速抓取 URL 内容(quick 模式)。纯 HTTP 请求，不支持 JS 渲染，有 SSRF 防护。", Parameters: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "url": map[string]interface{}{"type": "string", "description": "HTTP/HTTPS URL to fetch"}, - "max_chars": map[string]interface{}{"type": "integer", "description": "Max characters to return (default 20000)"}, + "url": map[string]interface{}{"type": "string", "description": "HTTP/HTTPS URL"}, + "max_chars": map[string]interface{}{"type": "integer", "description": "最大返回字符数(默认20000)"}, }, "required": []string{"url"}, }, }, p.handleFetch) s.RegisterTool(tp+"render", sdk.ToolDef{ - Name: tp + "render", - Description: "Render a web page using headless Chromium browser and extract the text content. Supports JavaScript-rendered pages. Returns title and first 5000 characters.", + Name: tp + "render", + Description: "无头 Chromium 渲染网页并提取文本(normal 模式)。支持 JS 渲染的页面。", Parameters: map[string]interface{}{ "type": "object", "properties": map[string]interface{}{ - "url": map[string]interface{}{"type": "string", "description": "URL to render"}, - "wait": map[string]interface{}{"type": "integer", "description": "Seconds to wait for JS rendering (default 0)"}, + "url": map[string]interface{}{"type": "string", "description": "URL"}, + "wait": map[string]interface{}{"type": "integer", "description": "等待 JS 渲染的秒数(默认0)"}, }, "required": []string{"url"}, }, }, p.handleRender) + s.RegisterTool(tp+"start", sdk.ToolDef{ + Name: tp + "start", + Description: "启动交互式浏览器会话(interactive 模式)。通过 CDP 连接 Chromium，支持导航、截图、点击、输入等操作。返回会话 ID。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "url": map[string]interface{}{"type": "string", "description": "初始导航 URL（可选）"}, + "timeout": map[string]interface{}{"type": "string", "description": "会话超时(如 5m, 10m，默认 10m)"}, + }, + }, + }, p.handleBrowserStart) + + s.RegisterTool(tp+"navigate", sdk.ToolDef{ + Name: tp + "navigate", + Description: "在交互式浏览器中导航到指定 URL。自动等待页面 body 加载完成。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "url": map[string]interface{}{"type": "string", "description": "目标 URL"}, + "wait": map[string]interface{}{"type": "integer", "description": "页面加载后额外等待秒数(默认2，反爬页面建议5)"}, + }, + "required": []string{"id", "url"}, + }, + }, p.handleNavigate) + + s.RegisterTool(tp+"screenshot", sdk.ToolDef{ + Name: tp + "screenshot", + Description: "对交互式浏览器当前页面截图。返回 base64 编码的 PNG 图片。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "full": map[string]interface{}{"type": "boolean", "description": "是否全页截图(默认 false，仅视口)"}, + "format": map[string]interface{}{"type": "string", "description": "图片格式: png 或 jpeg(默认 png)"}, + }, + "required": []string{"id"}, + }, + }, p.handleScreenshot) + + s.RegisterTool(tp+"html", sdk.ToolDef{ + Name: tp + "html", + Description: "获取交互式浏览器当前页面 JS 渲染后的完整 HTML。用于模型分析页面结构、定位元素。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "max_chars": map[string]interface{}{"type": "integer", "description": "最大返回字符数(默认50000)"}, + }, + "required": []string{"id"}, + }, + }, p.handleHTML) + + s.RegisterTool(tp+"click", sdk.ToolDef{ + Name: tp + "click", + Description: "在交互式浏览器中点击元素。自动等待元素可见后再点击。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "selector": map[string]interface{}{"type": "string", "description": "CSS 选择器"}, + "wait": map[string]interface{}{"type": "integer", "description": "等待元素出现的超时毫秒数(默认3000)"}, + }, + "required": []string{"id", "selector"}, + }, + }, p.handleClick) + + s.RegisterTool(tp+"type", sdk.ToolDef{ + Name: tp + "type", + Description: "在交互式浏览器中向输入框输入文字。自动等待元素可见、聚焦后清空再输入。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "selector": map[string]interface{}{"type": "string", "description": "CSS 选择器"}, + "text": map[string]interface{}{"type": "string", "description": "要输入的文字"}, + "submit": map[string]interface{}{"type": "boolean", "description": "输入后按回车(默认 false)"}, + "wait": map[string]interface{}{"type": "integer", "description": "等待元素出现的超时毫秒数(默认3000)"}, + }, + "required": []string{"id", "selector", "text"}, + }, + }, p.handleType) + + s.RegisterTool(tp+"scroll", sdk.ToolDef{ + Name: tp + "scroll", + Description: "在交互式浏览器中滚动页面。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + "dir": map[string]interface{}{"type": "string", "description": "方向: up, down, left, right(默认 down)"}, + "amount": map[string]interface{}{"type": "integer", "description": "滚动像素数(默认 500)"}, + }, + "required": []string{"id"}, + }, + }, p.handleScroll) + + s.RegisterTool(tp+"close", sdk.ToolDef{ + Name: tp + "close", + Description: "关闭交互式浏览器会话，释放资源。", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "浏览器会话 ID"}, + }, + "required": []string{"id"}, + }, + }, p.handleBrowserClose) + + p.wg.Add(1) + go p.cleanupLoop() + log.Printf("[%s] started, timeout=%ds proxy=%q", p.name, p.timeout, p.proxy) return nil } func (p *Plugin) Stop() error { + close(p.stopCh) + p.wg.Wait() if p.client != nil { p.client.CloseIdleConnections() } + p.mu.Lock() + for _, s := range p.sessions { + s.Close() + } + p.sessions = nil + p.mu.Unlock() log.Printf("[%s] stopped", p.name) return nil } -func getSetting[T any](s sdk.SettingsAPI, key string, def T) T { - v, err := s.Get(key) - if err != nil || v == nil { return def } - val, ok := v.(T) - if !ok { return def } - return val -} - -func convInt64(v interface{}) (int64, error) { - switch x := v.(type) { - case float64: return int64(x), nil - case int64: return x, nil - case json.Number: return x.Int64() - default: return 0, fmt.Errorf("cannot convert %T to int64", v) - } -} - -func errorResult(msg string) map[string]interface{} { - return map[string]interface{}{"isError": true, "content": msg} -} - // ── SSRF ────────────────────────────────────────────────── var privateCIDRs []*net.IPNet @@ -165,25 +380,33 @@ func init() { "::1/128", "fc00::/7", "fe80::/10", } { _, n, _ := net.ParseCIDR(c) - if n != nil { privateCIDRs = append(privateCIDRs, n) } + if n != nil { + privateCIDRs = append(privateCIDRs, n) + } } } func isPrivateIP(ip net.IP) bool { for _, n := range privateCIDRs { - if n.Contains(ip) { return true } + if n.Contains(ip) { + return true + } } return false } func (p *Plugin) ssrfCheck(rawURL string) error { u, err := url.Parse(rawURL) - if err != nil { return fmt.Errorf("invalid URL: %w", err) } + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } if u.Scheme != "http" && u.Scheme != "https" { - return fmt.Errorf("only http/https URLs allowed, got: %s", u.Scheme) + return fmt.Errorf("only http/https allowed, got: %s", u.Scheme) } ips, err := net.LookupHost(u.Hostname()) - if err != nil { return fmt.Errorf("DNS lookup failed: %w", err) } + if err != nil { + return fmt.Errorf("DNS lookup failed: %w", err) + } for _, ip := range ips { if parsed := net.ParseIP(ip); parsed != nil && isPrivateIP(parsed) { return fmt.Errorf("blocked request to private IP: %s (%s)", u.Hostname(), ip) @@ -192,95 +415,77 @@ func (p *Plugin) ssrfCheck(rawURL string) error { return nil } -// ── DuckDuckGo Search ───────────────────────────────────── +// ── Bing Search ─────────────────────────────────────────── -type ddgResult struct { +type searchResult struct { Title, URL, Snippet string } -func (p *Plugin) ddgSearch(query string, count int) ([]ddgResult, error) { - form := url.Values{"q": {query}} - req, _ := http.NewRequest("POST", "https://html.duckduckgo.com/html/", strings.NewReader(form.Encode())) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") +func (p *Plugin) bingSearch(query string, count int) ([]searchResult, error) { + u := fmt.Sprintf("https://www.bing.com/search?q=%s&count=%d", url.QueryEscape(query), count) + req, _ := http.NewRequest("GET", u, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") resp, err := p.client.Do(req) - if err != nil { return nil, fmt.Errorf("request failed: %w", err) } + if err != nil { + return nil, fmt.Errorf("request failed: %w", err) + } defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) - return parseDDGResults(string(body), count), nil + return parseBingResults(string(body), count), nil } -func parseDDGResults(html string, count int) []ddgResult { - var results []ddgResult - marker := `result__body"` - for i := 0; i < len(html); i++ { - idx := strings.Index(html[i:], marker) - if idx < 0 { break } - i += idx - closeIdx := findClosingTag(html, i, "") - if closeIdx < 0 { break } - if r := parseSingleDDGResult(html[i : closeIdx+6]); r.URL != "" { - results = append(results, r) - if len(results) >= count { break } +func parseBingResults(html string, count int) []searchResult { + var results []searchResult + re := regexp.MustCompile(`
  • `) + matches := re.FindAllStringSubmatch(html, -1) + for _, m := range matches { + if len(results) >= count { + break + } + block := m[1] + var r searchResult + hrefRe := regexp.MustCompile(`]+href="([^"]+)"[^>]*>`) + if hm := hrefRe.FindStringSubmatch(block); len(hm) > 1 { + r.URL = hm[1] + } + titleRe := regexp.MustCompile(`]+href="[^"]+"[^>]*>(.*?)`) + if tm := titleRe.FindStringSubmatch(block); len(tm) > 1 { + r.Title = stripTags(tm[1]) + } + snipRe := regexp.MustCompile(`
    .*?

    (.*?)

    `) + if sm := snipRe.FindStringSubmatch(block); len(sm) > 1 { + r.Snippet = stripTags(sm[1]) + } + if r.URL != "" && r.Title != "" { + results = append(results, r) } - i = closeIdx + 6 } return results } -func findClosingTag(s string, start int, tag string) int { - depth := 1 - for pos := start; pos < len(s); { - nextOpen := strings.Index(s[pos:], `= 0 && nextOpen < nextClose { - depth++ - pos += nextOpen + 4 - } else { - depth-- - if depth == 0 { return pos + nextClose } - pos += nextClose + len(tag) - } - } - return -1 -} - -func parseSingleDDGResult(block string) ddgResult { - var r ddgResult - urlMarker := `class="result__a" href="` - if uIdx := strings.Index(block, urlMarker); uIdx >= 0 { - start := uIdx + len(urlMarker) - if end := strings.Index(block[start:], `"`); end >= 0 { - r.URL = block[start : start+end] - } - } - for _, marker := range []string{`= 0 { - if aStart := strings.Index(block[sIdx:], `>`); aStart >= 0 { - snipStart := sIdx + aStart + 1 - snipEnd := strings.Index(block[snipStart:], ``) - if snipEnd < 0 { snipEnd = strings.Index(block[snipStart:], `
    `) } - if snipEnd >= 0 { r.Snippet = stripTags(block[snipStart : snipStart+snipEnd]) } - } - break - } - } - return r -} - func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) { - query, _ := args["query"].(string) - if query == "" { return errorResult("query is required"), nil } + query := readArg(args, "query", "") + if query == "" { + return errResult("query is required"), nil + } count := 5 - if v, ok := args["count"].(float64); ok && v > 0 { count = int(v) } - if count < 1 { count = 1 } - if count > 20 { count = 20 } - - results, err := p.ddgSearch(query, count) - if err != nil { return errorResult("search failed: " + err.Error()), nil } - if len(results) == 0 { return map[string]interface{}{"content": "No results found."}, nil } - + if v, ok := args["count"].(float64); ok && v > 0 { + count = int(v) + } + if count < 1 { + count = 1 + } + if count > 20 { + count = 20 + } + results, err := p.bingSearch(query, count) + if err != nil { + return errResult("search failed: " + err.Error()), nil + } + if len(results) == 0 { + return map[string]interface{}{"content": "No results found."}, nil + } var sb strings.Builder sb.WriteString(fmt.Sprintf("Search results for %q:\n\n", query)) for i, r := range results { @@ -289,16 +494,20 @@ func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) return map[string]interface{}{"content": strings.TrimSpace(sb.String())}, nil } -// ── Web Fetch ───────────────────────────────────────────── +// ── Web Fetch (quick) ────────────────────────────────────── func htmlToText(html string) string { for _, tag := range []string{"" for { start := strings.Index(strings.ToLower(html), tag) - if start < 0 { break } + if start < 0 { + break + } end := strings.Index(html[start:], closing) - if end < 0 { break } + if end < 0 { + break + } html = html[:start] + html[start+end+len(closing):] } } @@ -312,19 +521,26 @@ func htmlToText(html string) string { } { html = strings.ReplaceAll(html, pair[0], pair[1]) } - lines := strings.Split(html, "\n") var cleaned []string for _, line := range lines { line = strings.TrimSpace(line) - if line == "" { continue } + if line == "" { + continue + } in := []rune(line) var out []rune space := false for _, r := range in { if unicode.IsSpace(r) { - if !space { out = append(out, ' '); space = true } - } else { out = append(out, r); space = false } + if !space { + out = append(out, ' ') + space = true + } + } else { + out = append(out, r) + space = false + } } cleaned = append(cleaned, string(out)) } @@ -335,63 +551,89 @@ func stripTags(s string) string { var out strings.Builder inTag := false for _, r := range s { - if r == '<' { inTag = true; continue } - if r == '>' { inTag = false; continue } - if !inTag { out.WriteRune(r) } + if r == '<' { + inTag = true + continue + } + if r == '>' { + inTag = false + continue + } + if !inTag { + out.WriteRune(r) + } } return out.String() } func (p *Plugin) handleFetch(args map[string]interface{}) (interface{}, error) { - rawURL, _ := args["url"].(string) - if rawURL == "" { return errorResult("url is required"), nil } + rawURL := readArg(args, "url", "") + if rawURL == "" { + return errResult("url is required"), nil + } maxChars := 20000 - if v, ok := args["max_chars"].(float64); ok && v > 0 { maxChars = int(v) } - if maxChars > 500000 { maxChars = 500000 } - if err := p.ssrfCheck(rawURL); err != nil { return errorResult(err.Error()), nil } - + if v, ok := args["max_chars"].(float64); ok && v > 0 { + maxChars = int(v) + } + if maxChars > 500000 { + maxChars = 500000 + } + if err := p.ssrfCheck(rawURL); err != nil { + return errResult(err.Error()), nil + } req, _ := http.NewRequest("GET", rawURL, nil) req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") resp, err := p.client.Do(req) - if err != nil { return errorResult("fetch failed: " + err.Error()), nil } + if err != nil { + return errResult("fetch failed: " + err.Error()), nil + } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return errorResult(fmt.Sprintf("HTTP %d: %s", resp.StatusCode, resp.Status)), nil + return errResult(fmt.Sprintf("HTTP %d: %s", resp.StatusCode, resp.Status)), nil } - body, _ := io.ReadAll(io.LimitReader(resp.Body, int64(maxChars)+50000)) rawText := string(body) ct := resp.Header.Get("Content-Type") - var extracted string if strings.Contains(ct, "text/html") { extracted = htmlToText(rawText) } else if strings.Contains(ct, "application/json") { var v interface{} if json.Unmarshal(body, &v) == nil { - if pretty, err := json.MarshalIndent(v, "", " "); err == nil { extracted = string(pretty) } + if pretty, err := json.MarshalIndent(v, "", " "); err == nil { + extracted = string(pretty) + } } - if extracted == "" { extracted = rawText } - } else { extracted = rawText } - + if extracted == "" { + extracted = rawText + } + } else { + extracted = rawText + } extracted = strings.TrimSpace(extracted) - if len(extracted) > maxChars { extracted = extracted[:maxChars] + "\n\n[Content truncated]" } - if extracted == "" { extracted = "(empty content)" } - + if len(extracted) > maxChars { + extracted = extracted[:maxChars] + "\n\n[Content truncated]" + } + if extracted == "" { + extracted = "(empty content)" + } return map[string]interface{}{ "content": extracted, "details": map[string]interface{}{"url": rawURL, "status": resp.StatusCode, "content_type": ct}, }, nil } -// ── Chromium Render ─────────────────────────────────────── +// ── Chromium Render (normal) ────────────────────────────── func (p *Plugin) handleRender(args map[string]interface{}) (interface{}, error) { - rawURL, _ := args["url"].(string) - if rawURL == "" { return nil, fmt.Errorf("url is required") } - waitSec, _ := convInt64(args["wait"]) - if waitSec > 0 { time.Sleep(time.Duration(waitSec) * time.Second) } - + rawURL := readArg(args, "url", "") + if rawURL == "" { + return errResult("url is required"), nil + } + waitSec := int64(readArg(args, "wait", float64(0))) + if waitSec > 0 { + time.Sleep(time.Duration(waitSec) * time.Second) + } var html string chromiumPath := "/usr/local/bin/chromium" if _, err := os.Stat(chromiumPath); err == nil { @@ -399,49 +641,341 @@ func (p *Plugin) handleRender(args map[string]interface{}) (interface{}, error) cmd := exec.Command(chromiumPath, "--headless", "--disable-gpu", "--no-sandbox", "--dump-dom", rawURL) cmd.Stdout = &out if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("chromium: %w", err) + return errResult("chromium: " + err.Error()), nil } html = out.String() } else { resp, err := http.Get(rawURL) - if err != nil { return nil, fmt.Errorf("http get: %w", err) } + if err != nil { + return errResult("http get: " + err.Error()), nil + } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { return nil, fmt.Errorf("read body: %w", err) } + body, _ := io.ReadAll(resp.Body) html = string(body) } - title := "" if m := regexp.MustCompile(`([^<]+)`).FindStringSubmatch(html); len(m) > 1 { title = m[1] } - - var textOut bytes.Buffer - pyCmd := exec.Command("python3", "-c", ` -import sys, re, html -raw = sys.stdin.read() -text = re.sub(r'<[^>]+>', ' ', raw) -text = re.sub(r'\s+', ' ', text).strip() -text = html.unescape(text) -sys.stdout.write(text) -`) - pyCmd.Stdin = strings.NewReader(html) - pyCmd.Stdout = &textOut - pyCmd.Run() - text := strings.TrimSpace(textOut.String()) - + text := htmlToText(html) origLen := len(text) truncated := origLen > 5000 - if truncated { text = text[:5000] } - + if truncated { + text = text[:5000] + } result := "" - if title != "" { result = fmt.Sprintf("标题: %s\nURL: %s\n\n", title, rawURL) } + if title != "" { + result = fmt.Sprintf("标题: %s\nURL: %s\n\n", title, rawURL) + } result += text - if truncated { result += fmt.Sprintf("\n\n...（内容过长，仅显示前 5000 字符，共 %d 字符）", origLen) } - + if truncated { + result += fmt.Sprintf("\n\n...(仅显示前 5000 字符，共 %d 字符)", origLen) + } return map[string]interface{}{"content": result, "title": title}, nil } -func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { - return &Plugin{name: name}, nil +// ── Interactive Browser Session (CDP) ───────────────────── + +func (p *Plugin) handleBrowserStart(args map[string]interface{}) (interface{}, error) { + timeoutStr := readArg(args, "timeout", "10m") + timeout, err := time.ParseDuration(timeoutStr) + if err != nil { + timeout = 10 * time.Minute + } + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("no-sandbox", true), + chromedp.WindowSize(1280, 800), + ) + if p.dataDir != "" { + opts = append(opts, chromedp.Flag("user-data-dir", p.dataDir)) + } + if p.proxy != "" { + opts = append(opts, chromedp.Flag("proxy-server", p.proxy)) + } + + allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) + ctx, _ := chromedp.NewContext(allocCtx) + + session := &BrowserSession{ + allocCtx: allocCtx, + cancel: cancel, + ctx: ctx, + createdAt: time.Now(), + timeout: timeout, + } + + p.mu.Lock() + p.nextID++ + id := fmt.Sprintf("browser_%d", p.nextID) + session.id = id + p.sessions[id] = session + p.mu.Unlock() + + initURL := readArg(args, "url", "") + if initURL != "" { + if err := chromedp.Run(ctx, + chromedp.Navigate(initURL), + chromedp.WaitReady("body"), + ); err != nil { + session.Close() + p.mu.Lock() + delete(p.sessions, id) + p.mu.Unlock() + return errResult("navigate failed: " + err.Error()), nil + } + session.currentURL = initURL + p.sdk.InjectText(p.name, p.name, fmt.Sprintf("[浏览器 %s 已打开 %s]", id, initURL)) + } + + log.Printf("[%s] created browser session %s: url=%s timeout=%v", p.name, id, initURL, timeout) + return map[string]interface{}{ + "id": id, + "status": "created", + "url": initURL, + "timeout": timeout.String(), + }, nil +} + +func (p *Plugin) getSession(id string) (*BrowserSession, error) { + p.mu.Lock() + s, ok := p.sessions[id] + p.mu.Unlock() + if !ok { + return nil, fmt.Errorf("浏览器会话 %s 不存在或已关闭", id) + } + return s, nil +} + +func (p *Plugin) handleNavigate(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + rawURL := readArg(args, "url", "") + if id == "" || rawURL == "" { + return errResult("id 和 url 不能为空"), nil + } + s, err := p.getSession(id) + if err != nil { + return errResult(err.Error()), nil + } + waitSec := int64(readArg(args, "wait", float64(2))) + ctx, cancel := context.WithTimeout(s.ctx, time.Duration(p.timeout+10)*time.Second) + defer cancel() + if err := chromedp.Run(ctx, + chromedp.Navigate(rawURL), + chromedp.WaitReady("body"), + chromedp.Sleep(time.Duration(waitSec)*time.Second), + ); err != nil { + return errResult("navigate failed: " + err.Error()), nil + } + s.currentURL = rawURL + p.sdk.InjectText(p.name, p.name, fmt.Sprintf("[浏览器 %s 已导航到 %s]", id, rawURL)) + return map[string]interface{}{"status": "ok", "url": rawURL}, nil +} + +func (p *Plugin) handleScreenshot(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return errResult("id is required"), nil + } + s, sessErr := p.getSession(id) + if sessErr != nil { + return errResult(sessErr.Error()), nil + } + full := false + if v, ok := args["full"].(bool); ok { + full = v + } + format := readArg(args, "format", "png") + var buf []byte + var err error + if full { + err = chromedp.Run(s.ctx, chromedp.FullScreenshot(&buf, 90)) + } else { + err = chromedp.Run(s.ctx, chromedp.Screenshot("body", &buf)) + } + if err != nil { + return errResult("screenshot failed: " + err.Error()), nil + } + b64 := base64.StdEncoding.EncodeToString(buf) + return map[string]interface{}{ + "status": "ok", + "format": format, + "size": len(buf), + "base64": b64, + "data_uri": fmt.Sprintf("data:image/%s;base64,%s", format, b64), + }, nil +} + +func (p *Plugin) handleHTML(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return errResult("id is required"), nil + } + s, err := p.getSession(id) + if err != nil { + return errResult(err.Error()), nil + } + maxChars := 50000 + if v, ok := args["max_chars"].(float64); ok && v > 0 { + maxChars = int(v) + } + var html string + if err := chromedp.Run(s.ctx, chromedp.OuterHTML("html", &html)); err != nil { + return errResult("get html failed: " + err.Error()), nil + } + var title, currentURL string + chromedp.Run(s.ctx, + chromedp.Title(&title), + chromedp.Location(¤tURL), + ) + truncated := len(html) > maxChars + if truncated { + html = html[:maxChars] + "\n\n[HTML truncated]" + } + return map[string]interface{}{ + "status": "ok", + "title": title, + "url": currentURL, + "html": html, + "length": len(html), + }, nil +} + +func (p *Plugin) handleClick(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + selector := readArg(args, "selector", "") + if id == "" || selector == "" { + return errResult("id 和 selector 不能为空"), nil + } + s, err := p.getSession(id) + if err != nil { + return errResult(err.Error()), nil + } + waitMs := int64(readArg(args, "wait", float64(3000))) + ctx, cancel := context.WithTimeout(s.ctx, time.Duration(waitMs)*time.Millisecond) + defer cancel() + if err := chromedp.Run(ctx, + chromedp.WaitVisible(selector), + chromedp.Click(selector), + ); err != nil { + return errResult("click failed (element may not exist or page blocking): " + err.Error()), nil + } + return map[string]interface{}{"status": "ok", "selector": selector}, nil +} + +func (p *Plugin) handleType(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + selector := readArg(args, "selector", "") + text := readArg(args, "text", "") + if id == "" || selector == "" || text == "" { + return errResult("id, selector, text 不能为空"), nil + } + s, err := p.getSession(id) + if err != nil { + return errResult(err.Error()), nil + } + waitMs := int64(readArg(args, "wait", float64(3000))) + ctx, cancel := context.WithTimeout(s.ctx, time.Duration(waitMs)*time.Millisecond) + defer cancel() + actions := []chromedp.Action{ + chromedp.WaitVisible(selector), + chromedp.Click(selector, chromedp.NodeVisible), + chromedp.Clear(selector), + chromedp.SendKeys(selector, text), + } + submit := false + if v, ok := args["submit"].(bool); ok { + submit = v + } + if submit { + actions = append(actions, chromedp.SendKeys(selector, "\r")) + } + if err := chromedp.Run(ctx, actions...); err != nil { + return errResult("type failed (element may not exist or page blocking): " + err.Error()), nil + } + return map[string]interface{}{"status": "ok", "selector": selector}, nil +} + +func (p *Plugin) handleScroll(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return errResult("id is required"), nil + } + s, err := p.getSession(id) + if err != nil { + return errResult(err.Error()), nil + } + dir := readArg(args, "dir", "down") + amount := readArg(args, "amount", float64(500)) + var scrollJS string + switch dir { + case "up": + scrollJS = fmt.Sprintf("window.scrollBy(0, -%d)", int(amount)) + case "down": + scrollJS = fmt.Sprintf("window.scrollBy(0, %d)", int(amount)) + case "left": + scrollJS = fmt.Sprintf("window.scrollBy(-%d, 0)", int(amount)) + case "right": + scrollJS = fmt.Sprintf("window.scrollBy(%d, 0)", int(amount)) + default: + return errResult("dir 必须是 up/down/left/right"), nil + } + if err := chromedp.Run(s.ctx, chromedp.Evaluate(scrollJS, nil)); err != nil { + return errResult("scroll failed: " + err.Error()), nil + } + return map[string]interface{}{"status": "ok", "dir": dir, "amount": amount}, nil +} + +func (p *Plugin) handleBrowserClose(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return errResult("id is required"), nil + } + p.mu.Lock() + s, ok := p.sessions[id] + if ok { + delete(p.sessions, id) + } + p.mu.Unlock() + if !ok { + return errResult(fmt.Sprintf("浏览器会话 %s 不存在或已关闭", id)), nil + } + s.Close() + log.Printf("[%s] closed browser session %s", p.name, id) + return map[string]interface{}{"status": "closed", "id": id}, nil +} + +func (s *BrowserSession) Close() { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return + } + s.closed = true + s.cancel() +} + +func (p *Plugin) cleanupLoop() { + defer p.wg.Done() + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-p.stopCh: + return + case <-ticker.C: + p.mu.Lock() + for id, s := range p.sessions { + if time.Since(s.createdAt) >= s.timeout { + log.Printf("[%s] cleanup: browser session %s expired", p.name, id) + delete(p.sessions, id) + go s.Close() + p.sdk.InjectText(p.name, p.name, fmt.Sprintf("[浏览器会话 %s 已超时关闭]", id)) + } + } + p.mu.Unlock() + } + } } diff --git a/example/calendar/README.md b/example/calendar/README.md new file mode 100644 index 0000000..91702aa --- /dev/null +++ b/example/calendar/README.md @@ -0,0 +1,13 @@ +# calendar + +calendar plugin + +## Build + +```bash +plugindev build +``` + +## Install + +Upload the .hmap file through the Plugin Manager API. diff --git a/example/calendar/go.mod b/example/calendar/go.mod new file mode 100644 index 0000000..86f7f8a --- /dev/null +++ b/example/calendar/go.mod @@ -0,0 +1,7 @@ +module calendar + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0 + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone diff --git a/example/calendar/plg.json b/example/calendar/plg.json new file mode 100644 index 0000000..72e39ed --- /dev/null +++ b/example/calendar/plg.json @@ -0,0 +1,11 @@ +{ + "name": "calendar", + "name_zh": "日历", + "name_en": "Calendar", + "version": "1.0.0", + "description": "日历事件管理，支持提醒和重复事件", + "author": "HomeAgent", + "entry": "plugin.so", + "tags": ["calendar", "event", "reminder", "schedule"], + "targets": "linux/amd64" +} \ No newline at end of file diff --git a/example/calendar/plugin.go b/example/calendar/plugin.go new file mode 100644 index 0000000..601345b --- /dev/null +++ b/example/calendar/plugin.go @@ -0,0 +1,1140 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +const ( + RepeatNone = "none" + RepeatDaily = "daily" + RepeatWeekday = "weekday" + RepeatWeekly = "weekly" + RepeatBiweekly = "biweekly" + RepeatMonthly = "monthly" + RepeatYearly = "yearly" + RepeatLunarYearly = "lunar_yearly" +) + +type CalendarEvent struct { + ID string `json:"id"` + Title string `json:"title"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time,omitempty"` + AllDay bool `json:"all_day,omitempty"` + Location string `json:"location,omitempty"` + Note string `json:"note,omitempty"` + Reminds []int `json:"reminds,omitempty"` + RemindAt []int64 `json:"remind_at,omitempty"` + Repeat string `json:"repeat,omitempty"` + ParentID string `json:"parent_id,omitempty"` + Lunar bool `json:"lunar,omitempty"` + LunarMonth int `json:"lunar_month,omitempty"` + LunarDay int `json:"lunar_day,omitempty"` +} + +type Plugin struct { + name string + sdk *sdk.PluginSDK + dataDir string + mu sync.RWMutex + events []CalendarEvent + nextEventID int + stopCh chan struct{} + wg sync.WaitGroup + remindTicker *time.Ticker +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name, stopCh: make(chan struct{})}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T { + v, err := s.Get(key) + if err == nil && v != nil { + if sv, ok := v.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + v2, err2 := s.GetCore("plugin." + "calendar" + "." + key) + if err2 == nil && v2 != nil { + if sv, ok := v2.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + return fallback +} + +func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T { + v, ok := args[key] + if !ok || v == nil { + return fallback + } + switch any(fallback).(type) { + case string: + if s, ok := v.(string); ok { + return any(s).(T) + } + case int64: + switch n := v.(type) { + case float64: + return any(int64(n)).(T) + case int64: + return any(n).(T) + case string: + if i, err := strconv.ParseInt(n, 10, 64); err == nil { + return any(i).(T) + } + } + case float64: + switch n := v.(type) { + case float64: + return any(n).(T) + case int64: + return any(float64(n)).(T) + case string: + if f, err := strconv.ParseFloat(n, 64); err == nil { + return any(f).(T) + } + } + } + return fallback +} + +// --- Time Helpers --- + +var shortWeekday = map[time.Weekday]string{ + time.Monday: "一", time.Tuesday: "二", time.Wednesday: "三", + time.Thursday: "四", time.Friday: "五", time.Saturday: "六", time.Sunday: "日", +} + +func parseEventTime(s string) (time.Time, bool, bool) { + t, err := time.ParseInLocation("2006-01-02 15:04", s, time.Local) + if err == nil { + return t, false, true + } + t, err = time.ParseInLocation("2006-01-02", s, time.Local) + if err == nil { + return t, true, true + } + return time.Time{}, false, false +} + +// --- Lunar Calendar Engine --- + +var lunarInfo = []int{ + 0x04bd8, 0x04ae0, 0x0a570, 0x054d5, 0x0d260, 0x0d950, 0x16554, 0x056a0, 0x09ad0, 0x055d2, + 0x04ae0, 0x0a5b6, 0x0a4d0, 0x0d250, 0x1d255, 0x0b540, 0x0d6a0, 0x0ada2, 0x095b0, 0x14977, + 0x04970, 0x0a4b0, 0x0b4b5, 0x06a50, 0x06d40, 0x1ab54, 0x02b60, 0x09570, 0x052f2, 0x04970, + 0x06566, 0x0d4a0, 0x0ea50, 0x06e95, 0x05ad0, 0x02b60, 0x186e3, 0x092e0, 0x1c8d7, 0x0c950, + 0x0d4a0, 0x1d8a6, 0x0b550, 0x056a0, 0x1a5b4, 0x025d0, 0x092d0, 0x0d2b2, 0x0a950, 0x0b557, + 0x06ca0, 0x0b550, 0x15355, 0x04da0, 0x0a5b0, 0x14573, 0x052b0, 0x0a9a8, 0x0e950, 0x06aa0, + 0x0aea6, 0x0ab50, 0x04b60, 0x0aae4, 0x0a570, 0x05260, 0x0f263, 0x0d950, 0x05b57, 0x056a0, + 0x096d0, 0x04dd5, 0x04ad0, 0x0a4d0, 0x0d4d4, 0x0d250, 0x0d558, 0x0b540, 0x0b6a0, 0x195a6, + 0x095b0, 0x049b0, 0x0a974, 0x0a4b0, 0x0b27a, 0x06a50, 0x06d40, 0x0af46, 0x0ab60, 0x09570, + 0x04af5, 0x04970, 0x064b0, 0x074a3, 0x0ea50, 0x06b58, 0x05ac0, 0x0ab60, 0x096d5, 0x092e0, + 0x0c960, 0x0d954, 0x0d4a0, 0x0da50, 0x07552, 0x056a0, 0x0abb7, 0x025d0, 0x092d0, 0x0cab5, + 0x0a950, 0x0b4a0, 0x0baa4, 0x0ad50, 0x055d9, 0x04ba0, 0x0a5b0, 0x15176, 0x052b0, 0x0a930, + 0x07954, 0x06aa0, 0x0ad50, 0x05b52, 0x04b60, 0x0a6e6, 0x0a4e0, 0x0d260, 0x0ea65, 0x0d530, + 0x05aa0, 0x076a3, 0x096d0, 0x04afb, 0x04ad0, 0x0a4d0, 0x1d0b6, 0x0d250, 0x0d520, 0x0dd45, + 0x0b5a0, 0x056d0, 0x055b2, 0x049b0, 0x0a577, 0x0a4b0, 0x0aa50, 0x1b255, 0x06d20, 0x0ada0, + 0x14b63, 0x09370, 0x049f8, 0x04970, 0x064b0, 0x168a6, 0x0ea50, 0x06aa0, 0x1a6c4, 0x0aae0, + 0x092e0, 0x0d2e3, 0x0c960, 0x0d557, 0x0d4a0, 0x0da50, 0x05d55, 0x056a0, 0x0a6d0, 0x055d4, + 0x052d0, 0x0a9b8, 0x0a950, 0x0b4a0, 0x0b6a6, 0x0ad50, 0x055a0, 0x0aba4, 0x0a5b0, 0x052b0, + 0x0b273, 0x06930, 0x07337, 0x06aa0, 0x0ad50, 0x14b55, 0x04b60, 0x0a570, 0x054e4, 0x0d160, + 0x0e968, 0x0d520, 0x0daa0, 0x16aa6, 0x056d0, 0x04ae0, 0x0a9d4, 0x0a4d0, 0x0d150, 0x0f252, + 0x0d520, +} + +func daysInLunarYear(year int) int { + if year < 1900 || year > 2100 { + return 365 + } + y := lunarInfo[year-1900] + sum := 0 + for i := 0x8000; i > 0; i >>= 1 { + if y&i > 0 { + sum += 30 + } else { + sum += 29 + } + } + return sum +} + +func leapMonth(year int) int { + if year < 1900 || year > 2100 { + return 0 + } + return lunarInfo[year-1900] & 0xf +} + +func leapDays(year int) int { + if year < 1900 || year > 2100 { + return 0 + } + if leapMonth(year) == 0 { + return 0 + } + if lunarInfo[year-1900]&0x10000 > 0 { + return 30 + } + return 29 +} + +func monthDays(year, month int) int { + if year < 1900 || year > 2100 || month < 1 || month > 12 { + return 30 + } + if lunarInfo[year-1900]&(0x10000>>month) > 0 { + return 30 + } + return 29 +} + +var baseSolar = func() time.Time { t, _ := time.ParseInLocation("2006-01-02", "1900-01-31", time.Local); return t }() + +func lunarToSolar(year, month, day int) (time.Time, bool) { + if year < 1900 || year > 2100 || month < 1 || month > 12 || day < 1 || day > 30 { + return time.Time{}, false + } + offset := 0 + for y := 1900; y < year; y++ { + offset += daysInLunarYear(y) + } + lm := leapMonth(year) + for m := 1; m < month; m++ { + offset += monthDays(year, m) + if m == lm { + offset += leapDays(year) + } + } + offset += day - 1 + solar := baseSolar.AddDate(0, 0, offset) + return solar, true +} + +func nextLunarYearly(targetMonth, targetDay int, after time.Time) (time.Time, bool) { + afterYear := after.Year() + for y := afterYear; y <= afterYear+2; y++ { + t, ok := lunarToSolar(y, targetMonth, targetDay) + if !ok { + continue + } + if t.After(after) || t.Equal(after) { + return t, true + } + } + return time.Time{}, false +} + +// --- Plugin Lifecycle --- + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + p.sdk = s + + dataHome := os.Getenv("HOME") + if dataHome == "" { + dataHome = "/tmp" + } + p.dataDir = filepath.Join(dataHome, ".homeagent", "calendar") + os.MkdirAll(p.dataDir, 0755) + p.loadEvents() + + tp := p.name + "_" + + s.RegisterTool(tp+"event_add", sdk.ToolDef{ + Name: tp + "event_add", Description: "Add a calendar event. Time: YYYY-MM-DD HH:MM or YYYY-MM-DD for all-day.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "title": map[string]interface{}{"type": "string", "description": "Event title"}, + "start_time": map[string]interface{}{"type": "string", "description": "Start time (YYYY-MM-DD HH:MM or YYYY-MM-DD)"}, + "end_time": map[string]interface{}{"type": "string", "description": "End time (optional)"}, + "location": map[string]interface{}{"type": "string", "description": "Location (optional)"}, + "note": map[string]interface{}{"type": "string", "description": "Notes (optional)"}, + "remind_before": map[string]interface{}{"type": "string", "description": "Reminder minutes before event. Multiple: comma-separated, e.g. '15,60,1440' for 15min + 1hr + 1day before. 0 or empty = no reminder."}, + "repeat": map[string]interface{}{"type": "string", "description": "Repeat: none, daily, weekday, weekly, biweekly, monthly, yearly, lunar_yearly"}, + "lunar": map[string]interface{}{"type": "boolean", "description": "Whether the date is lunar calendar. If true, repeat=lunar_yearly by default. Also set lunar_month and lunar_day."}, + "lunar_month": map[string]interface{}{"type": "integer", "description": "Lunar month (1-12), required when lunar=true"}, + "lunar_day": map[string]interface{}{"type": "integer", "description": "Lunar day (1-30), required when lunar=true"}, + }, + "required": []string{"title", "start_time"}, + }, + }, p.handleEventAdd) + + s.RegisterTool(tp+"event_list", sdk.ToolDef{ + Name: tp + "event_list", Description: "List upcoming events. Shows date, time, repeat pattern.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "days": map[string]interface{}{"type": "integer", "description": "Days ahead (default 7, max 365)"}, + }, + }, + }, p.handleEventList) + + s.RegisterTool(tp+"event_delete", sdk.ToolDef{ + Name: tp + "event_delete", Description: "Delete an event by ID. Deletes this and all future recurrences.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "Event ID"}, + }, + "required": []string{"id"}, + }, + }, p.handleEventDelete) + + s.RegisterTool(tp+"event_update", sdk.ToolDef{ + Name: tp + "event_update", Description: "Update an event. Only provided fields change. Resets reminder state.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "id": map[string]interface{}{"type": "string", "description": "Event ID"}, + "title": map[string]interface{}{"type": "string", "description": "New title"}, + "start_time": map[string]interface{}{"type": "string", "description": "New start time"}, + "end_time": map[string]interface{}{"type": "string", "description": "New end time"}, + "location": map[string]interface{}{"type": "string", "description": "New location"}, + "note": map[string]interface{}{"type": "string", "description": "New notes"}, + "remind_before": map[string]interface{}{"type": "string", "description": "New reminder minutes (comma-separated)"}, + "repeat": map[string]interface{}{"type": "string", "description": "New repeat type"}, + "lunar": map[string]interface{}{"type": "boolean", "description": "Whether lunar calendar"}, + "lunar_month": map[string]interface{}{"type": "integer", "description": "Lunar month 1-12"}, + "lunar_day": map[string]interface{}{"type": "integer", "description": "Lunar day 1-30"}, + }, + "required": []string{"id"}, + }, + }, p.handleEventUpdate) + + s.RegisterTool(tp+"today", sdk.ToolDef{ + Name: tp + "today", Description: "Show today's events with countdown.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleToday) + + s.RegisterTool(tp+"week", sdk.ToolDef{ + Name: tp + "week", Description: "Show this week's events grouped by day.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleWeek) + + s.RegisterTool(tp+"month", sdk.ToolDef{ + Name: tp + "month", Description: "Show a month calendar grid with event dots.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "year": map[string]interface{}{"type": "integer", "description": "Year (default: current)"}, + "month": map[string]interface{}{"type": "integer", "description": "Month 1-12 (default: current)"}, + }, + }, + }, p.handleMonth) + + s.RegisterTool(tp+"search", sdk.ToolDef{ + Name: tp + "search", Description: "Search events by keyword in title, location, or notes.", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "keyword": map[string]interface{}{"type": "string", "description": "Search keyword"}, + }, + "required": []string{"keyword"}, + }, + }, p.handleSearch) + + p.remindTicker = time.NewTicker(30 * time.Second) + p.wg.Add(1) + go p.remindLoop() + + fmt.Printf("[%s] started (%d events)\n", p.name, len(p.events)) + return nil +} + +func (p *Plugin) Stop() error { + p.remindTicker.Stop() + close(p.stopCh) + p.wg.Wait() + p.saveEvents() + fmt.Printf("[%s] stopped\n", p.name) + return nil +} + +// --- Reminder Loop --- + +func (p *Plugin) remindLoop() { + defer p.wg.Done() + for { + select { + case <-p.remindTicker.C: + p.checkReminders() + case <-p.stopCh: + return + } + } +} + +func (p *Plugin) checkReminders() { + now := time.Now() + + p.mu.Lock() + defer p.mu.Unlock() + + changed := false + + for i := range p.events { + e := &p.events[i] + evtTime, allDay, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if allDay || evtTime.Before(now) { + continue + } + + for ri, remindMin := range e.Reminds { + if remindMin <= 0 { + continue + } + if ri < len(e.RemindAt) && e.RemindAt[ri] > 0 { + continue + } + remindAt := evtTime.Add(-time.Duration(remindMin) * time.Minute) + if !now.After(remindAt) && !now.Equal(remindAt) { + continue + } + if len(e.RemindAt) <= ri { + e.RemindAt = append(e.RemindAt, make([]int64, ri+1-len(e.RemindAt))...) + } + e.RemindAt[ri] = remindAt.Unix() + changed = true + timeUntil := evtTime.Sub(now).Round(time.Minute) + msg := fmt.Sprintf("⏰ 提醒: %s (%s)", e.Title, e.StartTime) + if timeUntil > 0 { + msg += fmt.Sprintf(" (还有%s)", timeUntil) + } + if len(e.Reminds) > 1 { + msg += fmt.Sprintf(" [第%d次提醒]", ri+1) + } + if e.Location != "" { + msg += fmt.Sprintf("\n📍 %s", e.Location) + } + if e.Note != "" { + msg += fmt.Sprintf("\n📝 %s", e.Note) + } + go p.sdk.InjectInterruptText("calendar", "calendar", msg) + } + } + + newEvents := []CalendarEvent{} + for i := range p.events { + e := &p.events[i] + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if !now.After(evtTime) { + continue + } + if e.Repeat == "" || e.Repeat == RepeatNone { + continue + } + next := p.nextOccurrence(*e, evtTime) + if next != nil { + pid := e.ID + if e.ParentID != "" { + pid = e.ParentID + } + next.ParentID = pid + newEvents = append(newEvents, *next) + changed = true + } + } + if len(newEvents) > 0 { + p.events = append(p.events, newEvents...) + } + + p.cleanupPastEvents() + if changed { + p.saveEventsLocked() + } +} + +func (p *Plugin) nextOccurrence(e CalendarEvent, evtTime time.Time) *CalendarEvent { + var next time.Time + switch e.Repeat { + case RepeatDaily: + next = evtTime.AddDate(0, 0, 1) + case RepeatWeekday: + next = evtTime.AddDate(0, 0, 1) + for next.Weekday() == time.Saturday || next.Weekday() == time.Sunday { + next = next.AddDate(0, 0, 1) + } + case RepeatWeekly: + next = evtTime.AddDate(0, 0, 7) + case RepeatBiweekly: + next = evtTime.AddDate(0, 0, 14) + case RepeatMonthly: + next = evtTime.AddDate(0, 1, 0) + case RepeatYearly: + next = evtTime.AddDate(1, 0, 0) + case RepeatLunarYearly: + if e.LunarMonth > 0 && e.LunarDay > 0 { + t, ok := nextLunarYearly(e.LunarMonth, e.LunarDay, evtTime) + if ok { + next = t + } else { + return nil + } + } else { + return nil + } + default: + return nil + } + + timeStr := next.Format("2006-01-02 15:04") + if e.AllDay { + timeStr = next.Format("2006-01-02") + } + + reminds := make([]int, len(e.Reminds)) + copy(reminds, e.Reminds) + + return &CalendarEvent{ + ID: fmt.Sprintf("evt_%d_%d", next.Unix(), p.nextEventID), + Title: e.Title, + StartTime: timeStr, + EndTime: e.EndTime, + AllDay: e.AllDay, + Location: e.Location, + Note: e.Note, + Reminds: reminds, + Repeat: e.Repeat, + ParentID: e.ParentID, + } +} + +func (p *Plugin) cleanupPastEvents() { + now := time.Now() + keep := []CalendarEvent{} + for _, e := range p.events { + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if !now.After(evtTime) { + keep = append(keep, e) + continue + } + if e.Repeat != "" && e.Repeat != RepeatNone { + keep = append(keep, e) + } + } + p.events = keep +} + +// --- Persistence --- + +func (p *Plugin) eventsFile() string { + return filepath.Join(p.dataDir, "events.json") +} + +func (p *Plugin) loadEvents() { + p.mu.Lock() + defer p.mu.Unlock() + b, err := os.ReadFile(p.eventsFile()) + if err != nil { + p.events = nil + p.nextEventID = 1 + return + } + var data struct { + Events []CalendarEvent `json:"events"` + NextEventID int `json:"next_id"` + } + if json.Unmarshal(b, &data) != nil { + p.events = nil + p.nextEventID = 1 + return + } + p.events = data.Events + p.nextEventID = data.NextEventID + if p.nextEventID < 1 { + p.nextEventID = 1 + } + if p.events == nil { + p.events = []CalendarEvent{} + } +} + +func (p *Plugin) saveEvents() { + p.mu.RLock() + defer p.mu.RUnlock() + p.saveEventsLocked() +} + +func (p *Plugin) saveEventsLocked() { + data := struct { + Events []CalendarEvent `json:"events"` + NextEventID int `json:"next_id"` + }{ + Events: p.events, + NextEventID: p.nextEventID, + } + b, _ := json.MarshalIndent(data, "", " ") + os.WriteFile(p.eventsFile(), b, 0644) +} + +// --- Helper: parse remind_before --- + +func parseReminds(s string) []int { + s = strings.TrimSpace(s) + if s == "" || s == "0" { + return nil + } + parts := strings.Split(s, ",") + vals := make([]int, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + v, err := strconv.Atoi(p) + if err != nil || v <= 0 { + continue + } + vals = append(vals, v) + } + sort.Ints(vals) + return vals +} + +// --- Helper: format event duration --- + +func formatTimeUntil(t time.Time) string { + now := time.Now() + if t.Before(now) { + return "已开始" + } + d := t.Sub(now) + if d < time.Hour { + m := int(d.Minutes()) + return fmt.Sprintf("还有%d分钟", m) + } + if d < 24*time.Hour { + h := int(d.Hours()) + m := int(d.Minutes()) % 60 + if m > 0 { + return fmt.Sprintf("还有%d小时%d分", h, m) + } + return fmt.Sprintf("还有%d小时", h) + } + d2 := int(d.Hours() / 24) + return fmt.Sprintf("还有%d天", d2) +} + +// --- Tool: event_add --- + +func (p *Plugin) handleEventAdd(args map[string]interface{}) (interface{}, error) { + title := readArg(args, "title", "") + startTime := readArg(args, "start_time", "") + if title == "" || startTime == "" { + return map[string]interface{}{"isError": true, "content": "title and start_time are required"}, nil + } + + parsedStart, allDay, ok := parseEventTime(startTime) + if !ok { + return map[string]interface{}{"isError": true, "content": "Invalid start_time. Use YYYY-MM-DD HH:MM or YYYY-MM-DD."}, nil + } + + endTime := readArg(args, "end_time", "") + if endTime != "" { + if _, _, ok := parseEventTime(endTime); !ok { + return map[string]interface{}{"isError": true, "content": "Invalid end_time."}, nil + } + } + + location := readArg(args, "location", "") + note := readArg(args, "note", "") + remindStr := readArg(args, "remind_before", "") + reminds := parseReminds(remindStr) + lunar := false + if v := readArg(args, "lunar", ""); v == "true" { + lunar = true + } + lunarMonth := int(readArg(args, "lunar_month", int64(0))) + lunarDay := int(readArg(args, "lunar_day", int64(0))) + + repeat := readArg(args, "repeat", RepeatNone) + if lunar && repeat == RepeatNone { + repeat = RepeatLunarYearly + } + switch repeat { + case RepeatNone, RepeatDaily, RepeatWeekday, RepeatWeekly, RepeatBiweekly, RepeatMonthly, RepeatYearly, RepeatLunarYearly: + default: + repeat = RepeatNone + } + + if lunar && (lunarMonth < 1 || lunarMonth > 12 || lunarDay < 1 || lunarDay > 30) { + return map[string]interface{}{"isError": true, "content": "lunar_month (1-12) and lunar_day (1-30) required when lunar=true"}, nil + } + + event := CalendarEvent{ + ID: fmt.Sprintf("evt_%d_%d", parsedStart.Unix(), p.nextEventID), + Title: title, + StartTime: startTime, + EndTime: endTime, + AllDay: allDay, + Location: location, + Note: note, + Reminds: reminds, + Repeat: repeat, + Lunar: lunar, + LunarMonth: lunarMonth, + LunarDay: lunarDay, + } + + p.mu.Lock() + p.events = append(p.events, event) + p.nextEventID++ + p.mu.Unlock() + p.saveEvents() + + detail := fmt.Sprintf("Event added: %s (ID: %s)", title, event.ID) + if len(reminds) > 0 { + parts := make([]string, len(reminds)) + for i, r := range reminds { + parts[i] = fmt.Sprintf("%dmin", r) + } + detail += fmt.Sprintf(" | 提醒: %s", strings.Join(parts, ", ")) + } + if repeat != RepeatNone { + detail += " | 重复: " + repeat + } + return map[string]interface{}{"content": detail}, nil +} + +// --- Tool: event_list --- + +func (p *Plugin) handleEventList(args map[string]interface{}) (interface{}, error) { + days := int(readArg(args, "days", int64(7))) + if days < 1 { + days = 1 + } + if days > 365 { + days = 365 + } + + now := time.Now() + cutoff := now.AddDate(0, 0, days) + + p.mu.RLock() + upcoming := make([]CalendarEvent, 0) + for _, e := range p.events { + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if evtTime.Before(cutoff) && evtTime.After(now.Add(-24*time.Hour)) { + upcoming = append(upcoming, e) + } + } + p.mu.RUnlock() + + sort.Slice(upcoming, func(i, j int) bool { + return upcoming[i].StartTime < upcoming[j].StartTime + }) + + if len(upcoming) == 0 { + return map[string]interface{}{"content": fmt.Sprintf("No events in the next %d days.", days)}, nil + } + + var lines []string + lines = append(lines, fmt.Sprintf("📋 Events (%d):", len(upcoming))) + for _, e := range upcoming { + timeStr := e.StartTime + if e.EndTime != "" { + timeStr += " → " + e.EndTime + } + extra := "" + if e.Location != "" { + extra += " 📍" + e.Location + } + if len(e.Reminds) > 0 { + parts := make([]string, len(e.Reminds)) + for i, r := range e.Reminds { + parts[i] = fmt.Sprintf("%d′", r) + } + extra += " 🔔" + strings.Join(parts, ",") + } + if e.Lunar { + extra += fmt.Sprintf(" 🌙%d-%d", e.LunarMonth, e.LunarDay) + } + if e.Repeat != "" && e.Repeat != RepeatNone { + extra += " 🔄" + e.Repeat + } + if e.Note != "" { + extra += " 📝" + e.Note + } + lines = append(lines, fmt.Sprintf(" [%s] %s%s", timeStr, e.Title, extra)) + } + + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} + +// --- Tool: event_delete --- + +func (p *Plugin) handleEventDelete(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return map[string]interface{}{"isError": true, "content": "Event ID is required"}, nil + } + + p.mu.Lock() + defer p.mu.Unlock() + + found := false + remaining := []CalendarEvent{} + for _, e := range p.events { + if e.ID == id { + found = true + continue + } + pid := e.ParentID + if pid == "" { + pid = e.ID + } + if pid == id { + continue + } + remaining = append(remaining, e) + } + if !found { + return map[string]interface{}{"isError": true, "content": "Event not found: " + id}, nil + } + p.events = remaining + p.saveEventsLocked() + return map[string]interface{}{"content": "Deleted event and all recurrences: " + id}, nil +} + +// --- Tool: event_update --- + +func (p *Plugin) handleEventUpdate(args map[string]interface{}) (interface{}, error) { + id := readArg(args, "id", "") + if id == "" { + return map[string]interface{}{"isError": true, "content": "Event ID is required"}, nil + } + + p.mu.Lock() + defer p.mu.Unlock() + + for i := range p.events { + if p.events[i].ID != id { + continue + } + e := &p.events[i] + + if v := readArg(args, "title", ""); v != "" { + e.Title = v + } + if v := readArg(args, "start_time", ""); v != "" { + if _, allDay, ok := parseEventTime(v); ok { + e.StartTime = v + e.AllDay = allDay + } + } + if v := readArg(args, "end_time", ""); v != "" { + if _, _, ok := parseEventTime(v); ok { + e.EndTime = v + } + } + if v := readArg(args, "location", ""); v != "" { + e.Location = v + } + if v := readArg(args, "note", ""); v != "" { + e.Note = v + } + if v := readArg(args, "remind_before", ""); v != "" { + e.Reminds = parseReminds(v) + } + if v := readArg(args, "repeat", ""); v != "" { + switch v { + case RepeatNone, RepeatDaily, RepeatWeekday, RepeatWeekly, RepeatBiweekly, RepeatMonthly, RepeatYearly, RepeatLunarYearly: + e.Repeat = v + } + } + if v := readArg(args, "lunar", ""); v == "true" { + e.Lunar = true + } else if v == "false" { + e.Lunar = false + } + if v := readArg(args, "lunar_month", int64(0)); v > 0 { + e.LunarMonth = int(v) + } + if v := readArg(args, "lunar_day", int64(0)); v > 0 { + e.LunarDay = int(v) + } + e.RemindAt = nil + + p.saveEventsLocked() + return map[string]interface{}{"content": "Event updated: " + e.Title}, nil + } + + return map[string]interface{}{"isError": true, "content": "Event not found: " + id}, nil +} + +// --- Tool: today --- + +func (p *Plugin) handleToday(args map[string]interface{}) (interface{}, error) { + now := time.Now() + todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + todayEnd := todayStart.AddDate(0, 0, 1) + + p.mu.RLock() + events := make([]CalendarEvent, 0) + for _, e := range p.events { + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if evtTime.After(todayStart.Add(-time.Hour)) && evtTime.Before(todayEnd) { + events = append(events, e) + } + } + p.mu.RUnlock() + + sort.Slice(events, func(i, j int) bool { + return events[i].StartTime < events[j].StartTime + }) + + dateStr := now.Format("2006-01-02") + weekday := shortWeekday[now.Weekday()] + lines := []string{fmt.Sprintf("📅 %s 周%s — 今天", dateStr, weekday)} + + if len(events) == 0 { + lines = append(lines, " 今天没有事件") + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil + } + + for _, e := range events { + evtTime, _, _ := parseEventTime(e.StartTime) + timeStr := e.StartTime + if now.Format("2006-01-02") == evtTime.Format("2006-01-02") { + timeStr = evtTime.Format("15:04") + } + countdown := formatTimeUntil(evtTime) + detail := fmt.Sprintf(" %s — %s (%s)", timeStr, e.Title, countdown) + if e.AllDay { + detail = fmt.Sprintf(" 🌙 %s (全天)", e.Title) + } + if e.Location != "" { + detail += " 📍" + e.Location + } + lines = append(lines, detail) + } + + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} + +// --- Tool: week --- + +func (p *Plugin) handleWeek(args map[string]interface{}) (interface{}, error) { + now := time.Now() + weekStart := now.AddDate(0, 0, -int(now.Weekday()-time.Monday)) + if now.Weekday() == time.Sunday { + weekStart = now.AddDate(0, 0, -6) + } + weekEnd := weekStart.AddDate(0, 0, 7) + + p.mu.RLock() + dayEvents := make(map[string][]CalendarEvent) + for _, e := range p.events { + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if evtTime.After(weekStart.Add(-time.Hour)) && evtTime.Before(weekEnd) { + dayKey := evtTime.Format("2006-01-02") + dayEvents[dayKey] = append(dayEvents[dayKey], e) + } + } + p.mu.RUnlock() + + for k := range dayEvents { + sort.Slice(dayEvents[k], func(i, j int) bool { + return dayEvents[k][i].StartTime < dayEvents[k][j].StartTime + }) + } + + lines := []string{fmt.Sprintf("📅 %s ～ %s", weekStart.Format("01-02"), weekEnd.AddDate(0, 0, -1).Format("01-02"))} + eventCount := 0 + for i := 0; i < 7; i++ { + d := weekStart.AddDate(0, 0, i) + dayKey := d.Format("2006-01-02") + wd := shortWeekday[d.Weekday()] + prefix := " " + if d.Format("2006-01-02") == now.Format("2006-01-02") { + prefix = "▶" + } + line := fmt.Sprintf("%s %s %s", prefix, d.Format("01-02"), wd) + if evts, ok := dayEvents[dayKey]; ok && len(evts) > 0 { + titles := make([]string, len(evts)) + for i, e := range evts { + timeStr := e.StartTime + if !e.AllDay { + timeStr = parseTimeShort(e.StartTime) + } else { + timeStr = "全天" + } + titles[i] = fmt.Sprintf("%s %s", timeStr, e.Title) + eventCount++ + } + line += " " + strings.Join(titles, ", ") + } + lines = append(lines, line) + } + if eventCount == 0 { + lines = append(lines, " 本周没有事件") + } + + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} + +func parseTimeShort(s string) string { + t, _, ok := parseEventTime(s) + if !ok { + return s + } + return t.Format("15:04") +} + +// --- Tool: month --- + +func (p *Plugin) handleMonth(args map[string]interface{}) (interface{}, error) { + now := time.Now() + year := int(readArg(args, "year", int64(now.Year()))) + month := int(readArg(args, "month", int64(now.Month()))) + if month < 1 || month > 12 { + month = int(now.Month()) + } + + firstDay := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, now.Location()) + lastDay := firstDay.AddDate(0, 1, -1) + daysInMonth := lastDay.Day() + startWeekday := int(firstDay.Weekday()) + if startWeekday == 0 { + startWeekday = 7 + } + + p.mu.RLock() + daySet := make(map[int]bool) + for _, e := range p.events { + evtTime, _, ok := parseEventTime(e.StartTime) + if !ok { + continue + } + if evtTime.Year() == year && evtTime.Month() == time.Month(month) { + daySet[evtTime.Day()] = true + } + } + p.mu.RUnlock() + + monthName := firstDay.Format("January") + lines := []string{fmt.Sprintf("📅 %d年%d月 (%s)", year, month, monthName)} + lines = append(lines, " 一 二 三 四 五 六 日") + lines = append(lines, "") + + row := " " + for i := 1; i < startWeekday; i++ { + row += " " + } + for d := 1; d <= daysInMonth; d++ { + mark := " " + if daySet[d] { + mark = "•" + } + row += fmt.Sprintf(" %2d%s", d, mark) + wd := startWeekday - 1 + d + if wd%7 == 0 || d == daysInMonth { + lines = append(lines, row) + row = " " + } + } + + count := 0 + for d := range daySet { + count++ + _ = d + } + lines = append(lines, fmt.Sprintf("\n本月 %d 天有事件", count)) + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} + +// --- Tool: search --- + +func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) { + keyword := strings.ToLower(readArg(args, "keyword", "")) + if keyword == "" { + return map[string]interface{}{"isError": true, "content": "keyword is required"}, nil + } + + p.mu.RLock() + results := make([]CalendarEvent, 0) + for _, e := range p.events { + if strings.Contains(strings.ToLower(e.Title), keyword) || + strings.Contains(strings.ToLower(e.Location), keyword) || + strings.Contains(strings.ToLower(e.Note), keyword) { + results = append(results, e) + } + } + p.mu.RUnlock() + + sort.Slice(results, func(i, j int) bool { + return results[i].StartTime < results[j].StartTime + }) + + if len(results) == 0 { + return map[string]interface{}{"content": fmt.Sprintf("No events match: %s", keyword)}, nil + } + + var lines []string + lines = append(lines, fmt.Sprintf("🔍 Found %d events for \"%s\":", len(results), keyword)) + for _, e := range results { + lines = append(lines, fmt.Sprintf(" [%s] %s (ID: %s)", e.StartTime, e.Title, e.ID)) + } + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} diff --git a/example/music/go.mod b/example/music/go.mod new file mode 100644 index 0000000..2c34cda --- /dev/null +++ b/example/music/go.mod @@ -0,0 +1,7 @@ +module music + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v0.7.1 + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone \ No newline at end of file diff --git a/example/music/plg.json b/example/music/plg.json new file mode 100644 index 0000000..44ef173 --- /dev/null +++ b/example/music/plg.json @@ -0,0 +1,11 @@ +{ + "name": "music", + "name_zh": "音乐搜索", + "name_en": "Music Search", + "version": "0.1.0", + "description": "音乐搜索插件，支持搜索歌曲和查看歌词（基于网易云音乐）", + "author": "HomeAgent", + "entry": "plugin.so", + "tags": ["music", "song", "lyrics", "网易云"], + "targets": "linux/amd64" +} \ No newline at end of file diff --git a/example/music/plugin.go b/example/music/plugin.go new file mode 100644 index 0000000..8021434 --- /dev/null +++ b/example/music/plugin.go @@ -0,0 +1,320 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +type Plugin struct { + name string + sdk *sdk.PluginSDK + cli *http.Client +} + +type searchResp struct { + Result *struct { + Songs []songItem `json:"songs"` + SongCount int `json:"songCount"` + } `json:"result"` + Code int `json:"code"` +} + +type songItem struct { + ID int64 `json:"id"` + Name string `json:"name"` + Artists []artist `json:"artists"` + Album albumInfo `json:"album"` + Duration int `json:"duration"` + Mvid int `json:"mvid"` + Fee int `json:"fee"` +} + +type artist struct { + ID int64 `json:"id"` + Name string `json:"name"` +} + +type albumInfo struct { + ID int64 `json:"id"` + Name string `json:"name"` +} + +type lyricResp struct { + Lrc *lyricData `json:"lrc"` + TLrc *lyricData `json:"tlyric"` + Code int `json:"code"` +} + +type lyricData struct { + Lyric string `json:"lyric"` +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + s.SetAutoRestart(true) + p.sdk = s + p.cli = &http.Client{Timeout: 15 * time.Second} + + s.RegisterTool(p.name+"_search", sdk.ToolDef{ + Name: p.name + "_search", + Description: "搜索歌曲，通过关键词查找音乐，返回歌曲列表", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "keyword": map[string]interface{}{ + "type": "string", + "description": "搜索关键词，如歌曲名、歌手名", + }, + "limit": map[string]interface{}{ + "type": "integer", + "description": "返回结果数量（1-50），默认10", + }, + }, + "required": []string{"keyword"}, + }, + }, p.handleSearch) + + s.RegisterTool(p.name+"_lyrics", sdk.ToolDef{ + Name: p.name + "_lyrics", + Description: "获取歌曲歌词，通过歌曲ID查看歌词内容", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "song_id": map[string]interface{}{ + "type": "integer", + "description": "歌曲ID（从搜索结果的 id 字段获取）", + }, + }, + "required": []string{"song_id"}, + }, + }, p.handleLyrics) + + return nil +} + +func (p *Plugin) Stop() error { return nil } + +func (p *Plugin) neRequest(path string, params map[string]string) ([]byte, error) { + base := "https://music.163.com/api" + path + reqURL := base + "?" + urlValues(params).Encode() + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + req.Header.Set("Referer", "https://music.163.com/") + resp, err := p.cli.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) +} + +func urlValues(m map[string]string) url.Values { + v := url.Values{} + for k, val := range m { + v.Set(k, val) + } + return v +} + +func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) { + keyword, _ := args["keyword"].(string) + keyword = strings.TrimSpace(keyword) + if keyword == "" { + return map[string]interface{}{ + "content": "请输入搜索关键词", + "isError": true, + }, nil + } + + limit := 10 + if v, ok := args["limit"].(float64); ok { + limit = int(v) + if limit < 1 { + limit = 1 + } + if limit > 50 { + limit = 50 + } + } + + body, err := p.neRequest("/search/get", map[string]string{ + "s": keyword, + "type": "1", + "limit": fmt.Sprint(limit), + }) + if err != nil { + return map[string]interface{}{ + "content": fmt.Sprintf("搜索失败：%v", err), + "isError": true, + }, nil + } + + var resp searchResp + if err := json.Unmarshal(body, &resp); err != nil { + return map[string]interface{}{ + "content": fmt.Sprintf("解析响应失败：%v", err), + "isError": true, + }, nil + } + + if resp.Code != 200 || resp.Result == nil { + return map[string]interface{}{ + "content": fmt.Sprintf("搜索失败，响应码：%d", resp.Code), + "isError": true, + }, nil + } + + songs := resp.Result.Songs + if len(songs) == 0 { + return map[string]interface{}{ + "content": fmt.Sprintf("未找到与「%s」相关的歌曲", keyword), + }, nil + } + + var lines []string + lines = append(lines, fmt.Sprintf("找到 %d 首与「%s」相关的歌曲：\n", resp.Result.SongCount, keyword)) + for i, s := range songs { + var artists []string + for _, a := range s.Artists { + artists = append(artists, a.Name) + } + dur := time.Duration(s.Duration) * time.Millisecond + minutes := int(dur.Minutes()) + seconds := int(dur.Seconds()) % 60 + lines = append(lines, fmt.Sprintf("%d. %s - %s [%02d:%02d] (ID: %d)", + i+1, s.Name, strings.Join(artists, "/"), minutes, seconds, s.ID)) + } + + type songResult struct { + ID int64 `json:"id"` + Name string `json:"name"` + Artists []string `json:"artists"` + Album string `json:"album"` + Duration int `json:"duration"` + } + + var results []songResult + for _, s := range songs { + var artists []string + for _, a := range s.Artists { + artists = append(artists, a.Name) + } + results = append(results, songResult{ + ID: s.ID, + Name: s.Name, + Artists: artists, + Album: s.Album.Name, + Duration: s.Duration, + }) + } + + return map[string]interface{}{ + "content": strings.Join(lines, "\n"), + "songs": results, + "total": resp.Result.SongCount, + }, nil +} + +func (p *Plugin) handleLyrics(args map[string]interface{}) (interface{}, error) { + songID, ok := args["song_id"].(float64) + if !ok { + return map[string]interface{}{ + "content": "请提供有效的歌曲ID", + "isError": true, + }, nil + } + + id := int64(songID) + body, err := p.neRequest("/song/lyric", map[string]string{ + "id": fmt.Sprint(id), + "lv": "-1", + "kv": "-1", + "tv": "-1", + }) + if err != nil { + return map[string]interface{}{ + "content": fmt.Sprintf("获取歌词失败：%v", err), + "isError": true, + }, nil + } + + var resp lyricResp + if err := json.Unmarshal(body, &resp); err != nil { + return map[string]interface{}{ + "content": fmt.Sprintf("解析歌词失败：%v", err), + "isError": true, + }, nil + } + + if resp.Code != 200 { + return map[string]interface{}{ + "content": fmt.Sprintf("获取歌词失败，响应码：%d", resp.Code), + "isError": true, + }, nil + } + + lyric := "" + if resp.Lrc != nil { + lyric = resp.Lrc.Lyric + } + + if lyric == "" { + return map[string]interface{}{ + "content": fmt.Sprintf("歌曲 %d 暂无歌词", id), + }, nil + } + + // Clean up lyrics metadata lines and limit length + lyric = cleanLyrics(lyric) + if len(lyric) > 3000 { + lyric = lyric[:3000] + "\n...（歌词过长已截断）" + } + + tLyric := "" + if resp.TLrc != nil && resp.TLrc.Lyric != "" { + tLyric = cleanLyrics(resp.TLrc.Lyric) + if len(tLyric) > 1000 { + tLyric = tLyric[:1000] + "\n...（翻译过长已截断）" + } + } + + result := fmt.Sprintf("歌词：\n%s", lyric) + if tLyric != "" { + result += fmt.Sprintf("\n翻译：\n%s", tLyric) + } + + return map[string]interface{}{ + "content": result, + "lyric": lyric, + "tlyric": tLyric, + }, nil +} + +func cleanLyrics(l string) string { + lines := strings.Split(l, "\n") + var cleaned []string + for _, line := range lines { + // Skip metadata lines like [ti:...], [ar:...], [al:...], [by:...], [offset:...] + trimmed := strings.TrimSpace(line) + if trimmed == "" { + continue + } + cleaned = append(cleaned, line) + } + return strings.Join(cleaned, "\n") +} diff --git a/example/qq/plugin.go b/example/qq/plugin.go index a751ecd..1011aee 100644 --- a/example/qq/plugin.go +++ b/example/qq/plugin.go @@ -808,7 +808,7 @@ func (p *Plugin) handleGetMessage(args map[string]interface{}) (interface{}, err "group_id": found.GroupID, "nickname": found.Nickname, "message_type": found.MessageType, - "time": time.Unix(found.Time, 0).Format("15:04:05"), + "time": time.Unix(found.Time, 0).Format("2006-01-02 15:04:05"), } if found.RawText != "" { result["raw_text"] = found.RawText @@ -1103,7 +1103,7 @@ func (p *Plugin) handleGetHistory(args map[string]interface{}) (interface{}, err } ts := "" if t, ok := msg["time"].(float64); ok { - ts = time.Unix(int64(t), 0).Format("15:04") + ts = time.Unix(int64(t), 0).Format("2006-01-02 15:04") } line := msgText if sender != "" { diff --git a/example/rss/README.md b/example/rss/README.md new file mode 100644 index 0000000..34d77a5 --- /dev/null +++ b/example/rss/README.md @@ -0,0 +1,13 @@ +# rss + +rss plugin + +## Build + +```bash +plugindev build +``` + +## Install + +Upload the .hmap file through the Plugin Manager API. diff --git a/example/rss/go.mod b/example/rss/go.mod new file mode 100644 index 0000000..da9d63a --- /dev/null +++ b/example/rss/go.mod @@ -0,0 +1,14 @@ +module rss + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0 + +require ( + github.com/mmcdole/gofeed v1.4.0 // indirect + github.com/mmcdole/goxpp/v2 v2.0.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/text v0.38.0 // indirect +) + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone diff --git a/example/rss/go.sum b/example/rss/go.sum new file mode 100644 index 0000000..869cbea --- /dev/null +++ b/example/rss/go.sum @@ -0,0 +1,8 @@ +github.com/mmcdole/gofeed v1.4.0 h1:+efDmI/yJXJgTfa8we5zg9GAKsU+2d7tnpt9QZwvjLQ= +github.com/mmcdole/gofeed v1.4.0/go.mod h1:ngV5MTB7UJko6fH3/fG5AkB/ABUGK1ZTePF9iRhzu/c= +github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= +github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= diff --git a/example/rss/plg.json b/example/rss/plg.json new file mode 100644 index 0000000..cbe1ec8 --- /dev/null +++ b/example/rss/plg.json @@ -0,0 +1,11 @@ +{ + "name": "rss", + "name_zh": "RSS订阅", + "name_en": "RSS", + "version": "1.0.0", + "description": "RSS/Atom 订阅监控插件，自动检测更新并推送通知", + "author": "HomeAgent", + "entry": "plugin.so", + "tags": ["rss", "feed", "subscription", "monitor"], + "targets": "linux/amd64" +} \ No newline at end of file diff --git a/example/rss/plugin.go b/example/rss/plugin.go new file mode 100644 index 0000000..edc23c7 --- /dev/null +++ b/example/rss/plugin.go @@ -0,0 +1,457 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" + "github.com/mmcdole/gofeed" +) + +type FeedSub struct { + URL string `json:"url"` + Title string `json:"title"` + AddedAt string `json:"added_at"` + Interval int `json:"interval"` +} + +type Plugin struct { + name string + sdk *sdk.PluginSDK + client *http.Client + fp *gofeed.Parser + dataDir string + mu sync.RWMutex + feeds []FeedSub + seenGUIDs map[string]bool + stopCh chan struct{} + wg sync.WaitGroup + pollTicker *time.Ticker +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T { + v, err := s.Get(key) + if err == nil && v != nil { + if sv, ok := v.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + v2, err2 := s.GetCore("plugin." + "rss" + "." + key) + if err2 == nil && v2 != nil { + if sv, ok := v2.(string); ok && sv != "" { + switch any(fallback).(type) { + case string: + return any(sv).(T) + case int64: + if n, err := strconv.ParseInt(sv, 10, 64); err == nil { + return any(n).(T) + } + case float64: + if n, err := strconv.ParseFloat(sv, 64); err == nil { + return any(n).(T) + } + } + } + } + return fallback +} + +func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T { + v, ok := args[key] + if !ok || v == nil { + return fallback + } + switch any(fallback).(type) { + case string: + if s, ok := v.(string); ok { + return any(s).(T) + } + case int64: + switch n := v.(type) { + case float64: + return any(int64(n)).(T) + case int64: + return any(n).(T) + case string: + if i, err := strconv.ParseInt(n, 10, 64); err == nil { + return any(i).(T) + } + } + case float64: + switch n := v.(type) { + case float64: + return any(n).(T) + case int64: + return any(float64(n)).(T) + case string: + if f, err := strconv.ParseFloat(n, 64); err == nil { + return any(f).(T) + } + } + } + return fallback +} + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + p.sdk = s + p.client = &http.Client{Timeout: 30 * time.Second} + p.fp = gofeed.NewParser() + p.stopCh = make(chan struct{}) + p.seenGUIDs = make(map[string]bool) + p.feeds = []FeedSub{} + + dataHome := os.Getenv("HOME") + if dataHome == "" { + dataHome = "/tmp" + } + p.dataDir = filepath.Join(dataHome, ".homeagent", "rss") + os.MkdirAll(p.dataDir, 0755) + p.loadData() + + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.rss.poll_interval", Default: "30", Type: "string", + DisplayName: "Poll Interval", Description: "Default polling interval in minutes (default: 30)", + Category: "rss", + }) + + tp := p.name + "_" + s.RegisterTool(tp+"subscribe", sdk.ToolDef{ + Name: tp + "subscribe", Description: "Subscribe to an RSS/Atom feed URL", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "url": map[string]interface{}{"type": "string", "description": "Feed URL"}, + "interval": map[string]interface{}{"type": "integer", "description": "Poll interval in minutes (default: 30, minimum: 5)"}, + }, + "required": []string{"url"}, + }, + }, p.handleSubscribe) + + s.RegisterTool(tp+"unsubscribe", sdk.ToolDef{ + Name: tp + "unsubscribe", Description: "Unsubscribe from a feed", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "url": map[string]interface{}{"type": "string", "description": "Feed URL to unsubscribe"}, + }, + "required": []string{"url"}, + }, + }, p.handleUnsubscribe) + + s.RegisterTool(tp+"list", sdk.ToolDef{ + Name: tp + "list", Description: "List all subscribed feeds", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleList) + + s.RegisterTool(tp+"check_now", sdk.ToolDef{ + Name: tp + "check_now", Description: "Manually check all feeds for new articles now", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{}, + }, + }, p.handleCheckNow) + + pollMin := int(readCfg(s.Settings(), "poll_interval", int64(30))) + if pollMin < 5 { + pollMin = 5 + } + p.pollTicker = time.NewTicker(time.Duration(pollMin) * time.Minute) + + p.wg.Add(1) + go p.pollLoop() + + fmt.Printf("[%s] started (%d feeds, poll every %dm)\n", p.name, len(p.feeds), pollMin) + return nil +} + +func (p *Plugin) Stop() error { + close(p.stopCh) + p.pollTicker.Stop() + p.wg.Wait() + p.saveData() + fmt.Printf("[%s] stopped\n", p.name) + return nil +} + +func (p *Plugin) pollLoop() { + defer p.wg.Done() + + p.checkAllFeeds() + + for { + select { + case <-p.pollTicker.C: + p.checkAllFeeds() + case <-p.stopCh: + return + } + } +} + +func (p *Plugin) checkAllFeeds() { + p.mu.RLock() + feeds := make([]FeedSub, len(p.feeds)) + copy(feeds, p.feeds) + p.mu.RUnlock() + + for _, feed := range feeds { + select { + case <-p.stopCh: + return + default: + } + p.checkFeed(feed) + } +} + +func (p *Plugin) checkFeed(sub FeedSub) { + parsed, err := p.fp.ParseURL(sub.URL) + if err != nil { + return + } + + title := parsed.Title + if title == "" { + title = sub.URL + } + + var newArticles []*gofeed.Item + for _, item := range parsed.Items { + guid := item.GUID + if guid == "" { + guid = item.Link + } + if guid == "" { + continue + } + guid = sub.URL + "|" + guid + p.mu.RLock() + seen := p.seenGUIDs[guid] + p.mu.RUnlock() + if !seen { + newArticles = append(newArticles, item) + } + } + + if len(newArticles) == 0 { + return + } + + var lines []string + lines = append(lines, fmt.Sprintf("📡 %s (%s) — %d 篇新文章:", title, sub.URL, len(newArticles))) + for _, item := range newArticles { + pubDate := "" + if item.PublishedParsed != nil { + pubDate = item.PublishedParsed.Format("01-02 15:04") + } + line := fmt.Sprintf(" • %s", item.Title) + if pubDate != "" { + line += fmt.Sprintf(" [%s]", pubDate) + } + if item.Link != "" { + line += "\n " + item.Link + } + lines = append(lines, line) + } + + p.sdk.InjectInterruptText("rss", "rss", strings.Join(lines, "\n")) + + p.mu.Lock() + for _, item := range newArticles { + guid := item.GUID + if guid == "" { + guid = item.Link + } + if guid == "" { + continue + } + p.seenGUIDs[sub.URL+"|"+guid] = true + } + p.mu.Unlock() + p.saveData() +} + +func (p *Plugin) handleSubscribe(args map[string]interface{}) (interface{}, error) { + url := readArg(args, "url", "") + if url == "" { + return map[string]interface{}{"isError": true, "content": "URL is required"}, nil + } + + p.mu.RLock() + for _, f := range p.feeds { + if f.URL == url { + p.mu.RUnlock() + return map[string]interface{}{"isError": true, "content": "Already subscribed to: " + url}, nil + } + } + p.mu.RUnlock() + + interval := int(readArg(args, "interval", int64(30))) + if interval < 5 { + interval = 5 + } + + parsed, err := p.fp.ParseURL(url) + if err != nil { + return map[string]interface{}{"isError": true, "content": "Failed to parse feed: " + err.Error()}, nil + } + + feedTitle := parsed.Title + if feedTitle == "" { + feedTitle = url + } + + sub := FeedSub{ + URL: url, + Title: feedTitle, + AddedAt: time.Now().Format("2006-01-02 15:04"), + Interval: interval, + } + + guidCount := 0 + for _, item := range parsed.Items { + guid := item.GUID + if guid == "" { + guid = item.Link + } + if guid == "" { + continue + } + p.seenGUIDs[url+"|"+guid] = true + guidCount++ + } + + p.mu.Lock() + p.feeds = append(p.feeds, sub) + p.mu.Unlock() + p.saveData() + + return map[string]interface{}{ + "content": fmt.Sprintf("Subscribed to: %s\nTitle: %s\nArticles found: %d\nPoll interval: %d min", url, feedTitle, guidCount, interval), + }, nil +} + +func (p *Plugin) handleUnsubscribe(args map[string]interface{}) (interface{}, error) { + url := readArg(args, "url", "") + if url == "" { + return map[string]interface{}{"isError": true, "content": "URL is required"}, nil + } + + p.mu.Lock() + found := false + for i, f := range p.feeds { + if f.URL == url { + p.feeds = append(p.feeds[:i], p.feeds[i+1:]...) + found = true + break + } + } + if !found { + p.mu.Unlock() + return map[string]interface{}{"isError": true, "content": "Not subscribed to: " + url}, nil + } + + for guid := range p.seenGUIDs { + if strings.HasPrefix(guid, url+"|") { + delete(p.seenGUIDs, guid) + } + } + p.mu.Unlock() + p.saveData() + + return map[string]interface{}{"content": "Unsubscribed: " + url}, nil +} + +func (p *Plugin) handleList(args map[string]interface{}) (interface{}, error) { + p.mu.RLock() + defer p.mu.RUnlock() + + if len(p.feeds) == 0 { + return map[string]interface{}{"content": "No subscriptions. Use rss_subscribe to add one."}, nil + } + + sort.Slice(p.feeds, func(i, j int) bool { + return p.feeds[i].Title < p.feeds[j].Title + }) + + var lines []string + lines = append(lines, fmt.Sprintf("📡 Subscriptions (%d):", len(p.feeds))) + for _, f := range p.feeds { + lines = append(lines, fmt.Sprintf(" • %s\n %s (every %dm, added %s)", f.Title, f.URL, f.Interval, f.AddedAt)) + } + + return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil +} + +func (p *Plugin) handleCheckNow(args map[string]interface{}) (interface{}, error) { + go p.checkAllFeeds() + return map[string]interface{}{"content": "Checking all feeds for updates..."}, nil +} + +func (p *Plugin) dataFile() string { + return filepath.Join(p.dataDir, "feeds.json") +} + +func (p *Plugin) loadData() { + b, err := os.ReadFile(p.dataFile()) + if err != nil { + return + } + var data struct { + Feeds []FeedSub `json:"feeds"` + SeenGUIDs map[string]bool `json:"seen"` + } + if json.Unmarshal(b, &data) != nil { + return + } + if data.Feeds != nil { + p.feeds = data.Feeds + } + if data.SeenGUIDs != nil { + p.seenGUIDs = data.SeenGUIDs + } +} + +func (p *Plugin) saveData() { + p.mu.RLock() + defer p.mu.RUnlock() + data := struct { + Feeds []FeedSub `json:"feeds"` + SeenGUIDs map[string]bool `json:"seen"` + }{ + Feeds: p.feeds, + SeenGUIDs: p.seenGUIDs, + } + b, _ := json.MarshalIndent(data, "", " ") + os.WriteFile(p.dataFile(), b, 0644) +} + + diff --git a/example/weather/README.md b/example/weather/README.md new file mode 100644 index 0000000..3dab20d --- /dev/null +++ b/example/weather/README.md @@ -0,0 +1,13 @@ +# weather + +weather plugin + +## Build + +```bash +plugindev build +``` + +## Install + +Upload the .hmap file through the Plugin Manager API. diff --git a/example/weather/go.mod b/example/weather/go.mod new file mode 100644 index 0000000..847bb7b --- /dev/null +++ b/example/weather/go.mod @@ -0,0 +1,7 @@ +module weather + +go 1.25.0 + +require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0 + +replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone diff --git a/example/weather/plg.json b/example/weather/plg.json new file mode 100644 index 0000000..d91b5b8 --- /dev/null +++ b/example/weather/plg.json @@ -0,0 +1,11 @@ +{ + "name": "weather", + "name_zh": "天气查询", + "name_en": "Weather", + "version": "1.0.0", + "description": "天气查询插件（基于 wttr.in），支持实时天气和未来预报", + "author": "HomeAgent", + "entry": "plugin.so", + "tags": ["weather", "forecast", "wttr"], + "targets": "linux/amd64" +} \ No newline at end of file diff --git a/example/weather/plugin.go b/example/weather/plugin.go new file mode 100644 index 0000000..7b46a46 --- /dev/null +++ b/example/weather/plugin.go @@ -0,0 +1,388 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk" +) + +type Plugin struct { + name string + sdk *sdk.PluginSDK + client *http.Client + defaultLoc string +} + +func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) { + return &Plugin{name: name}, nil +} + +func (p *Plugin) Name() string { return p.name } + +func (p *Plugin) Start(s *sdk.PluginSDK) error { + s.SetAutoRestart(true) + p.sdk = s + p.client = &http.Client{Timeout: 15 * time.Second} + + loc, err := s.Settings().Get("default_location") + if err == nil && loc != nil { + if v, ok := loc.(string); ok && v != "" { + p.defaultLoc = v + } + } + if p.defaultLoc == "" { + v, err := s.Settings().GetCore("plugin.weather.default_location") + if err == nil && v != nil { + if vs, ok := v.(string); ok && vs != "" { + p.defaultLoc = vs + } + } + } + + dataHome := os.Getenv("HOME") + if dataHome == "" { + dataHome = "/tmp" + } + dataDir := filepath.Join(dataHome, ".homeagent", "weather") + os.MkdirAll(dataDir, 0755) + + s.Settings().RegisterDef(sdk.ConfigDef{ + Key: "plugin.weather.default_location", Default: "", Type: "string", + DisplayName: "Default Location", Description: "Default city name for weather queries, e.g. Beijing", + Category: "weather", + }) + + tp := p.name + "_" + s.RegisterTool(tp+"current", sdk.ToolDef{ + Name: tp + "current", Description: "Get current weather for a city", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "location": map[string]interface{}{"type": "string", "description": "City name (e.g. Beijing, Shanghai, London). Uses default if omitted."}, + "units": map[string]interface{}{"type": "string", "description": "Units: metric (celsius) or imperial (fahrenheit), default metric"}, + }, + }, + }, p.handleCurrent) + + s.RegisterTool(tp+"forecast", sdk.ToolDef{ + Name: tp + "forecast", Description: "Get weather forecast for next several days", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "location": map[string]interface{}{"type": "string", "description": "City name. Uses default if omitted."}, + "days": map[string]interface{}{"type": "integer", "description": "Number of days (1-7), default 3"}, + "units": map[string]interface{}{"type": "string", "description": "Units: metric or imperial, default metric"}, + }, + }, + }, p.handleForecast) + + s.RegisterTool(tp+"set_location", sdk.ToolDef{ + Name: tp + "set_location", Description: "Set default weather location", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "location": map[string]interface{}{"type": "string", "description": "City name to set as default"}, + }, + "required": []string{"location"}, + }, + }, p.handleSetLocation) + + fmt.Printf("[%s] started\n", p.name) + return nil +} + +func (p *Plugin) Stop() error { + fmt.Printf("[%s] stopped\n", p.name) + return nil +} + +type wttrResp struct { + CurrentCondition []struct { + TempC string `json:"temp_C"` + FeelsLikeC string `json:"FeelsLikeC"` + Humidity string `json:"humidity"` + WindspeedKmph string `json:"windspeedKmph"` + Winddir16Point string `json:"winddir16Point"` + Pressure string `json:"pressure"` + Visibility string `json:"visibility"` + WeatherDesc []struct { + Value string `json:"value"` + } `json:"weatherDesc"` + LocalObsDateTime string `json:"localObsDateTime"` + } `json:"current_condition"` + NearestArea []struct { + AreaName []struct { + Value string `json:"value"` + } `json:"areaName"` + Country []struct { + Value string `json:"value"` + } `json:"country"` + Region []struct { + Value string `json:"value"` + } `json:"region"` + } `json:"nearest_area"` + Weather []wttrDay `json:"weather"` +} + +type wttrDay struct { + Date string `json:"date"` + Astronomy []struct { + Sunrise string `json:"sunrise"` + Sunset string `json:"sunset"` + } `json:"astronomy"` + MaxtempC string `json:"maxtempC"` + MintempC string `json:"mintempC"` + Hourly []struct { + TempC string `json:"tempC"` + WeatherDesc []struct { + Value string `json:"value"` + } `json:"weatherDesc"` + WindspeedKmph string `json:"windspeedKmph"` + Winddir16Point string `json:"winddir16Point"` + Humidity string `json:"humidity"` + FeelsLikeC string `json:"FeelsLikeC"` + PrecipMM string `json:"precipMM"` + Visibility string `json:"visibility"` + } `json:"hourly"` +} + +func (p *Plugin) getLoc(args map[string]interface{}) string { + if v, ok := args["location"].(string); ok && v != "" { + return v + } + return p.defaultLoc +} + +func (p *Plugin) getUnits(args map[string]interface{}) string { + if v, ok := args["units"].(string); ok && (v == "imperial" || v == "metric") { + return v + } + return "metric" +} + +func (p *Plugin) fetchWttr(location string) (*wttrResp, error) { + url := fmt.Sprintf("https://wttr.in/%s?format=j1", strings.ReplaceAll(location, " ", "%20")) + resp, err := p.client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + var data wttrResp + if err := json.Unmarshal(body, &data); err != nil { + return nil, err + } + if len(data.CurrentCondition) == 0 { + return nil, fmt.Errorf("no weather data for: %s", location) + } + return &data, nil +} + +func (p *Plugin) displayName(data *wttrResp) string { + if len(data.NearestArea) == 0 { + return "Unknown" + } + area := data.NearestArea[0] + name := "" + if len(area.AreaName) > 0 { + name = area.AreaName[0].Value + } + region := "" + if len(area.Region) > 0 { + region = area.Region[0].Value + } + country := "" + if len(area.Country) > 0 { + country = area.Country[0].Value + } + var parts []string + if name != "" { + parts = append(parts, name) + } + if region != "" && region != name { + parts = append(parts, region) + } + if country != "" { + parts = append(parts, country) + } + return strings.Join(parts, ", ") +} + +func convertCtoF(c string) string { + if v, err := strconv.ParseFloat(c, 64); err == nil { + return fmt.Sprintf("%.0f", v*9/5+32) + } + return c +} + +func (p *Plugin) handleCurrent(args map[string]interface{}) (interface{}, error) { + location := p.getLoc(args) + if location == "" { + return map[string]interface{}{"isError": true, "content": "No location specified. Provide a city name or set default_location."}, nil + } + + units := p.getUnits(args) + + data, err := p.fetchWttr(location) + if err != nil { + return map[string]interface{}{"isError": true, "content": "Weather request failed: " + err.Error()}, nil + } + + cc := data.CurrentCondition[0] + place := p.displayName(data) + + desc := "" + if len(cc.WeatherDesc) > 0 { + desc = cc.WeatherDesc[0].Value + } + + unitStr := "°C" + windUnit := "km/h" + tempStr := cc.TempC + feelsStr := cc.FeelsLikeC + if units == "imperial" { + unitStr = "°F" + windUnit = "mph" + tempStr = convertCtoF(tempStr) + feelsStr = convertCtoF(feelsStr) + } + + obsTime := cc.LocalObsDateTime + if len(obsTime) > 16 { + obsTime = obsTime[:16] + } + + result := fmt.Sprintf("🌤 %s — %s\n🌡 %s%s (体感 %s%s)\n💧 湿度 %s%% | 💨 风速 %s %s %s\n🕐 %s", + place, desc, + tempStr, unitStr, feelsStr, unitStr, + cc.Humidity, cc.WindspeedKmph, windUnit, cc.Winddir16Point, + obsTime) + + return map[string]interface{}{ + "content": result, + "location": place, + "temp": cc.TempC, + "feels_like": cc.FeelsLikeC, + "humidity": cc.Humidity, + "wind_speed": cc.WindspeedKmph, + "weather": desc, + "observed": obsTime, + }, nil +} + +func (p *Plugin) handleForecast(args map[string]interface{}) (interface{}, error) { + location := p.getLoc(args) + if location == "" { + return map[string]interface{}{"isError": true, "content": "No location specified."}, nil + } + + days := 3 + if v, ok := args["days"].(float64); ok { + d := int(v) + if d >= 1 && d <= 7 { + days = d + } + } + + units := p.getUnits(args) + + data, err := p.fetchWttr(location) + if err != nil { + return map[string]interface{}{"isError": true, "content": "Forecast request failed: " + err.Error()}, nil + } + + place := p.displayName(data) + + unitStr := "°C" + if units == "imperial" { + unitStr = "°F" + } + + dayCount := days + if dayCount > len(data.Weather) { + dayCount = len(data.Weather) + } + daysData := data.Weather[:dayCount] + + var lines []string + lines = append(lines, fmt.Sprintf("📅 %d日天气预报 — %s", days, place)) + for _, day := range daysData { + t, err := time.Parse("2006-01-02", day.Date) + if err != nil { + continue + } + weekday := t.Weekday().String()[:3] + + maxT := day.MaxtempC + minT := day.MintempC + desc := "" + precip := "" + + if len(day.Hourly) > 0 { + mid := len(day.Hourly) / 2 + if len(day.Hourly[mid].WeatherDesc) > 0 { + desc = day.Hourly[mid].WeatherDesc[0].Value + } + totalPrecip := 0.0 + for _, h := range day.Hourly { + if pv, err := strconv.ParseFloat(h.PrecipMM, 64); err == nil { + totalPrecip += pv + } + } + if totalPrecip > 0 { + precip = fmt.Sprintf(" 🌧%.1fmm", totalPrecip) + } + } + + if units == "imperial" { + maxT = convertCtoF(maxT) + minT = convertCtoF(minT) + } + + sunrise, sunset := "", "" + if len(day.Astronomy) > 0 { + sunrise = day.Astronomy[0].Sunrise + sunset = day.Astronomy[0].Sunset + } + + line := fmt.Sprintf(" %s %s/%s — %s~%s%s %s", weekday, day.Date[5:], day.Date[8:], minT, maxT, unitStr, desc) + if precip != "" { + line += precip + } + if sunrise != "" && sunset != "" { + line += fmt.Sprintf(" 🌅%s 🌇%s", sunrise, sunset) + } + lines = append(lines, line) + } + + cc := data.CurrentCondition[0] + nowDesc := "" + if len(cc.WeatherDesc) > 0 { + nowDesc = cc.WeatherDesc[0].Value + } + lines = append(lines, fmt.Sprintf("\n当前：%s %s°C", nowDesc, cc.TempC)) + + return map[string]interface{}{ + "content": strings.Join(lines, "\n"), + "location": place, + }, nil +} + +func (p *Plugin) handleSetLocation(args map[string]interface{}) (interface{}, error) { + loc, _ := args["location"].(string) + if loc == "" { + return map[string]interface{}{"isError": true, "content": "Location is required"}, nil + } + + p.sdk.Settings().SetCore("plugin.weather.default_location", loc) + p.defaultLoc = loc + return map[string]interface{}{"content": fmt.Sprintf("Default location set to: %s", loc)}, nil +} diff --git a/tools/gengskill/.gitignore b/tools/gengskill/.gitignore new file mode 100644 index 0000000..f257c24 --- /dev/null +++ b/tools/gengskill/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +.DS_Store +report/ +figures/ +*.egg-info/ diff --git a/tools/gengskill/LICENSE b/tools/gengskill/LICENSE new file mode 100644 index 0000000..858f61f --- /dev/null +++ b/tools/gengskill/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Geng Skill Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/gengskill/README.md b/tools/gengskill/README.md new file mode 100644 index 0000000..7c04874 --- /dev/null +++ b/tools/gengskill/README.md @@ -0,0 +1,319 @@ +

    + Geng Skill Banner +

    + +

    🔬 Geng Skill — 学术数据打假检测工具

    + +

    + 用数据说话，让造假无所遁形 · Inspired by "耿同学讲故事" +

    + +

    + Python 3.8+ + MIT License + Version 2.0.0 + Tests Passing +

    + +

    + Quick Start • + How It Works • + Input Modes • + Detection Modules • + Example • + Documentation +

    + +--- + +## 🌟 这是什么？ + +**Geng Skill** 是一套基于统计学原理的学术论文数据造假检测工具包。 + +2026 年 4 月起，科普博主"耿同学讲故事"凭一台电脑和几个统计方法，连续揪出多所 985 高校顶尖学者的论文造假——同济大学 Nature 论文院长免职、南开大学 Nature 子刊正在调查……他证明了一件事：**造假的数据一定会留下统计学破绽。** + +本项目将"耿同学"的技术方法论系统化、工具化，让任何人都能一键检测论文数据是否存在造假嫌疑。 + +> 💡 **核心原理**：真实实验数据具有随机性；人为编造的数据会呈现不自然的数学规律。 + +--- + +## 🚀 快速开始 + +```bash +# 克隆仓库 +git clone https://github.com/YOUR_USERNAME/geng-skill.git +cd geng-skill +pip install -r requirements.txt + +# 一键自动扫描（Scale 模式） +python3 scripts/input_pipeline.py --input your_data.csv --mode scale +``` + +就这么简单。Scale 模式会**自动扫描所有数值列**，运行 6 种检测算法，并按嫌疑程度从高到低排列结果。 + +--- + +## 🧠 工作原理 + +

    + System Architecture +

    + +系统分为三层： + +### 第一层 · 数据输入 Input Pipeline + +支持 **PDF 论文**、**Excel 原始数据**、**CSV 表格** 三种格式。自动提取表格、识别数值列、标准化数据格式。 + +### 第二层 · 检测引擎 Detection Engine + +6 个独立的统计检测模块，从不同角度分析数据异常： + +| 模块 | 检测目标 | 方法 | +|------|----------|------| +| **末位数字检测** Last Digit | 末位数字集中度异常 | Chi-squared vs 均匀分布 | +| **本福特定律** Benford's Law | 首位数字分布偏离 | Chi-squared + MAD | +| **GRIM 测试** | 均值与样本量不兼容 | 离散粒度校验 | +| **固定关系检测** Fixed Ratio ⭐ | 实验组间存在完美数学关系 | 比值/回归分析 | +| **小数位一致性** Decimal | 小数部分模式重复 | 自相关 + 熵分析 | +| **图像重复检测** Image Dup | 同一图片在不同条件下重复使用 | 感知哈希 + SSIM | + +### 第三层 · 输出报告 Output + +生成出版级可视化图表、综合风险评分（0–100）、以及详细的 HTML/Markdown 检测报告，精确标注每个可疑数据点。 + +--- + +## 📥 三种输入模式 + +

    + Workflow +

    + +### 模式一：论文 PDF 输入 + +直接从 PDF 论文中提取数据表格： + +```bash +python3 scripts/input_pipeline.py --input paper.pdf --mode extract +``` + +### 模式二：Excel / CSV 原始数据 + +处理从论文下载的 Supplementary Data： + +```bash +python3 scripts/input_pipeline.py --input supplementary_data.xlsx --mode extract +python3 scripts/input_pipeline.py --input table_s1.csv --mode extract +``` + +### 模式三：Scale 自动扫描 ⭐（推荐） + +**让工具自己去找问题。** 无需指定检测哪些列、用哪些方法——全部自动： + +```bash +python3 scripts/input_pipeline.py --input data.csv --mode scale +``` + +Scale 模式会： +1. 自动识别所有数值列 +2. 运行 6 大检测模块 +3. 交叉验证各模块结果 +4. 输出 **suspicion_ranking**（嫌疑排名榜） + +--- + +## 🔬 检测模块 + +### ⭐ 固定关系检测（核心 · 耿同学的看家本领） + +这是最致命的检测手段。如果两组"独立实验"数据之间存在**完美的固定数学关系**（比如每个样本 Treatment = Control × 2.0），那几乎可以断定是造假。 + +```python +from fixed_relation_test import fixed_relation_test + +result = fixed_relation_test(control_data, treatment_data) +# result['risk_score'] = 95 → 检测到精确 2.0 倍关系！ +``` + +**为什么这能定性？** 即使药物真的让蛋白表达提高 2 倍，每个样本也会有生物学个体差异。20 个样本**全部**精确到小数点后多位都是 2.000 倍——概率趋近于零。 + +### 本福特定律检测 Benford's Law + +跨越多个数量级的自然数据，首位数字遵循特定概率分布（1 最多，9 最少）。人为编造的数据会偏离： + +```python +from benford_test import benford_test +result = benford_test(values) +``` + +### GRIM 测试 + +对于整数取值的数据（如李克特量表 1–5 分），给定样本量 n，并非所有均值都是数学上可能的： + +```python +from grim_test import grim_test_single +result = grim_test_single(mean='3.47', n=25, decimals=2) +# result['consistent'] = False → 这个均值是不可能存在的！ +``` + +--- + +## 🧪 实战案例 + +### 输入：一篇可疑的生物医学论文数据 + +论文声称对小鼠进行了三组独立实验（Control / Treatment A / Treatment B），数据如下： + +```csv +sample_id,control,treatment_a,treatment_b +1,2.34,4.68,7.02 +2,3.12,6.24,9.36 +3,1.87,3.74,5.61 +4,4.56,9.12,13.68 +5,2.98,5.96,8.94 +... +``` + +### 运行 Scale 模式自动检测 + +```bash +python3 scripts/input_pipeline.py --input data.csv --mode scale +``` + +### 输出：精准定位问题 + +``` +=== Scale Mode: 自动检测结果 === + + 🔴 [1.00] control ↔ treatment_a ← 精确固定比值 = 2.000 + 🔴 [1.00] control ↔ treatment_b ← 精确固定比值 = 3.000 + 🔴 [1.00] treatment_a ↔ treatment_b ← 精确固定比值 = 1.500 + 🟠 [0.99] treatment_a ← 末位数字分布异常 + + ╔══════════════════════════════════════════════════════════════╗ + ║ 综合风险评分: 92/100 🔴 极高风险 ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ 三组"独立实验"数据之间存在完美的整数倍关系。 ║ + ║ 在真实生物实验中，这种情况出现的概率约等于零。 ║ + ║ 数据极大概率为人工编造。 ║ + ╚══════════════════════════════════════════════════════════════╝ +``` + +--- + +## 📊 风险评分体系 + +| 分数 | 等级 | 含义 | 建议行动 | +|------|------|------|----------| +| 0–25 | 🟢 低风险 | 未发现异常 | 无需干预 | +| 26–50 | 🟡 中等 | 存在轻微模式，可能是正常波动 | 建议复核 | +| 51–75 | 🟠 高风险 | 多项指标异常 | 深入调查 | +| 76–100 | 🔴 极高风险 | 系统性异常 | 正式举报 | + +**置信度规则：** +- 单一模块报警 → 标注为"待确认线索" +- 2 个以上独立模块同时报警 → 标注为"高度可疑" +- 仅当综合分 > 75 **且**多模块交叉确认时，才标注"极高风险" + +--- + +## ⚖️ 准确性与免责声明 + +### ✅ 本工具能做什么 + +- 检测不同数据列之间的固定数学关系 +- 识别数字分布的统计学异常 +- 标记数学上不可能的统计报告值 +- 发现重复使用/篡改的论文图片 +- 提供量化的风险评估和置信度等级 + +### ❌ 本工具不能做什么 + +- 不能证明造假的主观意图 +- 不能检测加了随机噪声的"高明造假" +- 不能替代领域专家的判断 +- 不具备法律效力 + +### ⚠️ 重要声明 + +> **本工具仅提供统计学层面的异常筛查功能。** +> 输出结果为"疑点线索"（Suspicious Indicators），而非"造假判定"（Fraud Determination）。 +> +> - 统计异常可能有合理的科学解释（仪器精度限制、数据标准化处理、单位转换等） +> - 最终判定需要领域专家复核和正式调查程序 +> - 通过全部检测 ≠ 数据一定真实（某些造假无法被统计方法捕获） +> - 使用者需自行承担因不当使用（如公开发布未经验证的指控）造成的一切后果 + +--- + +## 📁 项目结构 + +``` +geng-skill/ +├── scripts/ 核心引擎 +│ ├── input_pipeline.py 统一输入（PDF/Excel/CSV + Scale 模式） +│ ├── visualization.py 出版级可视化图表 +│ ├── report_generator.py HTML + Markdown 报告生成 +│ ├── geng_assess.py 综合评估引擎 +│ ├── last_digit_test.py 末位数字检测 +│ ├── benford_test.py 本福特定律检测 +│ ├── grim_test.py GRIM 均值一致性测试 +│ ├── fixed_relation_test.py 固定关系检测 ⭐ +│ ├── decimal_consistency_test.py 小数位一致性检测 +│ └── image_duplicate_test.py 图像重复检测 +├── docs/ 完整文档 +│ ├── USAGE_GUIDE.md 多平台使用指南（Claude/Cursor/GPT/Codex 等） +│ ├── DATA_SOURCES.md 学术参考文献 + 数据标准 + 伦理合规 +│ ├── ANNOTATIONS.md 架构图 + API 接口 + 代码注释规范 +│ └── EXAMPLE_WALKTHROUGH.md 端到端完整教程 +├── examples/ 示例数据 +├── tests/ 单元测试（24/24 通过） +└── assets/ README 配图 +``` + +--- + +## 📖 文档 + +| 文档 | 内容 | +|------|------| +| [USAGE_GUIDE.md](docs/USAGE_GUIDE.md) | 在 Claude / Cursor / GPT / Codex / Jupyter / Docker 等平台上的使用方法 | +| [DATA_SOURCES.md](docs/DATA_SOURCES.md) | 学术参考文献、数据标准、伦理合规框架 | +| [ANNOTATIONS.md](docs/ANNOTATIONS.md) | 系统架构、API 接口规范、代码注释标准 | +| [EXAMPLE_WALKTHROUGH.md](docs/EXAMPLE_WALKTHROUGH.md) | 从一篇论文到检测报告的完整教程 | +| [SKILL.md](SKILL.md) | Skill 核心技术文档 | + +--- + +## 🔗 学术参考 + +1. Benford, F. (1938). The law of anomalous numbers. *Proc. APS*, 78(4), 551–572. +2. Brown, N.J.L. & Heathers, J.A.J. (2017). The GRIM Test. *SPPS*, 8(4), 363–369. +3. Bik, E.M. et al. (2016). Image duplication in biomedical research. *mBio*, 7(3). +4. Nigrini, M.J. (2012). *Benford's Law*. Wiley. ISBN: 978-1118152850. +5. 余菁等 (2021). 科技论文数据造假的核查策略. *中国科技期刊研究*, 32(6), 770–776. + +--- + +## 🙏 致谢 + +本项目的灵感来源于 **"耿同学讲故事"** —— 一位吉林大学生物学硕士、北航退学博士，从 2026 年 4 月开始，仅凭一台电脑和统计学方法，就揪出了多所顶尖高校教授的论文数据造假。他的工作证明了：**学术诚信监督不仅必要，而且完全可行。** + +> "如果论文里的数据存在规律性，那么就明显不是在实验室实际测量的情况下生成的。" +> +> —— 耿同学 + +--- + +## 📄 开源许可 + +MIT License — 详见 [LICENSE](LICENSE) + +--- + +

    + 让学术回归诚信，让数据说出真相。
    + Let academic integrity prevail. Let data speak the truth. +

    diff --git a/tools/gengskill/README_CN.md b/tools/gengskill/README_CN.md new file mode 100644 index 0000000..cb96707 --- /dev/null +++ b/tools/gengskill/README_CN.md @@ -0,0 +1,319 @@ +

    + Geng Skill Banner +

    + +

    🔬 Geng Skill — 学术数据打假检测工具

    + +

    + 用数据说话，让造假无所遁形 · Inspired by "耿同学讲故事" +

    + +

    + Python 3.8+ + MIT License + Version 2.0.0 + Tests Passing +

    + +

    + Quick Start • + How It Works • + Input Modes • + Detection Modules • + Example • + Documentation +

    + +--- + +## 🌟 这是什么？ + +**Geng Skill** 是一套基于统计学原理的学术论文数据造假检测工具包。 + +2026 年 4 月起，科普博主"耿同学讲故事"凭一台电脑和几个统计方法，连续揪出多所 985 高校顶尖学者的论文造假——同济大学 Nature 论文院长免职、南开大学 Nature 子刊正在调查……他证明了一件事：**造假的数据一定会留下统计学破绽。** + +本项目将"耿同学"的技术方法论系统化、工具化，让任何人都能一键检测论文数据是否存在造假嫌疑。 + +> 💡 **核心原理**：真实实验数据具有随机性；人为编造的数据会呈现不自然的数学规律。 + +--- + +## 🚀 快速开始 + +```bash +# 克隆仓库 +git clone https://github.com/YOUR_USERNAME/geng-skill.git +cd geng-skill +pip install -r requirements.txt + +# 一键自动扫描（Scale 模式） +python3 scripts/input_pipeline.py --input your_data.csv --mode scale +``` + +就这么简单。Scale 模式会**自动扫描所有数值列**，运行 6 种检测算法，并按嫌疑程度从高到低排列结果。 + +--- + +## 🧠 工作原理 + +

    + System Architecture +

    + +系统分为三层： + +### 第一层 · 数据输入 Input Pipeline + +支持 **PDF 论文**、**Excel 原始数据**、**CSV 表格** 三种格式。自动提取表格、识别数值列、标准化数据格式。 + +### 第二层 · 检测引擎 Detection Engine + +6 个独立的统计检测模块，从不同角度分析数据异常： + +| 模块 | 检测目标 | 方法 | +|------|----------|------| +| **末位数字检测** Last Digit | 末位数字集中度异常 | Chi-squared vs 均匀分布 | +| **本福特定律** Benford's Law | 首位数字分布偏离 | Chi-squared + MAD | +| **GRIM 测试** | 均值与样本量不兼容 | 离散粒度校验 | +| **固定关系检测** Fixed Ratio ⭐ | 实验组间存在完美数学关系 | 比值/回归分析 | +| **小数位一致性** Decimal | 小数部分模式重复 | 自相关 + 熵分析 | +| **图像重复检测** Image Dup | 同一图片在不同条件下重复使用 | 感知哈希 + SSIM | + +### 第三层 · 输出报告 Output + +生成出版级可视化图表、综合风险评分（0–100）、以及详细的 HTML/Markdown 检测报告，精确标注每个可疑数据点。 + +--- + +## 📥 三种输入模式 + +

    + Workflow +

    + +### 模式一：论文 PDF 输入 + +直接从 PDF 论文中提取数据表格： + +```bash +python3 scripts/input_pipeline.py --input paper.pdf --mode extract +``` + +### 模式二：Excel / CSV 原始数据 + +处理从论文下载的 Supplementary Data： + +```bash +python3 scripts/input_pipeline.py --input supplementary_data.xlsx --mode extract +python3 scripts/input_pipeline.py --input table_s1.csv --mode extract +``` + +### 模式三：Scale 自动扫描 ⭐（推荐） + +**让工具自己去找问题。** 无需指定检测哪些列、用哪些方法——全部自动： + +```bash +python3 scripts/input_pipeline.py --input data.csv --mode scale +``` + +Scale 模式会： +1. 自动识别所有数值列 +2. 运行 6 大检测模块 +3. 交叉验证各模块结果 +4. 输出 **suspicion_ranking**（嫌疑排名榜） + +--- + +## 🔬 检测模块 + +### ⭐ 固定关系检测（核心 · 耿同学的看家本领） + +这是最致命的检测手段。如果两组"独立实验"数据之间存在**完美的固定数学关系**（比如每个样本 Treatment = Control × 2.0），那几乎可以断定是造假。 + +```python +from fixed_relation_test import fixed_relation_test + +result = fixed_relation_test(control_data, treatment_data) +# result['risk_score'] = 95 → 检测到精确 2.0 倍关系！ +``` + +**为什么这能定性？** 即使药物真的让蛋白表达提高 2 倍，每个样本也会有生物学个体差异。20 个样本**全部**精确到小数点后多位都是 2.000 倍——概率趋近于零。 + +### 本福特定律检测 Benford's Law + +跨越多个数量级的自然数据，首位数字遵循特定概率分布（1 最多，9 最少）。人为编造的数据会偏离： + +```python +from benford_test import benford_test +result = benford_test(values) +``` + +### GRIM 测试 + +对于整数取值的数据（如李克特量表 1–5 分），给定样本量 n，并非所有均值都是数学上可能的： + +```python +from grim_test import grim_test_single +result = grim_test_single(mean='3.47', n=25, decimals=2) +# result['consistent'] = False → 这个均值是不可能存在的！ +``` + +--- + +## 🧪 实战案例 + +### 输入：一篇可疑的生物医学论文数据 + +论文声称对小鼠进行了三组独立实验（Control / Treatment A / Treatment B），数据如下： + +```csv +sample_id,control,treatment_a,treatment_b +1,2.34,4.68,7.02 +2,3.12,6.24,9.36 +3,1.87,3.74,5.61 +4,4.56,9.12,13.68 +5,2.98,5.96,8.94 +... +``` + +### 运行 Scale 模式自动检测 + +```bash +python3 scripts/input_pipeline.py --input data.csv --mode scale +``` + +### 输出：精准定位问题 + +``` +=== Scale Mode: 自动检测结果 === + + 🔴 [1.00] control ↔ treatment_a ← 精确固定比值 = 2.000 + 🔴 [1.00] control ↔ treatment_b ← 精确固定比值 = 3.000 + 🔴 [1.00] treatment_a ↔ treatment_b ← 精确固定比值 = 1.500 + 🟠 [0.99] treatment_a ← 末位数字分布异常 + + ╔══════════════════════════════════════════════════════════════╗ + ║ 综合风险评分: 92/100 🔴 极高风险 ║ + ╠══════════════════════════════════════════════════════════════╣ + ║ 三组"独立实验"数据之间存在完美的整数倍关系。 ║ + ║ 在真实生物实验中，这种情况出现的概率约等于零。 ║ + ║ 数据极大概率为人工编造。 ║ + ╚══════════════════════════════════════════════════════════════╝ +``` + +--- + +## 📊 风险评分体系 + +| 分数 | 等级 | 含义 | 建议行动 | +|------|------|------|----------| +| 0–25 | 🟢 低风险 | 未发现异常 | 无需干预 | +| 26–50 | 🟡 中等 | 存在轻微模式，可能是正常波动 | 建议复核 | +| 51–75 | 🟠 高风险 | 多项指标异常 | 深入调查 | +| 76–100 | 🔴 极高风险 | 系统性异常 | 正式举报 | + +**置信度规则：** +- 单一模块报警 → 标注为"待确认线索" +- 2 个以上独立模块同时报警 → 标注为"高度可疑" +- 仅当综合分 > 75 **且**多模块交叉确认时，才标注"极高风险" + +--- + +## ⚖️ 准确性与免责声明 + +### ✅ 本工具能做什么 + +- 检测不同数据列之间的固定数学关系 +- 识别数字分布的统计学异常 +- 标记数学上不可能的统计报告值 +- 发现重复使用/篡改的论文图片 +- 提供量化的风险评估和置信度等级 + +### ❌ 本工具不能做什么 + +- 不能证明造假的主观意图 +- 不能检测加了随机噪声的"高明造假" +- 不能替代领域专家的判断 +- 不具备法律效力 + +### ⚠️ 重要声明 + +> **本工具仅提供统计学层面的异常筛查功能。** +> 输出结果为"疑点线索"（Suspicious Indicators），而非"造假判定"（Fraud Determination）。 +> +> - 统计异常可能有合理的科学解释（仪器精度限制、数据标准化处理、单位转换等） +> - 最终判定需要领域专家复核和正式调查程序 +> - 通过全部检测 ≠ 数据一定真实（某些造假无法被统计方法捕获） +> - 使用者需自行承担因不当使用（如公开发布未经验证的指控）造成的一切后果 + +--- + +## 📁 项目结构 + +``` +geng-skill/ +├── scripts/ 核心引擎 +│ ├── input_pipeline.py 统一输入（PDF/Excel/CSV + Scale 模式） +│ ├── visualization.py 出版级可视化图表 +│ ├── report_generator.py HTML + Markdown 报告生成 +│ ├── geng_assess.py 综合评估引擎 +│ ├── last_digit_test.py 末位数字检测 +│ ├── benford_test.py 本福特定律检测 +│ ├── grim_test.py GRIM 均值一致性测试 +│ ├── fixed_relation_test.py 固定关系检测 ⭐ +│ ├── decimal_consistency_test.py 小数位一致性检测 +│ └── image_duplicate_test.py 图像重复检测 +├── docs/ 完整文档 +│ ├── USAGE_GUIDE.md 多平台使用指南（Claude/Cursor/GPT/Codex 等） +│ ├── DATA_SOURCES.md 学术参考文献 + 数据标准 + 伦理合规 +│ ├── ANNOTATIONS.md 架构图 + API 接口 + 代码注释规范 +│ └── EXAMPLE_WALKTHROUGH.md 端到端完整教程 +├── examples/ 示例数据 +├── tests/ 单元测试（24/24 通过） +└── assets/ README 配图 +``` + +--- + +## 📖 文档 + +| 文档 | 内容 | +|------|------| +| [USAGE_GUIDE.md](docs/USAGE_GUIDE.md) | 在 Claude / Cursor / GPT / Codex / Jupyter / Docker 等平台上的使用方法 | +| [DATA_SOURCES.md](docs/DATA_SOURCES.md) | 学术参考文献、数据标准、伦理合规框架 | +| [ANNOTATIONS.md](docs/ANNOTATIONS.md) | 系统架构、API 接口规范、代码注释标准 | +| [EXAMPLE_WALKTHROUGH.md](docs/EXAMPLE_WALKTHROUGH.md) | 从一篇论文到检测报告的完整教程 | +| [SKILL.md](SKILL.md) | Skill 核心技术文档 | + +--- + +## 🔗 学术参考 + +1. Benford, F. (1938). The law of anomalous numbers. *Proc. APS*, 78(4), 551–572. +2. Brown, N.J.L. & Heathers, J.A.J. (2017). The GRIM Test. *SPPS*, 8(4), 363–369. +3. Bik, E.M. et al. (2016). Image duplication in biomedical research. *mBio*, 7(3). +4. Nigrini, M.J. (2012). *Benford's Law*. Wiley. ISBN: 978-1118152850. +5. 余菁等 (2021). 科技论文数据造假的核查策略. *中国科技期刊研究*, 32(6), 770–776. + +--- + +## 🙏 致谢 + +本项目的灵感来源于 **"耿同学讲故事"**，从 2026 年 4 月开始，仅凭一台电脑和统计学方法，就揪出了多所顶尖高校教授的论文数据造假。他的工作证明了：**学术诚信监督不仅必要，而且完全可行。** + +> "如果论文里的数据存在规律性，那么就明显不是在实验室实际测量的情况下生成的。" +> +> —— 耿同学 + +--- + +## 📄 开源许可 + +MIT License — 详见 [LICENSE](LICENSE) + +--- + +

    + 让学术回归诚信，让数据说出真相。
    + Let academic integrity prevail. Let data speak the truth. +

    diff --git a/tools/gengskill/SKILL.md b/tools/gengskill/SKILL.md new file mode 100644 index 0000000..93fd9f0 --- /dev/null +++ b/tools/gengskill/SKILL.md @@ -0,0 +1,139 @@ +# Geng Skill — 学术数据打假检测工具 + +> 致敬"耿同学讲故事"——用数据说话，让造假无所遁形。 + +## 概述 + +本 Skill 实现了一套**基于统计学原理的学术论文数据造假检测方法**，灵感来源于科普博主"耿同学"的技术流打假方法论。该工具从数据层面对论文中的实验数据进行多维度异常检测，适用于生物医学、化学、物理、社会科学等多个领域。 + +## 核心原理 + +**自然数据具有随机性，人为编造的数据会呈现不自然的规律性。** + +当研究者伪造实验数据时，由于人脑无法真正生成随机数，编造的数据往往会暴露以下统计学破绽： + +1. **末位数字分布异常** — 自然数据末位数字应近似均匀分布 +2. **固定差值/比例关系** — 不同实验组数据间存在恒定数学关系 +3. **小数位一致性过高** — 多组数据小数点后位数高度一致 +4. **本福特定律偏离** — 首位数字分布严重偏离理论预期 +5. **GRIM/SPRITE 不一致** — 报告的平均值与样本量不兼容 +6. **图像重复/篡改** — 同一图片出现在不同实验条件下 + +## 适用领域 + +| 领域 | 检测重点 | 典型数据类型 | +|------|----------|--------------| +| 生物医学 | Western blot、流式细胞术、动物实验数据 | 连续测量值、图像 | +| 化学 | 光谱数据、反应产率、催化活性 | 数值序列 | +| 物理/材料 | 性能曲线、电学/力学测试数据 | 时间序列 | +| 社会科学 | 问卷数据、量表得分 | 离散整数值 | +| 临床医学 | 生存数据、临床指标 | 分组统计量 | + +## 使用方法 + +### 输入 + +- 论文 PDF 文件（或提取的数据表格） +- 补充材料 / Source Data（如有） +- 指定检测领域（用于选择合适的检测策略） + +### 检测流程 + +``` +输入论文 → 数据提取 → 多维度异常检测 → 综合评分 → 生成报告 +``` + +### 输出 + +- **异常检测报告**：每项检测的结果、p值、置信度 +- **综合风险评分**：0-100 分，分为低/中/高/极高风险 +- **可视化图表**：分布直方图、偏离热力图 +- **建议行动**：需要进一步核查的具体数据点 + +## 检测模块 + +### Module 1: 末位数字检测 (Last Digit Test) + +```bash +python3 scripts/last_digit_test.py --input data.csv --column "value" +``` + +原理：自然实验数据的末位数字（0-9）应近似均匀分布。卡方检验判断偏离程度。 + +### Module 2: 本福特定律检测 (Benford's Law Test) + +```bash +python3 scripts/benford_test.py --input data.csv --column "value" +``` + +原理：多数量级跨度的自然数据，首位数字以1开头的概率约30.1%，逐位递减至9的4.6%。 + +### Module 3: GRIM 测试 (Granularity-Related Inconsistency of Means) + +```bash +python3 scripts/grim_test.py --mean 3.47 --n 25 --scale "1-5" --decimals 2 +``` + +原理：对于整数取值数据，给定样本量 n，合法的平均值只能取特定的有限集合。 + +### Module 4: 固定关系检测 (Fixed Relationship Detection) + +```bash +python3 scripts/fixed_relation_test.py --input data.csv --col1 "group_a" --col2 "group_b" +``` + +原理：两组独立实验数据之间不应存在恒定的差值、比值或线性关系。 + +### Module 5: 小数位一致性检测 (Decimal Consistency Test) + +```bash +python3 scripts/decimal_consistency_test.py --input data.csv --column "value" +``` + +原理：实验测量数据的小数位后数字应具有随机性，过度一致暗示人为编造。 + +### Module 6: 图像重复检测 (Image Duplication Detection) + +```bash +python3 scripts/image_duplicate_test.py --input_dir ./figures/ --threshold 0.85 +``` + +原理：基于感知哈希和 SSIM 相似度，检测论文图片中是否存在重复使用或篡改。 + +### Module 7: 综合评估引擎 (Comprehensive Assessment) + +```bash +python3 scripts/geng_assess.py --input data.csv --domain "biomedical" --output report/ +``` + +一键运行所有适用模块，生成综合报告。 + +## 风险评分体系 + +| 等级 | 分数 | 含义 | +|------|------|------| +| 🟢 低风险 | 0-25 | 数据未发现明显异常 | +| 🟡 中风险 | 26-50 | 存在可疑模式，建议人工复核 | +| 🟠 高风险 | 51-75 | 多项检测异常，强烈建议深入调查 | +| 🔴 极高风险 | 76-100 | 系统性异常，高度疑似数据造假 | + +## 重要声明 + +⚠️ **本工具仅用于辅助筛查，不能作为造假的最终判定依据。** + +- 数据异常 ≠ 数据造假（可能是仪器校准、单位转换、排版错误等） +- 检测结果需要领域专家复核 +- 不应基于单一检测模块的结果下结论 +- 使用本工具时应遵守学术伦理和法律法规 +- 建议将检测结果提交给相关机构进行正式调查 + +## 参考文献 + +1. Benford, F. (1938). The law of anomalous numbers. *Proceedings of the American Philosophical Society*, 78(4), 551-572. +2. Brown, N.J.L., & Heathers, J.A.J. (2017). The GRIM Test: A Simple Technique Detects Numerous Anomalies in the Reporting of Results in Psychology. *Social Psychological and Personality Science*, 8(4), 363-369. +3. 余菁等 (2021). 科技论文数据造假的核查策略和统计学方法验证. *中国科技期刊研究*, 32(6), 770-776. +4. Bik, E.M., et al. (2016). The prevalence of inappropriate image duplication in biomedical research publications. *mBio*, 7(3), e00809-16. + +## 版本 + +- v1.0.0 — 2026-05-20 — 初始版本，致敬耿同学 diff --git a/tools/gengskill/assets/architecture.png b/tools/gengskill/assets/architecture.png new file mode 100644 index 0000000..196421e Binary files /dev/null and b/tools/gengskill/assets/architecture.png differ diff --git a/tools/gengskill/assets/banner.png b/tools/gengskill/assets/banner.png new file mode 100644 index 0000000..ce1193c Binary files /dev/null and b/tools/gengskill/assets/banner.png differ diff --git a/tools/gengskill/assets/workflow.png b/tools/gengskill/assets/workflow.png new file mode 100644 index 0000000..73329c0 Binary files /dev/null and b/tools/gengskill/assets/workflow.png differ diff --git a/tools/gengskill/docs/ANNOTATIONS.md b/tools/gengskill/docs/ANNOTATIONS.md new file mode 100644 index 0000000..fd94166 --- /dev/null +++ b/tools/gengskill/docs/ANNOTATIONS.md @@ -0,0 +1,514 @@ +# 🏷️ Geng Skill 代码注释规范与架构说明 + +> 本文档提供完整的代码注释体系、模块间关系、接口规范，供开发者和 AI Agent 使用。 + +--- + +## 1. 项目架构总览 + +``` + ┌──────────────────────────────┐ + │ geng_assess.py │ + │ (综合评估引擎 / 主入口) │ + └──────────────┬───────────────┘ + │ + ┌───────────┬───────────┼───────────┬───────────┐ + ▼ ▼ ▼ ▼ ▼ + ┌────────────┐ ┌────────────┐ ┌────────────┐ ┌────────────┐ ┌──────────────┐ + │last_digit │ │ benford │ │ grim │ │fixed_rel │ │decimal_cons │ + │_test.py │ │_test.py │ │_test.py │ │_test.py │ │_test.py │ + │ │ │ │ │ │ │ │ │ │ + │末位数字检测│ │本福特定律 │ │均值一致性 │ │固定关系检测│ │小数位一致性 │ + └────────────┘ └────────────┘ └────────────┘ └────────────┘ └──────────────┘ + │ │ + │ ┌────────────────┐ │ + └───────────▶│image_duplicate │◀─────────────┘ + │_test.py │ + │图像重复检测 │ + └────────────────┘ +``` + +--- + +## 2. 模块接口规范 (API Contract) + +### 2.1 通用接口模式 + +每个检测模块都遵循统一的函数签名模式： + +```python +def _test( + values: List[str | float], # 输入数据 + **kwargs # 模块特定参数 +) -> Dict[str, Any]: # 标准化输出 + """ + [模块名称] — [一句话描述] + + Parameters + ---------- + values : list + 待检测数据。字符串形式传入以保留原始精度。 + **kwargs : dict + 模块特定参数（详见各模块文档） + + Returns + ------- + dict + 标准化输出，必含字段： + - test_name : str — 模块名称（中英文） + - status : str — "completed" | "insufficient_data" | "error" + - risk_level : str — "low" | "medium" | "medium-high" | "high" + - risk_score : float — 0-100 风险评分 + - interpretation : str — 中文可读解释 + """ +``` + +### 2.2 各模块特定接口 + +#### Module 1: `last_digit_test()` + +```python +def last_digit_test( + values: List[str], + method: str = 'all_digits' # 'all_digits' | 'decimal_last' +) -> Dict: + """ + 末位数字检测 + + 特定输出字段: + - digit_distribution : Dict[str, int] — 0-9 各数字出现次数 + - chi_square : float — 卡方统计量 + - p_value : float — p值 + - most_frequent_digit : int — 出现最多的数字 + - most_frequent_proportion : float — 最高频率 + - uniformity_deviation : float — 偏离均匀度 (0-1) + """ +``` + +#### Module 2: `benford_test()` + +```python +def benford_test( + values: List[str], + order: int = 1 # 1=首位, 2=前两位 +) -> Dict: + """ + 本福特定律检测 + + 前提条件: 数据应跨越至少1个数量级 + + 特定输出字段: + - distribution : Dict[str, Dict] — 各位数字观测/期望频率 + - mean_absolute_deviation : float — MAD (Nigrini 判定标准) + - conformity : str — 'close'|'acceptable'|'marginal'|'nonconforming' + - conformity_cn : str — 中文符合性判定 + """ +``` + +#### Module 3: `grim_test_single()` / `grim_test_batch()` + +```python +def grim_test_single( + mean: str, # 报告的平均值（字符串保留精度） + n: int, # 样本量 + decimals: int = 2, # 报告的小数位数 + scale_min: int = None, # 量表下限 + scale_max: int = None # 量表上限 +) -> Dict: + """ + GRIM 单项测试 + + 特定输出字段: + - consistent : bool — 是否通过一致性检验 + - computed_sum : float — 计算的总和 (mean × n) + - nearest_valid_mean : str — 最近的合法均值 + - difference : float — 与最近合法均值的差距 + """ + +def grim_test_batch( + items: List[Dict] # 批量项目列表 +) -> Dict: + """ + GRIM 批量测试 + + items 格式: [{"mean": "3.47", "n": 25, "decimals": 2, "label": "Table 1"}, ...] + + 特定输出字段: + - total_items : int + - inconsistent_items : int + - inconsistency_rate : float + - details : List[Dict] — 每项的详细结果 + """ +``` + +#### Module 4: `fixed_relation_test()` + +```python +def fixed_relation_test( + col1: List[float], # 第一列数据 + col2: List[float], # 第二列数据 + col1_name: str = 'A', # 列名标签 + col2_name: str = 'B' # 列名标签 +) -> Dict: + """ + 固定关系检测 — ⭐ 核心模块（耿同学最常用的方法） + + 检测内容: + 1. 固定差值 (col2 - col1 = 常数?) + 2. 固定比值 (col2 / col1 = 常数?) + 3. 完美线性关系 (R² → 1.0?) + 4. 小数模式一致性 + + 特定输出字段: + - detections : Dict — 各子检测结果 + - fixed_difference : {is_fixed, is_exact, mean_difference, std_difference} + - fixed_ratio : {is_fixed, is_exact, mean_ratio, std_ratio} + - linear_relationship : {r_squared, slope, intercept, is_suspicious} + - decimal_pattern : {match_rate, is_suspicious} + - n_suspicious_patterns : int + """ +``` + +#### Module 5: `decimal_consistency_test()` + +```python +def decimal_consistency_test( + values: List[str] # 保留原始字符串精度 +) -> Dict: + """ + 小数位一致性检测 + + 特定输出字段: + - decimal_places_analysis : Dict — 小数位数分布 + - decimal_repetition : Dict — 小数模式重复度 + - position_digit_analysis : Dict — 各位数字分布检验 + - autocorrelation : float — 小数部分自相关 + - risk_factors : List[str] — 触发的风险因子 + """ +``` + +#### Module 6: `find_duplicates()` + +```python +def find_duplicates( + image_dir: str, # 图片目录 + threshold: float = 0.85, # 相似度阈值 + extensions: List[str] = None # 图片格式 +) -> Dict: + """ + 图像重复检测 + + 依赖: Pillow, scikit-image (可选, 用于SSIM) + + 特定输出字段: + - n_images_scanned : int + - n_duplicate_pairs : int + - duplicates : List[Dict] — 每对疑似重复图片 + - file_1, file_2 : str + - avg_hash_similarity : float + - diff_hash_similarity : float + - combined_similarity : float + - rotation_check : Dict — 旋转/翻转匹配结果 + - ssim : float (如果 scikit-image 可用) + """ +``` + +--- + +## 3. 代码注释规范 + +### 3.1 文件头注释模板 + +每个 Python 文件必须包含以下格式的文件头： + +```python +#!/usr/bin/env python3 +""" +[模块名称中文] ([Module Name English]) +{'='*len(module_name)} + +原理：[一段话描述检测原理] + +方法：[具体使用的统计方法] + +参考：[关键参考文献，一行一条] + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" +``` + +### 3.2 函数注释规范 (NumPy Style) + +```python +def function_name(param1, param2, param3=default): + """ + [一句话功能描述] + + [详细说明段落，解释为什么需要这个函数、在什么场景下使用] + + Parameters + ---------- + param1 : type + 参数说明 + param2 : type + 参数说明 + param3 : type, optional + 参数说明（默认值：default） + + Returns + ------- + return_type + 返回值说明 + + Raises + ------ + ValueError + 何时抛出此异常 + + Examples + -------- + >>> result = function_name([1, 2, 3]) + >>> print(result['risk_score']) + 15.3 + + Notes + ----- + [重要注意事项、使用限制、已知问题] + + References + ---------- + [1] Author (Year). Title. Journal. DOI. + """ +``` + +### 3.3 行内注释规范 + +```python +# ✅ 好的注释 — 解释"为什么" +# 本福特定律只适用于跨数量级的数据，pH值（0-14）不适用 +if value_range < 10: + return skip_benford() + +# ❌ 差的注释 — 重复代码 +# 计算平均值 +mean = sum(values) / len(values) + +# ✅ 好的注释 — 标注算法来源 +# MAD 阈值参考 Nigrini (2012), Table 7.1 +# Close conformity: MAD < 0.006 +MAD_THRESHOLD_CLOSE = 0.006 +``` + +--- + +## 4. 错误处理与边界条件 + +### 4.1 标准错误返回 + +```python +# 数据不足 +if len(values) < MIN_REQUIRED: + return { + 'status': 'insufficient_data', + 'message': f'数据量不足（仅{len(values)}个），需要至少{MIN_REQUIRED}个', + 'n_valid': len(values) + } + +# 输入格式错误 +if not valid_input: + return { + 'status': 'error', + 'message': f'无效输入: {error_detail}' + } +``` + +### 4.2 边界条件处理 + +| 场景 | 处理方式 | +|------|----------| +| 全部值为0 | 跳过本福特检测（返回 status='not_applicable'） | +| 无小数部分 | 跳过小数位检测 | +| 仅1列数值 | 跳过固定关系检测 | +| 图片目录为空 | 返回 insufficient_data | +| 极端离群值 | 不剔除，但在 notes 中标注 | +| NaN/无效值 | 静默跳过，在 n_valid 中反映 | + +--- + +## 5. 风险评分算法详解 + +### 5.1 单模块评分 + +```python +""" +风险评分映射逻辑（以末位数字检测为例）： + + p >= 0.05 → risk_score = 40 * (1 - p) ∈ [0, ~38] → "low" + 0.01 <= p < 0.05 → risk_score = 40 + ... ∈ [40, 60] → "medium" + 0.001 <= p < 0.01 → risk_score = 60 + ... ∈ [60, 80] → "medium-high" + p < 0.001 → risk_score = 80 + ... ∈ [80, 100] → "high" + +设计考量： +- 不直接使用 1-p 作为分数（会导致 p=0.04 和 p=0.06 差距过小） +- 分段线性映射，确保跨越统计显著性阈值时有明显跳变 +- 上限 100 永远不精确达到（留有余地表示"不确定性"） +""" +``` + +### 5.2 综合评分算法 + +```python +""" +综合评分 = 0.6 × max(各模块分数) + 0.4 × mean(各模块分数) + +设计理由： +- 加权最大值确保"只要有一个模块高度异常，综合分就不会太低" +- 加权平均确保"如果多个模块都略有异常，综合分会累积上升" +- 0.6/0.4 比例经验性确定，偏向保守（避免漏检重于避免误报） + +特殊规则： +- 如果固定关系检测发现 is_exact=True，直接 risk_score = max(score, 90) +- 如果图像检测发现 similarity > 0.98，直接 risk_score = max(score, 90) +""" +``` + +--- + +## 6. 测试用例规范 + +### 6.1 单元测试结构 + +```python +# tests/test_modules.py + +""" +测试策略： +1. 已知正常数据 → 应返回 low risk +2. 已知造假数据 → 应返回 high risk +3. 边界条件 → 应优雅处理 +4. 回归测试 → 固定输入，固定输出 +""" + +def test_last_digit_uniform_data(): + """均匀分布数据应返回低风险""" + import random + random.seed(42) + values = [str(random.uniform(1, 100)) for _ in range(100)] + result = last_digit_test(values) + assert result['risk_level'] == 'low' + assert result['risk_score'] < 30 + +def test_fixed_relation_exact_ratio(): + """精确固定比值应返回极高风险""" + col1 = [1.23, 2.34, 3.45, 4.56, 5.67] + col2 = [2.46, 4.68, 6.90, 9.12, 11.34] # 精确 ×2 + result = fixed_relation_test(col1, col2) + assert result['risk_level'] == 'high' + assert result['risk_score'] >= 85 + +def test_grim_consistent(): + """合法均值应通过 GRIM""" + # n=20, 整数数据, mean=3.40 → sum=68 ✓ + result = grim_test_single('3.40', 20, decimals=2) + assert result['consistent'] == True + +def test_grim_inconsistent(): + """非法均值应失败""" + # n=20, 整数数据, mean=3.47 → sum=69.4 ✗ + result = grim_test_single('3.47', 20, decimals=2) + assert result['consistent'] == False +``` + +--- + +## 7. AI Agent 集成注释 + +### 7.1 Prompt Engineering 标注 + +每个模块的 docstring 设计为可被 AI Agent 直接解析： + +```python +""" +[AGENT_INSTRUCTION] +当用户要求检测数据造假时，按以下优先级选择模块： +1. 如果用户提供了两组"应该独立"的数据 → fixed_relation_test() +2. 如果数据跨越多个数量级 → benford_test() +3. 如果数据含小数 → decimal_consistency_test() + last_digit_test() +4. 如果用户提供了均值和样本量 → grim_test_single() +5. 如果有图片文件 → find_duplicates() +6. 一键全检 → geng_assess.py + +[AGENT_OUTPUT_FORMAT] +向用户展示结果时，使用以下格式： +- 先给出综合评分和风险等级（一句话） +- 然后列出关键发现（使用 emoji 标注严重度） +- 最后给出建议行动（编号列表） +- 始终附上免责声明 +""" +``` + +### 7.2 Tool Definition 标注 + +```python +""" +[TOOL_DEFINITION] +name: geng_fraud_detection +description: | + 基于统计学原理检测学术论文数据是否存在造假迹象。 + 支持末位数字检测、本福特定律、GRIM测试、固定关系检测、 + 小数位一致性检测和图像重复检测。 + 灵感来源于2026年"耿同学讲故事"的技术流打假方法论。 +input_schema: + type: object + properties: + data: + type: array + description: 数据行列表，或 CSV 文件路径 + domain: + type: string + enum: [biomedical, chemistry, physics, social_science, clinical, general] + modules: + type: array + items: + type: string + enum: [last_digit, benford, grim, fixed_relation, decimal, image] + description: 指定运行哪些模块（默认全部） +output_schema: + type: object + properties: + overall_risk_score: {type: number, min: 0, max: 100} + overall_risk_level: {type: string} + findings: {type: array, items: {type: string}} + recommendations: {type: array, items: {type: string}} +""" +``` + +--- + +## 8. 性能与限制 + +### 8.1 时间复杂度 + +| 模块 | 时间复杂度 | 1000行数据耗时 | +|------|-----------|---------------| +| last_digit_test | O(n) | <10ms | +| benford_test | O(n) | <10ms | +| grim_test_batch | O(k) per item | <1ms/item | +| fixed_relation_test | O(n) per pair | <10ms | +| decimal_consistency_test | O(n) | <20ms | +| image_duplicate_test | O(m²) m=图片数 | ~1s/100张 | +| geng_assess (综合) | O(n × c²) c=列数 | <500ms | + +### 8.2 已知限制 + +| 限制 | 影响 | 缓解方案 | +|------|------|----------| +| 数据量<30时统计效力低 | 本福特检测可能不准 | 自动标注 "统计效力有限" | +| 不支持时间序列自相关 | 遗漏趋势数据伪造 | v1.1 计划增加 | +| 固定关系仅检测两列 | 三列以上复杂关系漏检 | 通过两两组合覆盖 | +| 图像检测仅用全局特征 | 局部篡改可能漏检 | v1.2 计划增加分块检测 | +| 无法检测"高明造假" | 统计上完美的伪造数据 | 无银弹，需多维度交叉 | + +--- + +*Geng Skill v1.0.0 — 代码注释与架构标准化文档* diff --git a/tools/gengskill/docs/DATA_SOURCES.md b/tools/gengskill/docs/DATA_SOURCES.md new file mode 100644 index 0000000..fa96cc2 --- /dev/null +++ b/tools/gengskill/docs/DATA_SOURCES.md @@ -0,0 +1,294 @@ +# 📚 数据来源与参考文献标准化文档 + +> Geng Skill v1.0.0 — 学术数据打假检测工具 + +--- + +## 1. 方法论来源 + +### 1.1 直接灵感来源 + +| 来源 | 描述 | 时间 | +|------|------|------| +| **耿同学讲故事** (B站/抖音科普博主) | 吉林大学生物学硕士、北航博士五年级退学。2026年4月起连续举报多所985高校教授论文造假，核心方法：末位数字集中度检测、固定差值/比例关系检测、AI图片查重 | 2026-04 至今 | +| **澎湃新闻评论** | 《学术打假需要"耿同学"，更需要长效机制建设》— 详述耿同学方法论 | 2026-05-16 | +| **虎嗅网** | 《我Skill化了耿同学的"学术打假方法论"，致敬》— 方法论结构化梳理 | 2026-05-08 | + +### 1.2 耿同学核心方法总结 + +``` +┌────────────────────────────────────────────────────────────────┐ +│ 耿同学打假方法论（从公开报道中提取） │ +├────────────────────────────────────────────────────────────────┤ +│ 1. 末位数字集中度 — 某些数字出现频率异常高 │ +│ 2. 两列数据间固定差值/比例 — 不同组数据存在恒定数学关系 │ +│ 3. 小数点后位数高度一致 — 编造数据的小数位呈现不自然规律 │ +│ 4. AI图片查重 — 同一图片在不同实验条件下重复使用 │ +│ 5. 从PDF/Source Data/图片/表格多维度扒取证据 │ +│ 6. 卡方检验等统计学方法验证异常的显著性 │ +└────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 2. 统计学理论基础 + +### 2.1 本福特定律 (Benford's Law) + +| 字段 | 内容 | +|------|------| +| **原始论文** | Benford, F. (1938). The law of anomalous numbers. *Proceedings of the American Philosophical Society*, 78(4), 551-572. | +| **数学表述** | P(d) = log₁₀(1 + 1/d), d ∈ {1,2,...,9} | +| **适用条件** | 数据跨越多个数量级（至少1个）；数据量≥100为佳 | +| **不适用场景** | 范围有限的数据（百分比、pH值）；人为截断的数据 | +| **权威教材** | Nigrini, M.J. (2012). *Benford's Law: Applications for Forensic Accounting, Auditing, and Fraud Detection*. Wiley. ISBN: 978-1118152850 | +| **审计应用** | 美国注册欺诈审查师协会(ACFE)推荐用于财务审计 | +| **学术验证** | Diekmann, A. (2007). Not the first digit! Using Benford's law to detect fraudulent scientific data. *Journal of Applied Statistics*, 34(3), 321-329. | + +### 2.2 GRIM 测试 (Granularity-Related Inconsistency of Means) + +| 字段 | 内容 | +|------|------| +| **原始论文** | Brown, N.J.L., & Heathers, J.A.J. (2017). The GRIM Test: A Simple Technique Detects Numerous Anomalies in the Reporting of Results in Psychology. *Social Psychological and Personality Science*, 8(4), 363-369. DOI: 10.1177/1948550616673876 | +| **数学原理** | 对于整数取值数据，样本量为n时，合法均值只能是 k/n 形式（k为整数） | +| **适用条件** | 离散整数取值数据（李克特量表、计数数据） | +| **扩展** | SPRITE (Sample Parameter Reconstruction via Iterative TEchniques) — 更完整的数据重构验证 | +| **参考** | Heathers, J.A.J., et al. (2018). SPRITE: A Response to Anaya's Critique. DOI: 10.31234/osf.io/qfk7d | + +### 2.3 末位数字均匀分布检验 + +| 字段 | 内容 | +|------|------| +| **理论基础** | 连续测量数据在足够精度下，末位数字应服从离散均匀分布 U(0,9) | +| **检验方法** | 皮尔逊卡方检验 (Pearson's chi-squared test), df=9 | +| **参考文献** | Mosimann, J.E., et al. (2002). Terminal digits and the examination of questioned data. *Accountability in Research*, 9(2), 75-92. | +| **典型案例** | Hill, T.P. (1998). The first digit phenomenon. *American Scientist*, 86, 358-363. | + +### 2.4 图像重复检测 + +| 字段 | 内容 | +|------|------| +| **里程碑论文** | Bik, E.M., Casadevall, A., & Fang, F.C. (2016). The prevalence of inappropriate image duplication in biomedical research publications. *mBio*, 7(3), e00809-16. DOI: 10.1128/mBio.00809-16 | +| **发现** | 分析20,621篇论文，3.8%存在图片问题 | +| **技术方法** | 感知哈希(pHash)、差异哈希(dHash)、结构相似性(SSIM) | +| **工具参考** | ImageTwin, Proofig, STM Integrity Hub | + +### 2.5 数据一致性综合检验 + +| 字段 | 内容 | +|------|------| +| **中文权威** | 余菁, 邬加佳, 孙慧兰等 (2021). 科技论文数据造假的核查策略和统计学方法验证. *中国科技期刊研究*, 32(6), 770-776. DOI: 10.11946/cjstp.202012221043 | +| **方法体系** | t检验、F检验、卡方检验、生存分析一致性 | +| **国际标准** | COPE (Committee on Publication Ethics) Guidelines on Research Data | + +--- + +## 3. 检测模块与理论对应关系 + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ 模块名称 │ 理论基础 │ 统计方法 │ 适用领域 │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ Last Digit Test │ 末位均匀分布 │ χ² 检验 │ 全领域 │ +│ Benford's Law Test │ 本福特定律 │ χ² + MAD │ 跨数量级 │ +│ GRIM Test │ 离散粒度一致性 │ 整除验证 │ 社科/量表 │ +│ Fixed Relation Test │ 独立性原理 │ 比值/回归 │ 全领域(核心) │ +│ Decimal Consistency │ 随机性原理 │ 自相关+χ² │ 全领域 │ +│ Image Duplication │ 唯一性原理 │ 哈希+SSIM │ 生物医学 │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 4. 已验证的真实案例 + +### 4.1 耿同学举报案例（2026年，已被机构确认） + +| 案例 | 机构 | 期刊 | 问题类型 | 结果 | +|------|------|------|----------|------| +| 王平团队 | 同济大学生科院 | *Nature* | 系统性数据造假（固定数学关系、图片重复） | ✅ 确认，院长免职，第一作者解聘 | +| 陈佺团队 | 南开大学生科院 | *Nature* 子刊 | 数据异常 | 🔄 调查中 | +| 上海大学案例 | 上海大学 | — | 数据异常 | 🔄 调查中 | +| 中山大学案例 | 中山大学 | — | 数据异常 | 🔄 调查中 | + +### 4.2 国际经典案例 + +| 案例 | 方法 | 年份 | +|------|------|------| +| Diederik Stapel (社会心理学) | GRIM + 统计不一致性 | 2011 | +| Paolo Macchiarini (再生医学) | 图像重复 + 数据伪造 | 2016 | +| Hwang Woo-suk (干细胞) | 图像篡改检测 | 2005 | +| Jan Hendrik Schön (物理) | 数据重复模式 | 2002 | + +--- + +## 5. 数据标准与输入规范 + +### 5.1 CSV 输入格式标准 + +``` +编码: UTF-8 (支持 UTF-8-BOM) +分隔符: 逗号 (默认), 可配置为 TAB/分号 +表头: 必须有列名作为第一行 +数值: 支持整数、小数、科学计数法 +缺失值: 空字符串 (跳过处理) +``` + +**标准示例：** +```csv +sample_id,group,value,measurement,timepoint +1,control,2.34,12.5,0 +2,control,3.12,15.8,0 +3,treatment,4.68,25.0,24 +``` + +### 5.2 GRIM 批量输入 JSON 格式 + +```json +[ + { + "label": "Table 1, Row 1", + "mean": "3.47", + "n": 25, + "decimals": 2, + "scale_min": 1, + "scale_max": 5 + }, + { + "label": "Table 1, Row 2", + "mean": "4.12", + "n": 30, + "decimals": 2, + "scale_min": 1, + "scale_max": 5 + } +] +``` + +### 5.3 图像输入规范 + +``` +支持格式: PNG, JPG, JPEG, TIF, TIFF, BMP, GIF +最小尺寸: 32×32 像素 +推荐: 原始分辨率（不要人为缩放） +组织方式: 所有待比较图片放在同一目录下 +``` + +--- + +## 6. 输出标准化 + +### 6.1 JSON 输出 Schema + +所有模块遵循统一输出结构： + +```json +{ + "$schema": "geng-skill-output-v1", + "test_name": "string — 模块名称（中英双语）", + "status": "enum: completed | insufficient_data | error", + "n_values": "integer — 有效数据点数", + "risk_level": "enum: low | medium | medium-high | high", + "risk_score": "number 0-100 — 风险评分", + "p_value": "number — 统计检验p值（如适用）", + "interpretation": "string — 中文可读解释（含emoji状态标识）", + "...": "模块特定字段" +} +``` + +### 6.2 风险评分映射标准 + +| p-value 范围 | 风险等级 | 评分范围 | 颜色代码 | 建议动作 | +|-------------|----------|----------|----------|----------| +| p > 0.05 | low | 0-25 | 🟢 #00C853 | 无需干预 | +| 0.01 < p ≤ 0.05 | medium | 26-50 | 🟡 #FFD600 | 人工复核 | +| 0.001 < p ≤ 0.01 | medium-high | 51-75 | 🟠 #FF6D00 | 深入调查 | +| p ≤ 0.001 | high | 76-100 | 🔴 #D50000 | 正式举报 | + +### 6.3 Markdown 报告标准 + +综合报告遵循以下结构： + +```markdown +# 📋 Geng 学术数据打假检测报告 +## 📊 综合评估结果（表格） +## 📝 结论（一段话） +## 💡 建议（编号列表） +## 🔬 各模块检测详情 +### Module N: 模块名称 +## ⚠️ 重要声明 +``` + +--- + +## 7. 学术伦理与法律合规 + +### 7.1 合规框架 + +| 标准/规范 | 发布机构 | 相关性 | +|-----------|----------|--------| +| COPE Retraction Guidelines | 出版伦理委员会 | 论文撤稿/更正流程 | +| 科研诚信案件调查处理规则 | 中国科技部 (2019) | 国内学术不端处理 | +| ORI Research Integrity Guidelines | 美国研究诚信办公室 | 国际标准 | +| Singapore Statement | 全球科研诚信大会 | 负责任研究行为 | + +### 7.2 使用伦理准则 + +1. **比例原则** — 检测强度应与嫌疑程度成正比 +2. **无罪推定** — 异常 ≠ 造假，需完整证据链 +3. **保密义务** — 未经确认的检测结果不应公开传播 +4. **正式渠道** — 确认后应通过机构/期刊正式途径举报 +5. **避免伤害** — 不应基于工具结果对个人进行网络攻击 + +### 7.3 免责声明 + +``` +本工具仅提供统计学层面的异常筛查功能，输出结果为"疑点线索"而非 +"造假定论"。使用者应当理解： +- 统计异常可能有合理解释（仪器精度、数据处理等） +- 本工具不具备法律效力 +- 最终判定需要领域专家、原始数据核查和正式调查程序 +- 使用者需自行承担因不当使用造成的后果 +``` + +--- + +## 8. 版本与更新日志 + +### v1.0.0 (2026-05-20) + +- 初始发布 +- 6个核心检测模块 +- 综合评估引擎 +- 多平台使用指南 +- 标准化输出格式 + +### 路线图 + +| 版本 | 计划功能 | +|------|----------| +| v1.1 | 增加 SPRITE 测试、生存数据一致性检验 | +| v1.2 | 支持 Excel 直接输入、PDF 表格自动提取 | +| v1.3 | Web UI 界面、RESTful API | +| v2.0 | AI 增强检测（LLM 辅助判断上下文合理性） | + +--- + +## 9. 引用本工具 + +如果在学术工作中使用了本工具，请引用： + +```bibtex +@software{geng_skill_2026, + title = {Geng Skill: Academic Data Fraud Detection Toolkit}, + author = {Contributors}, + year = {2026}, + url = {https://github.com/YOUR_USERNAME/geng-skill}, + version = {1.0.0}, + note = {Inspired by the methodology of "Geng Tongxue" (耿同学讲故事)} +} +``` + +--- + +*Geng Skill — 让学术回归诚信，让数据说出真相。* diff --git a/tools/gengskill/docs/EXAMPLE_WALKTHROUGH.md b/tools/gengskill/docs/EXAMPLE_WALKTHROUGH.md new file mode 100644 index 0000000..bd00a41 --- /dev/null +++ b/tools/gengskill/docs/EXAMPLE_WALKTHROUGH.md @@ -0,0 +1,258 @@ +# 🧪 完整示例：从数据到报告的端到端演示 + +> 本文档通过一个完整的假数据检测案例，演示 Geng Skill 的全部使用流程。 + +--- + +## 场景设定 + +假设你在审阅一篇生物医学论文，论文声称： + +> "我们分别对小鼠进行了 Control、Treatment A、Treatment B 三组实验处理， +> 测量了各组的蛋白表达水平（相对定量）。结果显示 Treatment A 和 Treatment B +> 均显著提高了蛋白表达水平。" + +论文提供了以下数据（摘自 Supplementary Table 1）： + +```csv +sample_id,control_group,treatment_a,treatment_b,measurement +1,2.34,4.68,7.02,12.5 +2,3.12,6.24,9.36,15.8 +3,1.87,3.74,5.61,8.9 +4,4.56,9.12,13.68,22.1 +5,2.98,5.96,8.94,14.3 +6,3.45,6.90,10.35,17.2 +7,1.23,2.46,3.69,6.8 +8,5.67,11.34,17.01,28.4 +9,2.01,4.02,6.03,10.1 +10,3.89,7.78,11.67,19.5 +11,4.12,8.24,12.36,20.8 +12,1.56,3.12,4.68,7.9 +13,2.78,5.56,8.34,13.6 +14,3.34,6.68,10.02,16.7 +15,4.90,9.80,14.70,24.5 +16,1.45,2.90,4.35,7.2 +17,2.67,5.34,8.01,13.1 +18,3.56,7.12,10.68,17.8 +19,4.23,8.46,12.69,21.2 +20,1.89,3.78,5.67,9.4 +``` + +--- + +## 步骤 1：初步目视检查 + +一位细心的审稿人可能注意到： +- Treatment A 的数值似乎都是 Control 的两倍 +- Treatment B 的数值似乎都是 Control 的三倍 + +但仅凭目测无法确认。让我们用 Geng Skill 做系统化检测。 + +--- + +## 步骤 2：运行检测 + +### 2.1 命令行一键检测 + +```bash +cd geng-skill/scripts +python3 geng_assess.py \ + --input ../examples/fake_data_demo.csv \ + --domain biomedical \ + --output ../report/ +``` + +### 2.2 Python API 方式 + +```python +import sys +sys.path.insert(0, 'scripts') + +from last_digit_test import last_digit_test +from fixed_relation_test import fixed_relation_test +import csv + +# 加载数据 +with open('examples/fake_data_demo.csv') as f: + rows = list(csv.DictReader(f)) + +control = [float(r['control_group']) for r in rows] +treat_a = [float(r['treatment_a']) for r in rows] +treat_b = [float(r['treatment_b']) for r in rows] + +# 运行固定关系检测 +result = fixed_relation_test(control, treat_a, 'Control', 'Treatment A') +print(f"风险评分: {result['risk_score']}/100") +print(f"解释: {result['interpretation']}") +``` + +--- + +## 步骤 3：检测结果详解 + +### Module 1: 末位数字检测 + +``` +┌────────────────────────────────────────────────────────────────┐ +│ 检测列: control_group │ +│ 末位数字分布: {0:0, 1:1, 2:2, 3:2, 4:2, 5:2, 6:3, 7:3, 8:2, 9:3} │ +│ χ² = 4.00, p = 0.9114 │ +│ 结果: ✅ 正常 — 末位数字分布与均匀分布无显著差异 │ +│ 风险评分: 3.5/100 │ +└────────────────────────────────────────────────────────────────┘ +``` + +**解读**：造假者在编造 Control 组数据时，末位数字分布还算随机。这说明末位数字检测并非万能——它无法检测"有一定水平"的造假。 + +``` +┌────────────────────────────────────────────────────────────────┐ +│ 检测列: treatment_a │ +│ χ² = 21.00, p = 0.0127 │ +│ 结果: ⚠️ 异常 — 末位数字分布存在偏离 │ +│ 风险评分: 47.5/100 │ +└────────────────────────────────────────────────────────────────┘ +``` + +**解读**：Treatment A 的末位数字分布出现异常。这是因为 Control × 2 导致了末位数字的非均匀映射（如原数 .34 × 2 = .68，原数 .56 × 2 = .12）。 + +--- + +### Module 4: 固定关系检测 ⭐（核心发现） + +``` +┌────────────────────────────────────────────────────────────────┐ +│ 检测对: Control vs Treatment A │ +│ │ +│ 固定比值检测: │ +│ mean_ratio = 2.000000 │ +│ std_ratio = 0.000000 │ +│ → 🔴 完美固定比值！所有数据点 Treatment_A = Control × 2 │ +│ │ +│ 线性关系检测: │ +│ R² = 1.0000000000 │ +│ slope = 2.0000 │ +│ intercept = 0.000000 │ +│ → 🔴 完美线性关系，零残差 │ +│ │ +│ 风险评分: 95/100 — 极高风险 │ +└────────────────────────────────────────────────────────────────┘ + +┌────────────────────────────────────────────────────────────────┐ +│ 检测对: Control vs Treatment B │ +│ │ +│ 固定比值检测: │ +│ mean_ratio = 3.000000 │ +│ std_ratio = 0.000000 │ +│ → 🔴 完美固定比值！所有数据点 Treatment_B = Control × 3 │ +│ │ +│ 风险评分: 95/100 — 极高风险 │ +└────────────────────────────────────────────────────────────────┘ + +┌────────────────────────────────────────────────────────────────┐ +│ 检测对: Treatment A vs Treatment B │ +│ │ +│ 固定比值检测: │ +│ mean_ratio = 1.500000 │ +│ std_ratio = 0.000000 │ +│ → 🔴 完美固定比值！Treatment_B = Treatment_A × 1.5 │ +│ │ +│ 风险评分: 95/100 — 极高风险 │ +└────────────────────────────────────────────────────────────────┘ +``` + +**解读**：这是最致命的发现。三组"独立实验"数据之间存在精确的整数倍关系： +- Treatment A = Control × 2.000（精确到小数点后所有位） +- Treatment B = Control × 3.000（精确到小数点后所有位） +- Treatment B = Treatment A × 1.500（精确到小数点后所有位） + +**在真实生物实验中，这种完美的整数倍关系概率趋近于零。** 即使药物真的将蛋白表达提高了2倍，每个样本的响应也会有生物学变异（个体差异、实验误差等），绝不可能所有20个样本都精确地是2.000倍。 + +--- + +### 综合评估 + +``` +╔══════════════════════════════════════════════════════════════════╗ +║ 📋 综合评估结果 ║ +╠══════════════════════════════════════════════════════════════════╣ +║ ║ +║ 🔴 综合风险评分: 92/100 — 极高风险 ║ +║ ║ +║ 核心证据: ║ +║ • 三组"独立实验"数据存在精确整数倍关系（×2, ×3, ×1.5） ║ +║ • R² = 1.0，残差为零 ║ +║ • Treatment A 末位数字分布异常（p = 0.013） ║ +║ ║ +║ 结论: 数据极大概率为从单一数据源（Control组）通过简单 ║ +║ 乘法运算生成，而非独立实验获得。 ║ +║ ║ +╚══════════════════════════════════════════════════════════════════╝ +``` + +--- + +## 步骤 4：与正常数据对比 + +对 `real_data_demo.csv`（模拟的正常实验数据）运行同样的检测： + +``` +检测结果: +• 末位数字: ✅ 所有列 p > 0.05 +• 本福特定律: ✅ 符合 +• 固定关系: ✅ 无固定比值/差值（ratio_std > 1.5） +• 小数位一致性: ✅ 模式多样 + +综合风险评分: 8/100 — 🟢 低风险 +``` + +--- + +## 步骤 5：生成正式报告 + +运行完成后，`report/` 目录包含： + +``` +report/ +├── geng_assessment_report.json # 机器可读完整报告 +└── geng_assessment_report.md # 人类可读 Markdown 报告 +``` + +报告可直接用于： +- 向期刊提交 Letter of Concern +- 向机构科研诚信办公室提供技术证据 +- 审稿意见中引用具体检测结果 + +--- + +## 关键教训 + +| 教训 | 说明 | +|------|------| +| **单一检测不足以定论** | 末位数字检测对 Control 组未报警，但固定关系检测精准命中 | +| **多模块交叉验证更可靠** | 末位异常 + 固定比值 + 完美线性 = 综合证据链 | +| **异常不等于造假** | 需要排除合理解释（如数据预处理中的标准化操作） | +| **上下文很重要** | 同样的"固定比值"在"原始数据 vs 标准化后数据"场景中是正常的 | +| **工具是辅助，人是决策者** | 最终判断需要领域专家结合实验设计做出 | + +--- + +## 对审稿人/研究者的实用建议 + +### 何时应该怀疑数据？ + +1. ✋ 不同实验组数据太"干净"——没有离群值、没有异常 +2. ✋ 多组数据的 error bar 高度一致 +3. ✋ 不同条件下的重复次数完全一致 +4. ✋ 数据点"太完美"地落在预期曲线上 +5. ✋ 补充材料中的原始数据与正文图表不匹配 + +### 何时应该运行 Geng Skill？ + +1. 📊 审阅高利害关系的论文（顶刊/基金/职称） +2. 📊 收到学术不端举报后需要技术验证 +3. 📊 对自己团队数据做"造假预防"自查 +4. 📊 期刊编辑部建立投稿数据审查流程 + +--- + +*Geng Skill v1.0.0 — 完整示例演示* diff --git a/tools/gengskill/docs/USAGE_GUIDE.md b/tools/gengskill/docs/USAGE_GUIDE.md new file mode 100644 index 0000000..eec7e5c --- /dev/null +++ b/tools/gengskill/docs/USAGE_GUIDE.md @@ -0,0 +1,654 @@ +# 📖 Geng Skill 多平台使用指南 + +> 本文档详细说明 Geng Skill 在各主流 AI 编程助手和开发环境中的使用方法。 + +--- + +## 目录 + +1. [通用命令行使用](#1-通用命令行使用) +2. [在 Claude (Anthropic) 中使用](#2-在-claude-anthropic-中使用) +3. [在 Cursor 中使用](#3-在-cursor-中使用) +4. [在 ChatGPT / GPT-4 中使用](#4-在-chatgpt--gpt-4-中使用) +5. [在 OpenAI Codex / API 中使用](#5-在-openai-codex--api-中使用) +6. [在 GitHub Copilot 中使用](#6-在-github-copilot-中使用) +7. [在 Jupyter Notebook 中使用](#7-在-jupyter-notebook-中使用) +8. [作为 Python 库导入使用](#8-作为-python-库导入使用) +9. [CI/CD 自动化集成](#9-cicd-自动化集成) + +--- + +## 1. 通用命令行使用 + +### 1.1 安装 + +```bash +# 克隆仓库 +git clone https://github.com/YOUR_USERNAME/geng-skill.git +cd geng-skill + +# 安装依赖 +pip install -r requirements.txt +``` + +### 1.2 一键综合检测 + +```bash +cd scripts +python3 geng_assess.py \ + --input ../examples/fake_data_demo.csv \ + --domain biomedical \ + --output ../report/ +``` + +**参数说明：** + +| 参数 | 必填 | 说明 | 可选值 | +|------|------|------|--------| +| `--input` / `-i` | ✅ | 输入 CSV 文件路径 | 任意 .csv 文件 | +| `--domain` | ❌ | 研究领域（影响检测策略） | `biomedical`, `chemistry`, `physics`, `social_science`, `clinical`, `general` | +| `--output` / `-o` | ❌ | 输出报告目录 | 默认 `./report` | +| `--delimiter` / `-d` | ❌ | CSV 分隔符 | 默认 `,` | + +### 1.3 单项检测 + +```bash +# 末位数字检测 +python3 last_digit_test.py -i data.csv -c "column_name" -o result.json + +# 本福特定律检测 +python3 benford_test.py -i data.csv -c "measurement" -o result.json + +# GRIM 测试（单个均值） +python3 grim_test.py --mean 3.47 --n 25 --scale "1-5" --decimals 2 + +# GRIM 测试（批量，从 JSON） +python3 grim_test.py -i batch_means.json -o grim_results.json + +# 固定关系检测 +python3 fixed_relation_test.py -i data.csv --col1 "group_a" --col2 "group_b" + +# 小数位一致性检测 +python3 decimal_consistency_test.py -i data.csv -c "value" + +# 图像重复检测 +python3 image_duplicate_test.py -i ./figures/ -t 0.85 +``` + +### 1.4 输出格式 + +所有模块输出标准 JSON 格式，包含以下统一字段： + +```json +{ + "test_name": "检测模块名称（中英文）", + "status": "completed | insufficient_data | error", + "risk_level": "low | medium | medium-high | high", + "risk_score": 0-100, + "interpretation": "中文可读解释", + "...": "模块特定字段" +} +``` + +--- + +## 2. 在 Claude (Anthropic) 中使用 + +### 2.1 Claude Web / Claude Pro + +**方法 A：直接粘贴数据让 Claude 分析** + +``` +我有以下实验数据，请用"耿同学"的方法帮我检查是否存在数据造假迹象： + +sample,control,treatment_a,treatment_b +1,2.34,4.68,7.02 +2,3.12,6.24,9.36 +3,1.87,3.74,5.61 +... + +请检查： +1. 末位数字分布是否均匀 +2. 各组数据之间是否存在固定比值或差值关系 +3. 小数位模式是否异常 +``` + +**方法 B：上传 CSV 文件让 Claude 用代码分析** + +``` +请帮我运行学术数据打假检测。我上传的 CSV 文件包含论文中的实验数据。 +请用以下方法逐一检测： +- Last Digit Test（末位数字检测） +- Benford's Law Test（本福特定律检测） +- Fixed Relationship Detection（固定关系检测） +- Decimal Consistency Test（小数位一致性检测） + +最后给出综合风险评分和建议。 +``` + +### 2.2 Claude API (Artifacts / Tool Use) + +```python +import anthropic + +client = anthropic.Anthropic() + +# 将 Geng Skill 的 SKILL.md 作为 system prompt +with open('SKILL.md', 'r') as f: + skill_doc = f.read() + +message = client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=4096, + system=f"你是学术数据打假助手。请严格按照以下 Skill 文档执行检测：\n\n{skill_doc}", + messages=[{ + "role": "user", + "content": "请对以下数据执行完整的 Geng 打假检测...[数据]" + }] +) +``` + +### 2.3 Claude MCP (Model Context Protocol) + +将 Geng Skill 注册为 MCP Server： + +```json +// claude_desktop_config.json +{ + "mcpServers": { + "geng-skill": { + "command": "python3", + "args": ["/path/to/geng-skill/scripts/mcp_server.py"], + "env": {} + } + } +} +``` + +--- + +## 3. 在 Cursor 中使用 + +### 3.1 作为 Cursor Rules 使用 + +在项目根目录创建 `.cursor/rules/geng-skill.mdc`： + +```markdown +--- +description: 学术数据打假检测工具 +globs: ["*.csv", "*.xlsx", "data/**"] +alwaysApply: false +--- + +# Geng Skill — 学术数据打假检测 + +当用户要求检测数据是否造假时，按以下步骤执行： + +1. 确认数据格式（CSV/Excel/直接粘贴） +2. 识别数值列 +3. 对每个数值列执行： + - 末位数字检测（卡方检验 vs 均匀分布） + - 本福特定律检测（适用于跨数量级数据） + - 小数位一致性检测 +4. 对数值列两两执行： + - 固定关系检测（差值/比值/线性） +5. 综合评分（0-100）并给出建议 + +核心原则：自然数据具有随机性，人为编造的数据会呈现不自然的规律性。 +``` + +### 3.2 在 Cursor Chat 中使用 + +``` +@geng-skill 请检测这份数据文件 data/experiment_results.csv 是否存在造假迹象 + +重点关注： +- 不同实验组之间是否有固定数学关系 +- 末位数字分布是否正常 +- 小数位模式是否异常 +``` + +### 3.3 Cursor Composer 自动化 + +在 Cursor Composer 中直接引用脚本： + +``` +请运行 geng-skill/scripts/geng_assess.py 对 data/paper_results.csv 进行检测， +领域设为 biomedical，输出到 report/ 目录。 +然后帮我解读报告中的关键发现。 +``` + +--- + +## 4. 在 ChatGPT / GPT-4 中使用 + +### 4.1 ChatGPT Web (Code Interpreter / Advanced Data Analysis) + +**步骤：** +1. 上传 CSV 数据文件 +2. 同时上传 `scripts/` 目录下的 Python 脚本 +3. 提示词： + +``` +我上传了一组学术论文数据和几个检测脚本。请按照以下步骤执行学术数据打假检测： + +1. 先读取 CSV 数据，识别所有数值列 +2. 对每个数值列运行 last_digit_test.py 中的 last_digit_test() 函数 +3. 对适用的列运行 benford_test.py 中的 benford_test() 函数 +4. 对所有数值列对运行 fixed_relation_test.py 中的 fixed_relation_test() 函数 +5. 对每个数值列运行 decimal_consistency_test.py 中的 decimal_consistency_test() 函数 + +最后综合所有结果，给出： +- 综合风险评分（0-100） +- 关键发现（哪些数据可疑，为什么） +- 建议行动 +``` + +### 4.2 GPT-4 API + Function Calling + +```python +import openai +import json + +# 定义 Geng Skill 工具 +tools = [ + { + "type": "function", + "function": { + "name": "geng_last_digit_test", + "description": "检测数据末位数字是否偏离均匀分布。自然数据末位应均匀分布，造假数据往往集中在某些数字。", + "parameters": { + "type": "object", + "properties": { + "values": { + "type": "array", + "items": {"type": "string"}, + "description": "待检测的数值列表（字符串形式保留精度）" + }, + "method": { + "type": "string", + "enum": ["all_digits", "decimal_last"], + "description": "检测方法" + } + }, + "required": ["values"] + } + } + }, + { + "type": "function", + "function": { + "name": "geng_fixed_relation_test", + "description": "检测两组数据间是否存在固定差值、比值或完美线性关系。独立实验数据不应有精确数学关系。", + "parameters": { + "type": "object", + "properties": { + "col1": {"type": "array", "items": {"type": "number"}, "description": "第一列数据"}, + "col2": {"type": "array", "items": {"type": "number"}, "description": "第二列数据"}, + "col1_name": {"type": "string"}, + "col2_name": {"type": "string"} + }, + "required": ["col1", "col2"] + } + } + }, + { + "type": "function", + "function": { + "name": "geng_benford_test", + "description": "检测数据首位数字是否符合本福特定律。适用于跨多个数量级的自然数据。", + "parameters": { + "type": "object", + "properties": { + "values": {"type": "array", "items": {"type": "string"}, "description": "数值列表"} + }, + "required": ["values"] + } + } + } +] + +# 调用 GPT-4 带工具 +response = openai.chat.completions.create( + model="gpt-4", + messages=[ + {"role": "system", "content": "你是学术数据打假助手，使用 Geng Skill 检测论文数据。"}, + {"role": "user", "content": "请检测以下数据..."} + ], + tools=tools, + tool_choice="auto" +) +``` + +### 4.3 Custom GPT (GPTs Store) + +创建自定义 GPT，在 Instructions 中粘贴完整的 `SKILL.md` 内容，并上传所有脚本文件作为 Knowledge。 + +**GPT 名称建议**: "学术数据卫士 — Geng Fraud Detector" + +**Instructions 要点**： +``` +你是基于"耿同学"方法论的学术数据打假检测 GPT。 +当用户上传数据或粘贴数据时，自动执行以下检测流程... +``` + +--- + +## 5. 在 OpenAI Codex / API 中使用 + +### 5.1 Codex CLI + +```bash +# 安装 Codex CLI +npm install -g @openai/codex + +# 使用 Geng Skill 检测数据 +codex "请对 data.csv 文件运行学术数据打假检测：\ + 1. 读取所有数值列 \ + 2. 检测末位数字分布 \ + 3. 检测列间固定关系 \ + 4. 给出风险评分" \ + --file data.csv \ + --file scripts/last_digit_test.py \ + --file scripts/fixed_relation_test.py +``` + +### 5.2 Codex 作为自动化 Agent + +```python +# codex_geng_agent.py +""" +将 Geng Skill 封装为 Codex Agent 可调用的工具链 +""" +import subprocess +import json + +def run_geng_assessment(csv_path, domain="general"): + """调用 Geng 综合评估引擎""" + result = subprocess.run( + ["python3", "scripts/geng_assess.py", + "--input", csv_path, + "--domain", domain, + "--output", "./report/"], + capture_output=True, text=True + ) + + # 读取报告 + with open("./report/geng_assessment_report.json", "r") as f: + report = json.load(f) + + return report +``` + +--- + +## 6. 在 GitHub Copilot 中使用 + +### 6.1 Copilot Chat in VS Code + +在 VS Code 中打开数据文件，然后使用 Copilot Chat： + +``` +@workspace /explain 请分析 data.csv 中的数据是否存在学术造假迹象， +使用 geng-skill/scripts/ 中的检测模块 +``` + +### 6.2 Copilot in Terminal + +```bash +# GitHub Copilot CLI +gh copilot suggest "run geng academic fraud detection on experiment_data.csv" +``` + +### 6.3 作为 GitHub Action + +```yaml +# .github/workflows/geng-check.yml +name: Academic Data Integrity Check + +on: + pull_request: + paths: + - 'data/**/*.csv' + +jobs: + geng-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install -r geng-skill/requirements.txt + + - name: Run Geng Assessment + run: | + cd geng-skill/scripts + for csv_file in $(find ../../data -name "*.csv"); do + echo "🔍 Checking: $csv_file" + python3 geng_assess.py -i "$csv_file" -o ../../report/ + done + + - name: Upload Report + uses: actions/upload-artifact@v4 + with: + name: geng-report + path: report/ +``` + +--- + +## 7. 在 Jupyter Notebook 中使用 + +```python +# Cell 1: 安装与导入 +!pip install numpy scipy Pillow scikit-image -q + +import sys +sys.path.insert(0, '../scripts') + +from last_digit_test import last_digit_test +from benford_test import benford_test +from fixed_relation_test import fixed_relation_test +from decimal_consistency_test import decimal_consistency_test +from grim_test import grim_test_single, grim_test_batch + +import pandas as pd +import json + +# Cell 2: 加载数据 +df = pd.read_csv('../examples/fake_data_demo.csv') +print(f"数据形状: {df.shape}") +df.head() + +# Cell 3: 末位数字检测 +result = last_digit_test(df['control_group'].astype(str).tolist()) +print(json.dumps(result, ensure_ascii=False, indent=2)) + +# Cell 4: 固定关系检测 +result = fixed_relation_test( + df['control_group'].tolist(), + df['treatment_a'].tolist(), + 'control_group', 'treatment_a' +) +print(f"🎯 风险评分: {result['risk_score']}/100") +print(f"📝 {result['interpretation']}") + +# Cell 5: 可视化 +import matplotlib.pyplot as plt +import numpy as np + +fig, axes = plt.subplots(1, 3, figsize=(15, 4)) + +# 散点图：展示固定比值关系 +axes[0].scatter(df['control_group'], df['treatment_a'], c='red', alpha=0.7) +axes[0].set_xlabel('Control Group') +axes[0].set_ylabel('Treatment A') +axes[0].set_title('⚠️ 完美 2x 关系') +axes[0].plot([0, 6], [0, 12], 'k--', alpha=0.3) + +# 末位数字分布 +from collections import Counter +digits = [int(str(v)[-1]) for v in df['control_group'].astype(str)] +counts = Counter(digits) +axes[1].bar(range(10), [counts.get(i, 0) for i in range(10)]) +axes[1].axhline(y=len(digits)/10, color='r', linestyle='--', label='期望值') +axes[1].set_xlabel('末位数字') +axes[1].set_ylabel('频次') +axes[1].set_title('末位数字分布') +axes[1].legend() + +# 比值分布 +ratios = df['treatment_a'] / df['control_group'] +axes[2].hist(ratios, bins=20, edgecolor='black') +axes[2].set_xlabel('Treatment_A / Control') +axes[2].set_ylabel('频次') +axes[2].set_title(f'⚠️ 比值全部 = {ratios.mean():.1f}') + +plt.tight_layout() +plt.savefig('../report/detection_visualization.png', dpi=150) +plt.show() +``` + +--- + +## 8. 作为 Python 库导入使用 + +### 8.1 基础用法 + +```python +import sys +sys.path.insert(0, '/path/to/geng-skill/scripts') + +from last_digit_test import last_digit_test +from benford_test import benford_test +from fixed_relation_test import fixed_relation_test +from decimal_consistency_test import decimal_consistency_test +from grim_test import grim_test_single + +# 单列检测 +values = ['2.34', '3.12', '1.87', '4.56', '2.98'] +result = last_digit_test(values) +print(f"风险评分: {result['risk_score']}") + +# 两列关系检测 +col_a = [2.34, 3.12, 1.87, 4.56, 2.98] +col_b = [4.68, 6.24, 3.74, 9.12, 5.96] +result = fixed_relation_test(col_a, col_b, 'GroupA', 'GroupB') +print(f"风险等级: {result['risk_level']}") + +# GRIM 测试 +result = grim_test_single(mean='3.47', n=25, decimals=2, scale_min=1, scale_max=5) +print(f"一致性: {result['consistent']}") +``` + +### 8.2 批量处理多篇论文 + +```python +import os +import glob +import json +from geng_assess import run_assessment + +# 批量检测目录下所有 CSV +csv_files = glob.glob('/path/to/papers/*/data.csv') + +results = [] +for csv_path in csv_files: + paper_name = os.path.basename(os.path.dirname(csv_path)) + report = run_assessment(csv_path, domain='biomedical') + results.append({ + 'paper': paper_name, + 'score': report['summary']['overall_risk_score'], + 'level': report['summary']['overall_risk_level'] + }) + print(f" {paper_name}: {report['summary']['overall_risk_level_cn']}") + +# 排序输出高风险论文 +results.sort(key=lambda x: x['score'], reverse=True) +print("\n🔴 高风险论文：") +for r in results: + if r['score'] >= 50: + print(f" [{r['score']:.0f}] {r['paper']}") +``` + +--- + +## 9. CI/CD 自动化集成 + +### 9.1 Pre-commit Hook + +```yaml +# .pre-commit-config.yaml +repos: + - repo: local + hooks: + - id: geng-data-check + name: Geng Academic Data Check + entry: python3 geng-skill/scripts/geng_assess.py + language: python + files: '\.csv$' + args: ['--input'] +``` + +### 9.2 Docker 容器化 + +```dockerfile +# Dockerfile +FROM python:3.10-slim + +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY scripts/ ./scripts/ +COPY SKILL.md . + +ENTRYPOINT ["python3", "scripts/geng_assess.py"] +CMD ["--help"] +``` + +```bash +# 构建与运行 +docker build -t geng-skill . +docker run -v $(pwd)/data:/data geng-skill -i /data/paper.csv -o /data/report/ +``` + +--- + +## 常见问题 + +### Q: 数据量有什么要求？ + +| 检测模块 | 最小数据量 | 推荐数据量 | +|----------|-----------|-----------| +| 末位数字检测 | 10 | 50+ | +| 本福特定律 | 30 | 100+ | +| GRIM 测试 | 1（单项） | N/A | +| 固定关系检测 | 5对 | 20+ 对 | +| 小数位一致性 | 5 | 30+ | +| 图像重复 | 2张 | 10+ 张 | + +### Q: 支持什么输入格式？ + +- ✅ CSV（默认逗号分隔，可指定其他分隔符） +- ✅ 直接传入数值列表（Python API） +- ✅ JSON（GRIM 批量测试） +- ✅ 图片目录（PNG/JPG/TIF/BMP） +- ❌ Excel（需先转 CSV） +- ❌ PDF（需先提取数据表格） + +### Q: 如何降低误报率？ + +1. 确认数据范围是否适合该检测（如本福特需跨数量级） +2. 多模块交叉验证，不要仅凭单一结果下结论 +3. 考虑合理解释：仪器精度限制、数据预处理步骤等 +4. 结果需领域专家复核 + +--- + +*Geng Skill v1.0.0 — 致敬"耿同学讲故事"* diff --git a/tools/gengskill/examples/fake_data_demo.csv b/tools/gengskill/examples/fake_data_demo.csv new file mode 100644 index 0000000..0205a30 --- /dev/null +++ b/tools/gengskill/examples/fake_data_demo.csv @@ -0,0 +1,21 @@ +sample_id,control_group,treatment_a,treatment_b,measurement +1,2.34,4.68,7.02,12.5 +2,3.12,6.24,9.36,15.8 +3,1.87,3.74,5.61,8.9 +4,4.56,9.12,13.68,22.1 +5,2.98,5.96,8.94,14.3 +6,3.45,6.90,10.35,17.2 +7,1.23,2.46,3.69,6.8 +8,5.67,11.34,17.01,28.4 +9,2.01,4.02,6.03,10.1 +10,3.89,7.78,11.67,19.5 +11,4.12,8.24,12.36,20.8 +12,1.56,3.12,4.68,7.9 +13,2.78,5.56,8.34,13.6 +14,3.34,6.68,10.02,16.7 +15,4.90,9.80,14.70,24.5 +16,1.45,2.90,4.35,7.2 +17,2.67,5.34,8.01,13.1 +18,3.56,7.12,10.68,17.8 +19,4.23,8.46,12.69,21.2 +20,1.89,3.78,5.67,9.4 diff --git a/tools/gengskill/examples/real_data_demo.csv b/tools/gengskill/examples/real_data_demo.csv new file mode 100644 index 0000000..3e44d1c --- /dev/null +++ b/tools/gengskill/examples/real_data_demo.csv @@ -0,0 +1,31 @@ +sample_id,weight_g,blood_glucose,tumor_volume,survival_days +1,23.4,112,45.2,89 +2,25.1,98,32.8,124 +3,21.8,135,67.3,56 +4,24.7,107,51.9,78 +5,22.3,121,39.4,103 +6,26.5,89,28.7,145 +7,23.9,143,72.1,42 +8,25.8,101,44.6,91 +9,22.1,118,55.3,67 +10,24.2,96,36.2,112 +11,23.6,128,48.7,74 +12,25.4,105,41.3,98 +13,21.5,137,63.8,51 +14,24.9,92,30.5,131 +15,22.8,115,57.6,62 +16,26.1,99,35.4,118 +17,23.3,141,69.2,45 +18,25.7,108,42.8,86 +19,22.6,123,53.1,71 +20,24.4,94,37.9,107 +21,23.1,130,46.5,82 +22,25.2,103,39.7,95 +23,21.9,119,61.4,58 +24,24.6,97,33.6,126 +25,22.4,134,54.8,64 +26,26.3,91,29.3,139 +27,23.8,126,50.2,76 +28,25.5,110,43.1,88 +29,22.0,116,58.9,60 +30,24.1,100,36.7,115 diff --git a/tools/gengskill/requirements.txt b/tools/gengskill/requirements.txt new file mode 100644 index 0000000..8b07495 --- /dev/null +++ b/tools/gengskill/requirements.txt @@ -0,0 +1,4 @@ +numpy>=1.20.0 +scipy>=1.7.0 +Pillow>=9.0.0 +scikit-image>=0.19.0 diff --git a/tools/gengskill/scripts/benford_test.py b/tools/gengskill/scripts/benford_test.py new file mode 100644 index 0000000..454ab78 --- /dev/null +++ b/tools/gengskill/scripts/benford_test.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +""" +本福特定律检测 (Benford's Law Test) +==================================== +原理：跨越多个数量级的自然数据，首位数字遵循特定概率分布： +P(d) = log10(1 + 1/d), d = 1,2,...,9 + +人为编造的数据往往偏离这一分布（倾向于均匀分布或集中在某些数字）。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import math +import numpy as np +from collections import Counter +from scipy import stats + + +# 本福特定律理论概率 +BENFORD_PROBS = {d: math.log10(1 + 1/d) for d in range(1, 10)} + + +def get_first_digit(value): + """提取数值的首位有效数字（1-9）""" + try: + num = abs(float(value)) + if num == 0: + return None + # 转为科学计数法取首位 + s = f"{num:.10e}" + first = int(s[0]) + if 1 <= first <= 9: + return first + except (ValueError, TypeError): + pass + return None + + +def get_first_two_digits(value): + """提取前两位有效数字（10-99）""" + try: + num = abs(float(value)) + if num == 0: + return None + while num < 10: + num *= 10 + while num >= 100: + num /= 10 + return int(num) + except (ValueError, TypeError): + pass + return None + + +def benford_test(values, order=1): + """ + 执行本福特定律检测 + + Parameters + ---------- + values : list + 待检测的数值列表 + order : int + 1 = 首位数字检测, 2 = 前两位数字检测 + + Returns + ------- + dict : 检测结果 + """ + if order == 1: + digits = [] + for v in values: + d = get_first_digit(v) + if d is not None: + digits.append(d) + + if len(digits) < 30: + return { + 'status': 'insufficient_data', + 'message': f'数据量不足（仅{len(digits)}个有效值），本福特定律检测建议至少100个数据点', + 'n_valid': len(digits) + } + + # 统计观测频率 + digit_counts = Counter(digits) + observed = np.array([digit_counts.get(d, 0) for d in range(1, 10)]) + expected = np.array([BENFORD_PROBS[d] * len(digits) for d in range(1, 10)]) + + # 卡方检验 + chi2, p_value = stats.chisquare(observed, expected) + + # Kolmogorov-Smirnov 检验 + observed_freq = observed / len(digits) + expected_freq = np.array([BENFORD_PROBS[d] for d in range(1, 10)]) + + # 最大绝对偏差 (MAD) + mad = np.mean(np.abs(observed_freq - expected_freq)) + + # MAD 阈值参考 (Nigrini 2012) + # Close conformity: MAD < 0.006 + # Acceptable conformity: 0.006 <= MAD < 0.012 + # Marginally acceptable: 0.012 <= MAD < 0.015 + # Nonconformity: MAD >= 0.015 + + if mad < 0.006: + conformity = 'close' + conformity_cn = '高度符合' + elif mad < 0.012: + conformity = 'acceptable' + conformity_cn = '可接受' + elif mad < 0.015: + conformity = 'marginal' + conformity_cn = '边缘' + else: + conformity = 'nonconforming' + conformity_cn = '不符合' + + # 风险评分 + if p_value < 0.001 and mad >= 0.015: + risk_level = 'high' + risk_score = 75 + min(25, mad * 500) + elif p_value < 0.01: + risk_level = 'medium-high' + risk_score = 55 + min(20, mad * 400) + elif p_value < 0.05: + risk_level = 'medium' + risk_score = 35 + min(20, mad * 300) + else: + risk_level = 'low' + risk_score = max(0, mad * 200) + + distribution = { + str(d): { + 'observed': int(observed[d-1]), + 'observed_freq': round(float(observed_freq[d-1]), 4), + 'expected_freq': round(float(expected_freq[d-1]), 4), + 'deviation': round(float(observed_freq[d-1] - expected_freq[d-1]), 4) + } + for d in range(1, 10) + } + + result = { + 'test_name': "Benford's Law Test (本福特定律检测)", + 'status': 'completed', + 'order': order, + 'n_values': len(digits), + 'distribution': distribution, + 'chi_square': round(float(chi2), 4), + 'p_value': float(p_value), + 'degrees_of_freedom': 8, + 'mean_absolute_deviation': round(float(mad), 6), + 'conformity': conformity, + 'conformity_cn': conformity_cn, + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_benford(p_value, mad, conformity_cn, len(digits)), + 'note': '本福特定律适用于跨多个数量级的自然数据集。对于范围有限的数据（如百分比、pH值），该检测可能不适用。' + } + + return result + + else: + return {'status': 'error', 'message': '目前仅支持首位数字检测（order=1）'} + + +def _interpret_benford(p_value, mad, conformity_cn, n): + """生成可读的解释""" + if p_value < 0.001 and mad >= 0.015: + return ( + f"⚠️ 数据首位数字分布严重偏离本福特定律（p < 0.001, MAD = {mad:.4f}）。" + f"符合性判定：{conformity_cn}。" + f"这种偏离在{n}个数据点的样本中非常显著，强烈建议核查数据来源。" + f"注意：需确认数据是否适用本福特定律（需跨越多个数量级）。" + ) + elif p_value < 0.01: + return ( + f"⚠️ 数据首位数字分布显著偏离本福特定律（p < 0.01, MAD = {mad:.4f}）。" + f"符合性判定：{conformity_cn}。建议进一步检查。" + ) + elif p_value < 0.05: + return ( + f"⚡ 数据首位数字分布存在一定偏离（p < 0.05, MAD = {mad:.4f}）。" + f"符合性判定：{conformity_cn}。可能是正常波动，建议结合其他检测综合判断。" + ) + else: + return ( + f"✅ 数据首位数字分布符合本福特定律（p = {p_value:.4f}, MAD = {mad:.4f}）。" + f"符合性判定：{conformity_cn}。未发现异常。" + ) + + +def load_data(input_file, column=None, delimiter=','): + """从CSV文件加载数据""" + import csv + + values = [] + with open(input_file, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f, delimiter=delimiter) + if column and column in reader.fieldnames: + for row in reader: + try: + val = row[column].strip() + if val: + float(val) + values.append(val) + except (ValueError, KeyError): + continue + else: + for row in reader: + for key, val in row.items(): + try: + val = val.strip() + if val: + float(val) + values.append(val) + except (ValueError, AttributeError): + continue + return values + + +def main(): + parser = argparse.ArgumentParser( + description="本福特定律检测 - 检测首位数字是否符合Benford's Law" + ) + parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径') + parser.add_argument('--column', '-c', help='要检测的列名') + parser.add_argument('--order', type=int, default=1, choices=[1, 2], + help='检测阶数：1=首位, 2=前两位') + parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + values = load_data(args.input, args.column, args.delimiter) + + if not values: + print("错误：未能加载有效数据", file=sys.stderr) + sys.exit(1) + + result = benford_test(values, order=args.order) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/decimal_consistency_test.py b/tools/gengskill/scripts/decimal_consistency_test.py new file mode 100644 index 0000000..9909b40 --- /dev/null +++ b/tools/gengskill/scripts/decimal_consistency_test.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +""" +小数位一致性检测 (Decimal Consistency Test) +============================================ +原理：实验测量数据的小数位后数字应具有一定的随机性。 +如果大量数据点的小数部分高度一致（如小数后两位总是相同）， +或小数位数模式过于规律，则暗示数据可能是人为编造的。 + +这是"耿同学"常用的一个检测手段——造假者编造数据时， +小数点后的位数往往呈现不自然的一致性。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import numpy as np +from collections import Counter +from scipy import stats + + +def get_decimal_digits(value, max_digits=4): + """提取数值的小数部分各位数字""" + s = str(value).strip() + if '.' not in s: + return [] + decimal_part = s.split('.')[1] + return [int(d) for d in decimal_part[:max_digits]] + + +def get_decimal_string(value): + """提取数值的完整小数字符串""" + s = str(value).strip() + if '.' not in s: + return '' + return s.split('.')[1] + + +def count_decimal_places(value): + """计算数值的小数位数""" + s = str(value).strip() + if '.' not in s: + return 0 + return len(s.split('.')[1]) + + +def decimal_consistency_test(values): + """ + 执行小数位一致性检测 + + Parameters + ---------- + values : list + 待检测的数值列表（字符串形式保留原始精度） + + Returns + ------- + dict : 检测结果 + """ + if len(values) < 5: + return { + 'status': 'insufficient_data', + 'message': f'数据量不足（仅{len(values)}个值），需要至少5个数据点' + } + + # 分析1: 小数位数一致性 + decimal_places = [count_decimal_places(v) for v in values] + places_counter = Counter(decimal_places) + most_common_places = places_counter.most_common(1)[0] + places_uniformity = most_common_places[1] / len(values) + + # 分析2: 小数部分重复度 + decimal_strings = [get_decimal_string(v) for v in values if '.' in str(v)] + if decimal_strings: + decimal_counter = Counter(decimal_strings) + n_unique_decimals = len(decimal_counter) + most_repeated = decimal_counter.most_common(1)[0] + max_repetition_rate = most_repeated[1] / len(decimal_strings) + else: + n_unique_decimals = 0 + max_repetition_rate = 0 + most_repeated = ('N/A', 0) + + # 分析3: 各小数位数字分布 + position_analyses = {} + max_positions = max(decimal_places) if decimal_places else 0 + + for pos in range(min(max_positions, 4)): + digits_at_pos = [] + for v in values: + decs = get_decimal_digits(v) + if len(decs) > pos: + digits_at_pos.append(decs[pos]) + + if len(digits_at_pos) >= 10: + digit_counts = Counter(digits_at_pos) + observed = np.array([digit_counts.get(i, 0) for i in range(10)]) + expected = np.full(10, len(digits_at_pos) / 10.0) + chi2, p_value = stats.chisquare(observed, expected) + + position_analyses[f'position_{pos+1}'] = { + 'n_values': len(digits_at_pos), + 'distribution': {str(i): int(observed[i]) for i in range(10)}, + 'chi_square': round(float(chi2), 4), + 'p_value': float(p_value), + 'is_uniform': p_value > 0.05 + } + + # 分析4: 相邻数据小数部分相关性 + if len(decimal_strings) >= 5: + # 将小数部分转为数值进行自相关分析 + decimal_values = [] + for ds in decimal_strings: + try: + decimal_values.append(float('0.' + ds) if ds else 0.0) + except ValueError: + decimal_values.append(0.0) + + if len(decimal_values) >= 5: + # 计算一阶自相关 + x = np.array(decimal_values) + x_centered = x - np.mean(x) + if np.std(x) > 0: + autocorr = np.correlate(x_centered[:-1], x_centered[1:]) / (len(x_centered) - 1) / np.var(x) + autocorr_val = float(autocorr[0]) if len(autocorr) > 0 else 0 + else: + autocorr_val = 1.0 # 完全一致 + else: + autocorr_val = None + else: + autocorr_val = None + + # 综合风险评分 + risk_factors = [] + + # 因子1: 小数位数过于一致 + if places_uniformity > 0.95 and len(values) > 10: + risk_factors.append(('decimal_places_uniform', 20)) + + # 因子2: 小数部分重复度过高 + if max_repetition_rate > 0.5: + risk_factors.append(('high_repetition', 30)) + elif max_repetition_rate > 0.3: + risk_factors.append(('moderate_repetition', 15)) + + # 因子3: 某个位置数字分布异常 + for pos_key, pos_data in position_analyses.items(): + if pos_data['p_value'] < 0.001: + risk_factors.append((f'{pos_key}_nonuniform', 25)) + elif pos_data['p_value'] < 0.01: + risk_factors.append((f'{pos_key}_marginal', 10)) + + # 因子4: 自相关异常高 + if autocorr_val is not None and abs(autocorr_val) > 0.8: + risk_factors.append(('high_autocorrelation', 20)) + + risk_score = min(100, sum(score for _, score in risk_factors)) + + if risk_score >= 70: + risk_level = 'high' + elif risk_score >= 45: + risk_level = 'medium-high' + elif risk_score >= 25: + risk_level = 'medium' + else: + risk_level = 'low' + + result = { + 'test_name': 'Decimal Consistency Test (小数位一致性检测)', + 'status': 'completed', + 'n_values': len(values), + 'decimal_places_analysis': { + 'distribution': dict(places_counter), + 'most_common_places': most_common_places[0], + 'uniformity_rate': round(float(places_uniformity), 4) + }, + 'decimal_repetition': { + 'n_unique_patterns': n_unique_decimals, + 'most_repeated_pattern': most_repeated[0], + 'most_repeated_count': most_repeated[1], + 'max_repetition_rate': round(float(max_repetition_rate), 4) + }, + 'position_digit_analysis': position_analyses, + 'autocorrelation': round(float(autocorr_val), 4) if autocorr_val is not None else None, + 'risk_factors': [f for f, _ in risk_factors], + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated) + } + + return result + + +def _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated): + """生成可读的解释""" + if not risk_factors: + return "✅ 小数位分布未发现明显异常，数据的小数部分具有合理的随机性。" + + issues = [] + for factor, _ in risk_factors: + if 'repetition' in factor: + issues.append(f"小数部分 '{most_repeated[0]}' 重复出现 {most_repeated[1]} 次（{max_repetition_rate:.0%}）") + elif 'nonuniform' in factor: + issues.append("某些小数位的数字分布严重偏离均匀分布") + elif 'autocorrelation' in factor: + issues.append("相邻数据的小数部分存在异常高的自相关") + elif 'places_uniform' in factor: + issues.append(f"所有数据小数位数高度一致（{places_uniformity:.0%}相同）") + + issues_str = "；".join(issues) + + if len(risk_factors) >= 3: + return f"⚠️ 发现多项小数位异常：{issues_str}。这些模式在自然实验数据中非常罕见，强烈建议核查原始数据。" + elif len(risk_factors) >= 2: + return f"⚠️ 发现小数位可疑模式：{issues_str}。建议进一步检查。" + else: + return f"⚡ 发现轻微异常：{issues_str}。可能是测量精度限制导致，建议结合其他检测综合判断。" + + +def load_data(input_file, column=None, delimiter=','): + """从CSV文件加载数据（保留原始字符串精度）""" + import csv + + values = [] + with open(input_file, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f, delimiter=delimiter) + if column and column in reader.fieldnames: + for row in reader: + val = row[column].strip() + if val: + try: + float(val) + values.append(val) + except ValueError: + continue + else: + for row in reader: + for key, val in row.items(): + try: + val = val.strip() + if val: + float(val) + values.append(val) + except (ValueError, AttributeError): + continue + return values + + +def main(): + parser = argparse.ArgumentParser( + description='小数位一致性检测 - 检测数据小数部分是否存在异常模式' + ) + parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径') + parser.add_argument('--column', '-c', help='要检测的列名') + parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + values = load_data(args.input, args.column, args.delimiter) + + if not values: + print("错误：未能加载有效数据", file=sys.stderr) + sys.exit(1) + + result = decimal_consistency_test(values) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/fixed_relation_test.py b/tools/gengskill/scripts/fixed_relation_test.py new file mode 100644 index 0000000..0f2abee --- /dev/null +++ b/tools/gengskill/scripts/fixed_relation_test.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +固定关系检测 (Fixed Relationship Detection) +============================================ +原理：两组独立实验数据之间不应存在恒定的差值、比值或线性关系。 +如果不同实验条件下的数据存在固定的数学关系，暗示数据可能是 +从单一数据源通过简单数学运算生成的，而非独立实验获得。 + +这是"耿同学"打假方法中的核心策略之一——他发现许多造假论文中 +不同实验组的数据存在固定差值或固定比例关系。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import numpy as np +from scipy import stats + + +def detect_fixed_difference(col1, col2, tolerance=0.01): + """检测两列数据是否存在固定差值""" + differences = np.array(col2) - np.array(col1) + + if len(differences) < 3: + return None + + # 计算差值的变异系数 + mean_diff = np.mean(differences) + std_diff = np.std(differences) + + if abs(mean_diff) < 1e-10: + cv = float('inf') if std_diff > 0 else 0 + else: + cv = abs(std_diff / mean_diff) + + # 判断差值是否恒定（CV极小） + is_fixed = cv < tolerance and std_diff < tolerance * abs(mean_diff + 1e-10) + + # 检查差值是否完全相同 + unique_diffs = np.unique(np.round(differences, 6)) + is_exact = len(unique_diffs) == 1 + + return { + 'type': 'fixed_difference', + 'type_cn': '固定差值', + 'mean_difference': round(float(mean_diff), 6), + 'std_difference': round(float(std_diff), 6), + 'cv': round(float(cv), 6) if cv != float('inf') else 'inf', + 'is_fixed': bool(is_fixed or is_exact), + 'is_exact': bool(is_exact), + 'n_unique_differences': int(len(unique_diffs)) + } + + +def detect_fixed_ratio(col1, col2, tolerance=0.01): + """检测两列数据是否存在固定比值""" + col1 = np.array(col1, dtype=float) + col2 = np.array(col2, dtype=float) + + # 避免除以零 + mask = col1 != 0 + if np.sum(mask) < 3: + return None + + ratios = col2[mask] / col1[mask] + + mean_ratio = np.mean(ratios) + std_ratio = np.std(ratios) + + if abs(mean_ratio) < 1e-10: + cv = float('inf') if std_ratio > 0 else 0 + else: + cv = abs(std_ratio / mean_ratio) + + is_fixed = cv < tolerance + unique_ratios = np.unique(np.round(ratios, 6)) + is_exact = len(unique_ratios) == 1 + + return { + 'type': 'fixed_ratio', + 'type_cn': '固定比值', + 'mean_ratio': round(float(mean_ratio), 6), + 'std_ratio': round(float(std_ratio), 6), + 'cv': round(float(cv), 6) if cv != float('inf') else 'inf', + 'is_fixed': bool(is_fixed or is_exact), + 'is_exact': bool(is_exact), + 'n_unique_ratios': int(len(unique_ratios)) + } + + +def detect_linear_relationship(col1, col2): + """检测两列数据是否存在高度线性关系""" + col1 = np.array(col1, dtype=float) + col2 = np.array(col2, dtype=float) + + if len(col1) < 3: + return None + + # 线性回归 + slope, intercept, r_value, p_value, std_err = stats.linregress(col1, col2) + r_squared = r_value ** 2 + + # 残差分析 + predicted = slope * col1 + intercept + residuals = col2 - predicted + max_residual = np.max(np.abs(residuals)) + mean_residual = np.mean(np.abs(residuals)) + + # R² 非常接近1且残差极小 + is_suspicious = r_squared > 0.9999 and max_residual < 0.001 * np.std(col2) + + return { + 'type': 'linear_relationship', + 'type_cn': '线性关系', + 'slope': round(float(slope), 6), + 'intercept': round(float(intercept), 6), + 'r_squared': round(float(r_squared), 8), + 'p_value': float(p_value), + 'max_residual': round(float(max_residual), 8), + 'mean_residual': round(float(mean_residual), 8), + 'is_suspicious': bool(is_suspicious) + } + + +def detect_decimal_pattern(col1, col2): + """检测两列数据小数部分是否高度一致""" + col1 = np.array(col1, dtype=float) + col2 = np.array(col2, dtype=float) + + # 提取小数部分 + dec1 = col1 - np.floor(col1) + dec2 = col2 - np.floor(col2) + + # 检查小数部分是否一致 + dec_diff = np.abs(dec1 - dec2) + n_matching = np.sum(dec_diff < 0.001) + match_rate = n_matching / len(col1) + + return { + 'type': 'decimal_pattern', + 'type_cn': '小数位一致性', + 'n_matching_decimals': int(n_matching), + 'match_rate': round(float(match_rate), 4), + 'is_suspicious': match_rate > 0.8 + } + + +def fixed_relation_test(col1, col2, col1_name='Column A', col2_name='Column B'): + """ + 综合固定关系检测 + + Parameters + ---------- + col1 : list of float + 第一列数据 + col2 : list of float + 第二列数据 + + Returns + ------- + dict : 检测结果 + """ + if len(col1) != len(col2): + return {'status': 'error', 'message': '两列数据长度不一致'} + + if len(col1) < 3: + return {'status': 'insufficient_data', 'message': '数据量不足，至少需要3个数据点'} + + col1 = [float(x) for x in col1] + col2 = [float(x) for x in col2] + + # 执行各项检测 + results = {} + + diff_result = detect_fixed_difference(col1, col2) + if diff_result: + results['fixed_difference'] = diff_result + + ratio_result = detect_fixed_ratio(col1, col2) + if ratio_result: + results['fixed_ratio'] = ratio_result + + linear_result = detect_linear_relationship(col1, col2) + if linear_result: + results['linear_relationship'] = linear_result + + decimal_result = detect_decimal_pattern(col1, col2) + if decimal_result: + results['decimal_pattern'] = decimal_result + + # 综合风险评估 + n_suspicious = sum([ + 1 for r in results.values() + if r.get('is_fixed') or r.get('is_suspicious') + ]) + + if n_suspicious >= 3: + risk_level = 'high' + risk_score = 85 + elif n_suspicious == 2: + risk_level = 'medium-high' + risk_score = 65 + elif n_suspicious == 1: + risk_level = 'medium' + risk_score = 45 + else: + risk_level = 'low' + risk_score = 10 + + # 如果存在完全精确的固定关系，直接拉高风险 + if any(r.get('is_exact') for r in results.values()): + risk_level = 'high' + risk_score = max(risk_score, 90) + + summary = { + 'test_name': 'Fixed Relationship Detection (固定关系检测)', + 'status': 'completed', + 'n_data_points': len(col1), + 'column_1': col1_name, + 'column_2': col2_name, + 'detections': results, + 'n_suspicious_patterns': n_suspicious, + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name) + } + + return summary + + +def _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name): + """生成可读的解释""" + findings = [] + + if results.get('fixed_difference', {}).get('is_fixed'): + d = results['fixed_difference'] + findings.append(f"两列数据存在固定差值 {d['mean_difference']}") + + if results.get('fixed_ratio', {}).get('is_fixed'): + r = results['fixed_ratio'] + findings.append(f"两列数据存在固定比值 {r['mean_ratio']}") + + if results.get('linear_relationship', {}).get('is_suspicious'): + l = results['linear_relationship'] + findings.append(f"两列数据存在完美线性关系 (R² = {l['r_squared']})") + + if results.get('decimal_pattern', {}).get('is_suspicious'): + p = results['decimal_pattern'] + findings.append(f"两列数据小数部分高度一致 (匹配率 {p['match_rate']:.0%})") + + if not findings: + return f"✅ {col1_name} 与 {col2_name} 之间未发现固定数学关系，数据看起来是独立的。" + + findings_str = "；".join(findings) + return ( + f"⚠️ {col1_name} 与 {col2_name} 之间发现以下可疑模式：{findings_str}。" + f"独立实验数据通常不应存在如此精确的数学关系，建议核查数据是否来自独立实验。" + ) + + +def load_data(input_file, col1, col2, delimiter=','): + """从CSV文件加载两列数据""" + import csv + + data1, data2 = [], [] + with open(input_file, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f, delimiter=delimiter) + for row in reader: + try: + v1 = float(row[col1].strip()) + v2 = float(row[col2].strip()) + data1.append(v1) + data2.append(v2) + except (ValueError, KeyError): + continue + return data1, data2 + + +def main(): + parser = argparse.ArgumentParser( + description='固定关系检测 - 检测两组数据间是否存在不自然的数学关系' + ) + parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径') + parser.add_argument('--col1', required=True, help='第一列列名') + parser.add_argument('--col2', required=True, help='第二列列名') + parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + data1, data2 = load_data(args.input, args.col1, args.col2, args.delimiter) + + if not data1: + print("错误：未能加载有效数据", file=sys.stderr) + sys.exit(1) + + result = fixed_relation_test(data1, data2, args.col1, args.col2) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/geng_assess.py b/tools/gengskill/scripts/geng_assess.py new file mode 100644 index 0000000..3bd29be --- /dev/null +++ b/tools/gengskill/scripts/geng_assess.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +""" +Geng 综合评估引擎 (Comprehensive Assessment Engine) +===================================================== +一键运行所有适用的检测模块，生成综合学术数据打假报告。 + +支持的领域： +- biomedical: 生物医学（Western blot, 流式, 动物实验） +- chemistry: 化学（光谱, 产率, 催化） +- physics: 物理/材料（性能曲线, 电学/力学） +- social_science: 社会科学（问卷, 量表） +- clinical: 临床医学（生存数据, 临床指标） +- general: 通用（不指定领域） + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import os +import csv +import time +from datetime import datetime +from pathlib import Path + +# 导入各检测模块 +from last_digit_test import last_digit_test +from benford_test import benford_test +from decimal_consistency_test import decimal_consistency_test +from fixed_relation_test import fixed_relation_test +from grim_test import grim_test_batch + + +def load_csv_data(input_file, delimiter=','): + """加载CSV数据，返回列名和数据""" + columns = {} + with open(input_file, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f, delimiter=delimiter) + fieldnames = reader.fieldnames + for row in reader: + for col in fieldnames: + if col not in columns: + columns[col] = [] + columns[col].append(row[col].strip() if row[col] else '') + return columns, fieldnames + + +def identify_numeric_columns(columns): + """识别数值列""" + numeric_cols = {} + for col_name, values in columns.items(): + numeric_values = [] + for v in values: + try: + if v: + float(v) + numeric_values.append(v) + except ValueError: + continue + # 至少50%的值是数值 + if len(numeric_values) >= len(values) * 0.5 and len(numeric_values) >= 5: + numeric_cols[col_name] = numeric_values + return numeric_cols + + +def run_assessment(input_file, domain='general', output_dir=None, delimiter=','): + """ + 执行综合评估 + + Parameters + ---------- + input_file : str + 输入CSV文件 + domain : str + 研究领域 + output_dir : str + 输出目录 + delimiter : str + CSV分隔符 + """ + start_time = time.time() + + # 加载数据 + columns, fieldnames = load_csv_data(input_file, delimiter) + numeric_cols = identify_numeric_columns(columns) + + if not numeric_cols: + return { + 'status': 'error', + 'message': '未找到有效的数值列，请检查输入文件格式' + } + + report = { + 'meta': { + 'tool': 'Geng Academic Data Fraud Detection Tool', + 'version': '1.0.0', + 'timestamp': datetime.now().isoformat(), + 'input_file': os.path.basename(input_file), + 'domain': domain, + 'n_columns': len(fieldnames), + 'n_numeric_columns': len(numeric_cols), + 'numeric_columns': list(numeric_cols.keys()), + 'n_rows': max(len(v) for v in columns.values()) if columns else 0 + }, + 'module_results': {}, + 'summary': {} + } + + all_risk_scores = [] + + # ========================================== + # Module 1: 末位数字检测 (对每个数值列) + # ========================================== + print("🔍 执行末位数字检测...") + last_digit_results = {} + for col_name, values in numeric_cols.items(): + result = last_digit_test(values, method='all_digits') + if result.get('status') == 'completed': + last_digit_results[col_name] = result + all_risk_scores.append(result.get('risk_score', 0)) + + if last_digit_results: + report['module_results']['last_digit_test'] = { + 'module_name': '末位数字检测', + 'description': '检测数据末位数字是否偏离均匀分布', + 'columns_tested': len(last_digit_results), + 'results': last_digit_results + } + + # ========================================== + # Module 2: 本福特定律检测 (对每个数值列) + # ========================================== + print("🔍 执行本福特定律检测...") + benford_results = {} + for col_name, values in numeric_cols.items(): + # 本福特定律适用于跨多个数量级的数据 + try: + float_values = [float(v) for v in values if v] + value_range = max(float_values) / (min(v for v in float_values if v > 0) + 1e-10) + # 只对跨度超过1个数量级的列做本福特检测 + if value_range > 10: + result = benford_test(values, order=1) + if result.get('status') == 'completed': + benford_results[col_name] = result + all_risk_scores.append(result.get('risk_score', 0)) + except (ValueError, ZeroDivisionError): + continue + + if benford_results: + report['module_results']['benford_test'] = { + 'module_name': '本福特定律检测', + 'description': '检测首位数字是否符合Benford\'s Law', + 'columns_tested': len(benford_results), + 'results': benford_results + } + + # ========================================== + # Module 3: 小数位一致性检测 (对每个数值列) + # ========================================== + print("🔍 执行小数位一致性检测...") + decimal_results = {} + for col_name, values in numeric_cols.items(): + # 只检测包含小数的列 + has_decimal = any('.' in v for v in values if v) + if has_decimal: + result = decimal_consistency_test(values) + if result.get('status') == 'completed': + decimal_results[col_name] = result + all_risk_scores.append(result.get('risk_score', 0)) + + if decimal_results: + report['module_results']['decimal_consistency_test'] = { + 'module_name': '小数位一致性检测', + 'description': '检测数据小数部分是否存在异常模式', + 'columns_tested': len(decimal_results), + 'results': decimal_results + } + + # ========================================== + # Module 4: 固定关系检测 (两两比较数值列) + # ========================================== + print("🔍 执行固定关系检测...") + fixed_results = {} + col_names = list(numeric_cols.keys()) + + # 限制比较对数，避免组合爆炸 + max_pairs = min(10, len(col_names) * (len(col_names) - 1) // 2) + pair_count = 0 + + for i in range(len(col_names)): + if pair_count >= max_pairs: + break + for j in range(i + 1, len(col_names)): + if pair_count >= max_pairs: + break + col1_name = col_names[i] + col2_name = col_names[j] + + # 确保两列长度一致且有足够数据 + vals1 = numeric_cols[col1_name] + vals2 = numeric_cols[col2_name] + + # 配对：只取两列都有值的行 + paired_1, paired_2 = [], [] + for v1, v2 in zip(vals1, vals2): + try: + if v1 and v2: + float(v1) + float(v2) + paired_1.append(v1) + paired_2.append(v2) + except ValueError: + continue + + if len(paired_1) >= 5: + result = fixed_relation_test(paired_1, paired_2, col1_name, col2_name) + if result.get('status') == 'completed': + pair_key = f"{col1_name} vs {col2_name}" + fixed_results[pair_key] = result + all_risk_scores.append(result.get('risk_score', 0)) + pair_count += 1 + + if fixed_results: + report['module_results']['fixed_relation_test'] = { + 'module_name': '固定关系检测', + 'description': '检测不同列数据间是否存在不自然的数学关系', + 'pairs_tested': len(fixed_results), + 'results': fixed_results + } + + # ========================================== + # 综合评分 + # ========================================== + print("📊 生成综合评估...") + + if all_risk_scores: + # 综合评分：取各模块最高分的加权平均 + max_score = max(all_risk_scores) + mean_score = sum(all_risk_scores) / len(all_risk_scores) + # 综合分 = 60% 最高分 + 40% 平均分 + overall_score = 0.6 * max_score + 0.4 * mean_score + overall_score = min(100, overall_score) + else: + overall_score = 0 + + # 统计各风险等级 + high_risk_modules = [s for s in all_risk_scores if s >= 70] + medium_risk_modules = [s for s in all_risk_scores if 40 <= s < 70] + low_risk_modules = [s for s in all_risk_scores if s < 40] + + if overall_score >= 75: + overall_level = 'critical' + overall_level_cn = '🔴 极高风险' + overall_emoji = '🔴' + elif overall_score >= 50: + overall_level = 'high' + overall_level_cn = '🟠 高风险' + overall_emoji = '🟠' + elif overall_score >= 25: + overall_level = 'medium' + overall_level_cn = '🟡 中风险' + overall_emoji = '🟡' + else: + overall_level = 'low' + overall_level_cn = '🟢 低风险' + overall_emoji = '🟢' + + report['summary'] = { + 'overall_risk_score': round(float(overall_score), 1), + 'overall_risk_level': overall_level, + 'overall_risk_level_cn': overall_level_cn, + 'n_modules_run': len(report['module_results']), + 'n_tests_total': len(all_risk_scores), + 'n_high_risk': len(high_risk_modules), + 'n_medium_risk': len(medium_risk_modules), + 'n_low_risk': len(low_risk_modules), + 'execution_time_seconds': round(time.time() - start_time, 2), + 'conclusion': _generate_conclusion(overall_score, overall_level, report['module_results']), + 'recommendations': _generate_recommendations(overall_level, domain, report['module_results']) + } + + # 保存报告 + if output_dir: + os.makedirs(output_dir, exist_ok=True) + report_path = os.path.join(output_dir, 'geng_assessment_report.json') + with open(report_path, 'w', encoding='utf-8') as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + # 生成可读的 Markdown 报告 + md_path = os.path.join(output_dir, 'geng_assessment_report.md') + with open(md_path, 'w', encoding='utf-8') as f: + f.write(_generate_markdown_report(report)) + + print(f"\n📄 JSON报告已保存至: {report_path}") + print(f"📄 Markdown报告已保存至: {md_path}") + + return report + + +def _generate_conclusion(score, level, modules): + """生成结论""" + if level == 'critical': + return ( + "⚠️ 综合评估显示数据存在系统性异常，多项检测指标显著偏离正常预期。" + "强烈建议对原始实验数据进行全面核查。" + "注意：本工具仅提供线索筛查，最终判定需要领域专家复核。" + ) + elif level == 'high': + return ( + "⚠️ 数据中发现多处可疑模式，部分指标显著异常。" + "建议对标记为高风险的数据列进行重点核查。" + ) + elif level == 'medium': + return ( + "⚡ 数据存在一些轻微异常，但不足以构成造假的确证。" + "可能是测量精度限制、数据处理方式等正常因素导致。" + "建议结合论文方法学描述进行综合判断。" + ) + else: + return ( + "✅ 数据各项检测指标均在正常范围内，未发现明显的造假迹象。" + "注意：通过检测不代表数据一定真实，某些高明的造假可能无法被统计方法捕获。" + ) + + +def _generate_recommendations(level, domain, modules): + """生成建议""" + recs = [] + + if level in ('critical', 'high'): + recs.append("核查原始实验记录和数据记录本") + recs.append("验证数据是否来自独立实验") + recs.append("联系通讯作者要求提供原始数据") + if domain == 'biomedical': + recs.append("检查Western blot原始图片和流式原始FCS文件") + recs.append("考虑向期刊或机构提交正式质疑") + elif level == 'medium': + recs.append("仔细阅读论文方法学部分，确认数据采集方式") + recs.append("检查是否存在合理的解释（如仪器精度限制）") + recs.append("可考虑联系作者进行非正式沟通") + else: + recs.append("当前数据未发现明显异常") + recs.append("可考虑对补充材料中的数据做进一步检测") + + return recs + + +def _generate_markdown_report(report): + """生成Markdown格式报告""" + meta = report['meta'] + summary = report['summary'] + + md = [] + md.append("# 📋 Geng 学术数据打假检测报告\n") + md.append(f"> 生成时间: {meta['timestamp']}") + md.append(f"> 检测工具: {meta['tool']} v{meta['version']}") + md.append('> 致敬"耿同学讲故事"\n') + + md.append("## 📊 综合评估结果\n") + md.append(f"| 指标 | 结果 |") + md.append(f"|------|------|") + md.append(f"| **综合风险评分** | **{summary['overall_risk_score']}/100** |") + md.append(f"| **风险等级** | {summary['overall_risk_level_cn']} |") + md.append(f"| 输入文件 | {meta['input_file']} |") + md.append(f"| 检测领域 | {meta['domain']} |") + md.append(f"| 数值列数 | {meta['n_numeric_columns']} |") + md.append(f"| 数据行数 | {meta['n_rows']} |") + md.append(f"| 运行模块数 | {summary['n_modules_run']} |") + md.append(f"| 检测总数 | {summary['n_tests_total']} |") + md.append(f"| 高风险项 | {summary['n_high_risk']} |") + md.append(f"| 执行耗时 | {summary['execution_time_seconds']}s |\n") + + md.append("## 📝 结论\n") + md.append(f"{summary['conclusion']}\n") + + md.append("## 💡 建议\n") + for i, rec in enumerate(summary['recommendations'], 1): + md.append(f"{i}. {rec}") + md.append("") + + md.append("## 🔬 各模块检测详情\n") + + for module_key, module_data in report['module_results'].items(): + md.append(f"### {module_data['module_name']}\n") + md.append(f"_{module_data['description']}_\n") + + if 'columns_tested' in module_data: + md.append(f"- 检测列数: {module_data['columns_tested']}") + if 'pairs_tested' in module_data: + md.append(f"- 检测对数: {module_data['pairs_tested']}") + + # 列出各列/对的风险评分 + results = module_data.get('results', {}) + if results: + md.append(f"\n| 检测对象 | 风险评分 | 风险等级 | 说明 |") + md.append(f"|----------|----------|----------|------|") + for key, res in results.items(): + score = res.get('risk_score', 'N/A') + level = res.get('risk_level', 'N/A') + interp = res.get('interpretation', '')[:60] + md.append(f"| {key} | {score} | {level} | {interp}... |") + md.append("") + + md.append("---\n") + md.append("## ⚠️ 重要声明\n") + md.append("- 本工具仅用于辅助筛查，**不能作为造假的最终判定依据**") + md.append("- 数据异常 ≠ 数据造假（可能是仪器校准、单位转换、排版错误等）") + md.append("- 检测结果需要领域专家复核") + md.append("- 使用本工具时应遵守学术伦理和法律法规\n") + md.append("---\n") + md.append('*Powered by Geng Skill — 致敬"耿同学讲故事"*') + + return "\n".join(md) + + +def main(): + parser = argparse.ArgumentParser( + description='Geng 综合评估引擎 - 一键运行所有检测模块' + ) + parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径') + parser.add_argument('--domain', default='general', + choices=['biomedical', 'chemistry', 'physics', + 'social_science', 'clinical', 'general'], + help='研究领域') + parser.add_argument('--output', '-o', default='./report', help='输出目录') + parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符') + + args = parser.parse_args() + + if not os.path.isfile(args.input): + print(f"错误：文件不存在: {args.input}", file=sys.stderr) + sys.exit(1) + + print(f"{'='*60}") + print(f" Geng 学术数据打假检测工具 v1.0.0") + print(' 致敬"耿同学讲故事" — 用数据说话，让造假无所遁形') + print(f"{'='*60}") + print(f"\n📁 输入文件: {args.input}") + print(f"🔬 检测领域: {args.domain}") + print(f"📂 输出目录: {args.output}\n") + + report = run_assessment(args.input, args.domain, args.output, args.delimiter) + + if report.get('status') == 'error': + print(f"\n❌ 错误: {report['message']}", file=sys.stderr) + sys.exit(1) + + summary = report['summary'] + print(f"\n{'='*60}") + print(f" 综合评估结果") + print(f"{'='*60}") + print(f"\n {summary['overall_risk_level_cn']}") + print(f" 综合风险评分: {summary['overall_risk_score']}/100") + print(f" 高风险项: {summary['n_high_risk']} | " + f"中风险项: {summary['n_medium_risk']} | " + f"低风险项: {summary['n_low_risk']}") + print(f"\n {summary['conclusion']}") + print(f"\n{'='*60}") + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/grim_test.py b/tools/gengskill/scripts/grim_test.py new file mode 100644 index 0000000..4126545 --- /dev/null +++ b/tools/gengskill/scripts/grim_test.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +GRIM 测试 (Granularity-Related Inconsistency of Means) +======================================================== +原理：对于整数取值的数据（如李克特量表1-5、年龄等）， +给定样本量 n，合法的平均值只能取有限集合中的值。 +如果报告的平均值不在合法集合中，则数据存在不一致性。 + +例：n=25，数据取值为整数，则平均值只能是 k/25 的形式， +小数部分只能是 .00, .04, .08, .12, ..., .96 + +参考：Brown & Heathers (2017). The GRIM Test. SPPS. + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import math +from decimal import Decimal, ROUND_HALF_UP + + +def grim_test_single(mean, n, decimals=2, scale_min=None, scale_max=None): + """ + 对单个均值执行 GRIM 测试 + + Parameters + ---------- + mean : float or str + 报告的平均值 + n : int + 样本量 + decimals : int + 报告的小数位数 + scale_min : int, optional + 量表最小值（用于范围检查） + scale_max : int, optional + 量表最大值（用于范围检查） + + Returns + ------- + dict : 检测结果 + """ + try: + mean_val = Decimal(str(mean)) + n = int(n) + except (ValueError, TypeError) as e: + return {'status': 'error', 'message': f'无效输入: {e}'} + + if n <= 0: + return {'status': 'error', 'message': '样本量必须大于0'} + + # 范围检查 + if scale_min is not None and scale_max is not None: + if float(mean_val) < scale_min or float(mean_val) > scale_max: + return { + 'status': 'range_error', + 'message': f'均值 {mean} 超出量表范围 [{scale_min}, {scale_max}]', + 'consistent': False + } + + # 计算总和 = mean * n + total = mean_val * n + + # 对于整数取值数据，总和必须是整数 + # 考虑四舍五入误差：检查 total 是否足够接近某个整数 + granularity = Decimal(1) / Decimal(10 ** decimals) + + # 在四舍五入精度范围内检查 + # mean 可能是真实值四舍五入到 decimals 位的结果 + # 真实 mean 在 [mean - 0.5*granularity, mean + 0.5*granularity) 范围内 + lower_total = (mean_val - granularity / 2) * n + upper_total = (mean_val + granularity / 2) * n + + # 检查这个范围内是否包含整数 + lower_int = math.ceil(float(lower_total)) + upper_int = math.floor(float(upper_total)) + + consistent = lower_int <= upper_int + + # 计算最近的合法均值 + nearest_total = round(float(mean_val * n)) + nearest_mean = nearest_total / n + + # 格式化到指定小数位 + fmt = f"%.{decimals}f" + nearest_mean_str = fmt % nearest_mean + reported_mean_str = fmt % float(mean_val) + + result = { + 'reported_mean': str(mean), + 'sample_size': n, + 'decimals': decimals, + 'consistent': consistent, + 'computed_sum': float(mean_val * n), + 'nearest_valid_mean': nearest_mean_str, + 'difference': round(abs(float(mean_val) - nearest_mean), decimals + 2) + } + + if scale_min is not None and scale_max is not None: + result['scale_range'] = f"[{scale_min}, {scale_max}]" + + return result + + +def grim_test_batch(items): + """ + 批量 GRIM 测试 + + Parameters + ---------- + items : list of dict + 每个字典包含 'mean', 'n', 可选 'decimals', 'label' + + Returns + ------- + dict : 批量检测结果 + """ + results = [] + n_inconsistent = 0 + + for i, item in enumerate(items): + mean = item.get('mean') + n = item.get('n') + decimals = item.get('decimals', 2) + label = item.get('label', f'Item {i+1}') + scale_min = item.get('scale_min') + scale_max = item.get('scale_max') + + res = grim_test_single(mean, n, decimals, scale_min, scale_max) + res['label'] = label + results.append(res) + + if not res.get('consistent', True): + n_inconsistent += 1 + + # 整体评估 + total = len(results) + inconsistency_rate = n_inconsistent / total if total > 0 else 0 + + # 风险评分 + if inconsistency_rate > 0.5: + risk_level = 'high' + risk_score = 70 + inconsistency_rate * 30 + elif inconsistency_rate > 0.25: + risk_level = 'medium-high' + risk_score = 50 + inconsistency_rate * 40 + elif inconsistency_rate > 0: + risk_level = 'medium' + risk_score = 30 + inconsistency_rate * 40 + else: + risk_level = 'low' + risk_score = 0 + + summary = { + 'test_name': 'GRIM Test (均值粒度一致性检验)', + 'status': 'completed', + 'total_items': total, + 'inconsistent_items': n_inconsistent, + 'inconsistency_rate': round(inconsistency_rate, 4), + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'details': results, + 'interpretation': _interpret_grim(n_inconsistent, total, inconsistency_rate) + } + + return summary + + +def _interpret_grim(n_inconsistent, total, rate): + """生成可读的解释""" + if n_inconsistent == 0: + return f"✅ 全部 {total} 个均值通过 GRIM 检验，未发现数值不一致。" + elif rate > 0.5: + return ( + f"⚠️ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。" + f"超过半数均值与样本量不兼容，这是严重的数据不一致信号。" + f"强烈建议核查原始数据。" + ) + elif rate > 0.25: + return ( + f"⚠️ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。" + f"建议仔细核查这些不一致的数据点。" + ) + else: + return ( + f"⚡ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。" + f"少量不一致可能是四舍五入方式不同导致，建议结合其他检测综合判断。" + ) + + +def main(): + parser = argparse.ArgumentParser( + description='GRIM测试 - 检测报告均值与样本量的一致性' + ) + parser.add_argument('--mean', type=str, help='报告的平均值') + parser.add_argument('--n', type=int, help='样本量') + parser.add_argument('--decimals', type=int, default=2, help='小数位数') + parser.add_argument('--scale', type=str, help='量表范围，如 "1-5"') + parser.add_argument('--input', '-i', help='输入JSON文件（批量测试）') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + scale_min, scale_max = None, None + if args.scale: + parts = args.scale.split('-') + if len(parts) == 2: + scale_min, scale_max = int(parts[0]), int(parts[1]) + + if args.input: + # 批量模式 + with open(args.input, 'r', encoding='utf-8') as f: + items = json.load(f) + result = grim_test_batch(items) + elif args.mean and args.n: + # 单项模式 + res = grim_test_single(args.mean, args.n, args.decimals, scale_min, scale_max) + result = { + 'test_name': 'GRIM Test (均值粒度一致性检验)', + 'status': 'completed', + 'result': res, + 'interpretation': ( + f"✅ 均值 {args.mean} 与样本量 {args.n} 一致" if res.get('consistent') + else f"⚠️ 均值 {args.mean} 与样本量 {args.n} 不一致！最近合法均值为 {res.get('nearest_valid_mean')}" + ) + } + else: + parser.print_help() + sys.exit(1) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/image_duplicate_test.py b/tools/gengskill/scripts/image_duplicate_test.py new file mode 100644 index 0000000..1a3e84f --- /dev/null +++ b/tools/gengskill/scripts/image_duplicate_test.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +""" +图像重复检测 (Image Duplication Detection) +============================================ +原理：检测论文图片中是否存在重复使用或篡改的图像。 +使用感知哈希(pHash)和结构相似性(SSIM)来识别： +1. 完全相同的图片出现在不同实验条件下 +2. 经过旋转、翻转、裁剪后重复使用的图片 +3. 调整亮度/对比度后复用的图片 + +这是学术造假中常见的手段，尤其在Western blot、 +显微镜图片、流式细胞术散点图等场景中。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import os +from pathlib import Path + +try: + from PIL import Image + import numpy as np + HAS_PILLOW = True +except ImportError: + HAS_PILLOW = False + +try: + from skimage.metrics import structural_similarity as ssim + HAS_SKIMAGE = True +except ImportError: + HAS_SKIMAGE = False + + +def average_hash(image, hash_size=16): + """计算平均感知哈希""" + img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS) + pixels = np.array(img) + mean = pixels.mean() + return (pixels > mean).flatten() + + +def difference_hash(image, hash_size=16): + """计算差异感知哈希""" + img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS) + pixels = np.array(img) + return (pixels[:, 1:] > pixels[:, :-1]).flatten() + + +def hamming_distance(hash1, hash2): + """计算汉明距离（归一化到0-1）""" + return np.sum(hash1 != hash2) / len(hash1) + + +def compute_ssim(img1, img2, target_size=(256, 256)): + """计算结构相似性指数""" + if not HAS_SKIMAGE: + return None + + # 统一尺寸 + img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS) + img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS) + + arr1 = np.array(img1_resized) + arr2 = np.array(img2_resized) + + score = ssim(arr1, arr2) + return float(score) + + +def check_rotations(img1, img2, threshold=0.85): + """检查图像经过旋转/翻转后是否匹配""" + transformations = [ + ('original', lambda x: x), + ('rotate_90', lambda x: x.rotate(90, expand=True)), + ('rotate_180', lambda x: x.rotate(180, expand=True)), + ('rotate_270', lambda x: x.rotate(270, expand=True)), + ('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)), + ('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)), + ] + + best_match = None + best_score = 0 + + hash1 = average_hash(img1) + + for name, transform in transformations: + transformed = transform(img2) + hash2 = average_hash(transformed) + similarity = 1 - hamming_distance(hash1, hash2) + + if similarity > best_score: + best_score = similarity + best_match = name + + return { + 'best_transformation': best_match, + 'best_similarity': round(best_score, 4), + 'is_match': best_score >= threshold + } + + +def find_duplicates(image_dir, threshold=0.85, extensions=None): + """ + 在目录中查找重复或相似的图片 + + Parameters + ---------- + image_dir : str + 图片目录路径 + threshold : float + 相似度阈值（0-1），超过此值判定为重复 + extensions : list + 支持的图片格式 + + Returns + ------- + dict : 检测结果 + """ + if not HAS_PILLOW: + return { + 'status': 'error', + 'message': '需要安装 Pillow: pip install Pillow' + } + + if extensions is None: + extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif'] + + # 收集所有图片文件 + image_files = [] + for ext in extensions: + image_files.extend(Path(image_dir).glob(f'*{ext}')) + image_files.extend(Path(image_dir).glob(f'*{ext.upper()}')) + + image_files = sorted(set(image_files)) + + if len(image_files) < 2: + return { + 'status': 'insufficient_data', + 'message': f'目录中仅找到 {len(image_files)} 张图片，需要至少2张进行比较', + 'n_images': len(image_files) + } + + # 计算所有图片的哈希 + hashes = {} + for img_path in image_files: + try: + img = Image.open(img_path) + hashes[str(img_path)] = { + 'avg_hash': average_hash(img), + 'diff_hash': difference_hash(img), + 'size': img.size, + 'image': img + } + except Exception as e: + continue + + # 两两比较 + duplicates = [] + paths = list(hashes.keys()) + + for i in range(len(paths)): + for j in range(i + 1, len(paths)): + path1, path2 = paths[i], paths[j] + h1, h2 = hashes[path1], hashes[path2] + + # 平均哈希相似度 + avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash']) + + # 差异哈希相似度 + diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash']) + + # 综合相似度 + combined_sim = max(avg_sim, diff_sim) + + if combined_sim >= threshold: + pair_result = { + 'file_1': os.path.basename(path1), + 'file_2': os.path.basename(path2), + 'avg_hash_similarity': round(float(avg_sim), 4), + 'diff_hash_similarity': round(float(diff_sim), 4), + 'combined_similarity': round(float(combined_sim), 4), + } + + # 检查旋转/翻转匹配 + rotation_check = check_rotations( + h1['image'], h2['image'], threshold + ) + pair_result['rotation_check'] = rotation_check + + # SSIM（如果可用） + if HAS_SKIMAGE: + ssim_score = compute_ssim(h1['image'], h2['image']) + pair_result['ssim'] = round(ssim_score, 4) + + duplicates.append(pair_result) + + # 关闭所有图片 + for h in hashes.values(): + h['image'].close() + + # 风险评分 + n_duplicates = len(duplicates) + n_images = len(image_files) + + if n_duplicates == 0: + risk_level = 'low' + risk_score = 0 + elif n_duplicates <= 1: + risk_level = 'medium' + risk_score = 40 + elif n_duplicates <= 3: + risk_level = 'medium-high' + risk_score = 60 + else: + risk_level = 'high' + risk_score = 80 + min(20, n_duplicates * 3) + + # 如果有完美匹配（相似度>0.98），直接拉高 + perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98] + if perfect_matches: + risk_score = max(risk_score, 90) + risk_level = 'high' + + result = { + 'test_name': 'Image Duplication Detection (图像重复检测)', + 'status': 'completed', + 'n_images_scanned': n_images, + 'n_duplicate_pairs': n_duplicates, + 'threshold': threshold, + 'duplicates': duplicates, + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_image(n_duplicates, n_images, duplicates) + } + + return result + + +def _interpret_image(n_duplicates, n_images, duplicates): + """生成可读的解释""" + if n_duplicates == 0: + return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。" + + perfect = [d for d in duplicates if d['combined_similarity'] > 0.98] + + if perfect: + return ( + f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片！" + f"这些图片可能是同一图片的重复使用，强烈建议核查是否为不同实验条件下的独立数据。" + ) + else: + return ( + f"⚡ 发现 {n_duplicates} 对高度相似的图片（共扫描 {n_images} 张）。" + f"可能存在图片复用或篡改，建议人工核查具体图片内容。" + ) + + +def main(): + parser = argparse.ArgumentParser( + description='图像重复检测 - 检测论文图片是否存在重复使用或篡改' + ) + parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径') + parser.add_argument('--threshold', '-t', type=float, default=0.85, + help='相似度阈值（0-1），默认0.85') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + if not os.path.isdir(args.input_dir): + print(f"错误：目录不存在: {args.input_dir}", file=sys.stderr) + sys.exit(1) + + result = find_duplicates(args.input_dir, args.threshold) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/image_duplicate_test_patched.py b/tools/gengskill/scripts/image_duplicate_test_patched.py new file mode 100644 index 0000000..97a26a0 --- /dev/null +++ b/tools/gengskill/scripts/image_duplicate_test_patched.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +""" +图像重复检测 (Image Duplication Detection) +============================================ +原理：检测论文图片中是否存在重复使用或篡改的图像。 +使用感知哈希(pHash)和结构相似性(SSIM)来识别： +1. 完全相同的图片出现在不同实验条件下 +2. 经过旋转、翻转、裁剪后重复使用的图片 +3. 调整亮度/对比度后复用的图片 + +这是学术造假中常见的手段，尤其在Western blot、 +显微镜图片、流式细胞术散点图等场景中。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import os +from pathlib import Path + +try: + from PIL import Image + import numpy as np + + +class NumpyEncoder(json.JSONEncoder): + def default(self, obj): + import numpy as np + if isinstance(obj, (np.integer,)): return int(obj) + if isinstance(obj, (np.floating,)): return float(obj) + if isinstance(obj, (np.bool_,)): return bool(obj) + if isinstance(obj, np.ndarray): return obj.tolist() + if obj is None: return None + return super().default(obj) + + HAS_PILLOW = True +except ImportError: + HAS_PILLOW = False + +try: + from skimage.metrics import structural_similarity as ssim + HAS_SKIMAGE = True +except ImportError: + HAS_SKIMAGE = False + + +def average_hash(image, hash_size=16): + """计算平均感知哈希""" + img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS) + pixels = np.array(img) + mean = pixels.mean() + return (pixels > mean).flatten() + + +def difference_hash(image, hash_size=16): + """计算差异感知哈希""" + img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS) + pixels = np.array(img) + return (pixels[:, 1:] > pixels[:, :-1]).flatten() + + +def hamming_distance(hash1, hash2): + """计算汉明距离（归一化到0-1）""" + return np.sum(hash1 != hash2) / len(hash1) + + +def compute_ssim(img1, img2, target_size=(256, 256)): + """计算结构相似性指数""" + if not HAS_SKIMAGE: + return None + + # 统一尺寸 + img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS) + img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS) + + arr1 = np.array(img1_resized) + arr2 = np.array(img2_resized) + + score = ssim(arr1, arr2) + return float(score) + + +def check_rotations(img1, img2, threshold=0.85): + """检查图像经过旋转/翻转后是否匹配""" + transformations = [ + ('original', lambda x: x), + ('rotate_90', lambda x: x.rotate(90, expand=True)), + ('rotate_180', lambda x: x.rotate(180, expand=True)), + ('rotate_270', lambda x: x.rotate(270, expand=True)), + ('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)), + ('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)), + ] + + best_match = None + best_score = 0 + + hash1 = average_hash(img1) + + for name, transform in transformations: + transformed = transform(img2) + hash2 = average_hash(transformed) + similarity = 1 - hamming_distance(hash1, hash2) + + if similarity > best_score: + best_score = similarity + best_match = name + + return { + 'best_transformation': best_match, + 'best_similarity': round(best_score, 4), + 'is_match': best_score >= threshold + } + + +def find_duplicates(image_dir, threshold=0.85, extensions=None): + """ + 在目录中查找重复或相似的图片 + + Parameters + ---------- + image_dir : str + 图片目录路径 + threshold : float + 相似度阈值（0-1），超过此值判定为重复 + extensions : list + 支持的图片格式 + + Returns + ------- + dict : 检测结果 + """ + if not HAS_PILLOW: + return { + 'status': 'error', + 'message': '需要安装 Pillow: pip install Pillow' + } + + if extensions is None: + extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif'] + + # 收集所有图片文件 + image_files = [] + for ext in extensions: + image_files.extend(Path(image_dir).glob(f'*{ext}')) + image_files.extend(Path(image_dir).glob(f'*{ext.upper()}')) + + image_files = sorted(set(image_files)) + + if len(image_files) < 2: + return { + 'status': 'insufficient_data', + 'message': f'目录中仅找到 {len(image_files)} 张图片，需要至少2张进行比较', + 'n_images': len(image_files) + } + + # 计算所有图片的哈希 + hashes = {} + for img_path in image_files: + try: + img = Image.open(img_path) + hashes[str(img_path)] = { + 'avg_hash': average_hash(img), + 'diff_hash': difference_hash(img), + 'size': img.size, + 'image': img + } + except Exception as e: + continue + + # 两两比较 + duplicates = [] + paths = list(hashes.keys()) + + for i in range(len(paths)): + for j in range(i + 1, len(paths)): + path1, path2 = paths[i], paths[j] + h1, h2 = hashes[path1], hashes[path2] + + # 平均哈希相似度 + avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash']) + + # 差异哈希相似度 + diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash']) + + # 综合相似度 + combined_sim = max(avg_sim, diff_sim) + + if combined_sim >= threshold: + pair_result = { + 'file_1': os.path.basename(path1), + 'file_2': os.path.basename(path2), + 'avg_hash_similarity': round(float(avg_sim), 4), + 'diff_hash_similarity': round(float(diff_sim), 4), + 'combined_similarity': round(float(combined_sim), 4), + } + + # 检查旋转/翻转匹配 + rotation_check = check_rotations( + h1['image'], h2['image'], threshold + ) + pair_result['rotation_check'] = rotation_check + + # SSIM（如果可用） + if HAS_SKIMAGE: + ssim_score = compute_ssim(h1['image'], h2['image']) + pair_result['ssim'] = round(ssim_score, 4) + + duplicates.append(pair_result) + + # 关闭所有图片 + for h in hashes.values(): + h['image'].close() + + # 风险评分 + n_duplicates = len(duplicates) + n_images = len(image_files) + + if n_duplicates == 0: + risk_level = 'low' + risk_score = 0 + elif n_duplicates <= 1: + risk_level = 'medium' + risk_score = 40 + elif n_duplicates <= 3: + risk_level = 'medium-high' + risk_score = 60 + else: + risk_level = 'high' + risk_score = 80 + min(20, n_duplicates * 3) + + # 如果有完美匹配（相似度>0.98），直接拉高 + perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98] + if perfect_matches: + risk_score = max(risk_score, 90) + risk_level = 'high' + + result = { + 'test_name': 'Image Duplication Detection (图像重复检测)', + 'status': 'completed', + 'n_images_scanned': n_images, + 'n_duplicate_pairs': n_duplicates, + 'threshold': threshold, + 'duplicates': duplicates, + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_image(n_duplicates, n_images, duplicates) + } + + return result + + +def _interpret_image(n_duplicates, n_images, duplicates): + """生成可读的解释""" + if n_duplicates == 0: + return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。" + + perfect = [d for d in duplicates if d['combined_similarity'] > 0.98] + + if perfect: + return ( + f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片！" + f"这些图片可能是同一图片的重复使用，强烈建议核查是否为不同实验条件下的独立数据。" + ) + else: + return ( + f"⚡ 发现 {n_duplicates} 对高度相似的图片（共扫描 {n_images} 张）。" + f"可能存在图片复用或篡改，建议人工核查具体图片内容。" + ) + + +def main(): + parser = argparse.ArgumentParser( + description='图像重复检测 - 检测论文图片是否存在重复使用或篡改' + ) + parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径') + parser.add_argument('--threshold', '-t', type=float, default=0.85, + help='相似度阈值（0-1），默认0.85') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + + args = parser.parse_args() + + if not os.path.isdir(args.input_dir): + print(f"错误：目录不存在: {args.input_dir}", file=sys.stderr) + sys.exit(1) + + result = find_duplicates(args.input_dir, args.threshold) + + output_json = json.dumps(result, ensure_ascii=False, indent=2, cls=NumpyEncoder) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/input_pipeline.py b/tools/gengskill/scripts/input_pipeline.py new file mode 100644 index 0000000..07687de --- /dev/null +++ b/tools/gengskill/scripts/input_pipeline.py @@ -0,0 +1,1713 @@ +#!/usr/bin/env python3 +""" +Unified Input Pipeline for the Geng Skill Project +=================================================== + +A self-contained module that ingests data from PDFs, Excel files, and CSVs, +returning a standardized dictionary suitable for downstream statistical +forensics / anomaly-detection modules. + +Supports three primary modes: + - **extract**: Parse tables and numeric data from the input file. + - **scale**: Automated "scan" mode — ingests a CSV/Excel, runs ALL detection + modules, and highlights the most suspicious columns/pairs without user guidance. + - **info**: Return metadata about the input file without full extraction. + +CLI Usage +--------- + python3 input_pipeline.py --input paper.pdf --mode extract + python3 input_pipeline.py --input data.xlsx --mode scale + python3 input_pipeline.py --input results.csv --mode info + +Author: Geng Skill / BioMaster +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import warnings +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +# --------------------------------------------------------------------------- +# Lazy imports with install hints +# --------------------------------------------------------------------------- + +def _import_pdfplumber(): + """Lazily import pdfplumber with install hint on failure.""" + try: + import pdfplumber + return pdfplumber + except ImportError: + print( + "[input_pipeline] pdfplumber not found. Install with:\n" + " python3 -m pip install pdfplumber", + file=sys.stderr, + ) + return None + + +def _import_fitz(): + """Lazily import PyMuPDF (fitz) with install hint on failure.""" + try: + import fitz + return fitz + except ImportError: + print( + "[input_pipeline] PyMuPDF not found. Install with:\n" + " python3 -m pip install PyMuPDF", + file=sys.stderr, + ) + return None + + +def _import_tabula(): + """Lazily import tabula-py with install hint on failure.""" + try: + import tabula + return tabula + except ImportError: + print( + "[input_pipeline] tabula-py not found. Install with:\n" + " python3 -m pip install tabula-py\n" + " (also requires Java runtime: apt-get install default-jre)", + file=sys.stderr, + ) + return None + + +def _import_pandas(): + """Lazily import pandas with install hint on failure.""" + try: + import pandas as pd + return pd + except ImportError: + print( + "[input_pipeline] pandas not found. Install with:\n" + " python3 -m pip install pandas", + file=sys.stderr, + ) + sys.exit(1) + + +def _import_openpyxl(): + """Lazily import openpyxl with install hint on failure.""" + try: + import openpyxl + return openpyxl + except ImportError: + print( + "[input_pipeline] openpyxl not found. Install with:\n" + " python3 -m pip install openpyxl", + file=sys.stderr, + ) + return None + + +def _import_xlrd(): + """Lazily import xlrd with install hint on failure.""" + try: + import xlrd + return xlrd + except ImportError: + print( + "[input_pipeline] xlrd not found. Install with:\n" + " python3 -m pip install xlrd", + file=sys.stderr, + ) + return None + + +def _import_numpy(): + """Lazily import numpy with install hint on failure.""" + try: + import numpy as np + return np + except ImportError: + print( + "[input_pipeline] numpy not found. Install with:\n" + " python3 -m pip install numpy", + file=sys.stderr, + ) + sys.exit(1) + + +def _import_scipy(): + """Lazily import scipy with install hint on failure.""" + try: + import scipy + return scipy + except ImportError: + print( + "[input_pipeline] scipy not found. Install with:\n" + " python3 -m pip install scipy", + file=sys.stderr, + ) + return None + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +SUPPORTED_EXTENSIONS = {".pdf", ".xlsx", ".xls", ".csv", ".tsv"} + +# --------------------------------------------------------------------------- +# Utility helpers +# --------------------------------------------------------------------------- + + +def detect_file_type(filepath: str) -> str: + """ + Detect the logical file type from the file extension. + + Parameters + ---------- + filepath : str + Path to the input file. + + Returns + ------- + str + One of "pdf", "excel", "csv". + + Raises + ------ + ValueError + If the file extension is not supported. + """ + ext = Path(filepath).suffix.lower() + if ext == ".pdf": + return "pdf" + elif ext in (".xlsx", ".xls"): + return "excel" + elif ext in (".csv", ".tsv"): + return "csv" + else: + raise ValueError( + f"Unsupported file extension '{ext}'. " + f"Supported: {sorted(SUPPORTED_EXTENSIONS)}" + ) + + +def _dataframe_to_serializable(df) -> List[Dict[str, Any]]: + """ + Convert a pandas DataFrame to a list of row-dicts that is JSON-serializable. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to convert. + + Returns + ------- + list of dict + Each dict represents one row with column names as keys. + """ + pd = _import_pandas() + # Replace NaN/Inf with None for JSON compatibility + import numpy as np + df_clean = df.replace([np.inf, -np.inf], np.nan).where(df.notnull(), None) + records = df_clean.to_dict(orient="records") + # Ensure numpy types are converted to native Python types + clean_records = [] + for row in records: + clean_row = {} + for k, v in row.items(): + if hasattr(v, "item"): + clean_row[k] = v.item() + else: + clean_row[k] = v + clean_records.append(clean_row) + return clean_records + + +def _identify_numeric_columns(df) -> Dict[str, Dict[str, Any]]: + """ + Auto-detect numeric columns and compute summary statistics. + + Parameters + ---------- + df : pandas.DataFrame + Input dataframe. + + Returns + ------- + dict + Mapping of column name -> {dtype, count, mean, std, min, max, n_missing}. + """ + np = _import_numpy() + numeric_info: Dict[str, Dict[str, Any]] = {} + for col in df.columns: + # Attempt numeric coercion for mixed-type columns + series = df[col] + if not _is_numeric_dtype(series): + coerced = _try_coerce_numeric(series) + if coerced is None: + continue + series = coerced + + n_missing = int(series.isna().sum()) + valid = series.dropna() + if len(valid) == 0: + continue + + numeric_info[str(col)] = { + "dtype": str(series.dtype), + "count": int(len(valid)), + "mean": float(valid.mean()), + "std": float(valid.std()) if len(valid) > 1 else 0.0, + "min": float(valid.min()), + "max": float(valid.max()), + "n_missing": n_missing, + } + return numeric_info + + +def _is_numeric_dtype(series) -> bool: + """Check if a pandas Series has a numeric dtype.""" + pd = _import_pandas() + return pd.api.types.is_numeric_dtype(series) + + +def _try_coerce_numeric(series): + """ + Attempt to coerce a series to numeric, returning None if <50% convertible. + + Parameters + ---------- + series : pandas.Series + The series to attempt coercion on. + + Returns + ------- + pandas.Series or None + Coerced numeric series, or None if not predominantly numeric. + """ + pd = _import_pandas() + coerced = pd.to_numeric(series, errors="coerce") + valid_ratio = coerced.notna().sum() / max(len(series), 1) + if valid_ratio >= 0.5: + return coerced + return None + + +# --------------------------------------------------------------------------- +# PDF extraction +# --------------------------------------------------------------------------- + + +def extract_from_pdf( + filepath: str, + *, + pages: Optional[List[int]] = None, + password: Optional[str] = None, +) -> Dict[str, Any]: + """ + Extract tables and text data from an academic paper PDF. + + Uses pdfplumber as the primary extractor with fallback to PyMuPDF for + text extraction and tabula-py for table extraction. + + Parameters + ---------- + filepath : str + Path to the PDF file. + pages : list of int, optional + Specific 0-indexed page numbers to process. If None, all pages are + processed. + password : str, optional + Password for encrypted/protected PDFs. + + Returns + ------- + dict + Standardized result dictionary with keys: + - source_type : "pdf" + - tables : list of list-of-dicts (each table as records) + - numeric_columns : dict mapping column names to stats (aggregated) + - metadata : dict with page_count, extractor_used, warnings, etc. + + Raises + ------ + FileNotFoundError + If the PDF file does not exist. + RuntimeError + If no PDF extraction library is available. + + Notes + ----- + The function attempts extraction in the following order: + 1. pdfplumber (best for structured tables in academic papers) + 2. tabula-py (Java-based, good for complex table layouts) + 3. PyMuPDF/fitz (fallback for text-only extraction) + + Examples + -------- + >>> result = extract_from_pdf("paper.pdf") + >>> print(result["source_type"]) + 'pdf' + >>> print(len(result["tables"])) + 3 + """ + filepath = str(filepath) + if not os.path.isfile(filepath): + raise FileNotFoundError(f"PDF file not found: {filepath}") + + pd = _import_pandas() + np = _import_numpy() + + tables: List[List[Dict[str, Any]]] = [] + metadata: Dict[str, Any] = { + "filepath": filepath, + "filename": os.path.basename(filepath), + "file_size_bytes": os.path.getsize(filepath), + "extractor_used": None, + "page_count": None, + "warnings": [], + } + all_numeric_columns: Dict[str, Dict[str, Any]] = {} + + # --- Attempt 1: pdfplumber --- + pdfplumber = _import_pdfplumber() + if pdfplumber is not None: + try: + open_kwargs: Dict[str, Any] = {} + if password: + open_kwargs["password"] = password + + with pdfplumber.open(filepath, **open_kwargs) as pdf: + metadata["page_count"] = len(pdf.pages) + metadata["extractor_used"] = "pdfplumber" + + pages_to_process = pages if pages else range(len(pdf.pages)) + for page_idx in pages_to_process: + if page_idx >= len(pdf.pages): + metadata["warnings"].append( + f"Page {page_idx} out of range (total: {len(pdf.pages)})" + ) + continue + page = pdf.pages[page_idx] + page_tables = page.extract_tables() + if not page_tables: + continue + for raw_table in page_tables: + if not raw_table or len(raw_table) < 2: + continue + # First row as header + header = [ + str(c).strip() if c else f"col_{i}" + for i, c in enumerate(raw_table[0]) + ] + # Deduplicate headers + header = _deduplicate_headers(header) + rows = raw_table[1:] + df = pd.DataFrame(rows, columns=header) + # Attempt numeric coercion on all columns + for col in df.columns: + df[col] = pd.to_numeric(df[col], errors="ignore") + tables.append(_dataframe_to_serializable(df)) + col_info = _identify_numeric_columns(df) + all_numeric_columns.update(col_info) + + if tables: + return { + "source_type": "pdf", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + except Exception as e: + metadata["warnings"].append(f"pdfplumber failed: {str(e)}") + + # --- Attempt 2: tabula-py --- + tabula = _import_tabula() + if tabula is not None: + try: + tabula_pages = "all" + if pages: + # tabula uses 1-indexed pages + tabula_pages = [p + 1 for p in pages] + + kwargs: Dict[str, Any] = {"pages": tabula_pages, "multiple_tables": True} + if password: + kwargs["password"] = password + + dfs = tabula.read_pdf(filepath, **kwargs) + metadata["extractor_used"] = "tabula-py" + + for df in dfs: + if df.empty: + continue + tables.append(_dataframe_to_serializable(df)) + col_info = _identify_numeric_columns(df) + all_numeric_columns.update(col_info) + + if tables: + return { + "source_type": "pdf", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + except Exception as e: + metadata["warnings"].append(f"tabula-py failed: {str(e)}") + + # --- Attempt 3: PyMuPDF (text-only fallback) --- + fitz = _import_fitz() + if fitz is not None: + try: + doc = fitz.open(filepath) + if password and doc.is_encrypted: + if not doc.authenticate(password): + metadata["warnings"].append("PyMuPDF: password authentication failed") + doc.close() + raise RuntimeError("Cannot decrypt PDF with provided password") + + metadata["page_count"] = len(doc) + metadata["extractor_used"] = "PyMuPDF (text-only)" + + full_text_lines: List[str] = [] + pages_to_process = pages if pages else range(len(doc)) + for page_idx in pages_to_process: + if page_idx >= len(doc): + continue + page = doc[page_idx] + text = page.get_text() + full_text_lines.append(text) + + doc.close() + + # Attempt to parse tab/comma separated data from text + extracted_df = _parse_text_tables("\n".join(full_text_lines)) + if extracted_df is not None and not extracted_df.empty: + tables.append(_dataframe_to_serializable(extracted_df)) + all_numeric_columns = _identify_numeric_columns(extracted_df) + + metadata["text_length_chars"] = sum(len(t) for t in full_text_lines) + + return { + "source_type": "pdf", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + except Exception as e: + metadata["warnings"].append(f"PyMuPDF failed: {str(e)}") + + # --- All extractors failed --- + if not any([pdfplumber, tabula, fitz]): + raise RuntimeError( + "No PDF extraction library available. Install at least one:\n" + " python3 -m pip install pdfplumber\n" + " python3 -m pip install tabula-py\n" + " python3 -m pip install PyMuPDF" + ) + + return { + "source_type": "pdf", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + + +def _deduplicate_headers(headers: List[str]) -> List[str]: + """ + Ensure all column headers are unique by appending suffixes. + + Parameters + ---------- + headers : list of str + Raw header names (may contain duplicates). + + Returns + ------- + list of str + Deduplicated header names. + """ + seen: Dict[str, int] = {} + result: List[str] = [] + for h in headers: + if h in seen: + seen[h] += 1 + result.append(f"{h}_{seen[h]}") + else: + seen[h] = 0 + result.append(h) + return result + + +def _parse_text_tables(text: str): + """ + Heuristically parse tabular data from raw text (TSV or CSV-like). + + Parameters + ---------- + text : str + Raw text extracted from a PDF page. + + Returns + ------- + pandas.DataFrame or None + Parsed dataframe if a table-like structure is detected, else None. + """ + pd = _import_pandas() + import io + + lines = [l for l in text.strip().split("\n") if l.strip()] + if len(lines) < 3: + return None + + # Detect delimiter (tab > comma > multiple-spaces) + for delimiter in ["\t", ",", " "]: + counts = [l.count(delimiter) for l in lines[:10]] + if all(c > 0 for c in counts) and max(counts) - min(counts) <= 2: + try: + df = pd.read_csv( + io.StringIO("\n".join(lines)), + sep=delimiter if delimiter != " " else r"\s{2,}", + engine="python" if delimiter == " " else "c", + ) + if df.shape[1] >= 2 and df.shape[0] >= 2: + return df + except Exception: + continue + return None + + +# --------------------------------------------------------------------------- +# Excel extraction +# --------------------------------------------------------------------------- + + +def extract_from_excel( + filepath: str, + *, + sheet_names: Optional[List[str]] = None, + password: Optional[str] = None, +) -> Dict[str, Any]: + """ + Read tables from .xlsx/.xls files with auto-detection of numeric columns. + + Parameters + ---------- + filepath : str + Path to the Excel file. + sheet_names : list of str, optional + Specific sheet names to read. If None, all sheets are read. + password : str, optional + Password for protected workbooks (openpyxl only, limited support). + + Returns + ------- + dict + Standardized result dictionary with keys: + - source_type : "excel" + - tables : list of list-of-dicts (one per non-empty sheet) + - numeric_columns : dict mapping column names to stats (aggregated) + - metadata : dict with sheet_names_found, engine_used, warnings, etc. + + Raises + ------ + FileNotFoundError + If the Excel file does not exist. + RuntimeError + If no Excel reading library is available. + + Notes + ----- + Uses openpyxl for .xlsx and xlrd for .xls files. Falls back between + engines as needed. + + Examples + -------- + >>> result = extract_from_excel("data.xlsx") + >>> print(result["source_type"]) + 'excel' + >>> print(list(result["numeric_columns"].keys())) + ['age', 'score', 'p_value'] + """ + filepath = str(filepath) + if not os.path.isfile(filepath): + raise FileNotFoundError(f"Excel file not found: {filepath}") + + pd = _import_pandas() + np = _import_numpy() + + ext = Path(filepath).suffix.lower() + tables: List[List[Dict[str, Any]]] = [] + all_numeric_columns: Dict[str, Dict[str, Any]] = {} + metadata: Dict[str, Any] = { + "filepath": filepath, + "filename": os.path.basename(filepath), + "file_size_bytes": os.path.getsize(filepath), + "engine_used": None, + "sheet_names_found": [], + "sheets_processed": [], + "warnings": [], + } + + # Determine engine + engine = None + if ext == ".xlsx": + openpyxl = _import_openpyxl() + if openpyxl is not None: + engine = "openpyxl" + else: + metadata["warnings"].append("openpyxl not available for .xlsx") + elif ext == ".xls": + xlrd = _import_xlrd() + if xlrd is not None: + engine = "xlrd" + else: + metadata["warnings"].append("xlrd not available for .xls") + + if engine is None: + # Try pandas default + try: + _ = pd.ExcelFile(filepath) + engine = "auto" + except Exception as e: + raise RuntimeError( + f"No suitable Excel engine available for '{ext}'. " + "Install with:\n" + " python3 -m pip install openpyxl # for .xlsx\n" + " python3 -m pip install xlrd # for .xls" + ) from e + + metadata["engine_used"] = engine + + # Read Excel file + try: + read_kwargs: Dict[str, Any] = {"sheet_name": None} # Read all sheets + if engine != "auto": + read_kwargs["engine"] = engine + + # Handle password-protected xlsx (limited support) + if password and ext == ".xlsx": + try: + import msoffcrypto + import io + + decrypted = io.BytesIO() + with open(filepath, "rb") as f: + office_file = msoffcrypto.OfficeFile(f) + office_file.load_key(password=password) + office_file.decrypt(decrypted) + decrypted.seek(0) + sheets_dict = pd.read_excel(decrypted, **read_kwargs) + except ImportError: + metadata["warnings"].append( + "msoffcrypto not available for password-protected files. " + "Install with: python3 -m pip install msoffcrypto-tool" + ) + # Try without password + sheets_dict = pd.read_excel(filepath, **read_kwargs) + except Exception as e: + metadata["warnings"].append(f"Password decryption failed: {e}") + sheets_dict = pd.read_excel(filepath, **read_kwargs) + else: + sheets_dict = pd.read_excel(filepath, **read_kwargs) + + except Exception as e: + metadata["warnings"].append(f"Excel read failed: {str(e)}") + return { + "source_type": "excel", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + + metadata["sheet_names_found"] = list(sheets_dict.keys()) + + # Filter to requested sheets + sheets_to_process = sheet_names if sheet_names else list(sheets_dict.keys()) + + for sheet_name in sheets_to_process: + if sheet_name not in sheets_dict: + metadata["warnings"].append(f"Sheet '{sheet_name}' not found") + continue + + df = sheets_dict[sheet_name] + if df.empty: + metadata["warnings"].append(f"Sheet '{sheet_name}' is empty") + continue + + # Drop fully-empty rows and columns + df = df.dropna(how="all").dropna(axis=1, how="all") + if df.empty: + continue + + metadata["sheets_processed"].append(str(sheet_name)) + tables.append(_dataframe_to_serializable(df)) + + col_info = _identify_numeric_columns(df) + # Prefix with sheet name if multiple sheets + if len(sheets_to_process) > 1: + col_info = {f"{sheet_name}::{k}": v for k, v in col_info.items()} + all_numeric_columns.update(col_info) + + return { + "source_type": "excel", + "tables": tables, + "numeric_columns": all_numeric_columns, + "metadata": metadata, + } + + +# --------------------------------------------------------------------------- +# CSV extraction +# --------------------------------------------------------------------------- + + +def extract_from_csv(filepath: str) -> Dict[str, Any]: + """ + Read a CSV/TSV file and identify numeric columns. + + Parameters + ---------- + filepath : str + Path to the CSV or TSV file. + + Returns + ------- + dict + Standardized result dictionary with keys: + - source_type : "csv" + - tables : list containing one list-of-dicts + - numeric_columns : dict mapping column names to stats + - metadata : dict with delimiter_detected, row_count, col_count, etc. + + Raises + ------ + FileNotFoundError + If the CSV file does not exist. + + Examples + -------- + >>> result = extract_from_csv("results.csv") + >>> print(result["metadata"]["row_count"]) + 150 + """ + filepath = str(filepath) + if not os.path.isfile(filepath): + raise FileNotFoundError(f"CSV file not found: {filepath}") + + pd = _import_pandas() + + metadata: Dict[str, Any] = { + "filepath": filepath, + "filename": os.path.basename(filepath), + "file_size_bytes": os.path.getsize(filepath), + "delimiter_detected": None, + "row_count": 0, + "col_count": 0, + "warnings": [], + } + + ext = Path(filepath).suffix.lower() + sep = "\t" if ext == ".tsv" else "," + + # Auto-detect delimiter from first few lines + try: + with open(filepath, "r", encoding="utf-8", errors="replace") as f: + sample = f.read(4096) + if sep == "," and sample.count("\t") > sample.count(","): + sep = "\t" + elif sep == "," and sample.count(";") > sample.count(","): + sep = ";" + except Exception: + pass + + metadata["delimiter_detected"] = repr(sep) + + try: + df = pd.read_csv(filepath, sep=sep, engine="python", on_bad_lines="skip") + except Exception as e: + metadata["warnings"].append(f"CSV read failed: {str(e)}") + return { + "source_type": "csv", + "tables": [], + "numeric_columns": {}, + "metadata": metadata, + } + + # Drop fully-empty rows/cols + df = df.dropna(how="all").dropna(axis=1, how="all") + + metadata["row_count"] = len(df) + metadata["col_count"] = len(df.columns) + + tables = [_dataframe_to_serializable(df)] if not df.empty else [] + numeric_columns = _identify_numeric_columns(df) + + return { + "source_type": "csv", + "tables": tables, + "numeric_columns": numeric_columns, + "metadata": metadata, + } + + +# --------------------------------------------------------------------------- +# Scale mode — automated multi-module anomaly scan +# --------------------------------------------------------------------------- + + +def run_scale_mode(filepath: str) -> Dict[str, Any]: + """ + Automated "scale" scan: ingest a CSV/Excel, run ALL detection modules, + and highlight the most suspicious columns/pairs without user guidance. + + Detection modules applied: + 1. **Digit frequency (Benford's Law)** — first-digit distribution test + 2. **Terminal digit bias** — last-digit uniformity test + 3. **GRIM test** — granularity-consistent mean test for integer-sourced means + 4. **Duplicate pattern detection** — unusual repetition in numeric values + 5. **Correlation anomalies** — suspiciously perfect or impossible correlations + 6. **Distribution shape** — normality tests and outlier fraction + + Parameters + ---------- + filepath : str + Path to a CSV or Excel file. + + Returns + ------- + dict + Standardized result with additional key: + - scale_results : dict mapping module_name -> { + flagged_columns: list, + flagged_pairs: list, + scores: dict, + details: str + } + - suspicion_ranking : list of (column_or_pair, aggregate_score) sorted desc + + Notes + ----- + The scale mode is designed to be run without any prior knowledge of the + data. It is a screening tool; flagged columns should be investigated + further before drawing conclusions. + + Examples + -------- + >>> result = run_scale_mode("experiment_data.csv") + >>> for item in result["suspicion_ranking"][:5]: + ... print(item) + ('treatment_mean', 0.87) + ('control_mean', 0.72) + """ + # First, extract the data + file_type = detect_file_type(filepath) + if file_type == "csv": + extraction = extract_from_csv(filepath) + elif file_type == "excel": + extraction = extract_from_excel(filepath) + elif file_type == "pdf": + extraction = extract_from_pdf(filepath) + else: + raise ValueError(f"Scale mode does not support file type: {file_type}") + + pd = _import_pandas() + np = _import_numpy() + + # Reconstruct dataframes from extracted tables + all_dfs: List = [] + for table_records in extraction.get("tables", []): + if table_records: + df = pd.DataFrame(table_records) + all_dfs.append(df) + + if not all_dfs: + extraction["scale_results"] = {} + extraction["suspicion_ranking"] = [] + extraction["metadata"]["warnings"] = extraction.get("metadata", {}).get( + "warnings", [] + ) + ["No tables found for scale analysis"] + return extraction + + # Merge all tables for comprehensive analysis + # (if multiple tables, concatenate columns with unique naming) + combined_df = all_dfs[0] + for i, df in enumerate(all_dfs[1:], start=1): + df_renamed = df.add_prefix(f"table{i}_") + combined_df = pd.concat([combined_df, df_renamed], axis=1) + + # Run detection modules + scale_results: Dict[str, Dict[str, Any]] = {} + suspicion_scores: Dict[str, float] = {} + + # Module 1: Benford's Law (first-digit distribution) + benford_result = _module_benford(combined_df) + scale_results["benford_first_digit"] = benford_result + for col, score in benford_result.get("scores", {}).items(): + suspicion_scores[col] = suspicion_scores.get(col, 0.0) + score + + # Module 2: Terminal digit bias + terminal_result = _module_terminal_digits(combined_df) + scale_results["terminal_digit_bias"] = terminal_result + for col, score in terminal_result.get("scores", {}).items(): + suspicion_scores[col] = suspicion_scores.get(col, 0.0) + score + + # Module 3: GRIM test + grim_result = _module_grim(combined_df) + scale_results["grim_test"] = grim_result + for col, score in grim_result.get("scores", {}).items(): + suspicion_scores[col] = suspicion_scores.get(col, 0.0) + score + + # Module 4: Duplicate pattern detection + dup_result = _module_duplicate_patterns(combined_df) + scale_results["duplicate_patterns"] = dup_result + for col, score in dup_result.get("scores", {}).items(): + suspicion_scores[col] = suspicion_scores.get(col, 0.0) + score + + # Module 5: Correlation anomalies + corr_result = _module_correlation_anomalies(combined_df) + scale_results["correlation_anomalies"] = corr_result + for pair, score in corr_result.get("scores", {}).items(): + suspicion_scores[pair] = suspicion_scores.get(pair, 0.0) + score + + # Module 6: Distribution shape + dist_result = _module_distribution_shape(combined_df) + scale_results["distribution_shape"] = dist_result + for col, score in dist_result.get("scores", {}).items(): + suspicion_scores[col] = suspicion_scores.get(col, 0.0) + score + + # Normalize and rank + max_score = max(suspicion_scores.values()) if suspicion_scores else 1.0 + if max_score > 0: + normalized = {k: round(v / max_score, 3) for k, v in suspicion_scores.items()} + else: + normalized = suspicion_scores + + ranking = sorted(normalized.items(), key=lambda x: x[1], reverse=True) + + extraction["scale_results"] = scale_results + extraction["suspicion_ranking"] = ranking + return extraction + + +# --------------------------------------------------------------------------- +# Detection modules for scale mode +# --------------------------------------------------------------------------- + + +def _module_benford(df) -> Dict[str, Any]: + """ + Test first-digit distribution against Benford's Law. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_columns, scores, and details. + """ + np = _import_numpy() + scipy = _import_scipy() + + benford_expected = np.array([ + np.log10(1 + 1.0 / d) for d in range(1, 10) + ]) + + scores: Dict[str, float] = {} + flagged: List[str] = [] + details_parts: List[str] = [] + + for col in df.select_dtypes(include=["number"]).columns: + series = df[col].dropna() + if len(series) < 30: + continue + + # Extract first significant digit + abs_vals = series[series != 0].abs() + if len(abs_vals) < 30: + continue + + first_digits = abs_vals.apply( + lambda x: int(str(f"{x:.10e}")[0]) if x > 0 else 0 + ) + first_digits = first_digits[first_digits.between(1, 9)] + + if len(first_digits) < 20: + continue + + # Compute observed distribution + observed = np.zeros(9) + for d in range(1, 10): + observed[d - 1] = (first_digits == d).sum() + + total = observed.sum() + if total == 0: + continue + observed_freq = observed / total + + # Chi-squared test + if scipy is not None: + from scipy.stats import chisquare + expected_counts = benford_expected * total + # Avoid zero expected counts + mask = expected_counts > 0 + if mask.sum() >= 5: + stat, p_value = chisquare(observed[mask], expected_counts[mask]) + # Score: higher means more suspicious (low p-value) + score = max(0.0, 1.0 - p_value) + scores[str(col)] = round(score, 4) + if p_value < 0.01: + flagged.append(str(col)) + details_parts.append( + f" {col}: chi2={stat:.2f}, p={p_value:.4e} (FLAGGED)" + ) + else: + # Fallback: MAD from Benford + mad = np.mean(np.abs(observed_freq - benford_expected)) + score = min(1.0, mad * 10) # Scale heuristically + scores[str(col)] = round(score, 4) + if mad > 0.05: + flagged.append(str(col)) + + return { + "flagged_columns": flagged, + "flagged_pairs": [], + "scores": scores, + "details": ( + "Benford's Law first-digit test.\n" + "\n".join(details_parts) + if details_parts + else "Benford's Law first-digit test. No significant deviations." + ), + } + + +def _module_terminal_digits(df) -> Dict[str, Any]: + """ + Test for non-uniform terminal (last) digit distribution. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_columns, scores, and details. + """ + np = _import_numpy() + scipy = _import_scipy() + + scores: Dict[str, float] = {} + flagged: List[str] = [] + details_parts: List[str] = [] + + for col in df.select_dtypes(include=["number"]).columns: + series = df[col].dropna() + if len(series) < 20: + continue + + # Get terminal digits (last digit before decimal or last significant) + terminal_digits = [] + for val in series: + s = str(val).rstrip("0").rstrip(".") + if s and s[-1].isdigit(): + terminal_digits.append(int(s[-1])) + + if len(terminal_digits) < 20: + continue + + td_array = np.array(terminal_digits) + # Expected: uniform distribution over 0-9 + observed = np.array([(td_array == d).sum() for d in range(10)]) + total = observed.sum() + expected = np.full(10, total / 10.0) + + if scipy is not None: + from scipy.stats import chisquare + stat, p_value = chisquare(observed, expected) + score = max(0.0, 1.0 - p_value) + scores[str(col)] = round(score, 4) + if p_value < 0.01: + flagged.append(str(col)) + details_parts.append( + f" {col}: chi2={stat:.2f}, p={p_value:.4e} (non-uniform terminals)" + ) + else: + max_dev = np.max(np.abs(observed / total - 0.1)) + score = min(1.0, max_dev * 10) + scores[str(col)] = round(score, 4) + if max_dev > 0.1: + flagged.append(str(col)) + + return { + "flagged_columns": flagged, + "flagged_pairs": [], + "scores": scores, + "details": ( + "Terminal digit uniformity test.\n" + "\n".join(details_parts) + if details_parts + else "Terminal digit uniformity test. No significant bias detected." + ), + } + + +def _module_grim(df) -> Dict[str, Any]: + """ + GRIM (Granularity-Related Inconsistency of Means) test. + + For columns that appear to be sample means derived from integer data, + checks whether the reported mean is mathematically consistent with + the implied sample size. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_columns, scores, and details. + """ + np = _import_numpy() + + scores: Dict[str, float] = {} + flagged: List[str] = [] + details_parts: List[str] = [] + + # GRIM applies to means of integer-scale items + # Heuristic: columns with values like X.XX where denominator might be N + for col in df.select_dtypes(include=["number"]).columns: + series = df[col].dropna() + if len(series) < 5: + continue + + # Check if values look like means (between 1-7, typical Likert range) + if series.min() < 0 or series.max() > 100: + continue + + # Count GRIM-inconsistent values assuming various sample sizes + inconsistent_count = 0 + total_tested = 0 + for val in series: + # Determine decimal places + val_str = f"{val:.10f}".rstrip("0") + if "." in val_str: + decimals = len(val_str.split(".")[1]) + else: + decimals = 0 + + if decimals < 1 or decimals > 4: + continue + + # Test against common sample sizes (10-200) + is_consistent = False + for n in range(5, 201): + # For a mean of integers with sample size n, + # the mean must be a multiple of 1/n + granularity = 1.0 / n + remainder = abs(val % granularity) + if remainder < 1e-8 or abs(remainder - granularity) < 1e-8: + is_consistent = True + break + + total_tested += 1 + if not is_consistent: + inconsistent_count += 1 + + if total_tested >= 5: + inconsistency_rate = inconsistent_count / total_tested + score = min(1.0, inconsistency_rate * 2) # Scale up + scores[str(col)] = round(score, 4) + if inconsistency_rate > 0.5: + flagged.append(str(col)) + details_parts.append( + f" {col}: {inconsistent_count}/{total_tested} " + f"GRIM-inconsistent ({inconsistency_rate:.0%})" + ) + + return { + "flagged_columns": flagged, + "flagged_pairs": [], + "scores": scores, + "details": ( + "GRIM test for mean consistency.\n" + "\n".join(details_parts) + if details_parts + else "GRIM test. No inconsistencies detected." + ), + } + + +def _module_duplicate_patterns(df) -> Dict[str, Any]: + """ + Detect unusual repetition/duplication patterns in numeric columns. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_columns, scores, and details. + """ + np = _import_numpy() + + scores: Dict[str, float] = {} + flagged: List[str] = [] + details_parts: List[str] = [] + + for col in df.select_dtypes(include=["number"]).columns: + series = df[col].dropna() + if len(series) < 10: + continue + + n = len(series) + n_unique = series.nunique() + dup_ratio = 1.0 - (n_unique / n) + + # Also check for suspicious patterns (e.g., many values at round numbers) + round_count = sum(1 for v in series if v == round(v, 0)) + round_ratio = round_count / n + + # Consecutive duplicate runs + values = series.values + max_run = 1 + current_run = 1 + for i in range(1, len(values)): + if values[i] == values[i - 1]: + current_run += 1 + max_run = max(max_run, current_run) + else: + current_run = 1 + + # Score based on multiple signals + score = 0.0 + # High duplication + if dup_ratio > 0.5 and n_unique > 1: + score += dup_ratio * 0.5 + # Long consecutive runs (suspicious for continuous data) + expected_max_run = np.log2(n) if n > 1 else 1 + if max_run > expected_max_run * 2: + score += 0.3 + # Too many round numbers in presumably continuous data + if round_ratio > 0.8 and series.std() > 0.1: + score += 0.2 + + score = min(1.0, score) + if score > 0.1: + scores[str(col)] = round(score, 4) + if score > 0.5: + flagged.append(str(col)) + details_parts.append( + f" {col}: dup_ratio={dup_ratio:.2f}, max_run={max_run}, " + f"round_ratio={round_ratio:.2f}" + ) + + return { + "flagged_columns": flagged, + "flagged_pairs": [], + "scores": scores, + "details": ( + "Duplicate/repetition pattern analysis.\n" + "\n".join(details_parts) + if details_parts + else "Duplicate pattern analysis. No unusual patterns." + ), + } + + +def _module_correlation_anomalies(df) -> Dict[str, Any]: + """ + Detect suspiciously perfect or theoretically impossible correlations. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_pairs, scores, and details. + """ + np = _import_numpy() + + scores: Dict[str, float] = {} + flagged_pairs: List[str] = [] + details_parts: List[str] = [] + + numeric_cols = df.select_dtypes(include=["number"]).columns.tolist() + + if len(numeric_cols) < 2 or len(numeric_cols) > 100: + # Too few or too many columns + return { + "flagged_columns": [], + "flagged_pairs": flagged_pairs, + "scores": scores, + "details": "Correlation analysis: insufficient or too many columns.", + } + + # Compute correlation matrix + corr_matrix = df[numeric_cols].corr() + + for i in range(len(numeric_cols)): + for j in range(i + 1, len(numeric_cols)): + col_a = numeric_cols[i] + col_b = numeric_cols[j] + r = corr_matrix.iloc[i, j] + + if np.isnan(r): + continue + + pair_name = f"{col_a} <-> {col_b}" + abs_r = abs(r) + + # Flag suspiciously perfect correlations (|r| > 0.999) + if abs_r > 0.999: + score = 1.0 + scores[pair_name] = score + flagged_pairs.append(pair_name) + details_parts.append( + f" {pair_name}: r={r:.6f} (suspiciously perfect)" + ) + elif abs_r > 0.99: + score = 0.5 + scores[pair_name] = score + flagged_pairs.append(pair_name) + details_parts.append( + f" {pair_name}: r={r:.4f} (very high correlation)" + ) + + return { + "flagged_columns": [], + "flagged_pairs": flagged_pairs, + "scores": scores, + "details": ( + "Correlation anomaly detection.\n" + "\n".join(details_parts) + if details_parts + else "Correlation analysis. No anomalous pairs detected." + ), + } + + +def _module_distribution_shape(df) -> Dict[str, Any]: + """ + Test distribution normality and detect unusual outlier fractions. + + Parameters + ---------- + df : pandas.DataFrame + The dataframe to analyze. + + Returns + ------- + dict + Module result with flagged_columns, scores, and details. + """ + np = _import_numpy() + scipy = _import_scipy() + + scores: Dict[str, float] = {} + flagged: List[str] = [] + details_parts: List[str] = [] + + for col in df.select_dtypes(include=["number"]).columns: + series = df[col].dropna() + if len(series) < 20: + continue + + values = series.values + mean = np.mean(values) + std = np.std(values, ddof=1) + + if std == 0: + continue + + # Outlier fraction (beyond 3 sigma) + z_scores = np.abs((values - mean) / std) + outlier_frac = np.mean(z_scores > 3) + + # Expected ~0.3% for normal distribution + # Suspiciously low outlier rate might indicate trimming + score = 0.0 + + if scipy is not None: + from scipy.stats import shapiro, kurtosis, skew + + # Shapiro-Wilk test (on subsample if too large) + test_sample = values[:5000] if len(values) > 5000 else values + if len(test_sample) >= 8: + try: + stat, p_value = shapiro(test_sample) + # Very low p-value isn't inherently suspicious + # but combined with other signals matters + if p_value > 0.99: + # TOO normal — might be fabricated + score += 0.3 + details_parts.append( + f" {col}: Shapiro p={p_value:.4f} " + "(suspiciously normal)" + ) + except Exception: + pass + + # Kurtosis check + try: + kurt = float(kurtosis(values)) + if abs(kurt) > 10: + score += 0.2 + except Exception: + pass + + # Outlier fraction anomaly + if len(values) > 50: + if outlier_frac == 0 and len(values) > 200: + # Zero outliers in large sample — suspicious + score += 0.2 + elif outlier_frac > 0.05: + # Too many outliers + score += 0.2 + + score = min(1.0, score) + if score > 0.1: + scores[str(col)] = round(score, 4) + if score > 0.4: + flagged.append(str(col)) + + return { + "flagged_columns": flagged, + "flagged_pairs": [], + "scores": scores, + "details": ( + "Distribution shape and outlier analysis.\n" + "\n".join(details_parts) + if details_parts + else "Distribution analysis. No anomalies detected." + ), + } + + +# --------------------------------------------------------------------------- +# Unified pipeline entry point +# --------------------------------------------------------------------------- + + +def run_pipeline( + filepath: str, + *, + mode: str = "extract", + pages: Optional[List[int]] = None, + sheet_names: Optional[List[str]] = None, + password: Optional[str] = None, +) -> Dict[str, Any]: + """ + Unified entry point for the input pipeline. + + Determines the file type, applies the appropriate extraction method, + and optionally runs the full-scale anomaly detection suite. + + Parameters + ---------- + filepath : str + Path to the input file (PDF, Excel, or CSV). + mode : str, default "extract" + Processing mode: + - "extract" : Parse and return tables with numeric column detection. + - "scale" : Full automated anomaly scan (CSV/Excel only). + - "info" : Return metadata only (lightweight). + pages : list of int, optional + For PDFs: specific 0-indexed page numbers to process. + sheet_names : list of str, optional + For Excel: specific sheet names to read. + password : str, optional + Password for encrypted files. + + Returns + ------- + dict + Standardized result dictionary: + - source_type : "pdf" | "excel" | "csv" + - tables : list of list-of-dicts (each table as records) + - numeric_columns : dict mapping column names -> summary stats + - metadata : dict with file info, warnings, processing details + - scale_results : (only in "scale" mode) per-module detection results + - suspicion_ranking : (only in "scale" mode) ranked suspicious items + + Raises + ------ + FileNotFoundError + If the input file does not exist. + ValueError + If the file type is unsupported or mode is invalid. + + Examples + -------- + >>> result = run_pipeline("paper.pdf", mode="extract") + >>> print(result["source_type"]) + 'pdf' + + >>> result = run_pipeline("data.csv", mode="scale") + >>> print(result["suspicion_ranking"][:3]) + [('col_a', 0.95), ('col_b', 0.82), ('col_c <-> col_d', 0.71)] + """ + # Validate inputs + filepath = str(filepath) + if not os.path.isfile(filepath): + raise FileNotFoundError(f"Input file not found: {filepath}") + + valid_modes = ("extract", "scale", "info") + if mode not in valid_modes: + raise ValueError(f"Invalid mode '{mode}'. Must be one of {valid_modes}") + + file_type = detect_file_type(filepath) + + # Info mode: lightweight metadata only + if mode == "info": + metadata = { + "filepath": filepath, + "filename": os.path.basename(filepath), + "file_size_bytes": os.path.getsize(filepath), + "detected_type": file_type, + } + return { + "source_type": file_type, + "tables": [], + "numeric_columns": {}, + "metadata": metadata, + } + + # Scale mode + if mode == "scale": + return run_scale_mode(filepath) + + # Extract mode + if file_type == "pdf": + return extract_from_pdf(filepath, pages=pages, password=password) + elif file_type == "excel": + return extract_from_excel( + filepath, sheet_names=sheet_names, password=password + ) + elif file_type == "csv": + return extract_from_csv(filepath) + else: + raise ValueError(f"Unhandled file type: {file_type}") + + +# --------------------------------------------------------------------------- +# CLI interface +# --------------------------------------------------------------------------- + + +def _build_parser() -> argparse.ArgumentParser: + """ + Build the argument parser for CLI usage. + + Returns + ------- + argparse.ArgumentParser + Configured argument parser. + """ + parser = argparse.ArgumentParser( + prog="input_pipeline", + description=( + "Unified Input Pipeline for the Geng Skill Project.\n" + "Extracts tables and numeric data from PDFs, Excel files, and CSVs." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=( + "Examples:\n" + " python3 input_pipeline.py --input paper.pdf --mode extract\n" + " python3 input_pipeline.py --input data.xlsx --mode scale\n" + " python3 input_pipeline.py --input results.csv --mode info\n" + " python3 input_pipeline.py --input paper.pdf --pages 0 1 2\n" + " python3 input_pipeline.py --input data.xlsx --sheets Sheet1 Sheet2\n" + ), + ) + parser.add_argument( + "--input", "-i", + required=True, + help="Path to the input file (PDF, Excel, or CSV).", + ) + parser.add_argument( + "--mode", "-m", + choices=["extract", "scale", "info"], + default="extract", + help="Processing mode (default: extract).", + ) + parser.add_argument( + "--pages", + nargs="*", + type=int, + default=None, + help="For PDFs: 0-indexed page numbers to process (default: all).", + ) + parser.add_argument( + "--sheets", + nargs="*", + default=None, + help="For Excel: sheet names to read (default: all).", + ) + parser.add_argument( + "--password", + default=None, + help="Password for encrypted/protected files.", + ) + parser.add_argument( + "--output", "-o", + default=None, + help="Output JSON file path (default: stdout).", + ) + parser.add_argument( + "--pretty", + action="store_true", + help="Pretty-print JSON output.", + ) + return parser + + +def main(): + """ + CLI entry point. + + Parses command-line arguments, runs the pipeline, and outputs + results as JSON to stdout or a specified file. + """ + parser = _build_parser() + args = parser.parse_args() + + try: + result = run_pipeline( + args.input, + mode=args.mode, + pages=args.pages, + sheet_names=args.sheets, + password=args.password, + ) + except (FileNotFoundError, ValueError, RuntimeError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"UNEXPECTED ERROR: {type(e).__name__}: {e}", file=sys.stderr) + sys.exit(2) + + # Serialize output + indent = 2 if args.pretty else None + json_output = json.dumps(result, indent=indent, ensure_ascii=False, default=str) + + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json_output, encoding="utf-8") + print(f"Results written to: {args.output}", file=sys.stderr) + else: + print(json_output) + + +if __name__ == "__main__": + main() diff --git a/tools/gengskill/scripts/last_digit_test.py b/tools/gengskill/scripts/last_digit_test.py new file mode 100644 index 0000000..418a6ee --- /dev/null +++ b/tools/gengskill/scripts/last_digit_test.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +""" +末位数字检测 (Last Digit Test) +============================== +原理：自然实验数据的末位数字（0-9）应近似均匀分布。 +如果数据是人为编造的，末位数字往往会集中在某些特定值上。 + +使用卡方检验评估偏离均匀分布的程度。 + +致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。 +""" + +import argparse +import sys +import json +import numpy as np +from collections import Counter +from scipy import stats + + +def extract_last_digit(value): + """提取数值的末位有效数字""" + s = str(value).strip() + # 去除负号 + s = s.lstrip('-') + # 去除科学计数法 + if 'e' in s.lower(): + try: + value = float(s) + s = f"{value:.10f}".rstrip('0') + except ValueError: + return None + # 找到末位有效数字 + s = s.rstrip('0').rstrip('.') + if not s: + return 0 + for c in reversed(s): + if c.isdigit(): + return int(c) + return None + + +def extract_last_digit_with_decimals(value, use_decimal_last=True): + """ + 提取数值的末位数字 + use_decimal_last=True: 取小数点后最后一位非零数字 + use_decimal_last=False: 取整数部分的末位数字 + """ + s = str(value).strip() + s = s.lstrip('-') + + if '.' in s and use_decimal_last: + decimal_part = s.split('.')[1] + # 取小数部分的最后一位数字 + if decimal_part: + return int(decimal_part[-1]) + + # 取整数部分的末位 + integer_part = s.split('.')[0] if '.' in s else s + if integer_part: + return int(integer_part[-1]) + return None + + +def last_digit_test(values, method='all_digits'): + """ + 执行末位数字检测 + + Parameters + ---------- + values : list of float/str + 待检测的数值列表 + method : str + 'all_digits' - 检测最后一位有效数字 + 'decimal_last' - 检测小数末位 + + Returns + ------- + dict : 检测结果 + """ + # 提取末位数字 + last_digits = [] + for v in values: + try: + if method == 'decimal_last': + d = extract_last_digit_with_decimals(v, use_decimal_last=True) + else: + d = extract_last_digit(v) + if d is not None: + last_digits.append(d) + except (ValueError, TypeError): + continue + + if len(last_digits) < 10: + return { + 'status': 'insufficient_data', + 'message': f'数据量不足（仅{len(last_digits)}个有效值），需要至少10个数据点', + 'n_valid': len(last_digits) + } + + # 统计各数字出现频次 + digit_counts = Counter(last_digits) + observed = np.array([digit_counts.get(i, 0) for i in range(10)]) + expected = np.full(10, len(last_digits) / 10.0) + + # 卡方检验 + chi2, p_value = stats.chisquare(observed, expected) + + # 计算集中度指标 + max_digit = int(np.argmax(observed)) + max_freq = observed[max_digit] / len(last_digits) + + # 均匀性评分 (0=完全均匀, 1=完全集中) + uniformity_deviation = np.sqrt(np.sum((observed / len(last_digits) - 0.1) ** 2) / 10) / 0.3 + uniformity_deviation = min(uniformity_deviation, 1.0) + + # 风险评分 + if p_value < 0.001: + risk_level = 'high' + risk_score = min(80 + (1 - p_value) * 20, 100) + elif p_value < 0.01: + risk_level = 'medium-high' + risk_score = 60 + (0.01 - p_value) / 0.009 * 20 + elif p_value < 0.05: + risk_level = 'medium' + risk_score = 40 + (0.05 - p_value) / 0.04 * 20 + else: + risk_level = 'low' + risk_score = max(0, 40 * (1 - p_value)) + + result = { + 'test_name': 'Last Digit Test (末位数字检测)', + 'status': 'completed', + 'n_values': len(last_digits), + 'method': method, + 'digit_distribution': {str(i): int(observed[i]) for i in range(10)}, + 'chi_square': round(float(chi2), 4), + 'p_value': float(p_value), + 'degrees_of_freedom': 9, + 'most_frequent_digit': max_digit, + 'most_frequent_proportion': round(float(max_freq), 4), + 'uniformity_deviation': round(float(uniformity_deviation), 4), + 'risk_level': risk_level, + 'risk_score': round(float(risk_score), 1), + 'interpretation': _interpret_result(p_value, max_digit, max_freq, len(last_digits)) + } + + return result + + +def _interpret_result(p_value, max_digit, max_freq, n): + """生成可读的解释""" + if p_value < 0.001: + return ( + f"⚠️ 末位数字分布严重偏离均匀分布（p < 0.001）。" + f"数字 {max_digit} 出现频率为 {max_freq:.1%}（期望 10%），" + f"这种偏离在自然实验数据中极为罕见，强烈建议进一步核查原始数据。" + ) + elif p_value < 0.01: + return ( + f"⚠️ 末位数字分布显著偏离均匀分布（p < 0.01）。" + f"数字 {max_digit} 出现频率为 {max_freq:.1%}，建议关注并进行人工复核。" + ) + elif p_value < 0.05: + return ( + f"⚡ 末位数字分布存在一定偏离（p < 0.05）。" + f"可能是正常波动，也可能提示数据存在问题，建议结合其他检测结果综合判断。" + ) + else: + return ( + f"✅ 末位数字分布与均匀分布无显著差异（p = {p_value:.4f}），" + f"未发现明显异常。" + ) + + +def load_data(input_file, column=None, delimiter=','): + """从CSV文件加载数据""" + import csv + + values = [] + with open(input_file, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f, delimiter=delimiter) + if column and column in reader.fieldnames: + for row in reader: + try: + val = row[column].strip() + if val: + float(val) # 验证是数值 + values.append(val) + except (ValueError, KeyError): + continue + else: + # 如果没有指定列，尝试读取第一个数值列 + for row in reader: + for key, val in row.items(): + try: + val = val.strip() + if val: + float(val) + values.append(val) + except (ValueError, AttributeError): + continue + return values + + +def main(): + parser = argparse.ArgumentParser( + description='末位数字检测 - 检测数据末位数字是否偏离均匀分布' + ) + parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径') + parser.add_argument('--column', '-c', help='要检测的列名') + parser.add_argument('--method', '-m', default='all_digits', + choices=['all_digits', 'decimal_last'], + help='检测方法：all_digits=末位有效数字, decimal_last=小数末位') + parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符') + parser.add_argument('--output', '-o', help='输出JSON文件路径') + parser.add_argument('--values', nargs='+', type=str, + help='直接传入数值列表（不使用文件输入）') + + args = parser.parse_args() + + if args.values: + values = args.values + else: + values = load_data(args.input, args.column, args.delimiter) + + if not values: + print("错误：未能加载有效数据", file=sys.stderr) + sys.exit(1) + + result = last_digit_test(values, method=args.method) + + output_json = json.dumps(result, ensure_ascii=False, indent=2) + + if args.output: + with open(args.output, 'w', encoding='utf-8') as f: + f.write(output_json) + print(f"结果已保存至: {args.output}") + else: + print(output_json) + + +if __name__ == '__main__': + main() diff --git a/tools/gengskill/scripts/report_generator.py b/tools/gengskill/scripts/report_generator.py new file mode 100644 index 0000000..6edf65e --- /dev/null +++ b/tools/gengskill/scripts/report_generator.py @@ -0,0 +1,1775 @@ +#!/usr/bin/env python3 +""" +report_generator.py — Comprehensive HTML + Markdown Report Generator +for the Geng Skill Academic Fraud Detection Project. + +Generates professional, self-contained analysis reports from assessment JSON +produced by the detection pipeline. Supports two output formats: + - HTML (self-contained with embedded CSS/figures, printable) + - Markdown (for GitHub/documentation, figures as relative paths) + +Usage: + python3 report_generator.py --input assessment.json --figures figures/ --output report/ + +The input assessment.json is expected to have this structure: +{ + "metadata": { "source_file", "timestamp", "tool_version", "columns", "rows", ... }, + "overall_risk": { "score": 0-100, "level": "LOW|MEDIUM|HIGH|CRITICAL" }, + "data_overview": { "columns": [...], "preview": [...], "statistics": {...} }, + "modules": [ + { + "name": "...", + "description": "...", + "method": "...", + "results": { ... }, + "figures": ["fig1.png", ...], + "risk_level": "LOW|MEDIUM|HIGH|CRITICAL", + "p_value": ..., + "test_statistic": ..., + "evidence_summary": "..." + }, ... + ], + "suspicious_points": [ + { "row": ..., "column": "...", "value": ..., "reason": "...", "module": "..." }, ... + ], + "confidence": { "overall": ..., "intervals": {...}, "limitations": [...] }, + "recommendations": [ { "priority": 1, "action": "...", "rationale": "..." }, ... ] +} + +Author: BioMaster / Geng Skill Project +License: MIT +""" + +import argparse +import base64 +import json +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +TOOL_VERSION = "1.0.0" +TOOL_NAME = "Geng Skill Academic Data Integrity Analyzer" + +RISK_COLORS = { + "LOW": "#28a745", # green + "MEDIUM": "#ffc107", # yellow/amber + "HIGH": "#fd7e14", # orange + "CRITICAL": "#dc3545", # red +} + +RISK_EMOJI = { + "LOW": "🟢", + "MEDIUM": "🟡", + "HIGH": "🟠", + "CRITICAL": "🔴", +} + +RISK_LABELS = { + "LOW": "Low Risk", + "MEDIUM": "Medium Risk", + "HIGH": "High Risk", + "CRITICAL": "Critical Risk", +} + +METHODOLOGY_REFERENCES = { + "benford": { + "name": "Benford's Law (First-Digit Test)", + "description": ( + "Tests whether the distribution of leading digits in the dataset " + "conforms to the logarithmic distribution predicted by Benford's Law. " + "Fabricated data often shows uniform or biased digit distributions." + ), + "references": [ + "Benford, F. (1938). The law of anomalous numbers. Proc. Amer. Phil. Soc., 78(4), 551-572.", + "Nigrini, M.J. (2012). Benford's Law. Wiley.", + ], + }, + "terminal_digit": { + "name": "Terminal Digit Analysis", + "description": ( + "Examines the distribution of last digits in numeric data. " + "Authentic measurements typically show uniform terminal digit distribution, " + "while fabricated data often exhibits preference for certain digits (e.g., 0, 5)." + ), + "references": [ + "Mosimann, J.E., Wiseman, C.V., & Edelman, R.E. (1995). Data fabrication. " + "Chance, 8(2), 7-12.", + ], + }, + "grim": { + "name": "GRIM Test (Granularity-Related Inconsistency of Means)", + "description": ( + "Verifies whether reported means are mathematically possible given the " + "reported sample size and measurement granularity. Impossible means indicate " + "either reporting errors or data fabrication." + ), + "references": [ + "Brown, N.J.L., & Heathers, J.A.J. (2017). The GRIM test. " + "Social Psychological and Personality Science, 8(4), 363-369.", + ], + }, + "sprite": { + "name": "SPRITE (Sample Parameter Reconstruction via Iterative TEchniques)", + "description": ( + "Reconstructs possible raw data distributions consistent with reported " + "summary statistics. Flags cases where no valid distribution exists." + ), + "references": [ + "Heathers, J.A.J., & Brown, N.J.L. (2019). SPRITE. PeerJ Preprints.", + ], + }, + "distribution": { + "name": "Distribution Shape Analysis", + "description": ( + "Tests data against expected statistical distributions using " + "Kolmogorov-Smirnov, Shapiro-Wilk, or Anderson-Darling tests. " + "Fabricated data often shows abnormal distributional properties." + ), + "references": [ + "Simonsohn, U. (2013). Just post it. Psychological Science, 24(10), 1875-1888.", + ], + }, + "duplicates": { + "name": "Duplicate/Near-Duplicate Detection", + "description": ( + "Identifies exact and near-duplicate values, rows, or patterns that occur " + "more frequently than expected by chance." + ), + "references": [ + "Bik, E.M., Casadevall, A., & Fang, F.C. (2016). The prevalence of " + "inappropriate image duplication. mBio, 7(3), e00809-16.", + ], + }, + "variance": { + "name": "Variance Analysis (ANOVA / Levene's Test)", + "description": ( + "Examines whether variance patterns are consistent with genuine experimental " + "data. Fabricated data often shows abnormally low or uniform variance." + ), + "references": [ + "Carlisle, J.B. (2017). Data fabrication and other reasons for " + "non-random sampling. Anaesthesia, 72(8), 944-952.", + ], + }, + "correlation": { + "name": "Correlation Structure Analysis", + "description": ( + "Checks whether inter-variable correlations are biologically/experimentally " + "plausible. Fabricated data may show correlations that are too perfect or " + "internally inconsistent." + ), + "references": [ + "Simonsohn, U. (2014). Posterior-Hacking. Available at SSRN.", + ], + }, +} + +DISCLAIMER_EN = """ +**DISCLAIMER**: This report is generated by an automated statistical analysis tool and is intended +for preliminary screening purposes ONLY. The results do NOT constitute proof of misconduct. +Statistical anomalies can arise from legitimate methodological choices, measurement artifacts, +or natural data properties. Any findings should be interpreted by qualified experts and investigated +through proper institutional channels before any conclusions about research integrity are drawn. +This tool should NEVER be used as the sole basis for accusations of fraud or misconduct. +""".strip() + +DISCLAIMER_ZH = """ +**免责声明**：本报告由自动化统计分析工具生成，仅用于初步筛查目的。分析结果不构成学术不端的证据。 +统计异常可能源于合理的方法学选择、测量误差或数据的自然属性。任何发现都应由具备资质的专家解读， +并通过正规的机构渠道进行调查，方可得出关于研究诚信的结论。本工具绝不应作为指控欺诈或不端行为的 +唯一依据。 +""".strip() + +# --------------------------------------------------------------------------- +# HTML Template & CSS +# --------------------------------------------------------------------------- + +HTML_CSS = """ +:root { + --primary: #2c3e50; + --secondary: #34495e; + --accent: #3498db; + --bg: #ffffff; + --bg-alt: #f8f9fa; + --border: #dee2e6; + --text: #212529; + --text-muted: #6c757d; + --success: #28a745; + --warning: #ffc107; + --danger: #dc3545; + --orange: #fd7e14; +} + +* { box-sizing: border-box; } + +body { + font-family: 'Segoe UI', -apple-system, BlinkMacSystemFont, 'Helvetica Neue', Arial, sans-serif; + line-height: 1.6; + color: var(--text); + background: var(--bg); + margin: 0; + padding: 0; +} + +.container { + max-width: 1100px; + margin: 0 auto; + padding: 2rem; +} + +/* Header */ +.report-header { + border-bottom: 3px solid var(--primary); + padding-bottom: 1.5rem; + margin-bottom: 2rem; +} + +.report-header h1 { + font-size: 1.8rem; + color: var(--primary); + margin: 0 0 0.5rem 0; +} + +.report-header .subtitle { + font-size: 1rem; + color: var(--text-muted); + margin: 0; +} + +/* Risk Badge */ +.risk-badge { + display: inline-block; + padding: 0.4rem 1rem; + border-radius: 4px; + font-weight: 700; + font-size: 0.9rem; + color: #fff; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.risk-badge.low { background: var(--success); } +.risk-badge.medium { background: var(--warning); color: #212529; } +.risk-badge.high { background: var(--orange); } +.risk-badge.critical { background: var(--danger); } + +/* Score Meter */ +.score-meter { + width: 100%; + height: 24px; + background: #e9ecef; + border-radius: 12px; + overflow: hidden; + margin: 0.5rem 0; +} + +.score-meter .fill { + height: 100%; + border-radius: 12px; + transition: width 0.5s; + display: flex; + align-items: center; + justify-content: center; + font-size: 0.75rem; + font-weight: 700; + color: #fff; +} + +/* Sections */ +.section { + margin-bottom: 2.5rem; +} + +.section h2 { + font-size: 1.4rem; + color: var(--primary); + border-bottom: 2px solid var(--accent); + padding-bottom: 0.5rem; + margin-bottom: 1rem; +} + +.section h3 { + font-size: 1.1rem; + color: var(--secondary); + margin-top: 1.5rem; + margin-bottom: 0.5rem; +} + +/* Cards */ +.card { + background: var(--bg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 1.2rem; + margin-bottom: 1rem; + box-shadow: 0 1px 3px rgba(0,0,0,0.04); +} + +.card.risk-low { border-left: 4px solid var(--success); } +.card.risk-medium { border-left: 4px solid var(--warning); } +.card.risk-high { border-left: 4px solid var(--orange); } +.card.risk-critical { border-left: 4px solid var(--danger); } + +.card-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.8rem; +} + +.card-header h3 { + margin: 0; + font-size: 1.05rem; +} + +/* Tables */ +table { + width: 100%; + border-collapse: collapse; + margin: 1rem 0; + font-size: 0.9rem; +} + +th, td { + padding: 0.6rem 0.8rem; + text-align: left; + border-bottom: 1px solid var(--border); +} + +th { + background: var(--primary); + color: #fff; + font-weight: 600; + position: sticky; + top: 0; +} + +tr:nth-child(even) { + background: var(--bg-alt); +} + +tr:hover { + background: #e8f4fd; +} + +/* Figures */ +.figure-container { + text-align: center; + margin: 1rem 0; +} + +.figure-container img { + max-width: 100%; + height: auto; + border: 1px solid var(--border); + border-radius: 4px; +} + +.figure-caption { + font-size: 0.85rem; + color: var(--text-muted); + margin-top: 0.4rem; + font-style: italic; +} + +/* Stats Grid */ +.stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 1rem; + margin: 1rem 0; +} + +.stat-box { + background: var(--bg-alt); + border: 1px solid var(--border); + border-radius: 6px; + padding: 1rem; + text-align: center; +} + +.stat-box .stat-value { + font-size: 1.6rem; + font-weight: 700; + color: var(--primary); +} + +.stat-box .stat-label { + font-size: 0.8rem; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +/* Evidence */ +.evidence-list { + list-style: none; + padding: 0; +} + +.evidence-list li { + padding: 0.4rem 0; + padding-left: 1.5rem; + position: relative; +} + +.evidence-list li::before { + content: '•'; + position: absolute; + left: 0.5rem; + color: var(--accent); + font-weight: 700; +} + +/* Suspicious Points Table */ +.suspicious-row { + background: #fff3cd !important; +} + +/* Disclaimer */ +.disclaimer { + background: #f8d7da; + border: 1px solid #f5c6cb; + border-radius: 6px; + padding: 1.2rem; + margin: 2rem 0; + font-size: 0.9rem; +} + +.disclaimer h3 { + color: var(--danger); + margin-top: 0; +} + +/* Footer */ +.report-footer { + border-top: 2px solid var(--border); + padding-top: 1rem; + margin-top: 3rem; + font-size: 0.8rem; + color: var(--text-muted); + display: flex; + justify-content: space-between; + flex-wrap: wrap; +} + +/* Print Styles */ +@media print { + body { font-size: 10pt; } + .container { max-width: 100%; padding: 0; } + .card { break-inside: avoid; } + .section { break-inside: avoid; } + table { font-size: 8pt; } + .report-header { border-bottom-width: 2px; } +} + +/* Recommendations */ +.recommendation { + display: flex; + align-items: flex-start; + gap: 0.8rem; + padding: 0.8rem; + margin-bottom: 0.5rem; + background: var(--bg-alt); + border-radius: 6px; +} + +.recommendation .priority-num { + background: var(--accent); + color: #fff; + width: 28px; + height: 28px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + font-size: 0.85rem; + flex-shrink: 0; +} + +.recommendation .rec-content { + flex: 1; +} + +.recommendation .rec-action { + font-weight: 600; + margin-bottom: 0.2rem; +} + +.recommendation .rec-rationale { + font-size: 0.85rem; + color: var(--text-muted); +} + +/* Methodology */ +.method-entry { + margin-bottom: 1.2rem; + padding-left: 1rem; + border-left: 3px solid var(--accent); +} + +.method-entry .method-name { + font-weight: 700; + margin-bottom: 0.3rem; +} + +.method-entry .method-desc { + font-size: 0.9rem; + margin-bottom: 0.3rem; +} + +.method-entry .method-ref { + font-size: 0.8rem; + color: var(--text-muted); + font-style: italic; +} +""" + +# --------------------------------------------------------------------------- +# Helper Functions +# --------------------------------------------------------------------------- + + +def load_assessment(path: str) -> Dict[str, Any]: + """Load and validate the assessment JSON file. + + Args: + path: Path to the assessment JSON file. + + Returns: + Parsed assessment dictionary. + + Raises: + FileNotFoundError: If the assessment file doesn't exist. + json.JSONDecodeError: If the file is not valid JSON. + ValueError: If required fields are missing. + """ + filepath = Path(path) + if not filepath.exists(): + raise FileNotFoundError(f"Assessment file not found: {path}") + + with open(filepath, "r", encoding="utf-8") as f: + data = json.load(f) + + # Validate required top-level keys + required = ["metadata", "overall_risk", "modules"] + missing = [k for k in required if k not in data] + if missing: + raise ValueError(f"Assessment JSON missing required keys: {missing}") + + return data + + +def encode_figure_base64(figure_path: str, figures_dir: str) -> Optional[str]: + """Encode a figure file as base64 data URI for HTML embedding. + + Args: + figure_path: Filename or relative path of the figure. + figures_dir: Directory containing figures. + + Returns: + Base64 data URI string, or None if file not found. + """ + full_path = Path(figures_dir) / figure_path + if not full_path.exists(): + return None + + suffix = full_path.suffix.lower() + mime_map = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".svg": "image/svg+xml", + ".gif": "image/gif", + } + mime_type = mime_map.get(suffix, "image/png") + + with open(full_path, "rb") as f: + encoded = base64.b64encode(f.read()).decode("ascii") + + return f"data:{mime_type};base64,{encoded}" + + +def risk_level_to_css_class(level: str) -> str: + """Convert risk level string to CSS class name. + + Args: + level: Risk level (LOW, MEDIUM, HIGH, CRITICAL). + + Returns: + CSS class string. + """ + return level.lower() + + +def format_p_value(p: Optional[float]) -> str: + """Format a p-value for display with appropriate precision. + + Args: + p: The p-value to format, or None. + + Returns: + Formatted string representation. + """ + if p is None: + return "N/A" + if p < 0.001: + return f"< 0.001 (p = {p:.2e})" + elif p < 0.01: + return f"{p:.4f}" + elif p < 0.05: + return f"{p:.3f}" + else: + return f"{p:.3f}" + + +def score_to_color(score: float) -> str: + """Map a 0-100 risk score to a gradient color. + + Args: + score: Risk score (0-100). + + Returns: + CSS color string. + """ + if score <= 25: + return RISK_COLORS["LOW"] + elif score <= 50: + return RISK_COLORS["MEDIUM"] + elif score <= 75: + return RISK_COLORS["HIGH"] + else: + return RISK_COLORS["CRITICAL"] + + +def score_to_level(score: float) -> str: + """Map a 0-100 risk score to a risk level string. + + Args: + score: Risk score (0-100). + + Returns: + Risk level string. + """ + if score <= 25: + return "LOW" + elif score <= 50: + return "MEDIUM" + elif score <= 75: + return "HIGH" + else: + return "CRITICAL" + + +# --------------------------------------------------------------------------- +# HTML Report Generator +# --------------------------------------------------------------------------- + + +class HTMLReportGenerator: + """Generates a self-contained HTML report from assessment data. + + The report includes embedded CSS, base64-encoded figures, and is + designed to be printable without external dependencies. + + Attributes: + assessment: The assessment data dictionary. + figures_dir: Path to the directory containing figure files. + """ + + def __init__(self, assessment: Dict[str, Any], figures_dir: str): + """Initialize the HTML report generator. + + Args: + assessment: Parsed assessment dictionary. + figures_dir: Path to directory containing figure image files. + """ + self.assessment = assessment + self.figures_dir = figures_dir + + def generate(self) -> str: + """Generate the complete HTML report. + + Returns: + Complete HTML document as a string. + """ + parts = [ + self._html_head(), + '', + '
    ', + self._header(), + self._executive_summary(), + self._data_overview(), + self._module_results(), + self._suspicious_points(), + self._confidence_limitations(), + self._methodology(), + self._recommendations(), + self._disclaimer(), + self._footer(), + '
    ', + '', + '', + ] + return "\n".join(parts) + + def _html_head(self) -> str: + """Generate HTML head with embedded CSS.""" + title = f"{TOOL_NAME} — Analysis Report" + return f""" + + + + + {title} + +""" + + def _header(self) -> str: + """Generate report header section.""" + meta = self.assessment.get("metadata", {}) + source = meta.get("source_file", "Unknown") + timestamp = meta.get("timestamp", datetime.now(timezone.utc).isoformat()) + + return f""" +
    +

    📊 {TOOL_NAME}

    +

    Analysis Report — {source}

    +

    Generated: {timestamp}

    +
    """ + + def _executive_summary(self) -> str: + """Generate executive summary section with overall risk score.""" + risk = self.assessment.get("overall_risk", {}) + score = risk.get("score", 0) + level = risk.get("level", score_to_level(score)) + color = score_to_color(score) + emoji = RISK_EMOJI.get(level, "⚪") + label = RISK_LABELS.get(level, "Unknown") + + # Key findings + modules = self.assessment.get("modules", []) + flagged = [m for m in modules if m.get("risk_level", "LOW") in ("HIGH", "CRITICAL")] + suspicious_count = len(self.assessment.get("suspicious_points", [])) + + findings_html = "" + if flagged: + findings_html = "
      " + for m in flagged: + me = RISK_EMOJI.get(m.get("risk_level", "LOW"), "⚪") + findings_html += f'
    • {me} {m.get("name", "Unknown")}: {m.get("evidence_summary", "Anomaly detected")}
      • ' + findings_html += "
    " + else: + findings_html = "

    No high-risk anomalies detected across all modules.

    " + + # Conclusion + if level == "CRITICAL": + conclusion = ( + "Multiple strong statistical indicators suggest significant anomalies in this dataset. " + "The patterns observed are highly unlikely to arise from legitimate experimental data. " + "Further expert review is strongly recommended." + ) + elif level == "HIGH": + conclusion = ( + "Several statistical indicators show notable anomalies that warrant further investigation. " + "While not conclusive evidence of data integrity issues, the patterns deserve scrutiny." + ) + elif level == "MEDIUM": + conclusion = ( + "Some mild statistical anomalies were detected. These may reflect legitimate methodological " + "choices or minor reporting inconsistencies. Routine verification is advisable." + ) + else: + conclusion = ( + "The dataset shows no significant statistical anomalies across the tests performed. " + "The data patterns appear consistent with legitimate experimental measurements." + ) + + return f""" +
    +

    Executive Summary

    +
    +
    +
    {score}/100
    +
    Overall Risk Score
    +
    +
    +
    {emoji} {label}
    +
    Risk Level
    +
    +
    +
    {len(modules)}
    +
    Tests Performed
    +
    +
    +
    {suspicious_count}
    +
    Suspicious Data Points
    +
    +
    +
    +
    {score}%
    +
    +

    Key Findings

    + {findings_html} +

    Conclusion

    +

    {conclusion}

    +
    """ + + def _data_overview(self) -> str: + """Generate data overview section with column stats and preview.""" + overview = self.assessment.get("data_overview", {}) + meta = self.assessment.get("metadata", {}) + + # File info + source = meta.get("source_file", "Unknown") + rows = meta.get("rows", "N/A") + cols = meta.get("columns_count", len(meta.get("columns", []))) + file_size = meta.get("file_size", "N/A") + + info_html = f""" +
    +
    +
    {source}
    +
    Source File
    +
    +
    +
    {rows}
    +
    Rows
    +
    +
    +
    {cols}
    +
    Columns
    +
    +
    +
    {file_size}
    +
    File Size
    +
    +
    """ + + # Column statistics table + statistics = overview.get("statistics", {}) + stats_table = "" + if statistics: + stats_table = """ +

    Column Statistics

    + + """ + for col_name, stats in statistics.items(): + dtype = stats.get("dtype", "—") + non_null = stats.get("non_null", "—") + mean = stats.get("mean", "—") + std = stats.get("std", "—") + min_val = stats.get("min", "—") + max_val = stats.get("max", "—") + # Format numeric values + if isinstance(mean, float): + mean = f"{mean:.4g}" + if isinstance(std, float): + std = f"{std:.4g}" + if isinstance(min_val, float): + min_val = f"{min_val:.4g}" + if isinstance(max_val, float): + max_val = f"{max_val:.4g}" + stats_table += f"\n " + stats_table += "\n
    ColumnTypeNon-NullMeanStdMinMax
    {col_name}{dtype}{non_null}{mean}{std}{min_val}{max_val}
    " + + # Preview table + preview = overview.get("preview", []) + preview_html = "" + if preview: + columns = overview.get("columns", list(preview[0].keys()) if preview else []) + preview_html = "\n

    Data Preview (first rows)

    \n \n " + for col in columns: + preview_html += f"" + preview_html += "" + for row in preview[:10]: + preview_html += "\n " + for col in columns: + val = row.get(col, "—") + preview_html += f"" + preview_html += "" + preview_html += "\n
    {col}
    {val}
    " + + return f""" +
    +

    Data Overview

    + {info_html} + {stats_table} + {preview_html} +
    """ + + def _module_results(self) -> str: + """Generate module-by-module results section.""" + modules = self.assessment.get("modules", []) + if not modules: + return """ +
    +

    Module-by-Module Results

    +

    No detection modules were executed.

    +
    """ + + cards_html = "" + for i, module in enumerate(modules, 1): + name = module.get("name", f"Module {i}") + description = module.get("description", "No description available.") + method = module.get("method", "Unknown method") + risk_level = module.get("risk_level", "LOW") + p_value = module.get("p_value") + test_stat = module.get("test_statistic") + evidence = module.get("evidence_summary", "") + results = module.get("results", {}) + figures = module.get("figures", []) + + css_class = risk_level_to_css_class(risk_level) + emoji = RISK_EMOJI.get(risk_level, "⚪") + label = RISK_LABELS.get(risk_level, "Unknown") + color = RISK_COLORS.get(risk_level, "#6c757d") + + # Statistics row + stats_html = '
    ' + if p_value is not None: + stats_html += f""" +
    +
    {format_p_value(p_value)}
    +
    p-value
    +
    """ + if test_stat is not None: + ts_display = f"{test_stat:.4g}" if isinstance(test_stat, float) else str(test_stat) + stats_html += f""" +
    +
    {ts_display}
    +
    Test Statistic
    +
    """ + stats_html += f""" +
    +
    {emoji} {label}
    +
    Risk Assessment
    +
    +
    """ + + # Additional results + results_html = "" + if results: + results_html = "

    Detailed Results

      " + for key, val in results.items(): + if isinstance(val, float): + val = f"{val:.4g}" + results_html += f"
    • {key}: {val}
      • " + results_html += "
    " + + # Figures + figures_html = "" + for fig in figures: + b64 = encode_figure_base64(fig, self.figures_dir) + if b64: + figures_html += f""" +
    + {name} - {fig} +

    {fig}

    +
    """ + + # Evidence summary + evidence_html = "" + if evidence: + evidence_html = f"

    Evidence: {evidence}

    " + + cards_html += f""" +
    +
    +

    {emoji} {name}

    + {label} +
    +

    {description}

    +

    Method: {method}

    + {stats_html} + {evidence_html} + {results_html} + {figures_html} +
    """ + + return f""" +
    +

    Module-by-Module Results

    + {cards_html} +
    """ + + def _suspicious_points(self) -> str: + """Generate suspicious data points section.""" + points = self.assessment.get("suspicious_points", []) + + if not points: + return """ +
    +

    Suspicious Data Points

    +

    🟢 No individual data points were flagged as suspicious.

    +
    """ + + table_html = """ + + """ + + for i, point in enumerate(points, 1): + row = point.get("row", "—") + col = point.get("column", "—") + value = point.get("value", "—") + reason = point.get("reason", "—") + module = point.get("module", "—") + if isinstance(value, float): + value = f"{value:.6g}" + table_html += f""" + + + """ + + table_html += "\n
    #RowColumnValueReasonModule
    {i}{row}{col}{value}{reason}{module}
    " + + return f""" +
    +

    🔍 Suspicious Data Points

    +

    The following {len(points)} data point(s) triggered alerts. Each entry shows the + exact row/column reference, the observed value, and the reason for flagging.

    + {table_html} +
    """ + + def _confidence_limitations(self) -> str: + """Generate confidence and limitations section.""" + confidence = self.assessment.get("confidence", {}) + overall_conf = confidence.get("overall", "Not calculated") + intervals = confidence.get("intervals", {}) + limitations = confidence.get("limitations", []) + + # Confidence intervals + intervals_html = "" + if intervals: + intervals_html = """ +

    Confidence Intervals

    + + """ + for measure, data in intervals.items(): + est = data.get("estimate", "—") + lower = data.get("ci_lower", "—") + upper = data.get("ci_upper", "—") + if isinstance(est, float): + est = f"{est:.4g}" + if isinstance(lower, float): + lower = f"{lower:.4g}" + if isinstance(upper, float): + upper = f"{upper:.4g}" + intervals_html += f"\n " + intervals_html += "\n
    MeasureEstimate95% CI Lower95% CI Upper
    {measure}{est}{lower}{upper}
    " + + # Limitations + limitations_html = "" + if limitations: + limitations_html = "\n

    Limitations — What This Tool Cannot Detect

    \n
      " + for lim in limitations: + limitations_html += f"\n
    • {lim}
      • " + limitations_html += "\n
    " + else: + # Default limitations + limitations_html = """ +

    Limitations — What This Tool Cannot Detect

    +
      +
    • Selective reporting or HARKing (Hypothesizing After Results are Known)
      • +
    • Subtle p-hacking through flexible analysis choices
      • +
    • Data fabrication that perfectly mimics expected statistical properties
      • +
    • Image manipulation or duplication (requires specialized image forensics)
      • +
    • Plagiarism or text recycling
      • +
    • Errors in experimental design or methodology
      • +
    • Conflicts of interest or undisclosed funding
      • +
    • Small-scale selective data exclusion that preserves distributional properties
      • +
    """ + + # Overall confidence display + if isinstance(overall_conf, (int, float)): + conf_display = f"{overall_conf:.1%}" if overall_conf <= 1 else f"{overall_conf:.1f}%" + else: + conf_display = str(overall_conf) + + return f""" +
    +

    Confidence & Limitations

    +
    +
    {conf_display}
    +
    Overall Assessment Confidence
    +
    +

    Confidence reflects the reliability of the statistical tests given the data size, + quality, and number of applicable tests. Higher confidence means the results are + more likely to be meaningful rather than artifacts of small samples or noise.

    + {intervals_html} + {limitations_html} +
    """ + + def _methodology(self) -> str: + """Generate methodology section with academic references.""" + modules = self.assessment.get("modules", []) + methods_used = set() + for m in modules: + method_key = m.get("method_key", m.get("name", "").lower().replace(" ", "_")) + methods_used.add(method_key) + + entries_html = "" + for key in sorted(methods_used): + info = METHODOLOGY_REFERENCES.get(key) + if info: + refs_html = "
    ".join(info["references"]) + entries_html += f""" +
    +
    {info['name']}
    +
    {info['description']}
    +
    {refs_html}
    +
    """ + + # If no known methods matched, list from module descriptions + if not entries_html: + for m in modules: + name = m.get("name", "Unknown") + method = m.get("method", "Not specified") + entries_html += f""" +
    +
    {name}
    +
    Method: {method}
    +
    """ + + return f""" +
    +

    Methodology

    +

    The following statistical methods were applied during this analysis. + Each method targets a specific class of data integrity anomalies.

    + {entries_html} +
    """ + + def _recommendations(self) -> str: + """Generate prioritized recommendations section.""" + recs = self.assessment.get("recommendations", []) + + if not recs: + # Generate default recommendations based on risk level + risk = self.assessment.get("overall_risk", {}) + level = risk.get("level", "LOW") + if level in ("HIGH", "CRITICAL"): + recs = [ + {"priority": 1, "action": "Request raw data and analysis scripts from authors", + "rationale": "Direct verification of data provenance is the most reliable method."}, + {"priority": 2, "action": "Have an independent statistician review the flagged anomalies", + "rationale": "Expert review can distinguish genuine anomalies from methodological artifacts."}, + {"priority": 3, "action": "Check for corroborating evidence in supplementary materials", + "rationale": "Supplementary data may provide context that explains apparent anomalies."}, + {"priority": 4, "action": "Consider contacting the journal or institution", + "rationale": "If anomalies persist after review, formal investigation may be warranted."}, + ] + else: + recs = [ + {"priority": 1, "action": "Archive this report for reference", + "rationale": "Maintaining records supports longitudinal monitoring."}, + {"priority": 2, "action": "No immediate action required", + "rationale": "Current findings do not indicate significant integrity concerns."}, + ] + + recs_html = "" + for rec in sorted(recs, key=lambda r: r.get("priority", 99)): + priority = rec.get("priority", "—") + action = rec.get("action", "—") + rationale = rec.get("rationale", "") + recs_html += f""" +
    +
    {priority}
    +
    +
    {action}
    +
    {rationale}
    +
    +
    """ + + return f""" +
    +

    Recommendations

    + {recs_html} +
    """ + + def _disclaimer(self) -> str: + """Generate bilingual disclaimer section.""" + return f""" +
    +

    ⚠️ Disclaimer / 免责声明

    +

    {DISCLAIMER_EN}

    +
    +

    {DISCLAIMER_ZH}

    +
    """ + + def _footer(self) -> str: + """Generate report footer with metadata.""" + meta = self.assessment.get("metadata", {}) + source = meta.get("source_file", "Unknown") + version = meta.get("tool_version", TOOL_VERSION) + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + return f""" +
    + Generated by {TOOL_NAME} v{version} + Input: {source} + Report timestamp: {now} +
    """ + + +# --------------------------------------------------------------------------- +# Markdown Report Generator +# --------------------------------------------------------------------------- + + +class MarkdownReportGenerator: + """Generates a Markdown report from assessment data. + + Figures are referenced as relative paths (suitable for GitHub rendering). + + Attributes: + assessment: The assessment data dictionary. + figures_dir: Relative path to figures directory for Markdown links. + """ + + def __init__(self, assessment: Dict[str, Any], figures_dir: str): + """Initialize the Markdown report generator. + + Args: + assessment: Parsed assessment dictionary. + figures_dir: Relative path to figures directory for image links. + """ + self.assessment = assessment + self.figures_dir = figures_dir + + def generate(self) -> str: + """Generate the complete Markdown report. + + Returns: + Complete Markdown document as a string. + """ + parts = [ + self._header(), + self._executive_summary(), + self._data_overview(), + self._module_results(), + self._suspicious_points(), + self._confidence_limitations(), + self._methodology(), + self._recommendations(), + self._disclaimer(), + self._footer(), + ] + return "\n\n".join(parts) + + def _header(self) -> str: + """Generate Markdown header.""" + meta = self.assessment.get("metadata", {}) + source = meta.get("source_file", "Unknown") + timestamp = meta.get("timestamp", datetime.now(timezone.utc).isoformat()) + + return f"""# 📊 {TOOL_NAME} + +## Analysis Report + +- **Source File:** {source} +- **Generated:** {timestamp} +- **Tool Version:** {meta.get('tool_version', TOOL_VERSION)} + +---""" + + def _executive_summary(self) -> str: + """Generate executive summary in Markdown.""" + risk = self.assessment.get("overall_risk", {}) + score = risk.get("score", 0) + level = risk.get("level", score_to_level(score)) + emoji = RISK_EMOJI.get(level, "⚪") + label = RISK_LABELS.get(level, "Unknown") + + modules = self.assessment.get("modules", []) + flagged = [m for m in modules if m.get("risk_level", "LOW") in ("HIGH", "CRITICAL")] + suspicious_count = len(self.assessment.get("suspicious_points", [])) + + findings = "" + if flagged: + for m in flagged: + me = RISK_EMOJI.get(m.get("risk_level", "LOW"), "⚪") + findings += f"- {me} **{m.get('name', 'Unknown')}**: {m.get('evidence_summary', 'Anomaly detected')}\n" + else: + findings = "- No high-risk anomalies detected across all modules.\n" + + # Conclusion + if level == "CRITICAL": + conclusion = ( + "Multiple strong statistical indicators suggest significant anomalies in this dataset. " + "The patterns observed are highly unlikely to arise from legitimate experimental data. " + "Further expert review is strongly recommended." + ) + elif level == "HIGH": + conclusion = ( + "Several statistical indicators show notable anomalies that warrant further investigation. " + "While not conclusive evidence of data integrity issues, the patterns deserve scrutiny." + ) + elif level == "MEDIUM": + conclusion = ( + "Some mild statistical anomalies were detected. These may reflect legitimate methodological " + "choices or minor reporting inconsistencies. Routine verification is advisable." + ) + else: + conclusion = ( + "The dataset shows no significant statistical anomalies across the tests performed. " + "The data patterns appear consistent with legitimate experimental measurements." + ) + + return f"""## Executive Summary + +| Metric | Value | +|--------|-------| +| **Overall Risk Score** | {score}/100 | +| **Risk Level** | {emoji} {label} | +| **Tests Performed** | {len(modules)} | +| **Suspicious Data Points** | {suspicious_count} | + +### Key Findings + +{findings} +### Conclusion + +{conclusion}""" + + def _data_overview(self) -> str: + """Generate data overview in Markdown.""" + overview = self.assessment.get("data_overview", {}) + meta = self.assessment.get("metadata", {}) + + source = meta.get("source_file", "Unknown") + rows = meta.get("rows", "N/A") + cols = meta.get("columns_count", len(meta.get("columns", []))) + file_size = meta.get("file_size", "N/A") + + md = f"""## Data Overview + +| Property | Value | +|----------|-------| +| Source File | {source} | +| Rows | {rows} | +| Columns | {cols} | +| File Size | {file_size} | +""" + + # Column statistics + statistics = overview.get("statistics", {}) + if statistics: + md += "\n### Column Statistics\n\n" + md += "| Column | Type | Non-Null | Mean | Std | Min | Max |\n" + md += "|--------|------|----------|------|-----|-----|-----|\n" + for col_name, stats in statistics.items(): + dtype = stats.get("dtype", "—") + non_null = stats.get("non_null", "—") + mean = stats.get("mean", "—") + std = stats.get("std", "—") + min_val = stats.get("min", "—") + max_val = stats.get("max", "—") + if isinstance(mean, float): + mean = f"{mean:.4g}" + if isinstance(std, float): + std = f"{std:.4g}" + if isinstance(min_val, float): + min_val = f"{min_val:.4g}" + if isinstance(max_val, float): + max_val = f"{max_val:.4g}" + md += f"| {col_name} | {dtype} | {non_null} | {mean} | {std} | {min_val} | {max_val} |\n" + + # Preview + preview = overview.get("preview", []) + if preview: + columns = overview.get("columns", list(preview[0].keys()) if preview else []) + md += "\n### Data Preview\n\n" + md += "| " + " | ".join(str(c) for c in columns) + " |\n" + md += "| " + " | ".join("---" for _ in columns) + " |\n" + for row in preview[:10]: + vals = [str(row.get(c, "—")) for c in columns] + md += "| " + " | ".join(vals) + " |\n" + + return md + + def _module_results(self) -> str: + """Generate module results in Markdown.""" + modules = self.assessment.get("modules", []) + if not modules: + return "## Module-by-Module Results\n\nNo detection modules were executed." + + md = "## Module-by-Module Results\n" + + for i, module in enumerate(modules, 1): + name = module.get("name", f"Module {i}") + description = module.get("description", "No description available.") + method = module.get("method", "Unknown method") + risk_level = module.get("risk_level", "LOW") + p_value = module.get("p_value") + test_stat = module.get("test_statistic") + evidence = module.get("evidence_summary", "") + results = module.get("results", {}) + figures = module.get("figures", []) + + emoji = RISK_EMOJI.get(risk_level, "⚪") + label = RISK_LABELS.get(risk_level, "Unknown") + + md += f"\n### {emoji} {name}\n\n" + md += f"**Description:** {description}\n\n" + md += f"**Method:** {method}\n\n" + + # Statistics table + md += "| Metric | Value |\n|--------|-------|\n" + if p_value is not None: + md += f"| p-value | {format_p_value(p_value)} |\n" + if test_stat is not None: + ts = f"{test_stat:.4g}" if isinstance(test_stat, float) else str(test_stat) + md += f"| Test Statistic | {ts} |\n" + md += f"| Risk Assessment | {emoji} {label} |\n" + + if evidence: + md += f"\n**Evidence:** {evidence}\n" + + # Detailed results + if results: + md += "\n**Detailed Results:**\n\n" + for key, val in results.items(): + if isinstance(val, float): + val = f"{val:.4g}" + md += f"- **{key}**: {val}\n" + + # Figures + for fig in figures: + fig_path = f"{self.figures_dir}/{fig}" if self.figures_dir else fig + md += f"\n![{name} - {fig}]({fig_path})\n" + md += f"*Figure: {fig}*\n" + + md += "\n---\n" + + return md + + def _suspicious_points(self) -> str: + """Generate suspicious points section in Markdown.""" + points = self.assessment.get("suspicious_points", []) + + if not points: + return "## 🔍 Suspicious Data Points\n\n🟢 No individual data points were flagged as suspicious." + + md = f"## 🔍 Suspicious Data Points\n\n" + md += f"The following **{len(points)}** data point(s) triggered alerts:\n\n" + md += "| # | Row | Column | Value | Reason | Module |\n" + md += "|---|-----|--------|-------|--------|--------|\n" + + for i, point in enumerate(points, 1): + row = point.get("row", "—") + col = point.get("column", "—") + value = point.get("value", "—") + reason = point.get("reason", "—") + module = point.get("module", "—") + if isinstance(value, float): + value = f"{value:.6g}" + md += f"| {i} | {row} | {col} | {value} | {reason} | {module} |\n" + + return md + + def _confidence_limitations(self) -> str: + """Generate confidence and limitations section in Markdown.""" + confidence = self.assessment.get("confidence", {}) + overall_conf = confidence.get("overall", "Not calculated") + intervals = confidence.get("intervals", {}) + limitations = confidence.get("limitations", []) + + if isinstance(overall_conf, (int, float)): + conf_display = f"{overall_conf:.1%}" if overall_conf <= 1 else f"{overall_conf:.1f}%" + else: + conf_display = str(overall_conf) + + md = f"## Confidence & Limitations\n\n" + md += f"**Overall Assessment Confidence:** {conf_display}\n\n" + md += ( + "Confidence reflects the reliability of the statistical tests given the data size, " + "quality, and number of applicable tests.\n" + ) + + if intervals: + md += "\n### Confidence Intervals\n\n" + md += "| Measure | Estimate | 95% CI Lower | 95% CI Upper |\n" + md += "|---------|----------|--------------|-------------|\n" + for measure, data in intervals.items(): + est = data.get("estimate", "—") + lower = data.get("ci_lower", "—") + upper = data.get("ci_upper", "—") + if isinstance(est, float): + est = f"{est:.4g}" + if isinstance(lower, float): + lower = f"{lower:.4g}" + if isinstance(upper, float): + upper = f"{upper:.4g}" + md += f"| {measure} | {est} | {lower} | {upper} |\n" + + md += "\n### Limitations — What This Tool Cannot Detect\n\n" + if limitations: + for lim in limitations: + md += f"- {lim}\n" + else: + md += """- Selective reporting or HARKing (Hypothesizing After Results are Known) +- Subtle p-hacking through flexible analysis choices +- Data fabrication that perfectly mimics expected statistical properties +- Image manipulation or duplication (requires specialized image forensics) +- Plagiarism or text recycling +- Errors in experimental design or methodology +- Conflicts of interest or undisclosed funding +- Small-scale selective data exclusion that preserves distributional properties +""" + + return md + + def _methodology(self) -> str: + """Generate methodology section in Markdown.""" + modules = self.assessment.get("modules", []) + methods_used = set() + for m in modules: + method_key = m.get("method_key", m.get("name", "").lower().replace(" ", "_")) + methods_used.add(method_key) + + md = "## Methodology\n\n" + md += "The following statistical methods were applied during this analysis:\n\n" + + has_entries = False + for key in sorted(methods_used): + info = METHODOLOGY_REFERENCES.get(key) + if info: + has_entries = True + md += f"### {info['name']}\n\n" + md += f"{info['description']}\n\n" + md += "**References:**\n\n" + for ref in info["references"]: + md += f"- {ref}\n" + md += "\n" + + if not has_entries: + for m in modules: + name = m.get("name", "Unknown") + method = m.get("method", "Not specified") + md += f"### {name}\n\n" + md += f"Method: {method}\n\n" + + return md + + def _recommendations(self) -> str: + """Generate recommendations section in Markdown.""" + recs = self.assessment.get("recommendations", []) + + if not recs: + risk = self.assessment.get("overall_risk", {}) + level = risk.get("level", "LOW") + if level in ("HIGH", "CRITICAL"): + recs = [ + {"priority": 1, "action": "Request raw data and analysis scripts from authors", + "rationale": "Direct verification of data provenance is the most reliable method."}, + {"priority": 2, "action": "Have an independent statistician review the flagged anomalies", + "rationale": "Expert review can distinguish genuine anomalies from methodological artifacts."}, + {"priority": 3, "action": "Check for corroborating evidence in supplementary materials", + "rationale": "Supplementary data may provide context that explains apparent anomalies."}, + {"priority": 4, "action": "Consider contacting the journal or institution", + "rationale": "If anomalies persist after review, formal investigation may be warranted."}, + ] + else: + recs = [ + {"priority": 1, "action": "Archive this report for reference", + "rationale": "Maintaining records supports longitudinal monitoring."}, + {"priority": 2, "action": "No immediate action required", + "rationale": "Current findings do not indicate significant integrity concerns."}, + ] + + md = "## Recommendations\n\n" + for rec in sorted(recs, key=lambda r: r.get("priority", 99)): + priority = rec.get("priority", "—") + action = rec.get("action", "—") + rationale = rec.get("rationale", "") + md += f"**{priority}.** {action}\n" + if rationale: + md += f" > {rationale}\n" + md += "\n" + + return md + + def _disclaimer(self) -> str: + """Generate bilingual disclaimer in Markdown.""" + return f"""## ⚠️ Disclaimer / 免责声明 + +{DISCLAIMER_EN} + +--- + +{DISCLAIMER_ZH}""" + + def _footer(self) -> str: + """Generate footer in Markdown.""" + meta = self.assessment.get("metadata", {}) + source = meta.get("source_file", "Unknown") + version = meta.get("tool_version", TOOL_VERSION) + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + return f"""--- + +*Generated by {TOOL_NAME} v{version} | Input: {source} | Timestamp: {now}*""" + + +# --------------------------------------------------------------------------- +# Main / CLI +# --------------------------------------------------------------------------- + + +def generate_reports( + assessment_path: str, + figures_dir: str, + output_dir: str, + formats: Optional[List[str]] = None, +) -> Dict[str, str]: + """Generate reports in specified formats from an assessment JSON file. + + Args: + assessment_path: Path to the assessment JSON file. + figures_dir: Path to the directory containing figure files. + output_dir: Directory where output reports will be written. + formats: List of formats to generate ('html', 'markdown'). Defaults to both. + + Returns: + Dictionary mapping format names to output file paths. + + Raises: + FileNotFoundError: If assessment file doesn't exist. + ValueError: If assessment JSON is invalid. + """ + if formats is None: + formats = ["html", "markdown"] + + # Load assessment + assessment = load_assessment(assessment_path) + + # Ensure output directory exists + output_path = Path(output_dir) + output_path.mkdir(parents=True, exist_ok=True) + + # Determine base filename from source + meta = assessment.get("metadata", {}) + source = meta.get("source_file", "analysis") + base_name = Path(source).stem if source else "analysis" + + results = {} + + if "html" in formats: + html_gen = HTMLReportGenerator(assessment, figures_dir) + html_content = html_gen.generate() + html_path = output_path / f"{base_name}_report.html" + with open(html_path, "w", encoding="utf-8") as f: + f.write(html_content) + results["html"] = str(html_path) + print(f"✅ HTML report generated: {html_path}") + + if "markdown" in formats: + # For markdown, use relative path to figures + try: + rel_figures = os.path.relpath(figures_dir, output_dir) + except ValueError: + rel_figures = figures_dir + md_gen = MarkdownReportGenerator(assessment, rel_figures) + md_content = md_gen.generate() + md_path = output_path / f"{base_name}_report.md" + with open(md_path, "w", encoding="utf-8") as f: + f.write(md_content) + results["markdown"] = str(md_path) + print(f"✅ Markdown report generated: {md_path}") + + return results + + +def main(): + """CLI entry point for the report generator. + + Parses arguments and generates reports in the specified formats. + """ + parser = argparse.ArgumentParser( + description=f"{TOOL_NAME} — Report Generator", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate both HTML and Markdown reports + python3 report_generator.py --input assessment.json --figures figures/ --output report/ + + # Generate only HTML + python3 report_generator.py --input assessment.json --figures figures/ --output report/ --format html + + # Generate only Markdown + python3 report_generator.py --input assessment.json --output report/ --format markdown +""", + ) + + parser.add_argument( + "--input", "-i", + required=True, + help="Path to the assessment JSON file produced by the detection pipeline.", + ) + parser.add_argument( + "--figures", "-f", + default="figures/", + help="Directory containing figure image files (default: figures/).", + ) + parser.add_argument( + "--output", "-o", + default="report/", + help="Output directory for generated reports (default: report/).", + ) + parser.add_argument( + "--format", + choices=["html", "markdown", "both"], + default="both", + help="Output format: html, markdown, or both (default: both).", + ) + parser.add_argument( + "--version", "-v", + action="version", + version=f"%(prog)s {TOOL_VERSION}", + ) + + args = parser.parse_args() + + # Determine formats + if args.format == "both": + formats = ["html", "markdown"] + else: + formats = [args.format] + + try: + results = generate_reports( + assessment_path=args.input, + figures_dir=args.figures, + output_dir=args.output, + formats=formats, + ) + print(f"\n{'='*60}") + print(f"Report generation complete!") + print(f"{'='*60}") + for fmt, path in results.items(): + print(f" {fmt.upper():>10}: {path}") + print() + + except FileNotFoundError as e: + print(f"❌ Error: {e}", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError as e: + print(f"❌ Error: Invalid JSON in assessment file: {e}", file=sys.stderr) + sys.exit(1) + except ValueError as e: + print(f"❌ Error: {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"❌ Unexpected error: {e}", file=sys.stderr) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/tools/gengskill/scripts/run_dup_test.py b/tools/gengskill/scripts/run_dup_test.py new file mode 100644 index 0000000..ebde5a6 --- /dev/null +++ b/tools/gengskill/scripts/run_dup_test.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Wrapper to run image_duplicate_test with proper numpy handling""" +import sys +import json +import numpy as np + +# We'll monkey-patch json to handle numpy types +class NumpyEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, (np.integer,)): + return int(obj) + if isinstance(obj, (np.floating,)): + return float(obj) + if isinstance(obj, (np.bool_,)): + return bool(obj) + if isinstance(obj, np.ndarray): + return obj.tolist() + if obj is None: + return None + return super().default(obj) + +# Save original dumps +original_dumps = json.dumps + +def patched_dumps(obj, **kwargs): + kwargs.setdefault('cls', NumpyEncoder) + return original_dumps(obj, **kwargs) + +json.dumps = patched_dumps + +# Now import and run the original script +sys.argv = ['image_duplicate_test.py', + '--input_dir', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/figures', + '--threshold', '0.85', + '--output', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/duplicate_results.json'] + +exec(open('scripts/image_duplicate_test.py').read()) diff --git a/tools/gengskill/scripts/visualization.py b/tools/gengskill/scripts/visualization.py new file mode 100644 index 0000000..3ad79e5 --- /dev/null +++ b/tools/gengskill/scripts/visualization.py @@ -0,0 +1,1228 @@ +#!/usr/bin/env python3 +""" +visualization.py — Publication-quality figure generation for academic fraud detection reports. + +Part of the Geng Skill academic fraud detection project. Generates visualizations +for statistical tests including last-digit analysis, Benford's Law, fixed-ratio +detection, decimal pattern analysis, and comprehensive dashboards. + +Usage (CLI): + python3 visualization.py --input report.json --output figures/ + +Usage (API): + from visualization import plot_last_digit, plot_benford, plot_fixed_ratio + fig_path = plot_last_digit(test_result, output_dir="figures/") +""" + +import json +import argparse +import os +import sys +from pathlib import Path +from typing import Dict, Any, Optional, List, Tuple + +import numpy as np +import matplotlib +matplotlib.use("Agg") # Non-interactive backend for server environments +import matplotlib.pyplot as plt +import matplotlib.patches as mpatches +from matplotlib.patches import FancyBboxPatch, Arc, Wedge +from matplotlib.colors import LinearSegmentedColormap +import matplotlib.ticker as mticker +import seaborn as sns + +# --------------------------------------------------------------------------- +# Font Configuration for CJK (Chinese) + English support +# --------------------------------------------------------------------------- + +def _configure_fonts(): + """Configure matplotlib to support CJK characters with fallback.""" + # Try common CJK fonts available on Linux/macOS/Windows + cjk_fonts = [ + "WenQuanYi Micro Hei", + "WenQuanYi Zen Hei", + "Noto Sans CJK SC", + "SimHei", + "Microsoft YaHei", + "PingFang SC", + "Hiragino Sans GB", + "DejaVu Sans", + ] + + available_fonts = set( + f.name for f in matplotlib.font_manager.fontManager.ttflist + ) + + chosen_fonts = [] + for font in cjk_fonts: + if font in available_fonts: + chosen_fonts.append(font) + + # Fallback: always include DejaVu Sans for guaranteed rendering + if "DejaVu Sans" not in chosen_fonts: + chosen_fonts.append("DejaVu Sans") + + plt.rcParams["font.sans-serif"] = chosen_fonts + plt.rcParams.get("font.sans-serif", []) + plt.rcParams["axes.unicode_minus"] = False + plt.rcParams["font.family"] = "sans-serif" + + +_configure_fonts() + +# --------------------------------------------------------------------------- +# Style Configuration +# --------------------------------------------------------------------------- + +# Publication-quality defaults +STYLE_CONFIG = { + "figure.dpi": 300, + "savefig.dpi": 300, + "savefig.bbox": "tight", + "axes.spines.top": False, + "axes.spines.right": False, + "axes.labelsize": 11, + "axes.titlesize": 13, + "xtick.labelsize": 9, + "ytick.labelsize": 9, + "legend.fontsize": 9, + "figure.facecolor": "white", + "axes.facecolor": "white", + "axes.grid": False, +} +plt.rcParams.update(STYLE_CONFIG) + +# Color palette +COLORS = { + "observed": "#2196F3", # Blue + "expected": "#FF9800", # Orange + "highlight": "#F44336", # Red (anomaly) + "normal": "#4CAF50", # Green (normal) + "neutral": "#9E9E9E", # Gray + "regression": "#E91E63", # Pink + "scatter": "#3F51B5", # Indigo + "risk_low": "#4CAF50", # Green + "risk_medium": "#FFC107", # Amber + "risk_high": "#FF5722", # Deep Orange + "risk_critical": "#D32F2F", # Dark Red +} + +# Risk level thresholds +RISK_THRESHOLDS = { + "low": (0, 25), + "medium": (25, 50), + "high": (50, 75), + "critical": (75, 100), +} + + +# --------------------------------------------------------------------------- +# Utility Functions +# --------------------------------------------------------------------------- + +def _ensure_output_dir(output_dir: str) -> Path: + """Create output directory if it doesn't exist.""" + path = Path(output_dir) + path.mkdir(parents=True, exist_ok=True) + return path + + +def _save_figure(fig: plt.Figure, filepath: str, dpi: int = 300) -> str: + """Save figure to file and close it. + + Args: + fig: Matplotlib figure object. + filepath: Output file path. + dpi: Resolution in dots per inch. + + Returns: + Absolute path of the saved figure. + """ + fig.savefig(filepath, dpi=dpi, bbox_inches="tight", facecolor="white") + plt.close(fig) + return str(Path(filepath).resolve()) + + +def _annotate_pvalue(ax: plt.Axes, p_value: float, x: float = 0.95, y: float = 0.95): + """Add p-value annotation to axes with significance stars.""" + if p_value < 0.001: + stars = "***" + color = COLORS["highlight"] + elif p_value < 0.01: + stars = "**" + color = COLORS["risk_high"] + elif p_value < 0.05: + stars = "*" + color = COLORS["risk_medium"] + else: + stars = "ns" + color = COLORS["normal"] + + text = f"p = {p_value:.4f} {stars}" + ax.annotate( + text, + xy=(x, y), + xycoords="axes fraction", + fontsize=10, + fontweight="bold", + color=color, + ha="right", + va="top", + bbox=dict(boxstyle="round,pad=0.3", facecolor="lightyellow", alpha=0.8), + ) + + +# --------------------------------------------------------------------------- +# 1. Last Digit Distribution Plot +# --------------------------------------------------------------------------- + +def plot_last_digit( + result: Dict[str, Any], + output_dir: str = "figures/", + filename: str = "last_digit_distribution.png", + dpi: int = 300, +) -> str: + """Generate a bar chart of last digit distribution vs. expected uniform distribution. + + Shows observed frequencies of last digits (0-9) compared to the expected + uniform distribution (10% each), with chi-square test p-value annotation. + + Args: + result: Output dict from last-digit test module, expected keys: + - observed_freq (list/dict): Observed frequencies for digits 0-9. + - expected_freq (list/dict, optional): Expected frequencies. + - chi_square (float): Chi-square statistic. + - p_value (float): P-value from chi-square test. + - n_samples (int, optional): Total sample count. + - column_name (str, optional): Name of the analyzed column. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + # Extract data + observed = result.get("observed_freq", result.get("observed", [])) + if isinstance(observed, dict): + digits = sorted(observed.keys(), key=lambda x: int(x)) + obs_values = [observed[d] for d in digits] + digits = [int(d) for d in digits] + else: + obs_values = list(observed) + digits = list(range(len(obs_values))) + + # Normalize to proportions if raw counts + obs_array = np.array(obs_values, dtype=float) + if obs_array.sum() > 1.1: # Raw counts, convert to proportions + obs_array = obs_array / obs_array.sum() + + n_digits = len(digits) + expected_prop = 1.0 / n_digits # Uniform expectation + + chi_sq = result.get("chi_square", result.get("chi2", 0.0)) + p_value = result.get("p_value", result.get("pvalue", 1.0)) + n_samples = result.get("n_samples", result.get("n", "N/A")) + col_name = result.get("column_name", result.get("column", "")) + + # Create figure + fig, ax = plt.subplots(figsize=(8, 5)) + + x = np.arange(n_digits) + width = 0.35 + + # Bar chart + bars_obs = ax.bar( + x - width / 2, obs_array, width, + label="观测频率 Observed", color=COLORS["observed"], alpha=0.85, edgecolor="white" + ) + bars_exp = ax.bar( + x + width / 2, [expected_prop] * n_digits, width, + label=f"期望频率 Expected ({expected_prop:.1%})", color=COLORS["expected"], alpha=0.6, edgecolor="white" + ) + + # Reference line + ax.axhline(y=expected_prop, color=COLORS["neutral"], linestyle="--", linewidth=0.8, alpha=0.6) + + # Highlight anomalous digits (>2 standard deviations from expected) + std_threshold = 2.0 * np.sqrt(expected_prop * (1 - expected_prop) / max(n_samples if isinstance(n_samples, (int, float)) else 100, 1)) + for i, (obs_val, bar) in enumerate(zip(obs_array, bars_obs)): + if abs(obs_val - expected_prop) > std_threshold: + bar.set_edgecolor(COLORS["highlight"]) + bar.set_linewidth(2) + + # Labels and title + title = "末位数字分布检验 Last Digit Distribution Test" + if col_name: + title += f"\n[{col_name}]" + ax.set_title(title, fontsize=13, fontweight="bold", pad=15) + ax.set_xlabel("末位数字 Last Digit", fontsize=11) + ax.set_ylabel("频率 Frequency", fontsize=11) + ax.set_xticks(x) + ax.set_xticklabels(digits) + ax.set_ylim(0, max(obs_array.max(), expected_prop) * 1.35) + ax.legend(loc="upper left", framealpha=0.9) + + # Annotate statistics + _annotate_pvalue(ax, p_value) + stats_text = f"χ² = {chi_sq:.2f}\nn = {n_samples}" + ax.text( + 0.95, 0.80, stats_text, + transform=ax.transAxes, fontsize=9, + ha="right", va="top", color="#555555", + ) + + return _save_figure(fig, filepath, dpi) + + +# --------------------------------------------------------------------------- +# 2. Benford's Law Plot +# --------------------------------------------------------------------------- + +def plot_benford( + result: Dict[str, Any], + output_dir: str = "figures/", + filename: str = "benford_law.png", + dpi: int = 300, +) -> str: + """Generate a bar chart comparing observed first-digit frequencies to Benford's Law. + + Shows observed vs. theoretical Benford distribution with MAD (Mean Absolute + Deviation) annotation and conformity assessment. + + Args: + result: Output dict from Benford's Law test module, expected keys: + - observed_freq (list/dict): Observed proportions for digits 1-9. + - benford_freq (list/dict, optional): Theoretical Benford proportions. + - mad (float): Mean Absolute Deviation from Benford's Law. + - conformity (str, optional): Conformity level (e.g., "close", "acceptable"). + - chi_square (float, optional): Chi-square statistic. + - p_value (float, optional): P-value. + - column_name (str, optional): Name of the analyzed column. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + # Benford's theoretical distribution + benford_theoretical = { + d: np.log10(1 + 1 / d) for d in range(1, 10) + } + + # Extract observed data + observed = result.get("observed_freq", result.get("observed", {})) + if isinstance(observed, (list, np.ndarray)): + obs_values = list(observed) + digits = list(range(1, len(obs_values) + 1)) + else: + digits = sorted([int(k) for k in observed.keys()]) + obs_values = [observed[str(d)] if str(d) in observed else observed.get(d, 0) for d in digits] + + obs_array = np.array(obs_values, dtype=float) + if obs_array.sum() > 1.1: + obs_array = obs_array / obs_array.sum() + + # Benford expected + benford_expected = np.array([benford_theoretical.get(d, 0) for d in digits]) + + mad = result.get("mad", result.get("MAD", np.mean(np.abs(obs_array - benford_expected)))) + conformity = result.get("conformity", result.get("level", "")) + p_value = result.get("p_value", result.get("pvalue", None)) + col_name = result.get("column_name", result.get("column", "")) + + # Create figure + fig, ax = plt.subplots(figsize=(9, 5.5)) + + x = np.arange(len(digits)) + width = 0.35 + + # Bars + ax.bar( + x - width / 2, obs_array, width, + label="观测频率 Observed", color=COLORS["observed"], alpha=0.85, edgecolor="white" + ) + ax.bar( + x + width / 2, benford_expected, width, + label="Benford 理论值 Expected", color=COLORS["expected"], alpha=0.7, edgecolor="white" + ) + + # Benford curve overlay + ax.plot(x, benford_expected, "o-", color=COLORS["expected"], alpha=0.9, linewidth=1.5, markersize=4) + + # Title and labels + title = "Benford 定律检验 Benford's Law Test" + if col_name: + title += f"\n[{col_name}]" + ax.set_title(title, fontsize=13, fontweight="bold", pad=15) + ax.set_xlabel("首位数字 First Digit", fontsize=11) + ax.set_ylabel("频率 Frequency", fontsize=11) + ax.set_xticks(x) + ax.set_xticklabels(digits) + ax.set_ylim(0, max(obs_array.max(), benford_expected.max()) * 1.35) + ax.legend(loc="upper right", framealpha=0.9) + + # MAD annotation with color coding + # MAD thresholds (Nigrini 2012): <0.006 close, <0.012 acceptable, <0.015 marginally acceptable + if mad < 0.006: + mad_color = COLORS["normal"] + mad_label = "Close conformity" + elif mad < 0.012: + mad_color = COLORS["risk_medium"] + mad_label = "Acceptable conformity" + elif mad < 0.015: + mad_color = COLORS["risk_high"] + mad_label = "Marginally acceptable" + else: + mad_color = COLORS["highlight"] + mad_label = "Non-conformity" + + if conformity: + mad_label = conformity + + mad_text = f"MAD = {mad:.4f}\n({mad_label})" + ax.annotate( + mad_text, + xy=(0.02, 0.95), + xycoords="axes fraction", + fontsize=10, + fontweight="bold", + color=mad_color, + ha="left", + va="top", + bbox=dict(boxstyle="round,pad=0.4", facecolor="lightyellow", edgecolor=mad_color, alpha=0.9), + ) + + # P-value if available + if p_value is not None: + _annotate_pvalue(ax, p_value, x=0.98, y=0.95) + + return _save_figure(fig, filepath, dpi) + + +# --------------------------------------------------------------------------- +# 3. Fixed Ratio Scatter Plot +# --------------------------------------------------------------------------- + +def plot_fixed_ratio( + result: Dict[str, Any], + output_dir: str = "figures/", + filename: str = "fixed_ratio_scatter.png", + dpi: int = 300, +) -> str: + """Generate a scatter plot with regression line for fixed-ratio detection. + + Visualizes the relationship between two numeric columns, annotating R², + slope, and whether the ratio appears suspiciously fixed (unnaturally high R²). + + Args: + result: Output dict from fixed-ratio test module, expected keys: + - x_values (list): Values of the first column. + - y_values (list): Values of the second column. + - r_squared (float): R² value of linear fit. + - slope (float): Slope of regression line. + - intercept (float): Intercept of regression line. + - is_fixed (bool): Whether ratio is deemed suspiciously fixed. + - threshold (float, optional): R² threshold used for detection. + - x_column (str, optional): Name of x column. + - y_column (str, optional): Name of y column. + - p_value (float, optional): P-value for the regression. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + # Extract data + x_vals = np.array(result.get("x_values", result.get("x", [])), dtype=float) + y_vals = np.array(result.get("y_values", result.get("y", [])), dtype=float) + r_squared = result.get("r_squared", result.get("r2", 0.0)) + slope = result.get("slope", 0.0) + intercept = result.get("intercept", 0.0) + is_fixed = result.get("is_fixed", result.get("fixed", False)) + x_col = result.get("x_column", result.get("x_col", "X")) + y_col = result.get("y_column", result.get("y_col", "Y")) + threshold = result.get("threshold", 0.99) + + # Create figure + fig, ax = plt.subplots(figsize=(7, 6)) + + # Scatter plot + scatter_color = COLORS["highlight"] if is_fixed else COLORS["scatter"] + ax.scatter( + x_vals, y_vals, + c=scatter_color, alpha=0.5, s=30, edgecolors="white", linewidth=0.3, + label="数据点 Data points" + ) + + # Regression line + if len(x_vals) > 1: + x_fit = np.linspace(x_vals.min(), x_vals.max(), 100) + y_fit = slope * x_fit + intercept + ax.plot( + x_fit, y_fit, + color=COLORS["regression"], linewidth=2, linestyle="-", + label=f"回归线 y = {slope:.4f}x + {intercept:.4f}" + ) + + # Title + status_str = "⚠️ 比值固定 FIXED" if is_fixed else "✓ 正常 NORMAL" + title = f"固定比值检测 Fixed Ratio Detection\n{status_str}" + title_color = COLORS["highlight"] if is_fixed else COLORS["normal"] + ax.set_title(title, fontsize=13, fontweight="bold", pad=15, color=title_color) + + ax.set_xlabel(f"{x_col}", fontsize=11) + ax.set_ylabel(f"{y_col}", fontsize=11) + ax.legend(loc="lower right", framealpha=0.9) + + # Stats annotation box + stats_lines = [ + f"R² = {r_squared:.6f}", + f"Slope = {slope:.4f}", + f"Intercept = {intercept:.4f}", + f"Threshold = {threshold}", + f"判定 Verdict: {'固定 Fixed' if is_fixed else '正常 Normal'}", + ] + stats_text = "\n".join(stats_lines) + + box_color = "#FFEBEE" if is_fixed else "#E8F5E9" + border_color = COLORS["highlight"] if is_fixed else COLORS["normal"] + + ax.text( + 0.03, 0.97, stats_text, + transform=ax.transAxes, fontsize=9, + verticalalignment="top", fontfamily="monospace", + bbox=dict(boxstyle="round,pad=0.5", facecolor=box_color, edgecolor=border_color, alpha=0.9), + ) + + return _save_figure(fig, filepath, dpi) + + +# --------------------------------------------------------------------------- +# 4. Decimal Pattern Heatmap +# --------------------------------------------------------------------------- + +def plot_decimal_heatmap( + result: Dict[str, Any], + output_dir: str = "figures/", + filename: str = "decimal_pattern_heatmap.png", + dpi: int = 300, +) -> str: + """Generate a heatmap showing digit frequency at each decimal position. + + Visualizes the distribution of digits (0-9) at each decimal place, + highlighting positions with anomalously uniform or non-uniform patterns. + + Args: + result: Output dict from decimal pattern test module, expected keys: + - frequency_matrix (list[list] or dict): Digit frequencies per position. + Shape: (n_positions, 10) where columns are digits 0-9. + - positions (list, optional): Labels for decimal positions. + - anomalous_positions (list, optional): Positions flagged as anomalous. + - column_name (str, optional): Name of analyzed column. + - p_values (list, optional): Per-position p-values. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + # Extract frequency matrix + freq_matrix = result.get("frequency_matrix", result.get("matrix", [])) + if isinstance(freq_matrix, dict): + positions = sorted(freq_matrix.keys(), key=lambda x: int(x) if str(x).isdigit() else 0) + matrix = np.array([freq_matrix[p] for p in positions], dtype=float) + else: + matrix = np.array(freq_matrix, dtype=float) + positions = result.get("positions", [f"Pos {i+1}" for i in range(matrix.shape[0])]) + + # Normalize rows to proportions + row_sums = matrix.sum(axis=1, keepdims=True) + row_sums[row_sums == 0] = 1 # Avoid division by zero + matrix_prop = matrix / row_sums + + anomalous = set(result.get("anomalous_positions", result.get("anomalous", []))) + col_name = result.get("column_name", result.get("column", "")) + + # Create figure + n_positions = matrix_prop.shape[0] + fig_height = max(4, n_positions * 0.5 + 2) + fig, ax = plt.subplots(figsize=(10, fig_height)) + + # Heatmap with diverging colormap centered on 0.1 (uniform expectation) + vmin = 0.0 + vmax = max(0.25, matrix_prop.max()) + + # Custom colormap: green (under) -> white (expected=0.1) -> red (over) + cmap = sns.diverging_palette(145, 10, s=80, l=55, as_cmap=True) + + # Plot heatmap with deviation from expected (0.1) + deviation = matrix_prop - 0.1 + + sns.heatmap( + deviation, + ax=ax, + cmap=cmap, + center=0, + vmin=-0.15, + vmax=0.15, + annot=matrix_prop, + fmt=".3f", + linewidths=0.5, + linecolor="white", + xticklabels=[str(d) for d in range(10)], + yticklabels=[str(p) for p in positions], + cbar_kws={"label": "偏差 Deviation from expected (0.1)", "shrink": 0.8}, + ) + + # Highlight anomalous rows + for i, pos in enumerate(positions): + if pos in anomalous or i in anomalous or str(pos) in [str(a) for a in anomalous]: + ax.add_patch(plt.Rectangle((0, i), 10, 1, fill=False, edgecolor=COLORS["highlight"], linewidth=2.5)) + + # Title and labels + title = "小数位数字模式热图 Decimal Pattern Heatmap" + if col_name: + title += f"\n[{col_name}]" + ax.set_title(title, fontsize=13, fontweight="bold", pad=15) + ax.set_xlabel("数字 Digit", fontsize=11) + ax.set_ylabel("小数位置 Decimal Position", fontsize=11) + + # Annotation for anomalous positions + if anomalous: + ax.text( + 1.02, 0.02, + f"⚠ 异常位置: {len(anomalous)}", + transform=ax.transAxes, fontsize=9, + color=COLORS["highlight"], fontweight="bold", + va="bottom", + ) + + plt.tight_layout() + return _save_figure(fig, filepath, dpi) + + +# --------------------------------------------------------------------------- +# 5. Comprehensive Dashboard +# --------------------------------------------------------------------------- + +def plot_dashboard( + report: Dict[str, Any], + output_dir: str = "figures/", + filename: str = "comprehensive_dashboard.png", + dpi: int = 300, +) -> str: + """Generate a multi-panel comprehensive dashboard combining all tests. + + Creates a publication-ready figure with subplots for each statistical test, + plus an overall risk assessment panel. Suitable for report inclusion. + + Args: + report: Full report dict containing results from all tests, expected keys: + - last_digit (dict): Last digit test results. + - benford (dict): Benford's Law test results. + - fixed_ratio (dict, optional): Fixed ratio test results. + - decimal_pattern (dict, optional): Decimal pattern test results. + - risk_score (float): Overall risk score 0-100. + - summary (dict, optional): Summary statistics. + - dataset_name (str, optional): Name of the dataset. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + # Determine layout based on available tests + has_last_digit = "last_digit" in report + has_benford = "benford" in report + has_fixed_ratio = "fixed_ratio" in report + has_decimal = "decimal_pattern" in report + has_risk = "risk_score" in report + + # Create figure with GridSpec + fig = plt.figure(figsize=(16, 12)) + gs = fig.add_gridspec(3, 3, hspace=0.4, wspace=0.35) + + dataset_name = report.get("dataset_name", report.get("name", "Unknown Dataset")) + fig.suptitle( + f"学术数据异常检测综合报告 Fraud Detection Dashboard\n{dataset_name}", + fontsize=15, fontweight="bold", y=0.98, + ) + + # Panel 1: Last Digit Distribution (top-left) + if has_last_digit: + ax1 = fig.add_subplot(gs[0, 0]) + _draw_last_digit_panel(ax1, report["last_digit"]) + + # Panel 2: Benford's Law (top-center) + if has_benford: + ax2 = fig.add_subplot(gs[0, 1]) + _draw_benford_panel(ax2, report["benford"]) + + # Panel 3: Risk Score Gauge (top-right) + if has_risk: + ax3 = fig.add_subplot(gs[0, 2]) + _draw_risk_gauge_panel(ax3, report["risk_score"]) + + # Panel 4: Fixed Ratio (middle-left) + if has_fixed_ratio: + ax4 = fig.add_subplot(gs[1, 0]) + _draw_fixed_ratio_panel(ax4, report["fixed_ratio"]) + + # Panel 5: Decimal Pattern (middle-center + right) + if has_decimal: + ax5 = fig.add_subplot(gs[1, 1:]) + _draw_decimal_panel(ax5, report["decimal_pattern"]) + + # Panel 6: Summary table (bottom row) + ax6 = fig.add_subplot(gs[2, :]) + _draw_summary_panel(ax6, report) + + return _save_figure(fig, filepath, dpi) + + +def _draw_last_digit_panel(ax: plt.Axes, data: Dict): + """Draw last digit distribution as a mini panel.""" + observed = data.get("observed_freq", data.get("observed", [])) + if isinstance(observed, dict): + digits = sorted(observed.keys(), key=lambda x: int(x)) + obs_values = np.array([observed[d] for d in digits], dtype=float) + else: + obs_values = np.array(observed, dtype=float) + digits = list(range(len(obs_values))) + + if obs_values.sum() > 1.1: + obs_values = obs_values / obs_values.sum() + + n = len(digits) + expected = 1.0 / n + + colors = [COLORS["highlight"] if abs(v - expected) > 0.05 else COLORS["observed"] for v in obs_values] + ax.bar(range(n), obs_values, color=colors, alpha=0.8, edgecolor="white") + ax.axhline(expected, color=COLORS["expected"], linestyle="--", linewidth=1) + ax.set_title("末位数字 Last Digit", fontsize=10, fontweight="bold") + ax.set_xlabel("Digit", fontsize=8) + ax.set_ylabel("Freq", fontsize=8) + ax.set_xticks(range(n)) + ax.set_xticklabels(digits, fontsize=7) + + p_val = data.get("p_value", data.get("pvalue", None)) + if p_val is not None: + color = COLORS["highlight"] if p_val < 0.05 else COLORS["normal"] + ax.text(0.95, 0.9, f"p={p_val:.3f}", transform=ax.transAxes, fontsize=8, ha="right", color=color, fontweight="bold") + + +def _draw_benford_panel(ax: plt.Axes, data: Dict): + """Draw Benford's Law comparison as a mini panel.""" + observed = data.get("observed_freq", data.get("observed", {})) + if isinstance(observed, (list, np.ndarray)): + obs_values = np.array(observed, dtype=float) + else: + obs_values = np.array([observed.get(str(d), observed.get(d, 0)) for d in range(1, 10)], dtype=float) + + if obs_values.sum() > 1.1: + obs_values = obs_values / obs_values.sum() + + benford = np.array([np.log10(1 + 1/d) for d in range(1, 10)]) + x = np.arange(9) + + ax.bar(x - 0.15, obs_values, 0.3, label="Obs", color=COLORS["observed"], alpha=0.8) + ax.bar(x + 0.15, benford, 0.3, label="Benford", color=COLORS["expected"], alpha=0.6) + ax.set_title("Benford 定律", fontsize=10, fontweight="bold") + ax.set_xlabel("First Digit", fontsize=8) + ax.set_xticks(x) + ax.set_xticklabels(range(1, 10), fontsize=7) + ax.legend(fontsize=7, loc="upper right") + + mad = data.get("mad", data.get("MAD", 0)) + color = COLORS["highlight"] if mad > 0.015 else COLORS["normal"] + ax.text(0.95, 0.9, f"MAD={mad:.4f}", transform=ax.transAxes, fontsize=8, ha="right", color=color, fontweight="bold") + + +def _draw_risk_gauge_panel(ax: plt.Axes, risk_score: float): + """Draw a mini risk gauge.""" + ax.set_xlim(-1.2, 1.2) + ax.set_ylim(-0.3, 1.2) + ax.set_aspect("equal") + ax.axis("off") + + # Draw gauge arc segments + angles = np.linspace(180, 0, 100) + for i in range(len(angles) - 1): + frac = i / (len(angles) - 1) + if frac < 0.25: + color = COLORS["risk_low"] + elif frac < 0.5: + color = COLORS["risk_medium"] + elif frac < 0.75: + color = COLORS["risk_high"] + else: + color = COLORS["risk_critical"] + + theta1 = angles[i + 1] + theta2 = angles[i] + wedge = Wedge((0, 0), 1.0, theta1, theta2, width=0.3, facecolor=color, alpha=0.7) + ax.add_patch(wedge) + + # Needle + needle_angle = 180 - (risk_score / 100) * 180 + needle_rad = np.radians(needle_angle) + needle_x = 0.75 * np.cos(needle_rad) + needle_y = 0.75 * np.sin(needle_rad) + ax.annotate( + "", xy=(needle_x, needle_y), xytext=(0, 0), + arrowprops=dict(arrowstyle="-|>", color="#333333", lw=2), + ) + ax.plot(0, 0, "o", color="#333333", markersize=6) + + # Score text + if risk_score >= 75: + score_color = COLORS["risk_critical"] + elif risk_score >= 50: + score_color = COLORS["risk_high"] + elif risk_score >= 25: + score_color = COLORS["risk_medium"] + else: + score_color = COLORS["risk_low"] + + ax.text(0, -0.2, f"{risk_score:.0f}", fontsize=20, fontweight="bold", ha="center", color=score_color) + ax.set_title("风险评分 Risk Score", fontsize=10, fontweight="bold", pad=5) + + +def _draw_fixed_ratio_panel(ax: plt.Axes, data: Dict): + """Draw fixed ratio scatter as a mini panel.""" + x_vals = np.array(data.get("x_values", data.get("x", [])), dtype=float) + y_vals = np.array(data.get("y_values", data.get("y", [])), dtype=float) + r_sq = data.get("r_squared", data.get("r2", 0)) + is_fixed = data.get("is_fixed", data.get("fixed", False)) + + color = COLORS["highlight"] if is_fixed else COLORS["scatter"] + if len(x_vals) > 0 and len(y_vals) > 0: + ax.scatter(x_vals, y_vals, c=color, alpha=0.4, s=15, edgecolors="none") + + # Regression line + if len(x_vals) > 1: + slope = data.get("slope", 0) + intercept = data.get("intercept", 0) + x_fit = np.linspace(x_vals.min(), x_vals.max(), 50) + ax.plot(x_fit, slope * x_fit + intercept, color=COLORS["regression"], linewidth=1.5) + + status = "⚠ FIXED" if is_fixed else "✓ Normal" + ax.set_title(f"固定比值 {status}", fontsize=10, fontweight="bold", color=color) + ax.text(0.05, 0.9, f"R²={r_sq:.4f}", transform=ax.transAxes, fontsize=8, fontweight="bold") + + +def _draw_decimal_panel(ax: plt.Axes, data: Dict): + """Draw decimal pattern heatmap as a mini panel.""" + freq_matrix = data.get("frequency_matrix", data.get("matrix", [])) + if isinstance(freq_matrix, dict): + positions = sorted(freq_matrix.keys(), key=lambda x: int(x) if str(x).isdigit() else 0) + matrix = np.array([freq_matrix[p] for p in positions], dtype=float) + else: + matrix = np.array(freq_matrix, dtype=float) + positions = data.get("positions", [f"P{i+1}" for i in range(matrix.shape[0])]) + + if matrix.size == 0: + ax.text(0.5, 0.5, "No decimal data", ha="center", va="center", transform=ax.transAxes) + ax.set_title("小数模式 Decimal Pattern", fontsize=10, fontweight="bold") + return + + row_sums = matrix.sum(axis=1, keepdims=True) + row_sums[row_sums == 0] = 1 + matrix_prop = matrix / row_sums + + sns.heatmap( + matrix_prop, ax=ax, cmap="YlOrRd", + annot=True if matrix_prop.shape[0] <= 5 else False, + fmt=".2f", linewidths=0.3, + xticklabels=[str(d) for d in range(10)], + yticklabels=[str(p) for p in positions], + cbar_kws={"shrink": 0.7}, + ) + ax.set_title("小数模式 Decimal Pattern", fontsize=10, fontweight="bold") + ax.set_xlabel("Digit", fontsize=8) + ax.set_ylabel("Position", fontsize=8) + + +def _draw_summary_panel(ax: plt.Axes, report: Dict): + """Draw a summary table panel.""" + ax.axis("off") + + # Build summary rows + rows = [] + headers = ["检测项目 Test", "结果 Result", "指标 Metric", "判定 Verdict"] + + if "last_digit" in report: + ld = report["last_digit"] + p_val = ld.get("p_value", ld.get("pvalue", "N/A")) + verdict = "⚠ 异常" if (isinstance(p_val, (int, float)) and p_val < 0.05) else "✓ 正常" + rows.append(["末位数字 Last Digit", f"χ²={ld.get('chi_square', ld.get('chi2', 'N/A')):.2f}" if isinstance(ld.get('chi_square', ld.get('chi2')), (int, float)) else "N/A", f"p={p_val:.4f}" if isinstance(p_val, (int, float)) else str(p_val), verdict]) + + if "benford" in report: + bf = report["benford"] + mad = bf.get("mad", bf.get("MAD", "N/A")) + verdict = "⚠ 异常" if (isinstance(mad, (int, float)) and mad > 0.015) else "✓ 正常" + rows.append(["Benford 定律", f"MAD={mad:.4f}" if isinstance(mad, (int, float)) else str(mad), bf.get("conformity", ""), verdict]) + + if "fixed_ratio" in report: + fr = report["fixed_ratio"] + is_fixed = fr.get("is_fixed", fr.get("fixed", False)) + r2 = fr.get("r_squared", fr.get("r2", "N/A")) + verdict = "⚠ 固定" if is_fixed else "✓ 正常" + rows.append(["固定比值 Fixed Ratio", f"R²={r2:.6f}" if isinstance(r2, (int, float)) else str(r2), f"slope={fr.get('slope', 'N/A')}", verdict]) + + if "decimal_pattern" in report: + dp = report["decimal_pattern"] + n_anomalous = len(dp.get("anomalous_positions", dp.get("anomalous", []))) + verdict = f"⚠ {n_anomalous}处异常" if n_anomalous > 0 else "✓ 正常" + rows.append(["小数模式 Decimal", f"异常位置: {n_anomalous}", "", verdict]) + + if rows: + table = ax.table( + cellText=rows, + colLabels=headers, + cellLoc="center", + loc="center", + colWidths=[0.25, 0.25, 0.25, 0.25], + ) + table.auto_set_font_size(False) + table.set_fontsize(9) + table.scale(1.0, 1.5) + + # Style header + for j in range(len(headers)): + table[0, j].set_facecolor("#1976D2") + table[0, j].set_text_props(color="white", fontweight="bold") + + # Color verdict cells (last column = index 3) + n_cols = len(headers) + for i, row in enumerate(rows): + if "⚠" in row[-1]: + table[i + 1, n_cols - 1].set_facecolor("#FFEBEE") + else: + table[i + 1, n_cols - 1].set_facecolor("#E8F5E9") + + risk = report.get("risk_score", None) + if risk is not None: + ax.set_title( + f"综合评估 Overall Assessment | 风险评分 Risk Score: {risk:.0f}/100", + fontsize=11, fontweight="bold", pad=10, + ) + + +# --------------------------------------------------------------------------- +# 6. Risk Score Gauge +# --------------------------------------------------------------------------- + +def plot_risk_gauge( + risk_score: float, + output_dir: str = "figures/", + filename: str = "risk_score_gauge.png", + dpi: int = 300, + label: str = "", +) -> str: + """Generate a semi-circular gauge showing overall risk score (0-100). + + Creates a visually informative gauge with color gradient from green (low risk) + through yellow/orange to red (high risk), with a needle indicating the score. + + Args: + risk_score: Overall risk score between 0 and 100. + output_dir: Directory to save the figure. + filename: Output filename. + dpi: Resolution. + label: Optional label/dataset name to display. + + Returns: + Path to the saved figure file. + """ + output_path = _ensure_output_dir(output_dir) + filepath = str(output_path / filename) + + risk_score = float(np.clip(risk_score, 0, 100)) + + fig, ax = plt.subplots(figsize=(8, 5)) + ax.set_xlim(-1.5, 1.5) + ax.set_ylim(-0.5, 1.5) + ax.set_aspect("equal") + ax.axis("off") + + # Draw gauge background arc with color gradient + n_segments = 200 + angles = np.linspace(180, 0, n_segments + 1) + + for i in range(n_segments): + frac = i / n_segments + # Color interpolation: green -> yellow -> orange -> red + if frac < 0.25: + r, g, b = 0.30, 0.69, 0.31 # Green + f = frac / 0.25 + r = r + f * (1.0 - r) + g = g + f * (0.76 - g) + b = b + f * (0.03 - b) + elif frac < 0.5: + f = (frac - 0.25) / 0.25 + r, g, b = 1.0, 0.76 - f * 0.13, 0.03 + elif frac < 0.75: + f = (frac - 0.5) / 0.25 + r, g, b = 1.0 - f * 0.04, 0.63 - f * 0.29, 0.03 + f * 0.10 + else: + f = (frac - 0.75) / 0.25 + r, g, b = 0.96 - f * 0.13, 0.34 - f * 0.15, 0.13 + f * 0.06 + + theta1 = angles[i + 1] + theta2 = angles[i] + wedge = Wedge((0, 0), 1.2, theta1, theta2, width=0.35, facecolor=(r, g, b), alpha=0.85) + ax.add_patch(wedge) + + # Inner white circle for clean look + inner_circle = plt.Circle((0, 0), 0.82, color="white", zorder=2) + ax.add_patch(inner_circle) + + # Tick marks and labels + tick_values = [0, 25, 50, 75, 100] + tick_labels = ["0\n安全", "25\n低风险", "50\n中风险", "75\n高风险", "100\n极高"] + for val, lbl in zip(tick_values, tick_labels): + angle_rad = np.radians(180 - val / 100 * 180) + # Outer tick + x_outer = 1.28 * np.cos(angle_rad) + y_outer = 1.28 * np.sin(angle_rad) + x_inner = 1.18 * np.cos(angle_rad) + y_inner = 1.18 * np.sin(angle_rad) + ax.plot([x_inner, x_outer], [y_inner, y_outer], color="#333", linewidth=1.5) + # Label + x_label = 1.42 * np.cos(angle_rad) + y_label = 1.42 * np.sin(angle_rad) + ax.text(x_label, y_label, lbl, ha="center", va="center", fontsize=7, color="#555") + + # Needle + needle_angle = np.radians(180 - (risk_score / 100) * 180) + needle_length = 0.78 + needle_x = needle_length * np.cos(needle_angle) + needle_y = needle_length * np.sin(needle_angle) + + # Needle triangle (wider base) + base_angle1 = needle_angle + np.pi / 2 + base_angle2 = needle_angle - np.pi / 2 + base_r = 0.04 + triangle = plt.Polygon([ + [needle_x, needle_y], + [base_r * np.cos(base_angle1), base_r * np.sin(base_angle1)], + [base_r * np.cos(base_angle2), base_r * np.sin(base_angle2)], + ], closed=True, facecolor="#333333", zorder=5) + ax.add_patch(triangle) + + # Center dot + center_circle = plt.Circle((0, 0), 0.06, color="#333333", zorder=6) + ax.add_patch(center_circle) + + # Score display + if risk_score >= 75: + score_color = COLORS["risk_critical"] + risk_label = "极高风险 Critical Risk" + elif risk_score >= 50: + score_color = COLORS["risk_high"] + risk_label = "高风险 High Risk" + elif risk_score >= 25: + score_color = COLORS["risk_medium"] + risk_label = "中等风险 Medium Risk" + else: + score_color = COLORS["risk_low"] + risk_label = "低风险 Low Risk" + + ax.text(0, -0.15, f"{risk_score:.0f}", fontsize=32, fontweight="bold", + ha="center", va="center", color=score_color, zorder=7) + ax.text(0, -0.35, risk_label, fontsize=11, ha="center", va="center", + color=score_color, fontweight="bold") + + # Title + title = "学术数据风险评分 Academic Data Risk Score" + if label: + title += f"\n{label}" + ax.set_title(title, fontsize=13, fontweight="bold", pad=20, y=1.0) + + return _save_figure(fig, filepath, dpi) + + +# --------------------------------------------------------------------------- +# CLI Interface +# --------------------------------------------------------------------------- + +def generate_all_figures(report: Dict[str, Any], output_dir: str = "figures/") -> Dict[str, str]: + """Generate all available figures from a complete report. + + Args: + report: Full report dict containing results from all tests. + output_dir: Directory to save all figures. + + Returns: + Dictionary mapping figure type to saved file path. + """ + figures = {} + + if "last_digit" in report: + try: + path = plot_last_digit(report["last_digit"], output_dir=output_dir) + figures["last_digit"] = path + print(f" ✓ Last digit plot: {path}") + except Exception as e: + print(f" ✗ Last digit plot failed: {e}", file=sys.stderr) + + if "benford" in report: + try: + path = plot_benford(report["benford"], output_dir=output_dir) + figures["benford"] = path + print(f" ✓ Benford plot: {path}") + except Exception as e: + print(f" ✗ Benford plot failed: {e}", file=sys.stderr) + + if "fixed_ratio" in report: + try: + path = plot_fixed_ratio(report["fixed_ratio"], output_dir=output_dir) + figures["fixed_ratio"] = path + print(f" ✓ Fixed ratio plot: {path}") + except Exception as e: + print(f" ✗ Fixed ratio plot failed: {e}", file=sys.stderr) + + if "decimal_pattern" in report: + try: + path = plot_decimal_heatmap(report["decimal_pattern"], output_dir=output_dir) + figures["decimal_heatmap"] = path + print(f" ✓ Decimal heatmap: {path}") + except Exception as e: + print(f" ✗ Decimal heatmap failed: {e}", file=sys.stderr) + + if "risk_score" in report: + try: + label = report.get("dataset_name", "") + path = plot_risk_gauge(report["risk_score"], output_dir=output_dir, label=label) + figures["risk_gauge"] = path + print(f" ✓ Risk gauge: {path}") + except Exception as e: + print(f" ✗ Risk gauge failed: {e}", file=sys.stderr) + + # Comprehensive dashboard (needs at least 2 test results) + n_tests = sum(1 for k in ["last_digit", "benford", "fixed_ratio", "decimal_pattern"] if k in report) + if n_tests >= 2: + try: + path = plot_dashboard(report, output_dir=output_dir) + figures["dashboard"] = path + print(f" ✓ Dashboard: {path}") + except Exception as e: + print(f" ✗ Dashboard failed: {e}", file=sys.stderr) + + return figures + + +def main(): + """CLI entry point for batch figure generation.""" + parser = argparse.ArgumentParser( + description="Generate publication-quality figures for academic fraud detection reports.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python3 visualization.py --input report.json --output figures/ + python3 visualization.py --input report.json --output figures/ --dpi 600 + python3 visualization.py --input report.json --type benford --output figures/ + """, + ) + parser.add_argument( + "--input", "-i", required=True, + help="Path to JSON report file (output from fraud detection pipeline)." + ) + parser.add_argument( + "--output", "-o", default="figures/", + help="Output directory for generated figures (default: figures/)." + ) + parser.add_argument( + "--dpi", type=int, default=300, + help="Figure resolution in DPI (default: 300)." + ) + parser.add_argument( + "--type", "-t", choices=["all", "last_digit", "benford", "fixed_ratio", "decimal", "gauge", "dashboard"], + default="all", + help="Type of figure to generate (default: all)." + ) + + args = parser.parse_args() + + # Load report + input_path = Path(args.input) + if not input_path.exists(): + print(f"Error: Input file not found: {args.input}", file=sys.stderr) + sys.exit(1) + + with open(input_path, "r", encoding="utf-8") as f: + report = json.load(f) + + print(f"📊 Generating figures from: {args.input}") + print(f" Output directory: {args.output}") + print(f" DPI: {args.dpi}") + print(f" Type: {args.type}") + print("-" * 50) + + # Update global DPI + plt.rcParams["savefig.dpi"] = args.dpi + + if args.type == "all": + figures = generate_all_figures(report, output_dir=args.output) + elif args.type == "last_digit" and "last_digit" in report: + path = plot_last_digit(report["last_digit"], output_dir=args.output, dpi=args.dpi) + figures = {"last_digit": path} + print(f" ✓ Last digit plot: {path}") + elif args.type == "benford" and "benford" in report: + path = plot_benford(report["benford"], output_dir=args.output, dpi=args.dpi) + figures = {"benford": path} + print(f" ✓ Benford plot: {path}") + elif args.type == "fixed_ratio" and "fixed_ratio" in report: + path = plot_fixed_ratio(report["fixed_ratio"], output_dir=args.output, dpi=args.dpi) + figures = {"fixed_ratio": path} + print(f" ✓ Fixed ratio plot: {path}") + elif args.type == "decimal" and "decimal_pattern" in report: + path = plot_decimal_heatmap(report["decimal_pattern"], output_dir=args.output, dpi=args.dpi) + figures = {"decimal_heatmap": path} + print(f" ✓ Decimal heatmap: {path}") + elif args.type == "gauge" and "risk_score" in report: + label = report.get("dataset_name", "") + path = plot_risk_gauge(report["risk_score"], output_dir=args.output, dpi=args.dpi, label=label) + figures = {"risk_gauge": path} + print(f" ✓ Risk gauge: {path}") + elif args.type == "dashboard": + path = plot_dashboard(report, output_dir=args.output, dpi=args.dpi) + figures = {"dashboard": path} + print(f" ✓ Dashboard: {path}") + else: + print(f"Warning: No data available for type '{args.type}' in the report.", file=sys.stderr) + figures = {} + + print("-" * 50) + print(f"✅ Generated {len(figures)} figure(s).") + + if figures: + # Save figure manifest + manifest_path = Path(args.output) / "figures_manifest.json" + with open(manifest_path, "w", encoding="utf-8") as f: + json.dump(figures, f, indent=2, ensure_ascii=False) + print(f"📋 Manifest saved: {manifest_path}") + + +if __name__ == "__main__": + main() diff --git a/tools/gengskill/tests/test_modules.py b/tools/gengskill/tests/test_modules.py new file mode 100644 index 0000000..d0784dc --- /dev/null +++ b/tools/gengskill/tests/test_modules.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +""" +Geng Skill 单元测试 +==================== +验证各检测模块的正确性、边界条件处理和风险评分一致性。 + +运行方式: + cd geng-skill + python3 -m pytest tests/test_modules.py -v + +或直接运行: + python3 tests/test_modules.py +""" + +import sys +import os +import json +import random +import math + +# 添加 scripts 目录到路径 +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts')) + +from last_digit_test import last_digit_test, extract_last_digit +from benford_test import benford_test, get_first_digit +from grim_test import grim_test_single, grim_test_batch +from fixed_relation_test import fixed_relation_test +from decimal_consistency_test import decimal_consistency_test + + +# ============================================================ +# Test: Last Digit Test +# ============================================================ + +class TestLastDigitTest: + """末位数字检测模块测试""" + + def test_uniform_data_low_risk(self): + """均匀分布的末位数字应返回低风险""" + random.seed(42) + # 生成末位数字均匀的数据 + values = [f"{random.uniform(1, 100):.2f}" for _ in range(100)] + result = last_digit_test(values) + assert result['status'] == 'completed' + assert result['risk_level'] == 'low' + assert result['risk_score'] < 40 + + def test_concentrated_digits_high_risk(self): + """集中在某个数字的末位应返回高风险""" + # 80% 的末位是 "5" + values = ['1.25', '2.35', '3.45', '4.55', '5.65', + '6.75', '7.85', '8.95', '9.15', '10.25', + '11.35', '12.45', '13.55', '14.65', '15.75', + '16.85', '17.95', '18.05', '19.15', '20.55'] + result = last_digit_test(values) + assert result['status'] == 'completed' + assert result['most_frequent_digit'] == 5 + + def test_insufficient_data(self): + """数据量不足应返回 insufficient_data""" + values = ['1.23', '4.56', '7.89'] + result = last_digit_test(values) + assert result['status'] == 'insufficient_data' + + def test_extract_last_digit(self): + """末位数字提取正确性""" + assert extract_last_digit('3.14') == 4 + assert extract_last_digit('100') == 1 + assert extract_last_digit('-5.67') == 7 + + +# ============================================================ +# Test: Benford's Law Test +# ============================================================ + +class TestBenfordTest: + """本福特定律检测模块测试""" + + def test_benford_compliant_data(self): + """符合本福特定律的数据应返回低风险""" + # 生成符合本福特定律的数据 + random.seed(42) + values = [] + for _ in range(200): + # 对数均匀分布产生的数据符合本福特定律 + val = 10 ** (random.uniform(0, 4)) + values.append(f"{val:.2f}") + result = benford_test(values) + assert result['status'] == 'completed' + # 对数均匀数据应近似符合 + assert result['conformity'] in ('close', 'acceptable', 'marginal') + + def test_uniform_first_digit_high_risk(self): + """首位数字均匀分布应偏离本福特定律""" + # 人为造假数据：首位数字接近均匀 + values = [] + for d in range(1, 10): + for _ in range(20): + values.append(f"{d}{random.randint(10,99)}") + result = benford_test(values) + assert result['status'] == 'completed' + # 均匀分布会显著偏离本福特定律 + assert result['p_value'] < 0.05 + + def test_insufficient_data(self): + """数据量不足""" + values = ['123', '456'] + result = benford_test(values) + assert result['status'] == 'insufficient_data' + + def test_first_digit_extraction(self): + """首位数字提取""" + assert get_first_digit('314.15') == 3 + assert get_first_digit('0.0052') == 5 + assert get_first_digit('9999') == 9 + + +# ============================================================ +# Test: GRIM Test +# ============================================================ + +class TestGrimTest: + """GRIM 测试模块测试""" + + def test_consistent_mean(self): + """合法均值应通过""" + # n=20, 整数数据, mean=3.40 → sum=68 (整数) ✓ + result = grim_test_single('3.40', 20, decimals=2) + assert result['consistent'] == True + + def test_inconsistent_mean(self): + """非法均值应失败""" + # n=20, 整数数据, mean=3.47 → sum=69.4 (非整数) ✗ + result = grim_test_single('3.47', 20, decimals=2) + assert result['consistent'] == False + + def test_consistent_mean_n25(self): + """n=25 的合法均值""" + # n=25, mean=3.48 → sum=87 (整数) ✓ + result = grim_test_single('3.48', 25, decimals=2) + assert result['consistent'] == True + + def test_batch_mode(self): + """批量 GRIM 测试""" + items = [ + {'mean': '3.40', 'n': 20, 'decimals': 2, 'label': 'Item A'}, + {'mean': '3.47', 'n': 20, 'decimals': 2, 'label': 'Item B'}, + {'mean': '4.00', 'n': 10, 'decimals': 2, 'label': 'Item C'}, + ] + result = grim_test_batch(items) + assert result['status'] == 'completed' + assert result['total_items'] == 3 + assert result['inconsistent_items'] == 1 # 3.47/20 不一致 + + def test_range_check(self): + """量表范围检查""" + # 均值超出量表范围 + result = grim_test_single('6.50', 20, decimals=2, scale_min=1, scale_max=5) + assert result.get('consistent') == False or result.get('status') == 'range_error' + + +# ============================================================ +# Test: Fixed Relation Test +# ============================================================ + +class TestFixedRelationTest: + """固定关系检测模块测试""" + + def test_exact_ratio_detected(self): + """精确固定比值应被检测到""" + col1 = [1.23, 2.34, 3.45, 4.56, 5.67, 6.78, 7.89] + col2 = [2.46, 4.68, 6.90, 9.12, 11.34, 13.56, 15.78] # ×2 + result = fixed_relation_test(col1, col2) + assert result['status'] == 'completed' + assert result['risk_level'] == 'high' + assert result['risk_score'] >= 85 + assert result['detections']['fixed_ratio']['is_exact'] == True + + def test_exact_difference_detected(self): + """精确固定差值应被检测到""" + col1 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0] + col2 = [4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0] # +3 + result = fixed_relation_test(col1, col2) + assert result['detections']['fixed_difference']['is_exact'] == True + + def test_independent_data_low_risk(self): + """独立随机数据应返回低风险""" + random.seed(42) + col1 = [random.uniform(1, 10) for _ in range(30)] + col2 = [random.uniform(1, 10) for _ in range(30)] + result = fixed_relation_test(col1, col2) + assert result['risk_level'] == 'low' + assert result['risk_score'] < 30 + + def test_insufficient_data(self): + """数据不足""" + result = fixed_relation_test([1.0, 2.0], [3.0, 4.0]) + assert result['status'] == 'insufficient_data' + + def test_unequal_lengths(self): + """长度不一致应报错""" + result = fixed_relation_test([1, 2, 3], [4, 5]) + assert result['status'] == 'error' + + +# ============================================================ +# Test: Decimal Consistency Test +# ============================================================ + +class TestDecimalConsistencyTest: + """小数位一致性检测模块测试""" + + def test_diverse_decimals_low_risk(self): + """多样化小数模式应低风险""" + random.seed(42) + values = [f"{random.uniform(1, 100):.{random.randint(1,4)}f}" for _ in range(50)] + result = decimal_consistency_test(values) + assert result['status'] == 'completed' + assert result['risk_level'] in ('low', 'medium') + + def test_repeated_decimals_high_risk(self): + """高度重复的小数模式应高风险""" + # 所有值小数部分都是 .34 + values = [f"{i}.34" for i in range(1, 31)] + result = decimal_consistency_test(values) + assert result['status'] == 'completed' + assert result['risk_score'] >= 25 # 至少中风险 + + def test_insufficient_data(self): + """数据不足""" + values = ['1.23', '4.56'] + result = decimal_consistency_test(values) + assert result['status'] == 'insufficient_data' + + +# ============================================================ +# Integration Test +# ============================================================ + +class TestIntegration: + """集成测试：模拟完整检测流程""" + + def test_fake_data_high_risk(self): + """已知造假数据应返回高风险""" + # 模拟耿同学发现的典型造假：固定比例关系 + control = [2.34, 3.12, 1.87, 4.56, 2.98, 3.45, 1.23, 5.67, 2.01, 3.89] + treatment = [x * 2 for x in control] # 精确 ×2 + + result = fixed_relation_test(control, treatment, 'Control', 'Treatment') + assert result['risk_score'] >= 85 + assert 'fixed_ratio' in result['detections'] + assert result['detections']['fixed_ratio']['mean_ratio'] == 2.0 + + def test_real_data_low_risk(self): + """正常实验数据应返回低风险""" + random.seed(123) + # 模拟真实实验：基础值 + 效应 + 随机噪声 + control = [random.gauss(5, 1.5) for _ in range(20)] + treatment = [x * random.gauss(2, 0.3) for x in control] # ×2 但有变异 + + result = fixed_relation_test(control, treatment, 'Control', 'Treatment') + # 由于有随机噪声，不应该报告"精确"固定关系 + assert result['detections']['fixed_ratio']['is_exact'] == False + + def test_output_schema_compliance(self): + """输出格式应符合标准 schema""" + values = [f"{random.uniform(1, 100):.2f}" for _ in range(50)] + result = last_digit_test(values) + + # 必须包含的标准字段 + required_fields = ['test_name', 'status', 'risk_level', 'risk_score', 'interpretation'] + for field in required_fields: + assert field in result, f"缺少必需字段: {field}" + + # 风险评分范围 + assert 0 <= result['risk_score'] <= 100 + + # 风险等级合法值 + assert result['risk_level'] in ('low', 'medium', 'medium-high', 'high') + + +# ============================================================ +# Run tests +# ============================================================ + +def run_all_tests(): + """简易测试运行器（不依赖 pytest）""" + import traceback + + test_classes = [ + TestLastDigitTest, + TestBenfordTest, + TestGrimTest, + TestFixedRelationTest, + TestDecimalConsistencyTest, + TestIntegration, + ] + + total = 0 + passed = 0 + failed = 0 + errors = [] + + print("=" * 70) + print(" 🧪 Geng Skill 单元测试") + print("=" * 70) + print() + + for test_class in test_classes: + class_name = test_class.__name__ + print(f"▶ {class_name}") + + instance = test_class() + methods = [m for m in dir(instance) if m.startswith('test_')] + + for method_name in methods: + total += 1 + try: + getattr(instance, method_name)() + passed += 1 + print(f" ✅ {method_name}") + except AssertionError as e: + failed += 1 + errors.append((class_name, method_name, str(e))) + print(f" ❌ {method_name}: {e}") + except Exception as e: + failed += 1 + errors.append((class_name, method_name, traceback.format_exc())) + print(f" 💥 {method_name}: {type(e).__name__}: {e}") + print() + + print("=" * 70) + print(f" 结果: {passed} 通过 / {failed} 失败 / {total} 总计") + print("=" * 70) + + if errors: + print("\n❌ 失败详情:") + for cls, method, err in errors: + print(f" {cls}.{method}: {err[:200]}") + + return failed == 0 + + +if __name__ == '__main__': + success = run_all_tests() + sys.exit(0 if success else 1)