mirror of
https://gitcode.com/JianFeeeee/homeagent-sdk.git
synced 2026-09-20 00:48:12 +00:00
feat: add example plugins (ai_image, calendar, music, rss, weather), fix .gitignore, move gengskill to tools/
This commit is contained in:
9
.gitignore
vendored
9
.gitignore
vendored
@ -15,8 +15,11 @@ dist/
|
||||
testdist/
|
||||
|
||||
# Logs
|
||||
*.logz_bridge_gen.go\nz_entry.c\nbuild/\ndist/
|
||||
*.log
|
||||
|
||||
# Generated bridge files
|
||||
z_bridge_gen.go
|
||||
z_entry.c
|
||||
build/
|
||||
dist/
|
||||
|
||||
# Binary
|
||||
plugindev
|
||||
|
||||
13
example/ai_image/README.md
Normal file
13
example/ai_image/README.md
Normal file
@ -0,0 +1,13 @@
|
||||
# ai_image
|
||||
|
||||
ai_image plugin
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
plugindev build
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
Upload the .hmap file through the Plugin Manager API.
|
||||
7
example/ai_image/go.mod
Normal file
7
example/ai_image/go.mod
Normal file
@ -0,0 +1,7 @@
|
||||
module ai_image
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
11
example/ai_image/plg.json
Normal file
11
example/ai_image/plg.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "ai_image",
|
||||
"name_zh": "AI绘图",
|
||||
"name_en": "AI Image",
|
||||
"version": "1.0.0",
|
||||
"description": "AI 图像生成插件,支持 OpenAI DALL·E / Stable Diffusion",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["ai", "image", "draw", "generate"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
344
example/ai_image/plugin.go
Normal file
344
example/ai_image/plugin.go
Normal file
@ -0,0 +1,344 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||
)
|
||||
|
||||
type Plugin struct {
|
||||
name string
|
||||
sdk *sdk.PluginSDK
|
||||
client *http.Client
|
||||
apiKey string
|
||||
provider string
|
||||
model string
|
||||
size string
|
||||
}
|
||||
|
||||
func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) {
|
||||
return &Plugin{name: name}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Name() string { return p.name }
|
||||
|
||||
func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T {
|
||||
v, err := s.Get(key)
|
||||
if err == nil && v != nil {
|
||||
if sv, ok := v.(string); ok && sv != "" {
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
return any(sv).(T)
|
||||
case int64:
|
||||
if n, err := strconv.ParseInt(sv, 10, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
case float64:
|
||||
if n, err := strconv.ParseFloat(sv, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
v2, err2 := s.GetCore("plugin." + "ai_image" + "." + key)
|
||||
if err2 == nil && v2 != nil {
|
||||
if sv, ok := v2.(string); ok && sv != "" {
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
return any(sv).(T)
|
||||
case int64:
|
||||
if n, err := strconv.ParseInt(sv, 10, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
case float64:
|
||||
if n, err := strconv.ParseFloat(sv, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T {
|
||||
v, ok := args[key]
|
||||
if !ok || v == nil {
|
||||
return fallback
|
||||
}
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
if s, ok := v.(string); ok {
|
||||
return any(s).(T)
|
||||
}
|
||||
case int64:
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return any(int64(n)).(T)
|
||||
case int64:
|
||||
return any(n).(T)
|
||||
case string:
|
||||
if i, err := strconv.ParseInt(n, 10, 64); err == nil {
|
||||
return any(i).(T)
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return any(n).(T)
|
||||
case int64:
|
||||
return any(float64(n)).(T)
|
||||
case string:
|
||||
if f, err := strconv.ParseFloat(n, 64); err == nil {
|
||||
return any(f).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
p.sdk = s
|
||||
|
||||
p.apiKey = readCfg(s.Settings(), "api_key", "")
|
||||
p.provider = readCfg(s.Settings(), "provider", "openai")
|
||||
p.model = readCfg(s.Settings(), "model", "dall-e-3")
|
||||
p.size = readCfg(s.Settings(), "size", "1024x1024")
|
||||
|
||||
p.client = &http.Client{Timeout: 120 * time.Second}
|
||||
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.ai_image.api_key", Default: "", Type: "string",
|
||||
DisplayName: "API Key", Description: "OpenAI / Stable Diffusion API Key",
|
||||
Category: "ai_image", Secret: true,
|
||||
})
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.ai_image.provider", Default: "openai", Type: "string",
|
||||
DisplayName: "Provider", Description: "Image generation provider: openai / stability",
|
||||
Category: "ai_image",
|
||||
})
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.ai_image.model", Default: "dall-e-3", Type: "string",
|
||||
DisplayName: "Model", Description: "Model name (dall-e-3, sd-xl, etc.)",
|
||||
Category: "ai_image",
|
||||
})
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.ai_image.size", Default: "1024x1024", Type: "string",
|
||||
DisplayName: "Size", Description: "Default image size (1024x1024, 1024x1792, 1792x1024)",
|
||||
Category: "ai_image",
|
||||
})
|
||||
|
||||
tp := p.name + "_"
|
||||
s.RegisterTool(tp+"generate", sdk.ToolDef{
|
||||
Name: tp + "generate", Description: "Generate image from text prompt using AI. Returns image URL.",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"prompt": map[string]interface{}{"type": "string", "description": "Text description of the image to generate"},
|
||||
"size": map[string]interface{}{"type": "string", "description": "Image size (1024x1024, 1024x1792, 1792x1024), default from config"},
|
||||
"model": map[string]interface{}{"type": "string", "description": "Model override (dall-e-3, dall-e-2)"},
|
||||
"n": map[string]interface{}{"type": "integer", "description": "Number of images to generate (1-10), default 1"},
|
||||
},
|
||||
"required": []string{"prompt"},
|
||||
},
|
||||
}, p.handleGenerate)
|
||||
|
||||
fmt.Printf("[%s] started (provider=%s, model=%s)\n", p.name, p.provider, p.model)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Stop() error {
|
||||
fmt.Printf("[%s] stopped\n", p.name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type openAIReq struct {
|
||||
Model string `json:"model"`
|
||||
Prompt string `json:"prompt"`
|
||||
N int `json:"n"`
|
||||
Size string `json:"size"`
|
||||
ResponseFormat string `json:"response_format"`
|
||||
}
|
||||
|
||||
type openAIResp struct {
|
||||
Created int64 `json:"created"`
|
||||
Data []struct {
|
||||
RevisedPrompt string `json:"revised_prompt"`
|
||||
URL string `json:"url"`
|
||||
} `json:"data"`
|
||||
Error *struct {
|
||||
Message string `json:"message"`
|
||||
Type string `json:"type"`
|
||||
} `json:"error"`
|
||||
}
|
||||
|
||||
func (p *Plugin) handleGenerate(args map[string]interface{}) (interface{}, error) {
|
||||
prompt := readArg(args, "prompt", "")
|
||||
if prompt == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "prompt is required"}, nil
|
||||
}
|
||||
|
||||
p.apiKey = readCfg(p.sdk.Settings(), "api_key", p.apiKey)
|
||||
if p.apiKey == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "API key not configured. Set plugin.ai_image.api_key via CLI."}, nil
|
||||
}
|
||||
|
||||
provider := readCfg(p.sdk.Settings(), "provider", p.provider)
|
||||
model := readArg(args, "model", readCfg(p.sdk.Settings(), "model", p.model))
|
||||
size := readArg(args, "size", readCfg(p.sdk.Settings(), "size", p.size))
|
||||
n := readArg(args, "n", int64(1))
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
if n > 10 {
|
||||
n = 10
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case "openai":
|
||||
return p.generateOpenAI(prompt, model, size, int(n))
|
||||
case "stability":
|
||||
return p.generateStability(prompt, model, size, int(n))
|
||||
default:
|
||||
return map[string]interface{}{"isError": true, "content": "Unknown provider: " + provider + ". Supported: openai, stability"}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Plugin) generateOpenAI(prompt, model, size string, n int) (interface{}, error) {
|
||||
body := openAIReq{
|
||||
Model: model,
|
||||
Prompt: prompt,
|
||||
N: n,
|
||||
Size: size,
|
||||
ResponseFormat: "url",
|
||||
}
|
||||
|
||||
b, _ := json.Marshal(body)
|
||||
req, _ := http.NewRequest("POST", "https://api.openai.com/v1/images/generations", bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Request failed: " + err.Error()}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
var result openAIResp
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Failed to parse response: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
if result.Error != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "API error: " + result.Error.Message}, nil
|
||||
}
|
||||
|
||||
if len(result.Data) == 0 {
|
||||
return map[string]interface{}{"isError": true, "content": "No images returned"}, nil
|
||||
}
|
||||
|
||||
urls := make([]string, len(result.Data))
|
||||
for i, d := range result.Data {
|
||||
urls[i] = d.URL
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("Generated %d image(s) with model %s:\n%s", len(urls), model, strings.Join(urls, "\n")),
|
||||
"images": urls,
|
||||
"prompt": prompt,
|
||||
"model": model,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type stabilityReq struct {
|
||||
TextPrompts []stabilityPrompt `json:"text_prompts"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Samples int `json:"samples"`
|
||||
}
|
||||
|
||||
type stabilityPrompt struct {
|
||||
Text string `json:"text"`
|
||||
Weight float64 `json:"weight,omitempty"`
|
||||
}
|
||||
|
||||
type stabilityArtifact struct {
|
||||
Base64 string `json:"base64"`
|
||||
Seed int `json:"seed"`
|
||||
}
|
||||
|
||||
type stabilityResp struct {
|
||||
Artifacts []stabilityArtifact `json:"artifacts"`
|
||||
Message string `json:"message,omitempty"`
|
||||
}
|
||||
|
||||
func (p *Plugin) generateStability(prompt, model, size string, n int) (interface{}, error) {
|
||||
width, height := 1024, 1024
|
||||
if parts := strings.Split(size, "x"); len(parts) == 2 {
|
||||
if w, err := strconv.Atoi(parts[0]); err == nil {
|
||||
width = w
|
||||
}
|
||||
if h, err := strconv.Atoi(parts[1]); err == nil {
|
||||
height = h
|
||||
}
|
||||
}
|
||||
|
||||
body := stabilityReq{
|
||||
TextPrompts: []stabilityPrompt{{Text: prompt, Weight: 1.0}},
|
||||
Width: width,
|
||||
Height: height,
|
||||
Samples: n,
|
||||
}
|
||||
|
||||
apiURL := "https://api.stability.ai/v1/generation/stable-diffusion-xl-1024-v1-0/text-to-image"
|
||||
|
||||
b, _ := json.Marshal(body)
|
||||
req, _ := http.NewRequest("POST", apiURL, bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Request failed: " + err.Error()}, nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return map[string]interface{}{"isError": true, "content": fmt.Sprintf("API error (status %d): %s", resp.StatusCode, string(respBody))}, nil
|
||||
}
|
||||
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
var result stabilityResp
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Failed to parse response: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
if len(result.Artifacts) == 0 {
|
||||
msg := result.Message
|
||||
if msg == "" {
|
||||
msg = "No images returned"
|
||||
}
|
||||
return map[string]interface{}{"isError": true, "content": msg}, nil
|
||||
}
|
||||
|
||||
urls := make([]string, len(result.Artifacts))
|
||||
for i, a := range result.Artifacts {
|
||||
urls[i] = "data:image/png;base64," + a.Base64
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("Generated %d image(s) via Stability AI:\n%s", len(urls), strings.Join(urls, "\n")),
|
||||
"images": urls,
|
||||
"prompt": prompt,
|
||||
"model": model,
|
||||
}, nil
|
||||
}
|
||||
@ -1,7 +1,21 @@
|
||||
module browser
|
||||
module browser-plugin
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0-20260708004841-e9bdcf9304b0
|
||||
require (
|
||||
gitcode.com/JianFeeeee/homeagent-sdk v0.0.0
|
||||
github.com/chromedp/chromedp v0.9.5
|
||||
)
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => ../../.
|
||||
require (
|
||||
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect
|
||||
github.com/chromedp/sysutil v1.0.0 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.3.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
)
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
|
||||
23
example/browser/go.sum
Normal file
23
example/browser/go.sum
Normal file
@ -0,0 +1,23 @@
|
||||
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 h1:XYUCaZrW8ckGWlCRJKCSoh/iFwlpX316a8yY9IFEzv8=
|
||||
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||
github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg=
|
||||
github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y=
|
||||
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
|
||||
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q=
|
||||
github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "browser",
|
||||
"name_zh": "浏览器",
|
||||
"name_en": "browser",
|
||||
"version": "1.0.0",
|
||||
"description": "网络资源搜索与获取:搜索引擎查询(browser_search)、网页抓取(browser_fetch,SSRF防护)、无头浏览器渲染(browser_render)",
|
||||
"name_en": "Browser",
|
||||
"version": "2.0.0",
|
||||
"description": "统一浏览器插件:搜索、HTTP抓取(quick)、无头渲染(normal)、交互式浏览器(interactive/CDP)",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["web", "search", "fetch", "browser"],
|
||||
"tags": ["web", "search", "fetch", "browser", "cdp"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
13
example/calendar/README.md
Normal file
13
example/calendar/README.md
Normal file
@ -0,0 +1,13 @@
|
||||
# calendar
|
||||
|
||||
calendar plugin
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
plugindev build
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
Upload the .hmap file through the Plugin Manager API.
|
||||
7
example/calendar/go.mod
Normal file
7
example/calendar/go.mod
Normal file
@ -0,0 +1,7 @@
|
||||
module calendar
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
11
example/calendar/plg.json
Normal file
11
example/calendar/plg.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "calendar",
|
||||
"name_zh": "日历",
|
||||
"name_en": "Calendar",
|
||||
"version": "1.0.0",
|
||||
"description": "日历事件管理,支持提醒和重复事件",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["calendar", "event", "reminder", "schedule"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
1140
example/calendar/plugin.go
Normal file
1140
example/calendar/plugin.go
Normal file
File diff suppressed because it is too large
Load Diff
7
example/music/go.mod
Normal file
7
example/music/go.mod
Normal file
@ -0,0 +1,7 @@
|
||||
module music
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.7.1
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
11
example/music/plg.json
Normal file
11
example/music/plg.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "music",
|
||||
"name_zh": "音乐搜索",
|
||||
"name_en": "Music Search",
|
||||
"version": "0.1.0",
|
||||
"description": "音乐搜索插件,支持搜索歌曲和查看歌词(基于网易云音乐)",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["music", "song", "lyrics", "网易云"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
320
example/music/plugin.go
Normal file
320
example/music/plugin.go
Normal file
@ -0,0 +1,320 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||
)
|
||||
|
||||
type Plugin struct {
|
||||
name string
|
||||
sdk *sdk.PluginSDK
|
||||
cli *http.Client
|
||||
}
|
||||
|
||||
type searchResp struct {
|
||||
Result *struct {
|
||||
Songs []songItem `json:"songs"`
|
||||
SongCount int `json:"songCount"`
|
||||
} `json:"result"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
type songItem struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Artists []artist `json:"artists"`
|
||||
Album albumInfo `json:"album"`
|
||||
Duration int `json:"duration"`
|
||||
Mvid int `json:"mvid"`
|
||||
Fee int `json:"fee"`
|
||||
}
|
||||
|
||||
type artist struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type albumInfo struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type lyricResp struct {
|
||||
Lrc *lyricData `json:"lrc"`
|
||||
TLrc *lyricData `json:"tlyric"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
type lyricData struct {
|
||||
Lyric string `json:"lyric"`
|
||||
}
|
||||
|
||||
func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) {
|
||||
return &Plugin{name: name}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Name() string { return p.name }
|
||||
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
s.SetAutoRestart(true)
|
||||
p.sdk = s
|
||||
p.cli = &http.Client{Timeout: 15 * time.Second}
|
||||
|
||||
s.RegisterTool(p.name+"_search", sdk.ToolDef{
|
||||
Name: p.name + "_search",
|
||||
Description: "搜索歌曲,通过关键词查找音乐,返回歌曲列表",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"keyword": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "搜索关键词,如歌曲名、歌手名",
|
||||
},
|
||||
"limit": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "返回结果数量(1-50),默认10",
|
||||
},
|
||||
},
|
||||
"required": []string{"keyword"},
|
||||
},
|
||||
}, p.handleSearch)
|
||||
|
||||
s.RegisterTool(p.name+"_lyrics", sdk.ToolDef{
|
||||
Name: p.name + "_lyrics",
|
||||
Description: "获取歌曲歌词,通过歌曲ID查看歌词内容",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"song_id": map[string]interface{}{
|
||||
"type": "integer",
|
||||
"description": "歌曲ID(从搜索结果的 id 字段获取)",
|
||||
},
|
||||
},
|
||||
"required": []string{"song_id"},
|
||||
},
|
||||
}, p.handleLyrics)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Stop() error { return nil }
|
||||
|
||||
func (p *Plugin) neRequest(path string, params map[string]string) ([]byte, error) {
|
||||
base := "https://music.163.com/api" + path
|
||||
reqURL := base + "?" + urlValues(params).Encode()
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
req.Header.Set("Referer", "https://music.163.com/")
|
||||
resp, err := p.cli.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func urlValues(m map[string]string) url.Values {
|
||||
v := url.Values{}
|
||||
for k, val := range m {
|
||||
v.Set(k, val)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (p *Plugin) handleSearch(args map[string]interface{}) (interface{}, error) {
|
||||
keyword, _ := args["keyword"].(string)
|
||||
keyword = strings.TrimSpace(keyword)
|
||||
if keyword == "" {
|
||||
return map[string]interface{}{
|
||||
"content": "请输入搜索关键词",
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
limit := 10
|
||||
if v, ok := args["limit"].(float64); ok {
|
||||
limit = int(v)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 50 {
|
||||
limit = 50
|
||||
}
|
||||
}
|
||||
|
||||
body, err := p.neRequest("/search/get", map[string]string{
|
||||
"s": keyword,
|
||||
"type": "1",
|
||||
"limit": fmt.Sprint(limit),
|
||||
})
|
||||
if err != nil {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("搜索失败:%v", err),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var resp searchResp
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("解析响应失败:%v", err),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if resp.Code != 200 || resp.Result == nil {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("搜索失败,响应码:%d", resp.Code),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
songs := resp.Result.Songs
|
||||
if len(songs) == 0 {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("未找到与「%s」相关的歌曲", keyword),
|
||||
}, nil
|
||||
}
|
||||
|
||||
var lines []string
|
||||
lines = append(lines, fmt.Sprintf("找到 %d 首与「%s」相关的歌曲:\n", resp.Result.SongCount, keyword))
|
||||
for i, s := range songs {
|
||||
var artists []string
|
||||
for _, a := range s.Artists {
|
||||
artists = append(artists, a.Name)
|
||||
}
|
||||
dur := time.Duration(s.Duration) * time.Millisecond
|
||||
minutes := int(dur.Minutes())
|
||||
seconds := int(dur.Seconds()) % 60
|
||||
lines = append(lines, fmt.Sprintf("%d. %s - %s [%02d:%02d] (ID: %d)",
|
||||
i+1, s.Name, strings.Join(artists, "/"), minutes, seconds, s.ID))
|
||||
}
|
||||
|
||||
type songResult struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Artists []string `json:"artists"`
|
||||
Album string `json:"album"`
|
||||
Duration int `json:"duration"`
|
||||
}
|
||||
|
||||
var results []songResult
|
||||
for _, s := range songs {
|
||||
var artists []string
|
||||
for _, a := range s.Artists {
|
||||
artists = append(artists, a.Name)
|
||||
}
|
||||
results = append(results, songResult{
|
||||
ID: s.ID,
|
||||
Name: s.Name,
|
||||
Artists: artists,
|
||||
Album: s.Album.Name,
|
||||
Duration: s.Duration,
|
||||
})
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": strings.Join(lines, "\n"),
|
||||
"songs": results,
|
||||
"total": resp.Result.SongCount,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleLyrics(args map[string]interface{}) (interface{}, error) {
|
||||
songID, ok := args["song_id"].(float64)
|
||||
if !ok {
|
||||
return map[string]interface{}{
|
||||
"content": "请提供有效的歌曲ID",
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
id := int64(songID)
|
||||
body, err := p.neRequest("/song/lyric", map[string]string{
|
||||
"id": fmt.Sprint(id),
|
||||
"lv": "-1",
|
||||
"kv": "-1",
|
||||
"tv": "-1",
|
||||
})
|
||||
if err != nil {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("获取歌词失败:%v", err),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var resp lyricResp
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("解析歌词失败:%v", err),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
if resp.Code != 200 {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("获取歌词失败,响应码:%d", resp.Code),
|
||||
"isError": true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
lyric := ""
|
||||
if resp.Lrc != nil {
|
||||
lyric = resp.Lrc.Lyric
|
||||
}
|
||||
|
||||
if lyric == "" {
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("歌曲 %d 暂无歌词", id),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Clean up lyrics metadata lines and limit length
|
||||
lyric = cleanLyrics(lyric)
|
||||
if len(lyric) > 3000 {
|
||||
lyric = lyric[:3000] + "\n...(歌词过长已截断)"
|
||||
}
|
||||
|
||||
tLyric := ""
|
||||
if resp.TLrc != nil && resp.TLrc.Lyric != "" {
|
||||
tLyric = cleanLyrics(resp.TLrc.Lyric)
|
||||
if len(tLyric) > 1000 {
|
||||
tLyric = tLyric[:1000] + "\n...(翻译过长已截断)"
|
||||
}
|
||||
}
|
||||
|
||||
result := fmt.Sprintf("歌词:\n%s", lyric)
|
||||
if tLyric != "" {
|
||||
result += fmt.Sprintf("\n翻译:\n%s", tLyric)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": result,
|
||||
"lyric": lyric,
|
||||
"tlyric": tLyric,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanLyrics(l string) string {
|
||||
lines := strings.Split(l, "\n")
|
||||
var cleaned []string
|
||||
for _, line := range lines {
|
||||
// Skip metadata lines like [ti:...], [ar:...], [al:...], [by:...], [offset:...]
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, line)
|
||||
}
|
||||
return strings.Join(cleaned, "\n")
|
||||
}
|
||||
@ -808,7 +808,7 @@ func (p *Plugin) handleGetMessage(args map[string]interface{}) (interface{}, err
|
||||
"group_id": found.GroupID,
|
||||
"nickname": found.Nickname,
|
||||
"message_type": found.MessageType,
|
||||
"time": time.Unix(found.Time, 0).Format("15:04:05"),
|
||||
"time": time.Unix(found.Time, 0).Format("2006-01-02 15:04:05"),
|
||||
}
|
||||
if found.RawText != "" {
|
||||
result["raw_text"] = found.RawText
|
||||
@ -1103,7 +1103,7 @@ func (p *Plugin) handleGetHistory(args map[string]interface{}) (interface{}, err
|
||||
}
|
||||
ts := ""
|
||||
if t, ok := msg["time"].(float64); ok {
|
||||
ts = time.Unix(int64(t), 0).Format("15:04")
|
||||
ts = time.Unix(int64(t), 0).Format("2006-01-02 15:04")
|
||||
}
|
||||
line := msgText
|
||||
if sender != "" {
|
||||
|
||||
13
example/rss/README.md
Normal file
13
example/rss/README.md
Normal file
@ -0,0 +1,13 @@
|
||||
# rss
|
||||
|
||||
rss plugin
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
plugindev build
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
Upload the .hmap file through the Plugin Manager API.
|
||||
14
example/rss/go.mod
Normal file
14
example/rss/go.mod
Normal file
@ -0,0 +1,14 @@
|
||||
module rss
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0
|
||||
|
||||
require (
|
||||
github.com/mmcdole/gofeed v1.4.0 // indirect
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/text v0.38.0 // indirect
|
||||
)
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
8
example/rss/go.sum
Normal file
8
example/rss/go.sum
Normal file
@ -0,0 +1,8 @@
|
||||
github.com/mmcdole/gofeed v1.4.0 h1:+efDmI/yJXJgTfa8we5zg9GAKsU+2d7tnpt9QZwvjLQ=
|
||||
github.com/mmcdole/gofeed v1.4.0/go.mod h1:ngV5MTB7UJko6fH3/fG5AkB/ABUGK1ZTePF9iRhzu/c=
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ=
|
||||
github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
|
||||
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
|
||||
11
example/rss/plg.json
Normal file
11
example/rss/plg.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "rss",
|
||||
"name_zh": "RSS订阅",
|
||||
"name_en": "RSS",
|
||||
"version": "1.0.0",
|
||||
"description": "RSS/Atom 订阅监控插件,自动检测更新并推送通知",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["rss", "feed", "subscription", "monitor"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
457
example/rss/plugin.go
Normal file
457
example/rss/plugin.go
Normal file
@ -0,0 +1,457 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
type FeedSub struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
AddedAt string `json:"added_at"`
|
||||
Interval int `json:"interval"`
|
||||
}
|
||||
|
||||
type Plugin struct {
|
||||
name string
|
||||
sdk *sdk.PluginSDK
|
||||
client *http.Client
|
||||
fp *gofeed.Parser
|
||||
dataDir string
|
||||
mu sync.RWMutex
|
||||
feeds []FeedSub
|
||||
seenGUIDs map[string]bool
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
pollTicker *time.Ticker
|
||||
}
|
||||
|
||||
func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) {
|
||||
return &Plugin{name: name}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Name() string { return p.name }
|
||||
|
||||
func readCfg[T string | int64 | float64](s sdk.SettingsAPI, key string, fallback T) T {
|
||||
v, err := s.Get(key)
|
||||
if err == nil && v != nil {
|
||||
if sv, ok := v.(string); ok && sv != "" {
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
return any(sv).(T)
|
||||
case int64:
|
||||
if n, err := strconv.ParseInt(sv, 10, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
case float64:
|
||||
if n, err := strconv.ParseFloat(sv, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
v2, err2 := s.GetCore("plugin." + "rss" + "." + key)
|
||||
if err2 == nil && v2 != nil {
|
||||
if sv, ok := v2.(string); ok && sv != "" {
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
return any(sv).(T)
|
||||
case int64:
|
||||
if n, err := strconv.ParseInt(sv, 10, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
case float64:
|
||||
if n, err := strconv.ParseFloat(sv, 64); err == nil {
|
||||
return any(n).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func readArg[T string | int64 | float64](args map[string]interface{}, key string, fallback T) T {
|
||||
v, ok := args[key]
|
||||
if !ok || v == nil {
|
||||
return fallback
|
||||
}
|
||||
switch any(fallback).(type) {
|
||||
case string:
|
||||
if s, ok := v.(string); ok {
|
||||
return any(s).(T)
|
||||
}
|
||||
case int64:
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return any(int64(n)).(T)
|
||||
case int64:
|
||||
return any(n).(T)
|
||||
case string:
|
||||
if i, err := strconv.ParseInt(n, 10, 64); err == nil {
|
||||
return any(i).(T)
|
||||
}
|
||||
}
|
||||
case float64:
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return any(n).(T)
|
||||
case int64:
|
||||
return any(float64(n)).(T)
|
||||
case string:
|
||||
if f, err := strconv.ParseFloat(n, 64); err == nil {
|
||||
return any(f).(T)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
p.sdk = s
|
||||
p.client = &http.Client{Timeout: 30 * time.Second}
|
||||
p.fp = gofeed.NewParser()
|
||||
p.stopCh = make(chan struct{})
|
||||
p.seenGUIDs = make(map[string]bool)
|
||||
p.feeds = []FeedSub{}
|
||||
|
||||
dataHome := os.Getenv("HOME")
|
||||
if dataHome == "" {
|
||||
dataHome = "/tmp"
|
||||
}
|
||||
p.dataDir = filepath.Join(dataHome, ".homeagent", "rss")
|
||||
os.MkdirAll(p.dataDir, 0755)
|
||||
p.loadData()
|
||||
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.rss.poll_interval", Default: "30", Type: "string",
|
||||
DisplayName: "Poll Interval", Description: "Default polling interval in minutes (default: 30)",
|
||||
Category: "rss",
|
||||
})
|
||||
|
||||
tp := p.name + "_"
|
||||
s.RegisterTool(tp+"subscribe", sdk.ToolDef{
|
||||
Name: tp + "subscribe", Description: "Subscribe to an RSS/Atom feed URL",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"url": map[string]interface{}{"type": "string", "description": "Feed URL"},
|
||||
"interval": map[string]interface{}{"type": "integer", "description": "Poll interval in minutes (default: 30, minimum: 5)"},
|
||||
},
|
||||
"required": []string{"url"},
|
||||
},
|
||||
}, p.handleSubscribe)
|
||||
|
||||
s.RegisterTool(tp+"unsubscribe", sdk.ToolDef{
|
||||
Name: tp + "unsubscribe", Description: "Unsubscribe from a feed",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"url": map[string]interface{}{"type": "string", "description": "Feed URL to unsubscribe"},
|
||||
},
|
||||
"required": []string{"url"},
|
||||
},
|
||||
}, p.handleUnsubscribe)
|
||||
|
||||
s.RegisterTool(tp+"list", sdk.ToolDef{
|
||||
Name: tp + "list", Description: "List all subscribed feeds",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{},
|
||||
},
|
||||
}, p.handleList)
|
||||
|
||||
s.RegisterTool(tp+"check_now", sdk.ToolDef{
|
||||
Name: tp + "check_now", Description: "Manually check all feeds for new articles now",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{},
|
||||
},
|
||||
}, p.handleCheckNow)
|
||||
|
||||
pollMin := int(readCfg(s.Settings(), "poll_interval", int64(30)))
|
||||
if pollMin < 5 {
|
||||
pollMin = 5
|
||||
}
|
||||
p.pollTicker = time.NewTicker(time.Duration(pollMin) * time.Minute)
|
||||
|
||||
p.wg.Add(1)
|
||||
go p.pollLoop()
|
||||
|
||||
fmt.Printf("[%s] started (%d feeds, poll every %dm)\n", p.name, len(p.feeds), pollMin)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Stop() error {
|
||||
close(p.stopCh)
|
||||
p.pollTicker.Stop()
|
||||
p.wg.Wait()
|
||||
p.saveData()
|
||||
fmt.Printf("[%s] stopped\n", p.name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plugin) pollLoop() {
|
||||
defer p.wg.Done()
|
||||
|
||||
p.checkAllFeeds()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-p.pollTicker.C:
|
||||
p.checkAllFeeds()
|
||||
case <-p.stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Plugin) checkAllFeeds() {
|
||||
p.mu.RLock()
|
||||
feeds := make([]FeedSub, len(p.feeds))
|
||||
copy(feeds, p.feeds)
|
||||
p.mu.RUnlock()
|
||||
|
||||
for _, feed := range feeds {
|
||||
select {
|
||||
case <-p.stopCh:
|
||||
return
|
||||
default:
|
||||
}
|
||||
p.checkFeed(feed)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Plugin) checkFeed(sub FeedSub) {
|
||||
parsed, err := p.fp.ParseURL(sub.URL)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
title := parsed.Title
|
||||
if title == "" {
|
||||
title = sub.URL
|
||||
}
|
||||
|
||||
var newArticles []*gofeed.Item
|
||||
for _, item := range parsed.Items {
|
||||
guid := item.GUID
|
||||
if guid == "" {
|
||||
guid = item.Link
|
||||
}
|
||||
if guid == "" {
|
||||
continue
|
||||
}
|
||||
guid = sub.URL + "|" + guid
|
||||
p.mu.RLock()
|
||||
seen := p.seenGUIDs[guid]
|
||||
p.mu.RUnlock()
|
||||
if !seen {
|
||||
newArticles = append(newArticles, item)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newArticles) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
var lines []string
|
||||
lines = append(lines, fmt.Sprintf("📡 %s (%s) — %d 篇新文章:", title, sub.URL, len(newArticles)))
|
||||
for _, item := range newArticles {
|
||||
pubDate := ""
|
||||
if item.PublishedParsed != nil {
|
||||
pubDate = item.PublishedParsed.Format("01-02 15:04")
|
||||
}
|
||||
line := fmt.Sprintf(" • %s", item.Title)
|
||||
if pubDate != "" {
|
||||
line += fmt.Sprintf(" [%s]", pubDate)
|
||||
}
|
||||
if item.Link != "" {
|
||||
line += "\n " + item.Link
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
|
||||
p.sdk.InjectInterruptText("rss", "rss", strings.Join(lines, "\n"))
|
||||
|
||||
p.mu.Lock()
|
||||
for _, item := range newArticles {
|
||||
guid := item.GUID
|
||||
if guid == "" {
|
||||
guid = item.Link
|
||||
}
|
||||
if guid == "" {
|
||||
continue
|
||||
}
|
||||
p.seenGUIDs[sub.URL+"|"+guid] = true
|
||||
}
|
||||
p.mu.Unlock()
|
||||
p.saveData()
|
||||
}
|
||||
|
||||
func (p *Plugin) handleSubscribe(args map[string]interface{}) (interface{}, error) {
|
||||
url := readArg(args, "url", "")
|
||||
if url == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "URL is required"}, nil
|
||||
}
|
||||
|
||||
p.mu.RLock()
|
||||
for _, f := range p.feeds {
|
||||
if f.URL == url {
|
||||
p.mu.RUnlock()
|
||||
return map[string]interface{}{"isError": true, "content": "Already subscribed to: " + url}, nil
|
||||
}
|
||||
}
|
||||
p.mu.RUnlock()
|
||||
|
||||
interval := int(readArg(args, "interval", int64(30)))
|
||||
if interval < 5 {
|
||||
interval = 5
|
||||
}
|
||||
|
||||
parsed, err := p.fp.ParseURL(url)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Failed to parse feed: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
feedTitle := parsed.Title
|
||||
if feedTitle == "" {
|
||||
feedTitle = url
|
||||
}
|
||||
|
||||
sub := FeedSub{
|
||||
URL: url,
|
||||
Title: feedTitle,
|
||||
AddedAt: time.Now().Format("2006-01-02 15:04"),
|
||||
Interval: interval,
|
||||
}
|
||||
|
||||
guidCount := 0
|
||||
for _, item := range parsed.Items {
|
||||
guid := item.GUID
|
||||
if guid == "" {
|
||||
guid = item.Link
|
||||
}
|
||||
if guid == "" {
|
||||
continue
|
||||
}
|
||||
p.seenGUIDs[url+"|"+guid] = true
|
||||
guidCount++
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.feeds = append(p.feeds, sub)
|
||||
p.mu.Unlock()
|
||||
p.saveData()
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": fmt.Sprintf("Subscribed to: %s\nTitle: %s\nArticles found: %d\nPoll interval: %d min", url, feedTitle, guidCount, interval),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleUnsubscribe(args map[string]interface{}) (interface{}, error) {
|
||||
url := readArg(args, "url", "")
|
||||
if url == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "URL is required"}, nil
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
found := false
|
||||
for i, f := range p.feeds {
|
||||
if f.URL == url {
|
||||
p.feeds = append(p.feeds[:i], p.feeds[i+1:]...)
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
p.mu.Unlock()
|
||||
return map[string]interface{}{"isError": true, "content": "Not subscribed to: " + url}, nil
|
||||
}
|
||||
|
||||
for guid := range p.seenGUIDs {
|
||||
if strings.HasPrefix(guid, url+"|") {
|
||||
delete(p.seenGUIDs, guid)
|
||||
}
|
||||
}
|
||||
p.mu.Unlock()
|
||||
p.saveData()
|
||||
|
||||
return map[string]interface{}{"content": "Unsubscribed: " + url}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleList(args map[string]interface{}) (interface{}, error) {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
|
||||
if len(p.feeds) == 0 {
|
||||
return map[string]interface{}{"content": "No subscriptions. Use rss_subscribe to add one."}, nil
|
||||
}
|
||||
|
||||
sort.Slice(p.feeds, func(i, j int) bool {
|
||||
return p.feeds[i].Title < p.feeds[j].Title
|
||||
})
|
||||
|
||||
var lines []string
|
||||
lines = append(lines, fmt.Sprintf("📡 Subscriptions (%d):", len(p.feeds)))
|
||||
for _, f := range p.feeds {
|
||||
lines = append(lines, fmt.Sprintf(" • %s\n %s (every %dm, added %s)", f.Title, f.URL, f.Interval, f.AddedAt))
|
||||
}
|
||||
|
||||
return map[string]interface{}{"content": strings.Join(lines, "\n")}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleCheckNow(args map[string]interface{}) (interface{}, error) {
|
||||
go p.checkAllFeeds()
|
||||
return map[string]interface{}{"content": "Checking all feeds for updates..."}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) dataFile() string {
|
||||
return filepath.Join(p.dataDir, "feeds.json")
|
||||
}
|
||||
|
||||
func (p *Plugin) loadData() {
|
||||
b, err := os.ReadFile(p.dataFile())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var data struct {
|
||||
Feeds []FeedSub `json:"feeds"`
|
||||
SeenGUIDs map[string]bool `json:"seen"`
|
||||
}
|
||||
if json.Unmarshal(b, &data) != nil {
|
||||
return
|
||||
}
|
||||
if data.Feeds != nil {
|
||||
p.feeds = data.Feeds
|
||||
}
|
||||
if data.SeenGUIDs != nil {
|
||||
p.seenGUIDs = data.SeenGUIDs
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Plugin) saveData() {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
data := struct {
|
||||
Feeds []FeedSub `json:"feeds"`
|
||||
SeenGUIDs map[string]bool `json:"seen"`
|
||||
}{
|
||||
Feeds: p.feeds,
|
||||
SeenGUIDs: p.seenGUIDs,
|
||||
}
|
||||
b, _ := json.MarshalIndent(data, "", " ")
|
||||
os.WriteFile(p.dataFile(), b, 0644)
|
||||
}
|
||||
|
||||
|
||||
13
example/weather/README.md
Normal file
13
example/weather/README.md
Normal file
@ -0,0 +1,13 @@
|
||||
# weather
|
||||
|
||||
weather plugin
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
plugindev build
|
||||
```
|
||||
|
||||
## Install
|
||||
|
||||
Upload the .hmap file through the Plugin Manager API.
|
||||
7
example/weather/go.mod
Normal file
7
example/weather/go.mod
Normal file
@ -0,0 +1,7 @@
|
||||
module weather
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require gitcode.com/JianFeeeee/homeagent-sdk v0.0.0
|
||||
|
||||
replace gitcode.com/JianFeeeee/homeagent-sdk => /tmp/opencode/sdk-clone
|
||||
11
example/weather/plg.json
Normal file
11
example/weather/plg.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "weather",
|
||||
"name_zh": "天气查询",
|
||||
"name_en": "Weather",
|
||||
"version": "1.0.0",
|
||||
"description": "天气查询插件(基于 wttr.in),支持实时天气和未来预报",
|
||||
"author": "HomeAgent",
|
||||
"entry": "plugin.so",
|
||||
"tags": ["weather", "forecast", "wttr"],
|
||||
"targets": "linux/amd64"
|
||||
}
|
||||
388
example/weather/plugin.go
Normal file
388
example/weather/plugin.go
Normal file
@ -0,0 +1,388 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sdk "gitcode.com/JianFeeeee/homeagent-sdk/sdk"
|
||||
)
|
||||
|
||||
type Plugin struct {
|
||||
name string
|
||||
sdk *sdk.PluginSDK
|
||||
client *http.Client
|
||||
defaultLoc string
|
||||
}
|
||||
|
||||
func NewPlugin(name string, config map[string]interface{}) (sdk.Plugin, error) {
|
||||
return &Plugin{name: name}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Name() string { return p.name }
|
||||
|
||||
func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
s.SetAutoRestart(true)
|
||||
p.sdk = s
|
||||
p.client = &http.Client{Timeout: 15 * time.Second}
|
||||
|
||||
loc, err := s.Settings().Get("default_location")
|
||||
if err == nil && loc != nil {
|
||||
if v, ok := loc.(string); ok && v != "" {
|
||||
p.defaultLoc = v
|
||||
}
|
||||
}
|
||||
if p.defaultLoc == "" {
|
||||
v, err := s.Settings().GetCore("plugin.weather.default_location")
|
||||
if err == nil && v != nil {
|
||||
if vs, ok := v.(string); ok && vs != "" {
|
||||
p.defaultLoc = vs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dataHome := os.Getenv("HOME")
|
||||
if dataHome == "" {
|
||||
dataHome = "/tmp"
|
||||
}
|
||||
dataDir := filepath.Join(dataHome, ".homeagent", "weather")
|
||||
os.MkdirAll(dataDir, 0755)
|
||||
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "plugin.weather.default_location", Default: "", Type: "string",
|
||||
DisplayName: "Default Location", Description: "Default city name for weather queries, e.g. Beijing",
|
||||
Category: "weather",
|
||||
})
|
||||
|
||||
tp := p.name + "_"
|
||||
s.RegisterTool(tp+"current", sdk.ToolDef{
|
||||
Name: tp + "current", Description: "Get current weather for a city",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"location": map[string]interface{}{"type": "string", "description": "City name (e.g. Beijing, Shanghai, London). Uses default if omitted."},
|
||||
"units": map[string]interface{}{"type": "string", "description": "Units: metric (celsius) or imperial (fahrenheit), default metric"},
|
||||
},
|
||||
},
|
||||
}, p.handleCurrent)
|
||||
|
||||
s.RegisterTool(tp+"forecast", sdk.ToolDef{
|
||||
Name: tp + "forecast", Description: "Get weather forecast for next several days",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"location": map[string]interface{}{"type": "string", "description": "City name. Uses default if omitted."},
|
||||
"days": map[string]interface{}{"type": "integer", "description": "Number of days (1-7), default 3"},
|
||||
"units": map[string]interface{}{"type": "string", "description": "Units: metric or imperial, default metric"},
|
||||
},
|
||||
},
|
||||
}, p.handleForecast)
|
||||
|
||||
s.RegisterTool(tp+"set_location", sdk.ToolDef{
|
||||
Name: tp + "set_location", Description: "Set default weather location",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"location": map[string]interface{}{"type": "string", "description": "City name to set as default"},
|
||||
},
|
||||
"required": []string{"location"},
|
||||
},
|
||||
}, p.handleSetLocation)
|
||||
|
||||
fmt.Printf("[%s] started\n", p.name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Plugin) Stop() error {
|
||||
fmt.Printf("[%s] stopped\n", p.name)
|
||||
return nil
|
||||
}
|
||||
|
||||
type wttrResp struct {
|
||||
CurrentCondition []struct {
|
||||
TempC string `json:"temp_C"`
|
||||
FeelsLikeC string `json:"FeelsLikeC"`
|
||||
Humidity string `json:"humidity"`
|
||||
WindspeedKmph string `json:"windspeedKmph"`
|
||||
Winddir16Point string `json:"winddir16Point"`
|
||||
Pressure string `json:"pressure"`
|
||||
Visibility string `json:"visibility"`
|
||||
WeatherDesc []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"weatherDesc"`
|
||||
LocalObsDateTime string `json:"localObsDateTime"`
|
||||
} `json:"current_condition"`
|
||||
NearestArea []struct {
|
||||
AreaName []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"areaName"`
|
||||
Country []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"country"`
|
||||
Region []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"region"`
|
||||
} `json:"nearest_area"`
|
||||
Weather []wttrDay `json:"weather"`
|
||||
}
|
||||
|
||||
type wttrDay struct {
|
||||
Date string `json:"date"`
|
||||
Astronomy []struct {
|
||||
Sunrise string `json:"sunrise"`
|
||||
Sunset string `json:"sunset"`
|
||||
} `json:"astronomy"`
|
||||
MaxtempC string `json:"maxtempC"`
|
||||
MintempC string `json:"mintempC"`
|
||||
Hourly []struct {
|
||||
TempC string `json:"tempC"`
|
||||
WeatherDesc []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"weatherDesc"`
|
||||
WindspeedKmph string `json:"windspeedKmph"`
|
||||
Winddir16Point string `json:"winddir16Point"`
|
||||
Humidity string `json:"humidity"`
|
||||
FeelsLikeC string `json:"FeelsLikeC"`
|
||||
PrecipMM string `json:"precipMM"`
|
||||
Visibility string `json:"visibility"`
|
||||
} `json:"hourly"`
|
||||
}
|
||||
|
||||
func (p *Plugin) getLoc(args map[string]interface{}) string {
|
||||
if v, ok := args["location"].(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
return p.defaultLoc
|
||||
}
|
||||
|
||||
func (p *Plugin) getUnits(args map[string]interface{}) string {
|
||||
if v, ok := args["units"].(string); ok && (v == "imperial" || v == "metric") {
|
||||
return v
|
||||
}
|
||||
return "metric"
|
||||
}
|
||||
|
||||
func (p *Plugin) fetchWttr(location string) (*wttrResp, error) {
|
||||
url := fmt.Sprintf("https://wttr.in/%s?format=j1", strings.ReplaceAll(location, " ", "%20"))
|
||||
resp, err := p.client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var data wttrResp
|
||||
if err := json.Unmarshal(body, &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data.CurrentCondition) == 0 {
|
||||
return nil, fmt.Errorf("no weather data for: %s", location)
|
||||
}
|
||||
return &data, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) displayName(data *wttrResp) string {
|
||||
if len(data.NearestArea) == 0 {
|
||||
return "Unknown"
|
||||
}
|
||||
area := data.NearestArea[0]
|
||||
name := ""
|
||||
if len(area.AreaName) > 0 {
|
||||
name = area.AreaName[0].Value
|
||||
}
|
||||
region := ""
|
||||
if len(area.Region) > 0 {
|
||||
region = area.Region[0].Value
|
||||
}
|
||||
country := ""
|
||||
if len(area.Country) > 0 {
|
||||
country = area.Country[0].Value
|
||||
}
|
||||
var parts []string
|
||||
if name != "" {
|
||||
parts = append(parts, name)
|
||||
}
|
||||
if region != "" && region != name {
|
||||
parts = append(parts, region)
|
||||
}
|
||||
if country != "" {
|
||||
parts = append(parts, country)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func convertCtoF(c string) string {
|
||||
if v, err := strconv.ParseFloat(c, 64); err == nil {
|
||||
return fmt.Sprintf("%.0f", v*9/5+32)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (p *Plugin) handleCurrent(args map[string]interface{}) (interface{}, error) {
|
||||
location := p.getLoc(args)
|
||||
if location == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "No location specified. Provide a city name or set default_location."}, nil
|
||||
}
|
||||
|
||||
units := p.getUnits(args)
|
||||
|
||||
data, err := p.fetchWttr(location)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Weather request failed: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
cc := data.CurrentCondition[0]
|
||||
place := p.displayName(data)
|
||||
|
||||
desc := ""
|
||||
if len(cc.WeatherDesc) > 0 {
|
||||
desc = cc.WeatherDesc[0].Value
|
||||
}
|
||||
|
||||
unitStr := "°C"
|
||||
windUnit := "km/h"
|
||||
tempStr := cc.TempC
|
||||
feelsStr := cc.FeelsLikeC
|
||||
if units == "imperial" {
|
||||
unitStr = "°F"
|
||||
windUnit = "mph"
|
||||
tempStr = convertCtoF(tempStr)
|
||||
feelsStr = convertCtoF(feelsStr)
|
||||
}
|
||||
|
||||
obsTime := cc.LocalObsDateTime
|
||||
if len(obsTime) > 16 {
|
||||
obsTime = obsTime[:16]
|
||||
}
|
||||
|
||||
result := fmt.Sprintf("🌤 %s — %s\n🌡 %s%s (体感 %s%s)\n💧 湿度 %s%% | 💨 风速 %s %s %s\n🕐 %s",
|
||||
place, desc,
|
||||
tempStr, unitStr, feelsStr, unitStr,
|
||||
cc.Humidity, cc.WindspeedKmph, windUnit, cc.Winddir16Point,
|
||||
obsTime)
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": result,
|
||||
"location": place,
|
||||
"temp": cc.TempC,
|
||||
"feels_like": cc.FeelsLikeC,
|
||||
"humidity": cc.Humidity,
|
||||
"wind_speed": cc.WindspeedKmph,
|
||||
"weather": desc,
|
||||
"observed": obsTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleForecast(args map[string]interface{}) (interface{}, error) {
|
||||
location := p.getLoc(args)
|
||||
if location == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "No location specified."}, nil
|
||||
}
|
||||
|
||||
days := 3
|
||||
if v, ok := args["days"].(float64); ok {
|
||||
d := int(v)
|
||||
if d >= 1 && d <= 7 {
|
||||
days = d
|
||||
}
|
||||
}
|
||||
|
||||
units := p.getUnits(args)
|
||||
|
||||
data, err := p.fetchWttr(location)
|
||||
if err != nil {
|
||||
return map[string]interface{}{"isError": true, "content": "Forecast request failed: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
place := p.displayName(data)
|
||||
|
||||
unitStr := "°C"
|
||||
if units == "imperial" {
|
||||
unitStr = "°F"
|
||||
}
|
||||
|
||||
dayCount := days
|
||||
if dayCount > len(data.Weather) {
|
||||
dayCount = len(data.Weather)
|
||||
}
|
||||
daysData := data.Weather[:dayCount]
|
||||
|
||||
var lines []string
|
||||
lines = append(lines, fmt.Sprintf("📅 %d日天气预报 — %s", days, place))
|
||||
for _, day := range daysData {
|
||||
t, err := time.Parse("2006-01-02", day.Date)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
weekday := t.Weekday().String()[:3]
|
||||
|
||||
maxT := day.MaxtempC
|
||||
minT := day.MintempC
|
||||
desc := ""
|
||||
precip := ""
|
||||
|
||||
if len(day.Hourly) > 0 {
|
||||
mid := len(day.Hourly) / 2
|
||||
if len(day.Hourly[mid].WeatherDesc) > 0 {
|
||||
desc = day.Hourly[mid].WeatherDesc[0].Value
|
||||
}
|
||||
totalPrecip := 0.0
|
||||
for _, h := range day.Hourly {
|
||||
if pv, err := strconv.ParseFloat(h.PrecipMM, 64); err == nil {
|
||||
totalPrecip += pv
|
||||
}
|
||||
}
|
||||
if totalPrecip > 0 {
|
||||
precip = fmt.Sprintf(" 🌧%.1fmm", totalPrecip)
|
||||
}
|
||||
}
|
||||
|
||||
if units == "imperial" {
|
||||
maxT = convertCtoF(maxT)
|
||||
minT = convertCtoF(minT)
|
||||
}
|
||||
|
||||
sunrise, sunset := "", ""
|
||||
if len(day.Astronomy) > 0 {
|
||||
sunrise = day.Astronomy[0].Sunrise
|
||||
sunset = day.Astronomy[0].Sunset
|
||||
}
|
||||
|
||||
line := fmt.Sprintf(" %s %s/%s — %s~%s%s %s", weekday, day.Date[5:], day.Date[8:], minT, maxT, unitStr, desc)
|
||||
if precip != "" {
|
||||
line += precip
|
||||
}
|
||||
if sunrise != "" && sunset != "" {
|
||||
line += fmt.Sprintf(" 🌅%s 🌇%s", sunrise, sunset)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
|
||||
cc := data.CurrentCondition[0]
|
||||
nowDesc := ""
|
||||
if len(cc.WeatherDesc) > 0 {
|
||||
nowDesc = cc.WeatherDesc[0].Value
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("\n当前:%s %s°C", nowDesc, cc.TempC))
|
||||
|
||||
return map[string]interface{}{
|
||||
"content": strings.Join(lines, "\n"),
|
||||
"location": place,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *Plugin) handleSetLocation(args map[string]interface{}) (interface{}, error) {
|
||||
loc, _ := args["location"].(string)
|
||||
if loc == "" {
|
||||
return map[string]interface{}{"isError": true, "content": "Location is required"}, nil
|
||||
}
|
||||
|
||||
p.sdk.Settings().SetCore("plugin.weather.default_location", loc)
|
||||
p.defaultLoc = loc
|
||||
return map[string]interface{}{"content": fmt.Sprintf("Default location set to: %s", loc)}, nil
|
||||
}
|
||||
6
tools/gengskill/.gitignore
vendored
Normal file
6
tools/gengskill/.gitignore
vendored
Normal file
@ -0,0 +1,6 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.DS_Store
|
||||
report/
|
||||
figures/
|
||||
*.egg-info/
|
||||
21
tools/gengskill/LICENSE
Normal file
21
tools/gengskill/LICENSE
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Geng Skill Contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
319
tools/gengskill/README.md
Normal file
319
tools/gengskill/README.md
Normal file
@ -0,0 +1,319 @@
|
||||
<p align="center">
|
||||
<img src="assets/banner.png" alt="Geng Skill Banner" width="100%">
|
||||
</p>
|
||||
|
||||
<h1 align="center">🔬 Geng Skill — 学术数据打假检测工具</h1>
|
||||
|
||||
<p align="center">
|
||||
<strong>用数据说话,让造假无所遁形 · Inspired by "耿同学讲故事"</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/Python-3.8%2B-3776AB?style=for-the-badge&logo=python&logoColor=white" alt="Python 3.8+">
|
||||
<img src="https://img.shields.io/badge/License-MIT-22c55e?style=for-the-badge" alt="MIT License">
|
||||
<img src="https://img.shields.io/badge/Version-2.0.0-7c3aed?style=for-the-badge" alt="Version 2.0.0">
|
||||
<img src="https://img.shields.io/badge/Tests-24%2F24_Passed-22c55e?style=for-the-badge" alt="Tests Passing">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="#-快速开始">Quick Start</a> •
|
||||
<a href="#-工作原理">How It Works</a> •
|
||||
<a href="#-三种输入模式">Input Modes</a> •
|
||||
<a href="#-检测模块">Detection Modules</a> •
|
||||
<a href="#-实战案例">Example</a> •
|
||||
<a href="#-文档">Documentation</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 🌟 这是什么?
|
||||
|
||||
**Geng Skill** 是一套基于统计学原理的学术论文数据造假检测工具包。
|
||||
|
||||
2026 年 4 月起,科普博主"耿同学讲故事"凭一台电脑和几个统计方法,连续揪出多所 985 高校顶尖学者的论文造假——同济大学 Nature 论文院长免职、南开大学 Nature 子刊正在调查……他证明了一件事:**造假的数据一定会留下统计学破绽。**
|
||||
|
||||
本项目将"耿同学"的技术方法论系统化、工具化,让任何人都能一键检测论文数据是否存在造假嫌疑。
|
||||
|
||||
> 💡 **核心原理**:真实实验数据具有随机性;人为编造的数据会呈现不自然的数学规律。
|
||||
|
||||
---
|
||||
|
||||
## 🚀 快速开始
|
||||
|
||||
```bash
|
||||
# 克隆仓库
|
||||
git clone https://github.com/YOUR_USERNAME/geng-skill.git
|
||||
cd geng-skill
|
||||
pip install -r requirements.txt
|
||||
|
||||
# 一键自动扫描(Scale 模式)
|
||||
python3 scripts/input_pipeline.py --input your_data.csv --mode scale
|
||||
```
|
||||
|
||||
就这么简单。Scale 模式会**自动扫描所有数值列**,运行 6 种检测算法,并按嫌疑程度从高到低排列结果。
|
||||
|
||||
---
|
||||
|
||||
## 🧠 工作原理
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/architecture.png" alt="System Architecture" width="90%">
|
||||
</p>
|
||||
|
||||
系统分为三层:
|
||||
|
||||
### 第一层 · 数据输入 Input Pipeline
|
||||
|
||||
支持 **PDF 论文**、**Excel 原始数据**、**CSV 表格** 三种格式。自动提取表格、识别数值列、标准化数据格式。
|
||||
|
||||
### 第二层 · 检测引擎 Detection Engine
|
||||
|
||||
6 个独立的统计检测模块,从不同角度分析数据异常:
|
||||
|
||||
| 模块 | 检测目标 | 方法 |
|
||||
|------|----------|------|
|
||||
| **末位数字检测** Last Digit | 末位数字集中度异常 | Chi-squared vs 均匀分布 |
|
||||
| **本福特定律** Benford's Law | 首位数字分布偏离 | Chi-squared + MAD |
|
||||
| **GRIM 测试** | 均值与样本量不兼容 | 离散粒度校验 |
|
||||
| **固定关系检测** Fixed Ratio ⭐ | 实验组间存在完美数学关系 | 比值/回归分析 |
|
||||
| **小数位一致性** Decimal | 小数部分模式重复 | 自相关 + 熵分析 |
|
||||
| **图像重复检测** Image Dup | 同一图片在不同条件下重复使用 | 感知哈希 + SSIM |
|
||||
|
||||
### 第三层 · 输出报告 Output
|
||||
|
||||
生成出版级可视化图表、综合风险评分(0–100)、以及详细的 HTML/Markdown 检测报告,精确标注每个可疑数据点。
|
||||
|
||||
---
|
||||
|
||||
## 📥 三种输入模式
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/workflow.png" alt="Workflow" width="90%">
|
||||
</p>
|
||||
|
||||
### 模式一:论文 PDF 输入
|
||||
|
||||
直接从 PDF 论文中提取数据表格:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input paper.pdf --mode extract
|
||||
```
|
||||
|
||||
### 模式二:Excel / CSV 原始数据
|
||||
|
||||
处理从论文下载的 Supplementary Data:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input supplementary_data.xlsx --mode extract
|
||||
python3 scripts/input_pipeline.py --input table_s1.csv --mode extract
|
||||
```
|
||||
|
||||
### 模式三:Scale 自动扫描 ⭐(推荐)
|
||||
|
||||
**让工具自己去找问题。** 无需指定检测哪些列、用哪些方法——全部自动:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input data.csv --mode scale
|
||||
```
|
||||
|
||||
Scale 模式会:
|
||||
1. 自动识别所有数值列
|
||||
2. 运行 6 大检测模块
|
||||
3. 交叉验证各模块结果
|
||||
4. 输出 **suspicion_ranking**(嫌疑排名榜)
|
||||
|
||||
---
|
||||
|
||||
## 🔬 检测模块
|
||||
|
||||
### ⭐ 固定关系检测(核心 · 耿同学的看家本领)
|
||||
|
||||
这是最致命的检测手段。如果两组"独立实验"数据之间存在**完美的固定数学关系**(比如每个样本 Treatment = Control × 2.0),那几乎可以断定是造假。
|
||||
|
||||
```python
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
|
||||
result = fixed_relation_test(control_data, treatment_data)
|
||||
# result['risk_score'] = 95 → 检测到精确 2.0 倍关系!
|
||||
```
|
||||
|
||||
**为什么这能定性?** 即使药物真的让蛋白表达提高 2 倍,每个样本也会有生物学个体差异。20 个样本**全部**精确到小数点后多位都是 2.000 倍——概率趋近于零。
|
||||
|
||||
### 本福特定律检测 Benford's Law
|
||||
|
||||
跨越多个数量级的自然数据,首位数字遵循特定概率分布(1 最多,9 最少)。人为编造的数据会偏离:
|
||||
|
||||
```python
|
||||
from benford_test import benford_test
|
||||
result = benford_test(values)
|
||||
```
|
||||
|
||||
### GRIM 测试
|
||||
|
||||
对于整数取值的数据(如李克特量表 1–5 分),给定样本量 n,并非所有均值都是数学上可能的:
|
||||
|
||||
```python
|
||||
from grim_test import grim_test_single
|
||||
result = grim_test_single(mean='3.47', n=25, decimals=2)
|
||||
# result['consistent'] = False → 这个均值是不可能存在的!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 实战案例
|
||||
|
||||
### 输入:一篇可疑的生物医学论文数据
|
||||
|
||||
论文声称对小鼠进行了三组独立实验(Control / Treatment A / Treatment B),数据如下:
|
||||
|
||||
```csv
|
||||
sample_id,control,treatment_a,treatment_b
|
||||
1,2.34,4.68,7.02
|
||||
2,3.12,6.24,9.36
|
||||
3,1.87,3.74,5.61
|
||||
4,4.56,9.12,13.68
|
||||
5,2.98,5.96,8.94
|
||||
...
|
||||
```
|
||||
|
||||
### 运行 Scale 模式自动检测
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input data.csv --mode scale
|
||||
```
|
||||
|
||||
### 输出:精准定位问题
|
||||
|
||||
```
|
||||
=== Scale Mode: 自动检测结果 ===
|
||||
|
||||
🔴 [1.00] control ↔ treatment_a ← 精确固定比值 = 2.000
|
||||
🔴 [1.00] control ↔ treatment_b ← 精确固定比值 = 3.000
|
||||
🔴 [1.00] treatment_a ↔ treatment_b ← 精确固定比值 = 1.500
|
||||
🟠 [0.99] treatment_a ← 末位数字分布异常
|
||||
|
||||
╔══════════════════════════════════════════════════════════════╗
|
||||
║ 综合风险评分: 92/100 🔴 极高风险 ║
|
||||
╠══════════════════════════════════════════════════════════════╣
|
||||
║ 三组"独立实验"数据之间存在完美的整数倍关系。 ║
|
||||
║ 在真实生物实验中,这种情况出现的概率约等于零。 ║
|
||||
║ 数据极大概率为人工编造。 ║
|
||||
╚══════════════════════════════════════════════════════════════╝
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 风险评分体系
|
||||
|
||||
| 分数 | 等级 | 含义 | 建议行动 |
|
||||
|------|------|------|----------|
|
||||
| 0–25 | 🟢 低风险 | 未发现异常 | 无需干预 |
|
||||
| 26–50 | 🟡 中等 | 存在轻微模式,可能是正常波动 | 建议复核 |
|
||||
| 51–75 | 🟠 高风险 | 多项指标异常 | 深入调查 |
|
||||
| 76–100 | 🔴 极高风险 | 系统性异常 | 正式举报 |
|
||||
|
||||
**置信度规则:**
|
||||
- 单一模块报警 → 标注为"待确认线索"
|
||||
- 2 个以上独立模块同时报警 → 标注为"高度可疑"
|
||||
- 仅当综合分 > 75 **且**多模块交叉确认时,才标注"极高风险"
|
||||
|
||||
---
|
||||
|
||||
## ⚖️ 准确性与免责声明
|
||||
|
||||
### ✅ 本工具能做什么
|
||||
|
||||
- 检测不同数据列之间的固定数学关系
|
||||
- 识别数字分布的统计学异常
|
||||
- 标记数学上不可能的统计报告值
|
||||
- 发现重复使用/篡改的论文图片
|
||||
- 提供量化的风险评估和置信度等级
|
||||
|
||||
### ❌ 本工具不能做什么
|
||||
|
||||
- 不能证明造假的主观意图
|
||||
- 不能检测加了随机噪声的"高明造假"
|
||||
- 不能替代领域专家的判断
|
||||
- 不具备法律效力
|
||||
|
||||
### ⚠️ 重要声明
|
||||
|
||||
> **本工具仅提供统计学层面的异常筛查功能。**
|
||||
> 输出结果为"疑点线索"(Suspicious Indicators),而非"造假判定"(Fraud Determination)。
|
||||
>
|
||||
> - 统计异常可能有合理的科学解释(仪器精度限制、数据标准化处理、单位转换等)
|
||||
> - 最终判定需要领域专家复核和正式调查程序
|
||||
> - 通过全部检测 ≠ 数据一定真实(某些造假无法被统计方法捕获)
|
||||
> - 使用者需自行承担因不当使用(如公开发布未经验证的指控)造成的一切后果
|
||||
|
||||
---
|
||||
|
||||
## 📁 项目结构
|
||||
|
||||
```
|
||||
geng-skill/
|
||||
├── scripts/ 核心引擎
|
||||
│ ├── input_pipeline.py 统一输入(PDF/Excel/CSV + Scale 模式)
|
||||
│ ├── visualization.py 出版级可视化图表
|
||||
│ ├── report_generator.py HTML + Markdown 报告生成
|
||||
│ ├── geng_assess.py 综合评估引擎
|
||||
│ ├── last_digit_test.py 末位数字检测
|
||||
│ ├── benford_test.py 本福特定律检测
|
||||
│ ├── grim_test.py GRIM 均值一致性测试
|
||||
│ ├── fixed_relation_test.py 固定关系检测 ⭐
|
||||
│ ├── decimal_consistency_test.py 小数位一致性检测
|
||||
│ └── image_duplicate_test.py 图像重复检测
|
||||
├── docs/ 完整文档
|
||||
│ ├── USAGE_GUIDE.md 多平台使用指南(Claude/Cursor/GPT/Codex 等)
|
||||
│ ├── DATA_SOURCES.md 学术参考文献 + 数据标准 + 伦理合规
|
||||
│ ├── ANNOTATIONS.md 架构图 + API 接口 + 代码注释规范
|
||||
│ └── EXAMPLE_WALKTHROUGH.md 端到端完整教程
|
||||
├── examples/ 示例数据
|
||||
├── tests/ 单元测试(24/24 通过)
|
||||
└── assets/ README 配图
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📖 文档
|
||||
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [USAGE_GUIDE.md](docs/USAGE_GUIDE.md) | 在 Claude / Cursor / GPT / Codex / Jupyter / Docker 等平台上的使用方法 |
|
||||
| [DATA_SOURCES.md](docs/DATA_SOURCES.md) | 学术参考文献、数据标准、伦理合规框架 |
|
||||
| [ANNOTATIONS.md](docs/ANNOTATIONS.md) | 系统架构、API 接口规范、代码注释标准 |
|
||||
| [EXAMPLE_WALKTHROUGH.md](docs/EXAMPLE_WALKTHROUGH.md) | 从一篇论文到检测报告的完整教程 |
|
||||
| [SKILL.md](SKILL.md) | Skill 核心技术文档 |
|
||||
|
||||
---
|
||||
|
||||
## 🔗 学术参考
|
||||
|
||||
1. Benford, F. (1938). The law of anomalous numbers. *Proc. APS*, 78(4), 551–572.
|
||||
2. Brown, N.J.L. & Heathers, J.A.J. (2017). The GRIM Test. *SPPS*, 8(4), 363–369.
|
||||
3. Bik, E.M. et al. (2016). Image duplication in biomedical research. *mBio*, 7(3).
|
||||
4. Nigrini, M.J. (2012). *Benford's Law*. Wiley. ISBN: 978-1118152850.
|
||||
5. 余菁等 (2021). 科技论文数据造假的核查策略. *中国科技期刊研究*, 32(6), 770–776.
|
||||
|
||||
---
|
||||
|
||||
## 🙏 致谢
|
||||
|
||||
本项目的灵感来源于 **"耿同学讲故事"** —— 一位吉林大学生物学硕士、北航退学博士,从 2026 年 4 月开始,仅凭一台电脑和统计学方法,就揪出了多所顶尖高校教授的论文数据造假。他的工作证明了:**学术诚信监督不仅必要,而且完全可行。**
|
||||
|
||||
> "如果论文里的数据存在规律性,那么就明显不是在实验室实际测量的情况下生成的。"
|
||||
>
|
||||
> —— 耿同学
|
||||
|
||||
---
|
||||
|
||||
## 📄 开源许可
|
||||
|
||||
MIT License — 详见 [LICENSE](LICENSE)
|
||||
|
||||
---
|
||||
|
||||
<p align="center">
|
||||
<em>让学术回归诚信,让数据说出真相。</em><br>
|
||||
<em>Let academic integrity prevail. Let data speak the truth.</em>
|
||||
</p>
|
||||
319
tools/gengskill/README_CN.md
Normal file
319
tools/gengskill/README_CN.md
Normal file
@ -0,0 +1,319 @@
|
||||
<p align="center">
|
||||
<img src="assets/banner.png" alt="Geng Skill Banner" width="100%">
|
||||
</p>
|
||||
|
||||
<h1 align="center">🔬 Geng Skill — 学术数据打假检测工具</h1>
|
||||
|
||||
<p align="center">
|
||||
<strong>用数据说话,让造假无所遁形 · Inspired by "耿同学讲故事"</strong>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="https://img.shields.io/badge/Python-3.8%2B-3776AB?style=for-the-badge&logo=python&logoColor=white" alt="Python 3.8+">
|
||||
<img src="https://img.shields.io/badge/License-MIT-22c55e?style=for-the-badge" alt="MIT License">
|
||||
<img src="https://img.shields.io/badge/Version-2.0.0-7c3aed?style=for-the-badge" alt="Version 2.0.0">
|
||||
<img src="https://img.shields.io/badge/Tests-24%2F24_Passed-22c55e?style=for-the-badge" alt="Tests Passing">
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="#-快速开始">Quick Start</a> •
|
||||
<a href="#-工作原理">How It Works</a> •
|
||||
<a href="#-三种输入模式">Input Modes</a> •
|
||||
<a href="#-检测模块">Detection Modules</a> •
|
||||
<a href="#-实战案例">Example</a> •
|
||||
<a href="#-文档">Documentation</a>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
## 🌟 这是什么?
|
||||
|
||||
**Geng Skill** 是一套基于统计学原理的学术论文数据造假检测工具包。
|
||||
|
||||
2026 年 4 月起,科普博主"耿同学讲故事"凭一台电脑和几个统计方法,连续揪出多所 985 高校顶尖学者的论文造假——同济大学 Nature 论文院长免职、南开大学 Nature 子刊正在调查……他证明了一件事:**造假的数据一定会留下统计学破绽。**
|
||||
|
||||
本项目将"耿同学"的技术方法论系统化、工具化,让任何人都能一键检测论文数据是否存在造假嫌疑。
|
||||
|
||||
> 💡 **核心原理**:真实实验数据具有随机性;人为编造的数据会呈现不自然的数学规律。
|
||||
|
||||
---
|
||||
|
||||
## 🚀 快速开始
|
||||
|
||||
```bash
|
||||
# 克隆仓库
|
||||
git clone https://github.com/YOUR_USERNAME/geng-skill.git
|
||||
cd geng-skill
|
||||
pip install -r requirements.txt
|
||||
|
||||
# 一键自动扫描(Scale 模式)
|
||||
python3 scripts/input_pipeline.py --input your_data.csv --mode scale
|
||||
```
|
||||
|
||||
就这么简单。Scale 模式会**自动扫描所有数值列**,运行 6 种检测算法,并按嫌疑程度从高到低排列结果。
|
||||
|
||||
---
|
||||
|
||||
## 🧠 工作原理
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/architecture.png" alt="System Architecture" width="90%">
|
||||
</p>
|
||||
|
||||
系统分为三层:
|
||||
|
||||
### 第一层 · 数据输入 Input Pipeline
|
||||
|
||||
支持 **PDF 论文**、**Excel 原始数据**、**CSV 表格** 三种格式。自动提取表格、识别数值列、标准化数据格式。
|
||||
|
||||
### 第二层 · 检测引擎 Detection Engine
|
||||
|
||||
6 个独立的统计检测模块,从不同角度分析数据异常:
|
||||
|
||||
| 模块 | 检测目标 | 方法 |
|
||||
|------|----------|------|
|
||||
| **末位数字检测** Last Digit | 末位数字集中度异常 | Chi-squared vs 均匀分布 |
|
||||
| **本福特定律** Benford's Law | 首位数字分布偏离 | Chi-squared + MAD |
|
||||
| **GRIM 测试** | 均值与样本量不兼容 | 离散粒度校验 |
|
||||
| **固定关系检测** Fixed Ratio ⭐ | 实验组间存在完美数学关系 | 比值/回归分析 |
|
||||
| **小数位一致性** Decimal | 小数部分模式重复 | 自相关 + 熵分析 |
|
||||
| **图像重复检测** Image Dup | 同一图片在不同条件下重复使用 | 感知哈希 + SSIM |
|
||||
|
||||
### 第三层 · 输出报告 Output
|
||||
|
||||
生成出版级可视化图表、综合风险评分(0–100)、以及详细的 HTML/Markdown 检测报告,精确标注每个可疑数据点。
|
||||
|
||||
---
|
||||
|
||||
## 📥 三种输入模式
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/workflow.png" alt="Workflow" width="90%">
|
||||
</p>
|
||||
|
||||
### 模式一:论文 PDF 输入
|
||||
|
||||
直接从 PDF 论文中提取数据表格:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input paper.pdf --mode extract
|
||||
```
|
||||
|
||||
### 模式二:Excel / CSV 原始数据
|
||||
|
||||
处理从论文下载的 Supplementary Data:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input supplementary_data.xlsx --mode extract
|
||||
python3 scripts/input_pipeline.py --input table_s1.csv --mode extract
|
||||
```
|
||||
|
||||
### 模式三:Scale 自动扫描 ⭐(推荐)
|
||||
|
||||
**让工具自己去找问题。** 无需指定检测哪些列、用哪些方法——全部自动:
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input data.csv --mode scale
|
||||
```
|
||||
|
||||
Scale 模式会:
|
||||
1. 自动识别所有数值列
|
||||
2. 运行 6 大检测模块
|
||||
3. 交叉验证各模块结果
|
||||
4. 输出 **suspicion_ranking**(嫌疑排名榜)
|
||||
|
||||
---
|
||||
|
||||
## 🔬 检测模块
|
||||
|
||||
### ⭐ 固定关系检测(核心 · 耿同学的看家本领)
|
||||
|
||||
这是最致命的检测手段。如果两组"独立实验"数据之间存在**完美的固定数学关系**(比如每个样本 Treatment = Control × 2.0),那几乎可以断定是造假。
|
||||
|
||||
```python
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
|
||||
result = fixed_relation_test(control_data, treatment_data)
|
||||
# result['risk_score'] = 95 → 检测到精确 2.0 倍关系!
|
||||
```
|
||||
|
||||
**为什么这能定性?** 即使药物真的让蛋白表达提高 2 倍,每个样本也会有生物学个体差异。20 个样本**全部**精确到小数点后多位都是 2.000 倍——概率趋近于零。
|
||||
|
||||
### 本福特定律检测 Benford's Law
|
||||
|
||||
跨越多个数量级的自然数据,首位数字遵循特定概率分布(1 最多,9 最少)。人为编造的数据会偏离:
|
||||
|
||||
```python
|
||||
from benford_test import benford_test
|
||||
result = benford_test(values)
|
||||
```
|
||||
|
||||
### GRIM 测试
|
||||
|
||||
对于整数取值的数据(如李克特量表 1–5 分),给定样本量 n,并非所有均值都是数学上可能的:
|
||||
|
||||
```python
|
||||
from grim_test import grim_test_single
|
||||
result = grim_test_single(mean='3.47', n=25, decimals=2)
|
||||
# result['consistent'] = False → 这个均值是不可能存在的!
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 实战案例
|
||||
|
||||
### 输入:一篇可疑的生物医学论文数据
|
||||
|
||||
论文声称对小鼠进行了三组独立实验(Control / Treatment A / Treatment B),数据如下:
|
||||
|
||||
```csv
|
||||
sample_id,control,treatment_a,treatment_b
|
||||
1,2.34,4.68,7.02
|
||||
2,3.12,6.24,9.36
|
||||
3,1.87,3.74,5.61
|
||||
4,4.56,9.12,13.68
|
||||
5,2.98,5.96,8.94
|
||||
...
|
||||
```
|
||||
|
||||
### 运行 Scale 模式自动检测
|
||||
|
||||
```bash
|
||||
python3 scripts/input_pipeline.py --input data.csv --mode scale
|
||||
```
|
||||
|
||||
### 输出:精准定位问题
|
||||
|
||||
```
|
||||
=== Scale Mode: 自动检测结果 ===
|
||||
|
||||
🔴 [1.00] control ↔ treatment_a ← 精确固定比值 = 2.000
|
||||
🔴 [1.00] control ↔ treatment_b ← 精确固定比值 = 3.000
|
||||
🔴 [1.00] treatment_a ↔ treatment_b ← 精确固定比值 = 1.500
|
||||
🟠 [0.99] treatment_a ← 末位数字分布异常
|
||||
|
||||
╔══════════════════════════════════════════════════════════════╗
|
||||
║ 综合风险评分: 92/100 🔴 极高风险 ║
|
||||
╠══════════════════════════════════════════════════════════════╣
|
||||
║ 三组"独立实验"数据之间存在完美的整数倍关系。 ║
|
||||
║ 在真实生物实验中,这种情况出现的概率约等于零。 ║
|
||||
║ 数据极大概率为人工编造。 ║
|
||||
╚══════════════════════════════════════════════════════════════╝
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 风险评分体系
|
||||
|
||||
| 分数 | 等级 | 含义 | 建议行动 |
|
||||
|------|------|------|----------|
|
||||
| 0–25 | 🟢 低风险 | 未发现异常 | 无需干预 |
|
||||
| 26–50 | 🟡 中等 | 存在轻微模式,可能是正常波动 | 建议复核 |
|
||||
| 51–75 | 🟠 高风险 | 多项指标异常 | 深入调查 |
|
||||
| 76–100 | 🔴 极高风险 | 系统性异常 | 正式举报 |
|
||||
|
||||
**置信度规则:**
|
||||
- 单一模块报警 → 标注为"待确认线索"
|
||||
- 2 个以上独立模块同时报警 → 标注为"高度可疑"
|
||||
- 仅当综合分 > 75 **且**多模块交叉确认时,才标注"极高风险"
|
||||
|
||||
---
|
||||
|
||||
## ⚖️ 准确性与免责声明
|
||||
|
||||
### ✅ 本工具能做什么
|
||||
|
||||
- 检测不同数据列之间的固定数学关系
|
||||
- 识别数字分布的统计学异常
|
||||
- 标记数学上不可能的统计报告值
|
||||
- 发现重复使用/篡改的论文图片
|
||||
- 提供量化的风险评估和置信度等级
|
||||
|
||||
### ❌ 本工具不能做什么
|
||||
|
||||
- 不能证明造假的主观意图
|
||||
- 不能检测加了随机噪声的"高明造假"
|
||||
- 不能替代领域专家的判断
|
||||
- 不具备法律效力
|
||||
|
||||
### ⚠️ 重要声明
|
||||
|
||||
> **本工具仅提供统计学层面的异常筛查功能。**
|
||||
> 输出结果为"疑点线索"(Suspicious Indicators),而非"造假判定"(Fraud Determination)。
|
||||
>
|
||||
> - 统计异常可能有合理的科学解释(仪器精度限制、数据标准化处理、单位转换等)
|
||||
> - 最终判定需要领域专家复核和正式调查程序
|
||||
> - 通过全部检测 ≠ 数据一定真实(某些造假无法被统计方法捕获)
|
||||
> - 使用者需自行承担因不当使用(如公开发布未经验证的指控)造成的一切后果
|
||||
|
||||
---
|
||||
|
||||
## 📁 项目结构
|
||||
|
||||
```
|
||||
geng-skill/
|
||||
├── scripts/ 核心引擎
|
||||
│ ├── input_pipeline.py 统一输入(PDF/Excel/CSV + Scale 模式)
|
||||
│ ├── visualization.py 出版级可视化图表
|
||||
│ ├── report_generator.py HTML + Markdown 报告生成
|
||||
│ ├── geng_assess.py 综合评估引擎
|
||||
│ ├── last_digit_test.py 末位数字检测
|
||||
│ ├── benford_test.py 本福特定律检测
|
||||
│ ├── grim_test.py GRIM 均值一致性测试
|
||||
│ ├── fixed_relation_test.py 固定关系检测 ⭐
|
||||
│ ├── decimal_consistency_test.py 小数位一致性检测
|
||||
│ └── image_duplicate_test.py 图像重复检测
|
||||
├── docs/ 完整文档
|
||||
│ ├── USAGE_GUIDE.md 多平台使用指南(Claude/Cursor/GPT/Codex 等)
|
||||
│ ├── DATA_SOURCES.md 学术参考文献 + 数据标准 + 伦理合规
|
||||
│ ├── ANNOTATIONS.md 架构图 + API 接口 + 代码注释规范
|
||||
│ └── EXAMPLE_WALKTHROUGH.md 端到端完整教程
|
||||
├── examples/ 示例数据
|
||||
├── tests/ 单元测试(24/24 通过)
|
||||
└── assets/ README 配图
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📖 文档
|
||||
|
||||
| 文档 | 内容 |
|
||||
|------|------|
|
||||
| [USAGE_GUIDE.md](docs/USAGE_GUIDE.md) | 在 Claude / Cursor / GPT / Codex / Jupyter / Docker 等平台上的使用方法 |
|
||||
| [DATA_SOURCES.md](docs/DATA_SOURCES.md) | 学术参考文献、数据标准、伦理合规框架 |
|
||||
| [ANNOTATIONS.md](docs/ANNOTATIONS.md) | 系统架构、API 接口规范、代码注释标准 |
|
||||
| [EXAMPLE_WALKTHROUGH.md](docs/EXAMPLE_WALKTHROUGH.md) | 从一篇论文到检测报告的完整教程 |
|
||||
| [SKILL.md](SKILL.md) | Skill 核心技术文档 |
|
||||
|
||||
---
|
||||
|
||||
## 🔗 学术参考
|
||||
|
||||
1. Benford, F. (1938). The law of anomalous numbers. *Proc. APS*, 78(4), 551–572.
|
||||
2. Brown, N.J.L. & Heathers, J.A.J. (2017). The GRIM Test. *SPPS*, 8(4), 363–369.
|
||||
3. Bik, E.M. et al. (2016). Image duplication in biomedical research. *mBio*, 7(3).
|
||||
4. Nigrini, M.J. (2012). *Benford's Law*. Wiley. ISBN: 978-1118152850.
|
||||
5. 余菁等 (2021). 科技论文数据造假的核查策略. *中国科技期刊研究*, 32(6), 770–776.
|
||||
|
||||
---
|
||||
|
||||
## 🙏 致谢
|
||||
|
||||
本项目的灵感来源于 **"耿同学讲故事"**,从 2026 年 4 月开始,仅凭一台电脑和统计学方法,就揪出了多所顶尖高校教授的论文数据造假。他的工作证明了:**学术诚信监督不仅必要,而且完全可行。**
|
||||
|
||||
> "如果论文里的数据存在规律性,那么就明显不是在实验室实际测量的情况下生成的。"
|
||||
>
|
||||
> —— 耿同学
|
||||
|
||||
---
|
||||
|
||||
## 📄 开源许可
|
||||
|
||||
MIT License — 详见 [LICENSE](LICENSE)
|
||||
|
||||
---
|
||||
|
||||
<p align="center">
|
||||
<em>让学术回归诚信,让数据说出真相。</em><br>
|
||||
<em>Let academic integrity prevail. Let data speak the truth.</em>
|
||||
</p>
|
||||
139
tools/gengskill/SKILL.md
Normal file
139
tools/gengskill/SKILL.md
Normal file
@ -0,0 +1,139 @@
|
||||
# Geng Skill — 学术数据打假检测工具
|
||||
|
||||
> 致敬"耿同学讲故事"——用数据说话,让造假无所遁形。
|
||||
|
||||
## 概述
|
||||
|
||||
本 Skill 实现了一套**基于统计学原理的学术论文数据造假检测方法**,灵感来源于科普博主"耿同学"的技术流打假方法论。该工具从数据层面对论文中的实验数据进行多维度异常检测,适用于生物医学、化学、物理、社会科学等多个领域。
|
||||
|
||||
## 核心原理
|
||||
|
||||
**自然数据具有随机性,人为编造的数据会呈现不自然的规律性。**
|
||||
|
||||
当研究者伪造实验数据时,由于人脑无法真正生成随机数,编造的数据往往会暴露以下统计学破绽:
|
||||
|
||||
1. **末位数字分布异常** — 自然数据末位数字应近似均匀分布
|
||||
2. **固定差值/比例关系** — 不同实验组数据间存在恒定数学关系
|
||||
3. **小数位一致性过高** — 多组数据小数点后位数高度一致
|
||||
4. **本福特定律偏离** — 首位数字分布严重偏离理论预期
|
||||
5. **GRIM/SPRITE 不一致** — 报告的平均值与样本量不兼容
|
||||
6. **图像重复/篡改** — 同一图片出现在不同实验条件下
|
||||
|
||||
## 适用领域
|
||||
|
||||
| 领域 | 检测重点 | 典型数据类型 |
|
||||
|------|----------|--------------|
|
||||
| 生物医学 | Western blot、流式细胞术、动物实验数据 | 连续测量值、图像 |
|
||||
| 化学 | 光谱数据、反应产率、催化活性 | 数值序列 |
|
||||
| 物理/材料 | 性能曲线、电学/力学测试数据 | 时间序列 |
|
||||
| 社会科学 | 问卷数据、量表得分 | 离散整数值 |
|
||||
| 临床医学 | 生存数据、临床指标 | 分组统计量 |
|
||||
|
||||
## 使用方法
|
||||
|
||||
### 输入
|
||||
|
||||
- 论文 PDF 文件(或提取的数据表格)
|
||||
- 补充材料 / Source Data(如有)
|
||||
- 指定检测领域(用于选择合适的检测策略)
|
||||
|
||||
### 检测流程
|
||||
|
||||
```
|
||||
输入论文 → 数据提取 → 多维度异常检测 → 综合评分 → 生成报告
|
||||
```
|
||||
|
||||
### 输出
|
||||
|
||||
- **异常检测报告**:每项检测的结果、p值、置信度
|
||||
- **综合风险评分**:0-100 分,分为低/中/高/极高风险
|
||||
- **可视化图表**:分布直方图、偏离热力图
|
||||
- **建议行动**:需要进一步核查的具体数据点
|
||||
|
||||
## 检测模块
|
||||
|
||||
### Module 1: 末位数字检测 (Last Digit Test)
|
||||
|
||||
```bash
|
||||
python3 scripts/last_digit_test.py --input data.csv --column "value"
|
||||
```
|
||||
|
||||
原理:自然实验数据的末位数字(0-9)应近似均匀分布。卡方检验判断偏离程度。
|
||||
|
||||
### Module 2: 本福特定律检测 (Benford's Law Test)
|
||||
|
||||
```bash
|
||||
python3 scripts/benford_test.py --input data.csv --column "value"
|
||||
```
|
||||
|
||||
原理:多数量级跨度的自然数据,首位数字以1开头的概率约30.1%,逐位递减至9的4.6%。
|
||||
|
||||
### Module 3: GRIM 测试 (Granularity-Related Inconsistency of Means)
|
||||
|
||||
```bash
|
||||
python3 scripts/grim_test.py --mean 3.47 --n 25 --scale "1-5" --decimals 2
|
||||
```
|
||||
|
||||
原理:对于整数取值数据,给定样本量 n,合法的平均值只能取特定的有限集合。
|
||||
|
||||
### Module 4: 固定关系检测 (Fixed Relationship Detection)
|
||||
|
||||
```bash
|
||||
python3 scripts/fixed_relation_test.py --input data.csv --col1 "group_a" --col2 "group_b"
|
||||
```
|
||||
|
||||
原理:两组独立实验数据之间不应存在恒定的差值、比值或线性关系。
|
||||
|
||||
### Module 5: 小数位一致性检测 (Decimal Consistency Test)
|
||||
|
||||
```bash
|
||||
python3 scripts/decimal_consistency_test.py --input data.csv --column "value"
|
||||
```
|
||||
|
||||
原理:实验测量数据的小数位后数字应具有随机性,过度一致暗示人为编造。
|
||||
|
||||
### Module 6: 图像重复检测 (Image Duplication Detection)
|
||||
|
||||
```bash
|
||||
python3 scripts/image_duplicate_test.py --input_dir ./figures/ --threshold 0.85
|
||||
```
|
||||
|
||||
原理:基于感知哈希和 SSIM 相似度,检测论文图片中是否存在重复使用或篡改。
|
||||
|
||||
### Module 7: 综合评估引擎 (Comprehensive Assessment)
|
||||
|
||||
```bash
|
||||
python3 scripts/geng_assess.py --input data.csv --domain "biomedical" --output report/
|
||||
```
|
||||
|
||||
一键运行所有适用模块,生成综合报告。
|
||||
|
||||
## 风险评分体系
|
||||
|
||||
| 等级 | 分数 | 含义 |
|
||||
|------|------|------|
|
||||
| 🟢 低风险 | 0-25 | 数据未发现明显异常 |
|
||||
| 🟡 中风险 | 26-50 | 存在可疑模式,建议人工复核 |
|
||||
| 🟠 高风险 | 51-75 | 多项检测异常,强烈建议深入调查 |
|
||||
| 🔴 极高风险 | 76-100 | 系统性异常,高度疑似数据造假 |
|
||||
|
||||
## 重要声明
|
||||
|
||||
⚠️ **本工具仅用于辅助筛查,不能作为造假的最终判定依据。**
|
||||
|
||||
- 数据异常 ≠ 数据造假(可能是仪器校准、单位转换、排版错误等)
|
||||
- 检测结果需要领域专家复核
|
||||
- 不应基于单一检测模块的结果下结论
|
||||
- 使用本工具时应遵守学术伦理和法律法规
|
||||
- 建议将检测结果提交给相关机构进行正式调查
|
||||
|
||||
## 参考文献
|
||||
|
||||
1. Benford, F. (1938). The law of anomalous numbers. *Proceedings of the American Philosophical Society*, 78(4), 551-572.
|
||||
2. Brown, N.J.L., & Heathers, J.A.J. (2017). The GRIM Test: A Simple Technique Detects Numerous Anomalies in the Reporting of Results in Psychology. *Social Psychological and Personality Science*, 8(4), 363-369.
|
||||
3. 余菁等 (2021). 科技论文数据造假的核查策略和统计学方法验证. *中国科技期刊研究*, 32(6), 770-776.
|
||||
4. Bik, E.M., et al. (2016). The prevalence of inappropriate image duplication in biomedical research publications. *mBio*, 7(3), e00809-16.
|
||||
|
||||
## 版本
|
||||
|
||||
- v1.0.0 — 2026-05-20 — 初始版本,致敬耿同学
|
||||
BIN
tools/gengskill/assets/architecture.png
Normal file
BIN
tools/gengskill/assets/architecture.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1003 KiB |
BIN
tools/gengskill/assets/banner.png
Normal file
BIN
tools/gengskill/assets/banner.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.2 MiB |
BIN
tools/gengskill/assets/workflow.png
Normal file
BIN
tools/gengskill/assets/workflow.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1003 KiB |
514
tools/gengskill/docs/ANNOTATIONS.md
Normal file
514
tools/gengskill/docs/ANNOTATIONS.md
Normal file
@ -0,0 +1,514 @@
|
||||
# 🏷️ Geng Skill 代码注释规范与架构说明
|
||||
|
||||
> 本文档提供完整的代码注释体系、模块间关系、接口规范,供开发者和 AI Agent 使用。
|
||||
|
||||
---
|
||||
|
||||
## 1. 项目架构总览
|
||||
|
||||
```
|
||||
┌──────────────────────────────┐
|
||||
│ geng_assess.py │
|
||||
│ (综合评估引擎 / 主入口) │
|
||||
└──────────────┬───────────────┘
|
||||
│
|
||||
┌───────────┬───────────┼───────────┬───────────┐
|
||||
▼ ▼ ▼ ▼ ▼
|
||||
┌────────────┐ ┌────────────┐ ┌────────────┐ ┌────────────┐ ┌──────────────┐
|
||||
│last_digit │ │ benford │ │ grim │ │fixed_rel │ │decimal_cons │
|
||||
│_test.py │ │_test.py │ │_test.py │ │_test.py │ │_test.py │
|
||||
│ │ │ │ │ │ │ │ │ │
|
||||
│末位数字检测│ │本福特定律 │ │均值一致性 │ │固定关系检测│ │小数位一致性 │
|
||||
└────────────┘ └────────────┘ └────────────┘ └────────────┘ └──────────────┘
|
||||
│ │
|
||||
│ ┌────────────────┐ │
|
||||
└───────────▶│image_duplicate │◀─────────────┘
|
||||
│_test.py │
|
||||
│图像重复检测 │
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 模块接口规范 (API Contract)
|
||||
|
||||
### 2.1 通用接口模式
|
||||
|
||||
每个检测模块都遵循统一的函数签名模式:
|
||||
|
||||
```python
|
||||
def <module_name>_test(
|
||||
values: List[str | float], # 输入数据
|
||||
**kwargs # 模块特定参数
|
||||
) -> Dict[str, Any]: # 标准化输出
|
||||
"""
|
||||
[模块名称] — [一句话描述]
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list
|
||||
待检测数据。字符串形式传入以保留原始精度。
|
||||
**kwargs : dict
|
||||
模块特定参数(详见各模块文档)
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
标准化输出,必含字段:
|
||||
- test_name : str — 模块名称(中英文)
|
||||
- status : str — "completed" | "insufficient_data" | "error"
|
||||
- risk_level : str — "low" | "medium" | "medium-high" | "high"
|
||||
- risk_score : float — 0-100 风险评分
|
||||
- interpretation : str — 中文可读解释
|
||||
"""
|
||||
```
|
||||
|
||||
### 2.2 各模块特定接口
|
||||
|
||||
#### Module 1: `last_digit_test()`
|
||||
|
||||
```python
|
||||
def last_digit_test(
|
||||
values: List[str],
|
||||
method: str = 'all_digits' # 'all_digits' | 'decimal_last'
|
||||
) -> Dict:
|
||||
"""
|
||||
末位数字检测
|
||||
|
||||
特定输出字段:
|
||||
- digit_distribution : Dict[str, int] — 0-9 各数字出现次数
|
||||
- chi_square : float — 卡方统计量
|
||||
- p_value : float — p值
|
||||
- most_frequent_digit : int — 出现最多的数字
|
||||
- most_frequent_proportion : float — 最高频率
|
||||
- uniformity_deviation : float — 偏离均匀度 (0-1)
|
||||
"""
|
||||
```
|
||||
|
||||
#### Module 2: `benford_test()`
|
||||
|
||||
```python
|
||||
def benford_test(
|
||||
values: List[str],
|
||||
order: int = 1 # 1=首位, 2=前两位
|
||||
) -> Dict:
|
||||
"""
|
||||
本福特定律检测
|
||||
|
||||
前提条件: 数据应跨越至少1个数量级
|
||||
|
||||
特定输出字段:
|
||||
- distribution : Dict[str, Dict] — 各位数字观测/期望频率
|
||||
- mean_absolute_deviation : float — MAD (Nigrini 判定标准)
|
||||
- conformity : str — 'close'|'acceptable'|'marginal'|'nonconforming'
|
||||
- conformity_cn : str — 中文符合性判定
|
||||
"""
|
||||
```
|
||||
|
||||
#### Module 3: `grim_test_single()` / `grim_test_batch()`
|
||||
|
||||
```python
|
||||
def grim_test_single(
|
||||
mean: str, # 报告的平均值(字符串保留精度)
|
||||
n: int, # 样本量
|
||||
decimals: int = 2, # 报告的小数位数
|
||||
scale_min: int = None, # 量表下限
|
||||
scale_max: int = None # 量表上限
|
||||
) -> Dict:
|
||||
"""
|
||||
GRIM 单项测试
|
||||
|
||||
特定输出字段:
|
||||
- consistent : bool — 是否通过一致性检验
|
||||
- computed_sum : float — 计算的总和 (mean × n)
|
||||
- nearest_valid_mean : str — 最近的合法均值
|
||||
- difference : float — 与最近合法均值的差距
|
||||
"""
|
||||
|
||||
def grim_test_batch(
|
||||
items: List[Dict] # 批量项目列表
|
||||
) -> Dict:
|
||||
"""
|
||||
GRIM 批量测试
|
||||
|
||||
items 格式: [{"mean": "3.47", "n": 25, "decimals": 2, "label": "Table 1"}, ...]
|
||||
|
||||
特定输出字段:
|
||||
- total_items : int
|
||||
- inconsistent_items : int
|
||||
- inconsistency_rate : float
|
||||
- details : List[Dict] — 每项的详细结果
|
||||
"""
|
||||
```
|
||||
|
||||
#### Module 4: `fixed_relation_test()`
|
||||
|
||||
```python
|
||||
def fixed_relation_test(
|
||||
col1: List[float], # 第一列数据
|
||||
col2: List[float], # 第二列数据
|
||||
col1_name: str = 'A', # 列名标签
|
||||
col2_name: str = 'B' # 列名标签
|
||||
) -> Dict:
|
||||
"""
|
||||
固定关系检测 — ⭐ 核心模块(耿同学最常用的方法)
|
||||
|
||||
检测内容:
|
||||
1. 固定差值 (col2 - col1 = 常数?)
|
||||
2. 固定比值 (col2 / col1 = 常数?)
|
||||
3. 完美线性关系 (R² → 1.0?)
|
||||
4. 小数模式一致性
|
||||
|
||||
特定输出字段:
|
||||
- detections : Dict — 各子检测结果
|
||||
- fixed_difference : {is_fixed, is_exact, mean_difference, std_difference}
|
||||
- fixed_ratio : {is_fixed, is_exact, mean_ratio, std_ratio}
|
||||
- linear_relationship : {r_squared, slope, intercept, is_suspicious}
|
||||
- decimal_pattern : {match_rate, is_suspicious}
|
||||
- n_suspicious_patterns : int
|
||||
"""
|
||||
```
|
||||
|
||||
#### Module 5: `decimal_consistency_test()`
|
||||
|
||||
```python
|
||||
def decimal_consistency_test(
|
||||
values: List[str] # 保留原始字符串精度
|
||||
) -> Dict:
|
||||
"""
|
||||
小数位一致性检测
|
||||
|
||||
特定输出字段:
|
||||
- decimal_places_analysis : Dict — 小数位数分布
|
||||
- decimal_repetition : Dict — 小数模式重复度
|
||||
- position_digit_analysis : Dict — 各位数字分布检验
|
||||
- autocorrelation : float — 小数部分自相关
|
||||
- risk_factors : List[str] — 触发的风险因子
|
||||
"""
|
||||
```
|
||||
|
||||
#### Module 6: `find_duplicates()`
|
||||
|
||||
```python
|
||||
def find_duplicates(
|
||||
image_dir: str, # 图片目录
|
||||
threshold: float = 0.85, # 相似度阈值
|
||||
extensions: List[str] = None # 图片格式
|
||||
) -> Dict:
|
||||
"""
|
||||
图像重复检测
|
||||
|
||||
依赖: Pillow, scikit-image (可选, 用于SSIM)
|
||||
|
||||
特定输出字段:
|
||||
- n_images_scanned : int
|
||||
- n_duplicate_pairs : int
|
||||
- duplicates : List[Dict] — 每对疑似重复图片
|
||||
- file_1, file_2 : str
|
||||
- avg_hash_similarity : float
|
||||
- diff_hash_similarity : float
|
||||
- combined_similarity : float
|
||||
- rotation_check : Dict — 旋转/翻转匹配结果
|
||||
- ssim : float (如果 scikit-image 可用)
|
||||
"""
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 代码注释规范
|
||||
|
||||
### 3.1 文件头注释模板
|
||||
|
||||
每个 Python 文件必须包含以下格式的文件头:
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
[模块名称中文] ([Module Name English])
|
||||
{'='*len(module_name)}
|
||||
|
||||
原理:[一段话描述检测原理]
|
||||
|
||||
方法:[具体使用的统计方法]
|
||||
|
||||
参考:[关键参考文献,一行一条]
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
```
|
||||
|
||||
### 3.2 函数注释规范 (NumPy Style)
|
||||
|
||||
```python
|
||||
def function_name(param1, param2, param3=default):
|
||||
"""
|
||||
[一句话功能描述]
|
||||
|
||||
[详细说明段落,解释为什么需要这个函数、在什么场景下使用]
|
||||
|
||||
Parameters
|
||||
----------
|
||||
param1 : type
|
||||
参数说明
|
||||
param2 : type
|
||||
参数说明
|
||||
param3 : type, optional
|
||||
参数说明(默认值:default)
|
||||
|
||||
Returns
|
||||
-------
|
||||
return_type
|
||||
返回值说明
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
何时抛出此异常
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> result = function_name([1, 2, 3])
|
||||
>>> print(result['risk_score'])
|
||||
15.3
|
||||
|
||||
Notes
|
||||
-----
|
||||
[重要注意事项、使用限制、已知问题]
|
||||
|
||||
References
|
||||
----------
|
||||
[1] Author (Year). Title. Journal. DOI.
|
||||
"""
|
||||
```
|
||||
|
||||
### 3.3 行内注释规范
|
||||
|
||||
```python
|
||||
# ✅ 好的注释 — 解释"为什么"
|
||||
# 本福特定律只适用于跨数量级的数据,pH值(0-14)不适用
|
||||
if value_range < 10:
|
||||
return skip_benford()
|
||||
|
||||
# ❌ 差的注释 — 重复代码
|
||||
# 计算平均值
|
||||
mean = sum(values) / len(values)
|
||||
|
||||
# ✅ 好的注释 — 标注算法来源
|
||||
# MAD 阈值参考 Nigrini (2012), Table 7.1
|
||||
# Close conformity: MAD < 0.006
|
||||
MAD_THRESHOLD_CLOSE = 0.006
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 错误处理与边界条件
|
||||
|
||||
### 4.1 标准错误返回
|
||||
|
||||
```python
|
||||
# 数据不足
|
||||
if len(values) < MIN_REQUIRED:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足(仅{len(values)}个),需要至少{MIN_REQUIRED}个',
|
||||
'n_valid': len(values)
|
||||
}
|
||||
|
||||
# 输入格式错误
|
||||
if not valid_input:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': f'无效输入: {error_detail}'
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 边界条件处理
|
||||
|
||||
| 场景 | 处理方式 |
|
||||
|------|----------|
|
||||
| 全部值为0 | 跳过本福特检测(返回 status='not_applicable') |
|
||||
| 无小数部分 | 跳过小数位检测 |
|
||||
| 仅1列数值 | 跳过固定关系检测 |
|
||||
| 图片目录为空 | 返回 insufficient_data |
|
||||
| 极端离群值 | 不剔除,但在 notes 中标注 |
|
||||
| NaN/无效值 | 静默跳过,在 n_valid 中反映 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 风险评分算法详解
|
||||
|
||||
### 5.1 单模块评分
|
||||
|
||||
```python
|
||||
"""
|
||||
风险评分映射逻辑(以末位数字检测为例):
|
||||
|
||||
p >= 0.05 → risk_score = 40 * (1 - p) ∈ [0, ~38] → "low"
|
||||
0.01 <= p < 0.05 → risk_score = 40 + ... ∈ [40, 60] → "medium"
|
||||
0.001 <= p < 0.01 → risk_score = 60 + ... ∈ [60, 80] → "medium-high"
|
||||
p < 0.001 → risk_score = 80 + ... ∈ [80, 100] → "high"
|
||||
|
||||
设计考量:
|
||||
- 不直接使用 1-p 作为分数(会导致 p=0.04 和 p=0.06 差距过小)
|
||||
- 分段线性映射,确保跨越统计显著性阈值时有明显跳变
|
||||
- 上限 100 永远不精确达到(留有余地表示"不确定性")
|
||||
"""
|
||||
```
|
||||
|
||||
### 5.2 综合评分算法
|
||||
|
||||
```python
|
||||
"""
|
||||
综合评分 = 0.6 × max(各模块分数) + 0.4 × mean(各模块分数)
|
||||
|
||||
设计理由:
|
||||
- 加权最大值确保"只要有一个模块高度异常,综合分就不会太低"
|
||||
- 加权平均确保"如果多个模块都略有异常,综合分会累积上升"
|
||||
- 0.6/0.4 比例经验性确定,偏向保守(避免漏检重于避免误报)
|
||||
|
||||
特殊规则:
|
||||
- 如果固定关系检测发现 is_exact=True,直接 risk_score = max(score, 90)
|
||||
- 如果图像检测发现 similarity > 0.98,直接 risk_score = max(score, 90)
|
||||
"""
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. 测试用例规范
|
||||
|
||||
### 6.1 单元测试结构
|
||||
|
||||
```python
|
||||
# tests/test_modules.py
|
||||
|
||||
"""
|
||||
测试策略:
|
||||
1. 已知正常数据 → 应返回 low risk
|
||||
2. 已知造假数据 → 应返回 high risk
|
||||
3. 边界条件 → 应优雅处理
|
||||
4. 回归测试 → 固定输入,固定输出
|
||||
"""
|
||||
|
||||
def test_last_digit_uniform_data():
|
||||
"""均匀分布数据应返回低风险"""
|
||||
import random
|
||||
random.seed(42)
|
||||
values = [str(random.uniform(1, 100)) for _ in range(100)]
|
||||
result = last_digit_test(values)
|
||||
assert result['risk_level'] == 'low'
|
||||
assert result['risk_score'] < 30
|
||||
|
||||
def test_fixed_relation_exact_ratio():
|
||||
"""精确固定比值应返回极高风险"""
|
||||
col1 = [1.23, 2.34, 3.45, 4.56, 5.67]
|
||||
col2 = [2.46, 4.68, 6.90, 9.12, 11.34] # 精确 ×2
|
||||
result = fixed_relation_test(col1, col2)
|
||||
assert result['risk_level'] == 'high'
|
||||
assert result['risk_score'] >= 85
|
||||
|
||||
def test_grim_consistent():
|
||||
"""合法均值应通过 GRIM"""
|
||||
# n=20, 整数数据, mean=3.40 → sum=68 ✓
|
||||
result = grim_test_single('3.40', 20, decimals=2)
|
||||
assert result['consistent'] == True
|
||||
|
||||
def test_grim_inconsistent():
|
||||
"""非法均值应失败"""
|
||||
# n=20, 整数数据, mean=3.47 → sum=69.4 ✗
|
||||
result = grim_test_single('3.47', 20, decimals=2)
|
||||
assert result['consistent'] == False
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. AI Agent 集成注释
|
||||
|
||||
### 7.1 Prompt Engineering 标注
|
||||
|
||||
每个模块的 docstring 设计为可被 AI Agent 直接解析:
|
||||
|
||||
```python
|
||||
"""
|
||||
[AGENT_INSTRUCTION]
|
||||
当用户要求检测数据造假时,按以下优先级选择模块:
|
||||
1. 如果用户提供了两组"应该独立"的数据 → fixed_relation_test()
|
||||
2. 如果数据跨越多个数量级 → benford_test()
|
||||
3. 如果数据含小数 → decimal_consistency_test() + last_digit_test()
|
||||
4. 如果用户提供了均值和样本量 → grim_test_single()
|
||||
5. 如果有图片文件 → find_duplicates()
|
||||
6. 一键全检 → geng_assess.py
|
||||
|
||||
[AGENT_OUTPUT_FORMAT]
|
||||
向用户展示结果时,使用以下格式:
|
||||
- 先给出综合评分和风险等级(一句话)
|
||||
- 然后列出关键发现(使用 emoji 标注严重度)
|
||||
- 最后给出建议行动(编号列表)
|
||||
- 始终附上免责声明
|
||||
"""
|
||||
```
|
||||
|
||||
### 7.2 Tool Definition 标注
|
||||
|
||||
```python
|
||||
"""
|
||||
[TOOL_DEFINITION]
|
||||
name: geng_fraud_detection
|
||||
description: |
|
||||
基于统计学原理检测学术论文数据是否存在造假迹象。
|
||||
支持末位数字检测、本福特定律、GRIM测试、固定关系检测、
|
||||
小数位一致性检测和图像重复检测。
|
||||
灵感来源于2026年"耿同学讲故事"的技术流打假方法论。
|
||||
input_schema:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: array
|
||||
description: 数据行列表,或 CSV 文件路径
|
||||
domain:
|
||||
type: string
|
||||
enum: [biomedical, chemistry, physics, social_science, clinical, general]
|
||||
modules:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enum: [last_digit, benford, grim, fixed_relation, decimal, image]
|
||||
description: 指定运行哪些模块(默认全部)
|
||||
output_schema:
|
||||
type: object
|
||||
properties:
|
||||
overall_risk_score: {type: number, min: 0, max: 100}
|
||||
overall_risk_level: {type: string}
|
||||
findings: {type: array, items: {type: string}}
|
||||
recommendations: {type: array, items: {type: string}}
|
||||
"""
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 性能与限制
|
||||
|
||||
### 8.1 时间复杂度
|
||||
|
||||
| 模块 | 时间复杂度 | 1000行数据耗时 |
|
||||
|------|-----------|---------------|
|
||||
| last_digit_test | O(n) | <10ms |
|
||||
| benford_test | O(n) | <10ms |
|
||||
| grim_test_batch | O(k) per item | <1ms/item |
|
||||
| fixed_relation_test | O(n) per pair | <10ms |
|
||||
| decimal_consistency_test | O(n) | <20ms |
|
||||
| image_duplicate_test | O(m²) m=图片数 | ~1s/100张 |
|
||||
| geng_assess (综合) | O(n × c²) c=列数 | <500ms |
|
||||
|
||||
### 8.2 已知限制
|
||||
|
||||
| 限制 | 影响 | 缓解方案 |
|
||||
|------|------|----------|
|
||||
| 数据量<30时统计效力低 | 本福特检测可能不准 | 自动标注 "统计效力有限" |
|
||||
| 不支持时间序列自相关 | 遗漏趋势数据伪造 | v1.1 计划增加 |
|
||||
| 固定关系仅检测两列 | 三列以上复杂关系漏检 | 通过两两组合覆盖 |
|
||||
| 图像检测仅用全局特征 | 局部篡改可能漏检 | v1.2 计划增加分块检测 |
|
||||
| 无法检测"高明造假" | 统计上完美的伪造数据 | 无银弹,需多维度交叉 |
|
||||
|
||||
---
|
||||
|
||||
*Geng Skill v1.0.0 — 代码注释与架构标准化文档*
|
||||
294
tools/gengskill/docs/DATA_SOURCES.md
Normal file
294
tools/gengskill/docs/DATA_SOURCES.md
Normal file
@ -0,0 +1,294 @@
|
||||
# 📚 数据来源与参考文献标准化文档
|
||||
|
||||
> Geng Skill v1.0.0 — 学术数据打假检测工具
|
||||
|
||||
---
|
||||
|
||||
## 1. 方法论来源
|
||||
|
||||
### 1.1 直接灵感来源
|
||||
|
||||
| 来源 | 描述 | 时间 |
|
||||
|------|------|------|
|
||||
| **耿同学讲故事** (B站/抖音科普博主) | 吉林大学生物学硕士、北航博士五年级退学。2026年4月起连续举报多所985高校教授论文造假,核心方法:末位数字集中度检测、固定差值/比例关系检测、AI图片查重 | 2026-04 至今 |
|
||||
| **澎湃新闻评论** | 《学术打假需要"耿同学",更需要长效机制建设》— 详述耿同学方法论 | 2026-05-16 |
|
||||
| **虎嗅网** | 《我Skill化了耿同学的"学术打假方法论",致敬》— 方法论结构化梳理 | 2026-05-08 |
|
||||
|
||||
### 1.2 耿同学核心方法总结
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 耿同学打假方法论(从公开报道中提取) │
|
||||
├────────────────────────────────────────────────────────────────┤
|
||||
│ 1. 末位数字集中度 — 某些数字出现频率异常高 │
|
||||
│ 2. 两列数据间固定差值/比例 — 不同组数据存在恒定数学关系 │
|
||||
│ 3. 小数点后位数高度一致 — 编造数据的小数位呈现不自然规律 │
|
||||
│ 4. AI图片查重 — 同一图片在不同实验条件下重复使用 │
|
||||
│ 5. 从PDF/Source Data/图片/表格多维度扒取证据 │
|
||||
│ 6. 卡方检验等统计学方法验证异常的显著性 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 统计学理论基础
|
||||
|
||||
### 2.1 本福特定律 (Benford's Law)
|
||||
|
||||
| 字段 | 内容 |
|
||||
|------|------|
|
||||
| **原始论文** | Benford, F. (1938). The law of anomalous numbers. *Proceedings of the American Philosophical Society*, 78(4), 551-572. |
|
||||
| **数学表述** | P(d) = log₁₀(1 + 1/d), d ∈ {1,2,...,9} |
|
||||
| **适用条件** | 数据跨越多个数量级(至少1个);数据量≥100为佳 |
|
||||
| **不适用场景** | 范围有限的数据(百分比、pH值);人为截断的数据 |
|
||||
| **权威教材** | Nigrini, M.J. (2012). *Benford's Law: Applications for Forensic Accounting, Auditing, and Fraud Detection*. Wiley. ISBN: 978-1118152850 |
|
||||
| **审计应用** | 美国注册欺诈审查师协会(ACFE)推荐用于财务审计 |
|
||||
| **学术验证** | Diekmann, A. (2007). Not the first digit! Using Benford's law to detect fraudulent scientific data. *Journal of Applied Statistics*, 34(3), 321-329. |
|
||||
|
||||
### 2.2 GRIM 测试 (Granularity-Related Inconsistency of Means)
|
||||
|
||||
| 字段 | 内容 |
|
||||
|------|------|
|
||||
| **原始论文** | Brown, N.J.L., & Heathers, J.A.J. (2017). The GRIM Test: A Simple Technique Detects Numerous Anomalies in the Reporting of Results in Psychology. *Social Psychological and Personality Science*, 8(4), 363-369. DOI: 10.1177/1948550616673876 |
|
||||
| **数学原理** | 对于整数取值数据,样本量为n时,合法均值只能是 k/n 形式(k为整数) |
|
||||
| **适用条件** | 离散整数取值数据(李克特量表、计数数据) |
|
||||
| **扩展** | SPRITE (Sample Parameter Reconstruction via Iterative TEchniques) — 更完整的数据重构验证 |
|
||||
| **参考** | Heathers, J.A.J., et al. (2018). SPRITE: A Response to Anaya's Critique. DOI: 10.31234/osf.io/qfk7d |
|
||||
|
||||
### 2.3 末位数字均匀分布检验
|
||||
|
||||
| 字段 | 内容 |
|
||||
|------|------|
|
||||
| **理论基础** | 连续测量数据在足够精度下,末位数字应服从离散均匀分布 U(0,9) |
|
||||
| **检验方法** | 皮尔逊卡方检验 (Pearson's chi-squared test), df=9 |
|
||||
| **参考文献** | Mosimann, J.E., et al. (2002). Terminal digits and the examination of questioned data. *Accountability in Research*, 9(2), 75-92. |
|
||||
| **典型案例** | Hill, T.P. (1998). The first digit phenomenon. *American Scientist*, 86, 358-363. |
|
||||
|
||||
### 2.4 图像重复检测
|
||||
|
||||
| 字段 | 内容 |
|
||||
|------|------|
|
||||
| **里程碑论文** | Bik, E.M., Casadevall, A., & Fang, F.C. (2016). The prevalence of inappropriate image duplication in biomedical research publications. *mBio*, 7(3), e00809-16. DOI: 10.1128/mBio.00809-16 |
|
||||
| **发现** | 分析20,621篇论文,3.8%存在图片问题 |
|
||||
| **技术方法** | 感知哈希(pHash)、差异哈希(dHash)、结构相似性(SSIM) |
|
||||
| **工具参考** | ImageTwin, Proofig, STM Integrity Hub |
|
||||
|
||||
### 2.5 数据一致性综合检验
|
||||
|
||||
| 字段 | 内容 |
|
||||
|------|------|
|
||||
| **中文权威** | 余菁, 邬加佳, 孙慧兰等 (2021). 科技论文数据造假的核查策略和统计学方法验证. *中国科技期刊研究*, 32(6), 770-776. DOI: 10.11946/cjstp.202012221043 |
|
||||
| **方法体系** | t检验、F检验、卡方检验、生存分析一致性 |
|
||||
| **国际标准** | COPE (Committee on Publication Ethics) Guidelines on Research Data |
|
||||
|
||||
---
|
||||
|
||||
## 3. 检测模块与理论对应关系
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ 模块名称 │ 理论基础 │ 统计方法 │ 适用领域 │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ Last Digit Test │ 末位均匀分布 │ χ² 检验 │ 全领域 │
|
||||
│ Benford's Law Test │ 本福特定律 │ χ² + MAD │ 跨数量级 │
|
||||
│ GRIM Test │ 离散粒度一致性 │ 整除验证 │ 社科/量表 │
|
||||
│ Fixed Relation Test │ 独立性原理 │ 比值/回归 │ 全领域(核心) │
|
||||
│ Decimal Consistency │ 随机性原理 │ 自相关+χ² │ 全领域 │
|
||||
│ Image Duplication │ 唯一性原理 │ 哈希+SSIM │ 生物医学 │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 已验证的真实案例
|
||||
|
||||
### 4.1 耿同学举报案例(2026年,已被机构确认)
|
||||
|
||||
| 案例 | 机构 | 期刊 | 问题类型 | 结果 |
|
||||
|------|------|------|----------|------|
|
||||
| 王平团队 | 同济大学生科院 | *Nature* | 系统性数据造假(固定数学关系、图片重复) | ✅ 确认,院长免职,第一作者解聘 |
|
||||
| 陈佺团队 | 南开大学生科院 | *Nature* 子刊 | 数据异常 | 🔄 调查中 |
|
||||
| 上海大学案例 | 上海大学 | — | 数据异常 | 🔄 调查中 |
|
||||
| 中山大学案例 | 中山大学 | — | 数据异常 | 🔄 调查中 |
|
||||
|
||||
### 4.2 国际经典案例
|
||||
|
||||
| 案例 | 方法 | 年份 |
|
||||
|------|------|------|
|
||||
| Diederik Stapel (社会心理学) | GRIM + 统计不一致性 | 2011 |
|
||||
| Paolo Macchiarini (再生医学) | 图像重复 + 数据伪造 | 2016 |
|
||||
| Hwang Woo-suk (干细胞) | 图像篡改检测 | 2005 |
|
||||
| Jan Hendrik Schön (物理) | 数据重复模式 | 2002 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 数据标准与输入规范
|
||||
|
||||
### 5.1 CSV 输入格式标准
|
||||
|
||||
```
|
||||
编码: UTF-8 (支持 UTF-8-BOM)
|
||||
分隔符: 逗号 (默认), 可配置为 TAB/分号
|
||||
表头: 必须有列名作为第一行
|
||||
数值: 支持整数、小数、科学计数法
|
||||
缺失值: 空字符串 (跳过处理)
|
||||
```
|
||||
|
||||
**标准示例:**
|
||||
```csv
|
||||
sample_id,group,value,measurement,timepoint
|
||||
1,control,2.34,12.5,0
|
||||
2,control,3.12,15.8,0
|
||||
3,treatment,4.68,25.0,24
|
||||
```
|
||||
|
||||
### 5.2 GRIM 批量输入 JSON 格式
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"label": "Table 1, Row 1",
|
||||
"mean": "3.47",
|
||||
"n": 25,
|
||||
"decimals": 2,
|
||||
"scale_min": 1,
|
||||
"scale_max": 5
|
||||
},
|
||||
{
|
||||
"label": "Table 1, Row 2",
|
||||
"mean": "4.12",
|
||||
"n": 30,
|
||||
"decimals": 2,
|
||||
"scale_min": 1,
|
||||
"scale_max": 5
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
### 5.3 图像输入规范
|
||||
|
||||
```
|
||||
支持格式: PNG, JPG, JPEG, TIF, TIFF, BMP, GIF
|
||||
最小尺寸: 32×32 像素
|
||||
推荐: 原始分辨率(不要人为缩放)
|
||||
组织方式: 所有待比较图片放在同一目录下
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. 输出标准化
|
||||
|
||||
### 6.1 JSON 输出 Schema
|
||||
|
||||
所有模块遵循统一输出结构:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "geng-skill-output-v1",
|
||||
"test_name": "string — 模块名称(中英双语)",
|
||||
"status": "enum: completed | insufficient_data | error",
|
||||
"n_values": "integer — 有效数据点数",
|
||||
"risk_level": "enum: low | medium | medium-high | high",
|
||||
"risk_score": "number 0-100 — 风险评分",
|
||||
"p_value": "number — 统计检验p值(如适用)",
|
||||
"interpretation": "string — 中文可读解释(含emoji状态标识)",
|
||||
"...": "模块特定字段"
|
||||
}
|
||||
```
|
||||
|
||||
### 6.2 风险评分映射标准
|
||||
|
||||
| p-value 范围 | 风险等级 | 评分范围 | 颜色代码 | 建议动作 |
|
||||
|-------------|----------|----------|----------|----------|
|
||||
| p > 0.05 | low | 0-25 | 🟢 #00C853 | 无需干预 |
|
||||
| 0.01 < p ≤ 0.05 | medium | 26-50 | 🟡 #FFD600 | 人工复核 |
|
||||
| 0.001 < p ≤ 0.01 | medium-high | 51-75 | 🟠 #FF6D00 | 深入调查 |
|
||||
| p ≤ 0.001 | high | 76-100 | 🔴 #D50000 | 正式举报 |
|
||||
|
||||
### 6.3 Markdown 报告标准
|
||||
|
||||
综合报告遵循以下结构:
|
||||
|
||||
```markdown
|
||||
# 📋 Geng 学术数据打假检测报告
|
||||
## 📊 综合评估结果(表格)
|
||||
## 📝 结论(一段话)
|
||||
## 💡 建议(编号列表)
|
||||
## 🔬 各模块检测详情
|
||||
### Module N: 模块名称
|
||||
## ⚠️ 重要声明
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. 学术伦理与法律合规
|
||||
|
||||
### 7.1 合规框架
|
||||
|
||||
| 标准/规范 | 发布机构 | 相关性 |
|
||||
|-----------|----------|--------|
|
||||
| COPE Retraction Guidelines | 出版伦理委员会 | 论文撤稿/更正流程 |
|
||||
| 科研诚信案件调查处理规则 | 中国科技部 (2019) | 国内学术不端处理 |
|
||||
| ORI Research Integrity Guidelines | 美国研究诚信办公室 | 国际标准 |
|
||||
| Singapore Statement | 全球科研诚信大会 | 负责任研究行为 |
|
||||
|
||||
### 7.2 使用伦理准则
|
||||
|
||||
1. **比例原则** — 检测强度应与嫌疑程度成正比
|
||||
2. **无罪推定** — 异常 ≠ 造假,需完整证据链
|
||||
3. **保密义务** — 未经确认的检测结果不应公开传播
|
||||
4. **正式渠道** — 确认后应通过机构/期刊正式途径举报
|
||||
5. **避免伤害** — 不应基于工具结果对个人进行网络攻击
|
||||
|
||||
### 7.3 免责声明
|
||||
|
||||
```
|
||||
本工具仅提供统计学层面的异常筛查功能,输出结果为"疑点线索"而非
|
||||
"造假定论"。使用者应当理解:
|
||||
- 统计异常可能有合理解释(仪器精度、数据处理等)
|
||||
- 本工具不具备法律效力
|
||||
- 最终判定需要领域专家、原始数据核查和正式调查程序
|
||||
- 使用者需自行承担因不当使用造成的后果
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 版本与更新日志
|
||||
|
||||
### v1.0.0 (2026-05-20)
|
||||
|
||||
- 初始发布
|
||||
- 6个核心检测模块
|
||||
- 综合评估引擎
|
||||
- 多平台使用指南
|
||||
- 标准化输出格式
|
||||
|
||||
### 路线图
|
||||
|
||||
| 版本 | 计划功能 |
|
||||
|------|----------|
|
||||
| v1.1 | 增加 SPRITE 测试、生存数据一致性检验 |
|
||||
| v1.2 | 支持 Excel 直接输入、PDF 表格自动提取 |
|
||||
| v1.3 | Web UI 界面、RESTful API |
|
||||
| v2.0 | AI 增强检测(LLM 辅助判断上下文合理性) |
|
||||
|
||||
---
|
||||
|
||||
## 9. 引用本工具
|
||||
|
||||
如果在学术工作中使用了本工具,请引用:
|
||||
|
||||
```bibtex
|
||||
@software{geng_skill_2026,
|
||||
title = {Geng Skill: Academic Data Fraud Detection Toolkit},
|
||||
author = {Contributors},
|
||||
year = {2026},
|
||||
url = {https://github.com/YOUR_USERNAME/geng-skill},
|
||||
version = {1.0.0},
|
||||
note = {Inspired by the methodology of "Geng Tongxue" (耿同学讲故事)}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
*Geng Skill — 让学术回归诚信,让数据说出真相。*
|
||||
258
tools/gengskill/docs/EXAMPLE_WALKTHROUGH.md
Normal file
258
tools/gengskill/docs/EXAMPLE_WALKTHROUGH.md
Normal file
@ -0,0 +1,258 @@
|
||||
# 🧪 完整示例:从数据到报告的端到端演示
|
||||
|
||||
> 本文档通过一个完整的假数据检测案例,演示 Geng Skill 的全部使用流程。
|
||||
|
||||
---
|
||||
|
||||
## 场景设定
|
||||
|
||||
假设你在审阅一篇生物医学论文,论文声称:
|
||||
|
||||
> "我们分别对小鼠进行了 Control、Treatment A、Treatment B 三组实验处理,
|
||||
> 测量了各组的蛋白表达水平(相对定量)。结果显示 Treatment A 和 Treatment B
|
||||
> 均显著提高了蛋白表达水平。"
|
||||
|
||||
论文提供了以下数据(摘自 Supplementary Table 1):
|
||||
|
||||
```csv
|
||||
sample_id,control_group,treatment_a,treatment_b,measurement
|
||||
1,2.34,4.68,7.02,12.5
|
||||
2,3.12,6.24,9.36,15.8
|
||||
3,1.87,3.74,5.61,8.9
|
||||
4,4.56,9.12,13.68,22.1
|
||||
5,2.98,5.96,8.94,14.3
|
||||
6,3.45,6.90,10.35,17.2
|
||||
7,1.23,2.46,3.69,6.8
|
||||
8,5.67,11.34,17.01,28.4
|
||||
9,2.01,4.02,6.03,10.1
|
||||
10,3.89,7.78,11.67,19.5
|
||||
11,4.12,8.24,12.36,20.8
|
||||
12,1.56,3.12,4.68,7.9
|
||||
13,2.78,5.56,8.34,13.6
|
||||
14,3.34,6.68,10.02,16.7
|
||||
15,4.90,9.80,14.70,24.5
|
||||
16,1.45,2.90,4.35,7.2
|
||||
17,2.67,5.34,8.01,13.1
|
||||
18,3.56,7.12,10.68,17.8
|
||||
19,4.23,8.46,12.69,21.2
|
||||
20,1.89,3.78,5.67,9.4
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 步骤 1:初步目视检查
|
||||
|
||||
一位细心的审稿人可能注意到:
|
||||
- Treatment A 的数值似乎都是 Control 的两倍
|
||||
- Treatment B 的数值似乎都是 Control 的三倍
|
||||
|
||||
但仅凭目测无法确认。让我们用 Geng Skill 做系统化检测。
|
||||
|
||||
---
|
||||
|
||||
## 步骤 2:运行检测
|
||||
|
||||
### 2.1 命令行一键检测
|
||||
|
||||
```bash
|
||||
cd geng-skill/scripts
|
||||
python3 geng_assess.py \
|
||||
--input ../examples/fake_data_demo.csv \
|
||||
--domain biomedical \
|
||||
--output ../report/
|
||||
```
|
||||
|
||||
### 2.2 Python API 方式
|
||||
|
||||
```python
|
||||
import sys
|
||||
sys.path.insert(0, 'scripts')
|
||||
|
||||
from last_digit_test import last_digit_test
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
import csv
|
||||
|
||||
# 加载数据
|
||||
with open('examples/fake_data_demo.csv') as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
|
||||
control = [float(r['control_group']) for r in rows]
|
||||
treat_a = [float(r['treatment_a']) for r in rows]
|
||||
treat_b = [float(r['treatment_b']) for r in rows]
|
||||
|
||||
# 运行固定关系检测
|
||||
result = fixed_relation_test(control, treat_a, 'Control', 'Treatment A')
|
||||
print(f"风险评分: {result['risk_score']}/100")
|
||||
print(f"解释: {result['interpretation']}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 步骤 3:检测结果详解
|
||||
|
||||
### Module 1: 末位数字检测
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 检测列: control_group │
|
||||
│ 末位数字分布: {0:0, 1:1, 2:2, 3:2, 4:2, 5:2, 6:3, 7:3, 8:2, 9:3} │
|
||||
│ χ² = 4.00, p = 0.9114 │
|
||||
│ 结果: ✅ 正常 — 末位数字分布与均匀分布无显著差异 │
|
||||
│ 风险评分: 3.5/100 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**解读**:造假者在编造 Control 组数据时,末位数字分布还算随机。这说明末位数字检测并非万能——它无法检测"有一定水平"的造假。
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 检测列: treatment_a │
|
||||
│ χ² = 21.00, p = 0.0127 │
|
||||
│ 结果: ⚠️ 异常 — 末位数字分布存在偏离 │
|
||||
│ 风险评分: 47.5/100 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**解读**:Treatment A 的末位数字分布出现异常。这是因为 Control × 2 导致了末位数字的非均匀映射(如原数 .34 × 2 = .68,原数 .56 × 2 = .12)。
|
||||
|
||||
---
|
||||
|
||||
### Module 4: 固定关系检测 ⭐(核心发现)
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 检测对: Control vs Treatment A │
|
||||
│ │
|
||||
│ 固定比值检测: │
|
||||
│ mean_ratio = 2.000000 │
|
||||
│ std_ratio = 0.000000 │
|
||||
│ → 🔴 完美固定比值!所有数据点 Treatment_A = Control × 2 │
|
||||
│ │
|
||||
│ 线性关系检测: │
|
||||
│ R² = 1.0000000000 │
|
||||
│ slope = 2.0000 │
|
||||
│ intercept = 0.000000 │
|
||||
│ → 🔴 完美线性关系,零残差 │
|
||||
│ │
|
||||
│ 风险评分: 95/100 — 极高风险 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 检测对: Control vs Treatment B │
|
||||
│ │
|
||||
│ 固定比值检测: │
|
||||
│ mean_ratio = 3.000000 │
|
||||
│ std_ratio = 0.000000 │
|
||||
│ → 🔴 完美固定比值!所有数据点 Treatment_B = Control × 3 │
|
||||
│ │
|
||||
│ 风险评分: 95/100 — 极高风险 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ 检测对: Treatment A vs Treatment B │
|
||||
│ │
|
||||
│ 固定比值检测: │
|
||||
│ mean_ratio = 1.500000 │
|
||||
│ std_ratio = 0.000000 │
|
||||
│ → 🔴 完美固定比值!Treatment_B = Treatment_A × 1.5 │
|
||||
│ │
|
||||
│ 风险评分: 95/100 — 极高风险 │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**解读**:这是最致命的发现。三组"独立实验"数据之间存在精确的整数倍关系:
|
||||
- Treatment A = Control × 2.000(精确到小数点后所有位)
|
||||
- Treatment B = Control × 3.000(精确到小数点后所有位)
|
||||
- Treatment B = Treatment A × 1.500(精确到小数点后所有位)
|
||||
|
||||
**在真实生物实验中,这种完美的整数倍关系概率趋近于零。** 即使药物真的将蛋白表达提高了2倍,每个样本的响应也会有生物学变异(个体差异、实验误差等),绝不可能所有20个样本都精确地是2.000倍。
|
||||
|
||||
---
|
||||
|
||||
### 综合评估
|
||||
|
||||
```
|
||||
╔══════════════════════════════════════════════════════════════════╗
|
||||
║ 📋 综合评估结果 ║
|
||||
╠══════════════════════════════════════════════════════════════════╣
|
||||
║ ║
|
||||
║ 🔴 综合风险评分: 92/100 — 极高风险 ║
|
||||
║ ║
|
||||
║ 核心证据: ║
|
||||
║ • 三组"独立实验"数据存在精确整数倍关系(×2, ×3, ×1.5) ║
|
||||
║ • R² = 1.0,残差为零 ║
|
||||
║ • Treatment A 末位数字分布异常(p = 0.013) ║
|
||||
║ ║
|
||||
║ 结论: 数据极大概率为从单一数据源(Control组)通过简单 ║
|
||||
║ 乘法运算生成,而非独立实验获得。 ║
|
||||
║ ║
|
||||
╚══════════════════════════════════════════════════════════════════╝
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 步骤 4:与正常数据对比
|
||||
|
||||
对 `real_data_demo.csv`(模拟的正常实验数据)运行同样的检测:
|
||||
|
||||
```
|
||||
检测结果:
|
||||
• 末位数字: ✅ 所有列 p > 0.05
|
||||
• 本福特定律: ✅ 符合
|
||||
• 固定关系: ✅ 无固定比值/差值(ratio_std > 1.5)
|
||||
• 小数位一致性: ✅ 模式多样
|
||||
|
||||
综合风险评分: 8/100 — 🟢 低风险
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 步骤 5:生成正式报告
|
||||
|
||||
运行完成后,`report/` 目录包含:
|
||||
|
||||
```
|
||||
report/
|
||||
├── geng_assessment_report.json # 机器可读完整报告
|
||||
└── geng_assessment_report.md # 人类可读 Markdown 报告
|
||||
```
|
||||
|
||||
报告可直接用于:
|
||||
- 向期刊提交 Letter of Concern
|
||||
- 向机构科研诚信办公室提供技术证据
|
||||
- 审稿意见中引用具体检测结果
|
||||
|
||||
---
|
||||
|
||||
## 关键教训
|
||||
|
||||
| 教训 | 说明 |
|
||||
|------|------|
|
||||
| **单一检测不足以定论** | 末位数字检测对 Control 组未报警,但固定关系检测精准命中 |
|
||||
| **多模块交叉验证更可靠** | 末位异常 + 固定比值 + 完美线性 = 综合证据链 |
|
||||
| **异常不等于造假** | 需要排除合理解释(如数据预处理中的标准化操作) |
|
||||
| **上下文很重要** | 同样的"固定比值"在"原始数据 vs 标准化后数据"场景中是正常的 |
|
||||
| **工具是辅助,人是决策者** | 最终判断需要领域专家结合实验设计做出 |
|
||||
|
||||
---
|
||||
|
||||
## 对审稿人/研究者的实用建议
|
||||
|
||||
### 何时应该怀疑数据?
|
||||
|
||||
1. ✋ 不同实验组数据太"干净"——没有离群值、没有异常
|
||||
2. ✋ 多组数据的 error bar 高度一致
|
||||
3. ✋ 不同条件下的重复次数完全一致
|
||||
4. ✋ 数据点"太完美"地落在预期曲线上
|
||||
5. ✋ 补充材料中的原始数据与正文图表不匹配
|
||||
|
||||
### 何时应该运行 Geng Skill?
|
||||
|
||||
1. 📊 审阅高利害关系的论文(顶刊/基金/职称)
|
||||
2. 📊 收到学术不端举报后需要技术验证
|
||||
3. 📊 对自己团队数据做"造假预防"自查
|
||||
4. 📊 期刊编辑部建立投稿数据审查流程
|
||||
|
||||
---
|
||||
|
||||
*Geng Skill v1.0.0 — 完整示例演示*
|
||||
654
tools/gengskill/docs/USAGE_GUIDE.md
Normal file
654
tools/gengskill/docs/USAGE_GUIDE.md
Normal file
@ -0,0 +1,654 @@
|
||||
# 📖 Geng Skill 多平台使用指南
|
||||
|
||||
> 本文档详细说明 Geng Skill 在各主流 AI 编程助手和开发环境中的使用方法。
|
||||
|
||||
---
|
||||
|
||||
## 目录
|
||||
|
||||
1. [通用命令行使用](#1-通用命令行使用)
|
||||
2. [在 Claude (Anthropic) 中使用](#2-在-claude-anthropic-中使用)
|
||||
3. [在 Cursor 中使用](#3-在-cursor-中使用)
|
||||
4. [在 ChatGPT / GPT-4 中使用](#4-在-chatgpt--gpt-4-中使用)
|
||||
5. [在 OpenAI Codex / API 中使用](#5-在-openai-codex--api-中使用)
|
||||
6. [在 GitHub Copilot 中使用](#6-在-github-copilot-中使用)
|
||||
7. [在 Jupyter Notebook 中使用](#7-在-jupyter-notebook-中使用)
|
||||
8. [作为 Python 库导入使用](#8-作为-python-库导入使用)
|
||||
9. [CI/CD 自动化集成](#9-cicd-自动化集成)
|
||||
|
||||
---
|
||||
|
||||
## 1. 通用命令行使用
|
||||
|
||||
### 1.1 安装
|
||||
|
||||
```bash
|
||||
# 克隆仓库
|
||||
git clone https://github.com/YOUR_USERNAME/geng-skill.git
|
||||
cd geng-skill
|
||||
|
||||
# 安装依赖
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### 1.2 一键综合检测
|
||||
|
||||
```bash
|
||||
cd scripts
|
||||
python3 geng_assess.py \
|
||||
--input ../examples/fake_data_demo.csv \
|
||||
--domain biomedical \
|
||||
--output ../report/
|
||||
```
|
||||
|
||||
**参数说明:**
|
||||
|
||||
| 参数 | 必填 | 说明 | 可选值 |
|
||||
|------|------|------|--------|
|
||||
| `--input` / `-i` | ✅ | 输入 CSV 文件路径 | 任意 .csv 文件 |
|
||||
| `--domain` | ❌ | 研究领域(影响检测策略) | `biomedical`, `chemistry`, `physics`, `social_science`, `clinical`, `general` |
|
||||
| `--output` / `-o` | ❌ | 输出报告目录 | 默认 `./report` |
|
||||
| `--delimiter` / `-d` | ❌ | CSV 分隔符 | 默认 `,` |
|
||||
|
||||
### 1.3 单项检测
|
||||
|
||||
```bash
|
||||
# 末位数字检测
|
||||
python3 last_digit_test.py -i data.csv -c "column_name" -o result.json
|
||||
|
||||
# 本福特定律检测
|
||||
python3 benford_test.py -i data.csv -c "measurement" -o result.json
|
||||
|
||||
# GRIM 测试(单个均值)
|
||||
python3 grim_test.py --mean 3.47 --n 25 --scale "1-5" --decimals 2
|
||||
|
||||
# GRIM 测试(批量,从 JSON)
|
||||
python3 grim_test.py -i batch_means.json -o grim_results.json
|
||||
|
||||
# 固定关系检测
|
||||
python3 fixed_relation_test.py -i data.csv --col1 "group_a" --col2 "group_b"
|
||||
|
||||
# 小数位一致性检测
|
||||
python3 decimal_consistency_test.py -i data.csv -c "value"
|
||||
|
||||
# 图像重复检测
|
||||
python3 image_duplicate_test.py -i ./figures/ -t 0.85
|
||||
```
|
||||
|
||||
### 1.4 输出格式
|
||||
|
||||
所有模块输出标准 JSON 格式,包含以下统一字段:
|
||||
|
||||
```json
|
||||
{
|
||||
"test_name": "检测模块名称(中英文)",
|
||||
"status": "completed | insufficient_data | error",
|
||||
"risk_level": "low | medium | medium-high | high",
|
||||
"risk_score": 0-100,
|
||||
"interpretation": "中文可读解释",
|
||||
"...": "模块特定字段"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 在 Claude (Anthropic) 中使用
|
||||
|
||||
### 2.1 Claude Web / Claude Pro
|
||||
|
||||
**方法 A:直接粘贴数据让 Claude 分析**
|
||||
|
||||
```
|
||||
我有以下实验数据,请用"耿同学"的方法帮我检查是否存在数据造假迹象:
|
||||
|
||||
sample,control,treatment_a,treatment_b
|
||||
1,2.34,4.68,7.02
|
||||
2,3.12,6.24,9.36
|
||||
3,1.87,3.74,5.61
|
||||
...
|
||||
|
||||
请检查:
|
||||
1. 末位数字分布是否均匀
|
||||
2. 各组数据之间是否存在固定比值或差值关系
|
||||
3. 小数位模式是否异常
|
||||
```
|
||||
|
||||
**方法 B:上传 CSV 文件让 Claude 用代码分析**
|
||||
|
||||
```
|
||||
请帮我运行学术数据打假检测。我上传的 CSV 文件包含论文中的实验数据。
|
||||
请用以下方法逐一检测:
|
||||
- Last Digit Test(末位数字检测)
|
||||
- Benford's Law Test(本福特定律检测)
|
||||
- Fixed Relationship Detection(固定关系检测)
|
||||
- Decimal Consistency Test(小数位一致性检测)
|
||||
|
||||
最后给出综合风险评分和建议。
|
||||
```
|
||||
|
||||
### 2.2 Claude API (Artifacts / Tool Use)
|
||||
|
||||
```python
|
||||
import anthropic
|
||||
|
||||
client = anthropic.Anthropic()
|
||||
|
||||
# 将 Geng Skill 的 SKILL.md 作为 system prompt
|
||||
with open('SKILL.md', 'r') as f:
|
||||
skill_doc = f.read()
|
||||
|
||||
message = client.messages.create(
|
||||
model="claude-sonnet-4-20250514",
|
||||
max_tokens=4096,
|
||||
system=f"你是学术数据打假助手。请严格按照以下 Skill 文档执行检测:\n\n{skill_doc}",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "请对以下数据执行完整的 Geng 打假检测...[数据]"
|
||||
}]
|
||||
)
|
||||
```
|
||||
|
||||
### 2.3 Claude MCP (Model Context Protocol)
|
||||
|
||||
将 Geng Skill 注册为 MCP Server:
|
||||
|
||||
```json
|
||||
// claude_desktop_config.json
|
||||
{
|
||||
"mcpServers": {
|
||||
"geng-skill": {
|
||||
"command": "python3",
|
||||
"args": ["/path/to/geng-skill/scripts/mcp_server.py"],
|
||||
"env": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. 在 Cursor 中使用
|
||||
|
||||
### 3.1 作为 Cursor Rules 使用
|
||||
|
||||
在项目根目录创建 `.cursor/rules/geng-skill.mdc`:
|
||||
|
||||
```markdown
|
||||
---
|
||||
description: 学术数据打假检测工具
|
||||
globs: ["*.csv", "*.xlsx", "data/**"]
|
||||
alwaysApply: false
|
||||
---
|
||||
|
||||
# Geng Skill — 学术数据打假检测
|
||||
|
||||
当用户要求检测数据是否造假时,按以下步骤执行:
|
||||
|
||||
1. 确认数据格式(CSV/Excel/直接粘贴)
|
||||
2. 识别数值列
|
||||
3. 对每个数值列执行:
|
||||
- 末位数字检测(卡方检验 vs 均匀分布)
|
||||
- 本福特定律检测(适用于跨数量级数据)
|
||||
- 小数位一致性检测
|
||||
4. 对数值列两两执行:
|
||||
- 固定关系检测(差值/比值/线性)
|
||||
5. 综合评分(0-100)并给出建议
|
||||
|
||||
核心原则:自然数据具有随机性,人为编造的数据会呈现不自然的规律性。
|
||||
```
|
||||
|
||||
### 3.2 在 Cursor Chat 中使用
|
||||
|
||||
```
|
||||
@geng-skill 请检测这份数据文件 data/experiment_results.csv 是否存在造假迹象
|
||||
|
||||
重点关注:
|
||||
- 不同实验组之间是否有固定数学关系
|
||||
- 末位数字分布是否正常
|
||||
- 小数位模式是否异常
|
||||
```
|
||||
|
||||
### 3.3 Cursor Composer 自动化
|
||||
|
||||
在 Cursor Composer 中直接引用脚本:
|
||||
|
||||
```
|
||||
请运行 geng-skill/scripts/geng_assess.py 对 data/paper_results.csv 进行检测,
|
||||
领域设为 biomedical,输出到 report/ 目录。
|
||||
然后帮我解读报告中的关键发现。
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. 在 ChatGPT / GPT-4 中使用
|
||||
|
||||
### 4.1 ChatGPT Web (Code Interpreter / Advanced Data Analysis)
|
||||
|
||||
**步骤:**
|
||||
1. 上传 CSV 数据文件
|
||||
2. 同时上传 `scripts/` 目录下的 Python 脚本
|
||||
3. 提示词:
|
||||
|
||||
```
|
||||
我上传了一组学术论文数据和几个检测脚本。请按照以下步骤执行学术数据打假检测:
|
||||
|
||||
1. 先读取 CSV 数据,识别所有数值列
|
||||
2. 对每个数值列运行 last_digit_test.py 中的 last_digit_test() 函数
|
||||
3. 对适用的列运行 benford_test.py 中的 benford_test() 函数
|
||||
4. 对所有数值列对运行 fixed_relation_test.py 中的 fixed_relation_test() 函数
|
||||
5. 对每个数值列运行 decimal_consistency_test.py 中的 decimal_consistency_test() 函数
|
||||
|
||||
最后综合所有结果,给出:
|
||||
- 综合风险评分(0-100)
|
||||
- 关键发现(哪些数据可疑,为什么)
|
||||
- 建议行动
|
||||
```
|
||||
|
||||
### 4.2 GPT-4 API + Function Calling
|
||||
|
||||
```python
|
||||
import openai
|
||||
import json
|
||||
|
||||
# 定义 Geng Skill 工具
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "geng_last_digit_test",
|
||||
"description": "检测数据末位数字是否偏离均匀分布。自然数据末位应均匀分布,造假数据往往集中在某些数字。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"values": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "待检测的数值列表(字符串形式保留精度)"
|
||||
},
|
||||
"method": {
|
||||
"type": "string",
|
||||
"enum": ["all_digits", "decimal_last"],
|
||||
"description": "检测方法"
|
||||
}
|
||||
},
|
||||
"required": ["values"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "geng_fixed_relation_test",
|
||||
"description": "检测两组数据间是否存在固定差值、比值或完美线性关系。独立实验数据不应有精确数学关系。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"col1": {"type": "array", "items": {"type": "number"}, "description": "第一列数据"},
|
||||
"col2": {"type": "array", "items": {"type": "number"}, "description": "第二列数据"},
|
||||
"col1_name": {"type": "string"},
|
||||
"col2_name": {"type": "string"}
|
||||
},
|
||||
"required": ["col1", "col2"]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "geng_benford_test",
|
||||
"description": "检测数据首位数字是否符合本福特定律。适用于跨多个数量级的自然数据。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"values": {"type": "array", "items": {"type": "string"}, "description": "数值列表"}
|
||||
},
|
||||
"required": ["values"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# 调用 GPT-4 带工具
|
||||
response = openai.chat.completions.create(
|
||||
model="gpt-4",
|
||||
messages=[
|
||||
{"role": "system", "content": "你是学术数据打假助手,使用 Geng Skill 检测论文数据。"},
|
||||
{"role": "user", "content": "请检测以下数据..."}
|
||||
],
|
||||
tools=tools,
|
||||
tool_choice="auto"
|
||||
)
|
||||
```
|
||||
|
||||
### 4.3 Custom GPT (GPTs Store)
|
||||
|
||||
创建自定义 GPT,在 Instructions 中粘贴完整的 `SKILL.md` 内容,并上传所有脚本文件作为 Knowledge。
|
||||
|
||||
**GPT 名称建议**: "学术数据卫士 — Geng Fraud Detector"
|
||||
|
||||
**Instructions 要点**:
|
||||
```
|
||||
你是基于"耿同学"方法论的学术数据打假检测 GPT。
|
||||
当用户上传数据或粘贴数据时,自动执行以下检测流程...
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 在 OpenAI Codex / API 中使用
|
||||
|
||||
### 5.1 Codex CLI
|
||||
|
||||
```bash
|
||||
# 安装 Codex CLI
|
||||
npm install -g @openai/codex
|
||||
|
||||
# 使用 Geng Skill 检测数据
|
||||
codex "请对 data.csv 文件运行学术数据打假检测:\
|
||||
1. 读取所有数值列 \
|
||||
2. 检测末位数字分布 \
|
||||
3. 检测列间固定关系 \
|
||||
4. 给出风险评分" \
|
||||
--file data.csv \
|
||||
--file scripts/last_digit_test.py \
|
||||
--file scripts/fixed_relation_test.py
|
||||
```
|
||||
|
||||
### 5.2 Codex 作为自动化 Agent
|
||||
|
||||
```python
|
||||
# codex_geng_agent.py
|
||||
"""
|
||||
将 Geng Skill 封装为 Codex Agent 可调用的工具链
|
||||
"""
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
def run_geng_assessment(csv_path, domain="general"):
|
||||
"""调用 Geng 综合评估引擎"""
|
||||
result = subprocess.run(
|
||||
["python3", "scripts/geng_assess.py",
|
||||
"--input", csv_path,
|
||||
"--domain", domain,
|
||||
"--output", "./report/"],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
|
||||
# 读取报告
|
||||
with open("./report/geng_assessment_report.json", "r") as f:
|
||||
report = json.load(f)
|
||||
|
||||
return report
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. 在 GitHub Copilot 中使用
|
||||
|
||||
### 6.1 Copilot Chat in VS Code
|
||||
|
||||
在 VS Code 中打开数据文件,然后使用 Copilot Chat:
|
||||
|
||||
```
|
||||
@workspace /explain 请分析 data.csv 中的数据是否存在学术造假迹象,
|
||||
使用 geng-skill/scripts/ 中的检测模块
|
||||
```
|
||||
|
||||
### 6.2 Copilot in Terminal
|
||||
|
||||
```bash
|
||||
# GitHub Copilot CLI
|
||||
gh copilot suggest "run geng academic fraud detection on experiment_data.csv"
|
||||
```
|
||||
|
||||
### 6.3 作为 GitHub Action
|
||||
|
||||
```yaml
|
||||
# .github/workflows/geng-check.yml
|
||||
name: Academic Data Integrity Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'data/**/*.csv'
|
||||
|
||||
jobs:
|
||||
geng-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r geng-skill/requirements.txt
|
||||
|
||||
- name: Run Geng Assessment
|
||||
run: |
|
||||
cd geng-skill/scripts
|
||||
for csv_file in $(find ../../data -name "*.csv"); do
|
||||
echo "🔍 Checking: $csv_file"
|
||||
python3 geng_assess.py -i "$csv_file" -o ../../report/
|
||||
done
|
||||
|
||||
- name: Upload Report
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: geng-report
|
||||
path: report/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. 在 Jupyter Notebook 中使用
|
||||
|
||||
```python
|
||||
# Cell 1: 安装与导入
|
||||
!pip install numpy scipy Pillow scikit-image -q
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, '../scripts')
|
||||
|
||||
from last_digit_test import last_digit_test
|
||||
from benford_test import benford_test
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
from decimal_consistency_test import decimal_consistency_test
|
||||
from grim_test import grim_test_single, grim_test_batch
|
||||
|
||||
import pandas as pd
|
||||
import json
|
||||
|
||||
# Cell 2: 加载数据
|
||||
df = pd.read_csv('../examples/fake_data_demo.csv')
|
||||
print(f"数据形状: {df.shape}")
|
||||
df.head()
|
||||
|
||||
# Cell 3: 末位数字检测
|
||||
result = last_digit_test(df['control_group'].astype(str).tolist())
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
# Cell 4: 固定关系检测
|
||||
result = fixed_relation_test(
|
||||
df['control_group'].tolist(),
|
||||
df['treatment_a'].tolist(),
|
||||
'control_group', 'treatment_a'
|
||||
)
|
||||
print(f"🎯 风险评分: {result['risk_score']}/100")
|
||||
print(f"📝 {result['interpretation']}")
|
||||
|
||||
# Cell 5: 可视化
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
|
||||
|
||||
# 散点图:展示固定比值关系
|
||||
axes[0].scatter(df['control_group'], df['treatment_a'], c='red', alpha=0.7)
|
||||
axes[0].set_xlabel('Control Group')
|
||||
axes[0].set_ylabel('Treatment A')
|
||||
axes[0].set_title('⚠️ 完美 2x 关系')
|
||||
axes[0].plot([0, 6], [0, 12], 'k--', alpha=0.3)
|
||||
|
||||
# 末位数字分布
|
||||
from collections import Counter
|
||||
digits = [int(str(v)[-1]) for v in df['control_group'].astype(str)]
|
||||
counts = Counter(digits)
|
||||
axes[1].bar(range(10), [counts.get(i, 0) for i in range(10)])
|
||||
axes[1].axhline(y=len(digits)/10, color='r', linestyle='--', label='期望值')
|
||||
axes[1].set_xlabel('末位数字')
|
||||
axes[1].set_ylabel('频次')
|
||||
axes[1].set_title('末位数字分布')
|
||||
axes[1].legend()
|
||||
|
||||
# 比值分布
|
||||
ratios = df['treatment_a'] / df['control_group']
|
||||
axes[2].hist(ratios, bins=20, edgecolor='black')
|
||||
axes[2].set_xlabel('Treatment_A / Control')
|
||||
axes[2].set_ylabel('频次')
|
||||
axes[2].set_title(f'⚠️ 比值全部 = {ratios.mean():.1f}')
|
||||
|
||||
plt.tight_layout()
|
||||
plt.savefig('../report/detection_visualization.png', dpi=150)
|
||||
plt.show()
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. 作为 Python 库导入使用
|
||||
|
||||
### 8.1 基础用法
|
||||
|
||||
```python
|
||||
import sys
|
||||
sys.path.insert(0, '/path/to/geng-skill/scripts')
|
||||
|
||||
from last_digit_test import last_digit_test
|
||||
from benford_test import benford_test
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
from decimal_consistency_test import decimal_consistency_test
|
||||
from grim_test import grim_test_single
|
||||
|
||||
# 单列检测
|
||||
values = ['2.34', '3.12', '1.87', '4.56', '2.98']
|
||||
result = last_digit_test(values)
|
||||
print(f"风险评分: {result['risk_score']}")
|
||||
|
||||
# 两列关系检测
|
||||
col_a = [2.34, 3.12, 1.87, 4.56, 2.98]
|
||||
col_b = [4.68, 6.24, 3.74, 9.12, 5.96]
|
||||
result = fixed_relation_test(col_a, col_b, 'GroupA', 'GroupB')
|
||||
print(f"风险等级: {result['risk_level']}")
|
||||
|
||||
# GRIM 测试
|
||||
result = grim_test_single(mean='3.47', n=25, decimals=2, scale_min=1, scale_max=5)
|
||||
print(f"一致性: {result['consistent']}")
|
||||
```
|
||||
|
||||
### 8.2 批量处理多篇论文
|
||||
|
||||
```python
|
||||
import os
|
||||
import glob
|
||||
import json
|
||||
from geng_assess import run_assessment
|
||||
|
||||
# 批量检测目录下所有 CSV
|
||||
csv_files = glob.glob('/path/to/papers/*/data.csv')
|
||||
|
||||
results = []
|
||||
for csv_path in csv_files:
|
||||
paper_name = os.path.basename(os.path.dirname(csv_path))
|
||||
report = run_assessment(csv_path, domain='biomedical')
|
||||
results.append({
|
||||
'paper': paper_name,
|
||||
'score': report['summary']['overall_risk_score'],
|
||||
'level': report['summary']['overall_risk_level']
|
||||
})
|
||||
print(f" {paper_name}: {report['summary']['overall_risk_level_cn']}")
|
||||
|
||||
# 排序输出高风险论文
|
||||
results.sort(key=lambda x: x['score'], reverse=True)
|
||||
print("\n🔴 高风险论文:")
|
||||
for r in results:
|
||||
if r['score'] >= 50:
|
||||
print(f" [{r['score']:.0f}] {r['paper']}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. CI/CD 自动化集成
|
||||
|
||||
### 9.1 Pre-commit Hook
|
||||
|
||||
```yaml
|
||||
# .pre-commit-config.yaml
|
||||
repos:
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: geng-data-check
|
||||
name: Geng Academic Data Check
|
||||
entry: python3 geng-skill/scripts/geng_assess.py
|
||||
language: python
|
||||
files: '\.csv$'
|
||||
args: ['--input']
|
||||
```
|
||||
|
||||
### 9.2 Docker 容器化
|
||||
|
||||
```dockerfile
|
||||
# Dockerfile
|
||||
FROM python:3.10-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY scripts/ ./scripts/
|
||||
COPY SKILL.md .
|
||||
|
||||
ENTRYPOINT ["python3", "scripts/geng_assess.py"]
|
||||
CMD ["--help"]
|
||||
```
|
||||
|
||||
```bash
|
||||
# 构建与运行
|
||||
docker build -t geng-skill .
|
||||
docker run -v $(pwd)/data:/data geng-skill -i /data/paper.csv -o /data/report/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
### Q: 数据量有什么要求?
|
||||
|
||||
| 检测模块 | 最小数据量 | 推荐数据量 |
|
||||
|----------|-----------|-----------|
|
||||
| 末位数字检测 | 10 | 50+ |
|
||||
| 本福特定律 | 30 | 100+ |
|
||||
| GRIM 测试 | 1(单项) | N/A |
|
||||
| 固定关系检测 | 5对 | 20+ 对 |
|
||||
| 小数位一致性 | 5 | 30+ |
|
||||
| 图像重复 | 2张 | 10+ 张 |
|
||||
|
||||
### Q: 支持什么输入格式?
|
||||
|
||||
- ✅ CSV(默认逗号分隔,可指定其他分隔符)
|
||||
- ✅ 直接传入数值列表(Python API)
|
||||
- ✅ JSON(GRIM 批量测试)
|
||||
- ✅ 图片目录(PNG/JPG/TIF/BMP)
|
||||
- ❌ Excel(需先转 CSV)
|
||||
- ❌ PDF(需先提取数据表格)
|
||||
|
||||
### Q: 如何降低误报率?
|
||||
|
||||
1. 确认数据范围是否适合该检测(如本福特需跨数量级)
|
||||
2. 多模块交叉验证,不要仅凭单一结果下结论
|
||||
3. 考虑合理解释:仪器精度限制、数据预处理步骤等
|
||||
4. 结果需领域专家复核
|
||||
|
||||
---
|
||||
|
||||
*Geng Skill v1.0.0 — 致敬"耿同学讲故事"*
|
||||
21
tools/gengskill/examples/fake_data_demo.csv
Normal file
21
tools/gengskill/examples/fake_data_demo.csv
Normal file
@ -0,0 +1,21 @@
|
||||
sample_id,control_group,treatment_a,treatment_b,measurement
|
||||
1,2.34,4.68,7.02,12.5
|
||||
2,3.12,6.24,9.36,15.8
|
||||
3,1.87,3.74,5.61,8.9
|
||||
4,4.56,9.12,13.68,22.1
|
||||
5,2.98,5.96,8.94,14.3
|
||||
6,3.45,6.90,10.35,17.2
|
||||
7,1.23,2.46,3.69,6.8
|
||||
8,5.67,11.34,17.01,28.4
|
||||
9,2.01,4.02,6.03,10.1
|
||||
10,3.89,7.78,11.67,19.5
|
||||
11,4.12,8.24,12.36,20.8
|
||||
12,1.56,3.12,4.68,7.9
|
||||
13,2.78,5.56,8.34,13.6
|
||||
14,3.34,6.68,10.02,16.7
|
||||
15,4.90,9.80,14.70,24.5
|
||||
16,1.45,2.90,4.35,7.2
|
||||
17,2.67,5.34,8.01,13.1
|
||||
18,3.56,7.12,10.68,17.8
|
||||
19,4.23,8.46,12.69,21.2
|
||||
20,1.89,3.78,5.67,9.4
|
||||
|
31
tools/gengskill/examples/real_data_demo.csv
Normal file
31
tools/gengskill/examples/real_data_demo.csv
Normal file
@ -0,0 +1,31 @@
|
||||
sample_id,weight_g,blood_glucose,tumor_volume,survival_days
|
||||
1,23.4,112,45.2,89
|
||||
2,25.1,98,32.8,124
|
||||
3,21.8,135,67.3,56
|
||||
4,24.7,107,51.9,78
|
||||
5,22.3,121,39.4,103
|
||||
6,26.5,89,28.7,145
|
||||
7,23.9,143,72.1,42
|
||||
8,25.8,101,44.6,91
|
||||
9,22.1,118,55.3,67
|
||||
10,24.2,96,36.2,112
|
||||
11,23.6,128,48.7,74
|
||||
12,25.4,105,41.3,98
|
||||
13,21.5,137,63.8,51
|
||||
14,24.9,92,30.5,131
|
||||
15,22.8,115,57.6,62
|
||||
16,26.1,99,35.4,118
|
||||
17,23.3,141,69.2,45
|
||||
18,25.7,108,42.8,86
|
||||
19,22.6,123,53.1,71
|
||||
20,24.4,94,37.9,107
|
||||
21,23.1,130,46.5,82
|
||||
22,25.2,103,39.7,95
|
||||
23,21.9,119,61.4,58
|
||||
24,24.6,97,33.6,126
|
||||
25,22.4,134,54.8,64
|
||||
26,26.3,91,29.3,139
|
||||
27,23.8,126,50.2,76
|
||||
28,25.5,110,43.1,88
|
||||
29,22.0,116,58.9,60
|
||||
30,24.1,100,36.7,115
|
||||
|
4
tools/gengskill/requirements.txt
Normal file
4
tools/gengskill/requirements.txt
Normal file
@ -0,0 +1,4 @@
|
||||
numpy>=1.20.0
|
||||
scipy>=1.7.0
|
||||
Pillow>=9.0.0
|
||||
scikit-image>=0.19.0
|
||||
256
tools/gengskill/scripts/benford_test.py
Normal file
256
tools/gengskill/scripts/benford_test.py
Normal file
@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
本福特定律检测 (Benford's Law Test)
|
||||
====================================
|
||||
原理:跨越多个数量级的自然数据,首位数字遵循特定概率分布:
|
||||
P(d) = log10(1 + 1/d), d = 1,2,...,9
|
||||
|
||||
人为编造的数据往往偏离这一分布(倾向于均匀分布或集中在某些数字)。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
# 本福特定律理论概率
|
||||
BENFORD_PROBS = {d: math.log10(1 + 1/d) for d in range(1, 10)}
|
||||
|
||||
|
||||
def get_first_digit(value):
|
||||
"""提取数值的首位有效数字(1-9)"""
|
||||
try:
|
||||
num = abs(float(value))
|
||||
if num == 0:
|
||||
return None
|
||||
# 转为科学计数法取首位
|
||||
s = f"{num:.10e}"
|
||||
first = int(s[0])
|
||||
if 1 <= first <= 9:
|
||||
return first
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_first_two_digits(value):
|
||||
"""提取前两位有效数字(10-99)"""
|
||||
try:
|
||||
num = abs(float(value))
|
||||
if num == 0:
|
||||
return None
|
||||
while num < 10:
|
||||
num *= 10
|
||||
while num >= 100:
|
||||
num /= 10
|
||||
return int(num)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def benford_test(values, order=1):
|
||||
"""
|
||||
执行本福特定律检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list
|
||||
待检测的数值列表
|
||||
order : int
|
||||
1 = 首位数字检测, 2 = 前两位数字检测
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if order == 1:
|
||||
digits = []
|
||||
for v in values:
|
||||
d = get_first_digit(v)
|
||||
if d is not None:
|
||||
digits.append(d)
|
||||
|
||||
if len(digits) < 30:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足(仅{len(digits)}个有效值),本福特定律检测建议至少100个数据点',
|
||||
'n_valid': len(digits)
|
||||
}
|
||||
|
||||
# 统计观测频率
|
||||
digit_counts = Counter(digits)
|
||||
observed = np.array([digit_counts.get(d, 0) for d in range(1, 10)])
|
||||
expected = np.array([BENFORD_PROBS[d] * len(digits) for d in range(1, 10)])
|
||||
|
||||
# 卡方检验
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
# Kolmogorov-Smirnov 检验
|
||||
observed_freq = observed / len(digits)
|
||||
expected_freq = np.array([BENFORD_PROBS[d] for d in range(1, 10)])
|
||||
|
||||
# 最大绝对偏差 (MAD)
|
||||
mad = np.mean(np.abs(observed_freq - expected_freq))
|
||||
|
||||
# MAD 阈值参考 (Nigrini 2012)
|
||||
# Close conformity: MAD < 0.006
|
||||
# Acceptable conformity: 0.006 <= MAD < 0.012
|
||||
# Marginally acceptable: 0.012 <= MAD < 0.015
|
||||
# Nonconformity: MAD >= 0.015
|
||||
|
||||
if mad < 0.006:
|
||||
conformity = 'close'
|
||||
conformity_cn = '高度符合'
|
||||
elif mad < 0.012:
|
||||
conformity = 'acceptable'
|
||||
conformity_cn = '可接受'
|
||||
elif mad < 0.015:
|
||||
conformity = 'marginal'
|
||||
conformity_cn = '边缘'
|
||||
else:
|
||||
conformity = 'nonconforming'
|
||||
conformity_cn = '不符合'
|
||||
|
||||
# 风险评分
|
||||
if p_value < 0.001 and mad >= 0.015:
|
||||
risk_level = 'high'
|
||||
risk_score = 75 + min(25, mad * 500)
|
||||
elif p_value < 0.01:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 55 + min(20, mad * 400)
|
||||
elif p_value < 0.05:
|
||||
risk_level = 'medium'
|
||||
risk_score = 35 + min(20, mad * 300)
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = max(0, mad * 200)
|
||||
|
||||
distribution = {
|
||||
str(d): {
|
||||
'observed': int(observed[d-1]),
|
||||
'observed_freq': round(float(observed_freq[d-1]), 4),
|
||||
'expected_freq': round(float(expected_freq[d-1]), 4),
|
||||
'deviation': round(float(observed_freq[d-1] - expected_freq[d-1]), 4)
|
||||
}
|
||||
for d in range(1, 10)
|
||||
}
|
||||
|
||||
result = {
|
||||
'test_name': "Benford's Law Test (本福特定律检测)",
|
||||
'status': 'completed',
|
||||
'order': order,
|
||||
'n_values': len(digits),
|
||||
'distribution': distribution,
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'degrees_of_freedom': 8,
|
||||
'mean_absolute_deviation': round(float(mad), 6),
|
||||
'conformity': conformity,
|
||||
'conformity_cn': conformity_cn,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_benford(p_value, mad, conformity_cn, len(digits)),
|
||||
'note': '本福特定律适用于跨多个数量级的自然数据集。对于范围有限的数据(如百分比、pH值),该检测可能不适用。'
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
else:
|
||||
return {'status': 'error', 'message': '目前仅支持首位数字检测(order=1)'}
|
||||
|
||||
|
||||
def _interpret_benford(p_value, mad, conformity_cn, n):
|
||||
"""生成可读的解释"""
|
||||
if p_value < 0.001 and mad >= 0.015:
|
||||
return (
|
||||
f"⚠️ 数据首位数字分布严重偏离本福特定律(p < 0.001, MAD = {mad:.4f})。"
|
||||
f"符合性判定:{conformity_cn}。"
|
||||
f"这种偏离在{n}个数据点的样本中非常显著,强烈建议核查数据来源。"
|
||||
f"注意:需确认数据是否适用本福特定律(需跨越多个数量级)。"
|
||||
)
|
||||
elif p_value < 0.01:
|
||||
return (
|
||||
f"⚠️ 数据首位数字分布显著偏离本福特定律(p < 0.01, MAD = {mad:.4f})。"
|
||||
f"符合性判定:{conformity_cn}。建议进一步检查。"
|
||||
)
|
||||
elif p_value < 0.05:
|
||||
return (
|
||||
f"⚡ 数据首位数字分布存在一定偏离(p < 0.05, MAD = {mad:.4f})。"
|
||||
f"符合性判定:{conformity_cn}。可能是正常波动,建议结合其他检测综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"✅ 数据首位数字分布符合本福特定律(p = {p_value:.4f}, MAD = {mad:.4f})。"
|
||||
f"符合性判定:{conformity_cn}。未发现异常。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
try:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
else:
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="本福特定律检测 - 检测首位数字是否符合Benford's Law"
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--order', type=int, default=1, choices=[1, 2],
|
||||
help='检测阶数:1=首位, 2=前两位')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误:未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = benford_test(values, order=args.order)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
280
tools/gengskill/scripts/decimal_consistency_test.py
Normal file
280
tools/gengskill/scripts/decimal_consistency_test.py
Normal file
@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
小数位一致性检测 (Decimal Consistency Test)
|
||||
============================================
|
||||
原理:实验测量数据的小数位后数字应具有一定的随机性。
|
||||
如果大量数据点的小数部分高度一致(如小数后两位总是相同),
|
||||
或小数位数模式过于规律,则暗示数据可能是人为编造的。
|
||||
|
||||
这是"耿同学"常用的一个检测手段——造假者编造数据时,
|
||||
小数点后的位数往往呈现不自然的一致性。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def get_decimal_digits(value, max_digits=4):
|
||||
"""提取数值的小数部分各位数字"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return []
|
||||
decimal_part = s.split('.')[1]
|
||||
return [int(d) for d in decimal_part[:max_digits]]
|
||||
|
||||
|
||||
def get_decimal_string(value):
|
||||
"""提取数值的完整小数字符串"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return ''
|
||||
return s.split('.')[1]
|
||||
|
||||
|
||||
def count_decimal_places(value):
|
||||
"""计算数值的小数位数"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return 0
|
||||
return len(s.split('.')[1])
|
||||
|
||||
|
||||
def decimal_consistency_test(values):
|
||||
"""
|
||||
执行小数位一致性检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list
|
||||
待检测的数值列表(字符串形式保留原始精度)
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if len(values) < 5:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足(仅{len(values)}个值),需要至少5个数据点'
|
||||
}
|
||||
|
||||
# 分析1: 小数位数一致性
|
||||
decimal_places = [count_decimal_places(v) for v in values]
|
||||
places_counter = Counter(decimal_places)
|
||||
most_common_places = places_counter.most_common(1)[0]
|
||||
places_uniformity = most_common_places[1] / len(values)
|
||||
|
||||
# 分析2: 小数部分重复度
|
||||
decimal_strings = [get_decimal_string(v) for v in values if '.' in str(v)]
|
||||
if decimal_strings:
|
||||
decimal_counter = Counter(decimal_strings)
|
||||
n_unique_decimals = len(decimal_counter)
|
||||
most_repeated = decimal_counter.most_common(1)[0]
|
||||
max_repetition_rate = most_repeated[1] / len(decimal_strings)
|
||||
else:
|
||||
n_unique_decimals = 0
|
||||
max_repetition_rate = 0
|
||||
most_repeated = ('N/A', 0)
|
||||
|
||||
# 分析3: 各小数位数字分布
|
||||
position_analyses = {}
|
||||
max_positions = max(decimal_places) if decimal_places else 0
|
||||
|
||||
for pos in range(min(max_positions, 4)):
|
||||
digits_at_pos = []
|
||||
for v in values:
|
||||
decs = get_decimal_digits(v)
|
||||
if len(decs) > pos:
|
||||
digits_at_pos.append(decs[pos])
|
||||
|
||||
if len(digits_at_pos) >= 10:
|
||||
digit_counts = Counter(digits_at_pos)
|
||||
observed = np.array([digit_counts.get(i, 0) for i in range(10)])
|
||||
expected = np.full(10, len(digits_at_pos) / 10.0)
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
position_analyses[f'position_{pos+1}'] = {
|
||||
'n_values': len(digits_at_pos),
|
||||
'distribution': {str(i): int(observed[i]) for i in range(10)},
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'is_uniform': p_value > 0.05
|
||||
}
|
||||
|
||||
# 分析4: 相邻数据小数部分相关性
|
||||
if len(decimal_strings) >= 5:
|
||||
# 将小数部分转为数值进行自相关分析
|
||||
decimal_values = []
|
||||
for ds in decimal_strings:
|
||||
try:
|
||||
decimal_values.append(float('0.' + ds) if ds else 0.0)
|
||||
except ValueError:
|
||||
decimal_values.append(0.0)
|
||||
|
||||
if len(decimal_values) >= 5:
|
||||
# 计算一阶自相关
|
||||
x = np.array(decimal_values)
|
||||
x_centered = x - np.mean(x)
|
||||
if np.std(x) > 0:
|
||||
autocorr = np.correlate(x_centered[:-1], x_centered[1:]) / (len(x_centered) - 1) / np.var(x)
|
||||
autocorr_val = float(autocorr[0]) if len(autocorr) > 0 else 0
|
||||
else:
|
||||
autocorr_val = 1.0 # 完全一致
|
||||
else:
|
||||
autocorr_val = None
|
||||
else:
|
||||
autocorr_val = None
|
||||
|
||||
# 综合风险评分
|
||||
risk_factors = []
|
||||
|
||||
# 因子1: 小数位数过于一致
|
||||
if places_uniformity > 0.95 and len(values) > 10:
|
||||
risk_factors.append(('decimal_places_uniform', 20))
|
||||
|
||||
# 因子2: 小数部分重复度过高
|
||||
if max_repetition_rate > 0.5:
|
||||
risk_factors.append(('high_repetition', 30))
|
||||
elif max_repetition_rate > 0.3:
|
||||
risk_factors.append(('moderate_repetition', 15))
|
||||
|
||||
# 因子3: 某个位置数字分布异常
|
||||
for pos_key, pos_data in position_analyses.items():
|
||||
if pos_data['p_value'] < 0.001:
|
||||
risk_factors.append((f'{pos_key}_nonuniform', 25))
|
||||
elif pos_data['p_value'] < 0.01:
|
||||
risk_factors.append((f'{pos_key}_marginal', 10))
|
||||
|
||||
# 因子4: 自相关异常高
|
||||
if autocorr_val is not None and abs(autocorr_val) > 0.8:
|
||||
risk_factors.append(('high_autocorrelation', 20))
|
||||
|
||||
risk_score = min(100, sum(score for _, score in risk_factors))
|
||||
|
||||
if risk_score >= 70:
|
||||
risk_level = 'high'
|
||||
elif risk_score >= 45:
|
||||
risk_level = 'medium-high'
|
||||
elif risk_score >= 25:
|
||||
risk_level = 'medium'
|
||||
else:
|
||||
risk_level = 'low'
|
||||
|
||||
result = {
|
||||
'test_name': 'Decimal Consistency Test (小数位一致性检测)',
|
||||
'status': 'completed',
|
||||
'n_values': len(values),
|
||||
'decimal_places_analysis': {
|
||||
'distribution': dict(places_counter),
|
||||
'most_common_places': most_common_places[0],
|
||||
'uniformity_rate': round(float(places_uniformity), 4)
|
||||
},
|
||||
'decimal_repetition': {
|
||||
'n_unique_patterns': n_unique_decimals,
|
||||
'most_repeated_pattern': most_repeated[0],
|
||||
'most_repeated_count': most_repeated[1],
|
||||
'max_repetition_rate': round(float(max_repetition_rate), 4)
|
||||
},
|
||||
'position_digit_analysis': position_analyses,
|
||||
'autocorrelation': round(float(autocorr_val), 4) if autocorr_val is not None else None,
|
||||
'risk_factors': [f for f, _ in risk_factors],
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated):
|
||||
"""生成可读的解释"""
|
||||
if not risk_factors:
|
||||
return "✅ 小数位分布未发现明显异常,数据的小数部分具有合理的随机性。"
|
||||
|
||||
issues = []
|
||||
for factor, _ in risk_factors:
|
||||
if 'repetition' in factor:
|
||||
issues.append(f"小数部分 '{most_repeated[0]}' 重复出现 {most_repeated[1]} 次({max_repetition_rate:.0%})")
|
||||
elif 'nonuniform' in factor:
|
||||
issues.append("某些小数位的数字分布严重偏离均匀分布")
|
||||
elif 'autocorrelation' in factor:
|
||||
issues.append("相邻数据的小数部分存在异常高的自相关")
|
||||
elif 'places_uniform' in factor:
|
||||
issues.append(f"所有数据小数位数高度一致({places_uniformity:.0%}相同)")
|
||||
|
||||
issues_str = ";".join(issues)
|
||||
|
||||
if len(risk_factors) >= 3:
|
||||
return f"⚠️ 发现多项小数位异常:{issues_str}。这些模式在自然实验数据中非常罕见,强烈建议核查原始数据。"
|
||||
elif len(risk_factors) >= 2:
|
||||
return f"⚠️ 发现小数位可疑模式:{issues_str}。建议进一步检查。"
|
||||
else:
|
||||
return f"⚡ 发现轻微异常:{issues_str}。可能是测量精度限制导致,建议结合其他检测综合判断。"
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据(保留原始字符串精度)"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
try:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='小数位一致性检测 - 检测数据小数部分是否存在异常模式'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误:未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = decimal_consistency_test(values)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
312
tools/gengskill/scripts/fixed_relation_test.py
Normal file
312
tools/gengskill/scripts/fixed_relation_test.py
Normal file
@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
固定关系检测 (Fixed Relationship Detection)
|
||||
============================================
|
||||
原理:两组独立实验数据之间不应存在恒定的差值、比值或线性关系。
|
||||
如果不同实验条件下的数据存在固定的数学关系,暗示数据可能是
|
||||
从单一数据源通过简单数学运算生成的,而非独立实验获得。
|
||||
|
||||
这是"耿同学"打假方法中的核心策略之一——他发现许多造假论文中
|
||||
不同实验组的数据存在固定差值或固定比例关系。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def detect_fixed_difference(col1, col2, tolerance=0.01):
|
||||
"""检测两列数据是否存在固定差值"""
|
||||
differences = np.array(col2) - np.array(col1)
|
||||
|
||||
if len(differences) < 3:
|
||||
return None
|
||||
|
||||
# 计算差值的变异系数
|
||||
mean_diff = np.mean(differences)
|
||||
std_diff = np.std(differences)
|
||||
|
||||
if abs(mean_diff) < 1e-10:
|
||||
cv = float('inf') if std_diff > 0 else 0
|
||||
else:
|
||||
cv = abs(std_diff / mean_diff)
|
||||
|
||||
# 判断差值是否恒定(CV极小)
|
||||
is_fixed = cv < tolerance and std_diff < tolerance * abs(mean_diff + 1e-10)
|
||||
|
||||
# 检查差值是否完全相同
|
||||
unique_diffs = np.unique(np.round(differences, 6))
|
||||
is_exact = len(unique_diffs) == 1
|
||||
|
||||
return {
|
||||
'type': 'fixed_difference',
|
||||
'type_cn': '固定差值',
|
||||
'mean_difference': round(float(mean_diff), 6),
|
||||
'std_difference': round(float(std_diff), 6),
|
||||
'cv': round(float(cv), 6) if cv != float('inf') else 'inf',
|
||||
'is_fixed': bool(is_fixed or is_exact),
|
||||
'is_exact': bool(is_exact),
|
||||
'n_unique_differences': int(len(unique_diffs))
|
||||
}
|
||||
|
||||
|
||||
def detect_fixed_ratio(col1, col2, tolerance=0.01):
|
||||
"""检测两列数据是否存在固定比值"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
# 避免除以零
|
||||
mask = col1 != 0
|
||||
if np.sum(mask) < 3:
|
||||
return None
|
||||
|
||||
ratios = col2[mask] / col1[mask]
|
||||
|
||||
mean_ratio = np.mean(ratios)
|
||||
std_ratio = np.std(ratios)
|
||||
|
||||
if abs(mean_ratio) < 1e-10:
|
||||
cv = float('inf') if std_ratio > 0 else 0
|
||||
else:
|
||||
cv = abs(std_ratio / mean_ratio)
|
||||
|
||||
is_fixed = cv < tolerance
|
||||
unique_ratios = np.unique(np.round(ratios, 6))
|
||||
is_exact = len(unique_ratios) == 1
|
||||
|
||||
return {
|
||||
'type': 'fixed_ratio',
|
||||
'type_cn': '固定比值',
|
||||
'mean_ratio': round(float(mean_ratio), 6),
|
||||
'std_ratio': round(float(std_ratio), 6),
|
||||
'cv': round(float(cv), 6) if cv != float('inf') else 'inf',
|
||||
'is_fixed': bool(is_fixed or is_exact),
|
||||
'is_exact': bool(is_exact),
|
||||
'n_unique_ratios': int(len(unique_ratios))
|
||||
}
|
||||
|
||||
|
||||
def detect_linear_relationship(col1, col2):
|
||||
"""检测两列数据是否存在高度线性关系"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
if len(col1) < 3:
|
||||
return None
|
||||
|
||||
# 线性回归
|
||||
slope, intercept, r_value, p_value, std_err = stats.linregress(col1, col2)
|
||||
r_squared = r_value ** 2
|
||||
|
||||
# 残差分析
|
||||
predicted = slope * col1 + intercept
|
||||
residuals = col2 - predicted
|
||||
max_residual = np.max(np.abs(residuals))
|
||||
mean_residual = np.mean(np.abs(residuals))
|
||||
|
||||
# R² 非常接近1且残差极小
|
||||
is_suspicious = r_squared > 0.9999 and max_residual < 0.001 * np.std(col2)
|
||||
|
||||
return {
|
||||
'type': 'linear_relationship',
|
||||
'type_cn': '线性关系',
|
||||
'slope': round(float(slope), 6),
|
||||
'intercept': round(float(intercept), 6),
|
||||
'r_squared': round(float(r_squared), 8),
|
||||
'p_value': float(p_value),
|
||||
'max_residual': round(float(max_residual), 8),
|
||||
'mean_residual': round(float(mean_residual), 8),
|
||||
'is_suspicious': bool(is_suspicious)
|
||||
}
|
||||
|
||||
|
||||
def detect_decimal_pattern(col1, col2):
|
||||
"""检测两列数据小数部分是否高度一致"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
# 提取小数部分
|
||||
dec1 = col1 - np.floor(col1)
|
||||
dec2 = col2 - np.floor(col2)
|
||||
|
||||
# 检查小数部分是否一致
|
||||
dec_diff = np.abs(dec1 - dec2)
|
||||
n_matching = np.sum(dec_diff < 0.001)
|
||||
match_rate = n_matching / len(col1)
|
||||
|
||||
return {
|
||||
'type': 'decimal_pattern',
|
||||
'type_cn': '小数位一致性',
|
||||
'n_matching_decimals': int(n_matching),
|
||||
'match_rate': round(float(match_rate), 4),
|
||||
'is_suspicious': match_rate > 0.8
|
||||
}
|
||||
|
||||
|
||||
def fixed_relation_test(col1, col2, col1_name='Column A', col2_name='Column B'):
|
||||
"""
|
||||
综合固定关系检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
col1 : list of float
|
||||
第一列数据
|
||||
col2 : list of float
|
||||
第二列数据
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if len(col1) != len(col2):
|
||||
return {'status': 'error', 'message': '两列数据长度不一致'}
|
||||
|
||||
if len(col1) < 3:
|
||||
return {'status': 'insufficient_data', 'message': '数据量不足,至少需要3个数据点'}
|
||||
|
||||
col1 = [float(x) for x in col1]
|
||||
col2 = [float(x) for x in col2]
|
||||
|
||||
# 执行各项检测
|
||||
results = {}
|
||||
|
||||
diff_result = detect_fixed_difference(col1, col2)
|
||||
if diff_result:
|
||||
results['fixed_difference'] = diff_result
|
||||
|
||||
ratio_result = detect_fixed_ratio(col1, col2)
|
||||
if ratio_result:
|
||||
results['fixed_ratio'] = ratio_result
|
||||
|
||||
linear_result = detect_linear_relationship(col1, col2)
|
||||
if linear_result:
|
||||
results['linear_relationship'] = linear_result
|
||||
|
||||
decimal_result = detect_decimal_pattern(col1, col2)
|
||||
if decimal_result:
|
||||
results['decimal_pattern'] = decimal_result
|
||||
|
||||
# 综合风险评估
|
||||
n_suspicious = sum([
|
||||
1 for r in results.values()
|
||||
if r.get('is_fixed') or r.get('is_suspicious')
|
||||
])
|
||||
|
||||
if n_suspicious >= 3:
|
||||
risk_level = 'high'
|
||||
risk_score = 85
|
||||
elif n_suspicious == 2:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 65
|
||||
elif n_suspicious == 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 45
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = 10
|
||||
|
||||
# 如果存在完全精确的固定关系,直接拉高风险
|
||||
if any(r.get('is_exact') for r in results.values()):
|
||||
risk_level = 'high'
|
||||
risk_score = max(risk_score, 90)
|
||||
|
||||
summary = {
|
||||
'test_name': 'Fixed Relationship Detection (固定关系检测)',
|
||||
'status': 'completed',
|
||||
'n_data_points': len(col1),
|
||||
'column_1': col1_name,
|
||||
'column_2': col2_name,
|
||||
'detections': results,
|
||||
'n_suspicious_patterns': n_suspicious,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name)
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name):
|
||||
"""生成可读的解释"""
|
||||
findings = []
|
||||
|
||||
if results.get('fixed_difference', {}).get('is_fixed'):
|
||||
d = results['fixed_difference']
|
||||
findings.append(f"两列数据存在固定差值 {d['mean_difference']}")
|
||||
|
||||
if results.get('fixed_ratio', {}).get('is_fixed'):
|
||||
r = results['fixed_ratio']
|
||||
findings.append(f"两列数据存在固定比值 {r['mean_ratio']}")
|
||||
|
||||
if results.get('linear_relationship', {}).get('is_suspicious'):
|
||||
l = results['linear_relationship']
|
||||
findings.append(f"两列数据存在完美线性关系 (R² = {l['r_squared']})")
|
||||
|
||||
if results.get('decimal_pattern', {}).get('is_suspicious'):
|
||||
p = results['decimal_pattern']
|
||||
findings.append(f"两列数据小数部分高度一致 (匹配率 {p['match_rate']:.0%})")
|
||||
|
||||
if not findings:
|
||||
return f"✅ {col1_name} 与 {col2_name} 之间未发现固定数学关系,数据看起来是独立的。"
|
||||
|
||||
findings_str = ";".join(findings)
|
||||
return (
|
||||
f"⚠️ {col1_name} 与 {col2_name} 之间发现以下可疑模式:{findings_str}。"
|
||||
f"独立实验数据通常不应存在如此精确的数学关系,建议核查数据是否来自独立实验。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, col1, col2, delimiter=','):
|
||||
"""从CSV文件加载两列数据"""
|
||||
import csv
|
||||
|
||||
data1, data2 = [], []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
for row in reader:
|
||||
try:
|
||||
v1 = float(row[col1].strip())
|
||||
v2 = float(row[col2].strip())
|
||||
data1.append(v1)
|
||||
data2.append(v2)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
return data1, data2
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='固定关系检测 - 检测两组数据间是否存在不自然的数学关系'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--col1', required=True, help='第一列列名')
|
||||
parser.add_argument('--col2', required=True, help='第二列列名')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
data1, data2 = load_data(args.input, args.col1, args.col2, args.delimiter)
|
||||
|
||||
if not data1:
|
||||
print("错误:未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = fixed_relation_test(data1, data2, args.col1, args.col2)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
463
tools/gengskill/scripts/geng_assess.py
Normal file
463
tools/gengskill/scripts/geng_assess.py
Normal file
@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geng 综合评估引擎 (Comprehensive Assessment Engine)
|
||||
=====================================================
|
||||
一键运行所有适用的检测模块,生成综合学术数据打假报告。
|
||||
|
||||
支持的领域:
|
||||
- biomedical: 生物医学(Western blot, 流式, 动物实验)
|
||||
- chemistry: 化学(光谱, 产率, 催化)
|
||||
- physics: 物理/材料(性能曲线, 电学/力学)
|
||||
- social_science: 社会科学(问卷, 量表)
|
||||
- clinical: 临床医学(生存数据, 临床指标)
|
||||
- general: 通用(不指定领域)
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import csv
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# 导入各检测模块
|
||||
from last_digit_test import last_digit_test
|
||||
from benford_test import benford_test
|
||||
from decimal_consistency_test import decimal_consistency_test
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
from grim_test import grim_test_batch
|
||||
|
||||
|
||||
def load_csv_data(input_file, delimiter=','):
|
||||
"""加载CSV数据,返回列名和数据"""
|
||||
columns = {}
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
fieldnames = reader.fieldnames
|
||||
for row in reader:
|
||||
for col in fieldnames:
|
||||
if col not in columns:
|
||||
columns[col] = []
|
||||
columns[col].append(row[col].strip() if row[col] else '')
|
||||
return columns, fieldnames
|
||||
|
||||
|
||||
def identify_numeric_columns(columns):
|
||||
"""识别数值列"""
|
||||
numeric_cols = {}
|
||||
for col_name, values in columns.items():
|
||||
numeric_values = []
|
||||
for v in values:
|
||||
try:
|
||||
if v:
|
||||
float(v)
|
||||
numeric_values.append(v)
|
||||
except ValueError:
|
||||
continue
|
||||
# 至少50%的值是数值
|
||||
if len(numeric_values) >= len(values) * 0.5 and len(numeric_values) >= 5:
|
||||
numeric_cols[col_name] = numeric_values
|
||||
return numeric_cols
|
||||
|
||||
|
||||
def run_assessment(input_file, domain='general', output_dir=None, delimiter=','):
|
||||
"""
|
||||
执行综合评估
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入CSV文件
|
||||
domain : str
|
||||
研究领域
|
||||
output_dir : str
|
||||
输出目录
|
||||
delimiter : str
|
||||
CSV分隔符
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
# 加载数据
|
||||
columns, fieldnames = load_csv_data(input_file, delimiter)
|
||||
numeric_cols = identify_numeric_columns(columns)
|
||||
|
||||
if not numeric_cols:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '未找到有效的数值列,请检查输入文件格式'
|
||||
}
|
||||
|
||||
report = {
|
||||
'meta': {
|
||||
'tool': 'Geng Academic Data Fraud Detection Tool',
|
||||
'version': '1.0.0',
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'input_file': os.path.basename(input_file),
|
||||
'domain': domain,
|
||||
'n_columns': len(fieldnames),
|
||||
'n_numeric_columns': len(numeric_cols),
|
||||
'numeric_columns': list(numeric_cols.keys()),
|
||||
'n_rows': max(len(v) for v in columns.values()) if columns else 0
|
||||
},
|
||||
'module_results': {},
|
||||
'summary': {}
|
||||
}
|
||||
|
||||
all_risk_scores = []
|
||||
|
||||
# ==========================================
|
||||
# Module 1: 末位数字检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行末位数字检测...")
|
||||
last_digit_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
result = last_digit_test(values, method='all_digits')
|
||||
if result.get('status') == 'completed':
|
||||
last_digit_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
|
||||
if last_digit_results:
|
||||
report['module_results']['last_digit_test'] = {
|
||||
'module_name': '末位数字检测',
|
||||
'description': '检测数据末位数字是否偏离均匀分布',
|
||||
'columns_tested': len(last_digit_results),
|
||||
'results': last_digit_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 2: 本福特定律检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行本福特定律检测...")
|
||||
benford_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
# 本福特定律适用于跨多个数量级的数据
|
||||
try:
|
||||
float_values = [float(v) for v in values if v]
|
||||
value_range = max(float_values) / (min(v for v in float_values if v > 0) + 1e-10)
|
||||
# 只对跨度超过1个数量级的列做本福特检测
|
||||
if value_range > 10:
|
||||
result = benford_test(values, order=1)
|
||||
if result.get('status') == 'completed':
|
||||
benford_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
except (ValueError, ZeroDivisionError):
|
||||
continue
|
||||
|
||||
if benford_results:
|
||||
report['module_results']['benford_test'] = {
|
||||
'module_name': '本福特定律检测',
|
||||
'description': '检测首位数字是否符合Benford\'s Law',
|
||||
'columns_tested': len(benford_results),
|
||||
'results': benford_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 3: 小数位一致性检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行小数位一致性检测...")
|
||||
decimal_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
# 只检测包含小数的列
|
||||
has_decimal = any('.' in v for v in values if v)
|
||||
if has_decimal:
|
||||
result = decimal_consistency_test(values)
|
||||
if result.get('status') == 'completed':
|
||||
decimal_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
|
||||
if decimal_results:
|
||||
report['module_results']['decimal_consistency_test'] = {
|
||||
'module_name': '小数位一致性检测',
|
||||
'description': '检测数据小数部分是否存在异常模式',
|
||||
'columns_tested': len(decimal_results),
|
||||
'results': decimal_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 4: 固定关系检测 (两两比较数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行固定关系检测...")
|
||||
fixed_results = {}
|
||||
col_names = list(numeric_cols.keys())
|
||||
|
||||
# 限制比较对数,避免组合爆炸
|
||||
max_pairs = min(10, len(col_names) * (len(col_names) - 1) // 2)
|
||||
pair_count = 0
|
||||
|
||||
for i in range(len(col_names)):
|
||||
if pair_count >= max_pairs:
|
||||
break
|
||||
for j in range(i + 1, len(col_names)):
|
||||
if pair_count >= max_pairs:
|
||||
break
|
||||
col1_name = col_names[i]
|
||||
col2_name = col_names[j]
|
||||
|
||||
# 确保两列长度一致且有足够数据
|
||||
vals1 = numeric_cols[col1_name]
|
||||
vals2 = numeric_cols[col2_name]
|
||||
|
||||
# 配对:只取两列都有值的行
|
||||
paired_1, paired_2 = [], []
|
||||
for v1, v2 in zip(vals1, vals2):
|
||||
try:
|
||||
if v1 and v2:
|
||||
float(v1)
|
||||
float(v2)
|
||||
paired_1.append(v1)
|
||||
paired_2.append(v2)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if len(paired_1) >= 5:
|
||||
result = fixed_relation_test(paired_1, paired_2, col1_name, col2_name)
|
||||
if result.get('status') == 'completed':
|
||||
pair_key = f"{col1_name} vs {col2_name}"
|
||||
fixed_results[pair_key] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
pair_count += 1
|
||||
|
||||
if fixed_results:
|
||||
report['module_results']['fixed_relation_test'] = {
|
||||
'module_name': '固定关系检测',
|
||||
'description': '检测不同列数据间是否存在不自然的数学关系',
|
||||
'pairs_tested': len(fixed_results),
|
||||
'results': fixed_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# 综合评分
|
||||
# ==========================================
|
||||
print("📊 生成综合评估...")
|
||||
|
||||
if all_risk_scores:
|
||||
# 综合评分:取各模块最高分的加权平均
|
||||
max_score = max(all_risk_scores)
|
||||
mean_score = sum(all_risk_scores) / len(all_risk_scores)
|
||||
# 综合分 = 60% 最高分 + 40% 平均分
|
||||
overall_score = 0.6 * max_score + 0.4 * mean_score
|
||||
overall_score = min(100, overall_score)
|
||||
else:
|
||||
overall_score = 0
|
||||
|
||||
# 统计各风险等级
|
||||
high_risk_modules = [s for s in all_risk_scores if s >= 70]
|
||||
medium_risk_modules = [s for s in all_risk_scores if 40 <= s < 70]
|
||||
low_risk_modules = [s for s in all_risk_scores if s < 40]
|
||||
|
||||
if overall_score >= 75:
|
||||
overall_level = 'critical'
|
||||
overall_level_cn = '🔴 极高风险'
|
||||
overall_emoji = '🔴'
|
||||
elif overall_score >= 50:
|
||||
overall_level = 'high'
|
||||
overall_level_cn = '🟠 高风险'
|
||||
overall_emoji = '🟠'
|
||||
elif overall_score >= 25:
|
||||
overall_level = 'medium'
|
||||
overall_level_cn = '🟡 中风险'
|
||||
overall_emoji = '🟡'
|
||||
else:
|
||||
overall_level = 'low'
|
||||
overall_level_cn = '🟢 低风险'
|
||||
overall_emoji = '🟢'
|
||||
|
||||
report['summary'] = {
|
||||
'overall_risk_score': round(float(overall_score), 1),
|
||||
'overall_risk_level': overall_level,
|
||||
'overall_risk_level_cn': overall_level_cn,
|
||||
'n_modules_run': len(report['module_results']),
|
||||
'n_tests_total': len(all_risk_scores),
|
||||
'n_high_risk': len(high_risk_modules),
|
||||
'n_medium_risk': len(medium_risk_modules),
|
||||
'n_low_risk': len(low_risk_modules),
|
||||
'execution_time_seconds': round(time.time() - start_time, 2),
|
||||
'conclusion': _generate_conclusion(overall_score, overall_level, report['module_results']),
|
||||
'recommendations': _generate_recommendations(overall_level, domain, report['module_results'])
|
||||
}
|
||||
|
||||
# 保存报告
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
report_path = os.path.join(output_dir, 'geng_assessment_report.json')
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 生成可读的 Markdown 报告
|
||||
md_path = os.path.join(output_dir, 'geng_assessment_report.md')
|
||||
with open(md_path, 'w', encoding='utf-8') as f:
|
||||
f.write(_generate_markdown_report(report))
|
||||
|
||||
print(f"\n📄 JSON报告已保存至: {report_path}")
|
||||
print(f"📄 Markdown报告已保存至: {md_path}")
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _generate_conclusion(score, level, modules):
|
||||
"""生成结论"""
|
||||
if level == 'critical':
|
||||
return (
|
||||
"⚠️ 综合评估显示数据存在系统性异常,多项检测指标显著偏离正常预期。"
|
||||
"强烈建议对原始实验数据进行全面核查。"
|
||||
"注意:本工具仅提供线索筛查,最终判定需要领域专家复核。"
|
||||
)
|
||||
elif level == 'high':
|
||||
return (
|
||||
"⚠️ 数据中发现多处可疑模式,部分指标显著异常。"
|
||||
"建议对标记为高风险的数据列进行重点核查。"
|
||||
)
|
||||
elif level == 'medium':
|
||||
return (
|
||||
"⚡ 数据存在一些轻微异常,但不足以构成造假的确证。"
|
||||
"可能是测量精度限制、数据处理方式等正常因素导致。"
|
||||
"建议结合论文方法学描述进行综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
"✅ 数据各项检测指标均在正常范围内,未发现明显的造假迹象。"
|
||||
"注意:通过检测不代表数据一定真实,某些高明的造假可能无法被统计方法捕获。"
|
||||
)
|
||||
|
||||
|
||||
def _generate_recommendations(level, domain, modules):
|
||||
"""生成建议"""
|
||||
recs = []
|
||||
|
||||
if level in ('critical', 'high'):
|
||||
recs.append("核查原始实验记录和数据记录本")
|
||||
recs.append("验证数据是否来自独立实验")
|
||||
recs.append("联系通讯作者要求提供原始数据")
|
||||
if domain == 'biomedical':
|
||||
recs.append("检查Western blot原始图片和流式原始FCS文件")
|
||||
recs.append("考虑向期刊或机构提交正式质疑")
|
||||
elif level == 'medium':
|
||||
recs.append("仔细阅读论文方法学部分,确认数据采集方式")
|
||||
recs.append("检查是否存在合理的解释(如仪器精度限制)")
|
||||
recs.append("可考虑联系作者进行非正式沟通")
|
||||
else:
|
||||
recs.append("当前数据未发现明显异常")
|
||||
recs.append("可考虑对补充材料中的数据做进一步检测")
|
||||
|
||||
return recs
|
||||
|
||||
|
||||
def _generate_markdown_report(report):
|
||||
"""生成Markdown格式报告"""
|
||||
meta = report['meta']
|
||||
summary = report['summary']
|
||||
|
||||
md = []
|
||||
md.append("# 📋 Geng 学术数据打假检测报告\n")
|
||||
md.append(f"> 生成时间: {meta['timestamp']}")
|
||||
md.append(f"> 检测工具: {meta['tool']} v{meta['version']}")
|
||||
md.append('> 致敬"耿同学讲故事"\n')
|
||||
|
||||
md.append("## 📊 综合评估结果\n")
|
||||
md.append(f"| 指标 | 结果 |")
|
||||
md.append(f"|------|------|")
|
||||
md.append(f"| **综合风险评分** | **{summary['overall_risk_score']}/100** |")
|
||||
md.append(f"| **风险等级** | {summary['overall_risk_level_cn']} |")
|
||||
md.append(f"| 输入文件 | {meta['input_file']} |")
|
||||
md.append(f"| 检测领域 | {meta['domain']} |")
|
||||
md.append(f"| 数值列数 | {meta['n_numeric_columns']} |")
|
||||
md.append(f"| 数据行数 | {meta['n_rows']} |")
|
||||
md.append(f"| 运行模块数 | {summary['n_modules_run']} |")
|
||||
md.append(f"| 检测总数 | {summary['n_tests_total']} |")
|
||||
md.append(f"| 高风险项 | {summary['n_high_risk']} |")
|
||||
md.append(f"| 执行耗时 | {summary['execution_time_seconds']}s |\n")
|
||||
|
||||
md.append("## 📝 结论\n")
|
||||
md.append(f"{summary['conclusion']}\n")
|
||||
|
||||
md.append("## 💡 建议\n")
|
||||
for i, rec in enumerate(summary['recommendations'], 1):
|
||||
md.append(f"{i}. {rec}")
|
||||
md.append("")
|
||||
|
||||
md.append("## 🔬 各模块检测详情\n")
|
||||
|
||||
for module_key, module_data in report['module_results'].items():
|
||||
md.append(f"### {module_data['module_name']}\n")
|
||||
md.append(f"_{module_data['description']}_\n")
|
||||
|
||||
if 'columns_tested' in module_data:
|
||||
md.append(f"- 检测列数: {module_data['columns_tested']}")
|
||||
if 'pairs_tested' in module_data:
|
||||
md.append(f"- 检测对数: {module_data['pairs_tested']}")
|
||||
|
||||
# 列出各列/对的风险评分
|
||||
results = module_data.get('results', {})
|
||||
if results:
|
||||
md.append(f"\n| 检测对象 | 风险评分 | 风险等级 | 说明 |")
|
||||
md.append(f"|----------|----------|----------|------|")
|
||||
for key, res in results.items():
|
||||
score = res.get('risk_score', 'N/A')
|
||||
level = res.get('risk_level', 'N/A')
|
||||
interp = res.get('interpretation', '')[:60]
|
||||
md.append(f"| {key} | {score} | {level} | {interp}... |")
|
||||
md.append("")
|
||||
|
||||
md.append("---\n")
|
||||
md.append("## ⚠️ 重要声明\n")
|
||||
md.append("- 本工具仅用于辅助筛查,**不能作为造假的最终判定依据**")
|
||||
md.append("- 数据异常 ≠ 数据造假(可能是仪器校准、单位转换、排版错误等)")
|
||||
md.append("- 检测结果需要领域专家复核")
|
||||
md.append("- 使用本工具时应遵守学术伦理和法律法规\n")
|
||||
md.append("---\n")
|
||||
md.append('*Powered by Geng Skill — 致敬"耿同学讲故事"*')
|
||||
|
||||
return "\n".join(md)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Geng 综合评估引擎 - 一键运行所有检测模块'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--domain', default='general',
|
||||
choices=['biomedical', 'chemistry', 'physics',
|
||||
'social_science', 'clinical', 'general'],
|
||||
help='研究领域')
|
||||
parser.add_argument('--output', '-o', default='./report', help='输出目录')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isfile(args.input):
|
||||
print(f"错误:文件不存在: {args.input}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f" Geng 学术数据打假检测工具 v1.0.0")
|
||||
print(' 致敬"耿同学讲故事" — 用数据说话,让造假无所遁形')
|
||||
print(f"{'='*60}")
|
||||
print(f"\n📁 输入文件: {args.input}")
|
||||
print(f"🔬 检测领域: {args.domain}")
|
||||
print(f"📂 输出目录: {args.output}\n")
|
||||
|
||||
report = run_assessment(args.input, args.domain, args.output, args.delimiter)
|
||||
|
||||
if report.get('status') == 'error':
|
||||
print(f"\n❌ 错误: {report['message']}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
summary = report['summary']
|
||||
print(f"\n{'='*60}")
|
||||
print(f" 综合评估结果")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n {summary['overall_risk_level_cn']}")
|
||||
print(f" 综合风险评分: {summary['overall_risk_score']}/100")
|
||||
print(f" 高风险项: {summary['n_high_risk']} | "
|
||||
f"中风险项: {summary['n_medium_risk']} | "
|
||||
f"低风险项: {summary['n_low_risk']}")
|
||||
print(f"\n {summary['conclusion']}")
|
||||
print(f"\n{'='*60}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
244
tools/gengskill/scripts/grim_test.py
Normal file
244
tools/gengskill/scripts/grim_test.py
Normal file
@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GRIM 测试 (Granularity-Related Inconsistency of Means)
|
||||
========================================================
|
||||
原理:对于整数取值的数据(如李克特量表1-5、年龄等),
|
||||
给定样本量 n,合法的平均值只能取有限集合中的值。
|
||||
如果报告的平均值不在合法集合中,则数据存在不一致性。
|
||||
|
||||
例:n=25,数据取值为整数,则平均值只能是 k/25 的形式,
|
||||
小数部分只能是 .00, .04, .08, .12, ..., .96
|
||||
|
||||
参考:Brown & Heathers (2017). The GRIM Test. SPPS.
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
|
||||
|
||||
def grim_test_single(mean, n, decimals=2, scale_min=None, scale_max=None):
|
||||
"""
|
||||
对单个均值执行 GRIM 测试
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mean : float or str
|
||||
报告的平均值
|
||||
n : int
|
||||
样本量
|
||||
decimals : int
|
||||
报告的小数位数
|
||||
scale_min : int, optional
|
||||
量表最小值(用于范围检查)
|
||||
scale_max : int, optional
|
||||
量表最大值(用于范围检查)
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
try:
|
||||
mean_val = Decimal(str(mean))
|
||||
n = int(n)
|
||||
except (ValueError, TypeError) as e:
|
||||
return {'status': 'error', 'message': f'无效输入: {e}'}
|
||||
|
||||
if n <= 0:
|
||||
return {'status': 'error', 'message': '样本量必须大于0'}
|
||||
|
||||
# 范围检查
|
||||
if scale_min is not None and scale_max is not None:
|
||||
if float(mean_val) < scale_min or float(mean_val) > scale_max:
|
||||
return {
|
||||
'status': 'range_error',
|
||||
'message': f'均值 {mean} 超出量表范围 [{scale_min}, {scale_max}]',
|
||||
'consistent': False
|
||||
}
|
||||
|
||||
# 计算总和 = mean * n
|
||||
total = mean_val * n
|
||||
|
||||
# 对于整数取值数据,总和必须是整数
|
||||
# 考虑四舍五入误差:检查 total 是否足够接近某个整数
|
||||
granularity = Decimal(1) / Decimal(10 ** decimals)
|
||||
|
||||
# 在四舍五入精度范围内检查
|
||||
# mean 可能是真实值四舍五入到 decimals 位的结果
|
||||
# 真实 mean 在 [mean - 0.5*granularity, mean + 0.5*granularity) 范围内
|
||||
lower_total = (mean_val - granularity / 2) * n
|
||||
upper_total = (mean_val + granularity / 2) * n
|
||||
|
||||
# 检查这个范围内是否包含整数
|
||||
lower_int = math.ceil(float(lower_total))
|
||||
upper_int = math.floor(float(upper_total))
|
||||
|
||||
consistent = lower_int <= upper_int
|
||||
|
||||
# 计算最近的合法均值
|
||||
nearest_total = round(float(mean_val * n))
|
||||
nearest_mean = nearest_total / n
|
||||
|
||||
# 格式化到指定小数位
|
||||
fmt = f"%.{decimals}f"
|
||||
nearest_mean_str = fmt % nearest_mean
|
||||
reported_mean_str = fmt % float(mean_val)
|
||||
|
||||
result = {
|
||||
'reported_mean': str(mean),
|
||||
'sample_size': n,
|
||||
'decimals': decimals,
|
||||
'consistent': consistent,
|
||||
'computed_sum': float(mean_val * n),
|
||||
'nearest_valid_mean': nearest_mean_str,
|
||||
'difference': round(abs(float(mean_val) - nearest_mean), decimals + 2)
|
||||
}
|
||||
|
||||
if scale_min is not None and scale_max is not None:
|
||||
result['scale_range'] = f"[{scale_min}, {scale_max}]"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def grim_test_batch(items):
|
||||
"""
|
||||
批量 GRIM 测试
|
||||
|
||||
Parameters
|
||||
----------
|
||||
items : list of dict
|
||||
每个字典包含 'mean', 'n', 可选 'decimals', 'label'
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 批量检测结果
|
||||
"""
|
||||
results = []
|
||||
n_inconsistent = 0
|
||||
|
||||
for i, item in enumerate(items):
|
||||
mean = item.get('mean')
|
||||
n = item.get('n')
|
||||
decimals = item.get('decimals', 2)
|
||||
label = item.get('label', f'Item {i+1}')
|
||||
scale_min = item.get('scale_min')
|
||||
scale_max = item.get('scale_max')
|
||||
|
||||
res = grim_test_single(mean, n, decimals, scale_min, scale_max)
|
||||
res['label'] = label
|
||||
results.append(res)
|
||||
|
||||
if not res.get('consistent', True):
|
||||
n_inconsistent += 1
|
||||
|
||||
# 整体评估
|
||||
total = len(results)
|
||||
inconsistency_rate = n_inconsistent / total if total > 0 else 0
|
||||
|
||||
# 风险评分
|
||||
if inconsistency_rate > 0.5:
|
||||
risk_level = 'high'
|
||||
risk_score = 70 + inconsistency_rate * 30
|
||||
elif inconsistency_rate > 0.25:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 50 + inconsistency_rate * 40
|
||||
elif inconsistency_rate > 0:
|
||||
risk_level = 'medium'
|
||||
risk_score = 30 + inconsistency_rate * 40
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
|
||||
summary = {
|
||||
'test_name': 'GRIM Test (均值粒度一致性检验)',
|
||||
'status': 'completed',
|
||||
'total_items': total,
|
||||
'inconsistent_items': n_inconsistent,
|
||||
'inconsistency_rate': round(inconsistency_rate, 4),
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'details': results,
|
||||
'interpretation': _interpret_grim(n_inconsistent, total, inconsistency_rate)
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _interpret_grim(n_inconsistent, total, rate):
|
||||
"""生成可读的解释"""
|
||||
if n_inconsistent == 0:
|
||||
return f"✅ 全部 {total} 个均值通过 GRIM 检验,未发现数值不一致。"
|
||||
elif rate > 0.5:
|
||||
return (
|
||||
f"⚠️ {total} 个均值中有 {n_inconsistent} 个({rate:.0%})未通过 GRIM 检验。"
|
||||
f"超过半数均值与样本量不兼容,这是严重的数据不一致信号。"
|
||||
f"强烈建议核查原始数据。"
|
||||
)
|
||||
elif rate > 0.25:
|
||||
return (
|
||||
f"⚠️ {total} 个均值中有 {n_inconsistent} 个({rate:.0%})未通过 GRIM 检验。"
|
||||
f"建议仔细核查这些不一致的数据点。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ {total} 个均值中有 {n_inconsistent} 个({rate:.0%})未通过 GRIM 检验。"
|
||||
f"少量不一致可能是四舍五入方式不同导致,建议结合其他检测综合判断。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='GRIM测试 - 检测报告均值与样本量的一致性'
|
||||
)
|
||||
parser.add_argument('--mean', type=str, help='报告的平均值')
|
||||
parser.add_argument('--n', type=int, help='样本量')
|
||||
parser.add_argument('--decimals', type=int, default=2, help='小数位数')
|
||||
parser.add_argument('--scale', type=str, help='量表范围,如 "1-5"')
|
||||
parser.add_argument('--input', '-i', help='输入JSON文件(批量测试)')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
scale_min, scale_max = None, None
|
||||
if args.scale:
|
||||
parts = args.scale.split('-')
|
||||
if len(parts) == 2:
|
||||
scale_min, scale_max = int(parts[0]), int(parts[1])
|
||||
|
||||
if args.input:
|
||||
# 批量模式
|
||||
with open(args.input, 'r', encoding='utf-8') as f:
|
||||
items = json.load(f)
|
||||
result = grim_test_batch(items)
|
||||
elif args.mean and args.n:
|
||||
# 单项模式
|
||||
res = grim_test_single(args.mean, args.n, args.decimals, scale_min, scale_max)
|
||||
result = {
|
||||
'test_name': 'GRIM Test (均值粒度一致性检验)',
|
||||
'status': 'completed',
|
||||
'result': res,
|
||||
'interpretation': (
|
||||
f"✅ 均值 {args.mean} 与样本量 {args.n} 一致" if res.get('consistent')
|
||||
else f"⚠️ 均值 {args.mean} 与样本量 {args.n} 不一致!最近合法均值为 {res.get('nearest_valid_mean')}"
|
||||
)
|
||||
}
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
289
tools/gengskill/scripts/image_duplicate_test.py
Normal file
289
tools/gengskill/scripts/image_duplicate_test.py
Normal file
@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
图像重复检测 (Image Duplication Detection)
|
||||
============================================
|
||||
原理:检测论文图片中是否存在重复使用或篡改的图像。
|
||||
使用感知哈希(pHash)和结构相似性(SSIM)来识别:
|
||||
1. 完全相同的图片出现在不同实验条件下
|
||||
2. 经过旋转、翻转、裁剪后重复使用的图片
|
||||
3. 调整亮度/对比度后复用的图片
|
||||
|
||||
这是学术造假中常见的手段,尤其在Western blot、
|
||||
显微镜图片、流式细胞术散点图等场景中。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
HAS_PILLOW = True
|
||||
except ImportError:
|
||||
HAS_PILLOW = False
|
||||
|
||||
try:
|
||||
from skimage.metrics import structural_similarity as ssim
|
||||
HAS_SKIMAGE = True
|
||||
except ImportError:
|
||||
HAS_SKIMAGE = False
|
||||
|
||||
|
||||
def average_hash(image, hash_size=16):
|
||||
"""计算平均感知哈希"""
|
||||
img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
mean = pixels.mean()
|
||||
return (pixels > mean).flatten()
|
||||
|
||||
|
||||
def difference_hash(image, hash_size=16):
|
||||
"""计算差异感知哈希"""
|
||||
img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
return (pixels[:, 1:] > pixels[:, :-1]).flatten()
|
||||
|
||||
|
||||
def hamming_distance(hash1, hash2):
|
||||
"""计算汉明距离(归一化到0-1)"""
|
||||
return np.sum(hash1 != hash2) / len(hash1)
|
||||
|
||||
|
||||
def compute_ssim(img1, img2, target_size=(256, 256)):
|
||||
"""计算结构相似性指数"""
|
||||
if not HAS_SKIMAGE:
|
||||
return None
|
||||
|
||||
# 统一尺寸
|
||||
img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS)
|
||||
img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS)
|
||||
|
||||
arr1 = np.array(img1_resized)
|
||||
arr2 = np.array(img2_resized)
|
||||
|
||||
score = ssim(arr1, arr2)
|
||||
return float(score)
|
||||
|
||||
|
||||
def check_rotations(img1, img2, threshold=0.85):
|
||||
"""检查图像经过旋转/翻转后是否匹配"""
|
||||
transformations = [
|
||||
('original', lambda x: x),
|
||||
('rotate_90', lambda x: x.rotate(90, expand=True)),
|
||||
('rotate_180', lambda x: x.rotate(180, expand=True)),
|
||||
('rotate_270', lambda x: x.rotate(270, expand=True)),
|
||||
('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)),
|
||||
('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)),
|
||||
]
|
||||
|
||||
best_match = None
|
||||
best_score = 0
|
||||
|
||||
hash1 = average_hash(img1)
|
||||
|
||||
for name, transform in transformations:
|
||||
transformed = transform(img2)
|
||||
hash2 = average_hash(transformed)
|
||||
similarity = 1 - hamming_distance(hash1, hash2)
|
||||
|
||||
if similarity > best_score:
|
||||
best_score = similarity
|
||||
best_match = name
|
||||
|
||||
return {
|
||||
'best_transformation': best_match,
|
||||
'best_similarity': round(best_score, 4),
|
||||
'is_match': best_score >= threshold
|
||||
}
|
||||
|
||||
|
||||
def find_duplicates(image_dir, threshold=0.85, extensions=None):
|
||||
"""
|
||||
在目录中查找重复或相似的图片
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image_dir : str
|
||||
图片目录路径
|
||||
threshold : float
|
||||
相似度阈值(0-1),超过此值判定为重复
|
||||
extensions : list
|
||||
支持的图片格式
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if not HAS_PILLOW:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '需要安装 Pillow: pip install Pillow'
|
||||
}
|
||||
|
||||
if extensions is None:
|
||||
extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif']
|
||||
|
||||
# 收集所有图片文件
|
||||
image_files = []
|
||||
for ext in extensions:
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext}'))
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext.upper()}'))
|
||||
|
||||
image_files = sorted(set(image_files))
|
||||
|
||||
if len(image_files) < 2:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'目录中仅找到 {len(image_files)} 张图片,需要至少2张进行比较',
|
||||
'n_images': len(image_files)
|
||||
}
|
||||
|
||||
# 计算所有图片的哈希
|
||||
hashes = {}
|
||||
for img_path in image_files:
|
||||
try:
|
||||
img = Image.open(img_path)
|
||||
hashes[str(img_path)] = {
|
||||
'avg_hash': average_hash(img),
|
||||
'diff_hash': difference_hash(img),
|
||||
'size': img.size,
|
||||
'image': img
|
||||
}
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
# 两两比较
|
||||
duplicates = []
|
||||
paths = list(hashes.keys())
|
||||
|
||||
for i in range(len(paths)):
|
||||
for j in range(i + 1, len(paths)):
|
||||
path1, path2 = paths[i], paths[j]
|
||||
h1, h2 = hashes[path1], hashes[path2]
|
||||
|
||||
# 平均哈希相似度
|
||||
avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash'])
|
||||
|
||||
# 差异哈希相似度
|
||||
diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash'])
|
||||
|
||||
# 综合相似度
|
||||
combined_sim = max(avg_sim, diff_sim)
|
||||
|
||||
if combined_sim >= threshold:
|
||||
pair_result = {
|
||||
'file_1': os.path.basename(path1),
|
||||
'file_2': os.path.basename(path2),
|
||||
'avg_hash_similarity': round(float(avg_sim), 4),
|
||||
'diff_hash_similarity': round(float(diff_sim), 4),
|
||||
'combined_similarity': round(float(combined_sim), 4),
|
||||
}
|
||||
|
||||
# 检查旋转/翻转匹配
|
||||
rotation_check = check_rotations(
|
||||
h1['image'], h2['image'], threshold
|
||||
)
|
||||
pair_result['rotation_check'] = rotation_check
|
||||
|
||||
# SSIM(如果可用)
|
||||
if HAS_SKIMAGE:
|
||||
ssim_score = compute_ssim(h1['image'], h2['image'])
|
||||
pair_result['ssim'] = round(ssim_score, 4)
|
||||
|
||||
duplicates.append(pair_result)
|
||||
|
||||
# 关闭所有图片
|
||||
for h in hashes.values():
|
||||
h['image'].close()
|
||||
|
||||
# 风险评分
|
||||
n_duplicates = len(duplicates)
|
||||
n_images = len(image_files)
|
||||
|
||||
if n_duplicates == 0:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
elif n_duplicates <= 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40
|
||||
elif n_duplicates <= 3:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60
|
||||
else:
|
||||
risk_level = 'high'
|
||||
risk_score = 80 + min(20, n_duplicates * 3)
|
||||
|
||||
# 如果有完美匹配(相似度>0.98),直接拉高
|
||||
perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
if perfect_matches:
|
||||
risk_score = max(risk_score, 90)
|
||||
risk_level = 'high'
|
||||
|
||||
result = {
|
||||
'test_name': 'Image Duplication Detection (图像重复检测)',
|
||||
'status': 'completed',
|
||||
'n_images_scanned': n_images,
|
||||
'n_duplicate_pairs': n_duplicates,
|
||||
'threshold': threshold,
|
||||
'duplicates': duplicates,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_image(n_duplicates, n_images, duplicates)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_image(n_duplicates, n_images, duplicates):
|
||||
"""生成可读的解释"""
|
||||
if n_duplicates == 0:
|
||||
return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。"
|
||||
|
||||
perfect = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
|
||||
if perfect:
|
||||
return (
|
||||
f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片!"
|
||||
f"这些图片可能是同一图片的重复使用,强烈建议核查是否为不同实验条件下的独立数据。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ 发现 {n_duplicates} 对高度相似的图片(共扫描 {n_images} 张)。"
|
||||
f"可能存在图片复用或篡改,建议人工核查具体图片内容。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='图像重复检测 - 检测论文图片是否存在重复使用或篡改'
|
||||
)
|
||||
parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.85,
|
||||
help='相似度阈值(0-1),默认0.85')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.input_dir):
|
||||
print(f"错误:目录不存在: {args.input_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = find_duplicates(args.input_dir, args.threshold)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
301
tools/gengskill/scripts/image_duplicate_test_patched.py
Normal file
301
tools/gengskill/scripts/image_duplicate_test_patched.py
Normal file
@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
图像重复检测 (Image Duplication Detection)
|
||||
============================================
|
||||
原理:检测论文图片中是否存在重复使用或篡改的图像。
|
||||
使用感知哈希(pHash)和结构相似性(SSIM)来识别:
|
||||
1. 完全相同的图片出现在不同实验条件下
|
||||
2. 经过旋转、翻转、裁剪后重复使用的图片
|
||||
3. 调整亮度/对比度后复用的图片
|
||||
|
||||
这是学术造假中常见的手段,尤其在Western blot、
|
||||
显微镜图片、流式细胞术散点图等场景中。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
|
||||
class NumpyEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
import numpy as np
|
||||
if isinstance(obj, (np.integer,)): return int(obj)
|
||||
if isinstance(obj, (np.floating,)): return float(obj)
|
||||
if isinstance(obj, (np.bool_,)): return bool(obj)
|
||||
if isinstance(obj, np.ndarray): return obj.tolist()
|
||||
if obj is None: return None
|
||||
return super().default(obj)
|
||||
|
||||
HAS_PILLOW = True
|
||||
except ImportError:
|
||||
HAS_PILLOW = False
|
||||
|
||||
try:
|
||||
from skimage.metrics import structural_similarity as ssim
|
||||
HAS_SKIMAGE = True
|
||||
except ImportError:
|
||||
HAS_SKIMAGE = False
|
||||
|
||||
|
||||
def average_hash(image, hash_size=16):
|
||||
"""计算平均感知哈希"""
|
||||
img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
mean = pixels.mean()
|
||||
return (pixels > mean).flatten()
|
||||
|
||||
|
||||
def difference_hash(image, hash_size=16):
|
||||
"""计算差异感知哈希"""
|
||||
img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
return (pixels[:, 1:] > pixels[:, :-1]).flatten()
|
||||
|
||||
|
||||
def hamming_distance(hash1, hash2):
|
||||
"""计算汉明距离(归一化到0-1)"""
|
||||
return np.sum(hash1 != hash2) / len(hash1)
|
||||
|
||||
|
||||
def compute_ssim(img1, img2, target_size=(256, 256)):
|
||||
"""计算结构相似性指数"""
|
||||
if not HAS_SKIMAGE:
|
||||
return None
|
||||
|
||||
# 统一尺寸
|
||||
img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS)
|
||||
img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS)
|
||||
|
||||
arr1 = np.array(img1_resized)
|
||||
arr2 = np.array(img2_resized)
|
||||
|
||||
score = ssim(arr1, arr2)
|
||||
return float(score)
|
||||
|
||||
|
||||
def check_rotations(img1, img2, threshold=0.85):
|
||||
"""检查图像经过旋转/翻转后是否匹配"""
|
||||
transformations = [
|
||||
('original', lambda x: x),
|
||||
('rotate_90', lambda x: x.rotate(90, expand=True)),
|
||||
('rotate_180', lambda x: x.rotate(180, expand=True)),
|
||||
('rotate_270', lambda x: x.rotate(270, expand=True)),
|
||||
('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)),
|
||||
('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)),
|
||||
]
|
||||
|
||||
best_match = None
|
||||
best_score = 0
|
||||
|
||||
hash1 = average_hash(img1)
|
||||
|
||||
for name, transform in transformations:
|
||||
transformed = transform(img2)
|
||||
hash2 = average_hash(transformed)
|
||||
similarity = 1 - hamming_distance(hash1, hash2)
|
||||
|
||||
if similarity > best_score:
|
||||
best_score = similarity
|
||||
best_match = name
|
||||
|
||||
return {
|
||||
'best_transformation': best_match,
|
||||
'best_similarity': round(best_score, 4),
|
||||
'is_match': best_score >= threshold
|
||||
}
|
||||
|
||||
|
||||
def find_duplicates(image_dir, threshold=0.85, extensions=None):
|
||||
"""
|
||||
在目录中查找重复或相似的图片
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image_dir : str
|
||||
图片目录路径
|
||||
threshold : float
|
||||
相似度阈值(0-1),超过此值判定为重复
|
||||
extensions : list
|
||||
支持的图片格式
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if not HAS_PILLOW:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '需要安装 Pillow: pip install Pillow'
|
||||
}
|
||||
|
||||
if extensions is None:
|
||||
extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif']
|
||||
|
||||
# 收集所有图片文件
|
||||
image_files = []
|
||||
for ext in extensions:
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext}'))
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext.upper()}'))
|
||||
|
||||
image_files = sorted(set(image_files))
|
||||
|
||||
if len(image_files) < 2:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'目录中仅找到 {len(image_files)} 张图片,需要至少2张进行比较',
|
||||
'n_images': len(image_files)
|
||||
}
|
||||
|
||||
# 计算所有图片的哈希
|
||||
hashes = {}
|
||||
for img_path in image_files:
|
||||
try:
|
||||
img = Image.open(img_path)
|
||||
hashes[str(img_path)] = {
|
||||
'avg_hash': average_hash(img),
|
||||
'diff_hash': difference_hash(img),
|
||||
'size': img.size,
|
||||
'image': img
|
||||
}
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
# 两两比较
|
||||
duplicates = []
|
||||
paths = list(hashes.keys())
|
||||
|
||||
for i in range(len(paths)):
|
||||
for j in range(i + 1, len(paths)):
|
||||
path1, path2 = paths[i], paths[j]
|
||||
h1, h2 = hashes[path1], hashes[path2]
|
||||
|
||||
# 平均哈希相似度
|
||||
avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash'])
|
||||
|
||||
# 差异哈希相似度
|
||||
diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash'])
|
||||
|
||||
# 综合相似度
|
||||
combined_sim = max(avg_sim, diff_sim)
|
||||
|
||||
if combined_sim >= threshold:
|
||||
pair_result = {
|
||||
'file_1': os.path.basename(path1),
|
||||
'file_2': os.path.basename(path2),
|
||||
'avg_hash_similarity': round(float(avg_sim), 4),
|
||||
'diff_hash_similarity': round(float(diff_sim), 4),
|
||||
'combined_similarity': round(float(combined_sim), 4),
|
||||
}
|
||||
|
||||
# 检查旋转/翻转匹配
|
||||
rotation_check = check_rotations(
|
||||
h1['image'], h2['image'], threshold
|
||||
)
|
||||
pair_result['rotation_check'] = rotation_check
|
||||
|
||||
# SSIM(如果可用)
|
||||
if HAS_SKIMAGE:
|
||||
ssim_score = compute_ssim(h1['image'], h2['image'])
|
||||
pair_result['ssim'] = round(ssim_score, 4)
|
||||
|
||||
duplicates.append(pair_result)
|
||||
|
||||
# 关闭所有图片
|
||||
for h in hashes.values():
|
||||
h['image'].close()
|
||||
|
||||
# 风险评分
|
||||
n_duplicates = len(duplicates)
|
||||
n_images = len(image_files)
|
||||
|
||||
if n_duplicates == 0:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
elif n_duplicates <= 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40
|
||||
elif n_duplicates <= 3:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60
|
||||
else:
|
||||
risk_level = 'high'
|
||||
risk_score = 80 + min(20, n_duplicates * 3)
|
||||
|
||||
# 如果有完美匹配(相似度>0.98),直接拉高
|
||||
perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
if perfect_matches:
|
||||
risk_score = max(risk_score, 90)
|
||||
risk_level = 'high'
|
||||
|
||||
result = {
|
||||
'test_name': 'Image Duplication Detection (图像重复检测)',
|
||||
'status': 'completed',
|
||||
'n_images_scanned': n_images,
|
||||
'n_duplicate_pairs': n_duplicates,
|
||||
'threshold': threshold,
|
||||
'duplicates': duplicates,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_image(n_duplicates, n_images, duplicates)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_image(n_duplicates, n_images, duplicates):
|
||||
"""生成可读的解释"""
|
||||
if n_duplicates == 0:
|
||||
return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。"
|
||||
|
||||
perfect = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
|
||||
if perfect:
|
||||
return (
|
||||
f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片!"
|
||||
f"这些图片可能是同一图片的重复使用,强烈建议核查是否为不同实验条件下的独立数据。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ 发现 {n_duplicates} 对高度相似的图片(共扫描 {n_images} 张)。"
|
||||
f"可能存在图片复用或篡改,建议人工核查具体图片内容。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='图像重复检测 - 检测论文图片是否存在重复使用或篡改'
|
||||
)
|
||||
parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.85,
|
||||
help='相似度阈值(0-1),默认0.85')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.input_dir):
|
||||
print(f"错误:目录不存在: {args.input_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = find_duplicates(args.input_dir, args.threshold)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2, cls=NumpyEncoder)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1713
tools/gengskill/scripts/input_pipeline.py
Normal file
1713
tools/gengskill/scripts/input_pipeline.py
Normal file
File diff suppressed because it is too large
Load Diff
244
tools/gengskill/scripts/last_digit_test.py
Normal file
244
tools/gengskill/scripts/last_digit_test.py
Normal file
@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
末位数字检测 (Last Digit Test)
|
||||
==============================
|
||||
原理:自然实验数据的末位数字(0-9)应近似均匀分布。
|
||||
如果数据是人为编造的,末位数字往往会集中在某些特定值上。
|
||||
|
||||
使用卡方检验评估偏离均匀分布的程度。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话,让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def extract_last_digit(value):
|
||||
"""提取数值的末位有效数字"""
|
||||
s = str(value).strip()
|
||||
# 去除负号
|
||||
s = s.lstrip('-')
|
||||
# 去除科学计数法
|
||||
if 'e' in s.lower():
|
||||
try:
|
||||
value = float(s)
|
||||
s = f"{value:.10f}".rstrip('0')
|
||||
except ValueError:
|
||||
return None
|
||||
# 找到末位有效数字
|
||||
s = s.rstrip('0').rstrip('.')
|
||||
if not s:
|
||||
return 0
|
||||
for c in reversed(s):
|
||||
if c.isdigit():
|
||||
return int(c)
|
||||
return None
|
||||
|
||||
|
||||
def extract_last_digit_with_decimals(value, use_decimal_last=True):
|
||||
"""
|
||||
提取数值的末位数字
|
||||
use_decimal_last=True: 取小数点后最后一位非零数字
|
||||
use_decimal_last=False: 取整数部分的末位数字
|
||||
"""
|
||||
s = str(value).strip()
|
||||
s = s.lstrip('-')
|
||||
|
||||
if '.' in s and use_decimal_last:
|
||||
decimal_part = s.split('.')[1]
|
||||
# 取小数部分的最后一位数字
|
||||
if decimal_part:
|
||||
return int(decimal_part[-1])
|
||||
|
||||
# 取整数部分的末位
|
||||
integer_part = s.split('.')[0] if '.' in s else s
|
||||
if integer_part:
|
||||
return int(integer_part[-1])
|
||||
return None
|
||||
|
||||
|
||||
def last_digit_test(values, method='all_digits'):
|
||||
"""
|
||||
执行末位数字检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list of float/str
|
||||
待检测的数值列表
|
||||
method : str
|
||||
'all_digits' - 检测最后一位有效数字
|
||||
'decimal_last' - 检测小数末位
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
# 提取末位数字
|
||||
last_digits = []
|
||||
for v in values:
|
||||
try:
|
||||
if method == 'decimal_last':
|
||||
d = extract_last_digit_with_decimals(v, use_decimal_last=True)
|
||||
else:
|
||||
d = extract_last_digit(v)
|
||||
if d is not None:
|
||||
last_digits.append(d)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
if len(last_digits) < 10:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足(仅{len(last_digits)}个有效值),需要至少10个数据点',
|
||||
'n_valid': len(last_digits)
|
||||
}
|
||||
|
||||
# 统计各数字出现频次
|
||||
digit_counts = Counter(last_digits)
|
||||
observed = np.array([digit_counts.get(i, 0) for i in range(10)])
|
||||
expected = np.full(10, len(last_digits) / 10.0)
|
||||
|
||||
# 卡方检验
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
# 计算集中度指标
|
||||
max_digit = int(np.argmax(observed))
|
||||
max_freq = observed[max_digit] / len(last_digits)
|
||||
|
||||
# 均匀性评分 (0=完全均匀, 1=完全集中)
|
||||
uniformity_deviation = np.sqrt(np.sum((observed / len(last_digits) - 0.1) ** 2) / 10) / 0.3
|
||||
uniformity_deviation = min(uniformity_deviation, 1.0)
|
||||
|
||||
# 风险评分
|
||||
if p_value < 0.001:
|
||||
risk_level = 'high'
|
||||
risk_score = min(80 + (1 - p_value) * 20, 100)
|
||||
elif p_value < 0.01:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60 + (0.01 - p_value) / 0.009 * 20
|
||||
elif p_value < 0.05:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40 + (0.05 - p_value) / 0.04 * 20
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = max(0, 40 * (1 - p_value))
|
||||
|
||||
result = {
|
||||
'test_name': 'Last Digit Test (末位数字检测)',
|
||||
'status': 'completed',
|
||||
'n_values': len(last_digits),
|
||||
'method': method,
|
||||
'digit_distribution': {str(i): int(observed[i]) for i in range(10)},
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'degrees_of_freedom': 9,
|
||||
'most_frequent_digit': max_digit,
|
||||
'most_frequent_proportion': round(float(max_freq), 4),
|
||||
'uniformity_deviation': round(float(uniformity_deviation), 4),
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_result(p_value, max_digit, max_freq, len(last_digits))
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_result(p_value, max_digit, max_freq, n):
|
||||
"""生成可读的解释"""
|
||||
if p_value < 0.001:
|
||||
return (
|
||||
f"⚠️ 末位数字分布严重偏离均匀分布(p < 0.001)。"
|
||||
f"数字 {max_digit} 出现频率为 {max_freq:.1%}(期望 10%),"
|
||||
f"这种偏离在自然实验数据中极为罕见,强烈建议进一步核查原始数据。"
|
||||
)
|
||||
elif p_value < 0.01:
|
||||
return (
|
||||
f"⚠️ 末位数字分布显著偏离均匀分布(p < 0.01)。"
|
||||
f"数字 {max_digit} 出现频率为 {max_freq:.1%},建议关注并进行人工复核。"
|
||||
)
|
||||
elif p_value < 0.05:
|
||||
return (
|
||||
f"⚡ 末位数字分布存在一定偏离(p < 0.05)。"
|
||||
f"可能是正常波动,也可能提示数据存在问题,建议结合其他检测结果综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"✅ 末位数字分布与均匀分布无显著差异(p = {p_value:.4f}),"
|
||||
f"未发现明显异常。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
try:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
float(val) # 验证是数值
|
||||
values.append(val)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
else:
|
||||
# 如果没有指定列,尝试读取第一个数值列
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='末位数字检测 - 检测数据末位数字是否偏离均匀分布'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--method', '-m', default='all_digits',
|
||||
choices=['all_digits', 'decimal_last'],
|
||||
help='检测方法:all_digits=末位有效数字, decimal_last=小数末位')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
parser.add_argument('--values', nargs='+', type=str,
|
||||
help='直接传入数值列表(不使用文件输入)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.values:
|
||||
values = args.values
|
||||
else:
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误:未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = last_digit_test(values, method=args.method)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1775
tools/gengskill/scripts/report_generator.py
Normal file
1775
tools/gengskill/scripts/report_generator.py
Normal file
File diff suppressed because it is too large
Load Diff
37
tools/gengskill/scripts/run_dup_test.py
Normal file
37
tools/gengskill/scripts/run_dup_test.py
Normal file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrapper to run image_duplicate_test with proper numpy handling"""
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
# We'll monkey-patch json to handle numpy types
|
||||
class NumpyEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
if isinstance(obj, (np.floating,)):
|
||||
return float(obj)
|
||||
if isinstance(obj, (np.bool_,)):
|
||||
return bool(obj)
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
if obj is None:
|
||||
return None
|
||||
return super().default(obj)
|
||||
|
||||
# Save original dumps
|
||||
original_dumps = json.dumps
|
||||
|
||||
def patched_dumps(obj, **kwargs):
|
||||
kwargs.setdefault('cls', NumpyEncoder)
|
||||
return original_dumps(obj, **kwargs)
|
||||
|
||||
json.dumps = patched_dumps
|
||||
|
||||
# Now import and run the original script
|
||||
sys.argv = ['image_duplicate_test.py',
|
||||
'--input_dir', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/figures',
|
||||
'--threshold', '0.85',
|
||||
'--output', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/duplicate_results.json']
|
||||
|
||||
exec(open('scripts/image_duplicate_test.py').read())
|
||||
1228
tools/gengskill/scripts/visualization.py
Normal file
1228
tools/gengskill/scripts/visualization.py
Normal file
File diff suppressed because it is too large
Load Diff
346
tools/gengskill/tests/test_modules.py
Normal file
346
tools/gengskill/tests/test_modules.py
Normal file
@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geng Skill 单元测试
|
||||
====================
|
||||
验证各检测模块的正确性、边界条件处理和风险评分一致性。
|
||||
|
||||
运行方式:
|
||||
cd geng-skill
|
||||
python3 -m pytest tests/test_modules.py -v
|
||||
|
||||
或直接运行:
|
||||
python3 tests/test_modules.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import math
|
||||
|
||||
# 添加 scripts 目录到路径
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'scripts'))
|
||||
|
||||
from last_digit_test import last_digit_test, extract_last_digit
|
||||
from benford_test import benford_test, get_first_digit
|
||||
from grim_test import grim_test_single, grim_test_batch
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
from decimal_consistency_test import decimal_consistency_test
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test: Last Digit Test
|
||||
# ============================================================
|
||||
|
||||
class TestLastDigitTest:
|
||||
"""末位数字检测模块测试"""
|
||||
|
||||
def test_uniform_data_low_risk(self):
|
||||
"""均匀分布的末位数字应返回低风险"""
|
||||
random.seed(42)
|
||||
# 生成末位数字均匀的数据
|
||||
values = [f"{random.uniform(1, 100):.2f}" for _ in range(100)]
|
||||
result = last_digit_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['risk_level'] == 'low'
|
||||
assert result['risk_score'] < 40
|
||||
|
||||
def test_concentrated_digits_high_risk(self):
|
||||
"""集中在某个数字的末位应返回高风险"""
|
||||
# 80% 的末位是 "5"
|
||||
values = ['1.25', '2.35', '3.45', '4.55', '5.65',
|
||||
'6.75', '7.85', '8.95', '9.15', '10.25',
|
||||
'11.35', '12.45', '13.55', '14.65', '15.75',
|
||||
'16.85', '17.95', '18.05', '19.15', '20.55']
|
||||
result = last_digit_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['most_frequent_digit'] == 5
|
||||
|
||||
def test_insufficient_data(self):
|
||||
"""数据量不足应返回 insufficient_data"""
|
||||
values = ['1.23', '4.56', '7.89']
|
||||
result = last_digit_test(values)
|
||||
assert result['status'] == 'insufficient_data'
|
||||
|
||||
def test_extract_last_digit(self):
|
||||
"""末位数字提取正确性"""
|
||||
assert extract_last_digit('3.14') == 4
|
||||
assert extract_last_digit('100') == 1
|
||||
assert extract_last_digit('-5.67') == 7
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test: Benford's Law Test
|
||||
# ============================================================
|
||||
|
||||
class TestBenfordTest:
|
||||
"""本福特定律检测模块测试"""
|
||||
|
||||
def test_benford_compliant_data(self):
|
||||
"""符合本福特定律的数据应返回低风险"""
|
||||
# 生成符合本福特定律的数据
|
||||
random.seed(42)
|
||||
values = []
|
||||
for _ in range(200):
|
||||
# 对数均匀分布产生的数据符合本福特定律
|
||||
val = 10 ** (random.uniform(0, 4))
|
||||
values.append(f"{val:.2f}")
|
||||
result = benford_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
# 对数均匀数据应近似符合
|
||||
assert result['conformity'] in ('close', 'acceptable', 'marginal')
|
||||
|
||||
def test_uniform_first_digit_high_risk(self):
|
||||
"""首位数字均匀分布应偏离本福特定律"""
|
||||
# 人为造假数据:首位数字接近均匀
|
||||
values = []
|
||||
for d in range(1, 10):
|
||||
for _ in range(20):
|
||||
values.append(f"{d}{random.randint(10,99)}")
|
||||
result = benford_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
# 均匀分布会显著偏离本福特定律
|
||||
assert result['p_value'] < 0.05
|
||||
|
||||
def test_insufficient_data(self):
|
||||
"""数据量不足"""
|
||||
values = ['123', '456']
|
||||
result = benford_test(values)
|
||||
assert result['status'] == 'insufficient_data'
|
||||
|
||||
def test_first_digit_extraction(self):
|
||||
"""首位数字提取"""
|
||||
assert get_first_digit('314.15') == 3
|
||||
assert get_first_digit('0.0052') == 5
|
||||
assert get_first_digit('9999') == 9
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test: GRIM Test
|
||||
# ============================================================
|
||||
|
||||
class TestGrimTest:
|
||||
"""GRIM 测试模块测试"""
|
||||
|
||||
def test_consistent_mean(self):
|
||||
"""合法均值应通过"""
|
||||
# n=20, 整数数据, mean=3.40 → sum=68 (整数) ✓
|
||||
result = grim_test_single('3.40', 20, decimals=2)
|
||||
assert result['consistent'] == True
|
||||
|
||||
def test_inconsistent_mean(self):
|
||||
"""非法均值应失败"""
|
||||
# n=20, 整数数据, mean=3.47 → sum=69.4 (非整数) ✗
|
||||
result = grim_test_single('3.47', 20, decimals=2)
|
||||
assert result['consistent'] == False
|
||||
|
||||
def test_consistent_mean_n25(self):
|
||||
"""n=25 的合法均值"""
|
||||
# n=25, mean=3.48 → sum=87 (整数) ✓
|
||||
result = grim_test_single('3.48', 25, decimals=2)
|
||||
assert result['consistent'] == True
|
||||
|
||||
def test_batch_mode(self):
|
||||
"""批量 GRIM 测试"""
|
||||
items = [
|
||||
{'mean': '3.40', 'n': 20, 'decimals': 2, 'label': 'Item A'},
|
||||
{'mean': '3.47', 'n': 20, 'decimals': 2, 'label': 'Item B'},
|
||||
{'mean': '4.00', 'n': 10, 'decimals': 2, 'label': 'Item C'},
|
||||
]
|
||||
result = grim_test_batch(items)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['total_items'] == 3
|
||||
assert result['inconsistent_items'] == 1 # 3.47/20 不一致
|
||||
|
||||
def test_range_check(self):
|
||||
"""量表范围检查"""
|
||||
# 均值超出量表范围
|
||||
result = grim_test_single('6.50', 20, decimals=2, scale_min=1, scale_max=5)
|
||||
assert result.get('consistent') == False or result.get('status') == 'range_error'
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test: Fixed Relation Test
|
||||
# ============================================================
|
||||
|
||||
class TestFixedRelationTest:
|
||||
"""固定关系检测模块测试"""
|
||||
|
||||
def test_exact_ratio_detected(self):
|
||||
"""精确固定比值应被检测到"""
|
||||
col1 = [1.23, 2.34, 3.45, 4.56, 5.67, 6.78, 7.89]
|
||||
col2 = [2.46, 4.68, 6.90, 9.12, 11.34, 13.56, 15.78] # ×2
|
||||
result = fixed_relation_test(col1, col2)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['risk_level'] == 'high'
|
||||
assert result['risk_score'] >= 85
|
||||
assert result['detections']['fixed_ratio']['is_exact'] == True
|
||||
|
||||
def test_exact_difference_detected(self):
|
||||
"""精确固定差值应被检测到"""
|
||||
col1 = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]
|
||||
col2 = [4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0] # +3
|
||||
result = fixed_relation_test(col1, col2)
|
||||
assert result['detections']['fixed_difference']['is_exact'] == True
|
||||
|
||||
def test_independent_data_low_risk(self):
|
||||
"""独立随机数据应返回低风险"""
|
||||
random.seed(42)
|
||||
col1 = [random.uniform(1, 10) for _ in range(30)]
|
||||
col2 = [random.uniform(1, 10) for _ in range(30)]
|
||||
result = fixed_relation_test(col1, col2)
|
||||
assert result['risk_level'] == 'low'
|
||||
assert result['risk_score'] < 30
|
||||
|
||||
def test_insufficient_data(self):
|
||||
"""数据不足"""
|
||||
result = fixed_relation_test([1.0, 2.0], [3.0, 4.0])
|
||||
assert result['status'] == 'insufficient_data'
|
||||
|
||||
def test_unequal_lengths(self):
|
||||
"""长度不一致应报错"""
|
||||
result = fixed_relation_test([1, 2, 3], [4, 5])
|
||||
assert result['status'] == 'error'
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Test: Decimal Consistency Test
|
||||
# ============================================================
|
||||
|
||||
class TestDecimalConsistencyTest:
|
||||
"""小数位一致性检测模块测试"""
|
||||
|
||||
def test_diverse_decimals_low_risk(self):
|
||||
"""多样化小数模式应低风险"""
|
||||
random.seed(42)
|
||||
values = [f"{random.uniform(1, 100):.{random.randint(1,4)}f}" for _ in range(50)]
|
||||
result = decimal_consistency_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['risk_level'] in ('low', 'medium')
|
||||
|
||||
def test_repeated_decimals_high_risk(self):
|
||||
"""高度重复的小数模式应高风险"""
|
||||
# 所有值小数部分都是 .34
|
||||
values = [f"{i}.34" for i in range(1, 31)]
|
||||
result = decimal_consistency_test(values)
|
||||
assert result['status'] == 'completed'
|
||||
assert result['risk_score'] >= 25 # 至少中风险
|
||||
|
||||
def test_insufficient_data(self):
|
||||
"""数据不足"""
|
||||
values = ['1.23', '4.56']
|
||||
result = decimal_consistency_test(values)
|
||||
assert result['status'] == 'insufficient_data'
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Integration Test
|
||||
# ============================================================
|
||||
|
||||
class TestIntegration:
|
||||
"""集成测试:模拟完整检测流程"""
|
||||
|
||||
def test_fake_data_high_risk(self):
|
||||
"""已知造假数据应返回高风险"""
|
||||
# 模拟耿同学发现的典型造假:固定比例关系
|
||||
control = [2.34, 3.12, 1.87, 4.56, 2.98, 3.45, 1.23, 5.67, 2.01, 3.89]
|
||||
treatment = [x * 2 for x in control] # 精确 ×2
|
||||
|
||||
result = fixed_relation_test(control, treatment, 'Control', 'Treatment')
|
||||
assert result['risk_score'] >= 85
|
||||
assert 'fixed_ratio' in result['detections']
|
||||
assert result['detections']['fixed_ratio']['mean_ratio'] == 2.0
|
||||
|
||||
def test_real_data_low_risk(self):
|
||||
"""正常实验数据应返回低风险"""
|
||||
random.seed(123)
|
||||
# 模拟真实实验:基础值 + 效应 + 随机噪声
|
||||
control = [random.gauss(5, 1.5) for _ in range(20)]
|
||||
treatment = [x * random.gauss(2, 0.3) for x in control] # ×2 但有变异
|
||||
|
||||
result = fixed_relation_test(control, treatment, 'Control', 'Treatment')
|
||||
# 由于有随机噪声,不应该报告"精确"固定关系
|
||||
assert result['detections']['fixed_ratio']['is_exact'] == False
|
||||
|
||||
def test_output_schema_compliance(self):
|
||||
"""输出格式应符合标准 schema"""
|
||||
values = [f"{random.uniform(1, 100):.2f}" for _ in range(50)]
|
||||
result = last_digit_test(values)
|
||||
|
||||
# 必须包含的标准字段
|
||||
required_fields = ['test_name', 'status', 'risk_level', 'risk_score', 'interpretation']
|
||||
for field in required_fields:
|
||||
assert field in result, f"缺少必需字段: {field}"
|
||||
|
||||
# 风险评分范围
|
||||
assert 0 <= result['risk_score'] <= 100
|
||||
|
||||
# 风险等级合法值
|
||||
assert result['risk_level'] in ('low', 'medium', 'medium-high', 'high')
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Run tests
|
||||
# ============================================================
|
||||
|
||||
def run_all_tests():
|
||||
"""简易测试运行器(不依赖 pytest)"""
|
||||
import traceback
|
||||
|
||||
test_classes = [
|
||||
TestLastDigitTest,
|
||||
TestBenfordTest,
|
||||
TestGrimTest,
|
||||
TestFixedRelationTest,
|
||||
TestDecimalConsistencyTest,
|
||||
TestIntegration,
|
||||
]
|
||||
|
||||
total = 0
|
||||
passed = 0
|
||||
failed = 0
|
||||
errors = []
|
||||
|
||||
print("=" * 70)
|
||||
print(" 🧪 Geng Skill 单元测试")
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
for test_class in test_classes:
|
||||
class_name = test_class.__name__
|
||||
print(f"▶ {class_name}")
|
||||
|
||||
instance = test_class()
|
||||
methods = [m for m in dir(instance) if m.startswith('test_')]
|
||||
|
||||
for method_name in methods:
|
||||
total += 1
|
||||
try:
|
||||
getattr(instance, method_name)()
|
||||
passed += 1
|
||||
print(f" ✅ {method_name}")
|
||||
except AssertionError as e:
|
||||
failed += 1
|
||||
errors.append((class_name, method_name, str(e)))
|
||||
print(f" ❌ {method_name}: {e}")
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
errors.append((class_name, method_name, traceback.format_exc()))
|
||||
print(f" 💥 {method_name}: {type(e).__name__}: {e}")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print(f" 结果: {passed} 通过 / {failed} 失败 / {total} 总计")
|
||||
print("=" * 70)
|
||||
|
||||
if errors:
|
||||
print("\n❌ 失败详情:")
|
||||
for cls, method, err in errors:
|
||||
print(f" {cls}.{method}: {err[:200]}")
|
||||
|
||||
return failed == 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
success = run_all_tests()
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user