mirror of
https://gitcode.com/JianFeeeee/homeagent-sdk.git
synced 2026-09-21 09:28:04 +00:00
feat: add example plugins (ai_image, calendar, music, rss, weather), fix .gitignore, move gengskill to tools/
This commit is contained in:
root
256
tools/gengskill/scripts/benford_test.py
Normal file
256
tools/gengskill/scripts/benford_test.py
Normal file
@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
本福特定律检测 (Benford's Law Test)
|
||||
====================================
|
||||
原理：跨越多个数量级的自然数据，首位数字遵循特定概率分布：
|
||||
P(d) = log10(1 + 1/d), d = 1,2,...,9
|
||||
|
||||
人为编造的数据往往偏离这一分布（倾向于均匀分布或集中在某些数字）。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
# 本福特定律理论概率
|
||||
BENFORD_PROBS = {d: math.log10(1 + 1/d) for d in range(1, 10)}
|
||||
|
||||
|
||||
def get_first_digit(value):
|
||||
"""提取数值的首位有效数字（1-9）"""
|
||||
try:
|
||||
num = abs(float(value))
|
||||
if num == 0:
|
||||
return None
|
||||
# 转为科学计数法取首位
|
||||
s = f"{num:.10e}"
|
||||
first = int(s[0])
|
||||
if 1 <= first <= 9:
|
||||
return first
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_first_two_digits(value):
|
||||
"""提取前两位有效数字（10-99）"""
|
||||
try:
|
||||
num = abs(float(value))
|
||||
if num == 0:
|
||||
return None
|
||||
while num < 10:
|
||||
num *= 10
|
||||
while num >= 100:
|
||||
num /= 10
|
||||
return int(num)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def benford_test(values, order=1):
|
||||
"""
|
||||
执行本福特定律检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list
|
||||
待检测的数值列表
|
||||
order : int
|
||||
1 = 首位数字检测, 2 = 前两位数字检测
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if order == 1:
|
||||
digits = []
|
||||
for v in values:
|
||||
d = get_first_digit(v)
|
||||
if d is not None:
|
||||
digits.append(d)
|
||||
|
||||
if len(digits) < 30:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足（仅{len(digits)}个有效值），本福特定律检测建议至少100个数据点',
|
||||
'n_valid': len(digits)
|
||||
}
|
||||
|
||||
# 统计观测频率
|
||||
digit_counts = Counter(digits)
|
||||
observed = np.array([digit_counts.get(d, 0) for d in range(1, 10)])
|
||||
expected = np.array([BENFORD_PROBS[d] * len(digits) for d in range(1, 10)])
|
||||
|
||||
# 卡方检验
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
# Kolmogorov-Smirnov 检验
|
||||
observed_freq = observed / len(digits)
|
||||
expected_freq = np.array([BENFORD_PROBS[d] for d in range(1, 10)])
|
||||
|
||||
# 最大绝对偏差 (MAD)
|
||||
mad = np.mean(np.abs(observed_freq - expected_freq))
|
||||
|
||||
# MAD 阈值参考 (Nigrini 2012)
|
||||
# Close conformity: MAD < 0.006
|
||||
# Acceptable conformity: 0.006 <= MAD < 0.012
|
||||
# Marginally acceptable: 0.012 <= MAD < 0.015
|
||||
# Nonconformity: MAD >= 0.015
|
||||
|
||||
if mad < 0.006:
|
||||
conformity = 'close'
|
||||
conformity_cn = '高度符合'
|
||||
elif mad < 0.012:
|
||||
conformity = 'acceptable'
|
||||
conformity_cn = '可接受'
|
||||
elif mad < 0.015:
|
||||
conformity = 'marginal'
|
||||
conformity_cn = '边缘'
|
||||
else:
|
||||
conformity = 'nonconforming'
|
||||
conformity_cn = '不符合'
|
||||
|
||||
# 风险评分
|
||||
if p_value < 0.001 and mad >= 0.015:
|
||||
risk_level = 'high'
|
||||
risk_score = 75 + min(25, mad * 500)
|
||||
elif p_value < 0.01:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 55 + min(20, mad * 400)
|
||||
elif p_value < 0.05:
|
||||
risk_level = 'medium'
|
||||
risk_score = 35 + min(20, mad * 300)
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = max(0, mad * 200)
|
||||
|
||||
distribution = {
|
||||
str(d): {
|
||||
'observed': int(observed[d-1]),
|
||||
'observed_freq': round(float(observed_freq[d-1]), 4),
|
||||
'expected_freq': round(float(expected_freq[d-1]), 4),
|
||||
'deviation': round(float(observed_freq[d-1] - expected_freq[d-1]), 4)
|
||||
}
|
||||
for d in range(1, 10)
|
||||
}
|
||||
|
||||
result = {
|
||||
'test_name': "Benford's Law Test (本福特定律检测)",
|
||||
'status': 'completed',
|
||||
'order': order,
|
||||
'n_values': len(digits),
|
||||
'distribution': distribution,
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'degrees_of_freedom': 8,
|
||||
'mean_absolute_deviation': round(float(mad), 6),
|
||||
'conformity': conformity,
|
||||
'conformity_cn': conformity_cn,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_benford(p_value, mad, conformity_cn, len(digits)),
|
||||
'note': '本福特定律适用于跨多个数量级的自然数据集。对于范围有限的数据（如百分比、pH值），该检测可能不适用。'
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
else:
|
||||
return {'status': 'error', 'message': '目前仅支持首位数字检测（order=1）'}
|
||||
|
||||
|
||||
def _interpret_benford(p_value, mad, conformity_cn, n):
|
||||
"""生成可读的解释"""
|
||||
if p_value < 0.001 and mad >= 0.015:
|
||||
return (
|
||||
f"⚠️ 数据首位数字分布严重偏离本福特定律（p < 0.001, MAD = {mad:.4f}）。"
|
||||
f"符合性判定：{conformity_cn}。"
|
||||
f"这种偏离在{n}个数据点的样本中非常显著，强烈建议核查数据来源。"
|
||||
f"注意：需确认数据是否适用本福特定律（需跨越多个数量级）。"
|
||||
)
|
||||
elif p_value < 0.01:
|
||||
return (
|
||||
f"⚠️ 数据首位数字分布显著偏离本福特定律（p < 0.01, MAD = {mad:.4f}）。"
|
||||
f"符合性判定：{conformity_cn}。建议进一步检查。"
|
||||
)
|
||||
elif p_value < 0.05:
|
||||
return (
|
||||
f"⚡ 数据首位数字分布存在一定偏离（p < 0.05, MAD = {mad:.4f}）。"
|
||||
f"符合性判定：{conformity_cn}。可能是正常波动，建议结合其他检测综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"✅ 数据首位数字分布符合本福特定律（p = {p_value:.4f}, MAD = {mad:.4f}）。"
|
||||
f"符合性判定：{conformity_cn}。未发现异常。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
try:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
else:
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="本福特定律检测 - 检测首位数字是否符合Benford's Law"
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--order', type=int, default=1, choices=[1, 2],
|
||||
help='检测阶数：1=首位, 2=前两位')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误：未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = benford_test(values, order=args.order)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
280
tools/gengskill/scripts/decimal_consistency_test.py
Normal file
280
tools/gengskill/scripts/decimal_consistency_test.py
Normal file
@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
小数位一致性检测 (Decimal Consistency Test)
|
||||
============================================
|
||||
原理：实验测量数据的小数位后数字应具有一定的随机性。
|
||||
如果大量数据点的小数部分高度一致（如小数后两位总是相同），
|
||||
或小数位数模式过于规律，则暗示数据可能是人为编造的。
|
||||
|
||||
这是"耿同学"常用的一个检测手段——造假者编造数据时，
|
||||
小数点后的位数往往呈现不自然的一致性。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def get_decimal_digits(value, max_digits=4):
|
||||
"""提取数值的小数部分各位数字"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return []
|
||||
decimal_part = s.split('.')[1]
|
||||
return [int(d) for d in decimal_part[:max_digits]]
|
||||
|
||||
|
||||
def get_decimal_string(value):
|
||||
"""提取数值的完整小数字符串"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return ''
|
||||
return s.split('.')[1]
|
||||
|
||||
|
||||
def count_decimal_places(value):
|
||||
"""计算数值的小数位数"""
|
||||
s = str(value).strip()
|
||||
if '.' not in s:
|
||||
return 0
|
||||
return len(s.split('.')[1])
|
||||
|
||||
|
||||
def decimal_consistency_test(values):
|
||||
"""
|
||||
执行小数位一致性检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list
|
||||
待检测的数值列表（字符串形式保留原始精度）
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if len(values) < 5:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足（仅{len(values)}个值），需要至少5个数据点'
|
||||
}
|
||||
|
||||
# 分析1: 小数位数一致性
|
||||
decimal_places = [count_decimal_places(v) for v in values]
|
||||
places_counter = Counter(decimal_places)
|
||||
most_common_places = places_counter.most_common(1)[0]
|
||||
places_uniformity = most_common_places[1] / len(values)
|
||||
|
||||
# 分析2: 小数部分重复度
|
||||
decimal_strings = [get_decimal_string(v) for v in values if '.' in str(v)]
|
||||
if decimal_strings:
|
||||
decimal_counter = Counter(decimal_strings)
|
||||
n_unique_decimals = len(decimal_counter)
|
||||
most_repeated = decimal_counter.most_common(1)[0]
|
||||
max_repetition_rate = most_repeated[1] / len(decimal_strings)
|
||||
else:
|
||||
n_unique_decimals = 0
|
||||
max_repetition_rate = 0
|
||||
most_repeated = ('N/A', 0)
|
||||
|
||||
# 分析3: 各小数位数字分布
|
||||
position_analyses = {}
|
||||
max_positions = max(decimal_places) if decimal_places else 0
|
||||
|
||||
for pos in range(min(max_positions, 4)):
|
||||
digits_at_pos = []
|
||||
for v in values:
|
||||
decs = get_decimal_digits(v)
|
||||
if len(decs) > pos:
|
||||
digits_at_pos.append(decs[pos])
|
||||
|
||||
if len(digits_at_pos) >= 10:
|
||||
digit_counts = Counter(digits_at_pos)
|
||||
observed = np.array([digit_counts.get(i, 0) for i in range(10)])
|
||||
expected = np.full(10, len(digits_at_pos) / 10.0)
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
position_analyses[f'position_{pos+1}'] = {
|
||||
'n_values': len(digits_at_pos),
|
||||
'distribution': {str(i): int(observed[i]) for i in range(10)},
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'is_uniform': p_value > 0.05
|
||||
}
|
||||
|
||||
# 分析4: 相邻数据小数部分相关性
|
||||
if len(decimal_strings) >= 5:
|
||||
# 将小数部分转为数值进行自相关分析
|
||||
decimal_values = []
|
||||
for ds in decimal_strings:
|
||||
try:
|
||||
decimal_values.append(float('0.' + ds) if ds else 0.0)
|
||||
except ValueError:
|
||||
decimal_values.append(0.0)
|
||||
|
||||
if len(decimal_values) >= 5:
|
||||
# 计算一阶自相关
|
||||
x = np.array(decimal_values)
|
||||
x_centered = x - np.mean(x)
|
||||
if np.std(x) > 0:
|
||||
autocorr = np.correlate(x_centered[:-1], x_centered[1:]) / (len(x_centered) - 1) / np.var(x)
|
||||
autocorr_val = float(autocorr[0]) if len(autocorr) > 0 else 0
|
||||
else:
|
||||
autocorr_val = 1.0 # 完全一致
|
||||
else:
|
||||
autocorr_val = None
|
||||
else:
|
||||
autocorr_val = None
|
||||
|
||||
# 综合风险评分
|
||||
risk_factors = []
|
||||
|
||||
# 因子1: 小数位数过于一致
|
||||
if places_uniformity > 0.95 and len(values) > 10:
|
||||
risk_factors.append(('decimal_places_uniform', 20))
|
||||
|
||||
# 因子2: 小数部分重复度过高
|
||||
if max_repetition_rate > 0.5:
|
||||
risk_factors.append(('high_repetition', 30))
|
||||
elif max_repetition_rate > 0.3:
|
||||
risk_factors.append(('moderate_repetition', 15))
|
||||
|
||||
# 因子3: 某个位置数字分布异常
|
||||
for pos_key, pos_data in position_analyses.items():
|
||||
if pos_data['p_value'] < 0.001:
|
||||
risk_factors.append((f'{pos_key}_nonuniform', 25))
|
||||
elif pos_data['p_value'] < 0.01:
|
||||
risk_factors.append((f'{pos_key}_marginal', 10))
|
||||
|
||||
# 因子4: 自相关异常高
|
||||
if autocorr_val is not None and abs(autocorr_val) > 0.8:
|
||||
risk_factors.append(('high_autocorrelation', 20))
|
||||
|
||||
risk_score = min(100, sum(score for _, score in risk_factors))
|
||||
|
||||
if risk_score >= 70:
|
||||
risk_level = 'high'
|
||||
elif risk_score >= 45:
|
||||
risk_level = 'medium-high'
|
||||
elif risk_score >= 25:
|
||||
risk_level = 'medium'
|
||||
else:
|
||||
risk_level = 'low'
|
||||
|
||||
result = {
|
||||
'test_name': 'Decimal Consistency Test (小数位一致性检测)',
|
||||
'status': 'completed',
|
||||
'n_values': len(values),
|
||||
'decimal_places_analysis': {
|
||||
'distribution': dict(places_counter),
|
||||
'most_common_places': most_common_places[0],
|
||||
'uniformity_rate': round(float(places_uniformity), 4)
|
||||
},
|
||||
'decimal_repetition': {
|
||||
'n_unique_patterns': n_unique_decimals,
|
||||
'most_repeated_pattern': most_repeated[0],
|
||||
'most_repeated_count': most_repeated[1],
|
||||
'max_repetition_rate': round(float(max_repetition_rate), 4)
|
||||
},
|
||||
'position_digit_analysis': position_analyses,
|
||||
'autocorrelation': round(float(autocorr_val), 4) if autocorr_val is not None else None,
|
||||
'risk_factors': [f for f, _ in risk_factors],
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_decimal(risk_factors, places_uniformity, max_repetition_rate, most_repeated):
|
||||
"""生成可读的解释"""
|
||||
if not risk_factors:
|
||||
return "✅ 小数位分布未发现明显异常，数据的小数部分具有合理的随机性。"
|
||||
|
||||
issues = []
|
||||
for factor, _ in risk_factors:
|
||||
if 'repetition' in factor:
|
||||
issues.append(f"小数部分 '{most_repeated[0]}' 重复出现 {most_repeated[1]} 次（{max_repetition_rate:.0%}）")
|
||||
elif 'nonuniform' in factor:
|
||||
issues.append("某些小数位的数字分布严重偏离均匀分布")
|
||||
elif 'autocorrelation' in factor:
|
||||
issues.append("相邻数据的小数部分存在异常高的自相关")
|
||||
elif 'places_uniform' in factor:
|
||||
issues.append(f"所有数据小数位数高度一致（{places_uniformity:.0%}相同）")
|
||||
|
||||
issues_str = "；".join(issues)
|
||||
|
||||
if len(risk_factors) >= 3:
|
||||
return f"⚠️ 发现多项小数位异常：{issues_str}。这些模式在自然实验数据中非常罕见，强烈建议核查原始数据。"
|
||||
elif len(risk_factors) >= 2:
|
||||
return f"⚠️ 发现小数位可疑模式：{issues_str}。建议进一步检查。"
|
||||
else:
|
||||
return f"⚡ 发现轻微异常：{issues_str}。可能是测量精度限制导致，建议结合其他检测综合判断。"
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据（保留原始字符串精度）"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
try:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='小数位一致性检测 - 检测数据小数部分是否存在异常模式'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误：未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = decimal_consistency_test(values)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
312
tools/gengskill/scripts/fixed_relation_test.py
Normal file
312
tools/gengskill/scripts/fixed_relation_test.py
Normal file
@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
固定关系检测 (Fixed Relationship Detection)
|
||||
============================================
|
||||
原理：两组独立实验数据之间不应存在恒定的差值、比值或线性关系。
|
||||
如果不同实验条件下的数据存在固定的数学关系，暗示数据可能是
|
||||
从单一数据源通过简单数学运算生成的，而非独立实验获得。
|
||||
|
||||
这是"耿同学"打假方法中的核心策略之一——他发现许多造假论文中
|
||||
不同实验组的数据存在固定差值或固定比例关系。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def detect_fixed_difference(col1, col2, tolerance=0.01):
|
||||
"""检测两列数据是否存在固定差值"""
|
||||
differences = np.array(col2) - np.array(col1)
|
||||
|
||||
if len(differences) < 3:
|
||||
return None
|
||||
|
||||
# 计算差值的变异系数
|
||||
mean_diff = np.mean(differences)
|
||||
std_diff = np.std(differences)
|
||||
|
||||
if abs(mean_diff) < 1e-10:
|
||||
cv = float('inf') if std_diff > 0 else 0
|
||||
else:
|
||||
cv = abs(std_diff / mean_diff)
|
||||
|
||||
# 判断差值是否恒定（CV极小）
|
||||
is_fixed = cv < tolerance and std_diff < tolerance * abs(mean_diff + 1e-10)
|
||||
|
||||
# 检查差值是否完全相同
|
||||
unique_diffs = np.unique(np.round(differences, 6))
|
||||
is_exact = len(unique_diffs) == 1
|
||||
|
||||
return {
|
||||
'type': 'fixed_difference',
|
||||
'type_cn': '固定差值',
|
||||
'mean_difference': round(float(mean_diff), 6),
|
||||
'std_difference': round(float(std_diff), 6),
|
||||
'cv': round(float(cv), 6) if cv != float('inf') else 'inf',
|
||||
'is_fixed': bool(is_fixed or is_exact),
|
||||
'is_exact': bool(is_exact),
|
||||
'n_unique_differences': int(len(unique_diffs))
|
||||
}
|
||||
|
||||
|
||||
def detect_fixed_ratio(col1, col2, tolerance=0.01):
|
||||
"""检测两列数据是否存在固定比值"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
# 避免除以零
|
||||
mask = col1 != 0
|
||||
if np.sum(mask) < 3:
|
||||
return None
|
||||
|
||||
ratios = col2[mask] / col1[mask]
|
||||
|
||||
mean_ratio = np.mean(ratios)
|
||||
std_ratio = np.std(ratios)
|
||||
|
||||
if abs(mean_ratio) < 1e-10:
|
||||
cv = float('inf') if std_ratio > 0 else 0
|
||||
else:
|
||||
cv = abs(std_ratio / mean_ratio)
|
||||
|
||||
is_fixed = cv < tolerance
|
||||
unique_ratios = np.unique(np.round(ratios, 6))
|
||||
is_exact = len(unique_ratios) == 1
|
||||
|
||||
return {
|
||||
'type': 'fixed_ratio',
|
||||
'type_cn': '固定比值',
|
||||
'mean_ratio': round(float(mean_ratio), 6),
|
||||
'std_ratio': round(float(std_ratio), 6),
|
||||
'cv': round(float(cv), 6) if cv != float('inf') else 'inf',
|
||||
'is_fixed': bool(is_fixed or is_exact),
|
||||
'is_exact': bool(is_exact),
|
||||
'n_unique_ratios': int(len(unique_ratios))
|
||||
}
|
||||
|
||||
|
||||
def detect_linear_relationship(col1, col2):
|
||||
"""检测两列数据是否存在高度线性关系"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
if len(col1) < 3:
|
||||
return None
|
||||
|
||||
# 线性回归
|
||||
slope, intercept, r_value, p_value, std_err = stats.linregress(col1, col2)
|
||||
r_squared = r_value ** 2
|
||||
|
||||
# 残差分析
|
||||
predicted = slope * col1 + intercept
|
||||
residuals = col2 - predicted
|
||||
max_residual = np.max(np.abs(residuals))
|
||||
mean_residual = np.mean(np.abs(residuals))
|
||||
|
||||
# R² 非常接近1且残差极小
|
||||
is_suspicious = r_squared > 0.9999 and max_residual < 0.001 * np.std(col2)
|
||||
|
||||
return {
|
||||
'type': 'linear_relationship',
|
||||
'type_cn': '线性关系',
|
||||
'slope': round(float(slope), 6),
|
||||
'intercept': round(float(intercept), 6),
|
||||
'r_squared': round(float(r_squared), 8),
|
||||
'p_value': float(p_value),
|
||||
'max_residual': round(float(max_residual), 8),
|
||||
'mean_residual': round(float(mean_residual), 8),
|
||||
'is_suspicious': bool(is_suspicious)
|
||||
}
|
||||
|
||||
|
||||
def detect_decimal_pattern(col1, col2):
|
||||
"""检测两列数据小数部分是否高度一致"""
|
||||
col1 = np.array(col1, dtype=float)
|
||||
col2 = np.array(col2, dtype=float)
|
||||
|
||||
# 提取小数部分
|
||||
dec1 = col1 - np.floor(col1)
|
||||
dec2 = col2 - np.floor(col2)
|
||||
|
||||
# 检查小数部分是否一致
|
||||
dec_diff = np.abs(dec1 - dec2)
|
||||
n_matching = np.sum(dec_diff < 0.001)
|
||||
match_rate = n_matching / len(col1)
|
||||
|
||||
return {
|
||||
'type': 'decimal_pattern',
|
||||
'type_cn': '小数位一致性',
|
||||
'n_matching_decimals': int(n_matching),
|
||||
'match_rate': round(float(match_rate), 4),
|
||||
'is_suspicious': match_rate > 0.8
|
||||
}
|
||||
|
||||
|
||||
def fixed_relation_test(col1, col2, col1_name='Column A', col2_name='Column B'):
|
||||
"""
|
||||
综合固定关系检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
col1 : list of float
|
||||
第一列数据
|
||||
col2 : list of float
|
||||
第二列数据
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if len(col1) != len(col2):
|
||||
return {'status': 'error', 'message': '两列数据长度不一致'}
|
||||
|
||||
if len(col1) < 3:
|
||||
return {'status': 'insufficient_data', 'message': '数据量不足，至少需要3个数据点'}
|
||||
|
||||
col1 = [float(x) for x in col1]
|
||||
col2 = [float(x) for x in col2]
|
||||
|
||||
# 执行各项检测
|
||||
results = {}
|
||||
|
||||
diff_result = detect_fixed_difference(col1, col2)
|
||||
if diff_result:
|
||||
results['fixed_difference'] = diff_result
|
||||
|
||||
ratio_result = detect_fixed_ratio(col1, col2)
|
||||
if ratio_result:
|
||||
results['fixed_ratio'] = ratio_result
|
||||
|
||||
linear_result = detect_linear_relationship(col1, col2)
|
||||
if linear_result:
|
||||
results['linear_relationship'] = linear_result
|
||||
|
||||
decimal_result = detect_decimal_pattern(col1, col2)
|
||||
if decimal_result:
|
||||
results['decimal_pattern'] = decimal_result
|
||||
|
||||
# 综合风险评估
|
||||
n_suspicious = sum([
|
||||
1 for r in results.values()
|
||||
if r.get('is_fixed') or r.get('is_suspicious')
|
||||
])
|
||||
|
||||
if n_suspicious >= 3:
|
||||
risk_level = 'high'
|
||||
risk_score = 85
|
||||
elif n_suspicious == 2:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 65
|
||||
elif n_suspicious == 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 45
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = 10
|
||||
|
||||
# 如果存在完全精确的固定关系，直接拉高风险
|
||||
if any(r.get('is_exact') for r in results.values()):
|
||||
risk_level = 'high'
|
||||
risk_score = max(risk_score, 90)
|
||||
|
||||
summary = {
|
||||
'test_name': 'Fixed Relationship Detection (固定关系检测)',
|
||||
'status': 'completed',
|
||||
'n_data_points': len(col1),
|
||||
'column_1': col1_name,
|
||||
'column_2': col2_name,
|
||||
'detections': results,
|
||||
'n_suspicious_patterns': n_suspicious,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name)
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _interpret_fixed_relation(results, n_suspicious, col1_name, col2_name):
|
||||
"""生成可读的解释"""
|
||||
findings = []
|
||||
|
||||
if results.get('fixed_difference', {}).get('is_fixed'):
|
||||
d = results['fixed_difference']
|
||||
findings.append(f"两列数据存在固定差值 {d['mean_difference']}")
|
||||
|
||||
if results.get('fixed_ratio', {}).get('is_fixed'):
|
||||
r = results['fixed_ratio']
|
||||
findings.append(f"两列数据存在固定比值 {r['mean_ratio']}")
|
||||
|
||||
if results.get('linear_relationship', {}).get('is_suspicious'):
|
||||
l = results['linear_relationship']
|
||||
findings.append(f"两列数据存在完美线性关系 (R² = {l['r_squared']})")
|
||||
|
||||
if results.get('decimal_pattern', {}).get('is_suspicious'):
|
||||
p = results['decimal_pattern']
|
||||
findings.append(f"两列数据小数部分高度一致 (匹配率 {p['match_rate']:.0%})")
|
||||
|
||||
if not findings:
|
||||
return f"✅ {col1_name} 与 {col2_name} 之间未发现固定数学关系，数据看起来是独立的。"
|
||||
|
||||
findings_str = "；".join(findings)
|
||||
return (
|
||||
f"⚠️ {col1_name} 与 {col2_name} 之间发现以下可疑模式：{findings_str}。"
|
||||
f"独立实验数据通常不应存在如此精确的数学关系，建议核查数据是否来自独立实验。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, col1, col2, delimiter=','):
|
||||
"""从CSV文件加载两列数据"""
|
||||
import csv
|
||||
|
||||
data1, data2 = [], []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
for row in reader:
|
||||
try:
|
||||
v1 = float(row[col1].strip())
|
||||
v2 = float(row[col2].strip())
|
||||
data1.append(v1)
|
||||
data2.append(v2)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
return data1, data2
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='固定关系检测 - 检测两组数据间是否存在不自然的数学关系'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--col1', required=True, help='第一列列名')
|
||||
parser.add_argument('--col2', required=True, help='第二列列名')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
data1, data2 = load_data(args.input, args.col1, args.col2, args.delimiter)
|
||||
|
||||
if not data1:
|
||||
print("错误：未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = fixed_relation_test(data1, data2, args.col1, args.col2)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
463
tools/gengskill/scripts/geng_assess.py
Normal file
463
tools/gengskill/scripts/geng_assess.py
Normal file
@ -0,0 +1,463 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Geng 综合评估引擎 (Comprehensive Assessment Engine)
|
||||
=====================================================
|
||||
一键运行所有适用的检测模块，生成综合学术数据打假报告。
|
||||
|
||||
支持的领域：
|
||||
- biomedical: 生物医学（Western blot, 流式, 动物实验）
|
||||
- chemistry: 化学（光谱, 产率, 催化）
|
||||
- physics: 物理/材料（性能曲线, 电学/力学）
|
||||
- social_science: 社会科学（问卷, 量表）
|
||||
- clinical: 临床医学（生存数据, 临床指标）
|
||||
- general: 通用（不指定领域）
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
import csv
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# 导入各检测模块
|
||||
from last_digit_test import last_digit_test
|
||||
from benford_test import benford_test
|
||||
from decimal_consistency_test import decimal_consistency_test
|
||||
from fixed_relation_test import fixed_relation_test
|
||||
from grim_test import grim_test_batch
|
||||
|
||||
|
||||
def load_csv_data(input_file, delimiter=','):
|
||||
"""加载CSV数据，返回列名和数据"""
|
||||
columns = {}
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
fieldnames = reader.fieldnames
|
||||
for row in reader:
|
||||
for col in fieldnames:
|
||||
if col not in columns:
|
||||
columns[col] = []
|
||||
columns[col].append(row[col].strip() if row[col] else '')
|
||||
return columns, fieldnames
|
||||
|
||||
|
||||
def identify_numeric_columns(columns):
|
||||
"""识别数值列"""
|
||||
numeric_cols = {}
|
||||
for col_name, values in columns.items():
|
||||
numeric_values = []
|
||||
for v in values:
|
||||
try:
|
||||
if v:
|
||||
float(v)
|
||||
numeric_values.append(v)
|
||||
except ValueError:
|
||||
continue
|
||||
# 至少50%的值是数值
|
||||
if len(numeric_values) >= len(values) * 0.5 and len(numeric_values) >= 5:
|
||||
numeric_cols[col_name] = numeric_values
|
||||
return numeric_cols
|
||||
|
||||
|
||||
def run_assessment(input_file, domain='general', output_dir=None, delimiter=','):
|
||||
"""
|
||||
执行综合评估
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_file : str
|
||||
输入CSV文件
|
||||
domain : str
|
||||
研究领域
|
||||
output_dir : str
|
||||
输出目录
|
||||
delimiter : str
|
||||
CSV分隔符
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
# 加载数据
|
||||
columns, fieldnames = load_csv_data(input_file, delimiter)
|
||||
numeric_cols = identify_numeric_columns(columns)
|
||||
|
||||
if not numeric_cols:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '未找到有效的数值列，请检查输入文件格式'
|
||||
}
|
||||
|
||||
report = {
|
||||
'meta': {
|
||||
'tool': 'Geng Academic Data Fraud Detection Tool',
|
||||
'version': '1.0.0',
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'input_file': os.path.basename(input_file),
|
||||
'domain': domain,
|
||||
'n_columns': len(fieldnames),
|
||||
'n_numeric_columns': len(numeric_cols),
|
||||
'numeric_columns': list(numeric_cols.keys()),
|
||||
'n_rows': max(len(v) for v in columns.values()) if columns else 0
|
||||
},
|
||||
'module_results': {},
|
||||
'summary': {}
|
||||
}
|
||||
|
||||
all_risk_scores = []
|
||||
|
||||
# ==========================================
|
||||
# Module 1: 末位数字检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行末位数字检测...")
|
||||
last_digit_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
result = last_digit_test(values, method='all_digits')
|
||||
if result.get('status') == 'completed':
|
||||
last_digit_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
|
||||
if last_digit_results:
|
||||
report['module_results']['last_digit_test'] = {
|
||||
'module_name': '末位数字检测',
|
||||
'description': '检测数据末位数字是否偏离均匀分布',
|
||||
'columns_tested': len(last_digit_results),
|
||||
'results': last_digit_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 2: 本福特定律检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行本福特定律检测...")
|
||||
benford_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
# 本福特定律适用于跨多个数量级的数据
|
||||
try:
|
||||
float_values = [float(v) for v in values if v]
|
||||
value_range = max(float_values) / (min(v for v in float_values if v > 0) + 1e-10)
|
||||
# 只对跨度超过1个数量级的列做本福特检测
|
||||
if value_range > 10:
|
||||
result = benford_test(values, order=1)
|
||||
if result.get('status') == 'completed':
|
||||
benford_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
except (ValueError, ZeroDivisionError):
|
||||
continue
|
||||
|
||||
if benford_results:
|
||||
report['module_results']['benford_test'] = {
|
||||
'module_name': '本福特定律检测',
|
||||
'description': '检测首位数字是否符合Benford\'s Law',
|
||||
'columns_tested': len(benford_results),
|
||||
'results': benford_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 3: 小数位一致性检测 (对每个数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行小数位一致性检测...")
|
||||
decimal_results = {}
|
||||
for col_name, values in numeric_cols.items():
|
||||
# 只检测包含小数的列
|
||||
has_decimal = any('.' in v for v in values if v)
|
||||
if has_decimal:
|
||||
result = decimal_consistency_test(values)
|
||||
if result.get('status') == 'completed':
|
||||
decimal_results[col_name] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
|
||||
if decimal_results:
|
||||
report['module_results']['decimal_consistency_test'] = {
|
||||
'module_name': '小数位一致性检测',
|
||||
'description': '检测数据小数部分是否存在异常模式',
|
||||
'columns_tested': len(decimal_results),
|
||||
'results': decimal_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# Module 4: 固定关系检测 (两两比较数值列)
|
||||
# ==========================================
|
||||
print("🔍 执行固定关系检测...")
|
||||
fixed_results = {}
|
||||
col_names = list(numeric_cols.keys())
|
||||
|
||||
# 限制比较对数，避免组合爆炸
|
||||
max_pairs = min(10, len(col_names) * (len(col_names) - 1) // 2)
|
||||
pair_count = 0
|
||||
|
||||
for i in range(len(col_names)):
|
||||
if pair_count >= max_pairs:
|
||||
break
|
||||
for j in range(i + 1, len(col_names)):
|
||||
if pair_count >= max_pairs:
|
||||
break
|
||||
col1_name = col_names[i]
|
||||
col2_name = col_names[j]
|
||||
|
||||
# 确保两列长度一致且有足够数据
|
||||
vals1 = numeric_cols[col1_name]
|
||||
vals2 = numeric_cols[col2_name]
|
||||
|
||||
# 配对：只取两列都有值的行
|
||||
paired_1, paired_2 = [], []
|
||||
for v1, v2 in zip(vals1, vals2):
|
||||
try:
|
||||
if v1 and v2:
|
||||
float(v1)
|
||||
float(v2)
|
||||
paired_1.append(v1)
|
||||
paired_2.append(v2)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if len(paired_1) >= 5:
|
||||
result = fixed_relation_test(paired_1, paired_2, col1_name, col2_name)
|
||||
if result.get('status') == 'completed':
|
||||
pair_key = f"{col1_name} vs {col2_name}"
|
||||
fixed_results[pair_key] = result
|
||||
all_risk_scores.append(result.get('risk_score', 0))
|
||||
pair_count += 1
|
||||
|
||||
if fixed_results:
|
||||
report['module_results']['fixed_relation_test'] = {
|
||||
'module_name': '固定关系检测',
|
||||
'description': '检测不同列数据间是否存在不自然的数学关系',
|
||||
'pairs_tested': len(fixed_results),
|
||||
'results': fixed_results
|
||||
}
|
||||
|
||||
# ==========================================
|
||||
# 综合评分
|
||||
# ==========================================
|
||||
print("📊 生成综合评估...")
|
||||
|
||||
if all_risk_scores:
|
||||
# 综合评分：取各模块最高分的加权平均
|
||||
max_score = max(all_risk_scores)
|
||||
mean_score = sum(all_risk_scores) / len(all_risk_scores)
|
||||
# 综合分 = 60% 最高分 + 40% 平均分
|
||||
overall_score = 0.6 * max_score + 0.4 * mean_score
|
||||
overall_score = min(100, overall_score)
|
||||
else:
|
||||
overall_score = 0
|
||||
|
||||
# 统计各风险等级
|
||||
high_risk_modules = [s for s in all_risk_scores if s >= 70]
|
||||
medium_risk_modules = [s for s in all_risk_scores if 40 <= s < 70]
|
||||
low_risk_modules = [s for s in all_risk_scores if s < 40]
|
||||
|
||||
if overall_score >= 75:
|
||||
overall_level = 'critical'
|
||||
overall_level_cn = '🔴 极高风险'
|
||||
overall_emoji = '🔴'
|
||||
elif overall_score >= 50:
|
||||
overall_level = 'high'
|
||||
overall_level_cn = '🟠 高风险'
|
||||
overall_emoji = '🟠'
|
||||
elif overall_score >= 25:
|
||||
overall_level = 'medium'
|
||||
overall_level_cn = '🟡 中风险'
|
||||
overall_emoji = '🟡'
|
||||
else:
|
||||
overall_level = 'low'
|
||||
overall_level_cn = '🟢 低风险'
|
||||
overall_emoji = '🟢'
|
||||
|
||||
report['summary'] = {
|
||||
'overall_risk_score': round(float(overall_score), 1),
|
||||
'overall_risk_level': overall_level,
|
||||
'overall_risk_level_cn': overall_level_cn,
|
||||
'n_modules_run': len(report['module_results']),
|
||||
'n_tests_total': len(all_risk_scores),
|
||||
'n_high_risk': len(high_risk_modules),
|
||||
'n_medium_risk': len(medium_risk_modules),
|
||||
'n_low_risk': len(low_risk_modules),
|
||||
'execution_time_seconds': round(time.time() - start_time, 2),
|
||||
'conclusion': _generate_conclusion(overall_score, overall_level, report['module_results']),
|
||||
'recommendations': _generate_recommendations(overall_level, domain, report['module_results'])
|
||||
}
|
||||
|
||||
# 保存报告
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
report_path = os.path.join(output_dir, 'geng_assessment_report.json')
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 生成可读的 Markdown 报告
|
||||
md_path = os.path.join(output_dir, 'geng_assessment_report.md')
|
||||
with open(md_path, 'w', encoding='utf-8') as f:
|
||||
f.write(_generate_markdown_report(report))
|
||||
|
||||
print(f"\n📄 JSON报告已保存至: {report_path}")
|
||||
print(f"📄 Markdown报告已保存至: {md_path}")
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _generate_conclusion(score, level, modules):
|
||||
"""生成结论"""
|
||||
if level == 'critical':
|
||||
return (
|
||||
"⚠️ 综合评估显示数据存在系统性异常，多项检测指标显著偏离正常预期。"
|
||||
"强烈建议对原始实验数据进行全面核查。"
|
||||
"注意：本工具仅提供线索筛查，最终判定需要领域专家复核。"
|
||||
)
|
||||
elif level == 'high':
|
||||
return (
|
||||
"⚠️ 数据中发现多处可疑模式，部分指标显著异常。"
|
||||
"建议对标记为高风险的数据列进行重点核查。"
|
||||
)
|
||||
elif level == 'medium':
|
||||
return (
|
||||
"⚡ 数据存在一些轻微异常，但不足以构成造假的确证。"
|
||||
"可能是测量精度限制、数据处理方式等正常因素导致。"
|
||||
"建议结合论文方法学描述进行综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
"✅ 数据各项检测指标均在正常范围内，未发现明显的造假迹象。"
|
||||
"注意：通过检测不代表数据一定真实，某些高明的造假可能无法被统计方法捕获。"
|
||||
)
|
||||
|
||||
|
||||
def _generate_recommendations(level, domain, modules):
|
||||
"""生成建议"""
|
||||
recs = []
|
||||
|
||||
if level in ('critical', 'high'):
|
||||
recs.append("核查原始实验记录和数据记录本")
|
||||
recs.append("验证数据是否来自独立实验")
|
||||
recs.append("联系通讯作者要求提供原始数据")
|
||||
if domain == 'biomedical':
|
||||
recs.append("检查Western blot原始图片和流式原始FCS文件")
|
||||
recs.append("考虑向期刊或机构提交正式质疑")
|
||||
elif level == 'medium':
|
||||
recs.append("仔细阅读论文方法学部分，确认数据采集方式")
|
||||
recs.append("检查是否存在合理的解释（如仪器精度限制）")
|
||||
recs.append("可考虑联系作者进行非正式沟通")
|
||||
else:
|
||||
recs.append("当前数据未发现明显异常")
|
||||
recs.append("可考虑对补充材料中的数据做进一步检测")
|
||||
|
||||
return recs
|
||||
|
||||
|
||||
def _generate_markdown_report(report):
|
||||
"""生成Markdown格式报告"""
|
||||
meta = report['meta']
|
||||
summary = report['summary']
|
||||
|
||||
md = []
|
||||
md.append("# 📋 Geng 学术数据打假检测报告\n")
|
||||
md.append(f"> 生成时间: {meta['timestamp']}")
|
||||
md.append(f"> 检测工具: {meta['tool']} v{meta['version']}")
|
||||
md.append('> 致敬"耿同学讲故事"\n')
|
||||
|
||||
md.append("## 📊 综合评估结果\n")
|
||||
md.append(f"| 指标 | 结果 |")
|
||||
md.append(f"|------|------|")
|
||||
md.append(f"| **综合风险评分** | **{summary['overall_risk_score']}/100** |")
|
||||
md.append(f"| **风险等级** | {summary['overall_risk_level_cn']} |")
|
||||
md.append(f"| 输入文件 | {meta['input_file']} |")
|
||||
md.append(f"| 检测领域 | {meta['domain']} |")
|
||||
md.append(f"| 数值列数 | {meta['n_numeric_columns']} |")
|
||||
md.append(f"| 数据行数 | {meta['n_rows']} |")
|
||||
md.append(f"| 运行模块数 | {summary['n_modules_run']} |")
|
||||
md.append(f"| 检测总数 | {summary['n_tests_total']} |")
|
||||
md.append(f"| 高风险项 | {summary['n_high_risk']} |")
|
||||
md.append(f"| 执行耗时 | {summary['execution_time_seconds']}s |\n")
|
||||
|
||||
md.append("## 📝 结论\n")
|
||||
md.append(f"{summary['conclusion']}\n")
|
||||
|
||||
md.append("## 💡 建议\n")
|
||||
for i, rec in enumerate(summary['recommendations'], 1):
|
||||
md.append(f"{i}. {rec}")
|
||||
md.append("")
|
||||
|
||||
md.append("## 🔬 各模块检测详情\n")
|
||||
|
||||
for module_key, module_data in report['module_results'].items():
|
||||
md.append(f"### {module_data['module_name']}\n")
|
||||
md.append(f"_{module_data['description']}_\n")
|
||||
|
||||
if 'columns_tested' in module_data:
|
||||
md.append(f"- 检测列数: {module_data['columns_tested']}")
|
||||
if 'pairs_tested' in module_data:
|
||||
md.append(f"- 检测对数: {module_data['pairs_tested']}")
|
||||
|
||||
# 列出各列/对的风险评分
|
||||
results = module_data.get('results', {})
|
||||
if results:
|
||||
md.append(f"\n| 检测对象 | 风险评分 | 风险等级 | 说明 |")
|
||||
md.append(f"|----------|----------|----------|------|")
|
||||
for key, res in results.items():
|
||||
score = res.get('risk_score', 'N/A')
|
||||
level = res.get('risk_level', 'N/A')
|
||||
interp = res.get('interpretation', '')[:60]
|
||||
md.append(f"| {key} | {score} | {level} | {interp}... |")
|
||||
md.append("")
|
||||
|
||||
md.append("---\n")
|
||||
md.append("## ⚠️ 重要声明\n")
|
||||
md.append("- 本工具仅用于辅助筛查，**不能作为造假的最终判定依据**")
|
||||
md.append("- 数据异常 ≠ 数据造假（可能是仪器校准、单位转换、排版错误等）")
|
||||
md.append("- 检测结果需要领域专家复核")
|
||||
md.append("- 使用本工具时应遵守学术伦理和法律法规\n")
|
||||
md.append("---\n")
|
||||
md.append('*Powered by Geng Skill — 致敬"耿同学讲故事"*')
|
||||
|
||||
return "\n".join(md)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Geng 综合评估引擎 - 一键运行所有检测模块'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--domain', default='general',
|
||||
choices=['biomedical', 'chemistry', 'physics',
|
||||
'social_science', 'clinical', 'general'],
|
||||
help='研究领域')
|
||||
parser.add_argument('--output', '-o', default='./report', help='输出目录')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isfile(args.input):
|
||||
print(f"错误：文件不存在: {args.input}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"{'='*60}")
|
||||
print(f" Geng 学术数据打假检测工具 v1.0.0")
|
||||
print(' 致敬"耿同学讲故事" — 用数据说话，让造假无所遁形')
|
||||
print(f"{'='*60}")
|
||||
print(f"\n📁 输入文件: {args.input}")
|
||||
print(f"🔬 检测领域: {args.domain}")
|
||||
print(f"📂 输出目录: {args.output}\n")
|
||||
|
||||
report = run_assessment(args.input, args.domain, args.output, args.delimiter)
|
||||
|
||||
if report.get('status') == 'error':
|
||||
print(f"\n❌ 错误: {report['message']}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
summary = report['summary']
|
||||
print(f"\n{'='*60}")
|
||||
print(f" 综合评估结果")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n {summary['overall_risk_level_cn']}")
|
||||
print(f" 综合风险评分: {summary['overall_risk_score']}/100")
|
||||
print(f" 高风险项: {summary['n_high_risk']} | "
|
||||
f"中风险项: {summary['n_medium_risk']} | "
|
||||
f"低风险项: {summary['n_low_risk']}")
|
||||
print(f"\n {summary['conclusion']}")
|
||||
print(f"\n{'='*60}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
244
tools/gengskill/scripts/grim_test.py
Normal file
244
tools/gengskill/scripts/grim_test.py
Normal file
@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GRIM 测试 (Granularity-Related Inconsistency of Means)
|
||||
========================================================
|
||||
原理：对于整数取值的数据（如李克特量表1-5、年龄等），
|
||||
给定样本量 n，合法的平均值只能取有限集合中的值。
|
||||
如果报告的平均值不在合法集合中，则数据存在不一致性。
|
||||
|
||||
例：n=25，数据取值为整数，则平均值只能是 k/25 的形式，
|
||||
小数部分只能是 .00, .04, .08, .12, ..., .96
|
||||
|
||||
参考：Brown & Heathers (2017). The GRIM Test. SPPS.
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
from decimal import Decimal, ROUND_HALF_UP
|
||||
|
||||
|
||||
def grim_test_single(mean, n, decimals=2, scale_min=None, scale_max=None):
|
||||
"""
|
||||
对单个均值执行 GRIM 测试
|
||||
|
||||
Parameters
|
||||
----------
|
||||
mean : float or str
|
||||
报告的平均值
|
||||
n : int
|
||||
样本量
|
||||
decimals : int
|
||||
报告的小数位数
|
||||
scale_min : int, optional
|
||||
量表最小值（用于范围检查）
|
||||
scale_max : int, optional
|
||||
量表最大值（用于范围检查）
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
try:
|
||||
mean_val = Decimal(str(mean))
|
||||
n = int(n)
|
||||
except (ValueError, TypeError) as e:
|
||||
return {'status': 'error', 'message': f'无效输入: {e}'}
|
||||
|
||||
if n <= 0:
|
||||
return {'status': 'error', 'message': '样本量必须大于0'}
|
||||
|
||||
# 范围检查
|
||||
if scale_min is not None and scale_max is not None:
|
||||
if float(mean_val) < scale_min or float(mean_val) > scale_max:
|
||||
return {
|
||||
'status': 'range_error',
|
||||
'message': f'均值 {mean} 超出量表范围 [{scale_min}, {scale_max}]',
|
||||
'consistent': False
|
||||
}
|
||||
|
||||
# 计算总和 = mean * n
|
||||
total = mean_val * n
|
||||
|
||||
# 对于整数取值数据，总和必须是整数
|
||||
# 考虑四舍五入误差：检查 total 是否足够接近某个整数
|
||||
granularity = Decimal(1) / Decimal(10 ** decimals)
|
||||
|
||||
# 在四舍五入精度范围内检查
|
||||
# mean 可能是真实值四舍五入到 decimals 位的结果
|
||||
# 真实 mean 在 [mean - 0.5*granularity, mean + 0.5*granularity) 范围内
|
||||
lower_total = (mean_val - granularity / 2) * n
|
||||
upper_total = (mean_val + granularity / 2) * n
|
||||
|
||||
# 检查这个范围内是否包含整数
|
||||
lower_int = math.ceil(float(lower_total))
|
||||
upper_int = math.floor(float(upper_total))
|
||||
|
||||
consistent = lower_int <= upper_int
|
||||
|
||||
# 计算最近的合法均值
|
||||
nearest_total = round(float(mean_val * n))
|
||||
nearest_mean = nearest_total / n
|
||||
|
||||
# 格式化到指定小数位
|
||||
fmt = f"%.{decimals}f"
|
||||
nearest_mean_str = fmt % nearest_mean
|
||||
reported_mean_str = fmt % float(mean_val)
|
||||
|
||||
result = {
|
||||
'reported_mean': str(mean),
|
||||
'sample_size': n,
|
||||
'decimals': decimals,
|
||||
'consistent': consistent,
|
||||
'computed_sum': float(mean_val * n),
|
||||
'nearest_valid_mean': nearest_mean_str,
|
||||
'difference': round(abs(float(mean_val) - nearest_mean), decimals + 2)
|
||||
}
|
||||
|
||||
if scale_min is not None and scale_max is not None:
|
||||
result['scale_range'] = f"[{scale_min}, {scale_max}]"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def grim_test_batch(items):
|
||||
"""
|
||||
批量 GRIM 测试
|
||||
|
||||
Parameters
|
||||
----------
|
||||
items : list of dict
|
||||
每个字典包含 'mean', 'n', 可选 'decimals', 'label'
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 批量检测结果
|
||||
"""
|
||||
results = []
|
||||
n_inconsistent = 0
|
||||
|
||||
for i, item in enumerate(items):
|
||||
mean = item.get('mean')
|
||||
n = item.get('n')
|
||||
decimals = item.get('decimals', 2)
|
||||
label = item.get('label', f'Item {i+1}')
|
||||
scale_min = item.get('scale_min')
|
||||
scale_max = item.get('scale_max')
|
||||
|
||||
res = grim_test_single(mean, n, decimals, scale_min, scale_max)
|
||||
res['label'] = label
|
||||
results.append(res)
|
||||
|
||||
if not res.get('consistent', True):
|
||||
n_inconsistent += 1
|
||||
|
||||
# 整体评估
|
||||
total = len(results)
|
||||
inconsistency_rate = n_inconsistent / total if total > 0 else 0
|
||||
|
||||
# 风险评分
|
||||
if inconsistency_rate > 0.5:
|
||||
risk_level = 'high'
|
||||
risk_score = 70 + inconsistency_rate * 30
|
||||
elif inconsistency_rate > 0.25:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 50 + inconsistency_rate * 40
|
||||
elif inconsistency_rate > 0:
|
||||
risk_level = 'medium'
|
||||
risk_score = 30 + inconsistency_rate * 40
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
|
||||
summary = {
|
||||
'test_name': 'GRIM Test (均值粒度一致性检验)',
|
||||
'status': 'completed',
|
||||
'total_items': total,
|
||||
'inconsistent_items': n_inconsistent,
|
||||
'inconsistency_rate': round(inconsistency_rate, 4),
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'details': results,
|
||||
'interpretation': _interpret_grim(n_inconsistent, total, inconsistency_rate)
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _interpret_grim(n_inconsistent, total, rate):
|
||||
"""生成可读的解释"""
|
||||
if n_inconsistent == 0:
|
||||
return f"✅ 全部 {total} 个均值通过 GRIM 检验，未发现数值不一致。"
|
||||
elif rate > 0.5:
|
||||
return (
|
||||
f"⚠️ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。"
|
||||
f"超过半数均值与样本量不兼容，这是严重的数据不一致信号。"
|
||||
f"强烈建议核查原始数据。"
|
||||
)
|
||||
elif rate > 0.25:
|
||||
return (
|
||||
f"⚠️ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。"
|
||||
f"建议仔细核查这些不一致的数据点。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ {total} 个均值中有 {n_inconsistent} 个（{rate:.0%}）未通过 GRIM 检验。"
|
||||
f"少量不一致可能是四舍五入方式不同导致，建议结合其他检测综合判断。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='GRIM测试 - 检测报告均值与样本量的一致性'
|
||||
)
|
||||
parser.add_argument('--mean', type=str, help='报告的平均值')
|
||||
parser.add_argument('--n', type=int, help='样本量')
|
||||
parser.add_argument('--decimals', type=int, default=2, help='小数位数')
|
||||
parser.add_argument('--scale', type=str, help='量表范围，如 "1-5"')
|
||||
parser.add_argument('--input', '-i', help='输入JSON文件（批量测试）')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
scale_min, scale_max = None, None
|
||||
if args.scale:
|
||||
parts = args.scale.split('-')
|
||||
if len(parts) == 2:
|
||||
scale_min, scale_max = int(parts[0]), int(parts[1])
|
||||
|
||||
if args.input:
|
||||
# 批量模式
|
||||
with open(args.input, 'r', encoding='utf-8') as f:
|
||||
items = json.load(f)
|
||||
result = grim_test_batch(items)
|
||||
elif args.mean and args.n:
|
||||
# 单项模式
|
||||
res = grim_test_single(args.mean, args.n, args.decimals, scale_min, scale_max)
|
||||
result = {
|
||||
'test_name': 'GRIM Test (均值粒度一致性检验)',
|
||||
'status': 'completed',
|
||||
'result': res,
|
||||
'interpretation': (
|
||||
f"✅ 均值 {args.mean} 与样本量 {args.n} 一致" if res.get('consistent')
|
||||
else f"⚠️ 均值 {args.mean} 与样本量 {args.n} 不一致！最近合法均值为 {res.get('nearest_valid_mean')}"
|
||||
)
|
||||
}
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
289
tools/gengskill/scripts/image_duplicate_test.py
Normal file
289
tools/gengskill/scripts/image_duplicate_test.py
Normal file
@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
图像重复检测 (Image Duplication Detection)
|
||||
============================================
|
||||
原理：检测论文图片中是否存在重复使用或篡改的图像。
|
||||
使用感知哈希(pHash)和结构相似性(SSIM)来识别：
|
||||
1. 完全相同的图片出现在不同实验条件下
|
||||
2. 经过旋转、翻转、裁剪后重复使用的图片
|
||||
3. 调整亮度/对比度后复用的图片
|
||||
|
||||
这是学术造假中常见的手段，尤其在Western blot、
|
||||
显微镜图片、流式细胞术散点图等场景中。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
HAS_PILLOW = True
|
||||
except ImportError:
|
||||
HAS_PILLOW = False
|
||||
|
||||
try:
|
||||
from skimage.metrics import structural_similarity as ssim
|
||||
HAS_SKIMAGE = True
|
||||
except ImportError:
|
||||
HAS_SKIMAGE = False
|
||||
|
||||
|
||||
def average_hash(image, hash_size=16):
|
||||
"""计算平均感知哈希"""
|
||||
img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
mean = pixels.mean()
|
||||
return (pixels > mean).flatten()
|
||||
|
||||
|
||||
def difference_hash(image, hash_size=16):
|
||||
"""计算差异感知哈希"""
|
||||
img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
return (pixels[:, 1:] > pixels[:, :-1]).flatten()
|
||||
|
||||
|
||||
def hamming_distance(hash1, hash2):
|
||||
"""计算汉明距离（归一化到0-1）"""
|
||||
return np.sum(hash1 != hash2) / len(hash1)
|
||||
|
||||
|
||||
def compute_ssim(img1, img2, target_size=(256, 256)):
|
||||
"""计算结构相似性指数"""
|
||||
if not HAS_SKIMAGE:
|
||||
return None
|
||||
|
||||
# 统一尺寸
|
||||
img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS)
|
||||
img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS)
|
||||
|
||||
arr1 = np.array(img1_resized)
|
||||
arr2 = np.array(img2_resized)
|
||||
|
||||
score = ssim(arr1, arr2)
|
||||
return float(score)
|
||||
|
||||
|
||||
def check_rotations(img1, img2, threshold=0.85):
|
||||
"""检查图像经过旋转/翻转后是否匹配"""
|
||||
transformations = [
|
||||
('original', lambda x: x),
|
||||
('rotate_90', lambda x: x.rotate(90, expand=True)),
|
||||
('rotate_180', lambda x: x.rotate(180, expand=True)),
|
||||
('rotate_270', lambda x: x.rotate(270, expand=True)),
|
||||
('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)),
|
||||
('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)),
|
||||
]
|
||||
|
||||
best_match = None
|
||||
best_score = 0
|
||||
|
||||
hash1 = average_hash(img1)
|
||||
|
||||
for name, transform in transformations:
|
||||
transformed = transform(img2)
|
||||
hash2 = average_hash(transformed)
|
||||
similarity = 1 - hamming_distance(hash1, hash2)
|
||||
|
||||
if similarity > best_score:
|
||||
best_score = similarity
|
||||
best_match = name
|
||||
|
||||
return {
|
||||
'best_transformation': best_match,
|
||||
'best_similarity': round(best_score, 4),
|
||||
'is_match': best_score >= threshold
|
||||
}
|
||||
|
||||
|
||||
def find_duplicates(image_dir, threshold=0.85, extensions=None):
|
||||
"""
|
||||
在目录中查找重复或相似的图片
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image_dir : str
|
||||
图片目录路径
|
||||
threshold : float
|
||||
相似度阈值（0-1），超过此值判定为重复
|
||||
extensions : list
|
||||
支持的图片格式
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if not HAS_PILLOW:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '需要安装 Pillow: pip install Pillow'
|
||||
}
|
||||
|
||||
if extensions is None:
|
||||
extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif']
|
||||
|
||||
# 收集所有图片文件
|
||||
image_files = []
|
||||
for ext in extensions:
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext}'))
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext.upper()}'))
|
||||
|
||||
image_files = sorted(set(image_files))
|
||||
|
||||
if len(image_files) < 2:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'目录中仅找到 {len(image_files)} 张图片，需要至少2张进行比较',
|
||||
'n_images': len(image_files)
|
||||
}
|
||||
|
||||
# 计算所有图片的哈希
|
||||
hashes = {}
|
||||
for img_path in image_files:
|
||||
try:
|
||||
img = Image.open(img_path)
|
||||
hashes[str(img_path)] = {
|
||||
'avg_hash': average_hash(img),
|
||||
'diff_hash': difference_hash(img),
|
||||
'size': img.size,
|
||||
'image': img
|
||||
}
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
# 两两比较
|
||||
duplicates = []
|
||||
paths = list(hashes.keys())
|
||||
|
||||
for i in range(len(paths)):
|
||||
for j in range(i + 1, len(paths)):
|
||||
path1, path2 = paths[i], paths[j]
|
||||
h1, h2 = hashes[path1], hashes[path2]
|
||||
|
||||
# 平均哈希相似度
|
||||
avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash'])
|
||||
|
||||
# 差异哈希相似度
|
||||
diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash'])
|
||||
|
||||
# 综合相似度
|
||||
combined_sim = max(avg_sim, diff_sim)
|
||||
|
||||
if combined_sim >= threshold:
|
||||
pair_result = {
|
||||
'file_1': os.path.basename(path1),
|
||||
'file_2': os.path.basename(path2),
|
||||
'avg_hash_similarity': round(float(avg_sim), 4),
|
||||
'diff_hash_similarity': round(float(diff_sim), 4),
|
||||
'combined_similarity': round(float(combined_sim), 4),
|
||||
}
|
||||
|
||||
# 检查旋转/翻转匹配
|
||||
rotation_check = check_rotations(
|
||||
h1['image'], h2['image'], threshold
|
||||
)
|
||||
pair_result['rotation_check'] = rotation_check
|
||||
|
||||
# SSIM（如果可用）
|
||||
if HAS_SKIMAGE:
|
||||
ssim_score = compute_ssim(h1['image'], h2['image'])
|
||||
pair_result['ssim'] = round(ssim_score, 4)
|
||||
|
||||
duplicates.append(pair_result)
|
||||
|
||||
# 关闭所有图片
|
||||
for h in hashes.values():
|
||||
h['image'].close()
|
||||
|
||||
# 风险评分
|
||||
n_duplicates = len(duplicates)
|
||||
n_images = len(image_files)
|
||||
|
||||
if n_duplicates == 0:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
elif n_duplicates <= 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40
|
||||
elif n_duplicates <= 3:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60
|
||||
else:
|
||||
risk_level = 'high'
|
||||
risk_score = 80 + min(20, n_duplicates * 3)
|
||||
|
||||
# 如果有完美匹配（相似度>0.98），直接拉高
|
||||
perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
if perfect_matches:
|
||||
risk_score = max(risk_score, 90)
|
||||
risk_level = 'high'
|
||||
|
||||
result = {
|
||||
'test_name': 'Image Duplication Detection (图像重复检测)',
|
||||
'status': 'completed',
|
||||
'n_images_scanned': n_images,
|
||||
'n_duplicate_pairs': n_duplicates,
|
||||
'threshold': threshold,
|
||||
'duplicates': duplicates,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_image(n_duplicates, n_images, duplicates)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_image(n_duplicates, n_images, duplicates):
|
||||
"""生成可读的解释"""
|
||||
if n_duplicates == 0:
|
||||
return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。"
|
||||
|
||||
perfect = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
|
||||
if perfect:
|
||||
return (
|
||||
f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片！"
|
||||
f"这些图片可能是同一图片的重复使用，强烈建议核查是否为不同实验条件下的独立数据。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ 发现 {n_duplicates} 对高度相似的图片（共扫描 {n_images} 张）。"
|
||||
f"可能存在图片复用或篡改，建议人工核查具体图片内容。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='图像重复检测 - 检测论文图片是否存在重复使用或篡改'
|
||||
)
|
||||
parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.85,
|
||||
help='相似度阈值（0-1），默认0.85')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.input_dir):
|
||||
print(f"错误：目录不存在: {args.input_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = find_duplicates(args.input_dir, args.threshold)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
301
tools/gengskill/scripts/image_duplicate_test_patched.py
Normal file
301
tools/gengskill/scripts/image_duplicate_test_patched.py
Normal file
@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
图像重复检测 (Image Duplication Detection)
|
||||
============================================
|
||||
原理：检测论文图片中是否存在重复使用或篡改的图像。
|
||||
使用感知哈希(pHash)和结构相似性(SSIM)来识别：
|
||||
1. 完全相同的图片出现在不同实验条件下
|
||||
2. 经过旋转、翻转、裁剪后重复使用的图片
|
||||
3. 调整亮度/对比度后复用的图片
|
||||
|
||||
这是学术造假中常见的手段，尤其在Western blot、
|
||||
显微镜图片、流式细胞术散点图等场景中。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
|
||||
class NumpyEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
import numpy as np
|
||||
if isinstance(obj, (np.integer,)): return int(obj)
|
||||
if isinstance(obj, (np.floating,)): return float(obj)
|
||||
if isinstance(obj, (np.bool_,)): return bool(obj)
|
||||
if isinstance(obj, np.ndarray): return obj.tolist()
|
||||
if obj is None: return None
|
||||
return super().default(obj)
|
||||
|
||||
HAS_PILLOW = True
|
||||
except ImportError:
|
||||
HAS_PILLOW = False
|
||||
|
||||
try:
|
||||
from skimage.metrics import structural_similarity as ssim
|
||||
HAS_SKIMAGE = True
|
||||
except ImportError:
|
||||
HAS_SKIMAGE = False
|
||||
|
||||
|
||||
def average_hash(image, hash_size=16):
|
||||
"""计算平均感知哈希"""
|
||||
img = image.convert('L').resize((hash_size, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
mean = pixels.mean()
|
||||
return (pixels > mean).flatten()
|
||||
|
||||
|
||||
def difference_hash(image, hash_size=16):
|
||||
"""计算差异感知哈希"""
|
||||
img = image.convert('L').resize((hash_size + 1, hash_size), Image.LANCZOS)
|
||||
pixels = np.array(img)
|
||||
return (pixels[:, 1:] > pixels[:, :-1]).flatten()
|
||||
|
||||
|
||||
def hamming_distance(hash1, hash2):
|
||||
"""计算汉明距离（归一化到0-1）"""
|
||||
return np.sum(hash1 != hash2) / len(hash1)
|
||||
|
||||
|
||||
def compute_ssim(img1, img2, target_size=(256, 256)):
|
||||
"""计算结构相似性指数"""
|
||||
if not HAS_SKIMAGE:
|
||||
return None
|
||||
|
||||
# 统一尺寸
|
||||
img1_resized = img1.convert('L').resize(target_size, Image.LANCZOS)
|
||||
img2_resized = img2.convert('L').resize(target_size, Image.LANCZOS)
|
||||
|
||||
arr1 = np.array(img1_resized)
|
||||
arr2 = np.array(img2_resized)
|
||||
|
||||
score = ssim(arr1, arr2)
|
||||
return float(score)
|
||||
|
||||
|
||||
def check_rotations(img1, img2, threshold=0.85):
|
||||
"""检查图像经过旋转/翻转后是否匹配"""
|
||||
transformations = [
|
||||
('original', lambda x: x),
|
||||
('rotate_90', lambda x: x.rotate(90, expand=True)),
|
||||
('rotate_180', lambda x: x.rotate(180, expand=True)),
|
||||
('rotate_270', lambda x: x.rotate(270, expand=True)),
|
||||
('flip_horizontal', lambda x: x.transpose(Image.FLIP_LEFT_RIGHT)),
|
||||
('flip_vertical', lambda x: x.transpose(Image.FLIP_TOP_BOTTOM)),
|
||||
]
|
||||
|
||||
best_match = None
|
||||
best_score = 0
|
||||
|
||||
hash1 = average_hash(img1)
|
||||
|
||||
for name, transform in transformations:
|
||||
transformed = transform(img2)
|
||||
hash2 = average_hash(transformed)
|
||||
similarity = 1 - hamming_distance(hash1, hash2)
|
||||
|
||||
if similarity > best_score:
|
||||
best_score = similarity
|
||||
best_match = name
|
||||
|
||||
return {
|
||||
'best_transformation': best_match,
|
||||
'best_similarity': round(best_score, 4),
|
||||
'is_match': best_score >= threshold
|
||||
}
|
||||
|
||||
|
||||
def find_duplicates(image_dir, threshold=0.85, extensions=None):
|
||||
"""
|
||||
在目录中查找重复或相似的图片
|
||||
|
||||
Parameters
|
||||
----------
|
||||
image_dir : str
|
||||
图片目录路径
|
||||
threshold : float
|
||||
相似度阈值（0-1），超过此值判定为重复
|
||||
extensions : list
|
||||
支持的图片格式
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
if not HAS_PILLOW:
|
||||
return {
|
||||
'status': 'error',
|
||||
'message': '需要安装 Pillow: pip install Pillow'
|
||||
}
|
||||
|
||||
if extensions is None:
|
||||
extensions = ['.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif']
|
||||
|
||||
# 收集所有图片文件
|
||||
image_files = []
|
||||
for ext in extensions:
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext}'))
|
||||
image_files.extend(Path(image_dir).glob(f'*{ext.upper()}'))
|
||||
|
||||
image_files = sorted(set(image_files))
|
||||
|
||||
if len(image_files) < 2:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'目录中仅找到 {len(image_files)} 张图片，需要至少2张进行比较',
|
||||
'n_images': len(image_files)
|
||||
}
|
||||
|
||||
# 计算所有图片的哈希
|
||||
hashes = {}
|
||||
for img_path in image_files:
|
||||
try:
|
||||
img = Image.open(img_path)
|
||||
hashes[str(img_path)] = {
|
||||
'avg_hash': average_hash(img),
|
||||
'diff_hash': difference_hash(img),
|
||||
'size': img.size,
|
||||
'image': img
|
||||
}
|
||||
except Exception as e:
|
||||
continue
|
||||
|
||||
# 两两比较
|
||||
duplicates = []
|
||||
paths = list(hashes.keys())
|
||||
|
||||
for i in range(len(paths)):
|
||||
for j in range(i + 1, len(paths)):
|
||||
path1, path2 = paths[i], paths[j]
|
||||
h1, h2 = hashes[path1], hashes[path2]
|
||||
|
||||
# 平均哈希相似度
|
||||
avg_sim = 1 - hamming_distance(h1['avg_hash'], h2['avg_hash'])
|
||||
|
||||
# 差异哈希相似度
|
||||
diff_sim = 1 - hamming_distance(h1['diff_hash'], h2['diff_hash'])
|
||||
|
||||
# 综合相似度
|
||||
combined_sim = max(avg_sim, diff_sim)
|
||||
|
||||
if combined_sim >= threshold:
|
||||
pair_result = {
|
||||
'file_1': os.path.basename(path1),
|
||||
'file_2': os.path.basename(path2),
|
||||
'avg_hash_similarity': round(float(avg_sim), 4),
|
||||
'diff_hash_similarity': round(float(diff_sim), 4),
|
||||
'combined_similarity': round(float(combined_sim), 4),
|
||||
}
|
||||
|
||||
# 检查旋转/翻转匹配
|
||||
rotation_check = check_rotations(
|
||||
h1['image'], h2['image'], threshold
|
||||
)
|
||||
pair_result['rotation_check'] = rotation_check
|
||||
|
||||
# SSIM（如果可用）
|
||||
if HAS_SKIMAGE:
|
||||
ssim_score = compute_ssim(h1['image'], h2['image'])
|
||||
pair_result['ssim'] = round(ssim_score, 4)
|
||||
|
||||
duplicates.append(pair_result)
|
||||
|
||||
# 关闭所有图片
|
||||
for h in hashes.values():
|
||||
h['image'].close()
|
||||
|
||||
# 风险评分
|
||||
n_duplicates = len(duplicates)
|
||||
n_images = len(image_files)
|
||||
|
||||
if n_duplicates == 0:
|
||||
risk_level = 'low'
|
||||
risk_score = 0
|
||||
elif n_duplicates <= 1:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40
|
||||
elif n_duplicates <= 3:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60
|
||||
else:
|
||||
risk_level = 'high'
|
||||
risk_score = 80 + min(20, n_duplicates * 3)
|
||||
|
||||
# 如果有完美匹配（相似度>0.98），直接拉高
|
||||
perfect_matches = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
if perfect_matches:
|
||||
risk_score = max(risk_score, 90)
|
||||
risk_level = 'high'
|
||||
|
||||
result = {
|
||||
'test_name': 'Image Duplication Detection (图像重复检测)',
|
||||
'status': 'completed',
|
||||
'n_images_scanned': n_images,
|
||||
'n_duplicate_pairs': n_duplicates,
|
||||
'threshold': threshold,
|
||||
'duplicates': duplicates,
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_image(n_duplicates, n_images, duplicates)
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_image(n_duplicates, n_images, duplicates):
|
||||
"""生成可读的解释"""
|
||||
if n_duplicates == 0:
|
||||
return f"✅ 在 {n_images} 张图片中未发现重复或高度相似的图像对。"
|
||||
|
||||
perfect = [d for d in duplicates if d['combined_similarity'] > 0.98]
|
||||
|
||||
if perfect:
|
||||
return (
|
||||
f"⚠️ 发现 {len(perfect)} 对近乎完全相同的图片！"
|
||||
f"这些图片可能是同一图片的重复使用，强烈建议核查是否为不同实验条件下的独立数据。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"⚡ 发现 {n_duplicates} 对高度相似的图片（共扫描 {n_images} 张）。"
|
||||
f"可能存在图片复用或篡改，建议人工核查具体图片内容。"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='图像重复检测 - 检测论文图片是否存在重复使用或篡改'
|
||||
)
|
||||
parser.add_argument('--input_dir', '-i', required=True, help='图片目录路径')
|
||||
parser.add_argument('--threshold', '-t', type=float, default=0.85,
|
||||
help='相似度阈值（0-1），默认0.85')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.isdir(args.input_dir):
|
||||
print(f"错误：目录不存在: {args.input_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = find_duplicates(args.input_dir, args.threshold)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2, cls=NumpyEncoder)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1713
tools/gengskill/scripts/input_pipeline.py
Normal file
1713
tools/gengskill/scripts/input_pipeline.py
Normal file
File diff suppressed because it is too large
Load Diff
244
tools/gengskill/scripts/last_digit_test.py
Normal file
244
tools/gengskill/scripts/last_digit_test.py
Normal file
@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
末位数字检测 (Last Digit Test)
|
||||
==============================
|
||||
原理：自然实验数据的末位数字（0-9）应近似均匀分布。
|
||||
如果数据是人为编造的，末位数字往往会集中在某些特定值上。
|
||||
|
||||
使用卡方检验评估偏离均匀分布的程度。
|
||||
|
||||
致敬"耿同学讲故事" — 用数据说话，让造假无所遁形。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
from collections import Counter
|
||||
from scipy import stats
|
||||
|
||||
|
||||
def extract_last_digit(value):
|
||||
"""提取数值的末位有效数字"""
|
||||
s = str(value).strip()
|
||||
# 去除负号
|
||||
s = s.lstrip('-')
|
||||
# 去除科学计数法
|
||||
if 'e' in s.lower():
|
||||
try:
|
||||
value = float(s)
|
||||
s = f"{value:.10f}".rstrip('0')
|
||||
except ValueError:
|
||||
return None
|
||||
# 找到末位有效数字
|
||||
s = s.rstrip('0').rstrip('.')
|
||||
if not s:
|
||||
return 0
|
||||
for c in reversed(s):
|
||||
if c.isdigit():
|
||||
return int(c)
|
||||
return None
|
||||
|
||||
|
||||
def extract_last_digit_with_decimals(value, use_decimal_last=True):
|
||||
"""
|
||||
提取数值的末位数字
|
||||
use_decimal_last=True: 取小数点后最后一位非零数字
|
||||
use_decimal_last=False: 取整数部分的末位数字
|
||||
"""
|
||||
s = str(value).strip()
|
||||
s = s.lstrip('-')
|
||||
|
||||
if '.' in s and use_decimal_last:
|
||||
decimal_part = s.split('.')[1]
|
||||
# 取小数部分的最后一位数字
|
||||
if decimal_part:
|
||||
return int(decimal_part[-1])
|
||||
|
||||
# 取整数部分的末位
|
||||
integer_part = s.split('.')[0] if '.' in s else s
|
||||
if integer_part:
|
||||
return int(integer_part[-1])
|
||||
return None
|
||||
|
||||
|
||||
def last_digit_test(values, method='all_digits'):
|
||||
"""
|
||||
执行末位数字检测
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : list of float/str
|
||||
待检测的数值列表
|
||||
method : str
|
||||
'all_digits' - 检测最后一位有效数字
|
||||
'decimal_last' - 检测小数末位
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict : 检测结果
|
||||
"""
|
||||
# 提取末位数字
|
||||
last_digits = []
|
||||
for v in values:
|
||||
try:
|
||||
if method == 'decimal_last':
|
||||
d = extract_last_digit_with_decimals(v, use_decimal_last=True)
|
||||
else:
|
||||
d = extract_last_digit(v)
|
||||
if d is not None:
|
||||
last_digits.append(d)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
if len(last_digits) < 10:
|
||||
return {
|
||||
'status': 'insufficient_data',
|
||||
'message': f'数据量不足（仅{len(last_digits)}个有效值），需要至少10个数据点',
|
||||
'n_valid': len(last_digits)
|
||||
}
|
||||
|
||||
# 统计各数字出现频次
|
||||
digit_counts = Counter(last_digits)
|
||||
observed = np.array([digit_counts.get(i, 0) for i in range(10)])
|
||||
expected = np.full(10, len(last_digits) / 10.0)
|
||||
|
||||
# 卡方检验
|
||||
chi2, p_value = stats.chisquare(observed, expected)
|
||||
|
||||
# 计算集中度指标
|
||||
max_digit = int(np.argmax(observed))
|
||||
max_freq = observed[max_digit] / len(last_digits)
|
||||
|
||||
# 均匀性评分 (0=完全均匀, 1=完全集中)
|
||||
uniformity_deviation = np.sqrt(np.sum((observed / len(last_digits) - 0.1) ** 2) / 10) / 0.3
|
||||
uniformity_deviation = min(uniformity_deviation, 1.0)
|
||||
|
||||
# 风险评分
|
||||
if p_value < 0.001:
|
||||
risk_level = 'high'
|
||||
risk_score = min(80 + (1 - p_value) * 20, 100)
|
||||
elif p_value < 0.01:
|
||||
risk_level = 'medium-high'
|
||||
risk_score = 60 + (0.01 - p_value) / 0.009 * 20
|
||||
elif p_value < 0.05:
|
||||
risk_level = 'medium'
|
||||
risk_score = 40 + (0.05 - p_value) / 0.04 * 20
|
||||
else:
|
||||
risk_level = 'low'
|
||||
risk_score = max(0, 40 * (1 - p_value))
|
||||
|
||||
result = {
|
||||
'test_name': 'Last Digit Test (末位数字检测)',
|
||||
'status': 'completed',
|
||||
'n_values': len(last_digits),
|
||||
'method': method,
|
||||
'digit_distribution': {str(i): int(observed[i]) for i in range(10)},
|
||||
'chi_square': round(float(chi2), 4),
|
||||
'p_value': float(p_value),
|
||||
'degrees_of_freedom': 9,
|
||||
'most_frequent_digit': max_digit,
|
||||
'most_frequent_proportion': round(float(max_freq), 4),
|
||||
'uniformity_deviation': round(float(uniformity_deviation), 4),
|
||||
'risk_level': risk_level,
|
||||
'risk_score': round(float(risk_score), 1),
|
||||
'interpretation': _interpret_result(p_value, max_digit, max_freq, len(last_digits))
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _interpret_result(p_value, max_digit, max_freq, n):
|
||||
"""生成可读的解释"""
|
||||
if p_value < 0.001:
|
||||
return (
|
||||
f"⚠️ 末位数字分布严重偏离均匀分布（p < 0.001）。"
|
||||
f"数字 {max_digit} 出现频率为 {max_freq:.1%}（期望 10%），"
|
||||
f"这种偏离在自然实验数据中极为罕见，强烈建议进一步核查原始数据。"
|
||||
)
|
||||
elif p_value < 0.01:
|
||||
return (
|
||||
f"⚠️ 末位数字分布显著偏离均匀分布（p < 0.01）。"
|
||||
f"数字 {max_digit} 出现频率为 {max_freq:.1%}，建议关注并进行人工复核。"
|
||||
)
|
||||
elif p_value < 0.05:
|
||||
return (
|
||||
f"⚡ 末位数字分布存在一定偏离（p < 0.05）。"
|
||||
f"可能是正常波动，也可能提示数据存在问题，建议结合其他检测结果综合判断。"
|
||||
)
|
||||
else:
|
||||
return (
|
||||
f"✅ 末位数字分布与均匀分布无显著差异（p = {p_value:.4f}），"
|
||||
f"未发现明显异常。"
|
||||
)
|
||||
|
||||
|
||||
def load_data(input_file, column=None, delimiter=','):
|
||||
"""从CSV文件加载数据"""
|
||||
import csv
|
||||
|
||||
values = []
|
||||
with open(input_file, 'r', encoding='utf-8-sig') as f:
|
||||
reader = csv.DictReader(f, delimiter=delimiter)
|
||||
if column and column in reader.fieldnames:
|
||||
for row in reader:
|
||||
try:
|
||||
val = row[column].strip()
|
||||
if val:
|
||||
float(val) # 验证是数值
|
||||
values.append(val)
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
else:
|
||||
# 如果没有指定列，尝试读取第一个数值列
|
||||
for row in reader:
|
||||
for key, val in row.items():
|
||||
try:
|
||||
val = val.strip()
|
||||
if val:
|
||||
float(val)
|
||||
values.append(val)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
return values
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='末位数字检测 - 检测数据末位数字是否偏离均匀分布'
|
||||
)
|
||||
parser.add_argument('--input', '-i', required=True, help='输入CSV文件路径')
|
||||
parser.add_argument('--column', '-c', help='要检测的列名')
|
||||
parser.add_argument('--method', '-m', default='all_digits',
|
||||
choices=['all_digits', 'decimal_last'],
|
||||
help='检测方法：all_digits=末位有效数字, decimal_last=小数末位')
|
||||
parser.add_argument('--delimiter', '-d', default=',', help='CSV分隔符')
|
||||
parser.add_argument('--output', '-o', help='输出JSON文件路径')
|
||||
parser.add_argument('--values', nargs='+', type=str,
|
||||
help='直接传入数值列表（不使用文件输入）')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.values:
|
||||
values = args.values
|
||||
else:
|
||||
values = load_data(args.input, args.column, args.delimiter)
|
||||
|
||||
if not values:
|
||||
print("错误：未能加载有效数据", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
result = last_digit_test(values, method=args.method)
|
||||
|
||||
output_json = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_json)
|
||||
print(f"结果已保存至: {args.output}")
|
||||
else:
|
||||
print(output_json)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1775
tools/gengskill/scripts/report_generator.py
Normal file
1775
tools/gengskill/scripts/report_generator.py
Normal file
File diff suppressed because it is too large
Load Diff
37
tools/gengskill/scripts/run_dup_test.py
Normal file
37
tools/gengskill/scripts/run_dup_test.py
Normal file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrapper to run image_duplicate_test with proper numpy handling"""
|
||||
import sys
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
# We'll monkey-patch json to handle numpy types
|
||||
class NumpyEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
if isinstance(obj, (np.floating,)):
|
||||
return float(obj)
|
||||
if isinstance(obj, (np.bool_,)):
|
||||
return bool(obj)
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
if obj is None:
|
||||
return None
|
||||
return super().default(obj)
|
||||
|
||||
# Save original dumps
|
||||
original_dumps = json.dumps
|
||||
|
||||
def patched_dumps(obj, **kwargs):
|
||||
kwargs.setdefault('cls', NumpyEncoder)
|
||||
return original_dumps(obj, **kwargs)
|
||||
|
||||
json.dumps = patched_dumps
|
||||
|
||||
# Now import and run the original script
|
||||
sys.argv = ['image_duplicate_test.py',
|
||||
'--input_dir', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/figures',
|
||||
'--threshold', '0.85',
|
||||
'--output', '/home/program/qq-workspace/self-workplace/geng-skills/paper_images/duplicate_results.json']
|
||||
|
||||
exec(open('scripts/image_duplicate_test.py').read())
|
||||
1228
tools/gengskill/scripts/visualization.py
Normal file
1228
tools/gengskill/scripts/visualization.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user