规则库 3.8.1 + 排盘修复 + 评测体系
- 规则库 3.6→3.8.1（542→548 条）：剪枝上锁、R-杂-38 从格临界双轨、 R-事-10~13 冲刑方向判别、R-禁-23 自造必要条件之禁 - 排盘修复随 3.6.2 已发：港澳时区夏令时双向误拨、晚子时双重换日 - 新增 评测/ 目录（BaziQA 题答分离、判分工具、记分卡、with/without-skill 对照） 与 scripts/query_rules.py 条款检索；ZIPEC_EVAL 评测额度豁免 - manifest 重建并重新签名；VERSION 保持 3.6.2，不发版不打 tag
This commit is contained in:
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""评测判分脚本：抽取求解代理答案 → 比对答案键 → 输出统计与报告。
|
||||
|
||||
答案文件约定：<submissions>/<contest>_<person_id>.md，每命主末尾恰好 5 行
|
||||
`Q1: A` … `Q5: X`（X ∈ ABCD；答不出标 N 记 no_answer；行尾允许 # 注释）。
|
||||
题数以 questions/ 盲版为准（允许单命主不足 5 题，如 2021 届 P033/P034）。
|
||||
|
||||
数据源全部参数化、不依赖任何外部仓库路径：
|
||||
- --questions 盲版题目目录（枚举命主与题序）
|
||||
- --answers-key 答案键目录（person_id + 题序 + 答案字母 + 维度标签）
|
||||
- --submissions 待判答案 md 目录
|
||||
- --out-json / --out-md 结果输出
|
||||
|
||||
基线：BaziQA 官方基线（Gemini 跑分）内嵌于本脚本，口径见
|
||||
试题/BaziQA-全球算命师大赛/SOURCE.md。
|
||||
|
||||
零网络调用。用法：
|
||||
python run_eval.py [--questions <dir>] [--answers-key <dir>]
|
||||
[--submissions <dir>] [--out-json <file>] [--out-md <file>]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] # 评测/
|
||||
CONTEST_DIR = ROOT / "试题" / "BaziQA-全球算命师大赛"
|
||||
|
||||
# BaziQA 官方基线（来源见 SOURCE.md）：总体 + 分届 + 分维度
|
||||
OFFICIAL_TOTAL = {"total": 200, "correct": 69, "accuracy": 34.5}
|
||||
OFFICIAL_CONTEST = {
|
||||
"contest8_2021": {"total": 40, "accuracy": 27.5},
|
||||
"contest8_2022": {"total": 40, "accuracy": 40.0},
|
||||
"contest8_2023": {"total": 40, "accuracy": 27.5},
|
||||
"contest8_2024": {"total": 40, "accuracy": 35.0},
|
||||
"contest8_2025": {"total": 40, "accuracy": 42.5},
|
||||
}
|
||||
OFFICIAL_DIM = {
|
||||
"流年运势": {"total": 37, "accuracy": 40.5},
|
||||
"感情": {"total": 49, "accuracy": 36.7},
|
||||
"六亲": {"total": 29, "accuracy": 27.6},
|
||||
"事业": {"total": 25, "accuracy": 24.0},
|
||||
"性格": {"total": 20, "accuracy": 45.0},
|
||||
"学业": {"total": 12, "accuracy": 41.7},
|
||||
"财富": {"total": 13, "accuracy": 30.8},
|
||||
"健康": {"total": 13, "accuracy": 23.1},
|
||||
"其他": {"total": 2, "accuracy": 50.0},
|
||||
}
|
||||
DIM_ORDER = ["流年运势", "感情", "六亲", "事业", "性格", "学业", "财富", "健康", "其他"]
|
||||
|
||||
ANS_RE = re.compile(r"^\s*Q([1-5])\s*[:：]\s*([A-DNdn])\b", re.M)
|
||||
FILE_RE = re.compile(r"^(contest\d+_\d{4})_(.+)$")
|
||||
|
||||
|
||||
def load_questions(q_dir: Path) -> dict[str, list[dict]]:
|
||||
"""盲版题目：contest -> 命主数组（仅取 person_id 与题数，不读题面）。"""
|
||||
out = {}
|
||||
for f in sorted(q_dir.glob("contest8_*.json")):
|
||||
d = json.loads(f.read_text(encoding="utf-8"))
|
||||
out[d[0]["contest_id"]] = d[1:]
|
||||
return out
|
||||
|
||||
|
||||
def load_answer_key(k_dir: Path) -> dict[tuple[str, str], dict[int, dict]]:
|
||||
"""答案键：(contest, person_id) -> {题序: {answer, dimension}}。"""
|
||||
out = {}
|
||||
for f in sorted(k_dir.glob("contest8_*.json")):
|
||||
d = json.loads(f.read_text(encoding="utf-8"))
|
||||
contest = d[0]["contest_id"]
|
||||
for p in d[1:]:
|
||||
out[(contest, p["person_id"])] = {
|
||||
a["question_index"]: a for a in p["answers"]}
|
||||
return out
|
||||
|
||||
|
||||
def parse_answer_file(path: Path) -> dict[int, str]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
ans: dict[int, str] = {}
|
||||
for m in ANS_RE.finditer(text):
|
||||
q = int(m.group(1))
|
||||
if q not in ans: # 同一题取首次出现
|
||||
ans[q] = m.group(2).upper()
|
||||
return ans
|
||||
|
||||
|
||||
def acc(correct: int, total: int) -> float:
|
||||
return round(correct / total * 100, 1) if total else 0.0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="评测判分：submissions 答案文件 vs 答案键")
|
||||
ap.add_argument("--questions", type=Path, default=CONTEST_DIR / "questions")
|
||||
ap.add_argument("--answers-key", type=Path, default=CONTEST_DIR / "answers")
|
||||
ap.add_argument("--submissions", type=Path,
|
||||
default=ROOT / "ai答案" / "kimi-k3" / "with-skill")
|
||||
ap.add_argument("--out-json", type=Path, default=ROOT / "eval_result.json")
|
||||
ap.add_argument("--out-md", type=Path, default=ROOT / "eval_report.md")
|
||||
a = ap.parse_args()
|
||||
|
||||
datasets = load_questions(a.questions)
|
||||
keys = load_answer_key(a.answers_key)
|
||||
|
||||
# 索引答案文件：contest -> person_id -> {q: letter}
|
||||
files: dict[tuple[str, str], dict[int, str]] = {}
|
||||
if a.submissions.is_dir():
|
||||
for f in sorted(a.submissions.glob("*.md")):
|
||||
m = FILE_RE.match(f.stem)
|
||||
if not m:
|
||||
print(f"WARN: 跳过无法解析的文件名 {f.name}", file=sys.stderr)
|
||||
continue
|
||||
files[(m.group(1), m.group(2))] = parse_answer_file(f)
|
||||
|
||||
details = []
|
||||
per_contest: dict[str, dict[str, int]] = {}
|
||||
per_dim: dict[str, dict[str, int]] = {}
|
||||
tot = cor = noans = 0
|
||||
missing: list[str] = []
|
||||
for contest, persons in datasets.items():
|
||||
cs = per_contest.setdefault(contest, {"total": 0, "correct": 0, "no_answer": 0})
|
||||
for p in persons:
|
||||
pid = p["person_id"]
|
||||
got = files.get((contest, pid))
|
||||
if got is None:
|
||||
missing.append(f"{contest}/{pid}")
|
||||
got = {}
|
||||
key = keys.get((contest, pid), {})
|
||||
for i in range(1, len(p["questions"]) + 1):
|
||||
letter = got.get(i)
|
||||
unanswered = letter in (None, "N")
|
||||
k = key.get(i, {})
|
||||
ok = (not unanswered) and letter == k.get("answer")
|
||||
dim = k.get("dimension", "未标注")
|
||||
ds = per_dim.setdefault(dim, {"total": 0, "correct": 0, "no_answer": 0})
|
||||
for s in (cs, ds):
|
||||
s["total"] += 1
|
||||
s["correct"] += ok
|
||||
s["no_answer"] += unanswered
|
||||
tot += 1
|
||||
cor += ok
|
||||
noans += unanswered
|
||||
details.append({
|
||||
"dataset": contest, "person_id": pid, "question_index": i,
|
||||
"ai_answer": letter or "N",
|
||||
"correct_answer": k.get("answer", "?"), "is_correct": ok,
|
||||
"dimension": dim,
|
||||
})
|
||||
|
||||
if missing:
|
||||
print(f"WARN: {len(missing)} 位命主缺答案文件，按全未答计："
|
||||
+ ", ".join(missing), file=sys.stderr)
|
||||
|
||||
stats: dict = {"总体": {"total": tot, "correct": cor,
|
||||
"accuracy": acc(cor, tot), "no_answer": noans}}
|
||||
for c, s in per_contest.items():
|
||||
stats[c] = {**s, "accuracy": acc(s["correct"], s["total"])}
|
||||
dim_stats = {}
|
||||
for d in DIM_ORDER + sorted(set(per_dim) - set(DIM_ORDER)):
|
||||
if d in per_dim:
|
||||
s = per_dim[d]
|
||||
dim_stats[d] = {**s, "accuracy": acc(s["correct"], s["total"])}
|
||||
stats["分维度"] = dim_stats
|
||||
|
||||
result = {"stats": stats, "details": details}
|
||||
a.out_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out_json.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
ours = stats["总体"]
|
||||
lines = [
|
||||
"# BaziQA 评测报告", "",
|
||||
"## 总览", "",
|
||||
"| 口径 | 正确/总题数 | 准确率 | 未答 |", "|---|---|---|---|",
|
||||
f"| 本次评测 | {ours['correct']}/{ours['total']} | {ours['accuracy']}% | {ours['no_answer']} |",
|
||||
f"| 官方基线（Gemini） | {OFFICIAL_TOTAL['correct']}/{OFFICIAL_TOTAL['total']}"
|
||||
f" | {OFFICIAL_TOTAL['accuracy']}% | 0 |",
|
||||
"", f"> 基线口径见 试题/BaziQA-全球算命师大赛/SOURCE.md。",
|
||||
"", "## 分届准确率", "",
|
||||
"| 届别 | 本次 | 官方 |", "|---|---|---|",
|
||||
]
|
||||
for c in sorted(per_contest):
|
||||
s = stats[c]
|
||||
off = OFFICIAL_CONTEST.get(c, {})
|
||||
lines.append(f"| {c} | {s['correct']}/{s['total']} = {s['accuracy']}%"
|
||||
f"（未答 {s['no_answer']}） | {off.get('accuracy', '—')}% |")
|
||||
lines += ["", "## 分维度准确率", "",
|
||||
"| 维度 | 本次 | 官方 |", "|---|---|---|"]
|
||||
for d, s in dim_stats.items():
|
||||
off = OFFICIAL_DIM.get(d, {})
|
||||
lines.append(f"| {d} | {s['correct']}/{s['total']} = {s['accuracy']}%"
|
||||
f"（未答 {s['no_answer']}） | {off.get('accuracy', '—')}% |")
|
||||
lines += ["", "## 错题清单（错因留空，待人工/后续填写）", "",
|
||||
"| 届别 | 命主 | 题序 | 维度 | 我答 | 正解 | 错因 |",
|
||||
"|---|---|---|---|---|---|---|"]
|
||||
for d in details:
|
||||
if not d["is_correct"]:
|
||||
lines.append(f"| {d['dataset']} | {d['person_id']} | Q{d['question_index']}"
|
||||
f" | {d['dimension']} | {d['ai_answer']} | {d['correct_answer']} | |")
|
||||
a.out_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"总准确率: {ours['correct']}/{ours['total']} = {ours['accuracy']}%"
|
||||
f"（未答 {ours['no_answer']}；官方基线 {OFFICIAL_TOTAL['accuracy']}%）")
|
||||
print(f"写出 {a.out_json} / {a.out_md}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user