规则库 3.8.1 + 排盘修复 + 评测体系
- 规则库 3.6→3.8.1(542→548 条):剪枝上锁、R-杂-38 从格临界双轨、 R-事-10~13 冲刑方向判别、R-禁-23 自造必要条件之禁 - 排盘修复随 3.6.2 已发:港澳时区夏令时双向误拨、晚子时双重换日 - 新增 评测/ 目录(BaziQA 题答分离、判分工具、记分卡、with/without-skill 对照) 与 scripts/query_rules.py 条款检索;ZIPEC_EVAL 评测额度豁免 - manifest 重建并重新签名;VERSION 保持 3.6.2,不发版不打 tag
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""评测用地点扩展:给技能副本的 data/cities.json 追加境外/模糊地名的代理城市条目。
|
||||
|
||||
条目与首次评测(fate_benchmark 工作区)所用完全一致,原样搬来:
|
||||
- 境外城市按真实 IANA 时区(台北/吉隆坡/新加坡/宫崎/香港/澳门);
|
||||
- 模糊地名(美国/中国/中国大陆)代理到杭州 120.16E Asia/Shanghai,
|
||||
对齐官方 UTC+8 简化口径;港澳条目对齐母版修复口径(tzdata 自带
|
||||
1949–1979 夏令时、无 1986–1991 大陆夏令时)。
|
||||
|
||||
用法:
|
||||
python extend_cities.py [--cities <技能副本 data/cities.json>]
|
||||
[--scripts <技能副本 scripts 目录>]
|
||||
[--questions <盲版题目目录>]
|
||||
|
||||
追加后批跑校验:--questions 所指各届所有命主的 place 字符串逐一过
|
||||
resolve_location,全部命中才算通过。默认指向母版自身(scripts/、data/),
|
||||
评测复测时通常先复制一份技能再对副本执行本脚本,避免改动母版。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] # 评测/
|
||||
SKILL = Path(__file__).resolve().parents[2] # 技能母版根
|
||||
DEFAULT_QUESTIONS = ROOT / "试题" / "BaziQA-全球算命师大赛" / "questions"
|
||||
|
||||
TAIPEI = {"lon": 121.56, "lat": 25.03, "tz": "Asia/Taipei"}
|
||||
KL = {"lon": 101.69, "lat": 3.14, "tz": "Asia/Kuala_Lumpur"}
|
||||
SG = {"lon": 103.85, "lat": 1.29, "tz": "Asia/Singapore"}
|
||||
MIYAZAKI = {"lon": 131.42, "lat": 31.91, "tz": "Asia/Tokyo"}
|
||||
# 官方口径统一按 UTC+8 处理,美国/中国等模糊地名代理到杭州(120.16E)
|
||||
HANGZHOU_PROXY = {"lon": 120.16, "lat": 30.29, "tz": "Asia/Shanghai"}
|
||||
# 港澳对齐母版修复口径:港澳 tzdata 自带 1949–1979 夏令时、无 1986–1991 大陆夏令时
|
||||
HK = {"lon": 114.17, "lat": 22.32, "tz": "Asia/Hong_Kong"}
|
||||
MO = {"lon": 113.54, "lat": 22.2, "tz": "Asia/Macau"}
|
||||
GZ = {"lon": 113.26, "lat": 23.13, "tz": "Asia/Shanghai"}
|
||||
ST = {"lon": 116.68, "lat": 23.35, "tz": "Asia/Shanghai"}
|
||||
BJ = {"lon": 116.41, "lat": 39.9, "tz": "Asia/Shanghai"}
|
||||
|
||||
EXTRA = {
|
||||
"台北": TAIPEI, "台湾": TAIPEI, "中国台湾": TAIPEI, "台湾,中国": TAIPEI,
|
||||
"吉隆坡": KL, "马来西亚": KL,
|
||||
"新加坡": SG,
|
||||
"宫崎": MIYAZAKI, "日本": MIYAZAKI, "日本宫崎": MIYAZAKI, "日本宫崎县": MIYAZAKI,
|
||||
"美国": HANGZHOU_PROXY, "中国": HANGZHOU_PROXY, "中国大陆": HANGZHOU_PROXY,
|
||||
"中国香港": HK, "香港,中国": HK,
|
||||
"中国澳门": MO, "澳门,中国": MO,
|
||||
"广东": GZ, "广东,中国": GZ,
|
||||
"潮汕": ST, "广东潮汕,中国": ST,
|
||||
"北京,中国": BJ,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="评测用地点扩展(境外/模糊地名 → 代理城市)")
|
||||
ap.add_argument("--cities", type=Path, default=SKILL / "data" / "cities.json")
|
||||
ap.add_argument("--scripts", type=Path, default=SKILL / "scripts")
|
||||
ap.add_argument("--questions", type=Path, default=DEFAULT_QUESTIONS)
|
||||
a = ap.parse_args()
|
||||
|
||||
table = json.loads(a.cities.read_text(encoding="utf-8"))
|
||||
added = 0
|
||||
for k, v in EXTRA.items():
|
||||
if k not in table:
|
||||
table[k] = v
|
||||
added += 1
|
||||
a.cities.write_text(json.dumps(table, ensure_ascii=False, indent=1) + "\n",
|
||||
encoding="utf-8")
|
||||
print(f"追加 {added} 条(目标 {len(EXTRA)} 条),总条目 {len(table)}")
|
||||
|
||||
# 批跑校验:各届全部命主的 place 逐一解析
|
||||
sys.path.insert(0, str(a.scripts))
|
||||
from geo.cities import resolve_location # noqa: E402
|
||||
|
||||
ok = fail = 0
|
||||
for f in sorted(a.questions.glob("contest8_*.json")):
|
||||
for p in json.loads(f.read_text(encoding="utf-8"))[1:]:
|
||||
place = p["profile"]["birth"]["place"]
|
||||
try:
|
||||
loc = resolve_location(place)
|
||||
ok += 1
|
||||
print(f" OK {f.name} {p['person_id']:<40} {place!r} -> "
|
||||
f"{loc.name} {loc.lon}E {loc.tz}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
fail += 1
|
||||
print(f" FAIL {f.name} {p['person_id']:<40} {place!r}: {e}")
|
||||
print(f"地点解析: {ok} 成功 / {fail} 失败")
|
||||
return 1 if fail else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""试题原始包 → questions/(盲版)+ answers/(答案键)分离生成器。
|
||||
|
||||
输入为含官方答案的赛事原始 JSON(contest8_*.json 结构:第 0 项元数据,
|
||||
其后为命主数组,每命主 questions 列表,每题含 answer 字段)。输出:
|
||||
|
||||
- <out>/questions/<file>.json:与原版同构但剥离每题 answer 的盲版(求解端只准读这里);
|
||||
- <out>/answers/<file>.json:答案键,仅 [元数据, {person_id, answers: [
|
||||
{question_index, answer, dimension}, ...]} ...];维度标签可由 --scores
|
||||
指向的官方跑分目录(details 含 dimension 字段)合并,缺省标「未标注」。
|
||||
|
||||
用法(在本仓库根目录之外同样可用,路径全部参数化):
|
||||
python make_blind.py --src <原始包目录> [--out <赛事目录>]
|
||||
[--scores <官方跑分目录>]
|
||||
|
||||
校验:
|
||||
1. 结构断言:盲版任何位置(递归)不存在 "answer" 键;
|
||||
2. 文本扫描:盲版文件原文不出现 `"answer"` 字样;
|
||||
3. 等价性:盲版 + 答案键可无损还原原文件(确认只删了 answer);
|
||||
4. 题数:每届实际题数与元数据 total_questions 一致;
|
||||
5. 维度覆盖:打印未标注维度计数(有 --scores 时应为 0)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] # 评测/
|
||||
DEFAULT_OUT = ROOT / "试题" / "BaziQA-全球算命师大赛"
|
||||
|
||||
|
||||
def strip_answers(node, path: str = "$"):
|
||||
"""递归剥离题目里的 answer 键;返回剥离数量。非题目位置本就不该有 answer,发现即报错。"""
|
||||
n = 0
|
||||
if isinstance(node, dict):
|
||||
if "options" in node and "question_id" in node:
|
||||
assert "answer" in node, f"{path}: 题目缺少 answer 字段"
|
||||
del node["answer"]
|
||||
n += 1
|
||||
else:
|
||||
assert "answer" not in node, f"{path}: 非题目位置出现 answer 键"
|
||||
for k, v in node.items():
|
||||
n += strip_answers(v, f"{path}.{k}")
|
||||
elif isinstance(node, list):
|
||||
for i, v in enumerate(node):
|
||||
n += strip_answers(v, f"{path}[{i}]")
|
||||
return n
|
||||
|
||||
|
||||
def has_answer_key(node) -> bool:
|
||||
if isinstance(node, dict):
|
||||
return "answer" in node or any(has_answer_key(v) for v in node.values())
|
||||
if isinstance(node, list):
|
||||
return any(has_answer_key(v) for v in node)
|
||||
return False
|
||||
|
||||
|
||||
def load_dimensions(scores_dir: Path | None) -> dict[tuple[str, str, int], str]:
|
||||
"""从官方跑分 details 复用维度标签,键 (dataset, person_id, question_index)。"""
|
||||
dim = {}
|
||||
if not scores_dir or not scores_dir.is_dir():
|
||||
return dim
|
||||
for f in sorted(scores_dir.glob("contest8_*.json")):
|
||||
try:
|
||||
s = json.loads(f.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
for d in s.get("details", []):
|
||||
dim[(d["dataset"], d["person_id"], d["question_index"])] = \
|
||||
d.get("dimension", "未标注")
|
||||
return dim
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="试题原始包 → questions/+answers/ 分离")
|
||||
ap.add_argument("--src", type=Path, required=True, help="含 answer 的原始 contest8_*.json 目录")
|
||||
ap.add_argument("--out", type=Path, default=DEFAULT_OUT, help="赛事目录(其下建 questions/ 与 answers/)")
|
||||
ap.add_argument("--scores", type=Path, default=None, help="官方跑分目录(取 dimension 维度标签,可省)")
|
||||
a = ap.parse_args()
|
||||
|
||||
q_dir = a.out / "questions"
|
||||
k_dir = a.out / "answers"
|
||||
q_dir.mkdir(parents=True, exist_ok=True)
|
||||
k_dir.mkdir(parents=True, exist_ok=True)
|
||||
dims = load_dimensions(a.scores)
|
||||
|
||||
total = 0
|
||||
for src in sorted(a.src.glob("contest8_*.json")):
|
||||
original = json.loads(src.read_text(encoding="utf-8"))
|
||||
contest = original[0]["contest_id"]
|
||||
blind = copy.deepcopy(original)
|
||||
n = strip_answers(blind)
|
||||
|
||||
# 结构断言:盲版任何位置无 answer 键
|
||||
assert not has_answer_key(blind), f"{src.name}: 盲版仍含 answer 键"
|
||||
|
||||
# 答案键:仅 person_id + 题序 + 答案字母 + 维度标签
|
||||
missing_dim = 0
|
||||
key = [{
|
||||
"contest_id": contest,
|
||||
"total_questions": original[0].get("total_questions"),
|
||||
"note": "答案键:person_id + 题序 + 标准答案字母 + 维度标签;求解端禁读",
|
||||
}]
|
||||
for p in original[1:]:
|
||||
answers = []
|
||||
for i, q in enumerate(p["questions"], start=1):
|
||||
d = dims.get((contest, p["person_id"], i))
|
||||
if d is None:
|
||||
d, missing_dim = "未标注", missing_dim + 1
|
||||
answers.append({"question_index": i, "answer": q["answer"],
|
||||
"dimension": d})
|
||||
key.append({"person_id": p["person_id"], "answers": answers})
|
||||
|
||||
# 等价性:盲版 + 答案键 == 原版(确认除 answer 外未动任何字段)
|
||||
rebuilt = copy.deepcopy(blind)
|
||||
for p_o, p_r in zip(original[1:], rebuilt[1:]):
|
||||
for q_o, q_r in zip(p_o["questions"], p_r["questions"]):
|
||||
q_r["answer"] = q_o["answer"]
|
||||
assert rebuilt == original, f"{src.name}: 剥离 answer 后其余字段不一致"
|
||||
|
||||
# 题数与元数据一致
|
||||
actual = sum(len(p["questions"]) for p in original[1:])
|
||||
assert original[0].get("total_questions") in (None, actual), \
|
||||
f"{src.name}: 元数据 total_questions={original[0].get('total_questions')} 与实际 {actual} 不符"
|
||||
|
||||
q_text = json.dumps(blind, ensure_ascii=False, indent=1) + "\n"
|
||||
# 文本扫描:盲版原文不得出现 "answer" 字样
|
||||
assert '"answer"' not in q_text, f"{src.name}: 盲版文本含 answer 字样"
|
||||
(q_dir / src.name).write_text(q_text, encoding="utf-8")
|
||||
(k_dir / src.name).write_text(
|
||||
json.dumps(key, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
total += n
|
||||
print(f"{src.name}: {len(original) - 1} 命主 {n} 题 -> "
|
||||
f"questions/(盲版)+ answers/(答案键,维度未标注 {missing_dim})")
|
||||
print(f"OK: 共分离 {total} 题,questions/ 与 answers/ 已生成并校验通过")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""得分.json → 单文件 HTML 记分卡(模板 scorecard_template.html)。
|
||||
|
||||
沿用技能盘面图的品牌风格:logo 与品牌字体从母版 scripts/render/assets/
|
||||
以 base64 内嵌(单文件便携),亮色卡片风。数据全部由 得分.json 注入,
|
||||
模板不含任何硬编码分数。
|
||||
|
||||
得分.json 字段 schema(收录新模型时按此填写):
|
||||
model, skill_mode(with-skill / without-skill), skill_version, eval_date,
|
||||
benchmark,
|
||||
overall: {total, correct, accuracy, no_answer}
|
||||
baseline: {name, overall: {total, correct, accuracy},
|
||||
per_contest: {contest: accuracy}, per_dimension: {维度: accuracy}}
|
||||
per_contest: {contest: {total, correct, accuracy, no_answer}}
|
||||
per_dimension:{维度: {total, correct, accuracy, no_answer}}
|
||||
|
||||
用法:
|
||||
python render_scorecard.py [--score <得分.json>] [--out <输出.html>]
|
||||
[--template <模板>] [--assets <render/assets>]
|
||||
默认:--score 评测/ai模型纬度得分/kimi-k3/with-skill/得分.json,
|
||||
输出与得分.json 同目录的 得分.html。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from html import escape
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] # 评测/
|
||||
SKILL = Path(__file__).resolve().parents[2] # 技能母版根
|
||||
DEFAULT_SCORE = ROOT / "ai模型纬度得分" / "kimi-k3" / "with-skill" / "得分.json"
|
||||
DIM_ORDER = ["流年运势", "感情", "六亲", "事业", "性格", "学业", "财富", "健康", "其他"]
|
||||
|
||||
# 结构性占位符:值是装配层渲染的 HTML 片段或 base64 资源,填充时原样注入;
|
||||
# 其余占位符均为文本字段,一律 HTML 转义防注入(与盘面图装配层同口径)。
|
||||
STRUCTURAL = {"LOGO_B64", "FONT_B64", "DIM_BARS", "CONTEST_ROWS"}
|
||||
|
||||
|
||||
def asset_b64(assets: Path, name: str) -> str:
|
||||
f = assets / name
|
||||
return base64.b64encode(f.read_bytes()).decode("ascii") if f.is_file() else ""
|
||||
|
||||
|
||||
def dim_bars(score: dict) -> str:
|
||||
rows = []
|
||||
dims = score["per_dimension"]
|
||||
base = score["baseline"].get("per_dimension", {})
|
||||
for d in DIM_ORDER + sorted(set(dims) - set(DIM_ORDER)):
|
||||
if d not in dims:
|
||||
continue
|
||||
s = dims[d]
|
||||
b = base.get(d)
|
||||
name = escape(d)
|
||||
ours = f'<div class="bar ours" style="width:{s["accuracy"]}%"></div>'
|
||||
ours += (f'<div class="bar base" style="width:{b}%"></div>'
|
||||
if isinstance(b, (int, float)) else "")
|
||||
val = (f'<b>{s["accuracy"]}%</b>({s["correct"]}/{s["total"]})'
|
||||
+ (f' <span style="color:var(--muted)">基线 {b}%</span>'
|
||||
if isinstance(b, (int, float)) else ""))
|
||||
rows.append(f'<div class="dimrow"><div class="dimname">{name}</div>'
|
||||
f'<div class="dimtrack">{ours}</div>'
|
||||
f'<div class="dimval">{val}</div></div>')
|
||||
return "\n ".join(rows)
|
||||
|
||||
|
||||
def contest_rows(score: dict) -> str:
|
||||
rows = []
|
||||
base = score["baseline"].get("per_contest", {})
|
||||
for c in sorted(score["per_contest"]):
|
||||
s = score["per_contest"][c]
|
||||
b = base.get(c)
|
||||
if isinstance(b, (int, float)):
|
||||
diff = round(s["accuracy"] - b, 1)
|
||||
cls = "up" if diff >= 0 else "down"
|
||||
delta = (f'<span class="delta {cls}">{"+" if diff >= 0 else ""}'
|
||||
f'{diff}pp</span>')
|
||||
btxt = f"{b}%"
|
||||
else:
|
||||
delta, btxt = "—", "—"
|
||||
rows.append(
|
||||
f'<tr><td class="dim">{escape(c)}</td>'
|
||||
f'<td><b>{s["accuracy"]}%</b>({s["correct"]}/{s["total"]},'
|
||||
f'未答 {s.get("no_answer", 0)})</td>'
|
||||
f'<td>{btxt}</td><td>{delta}</td></tr>')
|
||||
return "\n ".join(rows)
|
||||
|
||||
|
||||
def render(score: dict, template: str, assets: Path, score_json_name: str) -> str:
|
||||
ov = score["overall"]
|
||||
bo = score["baseline"]["overall"]
|
||||
delta = round(ov["accuracy"] - bo["accuracy"], 1)
|
||||
values = {
|
||||
"TITLE": f"评测记分卡:{score['model']}({score['skill_mode']})",
|
||||
"LOGO_B64": asset_b64(assets, "lingyao-logo.png"),
|
||||
"FONT_B64": asset_b64(assets, "logo-font.ttf"),
|
||||
"MODEL": score["model"],
|
||||
"SKILL_MODE_LABEL": "加载技能" if score["skill_mode"] == "with-skill"
|
||||
else "裸答对照",
|
||||
"BENCHMARK": score["benchmark"],
|
||||
"SKILL_VERSION": score.get("skill_version", "—"),
|
||||
"EVAL_DATE": score.get("eval_date", "—"),
|
||||
"OVERALL_ACC": f'{ov["accuracy"]}',
|
||||
"OVERALL_CORRECT": f'{ov["correct"]}',
|
||||
"OVERALL_TOTAL": f'{ov["total"]}',
|
||||
"NO_ANSWER": f'{ov.get("no_answer", 0)}',
|
||||
"BASELINE_NAME": score["baseline"].get("name", "官方基线"),
|
||||
"BASELINE_ACC": f'{bo["accuracy"]}',
|
||||
"DELTA_CLASS": "up" if delta >= 0 else "down",
|
||||
"DELTA_TEXT": f'({"+" if delta >= 0 else ""}{delta}pp)',
|
||||
"DIM_BARS": dim_bars(score),
|
||||
"CONTEST_ROWS": contest_rows(score),
|
||||
"SCORE_JSON": score_json_name,
|
||||
}
|
||||
out = template
|
||||
for k, v in values.items():
|
||||
out = out.replace("{{" + k + "}}", v if k in STRUCTURAL else escape(str(v)))
|
||||
leftover = [p.split("}}")[0] for p in out.split("{{")[1:]]
|
||||
assert not leftover, f"模板存在未填充占位符: {leftover}"
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="得分.json → 单文件 HTML 记分卡")
|
||||
ap.add_argument("--score", type=Path, default=DEFAULT_SCORE)
|
||||
ap.add_argument("--out", type=Path, default=None,
|
||||
help="默认与得分.json 同目录的 得分.html")
|
||||
ap.add_argument("--template", type=Path,
|
||||
default=Path(__file__).resolve().parent / "scorecard_template.html")
|
||||
ap.add_argument("--assets", type=Path,
|
||||
default=SKILL / "scripts" / "render" / "assets")
|
||||
a = ap.parse_args()
|
||||
|
||||
score = json.loads(a.score.read_text(encoding="utf-8"))
|
||||
template = a.template.read_text(encoding="utf-8")
|
||||
out = a.out or a.score.with_name("得分.html")
|
||||
html = render(score, template, a.assets, a.score.name)
|
||||
out.write_text(html, encoding="utf-8")
|
||||
print(f"OK: {out}({len(html)} 字符,logo/字体"
|
||||
f"{'已内嵌' if asset_b64(a.assets, 'lingyao-logo.png') else '未找到,已留空'})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""评测判分脚本:抽取求解代理答案 → 比对答案键 → 输出统计与报告。
|
||||
|
||||
答案文件约定:<submissions>/<contest>_<person_id>.md,每命主末尾恰好 5 行
|
||||
`Q1: A` … `Q5: X`(X ∈ ABCD;答不出标 N 记 no_answer;行尾允许 # 注释)。
|
||||
题数以 questions/ 盲版为准(允许单命主不足 5 题,如 2021 届 P033/P034)。
|
||||
|
||||
数据源全部参数化、不依赖任何外部仓库路径:
|
||||
- --questions 盲版题目目录(枚举命主与题序)
|
||||
- --answers-key 答案键目录(person_id + 题序 + 答案字母 + 维度标签)
|
||||
- --submissions 待判答案 md 目录
|
||||
- --out-json / --out-md 结果输出
|
||||
|
||||
基线:BaziQA 官方基线(Gemini 跑分)内嵌于本脚本,口径见
|
||||
试题/BaziQA-全球算命师大赛/SOURCE.md。
|
||||
|
||||
零网络调用。用法:
|
||||
python run_eval.py [--questions <dir>] [--answers-key <dir>]
|
||||
[--submissions <dir>] [--out-json <file>] [--out-md <file>]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] # 评测/
|
||||
CONTEST_DIR = ROOT / "试题" / "BaziQA-全球算命师大赛"
|
||||
|
||||
# BaziQA 官方基线(来源见 SOURCE.md):总体 + 分届 + 分维度
|
||||
OFFICIAL_TOTAL = {"total": 200, "correct": 69, "accuracy": 34.5}
|
||||
OFFICIAL_CONTEST = {
|
||||
"contest8_2021": {"total": 40, "accuracy": 27.5},
|
||||
"contest8_2022": {"total": 40, "accuracy": 40.0},
|
||||
"contest8_2023": {"total": 40, "accuracy": 27.5},
|
||||
"contest8_2024": {"total": 40, "accuracy": 35.0},
|
||||
"contest8_2025": {"total": 40, "accuracy": 42.5},
|
||||
}
|
||||
OFFICIAL_DIM = {
|
||||
"流年运势": {"total": 37, "accuracy": 40.5},
|
||||
"感情": {"total": 49, "accuracy": 36.7},
|
||||
"六亲": {"total": 29, "accuracy": 27.6},
|
||||
"事业": {"total": 25, "accuracy": 24.0},
|
||||
"性格": {"total": 20, "accuracy": 45.0},
|
||||
"学业": {"total": 12, "accuracy": 41.7},
|
||||
"财富": {"total": 13, "accuracy": 30.8},
|
||||
"健康": {"total": 13, "accuracy": 23.1},
|
||||
"其他": {"total": 2, "accuracy": 50.0},
|
||||
}
|
||||
DIM_ORDER = ["流年运势", "感情", "六亲", "事业", "性格", "学业", "财富", "健康", "其他"]
|
||||
|
||||
ANS_RE = re.compile(r"^\s*Q([1-5])\s*[::]\s*([A-DNdn])\b", re.M)
|
||||
FILE_RE = re.compile(r"^(contest\d+_\d{4})_(.+)$")
|
||||
|
||||
|
||||
def load_questions(q_dir: Path) -> dict[str, list[dict]]:
|
||||
"""盲版题目:contest -> 命主数组(仅取 person_id 与题数,不读题面)。"""
|
||||
out = {}
|
||||
for f in sorted(q_dir.glob("contest8_*.json")):
|
||||
d = json.loads(f.read_text(encoding="utf-8"))
|
||||
out[d[0]["contest_id"]] = d[1:]
|
||||
return out
|
||||
|
||||
|
||||
def load_answer_key(k_dir: Path) -> dict[tuple[str, str], dict[int, dict]]:
|
||||
"""答案键:(contest, person_id) -> {题序: {answer, dimension}}。"""
|
||||
out = {}
|
||||
for f in sorted(k_dir.glob("contest8_*.json")):
|
||||
d = json.loads(f.read_text(encoding="utf-8"))
|
||||
contest = d[0]["contest_id"]
|
||||
for p in d[1:]:
|
||||
out[(contest, p["person_id"])] = {
|
||||
a["question_index"]: a for a in p["answers"]}
|
||||
return out
|
||||
|
||||
|
||||
def parse_answer_file(path: Path) -> dict[int, str]:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
ans: dict[int, str] = {}
|
||||
for m in ANS_RE.finditer(text):
|
||||
q = int(m.group(1))
|
||||
if q not in ans: # 同一题取首次出现
|
||||
ans[q] = m.group(2).upper()
|
||||
return ans
|
||||
|
||||
|
||||
def acc(correct: int, total: int) -> float:
|
||||
return round(correct / total * 100, 1) if total else 0.0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="评测判分:submissions 答案文件 vs 答案键")
|
||||
ap.add_argument("--questions", type=Path, default=CONTEST_DIR / "questions")
|
||||
ap.add_argument("--answers-key", type=Path, default=CONTEST_DIR / "answers")
|
||||
ap.add_argument("--submissions", type=Path,
|
||||
default=ROOT / "ai答案" / "kimi-k3" / "with-skill")
|
||||
ap.add_argument("--out-json", type=Path, default=ROOT / "eval_result.json")
|
||||
ap.add_argument("--out-md", type=Path, default=ROOT / "eval_report.md")
|
||||
a = ap.parse_args()
|
||||
|
||||
datasets = load_questions(a.questions)
|
||||
keys = load_answer_key(a.answers_key)
|
||||
|
||||
# 索引答案文件:contest -> person_id -> {q: letter}
|
||||
files: dict[tuple[str, str], dict[int, str]] = {}
|
||||
if a.submissions.is_dir():
|
||||
for f in sorted(a.submissions.glob("*.md")):
|
||||
m = FILE_RE.match(f.stem)
|
||||
if not m:
|
||||
print(f"WARN: 跳过无法解析的文件名 {f.name}", file=sys.stderr)
|
||||
continue
|
||||
files[(m.group(1), m.group(2))] = parse_answer_file(f)
|
||||
|
||||
details = []
|
||||
per_contest: dict[str, dict[str, int]] = {}
|
||||
per_dim: dict[str, dict[str, int]] = {}
|
||||
tot = cor = noans = 0
|
||||
missing: list[str] = []
|
||||
for contest, persons in datasets.items():
|
||||
cs = per_contest.setdefault(contest, {"total": 0, "correct": 0, "no_answer": 0})
|
||||
for p in persons:
|
||||
pid = p["person_id"]
|
||||
got = files.get((contest, pid))
|
||||
if got is None:
|
||||
missing.append(f"{contest}/{pid}")
|
||||
got = {}
|
||||
key = keys.get((contest, pid), {})
|
||||
for i in range(1, len(p["questions"]) + 1):
|
||||
letter = got.get(i)
|
||||
unanswered = letter in (None, "N")
|
||||
k = key.get(i, {})
|
||||
ok = (not unanswered) and letter == k.get("answer")
|
||||
dim = k.get("dimension", "未标注")
|
||||
ds = per_dim.setdefault(dim, {"total": 0, "correct": 0, "no_answer": 0})
|
||||
for s in (cs, ds):
|
||||
s["total"] += 1
|
||||
s["correct"] += ok
|
||||
s["no_answer"] += unanswered
|
||||
tot += 1
|
||||
cor += ok
|
||||
noans += unanswered
|
||||
details.append({
|
||||
"dataset": contest, "person_id": pid, "question_index": i,
|
||||
"ai_answer": letter or "N",
|
||||
"correct_answer": k.get("answer", "?"), "is_correct": ok,
|
||||
"dimension": dim,
|
||||
})
|
||||
|
||||
if missing:
|
||||
print(f"WARN: {len(missing)} 位命主缺答案文件,按全未答计:"
|
||||
+ ", ".join(missing), file=sys.stderr)
|
||||
|
||||
stats: dict = {"总体": {"total": tot, "correct": cor,
|
||||
"accuracy": acc(cor, tot), "no_answer": noans}}
|
||||
for c, s in per_contest.items():
|
||||
stats[c] = {**s, "accuracy": acc(s["correct"], s["total"])}
|
||||
dim_stats = {}
|
||||
for d in DIM_ORDER + sorted(set(per_dim) - set(DIM_ORDER)):
|
||||
if d in per_dim:
|
||||
s = per_dim[d]
|
||||
dim_stats[d] = {**s, "accuracy": acc(s["correct"], s["total"])}
|
||||
stats["分维度"] = dim_stats
|
||||
|
||||
result = {"stats": stats, "details": details}
|
||||
a.out_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out_json.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n",
|
||||
encoding="utf-8")
|
||||
|
||||
ours = stats["总体"]
|
||||
lines = [
|
||||
"# BaziQA 评测报告", "",
|
||||
"## 总览", "",
|
||||
"| 口径 | 正确/总题数 | 准确率 | 未答 |", "|---|---|---|---|",
|
||||
f"| 本次评测 | {ours['correct']}/{ours['total']} | {ours['accuracy']}% | {ours['no_answer']} |",
|
||||
f"| 官方基线(Gemini) | {OFFICIAL_TOTAL['correct']}/{OFFICIAL_TOTAL['total']}"
|
||||
f" | {OFFICIAL_TOTAL['accuracy']}% | 0 |",
|
||||
"", f"> 基线口径见 试题/BaziQA-全球算命师大赛/SOURCE.md。",
|
||||
"", "## 分届准确率", "",
|
||||
"| 届别 | 本次 | 官方 |", "|---|---|---|",
|
||||
]
|
||||
for c in sorted(per_contest):
|
||||
s = stats[c]
|
||||
off = OFFICIAL_CONTEST.get(c, {})
|
||||
lines.append(f"| {c} | {s['correct']}/{s['total']} = {s['accuracy']}%"
|
||||
f"(未答 {s['no_answer']}) | {off.get('accuracy', '—')}% |")
|
||||
lines += ["", "## 分维度准确率", "",
|
||||
"| 维度 | 本次 | 官方 |", "|---|---|---|"]
|
||||
for d, s in dim_stats.items():
|
||||
off = OFFICIAL_DIM.get(d, {})
|
||||
lines.append(f"| {d} | {s['correct']}/{s['total']} = {s['accuracy']}%"
|
||||
f"(未答 {s['no_answer']}) | {off.get('accuracy', '—')}% |")
|
||||
lines += ["", "## 错题清单(错因留空,待人工/后续填写)", "",
|
||||
"| 届别 | 命主 | 题序 | 维度 | 我答 | 正解 | 错因 |",
|
||||
"|---|---|---|---|---|---|---|"]
|
||||
for d in details:
|
||||
if not d["is_correct"]:
|
||||
lines.append(f"| {d['dataset']} | {d['person_id']} | Q{d['question_index']}"
|
||||
f" | {d['dimension']} | {d['ai_answer']} | {d['correct_answer']} | |")
|
||||
a.out_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"总准确率: {ours['correct']}/{ours['total']} = {ours['accuracy']}%"
|
||||
f"(未答 {ours['no_answer']};官方基线 {OFFICIAL_TOTAL['accuracy']}%)")
|
||||
print(f"写出 {a.out_json} / {a.out_md}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,187 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src data:; style-src 'unsafe-inline'; font-src data:; base-uri 'none'; form-action 'none'">
|
||||
<title>{{TITLE}}</title>
|
||||
<style>
|
||||
@font-face {
|
||||
font-family: 'LogoFont';
|
||||
src: url("data:font/ttf;base64,{{FONT_B64}}") format("truetype");
|
||||
font-weight: normal; font-style: normal; font-display: swap;
|
||||
}
|
||||
:root {
|
||||
/* 灵爻妙解网站浅色主题(与盘面图同一套 root.css :root.light 变量) */
|
||||
--ink: #222831;
|
||||
--muted: rgba(27, 27, 27, .52);
|
||||
--soft: rgba(27, 27, 27, .68);
|
||||
--line: rgba(193, 188, 235, .55);
|
||||
--card-line: rgba(193, 188, 235, .45);
|
||||
--accent: #dc7b28;
|
||||
--accent-2: #f97316;
|
||||
--base-grey: #b9b3d9;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0; padding: 28px 24px 32px; color: var(--ink);
|
||||
font-family: -apple-system, "PingFang SC", "Microsoft YaHei", sans-serif;
|
||||
background:
|
||||
radial-gradient(ellipse 72% 58% at 78% 20%, rgba(150, 125, 245, .18), transparent 68%),
|
||||
radial-gradient(ellipse 66% 48% at 20% 18%, rgba(112, 174, 255, .18), transparent 70%),
|
||||
radial-gradient(ellipse 82% 52% at 52% 92%, rgba(84, 210, 200, .11), transparent 72%),
|
||||
linear-gradient(135deg, #fbfcff 0%, #f5f8ff 42%, #f2edff 100%);
|
||||
background-attachment: fixed;
|
||||
}
|
||||
.page { max-width: 900px; margin: 0 auto; }
|
||||
|
||||
/* 顶部品牌区:与盘面图一致的居中品牌头 */
|
||||
.brand-head {
|
||||
display: flex; align-items: center; justify-content: center;
|
||||
gap: 14px; margin: 2px 0 20px;
|
||||
}
|
||||
.brand-logo { width: 52px; height: 52px; display: block; }
|
||||
.brand-name {
|
||||
font-family: 'LogoFont', 'Kaiti SC', 'STKaiti', 'KaiTi', serif;
|
||||
font-weight: 500; font-size: 30px; line-height: 1.25;
|
||||
color: var(--ink); letter-spacing: 1px;
|
||||
}
|
||||
.brand-slogan {
|
||||
margin-top: 3px; font-size: 11px; letter-spacing: 3.5px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.panel {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 12px;
|
||||
background: linear-gradient(135deg, rgba(255, 255, 255, 1), rgba(250, 252, 255, 1));
|
||||
box-shadow: 0 16px 38px rgba(48, 56, 96, .12);
|
||||
padding: 16px;
|
||||
}
|
||||
.card {
|
||||
background: #fff; border: 1px solid var(--card-line);
|
||||
border-radius: 8px; padding: 16px 18px; margin-bottom: 14px;
|
||||
box-shadow: 0 2px 8px rgba(48, 56, 96, .05);
|
||||
}
|
||||
.card:last-child { margin-bottom: 0; }
|
||||
|
||||
/* 头部信息 */
|
||||
.head { display: flex; align-items: center; gap: 10px; flex-wrap: wrap; }
|
||||
.title {
|
||||
font-size: 24px; font-weight: 700;
|
||||
font-family: "Kaiti SC", "STKaiti", "KaiTi", serif;
|
||||
}
|
||||
.chip {
|
||||
display: inline-block; padding: 2px 10px; border-radius: 999px;
|
||||
border: 1px solid var(--accent); color: var(--accent); font-size: 13px;
|
||||
}
|
||||
.chip.grey { border-color: var(--base-grey); color: var(--soft); }
|
||||
.meta { color: var(--muted); font-size: 13px; line-height: 1.9; margin-top: 6px; }
|
||||
.meta b { color: var(--ink); font-weight: 600; }
|
||||
|
||||
/* 大数字 */
|
||||
.hero { display: flex; align-items: flex-end; gap: 26px; flex-wrap: wrap; }
|
||||
.big {
|
||||
font-size: 64px; font-weight: 700; line-height: 1;
|
||||
font-family: "Kaiti SC", "STKaiti", "KaiTi", serif;
|
||||
color: var(--accent);
|
||||
}
|
||||
.big small { font-size: 26px; color: var(--accent-2); }
|
||||
.hero .sub { font-size: 14px; color: var(--soft); line-height: 1.9; padding-bottom: 6px; }
|
||||
.delta { font-weight: 700; }
|
||||
.delta.up { color: #2e7d32; }
|
||||
.delta.down { color: #c62828; }
|
||||
|
||||
/* 维度条形 */
|
||||
.sect { font-size: 15px; font-weight: 700; margin: 2px 0 10px;
|
||||
font-family: "Kaiti SC", "STKaiti", "KaiTi", serif; }
|
||||
.sect .note { font-size: 12px; color: var(--muted); font-weight: 400; margin-left: 8px; }
|
||||
.legend { font-size: 12px; color: var(--muted); margin: 0 0 10px; }
|
||||
.legend i {
|
||||
display: inline-block; width: 22px; height: 8px; border-radius: 4px;
|
||||
vertical-align: middle; margin: 0 5px 1px 12px;
|
||||
}
|
||||
.legend i.ours { background: linear-gradient(90deg, var(--accent), var(--accent-2)); }
|
||||
.legend i.base { background: var(--base-grey); }
|
||||
.dimrow { display: flex; align-items: center; gap: 10px; margin: 7px 0; }
|
||||
.dimname { width: 64px; text-align: right; font-size: 13px; color: var(--soft);
|
||||
flex: none; }
|
||||
.dimtrack { flex: 1; }
|
||||
.bar { height: 10px; border-radius: 5px; }
|
||||
.bar.ours { background: linear-gradient(90deg, var(--accent), var(--accent-2)); }
|
||||
.bar.base { background: var(--base-grey); height: 4px; margin-top: 3px; opacity: .8; }
|
||||
.dimval { width: 176px; flex: none; font-size: 12px; color: var(--soft);
|
||||
white-space: nowrap; }
|
||||
.dimval b { color: var(--ink); font-size: 13.5px; }
|
||||
|
||||
/* 分届表 */
|
||||
table.contest { width: 100%; border-collapse: collapse; }
|
||||
.contest th, .contest td {
|
||||
border: 1px solid var(--card-line); text-align: center;
|
||||
padding: 7px 4px; font-size: 13.5px;
|
||||
}
|
||||
.contest th { background: rgba(193, 188, 235, .18); color: var(--soft); font-weight: 600; }
|
||||
.contest td.dim { text-align: left; padding-left: 12px; }
|
||||
|
||||
.foot {
|
||||
margin-top: 14px; padding-top: 10px; font-size: 12px; color: var(--muted);
|
||||
border-top: 1px dashed var(--card-line); line-height: 1.9;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="page">
|
||||
<div class="brand-head">
|
||||
<img class="brand-logo" src="data:image/png;base64,{{LOGO_B64}}" alt="灵爻妙解">
|
||||
<div>
|
||||
<div class="brand-name">灵爻妙解</div>
|
||||
<div class="brand-slogan">传承易学智慧</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="card">
|
||||
<div class="head">
|
||||
<span class="title">AI 评测记分卡</span>
|
||||
<span class="chip">{{MODEL}}</span>
|
||||
<span class="chip">{{SKILL_MODE_LABEL}}</span>
|
||||
<span class="chip grey">{{BENCHMARK}}</span>
|
||||
</div>
|
||||
<div class="meta">
|
||||
技能版本 <b>{{SKILL_VERSION}}</b> ・ 评测日期 <b>{{EVAL_DATE}}</b> ・
|
||||
判分 <b>run_eval.py 脚本判分</b>(非模型裁判)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="hero">
|
||||
<div class="big">{{OVERALL_ACC}}<small>%</small></div>
|
||||
<div class="sub">
|
||||
总体准确率 <b>{{OVERALL_CORRECT}}/{{OVERALL_TOTAL}}</b>(未答 {{NO_ANSWER}})<br>
|
||||
{{BASELINE_NAME}} {{BASELINE_ACC}}%
|
||||
<span class="delta {{DELTA_CLASS}}">{{DELTA_TEXT}}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="sect">分维度准确率<span class="note">九维度,条下灰线为官方基线</span></div>
|
||||
<div class="legend"><i class="ours"></i>本次评测 <i class="base"></i>官方基线</div>
|
||||
{{DIM_BARS}}
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="sect">分届准确率</div>
|
||||
<table class="contest">
|
||||
<tr><th>届别</th><th>本次</th><th>官方基线</th><th>差值</th></tr>
|
||||
{{CONTEST_ROWS}}
|
||||
</table>
|
||||
<div class="foot">
|
||||
数据源:{{SCORE_JSON}} ・ 渲染:评测/工具/render_scorecard.py<br>
|
||||
盲测纪律与复测方法见 评测/README.md 与 评测/测试规则.md。
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user