Files
health/辅行诀五脏用药法要-药性探真/05_脚本工具/02_clean_yuanwen.py
T

329 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
辅行诀五脏用药法要-药性探真 资料库 ETL 第 2 步
01_来源数据/辅行诀脏腑用药法要_古本原文_20260915.md -> 02_加工数据/
用途:把用户提供的《辅行诀脏腑用药法要》古本原文导入资料库,作为全书原文依据参考。
本阶段仍只做数据清理,不做释义抽取 / 跨库关联 / 主题分析。
清理项:
A 版式残留:全角缩进(U+3000)、行尾空格、词间半角空格
B 校勘符号规范化:\\[X\\] -> 〔X〕(Markdown 转义残留 -> 传统校补号)
C 讹字/衍字修正:每条附核校依据(参校本=维基文库本;内证=本书条文自证)
D 标点规范化:重复标点、缺失括号、
E 存疑登记:原文不改,逐条附疑正与参校本异文
F 版本差异/缺文登记:与参校本结构不同处,不径改
输出:
02_加工数据/辅行诀脏腑用药法要_清洗版.md
02_加工数据/原文清洗报告.md (核对数据按篇/条列明)
02_加工数据/原文人工核对清单.md (按篇列明存疑 + 缺文与版本差异登记)
02_加工数据/原文结构_上下卷.json (卷 / 篇 / 方 三级 + 行号锚定)
幂等:可重复运行,输出覆盖生成。
"""
import os, re, json, hashlib, sys
BASE = os.path.expanduser("~/Documents/ai_agent_scraper_study/data/辅行诀五脏用药法要-药性探真")
SRC = os.path.join(BASE, "01_来源数据/辅行诀脏腑用药法要_古本原文_20260915.md")
REF_DIR = os.path.join(BASE, "01_来源数据/校核参考")
OUT_MD = os.path.join(BASE, "02_加工数据/辅行诀脏腑用药法要_清洗版.md")
OUT_REPORT = os.path.join(BASE, "02_加工数据/原文清洗报告.md")
OUT_CHECK = os.path.join(BASE, "02_加工数据/原文人工核对清单.md")
OUT_STRUCT = os.path.join(BASE, "02_加工数据/原文结构_上下卷.json")
REF_NAME = "维基文库本《辅行诀脏腑用药法要》"
# C 类:讹字 / 衍字修正(依据=参校本 + 内证;逐条留痕)
FIXES = [
("谨钭五脏", "谨将五脏", f"{REF_NAME}作「谨将」;钭/将形近致误"),
("耳有所闻", "耳无所闻", f"{REF_NAME}作「耳无所闻」;《素问·脏气法时论》「虚则目䀮䀮无所见,耳无所闻」"),
("大枣(十二个", "大枣(十二枚", f"{REF_NAME}作「十二枚」;大枣量词作枚"),
("薯蓣,;当从", "薯蓣,当从", "衍分号"),
("病心中前", "病心中痛", f"{REF_NAME}作「病心中痛」;《灵枢·五邪》「邪在心,则病心痛」"),
("崦调之", "而调之", f"{REF_NAME}作「而调之」;崦/而形近致误"),
("以白浆一斗煮取四或", "以白浆一斗煮取四升", f"{REF_NAME}作「煮取四升」;或/升形近致误(白浆/白酨浆之异文见存疑)"),
("每服二或", "每服二升", f"{REF_NAME}作「每服二升」;或/升形近致误"),
("眩中澹澹大动", "心中澹澹大动", f"{REF_NAME}作「心中澹澹大动」;眩/心形近致误"),
("以不八升", "以水八升", f"{REF_NAME}作「以水八升」;不/水形近致误"),
("怔惊不发者", "怔惊不安者", f"{REF_NAME}作「不安」;与下条「懊憹不安」同例"),
("懊憹不发", "懊憹不安", f"{REF_NAME}作「懊憹不安」;发/安形近致误"),
("殓泄", "飧泄", f"{REF_NAME}作「飧泄」;《素问》「脾实则腹满,飧泄」"),
("煮取二勤务员,温升再服", "煮取二升,温分再服", f"{REF_NAME}作「煮取二升,温分再服,日二」;OCR混入「勤务员」衍文,升/分互讹"),
("治烦热汗邮", "治烦热汗出", f"{REF_NAME}作「汗出」;邮/出形近致误"),
("取之勇泉", "取之涌泉", f"{REF_NAME}作「涌泉」;《灵枢·五邪》「取之涌泉、昆仑」"),
("肾甘燥", "肾苦燥", f"{REF_NAME}作「肾苦燥」;与「肝苦急、心苦缓、脾苦湿、肺苦气上逆」同例"),
("时足胫有者方", "时足胫肿者方", f"{REF_NAME}作「时足胫肿」;有/肿形近致误"),
("加伏龙肝如鸡子在", "加伏龙肝如鸡子大", f"{REF_NAME}作「如鸡子大」;在/大形近致误"),
("渐分四服", "温分四服", f"{REF_NAME}作「温分四服」;煎服法常语作「温分」"),
("止方五味", "上方五味", f"{REF_NAME}作「上方五味」;止/上形近致误"),
("以水五升,称煮椒", "以水五升,先煮椒", f"{REF_NAME}作「先煮椒」;称/先形近致误"),
("韭叶、区药", "韭叶、芍药", f"{REF_NAME}作「韭叶、芍药」;区/芍形近致误"),
("温分三他", "温分三服", f"{REF_NAME}作「温分三服」;他/服形近致误"),
("渐分三他", "温分三服", f"{REF_NAME}作「温分三服」;渐/温、他/服并误"),
("羸唐如柴", "羸瘦如柴", f"{REF_NAME}作「羸瘦如柴」;唐/瘦形近致误"),
("竹味(三把)", "竹叶(三把)", f"{REF_NAME}作「竹叶」;味/叶形近致误"),
("竹味为水", "竹叶为水", f"{REF_NAME}作「竹叶为水」;味/叶形近致误"),
("五茶为充", "五菜为充", f"{REF_NAME}作「五菜为充」;《素问·脏气法时论》「五菜为充」"),
("五畜为益,,尔乃", "五畜为益,尔乃", "衍顿号"),
("补有汤内加羊肝", "补肝汤内加羊肝", f"{REF_NAME}作「补肝汤内加羊肝」;有/肝形近致误"),
("亦百十十首", "亦百二十首", f"{REF_NAME}作「亦百二十首」;十/二形近致误"),
("可黩契经方", "可默契经方", f"{REF_NAME}作「可默契经方」;黩/默形近致误"),
("区药为土", "芍药为土", f"{REF_NAME}作「芍药为土」;区/芍形近致误"),
("身热之自愈也", "身热去自愈也", f"{REF_NAME}作「身热去自愈」;之/去形近致误"),
("黄芩(五两)人参桂枝生姜", "黄耆(五两)人参桂枝生姜", f"{REF_NAME}作「黄耆五两」;本篇末「阳旦者,升阳之方,以黄芪为主」自证"),
("煮取四或,去滓", "煮取四升,去滓", f"{REF_NAME}作「煮取四升」;或/升形近致误"),
("一斗二升,,煮取六升", "一斗二升,煮取六升", "衍顿号"),
("治天行热痛", "治天行热病", f"{REF_NAME}作「治天行热病」;本书他篇亦作「天行热病」"),
("皂角刮去皮絃", "皂角刮去皮弦", f"{REF_NAME}作「皮弦」;絃/弦异体,取通行字"),
("鸡子黄二枚)", "鸡子黄(二枚)", "夺前括号,据下文「阿胶(三锭)」例补"),
("内胶,列上火", "内胶,更上火", f"{REF_NAME}作「更上火」;列/更形近致误"),
("蠃瘦如柴", "羸瘦如柴", f"{REF_NAME}作「羸瘦如柴」;蠃/羸形近致误"),
("更上炎", "更上火", f"{REF_NAME}作「更上火」;炎/火形近致误"),
("附了(一枚", "附子(一枚", f"{REF_NAME}作「附子」;了/子形近致误"),
("少腹中冷嘲热讽", "少腹中冷", f"{REF_NAME}作「少腹中冷」;「嘲热讽」为成语混入之衍文"),
("日三夜一他", "日三夜一服", f"{REF_NAME}作「日三、夜一服」;他/服形近致误"),
("以鸡了黄为主", "以鸡子黄为主", f"{REF_NAME}作「鸡子黄」;了/子形近致误"),
("手足遂冷", "手足逆冷", f"{REF_NAME}作「手足逆冷」;遂/逆形近致误"),
("扑克其人", "看其人", f"{REF_NAME}作「看其人」;扑克/看形近致误"),
("得大吐鲁番即愈", "得大吐即愈", f"{REF_NAME}作「得大吐即愈」;「鲁番」为衍文"),
("少时小便通,愈)", "少时小便通,立愈)", f"{REF_NAME}有「立」字;夺文"),
(r"枚,炮\]干姜", "枚,炮)干姜", f"原文「\\]」为「)」之误;{REF_NAME}作「附子(一枚,炮)干姜」"),
("《《金匮要略》", "《金匮要略》", "书名号重复"),
]
# E/F 类:存疑与版本差异(不改原文)
SUSPECT_RULES = [
(r'白浆', "参校本作「白酨浆」,本书及集成网本作「白浆」,本书校注亦称「白浆」——异文待定"),
(r'()()为水', "二十五味药精「味辛」条缺字,参校本作「附子(一本此二字不明」;本书缺字未标"),
(r'视有余者尽取之', "参校本作「视有馀血者尽取之」,疑夺「血」字(《灵枢·五邪》「视有血者尽取之」)"),
(r'上七味,以水一斗,温服一升', "参校本亦同,疑夺「煮取」句(如「煮取四升」)"),
(r'绞去滓,顿。', "疑夺「服」字(参校本作「顿服」)"),
(r'日数传,生死', "参校本作「日数传变」,疑夺「变」字"),
(r'耎意为祛弱', "本书校注;疑当作「耎意为软弱」(《说文》「耎,弱也」)"),
(r'气结\n', "泻肺汤方证末夺「方:」(参校本作「气结方。」)"),
(r'汗出不止\n', "泻肾汤方证末夺「方:」(参校本作「汗出不止方。」)"),
(r'鸡子黄二枚)阿胶|黄连(四两)芍药(各二两)', "小朱鸟汤、大朱鸟汤药味疑夺「黄芩」(本方煎法作「先煮连、芩、芍三物」;参校本有黄芩)"),
(r'^弘景曰:“外感天行', "此段起首引号未收(参校本此段不用引号);原文不改,待核影印本"),
]
# F 类:结构缺题/版本差异(登记,不改)
VERSION_NOTES = [
("篇题缺失", "「辨心包络病证文并方」篇题(参校本有;本书与集成网本皆无,心胞条文直接接于心脏病证篇后)"),
("篇题缺失", "「五脏泻方」篇题(本书以「陶曰:又有泻方五首」起;参校本有篇题)"),
("篇题缺失", "「二旦六神大小汤」篇题(本书直接以「弘景曰:外感天行……」起;参校本有篇题)"),
("篇题缺失", "「救五脏中恶卒死方」篇题(本书以「陶经隐居云:中恶卒死者……」起;参校本有篇题)"),
("编排差异", "心胞补方编排:本书「小补心汤」=治胸痹心中痞满(栝蒌薤白半夏厚朴桂枝);参校本作「大补心汤」,另立「小补心汤」=治胸痹不得卧(栝蒌薤白半夏)"),
("内容节略", "大泻心汤煎服法节略:参校本有「上六味,以酢六升,先煮上五味,得三升许,去滓。内戎盐…」全法,本书仅存「上六味,服一升,当大吐……」"),
("图版缺失", "《汤液经法图》及其图注(「左阳进为补,其数七,火数也;右阴退为泻,其数六,水数也」)本书无图无注;参校本存图(SVG)与图注"),
("异体不校", "着/著(着舌、着散、着汤)、纳/内、郄/郤、憹/𢙐、芪/耆 等异体字,两本互见,本阶段不校"),
]
def clean(text):
lines = text.split("\n")
events, out = [], []
for idx, raw in enumerate(lines, start=1):
line = raw.replace("\u3000", "").rstrip() # A 全角缩进
if line != "" and not re.match(r'^梁[·・]?华阳隐居', line):
prev = None
while prev != line:
prev = line
line = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', line)
line = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[,。、;:?!“”()《》])', '', line)
line = re.sub(r'(?<=[,。、;:?!“”()《》])\s+(?=[\u4e00-\u9fff])', '', line)
# B 校勘符号
b = re.sub(r'\\\[([^\]\\]*)\\\]', r'〔\1〕', line)
if b != line:
events.append(dict(行=idx, 类型="B校勘符号", 原="\\[…\\]", 改="〔…〕",
依据="Markdown 转义残留 → 传统校补号"))
line = b
# C 讹字
for bad, good, why in FIXES:
if bad != good and bad in line:
events.append(dict(行=idx, 类型="C讹字修正", 原=bad, 改=good, 依据=why))
line = line.replace(bad, good)
# D 标点
d = re.sub(r'(,,|;;|。。)', lambda m: m.group(1)[0], line)
if d != line:
events.append(dict(行=idx, 类型="D标点", 原="重复标点", 改="单一标点", 依据="重复标点衍文"))
line = d
out.append(line)
cleaned = re.sub(r'\n{3,}', '\n\n', "\n".join(out)).strip() + "\n"
# E 存疑登记(在清洗后文本上定位)
suspects = []
clines = cleaned.split("\n")
for i, l in enumerate(clines, start=1):
for pat, why in SUSPECT_RULES:
if re.search(pat, l):
suspects.append({"行": i, "片段": l[:60], "说明": why})
seen, uniq = set(), []
for s in suspects:
k = (s["行"], s["说明"])
if k not in seen:
seen.add(k); uniq.append(s)
return cleaned, events, uniq
SECTIONS = [
(r'^辅行诀脏腑用药法要$', "上卷·总题"),
(r'^梁·华阳隐居', "上卷·撰者"),
(r'^辨肝脏病证文并方', "上卷·辨肝脏病证文并方"),
(r'^辨心脏病证文并方', "上卷·辨心脏病证文并方"),
(r'^心胞气实者', "上卷·心胞病证文(篇题缺)"),
(r'^辨脾脏病证文并方', "上卷·辨脾脏病证文并方"),
(r'^辨肺脏病证候文并方', "上卷·辨肺脏病证候文并方"),
(r'^辨肾脏病证文并方', "上卷·辨肾脏病证文并方"),
(r'^陶曰:又有泻方五首', "上卷·五脏泻方(篇题缺)"),
(r'^陶云:经方有救诸劳损病方', "下卷·救五脏诸劳损病方(引)"),
(r'^辅行诀脏腑用药法要(下)', "下卷·总题"),
(r'^养生补肝汤', "下卷·救五脏诸劳损病方"),
(r'^陶云:经云:毒药攻邪', "下卷·汤液经法(引)"),
(r'^味辛皆属木', "下卷·汤液经法(二十五味药精)"),
(r'^弘景曰:“外感天行|^弘景曰:外感天行', "下卷·二旦六神大小汤(篇题缺)"),
(r'^陶经隐居云:中恶卒死者', "下卷·救五脏中恶卒死方(篇题缺)"),
(r'^校注:$', "下卷·校注"),
]
def parse_struct(text):
lines = text.split("\n")
nodes = []
for i, l in enumerate(lines, start=1):
s = l.strip()
for pat, name in SECTIONS:
if re.match(pat, s):
nodes.append({"篇": name, "起始行": i})
break
for n, node in enumerate(nodes):
node["结束行"] = nodes[n + 1]["起始行"] - 1 if n + 1 < len(nodes) else len(lines)
node["条文数"] = sum(1 for i in range(node["起始行"], node["结束行"] + 1)
if lines[i - 1].strip())
return nodes
def main():
raw = open(SRC, encoding="utf-8").read()
cleaned, events, suspects = clean(raw)
nodes = parse_struct(cleaned)
def wf(path, content):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(content); f.flush(); os.fsync(f.fileno())
wf(OUT_MD, cleaned)
struct = {
"来源文件": os.path.basename(SRC), "来源文件md5": hashlib.md5(raw.encode()).hexdigest(),
"清洗版文件": os.path.basename(OUT_MD), "清洗版md5": hashlib.md5(cleaned.encode()).hexdigest(),
"参校本": REF_NAME,
"说明": "行号均指清洗版;本文件仅作清理配套定位索引,未做释义抽取与跨库关联",
"篇": nodes,
}
wf(OUT_STRUCT, json.dumps(struct, ensure_ascii=False, indent=2) + "\n")
def owner(line):
for n in nodes:
if n["起始行"] <= line <= n["结束行"]:
return n["篇"]
return "(篇外)"
per = {}
for e in events:
per.setdefault(owner(e["行"]), []).append(e)
per_sus = {}
for s in suspects:
per_sus.setdefault(owner(s["行"]), []).append(s)
cnt = lambda t: sum(1 for e in events if e["类型"].startswith(t))
rep = ["# 《辅行诀脏腑用药法要》古本原文 — 清洗报告", "",
"> 本库现阶段只做数据清理。本文本作为全书**原文依据参考**入库,未做释义抽取、"
"跨库关联与主题分析(用户 2026-09-15 指令)。", "",
"## 0 概览", "",
f"- 来源:`01_来源数据/{os.path.basename(SRC)}`(md5 `{struct['来源文件md5']}`)",
f"- 清洗版:`02_加工数据/{os.path.basename(OUT_MD)}`(md5 `{struct['清洗版md5']}`)",
f"- 参校本:{REF_NAME}(另存 `01_来源数据/校核参考/`,仅作清理核校用)",
f"- 字符数:{len(raw)} → {len(cleaned)};行数:{raw.count(chr(10))} → {cleaned.count(chr(10))}",
f"- 结构:上卷 + 下卷,共 {len(nodes)} 个可识别篇段", "",
"| 清理类别 | 处数 | 说明 |", "|---------|------|------|",
f"| A 版式残留 | 全量 | 全角缩进({sum(1 for l in raw.split(chr(10)) if l.startswith(chr(0x3000)))} 行)、行尾空格、词间半角空格 |",
f"| B 校勘符号规范化 | {cnt('B')} | `\\[X\\]` → `〔X〕`(补字保留语义,只去 Markdown 转义)|",
f"| C 讹字/衍字修正 | {cnt('C')} | 每条附核校依据(参校本 + 内证),见下方逐篇明细 |",
f"| D 标点规范化 | {cnt('D')} | 重复标点、缺失括号 |",
f"| E 存疑(未改原文) | {len(suspects)} | 逐条附疑正与参校本异文 |",
f"| F 版本差异/缺文(未改) | {len(VERSION_NOTES)} | 结构、编排、图版差异,不作校改 |", "",
"### 各篇清理量一览", "",
"| 篇 | 行范围 | 条文行 | B | C | D | E 存疑 |", "|----|--------|--------|---|---|---|--------|"]
for n in nodes:
evs = per.get(n["篇"], [])
c = lambda t: sum(1 for e in evs if e["类型"].startswith(t))
rep.append(f"| {n['篇']} | {n['起始行']}–{n['结束行']} | {n['条文数']} | {c('B')} | {c('C')} | {c('D')} | "
f"{len(per_sus.get(n['篇'], []))} |")
rep += ["", "---", "", "## 1 逐篇核对明细(按卷 / 篇列明)", ""]
for n in nodes:
evs = per.get(n["篇"], [])
sus = per_sus.get(n["篇"], [])
rep += [f"### {n['篇']}(清洗版行 {n['起始行']}–{n['结束行']})", "",
f"- 清理量:B {sum(1 for e in evs if e['类型'].startswith('B'))} / "
f"C {sum(1 for e in evs if e['类型'].startswith('C'))} / "
f"D {sum(1 for e in evs if e['类型'].startswith('D'))} / 存疑 {len(sus)}", ""]
if evs:
rep += ["| 行 | 类型 | 原文 | 处理后 | 核校依据 |", "|----|------|------|--------|---------|"]
for e in evs:
rep.append(f"| {e['行']} | {e['类型']} | {e['原'].replace('|','|')} | "
f"{e['改'].replace('|','|')} | {e['依据']} |")
rep.append("")
if sus:
rep += ["存疑项:", "", "| 行 | 片段 | 说明 |", "|----|------|------|"]
for s in sus:
rep.append(f"| {s['行']} | {s['片段'].replace('|','|')} | {s['说明']} |")
rep.append("")
if not evs and not sus:
rep += ["- 本篇无需修正条目。", ""]
rep += ["---", "", "## 2 版本差异 / 缺文登记(不校改)", "",
"| 类别 | 说明 |", "|------|------|"]
for k, v in VERSION_NOTES:
rep.append(f"| {k} | {v} |")
rep += ["", "## 3 篇段结构(清洗版行号)", "",
"| 卷·篇 | 起始行 | 结束行 | 条文行 |", "|-------|--------|--------|--------|"]
for n in nodes:
rep.append(f"| {n['篇']} | {n['起始行']} | {n['结束行']} | {n['条文数']} |")
wf(OUT_REPORT, "\n".join(rep) + "\n")
chk = ["# 原文人工核对清单 — 《辅行诀脏腑用药法要》古本原文", "",
"> 用途:需对照纸质原书/敦煌本影印本确认的条目。**按卷/篇列明**,本阶段不改动清洗版正文。",
"> 参校本:维基文库本《辅行诀脏腑用药法要》(存 `01_来源数据/校核参考/`)。", ""]
for n in nodes:
sus = per_sus.get(n["篇"], [])
chk += [f"## {n['篇']}(行 {n['起始行']}–{n['结束行']})", ""]
if sus:
chk += ["| 行 | 片段 | 疑正 / 参校本异文 |", "|----|------|------------------|"]
for s in sus:
chk.append(f"| {s['行']} | {s['片段'].replace('|','|')} | {s['说明']} |")
else:
chk.append("- 无存疑项。")
chk.append("")
chk += ["---", "", "## 版本差异 / 缺文登记(不校改,待核影印本)", "",
"| 类别 | 说明 |", "|------|------|"]
for k, v in VERSION_NOTES:
chk.append(f"| {k} | {v} |")
chk += ["", "## 书目与来源", "",
"- 文本:《辅行诀脏腑用药法要》(梁·华阳隐居 陶弘景撰;敦煌遗书系传本)",
"- 本次来源:`01_来源数据/辅行诀脏腑用药法要_古本原文_20260915.md`(用户 2026-09-15 提供)",
"- 参校本:维基文库本(另有集成网本,与本来源同系,讹字位置一致)",
"- 参校本快照:`01_来源数据/校核参考/`"]
wf(OUT_CHECK, "\n".join(chk) + "\n")
print("清洗版:", OUT_MD, os.path.getsize(OUT_MD), "bytes")
print("结构 :", OUT_STRUCT, os.path.getsize(OUT_STRUCT), "bytes")
print("报告 :", OUT_REPORT, os.path.getsize(OUT_REPORT), "bytes")
print("核对单:", OUT_CHECK, os.path.getsize(OUT_CHECK), "bytes")
print(f"B {cnt('B')} | C {cnt('C')} | D {cnt('D')} | 存疑 {len(suspects)} | 版本差异 {len(VERSION_NOTES)}")
for n in nodes:
evs = per.get(n["篇"], [])
print(f" {n['篇']} 行{n['起始行']}–{n['结束行']} 条文{n['条文数']} 清理项 {len(evs)}")
if __name__ == "__main__":
main()