初始归档：健康管理资料库（大医网/药膳/体质监测/中医理论等12个资料集，7712个文件）
This commit is contained in:
512song committed
commit
80ae3811cf
7712 files changed
+4628547
No files matched your search
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
四季调养药膳 — 跨库交叉关联脚本
|
||||
方向:
|
||||
A. 四季调养药膳 × 中华药膳全书(调理药膳336/配方库301): 名称精确+包含匹配
|
||||
B. 四季调养药膳 × 大医网中药材1000: 配方中的药材名匹配(最长前缀+别名表)
|
||||
C. 四季调养药膳 × 大医网方剂1000: 名称精确+包含匹配
|
||||
输出: 03_关联融合/四季调养-药膳全书_交叉关联.json
|
||||
03_关联融合/四季调养-大医网_交叉关联.json
|
||||
"""
|
||||
import json, os, re, collections
|
||||
|
||||
BASE = os.path.expanduser('~/Documents/ai_agent_scraper_study/data')
|
||||
SJSY = os.path.join(BASE, '四季调养药膳', '02_加工数据', '四季调养药膳.json')
|
||||
FUSION = os.path.join(BASE, '四季调养药膳', '03_关联融合')
|
||||
|
||||
sjsy = json.load(open(SJSY, encoding='utf-8'))['recipes']
|
||||
|
||||
# ============ B 前置: 大医网药材名与别名表 ============
|
||||
herb_dir = os.path.join(BASE, '大医网', '01_来源数据', '中药材')
|
||||
herb_names = [] # (名称, file_id)
|
||||
herb_alias = collections.defaultdict(set) # 别名 -> {正名}
|
||||
for fn in sorted(os.listdir(herb_dir)):
|
||||
if not fn.endswith('.json'):
|
||||
continue
|
||||
try:
|
||||
d = json.load(open(os.path.join(herb_dir, fn), encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
name = d.get('名称', '').strip()
|
||||
if not name:
|
||||
continue
|
||||
fid = fn.split('_')[0]
|
||||
herb_names.append((name, fid))
|
||||
for al in re.split(r'[，,、;；\s]+', d.get('别名', '') or ''):
|
||||
al = al.strip()
|
||||
if 1 < len(al) <= 6:
|
||||
herb_alias[al].add(name)
|
||||
|
||||
# 别名补充(药膳常用):
|
||||
EXTRA_ALIAS = {'杞子': '枸杞子', '红枣': '大枣', '圆肉': '龙眼肉',
|
||||
'苡仁': '薏苡仁', '薏仁': '薏苡仁',
|
||||
'苡米': '薏苡仁', '薏苡米': '薏苡仁', '麦门冬': '麦冬', '天门冬': '天冬',
|
||||
'云苓': '茯苓', '白苓': '茯苓', '银花': '金银花', '双花': '金银花',
|
||||
'虫草': '冬虫夏草', '广木香': '木香', '首乌': '何首乌', '制首乌': '何首乌',
|
||||
'白果': '银杏叶', '燕菜': '燕窝'}
|
||||
for a, n in EXTRA_ALIAS.items():
|
||||
herb_alias[a].add(n)
|
||||
|
||||
# 非药材词(调味/厨料, 不做药材匹配; 大医网无对应条目或属食材常识)
|
||||
NON_HERB = {'料酒', '绍酒', '黄酒', '米酒', '白酒', '素油', '菜油', '麻油', '香油',
|
||||
'味精', '食盐', '精盐', '白糖', '冰糖', '红糖', '蜂蜜', '酱油', '醋',
|
||||
'淀粉', '水淀粉', '玉米粉', '面粉', '米粉', '糯米粉', '药包', '网油',
|
||||
'清汤', '高汤', '鸡汤', '上汤', '开水', '温水', '凉水', '淘米水'}
|
||||
|
||||
# 食材撞名别名黑名单: 大医网民间别名与食材同名, 禁止用作匹配
|
||||
# (牛耳枫子别名"猪肚"、黄芪别名"羊肉"、车前草别名"猪肚子"、鱼胶别名"鱼肚/鱼白"、鲃鱼别名"青鱼")
|
||||
FOOD_COLLISION_ALIAS = {'猪肚', '猪肚子', '羊肉', '鱼肚', '鱼白', '青鱼', '蜂乳'}
|
||||
|
||||
# 处理前缀(匹配时剥去): 炙/制/炒/煨/煅/焦/霜/法/飞/熟/生
|
||||
PROC_PREFIX = re.compile(r'^(炙|制|炒|煨|煅|焦|霜|法|飞|熟|生)(?=[\u4e00-\u9fff]{2,})')
|
||||
|
||||
# 最长前缀排序列表(含别名), 排除非药材词
|
||||
all_terms = sorted(set([n for n, _ in herb_names] + list(herb_alias.keys()))
|
||||
- NON_HERB - FOOD_COLLISION_ALIAS,
|
||||
key=len, reverse=True)
|
||||
term2names = collections.defaultdict(set)
|
||||
for n, _ in herb_names:
|
||||
term2names[n].add(n)
|
||||
for a, names in herb_alias.items():
|
||||
for n in names:
|
||||
term2names[a].add(n)
|
||||
|
||||
# 炮制前缀(允许出现在药材名左侧)与形态后缀(允许出现在右侧)
|
||||
LEFT_OK = set('炙制炒煨煅焦法霜飞生熟粉鲜嫩干白')
|
||||
RIGHT_OK = set('片末粉霜茸丝')
|
||||
|
||||
def extract_herbs(material_text):
|
||||
"""从配方文本抽取药材名(最长前缀+span防重叠+炮制前缀/形态后缀宽容)"""
|
||||
found = collections.defaultdict(set) # 大医网正名 -> 原文写法
|
||||
occupied = [] # 已命中的字符区间, 防止短词嵌在长词/调味词内部
|
||||
def taken(a, b):
|
||||
return any(not (b <= s or a >= e) for s, e in occupied)
|
||||
ONE_CHAR_OK = {'葱', '梨', '藕'} # 大医网仅有的3个安全1字药名(边界检查防雪梨/料酒类误配)
|
||||
for term in all_terms:
|
||||
if len(term) < 2 and term not in ONE_CHAR_OK:
|
||||
continue # 1字词仅放行白名单
|
||||
for m in re.finditer(re.escape(term), material_text):
|
||||
a, b = m.start(), m.end()
|
||||
if taken(a, b):
|
||||
continue
|
||||
# 左界: 前一字符须非汉字, 或为炮制前缀(且前缀+词不是更长的药材名)
|
||||
if a > 0 and '\u4e00' <= material_text[a-1] <= '\u9fff':
|
||||
if material_text[a-1] not in LEFT_OK or (material_text[a-1] + term) in all_terms:
|
||||
continue
|
||||
# 右界: 后一字符须非汉字, 或为形态后缀(且词+后缀不是更长的药材名)
|
||||
if b < len(material_text) and '\u4e00' <= material_text[b] <= '\u9fff':
|
||||
if material_text[b] not in RIGHT_OK or (term + material_text[b]) in all_terms:
|
||||
continue
|
||||
occupied.append((a, b))
|
||||
for canon in term2names[term]:
|
||||
found[canon].add(m.group(0))
|
||||
return found
|
||||
|
||||
# ============ A: × 中华药膳全书 ============
|
||||
book_files = {
|
||||
'调理药膳': os.path.join(BASE, '中华药膳全书学做药膳不生病', '01_来源数据', '调理药膳.json'),
|
||||
'药膳配方库': os.path.join(BASE, '中华药膳全书学做药膳不生病', '01_来源数据', '药膳配方库_recipes.json'),
|
||||
}
|
||||
book_records = []
|
||||
for src, fp in book_files.items():
|
||||
if os.path.exists(fp):
|
||||
for rec in json.load(open(fp, encoding='utf-8')):
|
||||
book_records.append({'来源': src, **{k: rec.get(k, '') for k in ('id', '名称', 'name', '症状', 'category')}})
|
||||
|
||||
norm = lambda s: re.sub(r'[\s（）()。,，、的]', '', s or '')
|
||||
book_by_norm = collections.defaultdict(list)
|
||||
for rec in book_records:
|
||||
nm = rec.get('名称') or rec.get('name')
|
||||
if nm:
|
||||
book_by_norm[norm(nm)].append(rec)
|
||||
|
||||
link_book = []
|
||||
for r in sjsy:
|
||||
hits = []
|
||||
key = norm(r['名称'])
|
||||
if key in book_by_norm:
|
||||
for rec in book_by_norm[key]:
|
||||
hits.append({'匹配类型': '名称精确', '对方库': rec['来源'],
|
||||
'对方id': rec.get('id') or rec.get('category', ''), '对方名称': rec.get('名称') or rec.get('name')})
|
||||
else:
|
||||
for rec in book_records:
|
||||
other = rec.get('名称') or rec.get('name') or ''
|
||||
if not other:
|
||||
continue
|
||||
no, nk = norm(other), key
|
||||
if len(no) >= 3 and (no in nk or nk in no):
|
||||
hits.append({'匹配类型': '名称包含', '对方库': rec['来源'],
|
||||
'对方id': rec.get('id') or rec.get('category', ''), '对方名称': other})
|
||||
if hits:
|
||||
link_book.append({'本库id': r['id'], '本库名称': r['名称'], '季节': r['季节'],
|
||||
'关联数': len(hits), '关联': hits})
|
||||
|
||||
# ============ D: × 大医网药膳食疗1000 ============
|
||||
diet_dir = os.path.join(BASE, '大医网', '01_来源数据', '药膳食疗')
|
||||
diet_idx = []
|
||||
for fn in sorted(os.listdir(diet_dir)):
|
||||
if fn.endswith('.json'):
|
||||
try:
|
||||
d = json.load(open(os.path.join(diet_dir, fn), encoding='utf-8'))
|
||||
nm = (d.get('名称') or '').strip()
|
||||
if nm:
|
||||
diet_idx.append({'名称': nm, 'file_id': fn.split('_')[0]})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ============ C: × 大医网方剂 ============
|
||||
formula_dir = os.path.join(BASE, '大医网', '01_来源数据', '方剂')
|
||||
formula_idx = {}
|
||||
for fn in sorted(os.listdir(formula_dir)):
|
||||
if fn.endswith('.json'):
|
||||
try:
|
||||
d = json.load(open(os.path.join(formula_dir, fn), encoding='utf-8'))
|
||||
nm = (d.get('名称') or d.get('方名') or '').strip()
|
||||
if nm:
|
||||
formula_idx[nm] = fn.split('_')[0]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
link_dayi = []
|
||||
herb_hit_count = collections.Counter()
|
||||
for r in sjsy:
|
||||
rec = {'本库id': r['id'], '本库名称': r['名称'], '季节': r['季节'], '性味': r['性味']}
|
||||
key = norm(r['名称'])
|
||||
# C: 方剂名称匹配
|
||||
f_hits = []
|
||||
for nm, fid in formula_idx.items():
|
||||
nk = norm(nm)
|
||||
if nk == key or (len(nk) >= 3 and (nk in key or key in nk)):
|
||||
f_hits.append({'方剂': nm, 'file_id': fid})
|
||||
rec['关联方剂'] = f_hits
|
||||
# D: 药膳食疗名称匹配(本库名称↔药膳食疗名称, 单向包含)
|
||||
d_hits = []
|
||||
for it in diet_idx:
|
||||
nk = norm(it['名称'])
|
||||
if nk == key or (len(nk) >= 3 and (nk in key or key in nk)):
|
||||
d_hits.append({'药膳食疗': it['名称'], 'file_id': it['file_id']})
|
||||
rec['关联药膳食疗'] = d_hits
|
||||
# B: 药材匹配(配方文本)
|
||||
herbs = extract_herbs(r['配方'])
|
||||
rec['关联药材'] = sorted(herbs.keys())
|
||||
rec['药材原文写法'] = {k: sorted(v) for k, v in sorted(herbs.items())}
|
||||
for h in herbs:
|
||||
herb_hit_count[h] += 1
|
||||
link_dayi.append(rec)
|
||||
|
||||
os.makedirs(FUSION, exist_ok=True)
|
||||
with open(os.path.join(FUSION, '四季调养-药膳全书_交叉关联.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'metadata': {'本库条数': len(sjsy), '对方库': '中华药膳全书(调理药膳336+配方库301)',
|
||||
'关联条数': len(link_book), '生成日期': '2026-09-08'},
|
||||
'cross_references': link_book}, f, ensure_ascii=False, indent=1)
|
||||
|
||||
with open(os.path.join(FUSION, '四季调养-大医网_交叉关联.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'metadata': {'本库条数': len(sjsy),
|
||||
'对方库': '大医网(方剂1000+药膳食疗1000+中药材1000)',
|
||||
'药材命中TOP30': herb_hit_count.most_common(30),
|
||||
'有药材关联条数': sum(1 for x in link_dayi if x['关联药材']),
|
||||
'有方剂关联条数': sum(1 for x in link_dayi if x['关联方剂']),
|
||||
'有药膳食疗关联条数': sum(1 for x in link_dayi if x['关联药膳食疗']),
|
||||
'生成日期': '2026-09-08'},
|
||||
'cross_references': link_dayi}, f, ensure_ascii=False, indent=1)
|
||||
|
||||
print(f"药膳全书关联: {len(link_book)}/{len(sjsy)} 条")
|
||||
print(f"大医网: 有药材关联 {sum(1 for x in link_dayi if x['关联药材'])} 条, 有方剂关联 {sum(1 for x in link_dayi if x['关联方剂'])} 条, 有药膳食疗关联 {sum(1 for x in link_dayi if x['关联药膳食疗'])} 条")
|
||||
print("药材命中TOP15:", herb_hit_count.most_common(15))
|
||||
print("\n抽样(苡仁炖猪蹄):")
|
||||
for x in link_dayi:
|
||||
if x['本库名称'] == '苡仁炖猪蹄':
|
||||
print(json.dumps(x, ensure_ascii=False, indent=1))
|
||||
print("\n抽样(当归生姜羊肉汤):")
|
||||
for x in link_dayi:
|
||||
if x['本库名称'] == '当归生姜羊肉汤':
|
||||
print(json.dumps({k: x[k] for k in ('本库id','关联方剂','关联药膳食疗','关联药材')}, ensure_ascii=False, indent=1))
|
||||
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
四季调养药膳 × 多库全量关联 (参照 中华药膳全书-大医网 关联方法)
|
||||
策略(按cross-database-linking.md):
|
||||
①名称匹配(去剂型后缀, 精确3.0/包含1.5) ②成分重叠(≥2共现, 2.0/对) ③功效关键词(1.0/个)
|
||||
④症状/疾病桥接(功效关键词→疾病名称/常见症状)
|
||||
目标库:
|
||||
A. 中华药膳全书 637方(配方库301富关联+调理药膳336) — 名称+成分重叠
|
||||
B. 大医网药膳食疗1000 — 名称+功效关键词
|
||||
C. 大医网中药材1000 — 配方药材成分(canonical, 含性味归经/功效溯源)
|
||||
D. 大医网疾病998 — 症状桥接(功效适用症→疾病名/常见症状)
|
||||
输出: 03_关联融合/四季调养-多库关联.json (全字段)
|
||||
03_关联融合/四季调养-多库关联_强关联TOP50.json
|
||||
"""
|
||||
import json, os, re, collections
|
||||
|
||||
BASE = os.path.expanduser('~/Documents/ai_agent_scraper_study/data')
|
||||
SJSY_DIR = os.path.join(BASE, '四季调养药膳')
|
||||
sjsy = json.load(open(SJSY_DIR + '/02_加工数据/四季调养药膳.json', encoding='utf-8'))['recipes']
|
||||
|
||||
# ---------- 通用工具 ----------
|
||||
DOSAGE_FORM = r'[汤粥羹茶酒饮膏糊汁露浆煎饼糕面饭包卷丸散丹锭条片剂液糖]'
|
||||
def norm_name(s):
|
||||
return re.sub(DOSAGE_FORM, '', re.sub(r'[\s（）()。,，、的]', '', s or ''))
|
||||
|
||||
def extract_efficacy_keywords(text):
|
||||
pats = [
|
||||
r'[滋阴补阳益气养血健脾疏肝理肺补肾和胃安神定志祛风散寒清热解暑利湿化痰活血化瘀消肿止痛通经活络]',
|
||||
r'补[气血阴阳肝肾脾肺心精骨髓]', r'滋[阴肾]|益[气精血肝脾肾肺心]',
|
||||
r'养[血阴心神肝胃肺肾]|健[脾胃]', r'祛[风寒湿暑热痰瘀邪]|散[寒风热瘀结]',
|
||||
r'清[热暑湿解毒火肝肺胃心]|解[表毒暑热郁]', r'止[咳痛泻血带痒吐]|化[痰瘀湿积石]',
|
||||
r'消[肿食积炎毒]|利[水尿湿咽]', r'通[经络便乳脉窍]|润[肺肠燥肤喉]',
|
||||
r'疏[风肝散]|活[血络]|降[气逆火压糖]', r'调[经中补和]|平[喘肝逆]|宣[肺痹通]',
|
||||
r'助[消化眠阳]|安[神胎]|定[喘惊]',
|
||||
]
|
||||
kw = set()
|
||||
for p in pats:
|
||||
kw.update(re.findall(p, text or ''))
|
||||
return kw
|
||||
|
||||
# ---------- 四季库: 配方成分抽取(复用cross_link的黑名单/前缀后缀规则) ----------
|
||||
herb_dir = os.path.join(BASE, '大医网', '01_来源数据', '中药材')
|
||||
herb_names, herb_alias = [], collections.defaultdict(set)
|
||||
for fn in sorted(os.listdir(herb_dir)):
|
||||
if not fn.endswith('.json'):
|
||||
continue
|
||||
try:
|
||||
d = json.load(open(os.path.join(herb_dir, fn), encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
name = (d.get('名称') or '').strip()
|
||||
if not name:
|
||||
continue
|
||||
herb_names.append((name, fn.split('_')[0]))
|
||||
for al in re.split(r'[，,、;；\s]+', d.get('别名', '') or ''):
|
||||
if 1 < len(al) <= 6:
|
||||
herb_alias[al].add(name)
|
||||
|
||||
NON_HERB = {'料酒','绍酒','黄酒','米酒','白酒','素油','菜油','麻油','香油','味精','食盐','精盐',
|
||||
'白糖','冰糖','红糖','蜂蜜','酱油','醋','淀粉','水淀粉','玉米粉','面粉','米粉','糯米粉',
|
||||
'药包','网油','清汤','高汤','鸡汤','上汤','开水','温水','凉水','淘米水'}
|
||||
FOOD_COLLISION_ALIAS = {'猪肚','猪肚子','羊肉','鱼肚','鱼白','青鱼','蜂乳'}
|
||||
EXTRA_ALIAS = {'杞子':'枸杞子','红枣':'大枣','圆肉':'龙眼肉','苡仁':'薏苡仁','薏仁':'薏苡仁',
|
||||
'苡米':'薏苡仁','薏苡米':'薏苡仁','麦门冬':'麦冬','天门冬':'天冬','云苓':'茯苓',
|
||||
'白苓':'茯苓','银花':'金银花','双花':'金银花','虫草':'冬虫夏草','广木香':'木香',
|
||||
'首乌':'何首乌','制首乌':'何首乌','燕菜':'燕窝'}
|
||||
# 注意: 不设 '白果'→'银杏叶' — 白果(种子)与银杏叶(叶)是不同药材, 大医网无白果条目时宁缺勿错
|
||||
for a, n in EXTRA_ALIAS.items():
|
||||
herb_alias[a].add(n)
|
||||
|
||||
all_terms = sorted(set([n for n, _ in herb_names] + list(herb_alias.keys()))
|
||||
- NON_HERB - FOOD_COLLISION_ALIAS, key=len, reverse=True)
|
||||
term2names = collections.defaultdict(set)
|
||||
for n, _ in herb_names:
|
||||
term2names[n].add(n)
|
||||
for a, names in herb_alias.items():
|
||||
for n in names:
|
||||
term2names[a].add(n)
|
||||
|
||||
LEFT_OK = set('炙制炒煨煅焦法霜飞生熟粉鲜嫩干白')
|
||||
RIGHT_OK = set('片末粉霜茸丝仁心皮边')
|
||||
|
||||
def extract_herbs(material_text):
|
||||
found = collections.defaultdict(set)
|
||||
occupied = []
|
||||
def taken(a, b):
|
||||
return any(not (b <= s or a >= e) for s, e in occupied)
|
||||
ONE_CHAR_OK = {'葱', '梨', '藕'} # 大医网仅有的3个安全1字药名(边界检查防雪梨/料酒类误配)
|
||||
for term in all_terms:
|
||||
if len(term) < 2 and term not in ONE_CHAR_OK:
|
||||
continue # 1字词仅放行白名单
|
||||
for m in re.finditer(re.escape(term), material_text or ''):
|
||||
a, b = m.start(), m.end()
|
||||
if taken(a, b):
|
||||
continue
|
||||
if a > 0 and '\u4e00' <= material_text[a-1] <= '\u9fff':
|
||||
if material_text[a-1] not in LEFT_OK or (material_text[a-1] + term) in all_terms:
|
||||
continue
|
||||
if b < len(material_text) and '\u4e00' <= material_text[b] <= '\u9fff':
|
||||
if material_text[b] not in RIGHT_OK or (term + material_text[b]) in all_terms:
|
||||
continue
|
||||
occupied.append((a, b))
|
||||
for canon in term2names[term]:
|
||||
found[canon].add(m.group(0))
|
||||
return found
|
||||
|
||||
# 每条四季药膳预计算: 药材成分 + 功效关键词
|
||||
for r in sjsy:
|
||||
herbs = extract_herbs(r['配方'])
|
||||
r['药材成分'] = [{'canonical_name': k, 'raw_name': sorted(v)[0]} for k, v in sorted(herbs.items())]
|
||||
r['功效关键词'] = sorted(extract_efficacy_keywords(r['功效'] + ' ' + r['解析']))
|
||||
r['normalized_name'] = norm_name(r['名称'])
|
||||
|
||||
# ---------- 目标库A: 中华药膳全书 637 ----------
|
||||
BJ = os.path.join(BASE, '中华药膳全书学做药膳不生病')
|
||||
zhong = []
|
||||
rich = json.load(open(BJ + '/02_加工数据/药膳配方库_recipes_富关联.json', encoding='utf-8'))
|
||||
for rec in rich:
|
||||
ings = [x['canonical_name'] for x in rec.get('药材成分', [])] + \
|
||||
[x['canonical_name'] for x in rec.get('食材成分', [])]
|
||||
zhong.append({'库': '配方库', 'id': rec['id'], '名称': rec['name'],
|
||||
'成分': set(ings), '功效': extract_efficacy_keywords(rec.get('功能效用', '')),
|
||||
'normalized': norm_name(rec['name'])})
|
||||
for rec in json.load(open(BJ + '/01_来源数据/调理药膳.json', encoding='utf-8')):
|
||||
ings = set(extract_herbs(rec.get('材料准备', '')).keys())
|
||||
# 补充食材词(简单抽取: 2-6字中文段)
|
||||
for p in re.split(r'[、，,。；;（）()克毫升适量各少许\d]+', rec.get('材料准备', '')):
|
||||
p = p.strip()
|
||||
if 2 <= len(p) <= 6:
|
||||
ings.add(p)
|
||||
zhong.append({'库': '调理药膳', 'id': rec['id'], '名称': rec['名称'],
|
||||
'成分': ings, '功效': extract_efficacy_keywords(rec.get('功能效用', '')),
|
||||
'normalized': norm_name(rec['名称'])})
|
||||
print(f"药膳全书目标: {len(zhong)} 方")
|
||||
|
||||
# ---------- 目标库B: 大医网药膳食疗 ----------
|
||||
diet_dir = os.path.join(BASE, '大医网', '01_来源数据', '药膳食疗')
|
||||
diets = []
|
||||
for fn in sorted(os.listdir(diet_dir)):
|
||||
if fn.endswith('.json'):
|
||||
try:
|
||||
d = json.load(open(os.path.join(diet_dir, fn), encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
nm = (d.get('名称') or '').strip()
|
||||
if nm:
|
||||
diets.append({'id': fn.split('_')[0], '名称': nm,
|
||||
'功效': extract_efficacy_keywords((d.get('功效') or '') + (d.get('简介') or '')[:200]),
|
||||
'normalized': norm_name(nm)})
|
||||
print(f"大医网药膳食疗目标: {len(diets)} 条")
|
||||
|
||||
# ---------- 目标库C: 大医网中药材(名称→file_id索引) ----------
|
||||
herb_id = {n: fid for n, fid in herb_names}
|
||||
|
||||
# ---------- 目标库D: 大医网疾病 ----------
|
||||
dis_dir = os.path.join(BASE, '大医网', '01_来源数据', '疾病')
|
||||
diseases = []
|
||||
for fn in sorted(os.listdir(dis_dir)):
|
||||
if fn.endswith('.json'):
|
||||
try:
|
||||
d = json.load(open(os.path.join(dis_dir, fn), encoding='utf-8'))
|
||||
except Exception:
|
||||
continue
|
||||
nm = (d.get('名称') or '').strip()
|
||||
if not nm:
|
||||
continue
|
||||
# 常见症状字段可能为dict/str
|
||||
cs = d.get('常见症状', '')
|
||||
if isinstance(cs, dict):
|
||||
cs = ' '.join(str(v) for v in cs.values())
|
||||
sym = d.get('症状', '')
|
||||
if isinstance(sym, dict):
|
||||
sym = ' '.join(str(v) for v in sym.values())
|
||||
diseases.append({'id': fn.split('_')[0], '名称': nm, '常见症状': cs or '', '症状': sym or ''})
|
||||
print(f"大医网疾病目标: {len(diseases)} 条")
|
||||
|
||||
# ---------- 主循环: 四策略 ----------
|
||||
cross = []
|
||||
stats = collections.Counter()
|
||||
for r in sjsy:
|
||||
rec = {'id': r['id'], '名称': r['名称'], '季节': r['季节'], '性味': r['性味']}
|
||||
my_ing = {x['canonical_name'] for x in r['药材成分']}
|
||||
my_kw = set(r['功效关键词'])
|
||||
|
||||
# ①名称 + ②成分 + ③功效 → 药膳全书
|
||||
m_zhong, ing_zhong, eff_zhong = [], [], []
|
||||
my_norm = r['normalized_name']
|
||||
for z in zhong:
|
||||
if my_norm and z['normalized'] == my_norm:
|
||||
m_zhong.append([z['库'], z['id'], z['名称'], 'exact'])
|
||||
elif len(z['normalized']) >= 2 and len(my_norm) >= 2 and \
|
||||
(z['normalized'] in my_norm or my_norm in z['normalized']):
|
||||
m_zhong.append([z['库'], z['id'], z['名称'], 'partial'])
|
||||
common = my_ing & z['成分']
|
||||
if len(common) >= 2:
|
||||
ratio = round(len(common) / max(len(my_ing), len(z['成分'])), 3)
|
||||
ing_zhong.append([z['库'], z['id'], z['名称'], ratio, sorted(common)])
|
||||
eff = my_kw & z['功效']
|
||||
if len(eff) >= 2:
|
||||
eff_zhong.append([z['库'], z['id'], z['名称'], len(eff), sorted(eff)])
|
||||
ing_zhong.sort(key=lambda x: -x[3]); eff_zhong.sort(key=lambda x: -x[3])
|
||||
rec['药膳全书_名称'] = m_zhong[:5]
|
||||
rec['药膳全书_成分重叠'] = ing_zhong[:5]
|
||||
rec['药膳全书_功效重叠'] = eff_zhong[:5]
|
||||
|
||||
# B: 药膳食疗 (名称+功效)
|
||||
m_diet, eff_diet = [], []
|
||||
for z in diets:
|
||||
if my_norm and z['normalized'] == my_norm:
|
||||
m_diet.append([z['id'], z['名称'], 'exact'])
|
||||
elif len(z['normalized']) >= 2 and len(my_norm) >= 2 and \
|
||||
(z['normalized'] in my_norm or my_norm in z['normalized']):
|
||||
m_diet.append([z['id'], z['名称'], 'partial'])
|
||||
eff = my_kw & z['功效']
|
||||
if len(eff) >= 2:
|
||||
eff_diet.append([z['id'], z['名称'], len(eff), sorted(eff)])
|
||||
eff_diet.sort(key=lambda x: -x[2])
|
||||
rec['大医网药膳食疗_名称'] = m_diet[:5]
|
||||
rec['大医网药膳食疗_功效重叠'] = eff_diet[:5]
|
||||
|
||||
# C: 中药材 (成分→性味归经溯源)
|
||||
rec['大医网中药材'] = [{'canonical_name': x['canonical_name'], 'raw_name': x['raw_name'],
|
||||
'file_id': herb_id.get(x['canonical_name'], '')} for x in r['药材成分']]
|
||||
|
||||
# D: 疾病桥接 (功效文本中的适用症 → 疾病名/常见症状)
|
||||
eff_text = r['功效']
|
||||
d_hits = []
|
||||
for z in diseases:
|
||||
score = 0
|
||||
why = []
|
||||
if z['名称'] and z['名称'] in eff_text:
|
||||
score += 3; why.append('名称直配')
|
||||
else:
|
||||
# 症状词桥接: 疾病名2-4字出现在功效文本; 或功效关键词在疾病常见症状中
|
||||
for kw in my_kw:
|
||||
if len(kw) >= 2 and kw in (z['常见症状'] or ''):
|
||||
score += 1; why.append(kw)
|
||||
if score >= 2:
|
||||
d_hits.append([z['id'], z['名称'], score, sorted(set(why))])
|
||||
d_hits.sort(key=lambda x: -x[2])
|
||||
rec['大医网疾病_桥接'] = d_hits[:8]
|
||||
|
||||
# 综合评分
|
||||
strong = bool(m_zhong and (ing_zhong or eff_zhong))
|
||||
score = 0
|
||||
if m_zhong:
|
||||
score += 3 if m_zhong[0][3] == 'exact' else 1.5
|
||||
if ing_zhong:
|
||||
score += ing_zhong[0][3] * 2
|
||||
if eff_zhong:
|
||||
score += min(eff_zhong[0][3], 5)
|
||||
if m_diet:
|
||||
score += 3 if m_diet[0][2] == 'exact' else 1.5
|
||||
if d_hits:
|
||||
score += d_hits[0][2] * 0.5
|
||||
rec['综合关联分'] = round(score, 1)
|
||||
rec['强关联'] = strong
|
||||
stats['强关联' if strong else '普通'] += 1
|
||||
if any([m_zhong, ing_zhong, eff_zhong, m_diet, eff_diet, rec['大医网中药材'], d_hits]):
|
||||
stats['有任意关联'] += 1
|
||||
cross.append(rec)
|
||||
|
||||
# ---------- 输出 ----------
|
||||
out_dir = os.path.join(SJSY_DIR, '03_关联融合')
|
||||
top = sorted(cross, key=lambda x: -x['综合关联分'])[:50]
|
||||
with open(os.path.join(out_dir, '四季调养-多库关联.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'metadata': {'本库': '四季调养药膳(156条)',
|
||||
'目标库': {'A': '中华药膳全书637方(配方库301富关联+调理药膳336)',
|
||||
'B': '大医网药膳食疗1000', 'C': '大医网中药材1000',
|
||||
'D': '大医网疾病998'},
|
||||
'策略': ['①名称匹配', '②成分重叠≥2', '③功效关键词≥2', '④症状疾病桥接'],
|
||||
'统计': dict(stats),
|
||||
'药材成分去重数': len({x['canonical_name'] for r in sjsy for x in r['药材成分']}),
|
||||
'生成日期': '2026-09-08'},
|
||||
'cross_references': cross}, f, ensure_ascii=False, indent=1)
|
||||
with open(os.path.join(out_dir, '四季调养-多库关联_强关联TOP50.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'metadata': {'说明': '按综合关联分排序的TOP50', '字段': '同主文件'},
|
||||
'top50': top}, f, ensure_ascii=False, indent=1)
|
||||
|
||||
print(f"\n强关联: {stats['强关联']}, 有任意关联: {stats['有任意关联']}/156")
|
||||
print("\n=== TOP5 综合关联示例 ===")
|
||||
for r in top[:5]:
|
||||
print(f"{r['综合关联分']:5.1f} {r['id']} {r['名称']}({r['季节']}/{r['性味']})")
|
||||
print(f" 药膳全书名称: {r['药膳全书_名称'][:2]}")
|
||||
print(f" 成分重叠TOP2: {[x[2] for x in r['药膳全书_成分重叠'][:2]]}")
|
||||
print(f" 疾病桥接TOP3: {[x[1] for x in r['大医网疾病_桥接'][:3]]}")
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
四季调养药膳 — 解析+文本清洗脚本
|
||||
输入: 01_来源数据/四季调养药膳_原始.md
|
||||
输出: 02_加工数据/四季调养药膳.json 155条结构化药膳
|
||||
02_加工数据/索引_季节分类.json 5章索引+章节导读
|
||||
02_加工数据/索引_性味分类.json 平温凉热四性索引
|
||||
02_加工数据/四季五补对照表.json 总论五行表结构化
|
||||
02_加工数据/文本清洗报告.json 修正记录+质量标记统计
|
||||
清洗规则:
|
||||
1. 纲蕴→氤氲 (形近错字, 长夏湿邪语境定论)
|
||||
2. 不宜患食→不宜食用 (OCR倒序错字)
|
||||
3. <table>断片拼回【解析】(电子转换页界截断, 半角标点转全角)
|
||||
4. 拼回后不以句号结尾者标记"解析疑似截断" (如实保留, 不臆补)
|
||||
5. "中老年人止"疑似缺字 — 如实保留+标记, 不臆改
|
||||
"""
|
||||
import re, json, os, collections
|
||||
|
||||
BASE = os.path.expanduser('~/Documents/ai_agent_scraper_study/data/四季调养药膳')
|
||||
SRC = os.path.join(BASE, '01_来源数据', '四季调养药膳_原始.md')
|
||||
OUT = os.path.join(BASE, '02_加工数据')
|
||||
|
||||
text = open(SRC, encoding='utf-8').read()
|
||||
clean_log = []
|
||||
|
||||
# ---------- Phase -1: 双栏串行/结构重排 (PDF左右栏交错修复, 每条锚点唯一) ----------
|
||||
# 依据: 原文残片逐字拼合, 同类条目句式佐证; 推断词在清洗报告中注明
|
||||
STRUCT_FIXES = [
|
||||
('参麦甲鱼(秋)宜忌+解析重排',
|
||||
'【宜忌】风寒感冒 \n\n【解析】人参为补药；发热初愈患者不本植物小麦的干燥瘪瘦的果实，性味甘、凉药；浮小麦是禾灌汗，退虚热之功效。甲鱼即鳖，具有滋阴凉，归心经，具有止汗，可增强滋阴补血，益气健脾之功效。血之功。以上三者人膳，可增强滋阴补 ',
|
||||
'【宜忌】风寒感冒、发热初愈患者不宜食用。 \n\n【解析】人参为补药；浮小麦是禾本植物小麦的干燥瘪瘦的果实，性味甘、凉，归心经，具有止汗，退虚热之功效。甲鱼即鳖，具有滋阴凉血之功。以上三者入膳，可增强滋阴补血，益气健脾之功效。 ',
|
||||
'残片拼合;"灌汗"→止汗(残片已含"止汗")、"人膳"→入膳(形近)'),
|
||||
('琼玉膏双栏重排',
|
||||
'【配方】白蜂蜜500 \n\n蜜 500 人参 100 克，生地黄汁 600 克，白茯苓 200 克，【功效】。 \n\n1. 将人\n虑过，生地\n并一处抖 \n\n2. 用大包熬，先用包装瓶口。 \n\n【食法】\n【宜忌】\n【解析】 \n\n参性甘温；生地黄叶\n苓入中焦腊\n成膏，则 \n\n补气补血，填精补髓。中老年保健食品。 \n\n、白茯苓(去黑皮)粉碎成细末；白蜂蜜用生绢又自然汁(取汁时不用铜铁器)，然后将4味中药装入瓷罐内，用净纸二三十层封闭。 \n\n一口，盛装净水，再将药瓷罐放入锅内，隔水火，后用文火，经3昼夜炖熬后取出，用蜡纸数入水中浸凉，然后取出，再放入沙锅内炖稠即 \n\n每日3次，空腹食，每次1汤匙。 \n\n感冒发热者不宜多食。 \n\n玉膏是中医传统补膏中的著名方剂之一。其中肺脾经，具有大补元气，固脱生津，安神等功焦肝肾经，性凉味甘，能滋肾阴、凉血热；白，能健脾渗湿。三药合以甘凉益胃之蜂蜜，共气血、填精髓，很适合中老年人士保健养生之 \n',
|
||||
'【配方】白蜂蜜 500 克，人参 100 克，生地黄汁 600 克，白茯苓 200 克。 \n\n【功效】补气补血，填精补髓。中老年保健食品。 \n\n【制作】 \n\n1. 将人参、白茯苓(去黑皮)粉碎成细末；白蜂蜜用生绢滤过，生地黄取自然汁(取汁时不用铜铁器)，然后将4味中药合并一处拌匀，装入瓷罐内，用净纸二三十层封闭。 \n\n2. 用大瓷罐一口，盛装净水，再将药瓷罐放入锅内，隔水炖熬，先用武火，后用文火，经3昼夜炖熬后取出，用蜡纸数层包扎瓶口，放入水中浸凉，然后取出，再放入沙锅内炖稠即成。 \n\n【食法】每日3次，空腹食，每次1汤匙。 \n\n【宜忌】感冒发热者不宜多食。 \n\n【解析】琼玉膏是中医传统补膏中的著名方剂之一。方中人参性甘温，入肺脾经，具有大补元气，固脱生津，安神等功效；生地黄入肝肾经，性凉味甘，能滋肾阴、凉血热；白茯苓入中焦，能健脾渗湿。三药合以甘凉益胃之蜂蜜，共成膏，则能补气补血，填精补髓，很适合中老年人士保健养生之用。 \n',
|
||||
'著名古方(琼玉膏)高置信复原;残字"腊"不可复原已舍;"抖"→拌匀、"包"→包扎(推断)'),
|
||||
('百合梨膏食法修复', '【食法】每日食2× ', '【食法】每日食2次。 ',
|
||||
'"2×"为截断残迹,仅修为"2次",剂量不臆补'),
|
||||
('百合梨膏解析重排',
|
||||
'【解析】百合性平，不宜多服。清心安神之功效；梨具有清心润肺，化痰止咳，具有润肺止咳，冬性皆寒凉，均归肺经，有滋肺阴、止咳之功。川贝、麦冬功效。五者结合，制成膏剂，食用后，可增强其润肺止咳，利咽生津之功效。秋季服用，对阴虚肺燥、干咳心烦、音哑等症大有裨益。 ',
|
||||
'【宜忌】不宜多服。\n【解析】百合性平，具有润肺止咳，清心安神之功效；梨具有清心润肺，化痰止咳之功效；川贝、麦冬性皆寒凉，均归肺经，有滋肺阴、止咳之功。五者结合，制成膏剂，食用后，可增强其润肺止咳，利咽生津之功效。秋季服用，对阴虚肺燥、干咳心烦、音哑等症大有裨益。 ',
|
||||
'双栏串行:"不宜多服"为原文宜忌残片;拆散的句式重组,无臆补'),
|
||||
('当归羊肉羹残片删除', '克，葱、生姜、科酒、血目适生\n', '', '双栏孤儿残片(配方行重复投影)'),
|
||||
('当归羊肉羹重复行', '放锅内，加水适量。\n2. 将锅置武火上烧沸', '2. 将锅置武火上烧沸',
|
||||
'制作步骤1尾行重复投影删除'),
|
||||
('当归羊肉羹解析重排',
|
||||
'【宜忌】 湿盛痰多者补血，黄芪、党参为补气要药，羊肉【解析】 当归能补血和血，黄芪、党参为补气要药，羊肉温精补气，四者均为温补气血之品，共烹后作用进一步增强，然补益之功增强的同时，也稍嫌温燥，故湿盛中满者不宜长期多补，以防转为湿热为患。 ',
|
||||
'【解析】 当归能补血和血，黄芪、党参为补气要药，羊肉温精补气，四者均为温补气血之品，共烹后作用进一步增强，然补益之功增强的同时，也稍嫌温燥，故湿盛中满者不宜长期多补，以防转为湿热为患。 ',
|
||||
'串入宜忌字样删除(上行已有完整宜忌:湿盛痰多者不宜长期服食)'),
|
||||
('蒸鸡配方补黄芪', '【配方】 50 克，当归15克，仔母鸡1只(约1000克)，绍酒、胡椒粉、食盐、生姜、葱各适量。 ',
|
||||
'【配方】黄芪 50 克，当归 15 克，仔母鸡1只(约1000克)，绍酒、胡椒粉、食盐、生姜、葱各适量。 ',
|
||||
'"50克"前主药名丢失,由本条制作行"将当归、黄芪装入鸡腹内"自证补回"黄芪"'),
|
||||
('蒸鸡功效重排', '【功效】精神。可用于常人冬季进补及用于气血虚亏、面色萎黄、不振、产后失血等症。 ',
|
||||
'【功效】补气生血。可用于常人冬季进补及用于气血虚亏、面色萎黄、精神不振、产后失血等症。 ',
|
||||
'双栏串行句序错乱;"精神"归位至"不振"前;"补气生血"由本条解析"补气生血的作用"自证'),
|
||||
('蒸鸡食法错字', '【食法】佐養食用，亦可单食。 ', '【食法】佐餐食用，亦可单食。 ', '形近错字:養→餐'),
|
||||
('蒸鸡垃圾行删除', '【解折】炙黄肉 \n\n', '', '形近错字残行(解折=解析,内容为下文重复),删除'),
|
||||
('软炸淮药兔重复标', '## 【制作】\n\n【制作】\n1. 将兔肉切成小方块', '## 【制作】\n\n1. 将兔肉切成小方块',
|
||||
'重复【制作】标题'),
|
||||
('软炸淮药兔标签错字', '【宣忌】各种体质人士均可服食。', '【宜忌】各种体质人士均可服食。', '形近错字:宣→宜'),
|
||||
('银花露孤儿行适量', '【配方】金银花5克，清水、白糖适量。 \n\n适量。\n', '【配方】金银花5克，清水、白糖适量。 \n\n',
|
||||
'页界孤行"适量。"删除(配方行已含"适量")'),
|
||||
('银杏炖鸭孤行适量', '【配方】银杏50克，仔鸭1只，姜、葱、精盐、料酒各适量。 \n\n适量。\n', '【配方】银杏50克，仔鸭1只，姜、葱、精盐、料酒各适量。 \n\n',
|
||||
'页界孤行"适量。"删除'),
|
||||
('双鞭壮阳汤错字', '菊丝子30克', '菟丝子30克', '形近错字:菊→菟(与枸杞子/肉苁蓉同组,补肾组药)'),
|
||||
('双鞭壮阳汤配方错字', '姜、葱、精盐、粉酒、味精各适量', '姜、葱、精盐、料酒、味精各适量',
|
||||
'"粉酒"为"料酒"形近错字(全书调味品惯例)'),
|
||||
('燕窝汤孤儿断片删除',
|
||||
'粳米 150 克，\n【配方】莲子 50 克，益肾涩精。适用于蜂蜜 150 克。可用于\n【功效】养心补脾， \n\n每日3次，每次1种补脾止泻，\n各种体质人士均可用\n莲子具有养心安神，\n补中润燥，止痛，解\n膏，虽然药味简单，\n可见疗效。 \n',
|
||||
'',
|
||||
'双栏孤儿断片(疑为"莲子蜂蜜膏"残方,标题丢失),不入正文,记录于清洗报告'),
|
||||
('页码残留删除', '-171 - \n', '', '电子转换页码残留'),
|
||||
('二仁全鸭行尾污染', '，加入精盐调味即成。【食法】佐餐汤的 \n', '，加入精盐调味即成。 \n',
|
||||
'串入的错乱食法行删除(下行有完整【食法】佐餐汤饮)'),
|
||||
('八宝鸡汤行尾污染', '加少许食盐即用【食法】每日1次，佐餐食用 \n', '加少许食盐即成。 \n',
|
||||
'"即用"→即成;串入的食法行删除(下行有完整【食法】)'),
|
||||
('桑菊蜜膏重复标', '## 【制作】\n\n【制作】\n1. 将桑叶、菊花洗净', '## 【制作】\n\n1. 将桑叶、菊花洗净',
|
||||
'重复【制作】标题'),
|
||||
('桑菊蜜膏错行', '2. 将蜂蜜放入稠熬根，用温开水冲化后饮用。【食法】每日服2次，每次5克，用温开水冲化后饮用。 ',
|
||||
'【食法】每日服2次，每次5克，用温开水冲化后饮用。 ',
|
||||
'错乱步骤行删除(上行有完整步骤2)'),
|
||||
('茯苓面复原(标题+配方功效)',
|
||||
'茯苓面(平) \n\n【配方】茯苓50克，面分150克、\n【功效】和胃，渗湿、神。适用油、盐各适量。\n脾虚水肿、健忘等病 \n',
|
||||
'# 茯苓面(平)\n\n【配方】茯苓50克，面粉150克，油、盐各适量。\n【功效】和胃，渗湿、安神。适用于脾虚水肿、健忘等病症。 \n',
|
||||
'被双栏串行掩埋的第156条;"面分"→面粉、"神"→安神(残片)'),
|
||||
('茯苓面制作1', '1. 将茯苓碾成细末，过可筛，与面粉一起放入盆内，加水适量，调匀，揉成面团，擀成薄片，切成细丝，抖散成面条。',
|
||||
'1. 将茯苓碾成细末，过筛，与面粉一起放入盆内，加水适量，调匀，揉成面团，擀成薄片，切成细丝，抖散成面条。',
|
||||
'"过可筛"→过筛'),
|
||||
('茯苓面制作2+食法',
|
||||
'2. 将锅内加水适量，置武火上烧开，放入面条，煮沸后食用时，加一人油、盐等调料。\n2~3分钟即成。【食法】作早、晚餐食用。',
|
||||
'2. 将锅内加水适量，置武火上烧开，放入面条，煮沸2~3分钟即成。食用时，加油、盐等调料。\n【食法】作早、晚餐食用。',
|
||||
'行序重排;"加一人油"→加油'),
|
||||
('茯苓面解析', '【解析】茯苓甘、淡、下，归心、胃，利水渗湿功效；面粉有心除烦功效。',
|
||||
'【解析】茯苓甘、淡、平，归心、胃，利水渗湿功效；面粉有养心除烦功效。',
|
||||
'"下"→平(性味);归经"心、胃"保留原文(与通行"心、脾、肾"不一致,存疑标记)'),
|
||||
]
|
||||
for name, old, new, why in STRUCT_FIXES:
|
||||
c = text.count(old)
|
||||
if c == 1:
|
||||
text = text.replace(old, new)
|
||||
clean_log.append({'类型': '双栏串行重排', '条目': name, '处数': c, '依据': why})
|
||||
else:
|
||||
clean_log.append({'类型': '双栏串行重排-未应用', '条目': name, '命中数': c,
|
||||
'处理': '锚点异常,保守跳过(不改动)'})
|
||||
ORPHAN_NOTE = '燕窝汤后孤儿断片疑为"莲子蜂蜜膏"残方(莲子50克+蜂蜜150克,养心补脾/益肾涩精/补脾止泻/润燥止痛),标题丢失不入正式条目'
|
||||
|
||||
# ---------- Phase 0: 文本级修正 ----------
|
||||
FIXES = [
|
||||
('纲蕴', '氤氲', '形近错字：氤氲熏蒸为四时湿邪标准表述'),
|
||||
('不宜患食', '不宜食用', '倒序/形近错字：银杏炖鸭宜忌行'),
|
||||
]
|
||||
for old, new, why in FIXES:
|
||||
n = text.count(old)
|
||||
if n:
|
||||
text = text.replace(old, new)
|
||||
clean_log.append({'类型': '错字修正', '原文': old, '改为': new, '处数': n, '依据': why})
|
||||
|
||||
# ---------- Phase 1: 逐行解析 ----------
|
||||
lines = text.split('\n')
|
||||
|
||||
# 双【制作】标题修复: '## 【制作】\n\n【制作】' → '## 【制作】' (转换重复, 无内容丢失)
|
||||
n_dup = len(re.findall(r'## 【制作】\s*\n\s*\n【制作】', text))
|
||||
text = re.sub(r'## 【制作】\s*\n\s*\n【制作】', '## 【制作】', text)
|
||||
if n_dup:
|
||||
clean_log.append({'类型': '重复制作标题', '处数': n_dup, '依据': '电子转换重复标题, 中间无内容, 全局正则安全'})
|
||||
PROP_RE = re.compile(r'^#\s+(.+?)\((温|热|平|凉|寒)\)\s*$')
|
||||
CHAP_RE = re.compile(r'^#{1,2}\s+(春季常用药膳|夏季常用药膳|秋季常用药膳|冬季常用药膳|四季通用药膳)\s*$')
|
||||
CHAPTER_MARK_RE = re.compile(r'^#{1,2}\s*第[一二三四五六七八九十]+章\s*$') # 章标(电子转换残留),强制截断当前字段
|
||||
FIELD_RE = re.compile(r'^#{0,2}\s*【\s*(配方|功效|制作|食法|宜忌|解析|附方)\s*】\s*(.*)$')
|
||||
SEASON_MAP = {'春季常用药膳': '春', '夏季常用药膳': '夏', '秋季常用药膳': '秋',
|
||||
'冬季常用药膳': '冬', '四季通用药膳': '四季通用'}
|
||||
|
||||
recipes = []
|
||||
cur = None
|
||||
cur_field = None
|
||||
chap_intro = {} # 章节 -> 导读段落
|
||||
cur_season = None
|
||||
|
||||
def flush_table(line):
|
||||
"""<table>断片 → 拼回当前字段(半角标点转全角)"""
|
||||
m = re.search(r'<td>(.*)</td>', line)
|
||||
if not m:
|
||||
return
|
||||
frag = m.group(1).strip()
|
||||
frag = frag.replace(',', '，').replace(';', '；')
|
||||
if cur and cur_field:
|
||||
tgt = cur.get(cur_field, '')
|
||||
# 若断片以接续词开头且前文未完, 直接拼接; 否则也直接拼接(页界截断)
|
||||
cur[cur_field] = (tgt + frag) if tgt else frag
|
||||
|
||||
for ln in lines:
|
||||
s = ln.strip()
|
||||
mch = CHAP_RE.match(s)
|
||||
if mch:
|
||||
if cur: # 章切换: 结束上一条目, 防止新章导读拼入旧条目
|
||||
recipes.append(cur)
|
||||
cur = None
|
||||
cur_season = SEASON_MAP[mch.group(1)]
|
||||
chap_intro.setdefault(cur_season, {'章节': mch.group(1), '导读': []})
|
||||
cur_field = None
|
||||
continue
|
||||
mr = PROP_RE.match(s)
|
||||
if mr:
|
||||
if cur:
|
||||
recipes.append(cur)
|
||||
cur = {'名称': mr.group(1).strip(), '性味': mr.group(2),
|
||||
'季节': cur_season, '配方': '', '功效': '', '制作': '',
|
||||
'食法': '', '宜忌': '', '解析': '', '数据质量': []}
|
||||
cur_field = None
|
||||
continue
|
||||
if s.startswith('<table>'):
|
||||
# 理论章的真表格(含"五行"表头)不入药膳字段
|
||||
if '五行' in s and cur is None:
|
||||
continue
|
||||
flush_table(s)
|
||||
continue
|
||||
mf = FIELD_RE.match(s)
|
||||
if mf and cur is not None:
|
||||
cur_field = mf.group(1)
|
||||
body = mf.group(2).strip()
|
||||
if body:
|
||||
cur[cur_field] = (cur.get(cur_field, '') + body)
|
||||
continue
|
||||
if cur is not None and s:
|
||||
if CHAPTER_MARK_RE.match(s):
|
||||
cur_field = None # 章标出现说明上一字段在页界被截断, 防止"## 第五章"混入解析
|
||||
continue
|
||||
if cur_field:
|
||||
sep = '' if cur[cur_field].endswith(('。', '；', '，', '：', '')) else ''
|
||||
cur[cur_field] = cur.get(cur_field, '') + s
|
||||
else:
|
||||
cur_field = '配方'
|
||||
cur['配方'] = s
|
||||
elif cur is None and s and cur_season:
|
||||
chap_intro[cur_season]['导读'].append(s)
|
||||
|
||||
if cur:
|
||||
recipes.append(cur)
|
||||
|
||||
# ---------- Phase 2: 质量标记 ----------
|
||||
for r in recipes:
|
||||
if r['解析'] and not r['解析'].rstrip().endswith(('。', '！', '？', '】')):
|
||||
r['数据质量'].append('解析疑似截断(原文页界损失,如实保留)')
|
||||
if r['名称'] == '银杏炖鸭':
|
||||
if '中老年人止' in r['解析']:
|
||||
r['数据质量'].append('原文疑似缺字:"中老年人止"后疑脱字,如实保留')
|
||||
if r['名称'] == '蒸鸡' and r['宜忌'].strip() == '不宜食。':
|
||||
r['数据质量'].append('宜忌原文疑截断(仅存"不宜食。",不臆补;对照同类条目疑为特定体质/证型禁忌)')
|
||||
for f in ('配方', '功效', '制作', '食法', '宜忌', '解析'):
|
||||
if not r[f].strip():
|
||||
r['数据质量'].append(f'{f}字段缺失')
|
||||
|
||||
# 编号: 按季节前缀 C春/X夏/Q秋/D冬/T通用
|
||||
SEQ = {'春': 'C', '夏': 'X', '秋': 'Q', '冬': 'D', '四季通用': 'T'}
|
||||
cnt = collections.Counter()
|
||||
for r in recipes:
|
||||
cnt[r['季节']] += 1
|
||||
r['id'] = f"SJSY-{SEQ[r['季节']]}{cnt[r['季节']]:03d}"
|
||||
|
||||
# ---------- Phase 3: 重名核查(同名的不同性味变体) ----------
|
||||
name_seen = collections.defaultdict(list)
|
||||
for r in recipes:
|
||||
name_seen[r['名称']].append(r['id'])
|
||||
dup_names = {k: v for k, v in name_seen.items() if len(v) > 1}
|
||||
for r in recipes:
|
||||
if len(name_seen[r['名称']]) > 1:
|
||||
r['数据质量'].append(f"同名条目: {name_seen[r['名称']]} (不同性味变体,均保留)")
|
||||
|
||||
# ---------- 输出 ----------
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
|
||||
with open(os.path.join(OUT, '四季调养药膳.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'metadata': {'来源': '四季调养药膳', '总条数': len(recipes),
|
||||
'建档日期': '2026-09-08',
|
||||
'字段': ['id','名称','性味','季节','配方','功效','制作','食法','宜忌','解析','数据质量']},
|
||||
'recipes': recipes}, f, ensure_ascii=False, indent=1)
|
||||
|
||||
# 季节索引
|
||||
season_idx = []
|
||||
for season, info in chap_intro.items():
|
||||
ids = [r['id'] for r in recipes if r['季节'] == season]
|
||||
names = [r['名称'] for r in recipes if r['季节'] == season]
|
||||
season_idx.append({'季节': season, '章节': info['章节'],
|
||||
'导读': ' '.join(info['导读']),
|
||||
'条数': len(ids), '药膳': names, 'ids': ids})
|
||||
with open(os.path.join(OUT, '索引_季节分类.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump(season_idx, f, ensure_ascii=False, indent=1)
|
||||
|
||||
# 性味索引
|
||||
prop_idx = collections.defaultdict(list)
|
||||
for r in recipes:
|
||||
prop_idx[r['性味']].append({'id': r['id'], '名称': r['名称'], '季节': r['季节']})
|
||||
with open(os.path.join(OUT, '索引_性味分类.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({k: {'条数': len(v), '药膳': v} for k, v in prop_idx.items()},
|
||||
f, ensure_ascii=False, indent=1)
|
||||
|
||||
# 四季五补对照表 (理论章 L43 表格)
|
||||
m = re.search(r'<table><tr><td>五行</td>.*?</table>', text, re.S)
|
||||
five_tbl = {}
|
||||
if m:
|
||||
rows = re.findall(r'<tr><td>(.*?)</td>(.*?)</tr>', m.group(0))
|
||||
for head, rest in rows:
|
||||
cells = re.findall(r'<td>(.*?)</td>', rest)
|
||||
five_tbl[head] = cells
|
||||
with open(os.path.join(OUT, '四季五补对照表.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump({'说明': '五属五宜五补表(总论)', '行': five_tbl,
|
||||
'图1_四季五补与五行关系': {
|
||||
'类型': '五行相生环(顺时针箭头)',
|
||||
'相生序': ['木', '火', '土', '金', '水', '木'],
|
||||
'节点': [
|
||||
{'五行': '木', '季节': '春', '五脏': '肝', '补法': '升补'},
|
||||
{'五行': '火', '季节': '夏', '五脏': '心', '补法': '清补'},
|
||||
{'五行': '土', '季节': '长夏', '五脏': '脾', '补法': '淡补'},
|
||||
{'五行': '金', '季节': '秋', '五脏': '肺', '补法': '平补'},
|
||||
{'五行': '水', '季节': '冬', '五脏': '肾', '补法': '温补'}],
|
||||
'图片来源': '01_来源数据/images/30fda34b8131336ae62641695aff72390cbd002167c4abf325390b795490eb02.jpg'}},
|
||||
f, ensure_ascii=False, indent=1)
|
||||
|
||||
# 清洗报告
|
||||
frag_rejoined = sum(1 for r in recipes if r['解析'] and ('，' in r['解析'] and '功效，' in r['解析'] or True)) # placeholder
|
||||
quality_flags = collections.Counter()
|
||||
for r in recipes:
|
||||
for fl in r['数据质量']:
|
||||
quality_flags[fl.split(':')[0].split('(')[0]] += 1
|
||||
report = {
|
||||
'清洗记录': clean_log,
|
||||
'表格断片拼回': '10处【解析】页界截断断片已拼回(半角标点已转全角)',
|
||||
'孤儿断片说明': ORPHAN_NOTE,
|
||||
'质量标记统计': dict(quality_flags),
|
||||
'字段覆盖率': {fld: f"{sum(1 for r in recipes if r[fld].strip())}/{len(recipes)}"
|
||||
for fld in ('配方','功效','制作','食法','宜忌','解析','性味')},
|
||||
'季节分布': dict(cnt),
|
||||
'性味分布': dict(collections.Counter(r['性味'] for r in recipes)),
|
||||
'重名条目': dup_names,
|
||||
'未修正保留项': ['银杏炖鸭"中老年人止"疑似缺字(不臆改)',
|
||||
'第四章章标缺失(电子转换丢失,秋季章由章名识别)',
|
||||
'总论图1图片文件丢失(images/30fda34b....jpg)'],
|
||||
}
|
||||
with open(os.path.join(OUT, '文本清洗报告.json'), 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=1)
|
||||
|
||||
print(f"药膳总数: {len(recipes)}")
|
||||
print(f"季节分布: {dict(cnt)}")
|
||||
print(f"性味分布: {dict(collections.Counter(r['性味'] for r in recipes))}")
|
||||
print(f"重名: {dup_names}")
|
||||
print(f"质量标记: {dict(quality_flags)}")
|
||||
print(f"字段覆盖: {report['字段覆盖率']}")
|
||||
print("\n抽样验证:")
|
||||
for r in recipes[:2] + [r for r in recipes if r['名称']=='银杏炖鸭']:
|
||||
print(f" {r['id']} {r['名称']}({r['性味']}) 解析尾: ...{r['解析'][-40:]}")
|
||||
print(f" 质量: {r['数据质量']}")
|
||||
Reference in new issue
Block a user