#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
深度数据挖掘 - 精修修正版
核心修正: 
  1. 药材匹配仅使用药材主名（name），不含单字别名
  2. 药材名按长度降序排列，确保长名优先匹配
  3. 功效关键词过滤噪音（去除"本品""诸药"等解析残余）
  4. 药对/药组仅统计完整药材名
"""
import json
import os
import re
import glob
from collections import Counter, defaultdict, OrderedDict
from itertools import combinations

base = "/home/songyi/Documents/ai_agent_scraper_study/data/大医网"
OUT = os.path.join(base, "03_关联融合/深度挖掘迭代/分析_精修2")
os.makedirs(OUT, exist_ok=True)

# ========== 加载药材 ==========
print("=" * 70)
print("  大医网 方剂-中药材 深度数据挖掘 (精修修正版)")
print("=" * 70)
print(f"\n[1/8] 加载数据...")

# 药材主名集合（仅使用主名，不含别名）
herbs = {}  # canonical_name -> full_data
valid_herb_names = []  # 仅≥2字的主名，按长度降序排列

for f in glob.glob(os.path.join(base, "01_来源数据/中药材", "*.json")):
    with open(f, encoding='utf-8') as fh:
        d = json.load(fh)
    name = d.get('名称', '').strip()
    if name:
        herbs[name] = d

# 仅保留≥2个字的药材名作为匹配字典
for name, data in herbs.items():
    if len(name) >= 2:
        valid_herb_names.append(name)

# 按长度降序，确保长名优先匹配
valid_herb_names.sort(key=len, reverse=True)
herb_pattern = re.compile('|'.join(re.escape(n) for n in valid_herb_names))

print(f"  药材: {len(herbs)} 味 (匹配字典: {len(valid_herb_names)}味)")
print(f"  药材样本: {', '.join(valid_herb_names[:10])}")

# ========== 加载方剂 ==========
formulas = OrderedDict()
for f in glob.glob(os.path.join(base, "01_来源数据/方剂", "*.json")):
    with open(f, encoding='utf-8') as fh:
        d = json.load(fh)
    name = d.get('名称', os.path.basename(f).rsplit('_', 1)[0])
    formulas[name] = d

print(f"  方剂: {len(formulas)} 首")

# ========== 提取方剂中的药材 ==========
print(f"\n[2/8] 提取方剂中的药材...")

def extract_herbs_exact(text):
    """精确匹配：仅使用≥2字的主药材名"""
    if not text or len(text) < 3:
        return []
    
    # 去掉炮制括号 药量
    clean = re.sub(r'[（(][^())]*[）)]', '', text)
    clean = re.sub(r'\d+[两钱毫升克]', '', clean)
    
    # 用正则找到所有药材名
    found = []
    for match in herb_pattern.finditer(clean):
        herb = match.group(0)
        if herb and herb not in found:
            found.append(herb)
    
    return found

def get_all_text(data_dict):
    """拼接所有字段文本"""
    all_text = ''
    for val in data_dict.values():
        if isinstance(val, str):
            all_text += val
        elif isinstance(val, (dict, list)):
            all_text += json.dumps(val, ensure_ascii=False)
    return all_text

formula_herbs = OrderedDict()
for fname, data in formulas.items():
    all_text = get_all_text(data)
    herbs_found = extract_herbs_exact(all_text)
    formula_herbs[fname] = herbs_found

total_links = sum(len(h) for h in formula_herbs.values())
avg = total_links / max(1, len(formula_herbs))
print(f"  总关联对数: {total_links}")
print(f"  平均每方药材数: {avg:.1f}")

# 检查提取质量
print(f"\n  提取结果检查 (前15首方):")
for i, (fname, hlist) in enumerate(list(formula_herbs.items())[:15]):
    print(f"    [{i+1:2d}] {fname:30s} {len(hlist):2d}味 -> {hlist[:8]}")

# 逆向索引
herb_formulas = defaultdict(set)
for fname, hlist in formula_herbs.items():
    for h in hlist:
        herb_formulas[h].add(fname)

# ========== 3. 高频药材 ==========
print(f"\n[3/8] 高频药材分析...")
herb_freq = Counter()
for hlist in formula_herbs.values():
    for h in hlist:
        herb_freq[h] += 1

top30 = herb_freq.most_common(30)
print(f"\n  Top 30 高频药材:")
print(f"  {'药材':<12s} {'频次':>5s}  {'分类':<6s}  {'功效':<30s}")
print(f"  {'-'*12} {'-'*5}  {'-'*6}  {'-'*30}")
for h, c in top30:
    ct = herbs.get(h, {}).get('药材分类', '未知')
    func_data = herbs.get(h, {}).get('功效作用', {})
    func = func_data.get('功能', '')[:30] if isinstance(func_data, dict) else ''
    print(f"  {h:<12s} {c:>5d}  {ct:<6s}  {func}")

with open(os.path.join(OUT, "01_高频药材Top30.json"), 'w', encoding='utf-8') as f:
    json.dump([{"name": h, "count": c} for h, c in top30], f, ensure_ascii=False, indent=2)
print("\n  ✓ 01_高频药材Top30.json")

# ========== 4. 高频药对 ==========
print(f"\n[4/8] 高频药对 (2味) Top 20...")
pair_freq = Counter()
for hlist in formula_herbs.values():
    u = sorted(list(set(hlist)))
    if len(u) >= 2:
        for combo in combinations(u, 2):
            pair_freq[combo] += 1

# Top 50药对 (≥5次) 
top_pairs = [p for p, c in pair_freq.most_common(100) if c >= 5]
print(f"\n  Top 20 高频药对:")
print(f"  {'药对':<30s} {'频次':>5s}  {'药材1功效':<12s}  {'药材2功效':<12s}")
print(f"  {'-'*30} {'-'*5}  {'-'*12}  {'-'*12}")
for pair in top_pairs[:20]:
    h1, h2 = pair
    c = pair_freq[pair]
    func_data1 = herbs.get(h1, {}).get('功效作用', {})
    func_data2 = herbs.get(h2, {}).get('功效作用', {})
    func1 = func_data1.get('功能', '')[:12] if isinstance(func_data1, dict) else ''
    func2 = func_data2.get('功能', '')[:12] if isinstance(func_data2, dict) else ''
    print(f"  {h1:12s} + {h2:12s} {c:>5d}  [{func1[:8]}]  [{func2[:8]}]")

with open(os.path.join(OUT, "02_高频药对_Top20.json"), 'w', encoding='utf-8') as f:
    json.dump([{"pair": list(p), "count": c} for p, c in pair_freq.most_common(50)], f, ensure_ascii=False, indent=2)
print("\n  ✓ 02_高频药对_Top20.json")

# ========== 5. 核心药组 ==========
print(f"\n[5/8] 核心药组 (3-5味药) Top 10...")

for n in [3, 4, 5]:
    triple_freq = Counter()
    for hlist in formula_herbs.values():
        u = sorted(list(set(hlist)))
        if len(u) >= n:
            for combo in combinations(u, n):
                triple_freq[combo] += 1
    
    top_combs = [(t, c) for t, c in triple_freq.most_common(50) if c >= 5]
    print(f"\n  Top {n}味药组 (出现≥5次):")
    for combo, c in top_combs[:10]:
        names = "+".join(combo)
        print(f"    {names:>40s}: {c}首方")

with open(os.path.join(OUT, "03_核心药组3-5味_Top10.json"), 'w', encoding='utf-8') as f:
    res = {}
    for n in [3, 4, 5]:
        triple_freq = Counter()
        for hlist in formula_herbs.values():
            u = sorted(list(set(hlist)))
            if len(u) >= n:
                for combo in combinations(u, n):
                    triple_freq[combo] += 1
        res[f"{n}味药组"] = [{"group": "+".join(t), "count": c} for t, c in triple_freq.most_common(10) if c >= 5]
    json.dump(res, f, ensure_ascii=False, indent=2)
print("\n  ✓ 03_核心药组3-5味_Top10.json")

# ========== 6. 功效关键词网络 ==========
print(f"\n[6/8] 功效关键词网络...")

# 提取核心功效词（过滤噪音词）
noise_words = {'本品', '诸药', '的', '具有', '的功效', '功效', '。', '。'}

func_keywords = Counter()
func_formulas = defaultdict(set)

for h, data in herbs.items():
    if h not in herb_formulas:
        continue
    gua = data.get('功效作用', {})
    if not isinstance(gua, dict):
        continue
    func_str = gua.get('功能', '')
    if not func_str:
        continue
    
    # 按标点拆分
    keywords = re.split(r'[、,，、；;。]', func_str)
    for kw in keywords:
        kw = kw.strip()
        # 过滤噪音词
        if not kw or len(kw) < 2 or kw in noise_words:
            continue
        # 也检查是否全是药材名（如"甘草"也是功效词"甘草具有补脾益气"中的残留）
        if kw == h and isinstance(gua, dict) and func_str.startswith(kw):
            # 如果关键词就是药材名且紧跟药材名，跳过
            continue
        if kw in herbs:
            # 如果关键词本身是药材名且在功效描述中（非独立功效词），跳过
            continue
            
        func_keywords[kw] += 1
        for fname in herb_formulas[h]:
            func_formulas[kw].add(fname)

# 按方剂数排序
top_funcs = [(kw, len(vx)) for kw, vx in func_formulas.items() if len(vx) >= 10]
top_funcs.sort(key=lambda x: -x[1])

print(f"\n  Top 30 功效关键词 (按关联方剂数):")
print(f"  {'关键词':<15s} {'方剂数':>6s}  示例方剂")
print(f"  {'-'*15} {'-'*6}  {'-'*40}")
for kw, count in top_funcs[:30]:
    examples = list(func_formulas[kw])[:2]
    print(f"  {kw:<15s} {count:>6d}  {', '.join(examples)}")

with open(os.path.join(OUT, "04_功效关键词_Top30.json"), 'w', encoding='utf-8') as f:
    json.dump([{"keyword": kw, "formula_count": count} for kw, count in top_funcs[:30]], f, ensure_ascii=False, indent=2)
print("\n  ✓ 04_功效关键词_Top30.json")

# ========== 7. 方剂聚类 ==========
print(f"\n[7/8] 方剂聚类...")

clusters = defaultdict(list)
for fname, hlist in formula_herbs.items():
    kw_count = Counter()
    for herb in hlist:
        if herb in herbs:
            gua = herbs[herb].get('功效作用', {})
            if isinstance(gua, dict) and gua.get('功能'):
                kw_str = gua['功能']
                keywords = re.split(r'[、,，、；;。]', kw_str)
                for kw in keywords:
                    kw = kw.strip()
                    if kw and len(kw) >= 2 and kw not in noise_words:
                        kw_count[kw] += 1
    if kw_count:
        top_kw = kw_count.most_common(1)[0][0]
        clusters[top_kw].append(fname)

big_clusters = [(kw, flst) for kw, flst in clusters.items() if len(flst) >= 5]
big_clusters.sort(key=lambda x: -len(x[1]))

print(f"  发现 {len(clusters)} 个功效簇")
print(f"\n  Top 20 大功效簇:")
for kw, flst in big_clusters[:20]:
    # 找簇内的代表性药材
    sample_herbs = set()
    for fname in flst[:10]:
        for h in formula_herbs[fname]:
            sample_herbs.add(h)
    top_h = sample_herbs
    print(f"  '{kw}' ({len(flst)}首方): 代表药材 -> {', '.join(sorted(top_h)[:10])}")

with open(os.path.join(OUT, "05_方剂聚类_Top20.json"), 'w', encoding='utf-8') as f:
    json.dump([{"cluster": kw, "count": len(flst), "formulas": flst[:20]} for kw, flst in big_clusters[:20]], f, ensure_ascii=False, indent=2)
print("\n  ✓ 05_方剂聚类_Top20.json")

# ========== 8. 配伍禁忌 & 相似度 ==========
print(f"\n[8/8] 配伍禁忌验证 & 方剂相似度...")

# 十八反
fan = {
    '甘草': ['大戟', '芫花', '甘遂', '京大戟', '红大戟'],
    '乌头': ['半夏', '瓜蒌', '天花粉', '贝母', '平贝母', '川贝母', '浙贝母', '白蔹', '白及'],
    '藜芦': ['人参', '党参', '丹参', '玄参', '沙参', '苦参', '细辛', '白芍', '赤芍'],
}
wei = {
    '硫黄': ['朴硝', '芒硝', '牙硝'],
    '水银': ['铅丹', '砒霜'],
    '巴豆': ['牵牛', '牵牛子'],
    '丁香': ['郁金'],
    '人参': ['五灵脂'],
    '肉桂': ['石脂', '赤石脂'],
    '半夏': ['羊脂'],
    '厚朴': ['硝石', '滑石'],
}

# 检查禁忌药材是否在方剂中出现
contra_found = defaultdict(list)
all_contra_search = {}
for k, v in fan.items():
    for vv in v:
        if vv not in all_contra_search:
            all_contra_search[vv] = k
for k, v in wei.items():
    for vv in v:
        if vv not in all_contra_search:
            all_contra_search[vv] = k

print(f"\n  十八反/十九畏相关药材在方剂中出现情况:")
for herb_name, related in all_contra_search.items():
    if herb_name in herb_formulas:
        count = len(herb_formulas[herb_name])
        print(f"    ⚠ {herb_name} (反/畏{related}): 出现在 {count} 首方")
        
# 检查实际方剂中是否同时出现矛盾配对
contra_pairs_found = defaultdict(int)
contra_pairs_details = defaultdict(list)

for fname, hlist in formula_herbs.items():
    for herb, contra_list in fan.items():
        if herb in hlist:
            for ch in contra_list:
                if ch in hlist:
                    contra_pairs_found[(herb, ch)] += 1
                    contra_pairs_details[(herb, ch)].append(fname)
    for herb, contra_list in wei.items():
        if herb in hlist:
            for ch in contra_list:
                if ch in hlist:
                    contra_pairs_found[(herb, ch)] += 1
                    contra_pairs_details[(herb, ch)].append(fname)

top_contra = sorted(contra_pairs_found.items(), key=lambda x: -x[1])[:10]
if top_contra:
    print(f"\n  发现配伍禁忌:")
    for (h1, h2), count in top_contra:
        print(f"    ⚠️ {h1} + {h2}: {count}首方 ({', '.join(contra_pairs_details[(h1,h2)][:2])})")
else:
    print(f"\n  未发现十八反/十九畏的直接配对出现在同一首方剂中")

with open(os.path.join(OUT, "06_配伍禁忌.json"), 'w', encoding='utf-8') as f:
    json.dump({
        "十八反": fan,
        "十九畏": wei,
        "禁忌药材出现": {k: len(herb_formulas.get(k, set())) for k in all_contra_search.keys() if k in herb_formulas},
        "实际禁忌配对": {f"{k[0]}+{k[1]}": c for k, c in top_contra},
        "禁忌详情": {f"{k[0]}+{k[1]}": contra_pairs_details[k] for k in top_contra}
    }, f, ensure_ascii=False, indent=2)
print("\n  ✓ 06_配伍禁忌.json")

# 方剂相似度
print(f"\n  计算方剂相似度 (Jaccard >= 0.3)...")

def jaccard_fixed(s1, s2):
    if not s1 or not s2:
        return 0.0
    intersection = len(s1 & s2)
    if intersection == 0:
        return 0.0
    union = len(s1 | s2)
    return intersection / union if union > 0 else 0.0

similar_pairs = []
formula_list = list(formulas.keys())
count = 0
for i in range(len(formula_list)):
    f1 = formula_list[i]
    h1 = formula_herbs.get(f1, [])
    if not h1:
        continue
    s1 = set(h1)
    for j in range(i + 1, len(formula_list)):
        f2 = formula_list[j]
        h2 = formula_herbs.get(f2, [])
        if not h2:
            continue
        s2 = set(h2)
        sim = jaccard_fixed(s1, s2)
        if sim >= 0.3:
            shared = s1 & s2
            similar_pairs.append((f1, f2, sim, len(shared), sorted(shared)))
        count += 1

print(f"  检查 {count} 对, 找到 {len(similar_pairs)} 对相似方剂")
similar_pairs.sort(key=lambda x: (-x[2], -x[3]))
top_similar = similar_pairs[:20]

print(f"\n  Top 15 相似方剂对:")
for f1, f2, sim, shared_c, shared_h in top_similar[:15]:
    print(f"  {f1:30s} + {f2:30s} -> {sim:.3f} (共享{shared_c}味: {', '.join(shared_h)})")

with open(os.path.join(OUT, "07_方剂相似度_Top20.json"), 'w', encoding='utf-8') as f:
    json.dump([{"similarity": round(sim, 4), "shared_count": sc, "pair1": f1, "pair2": f2, "shared_herbs": sh} for f1, f2, sim, sc, sh in top_similar], f, ensure_ascii=False, indent=2)
print("\n  ✓ 07_方剂相似度_Top20.json")

# ========== 汇总 ==========
print(f"\n{'='*70}")
print(f"  深度挖掘分析完成！")
print(f"{'='*70}")

summary = OrderedDict()
summary["标题"] = "大医网 方剂-中药材 深度数据挖掘报告 (精修修正版)"
summary["数据规模"] = {
    "药材": len(herbs),
    "方剂": len(formulas),
    "总关联对数": total_links,
    "平均每方药材数": round(avg, 1),
    "成功提取方剂": sum(1 for h in formula_herbs.values() if h),
    "无法提取方剂": sum(1 for h in formula_herbs.values() if not h),
}
summary["高频药材_Top20"] = [{"name": h, "count": c} for h, c in top30[:20]]
summary["高频药对_Top10"] = [{"pair": list(p), "count": c} for p, c in pair_freq.most_common(10)]
summary["核心药组_Top10"] = {}
for n in [3, 4, 5]:
    triple_freq = Counter()
    for hlist in formula_herbs.values():
        u = sorted(list(set(hlist)))
        if len(u) >= n:
            for combo in combinations(u, n):
                triple_freq[combo] += 1
    summary["核心药组_Top10"][f"{n}味药组"] = [{"group": "+".join(t), "count": c} for t, c in triple_freq.most_common(10) if c >= 5]

summary["功效关键词_Top15"] = [{"keyword": kw, "count": c} for kw, c in top_funcs[:15]]
summary["方剂聚类_Top10"] = [{"cluster": kw, "count": len(flst)} for kw, flst in big_clusters[:10]]
summary["配伍禁忌"] = {
    "十八反": fan,
    "十九畏": wei,
    "实际发现": {f"{k[0]}+{k[1]}": c for k, c in top_contra},
}
summary["方剂相似度_Top10"] = [{"similarity": round(sim, 4), "pair": f"{f1} + {f2}"} for f1, f2, sim, sc, sh in top_similar[:10]]
summary["输出目录"] = os.path.abspath(OUT)

with open(os.path.join(OUT, "汇总报告.json"), 'w', encoding='utf-8') as f:
    json.dump(summary, f, ensure_ascii=False, indent=2)

print(f"\n  输出目录: {os.path.abspath(OUT)}")
print(f"\n  文件列表:")
for fn in sorted(os.listdir(OUT)):
    fp = os.path.join(OUT, fn)
    size = os.path.getsize(fp)
    print(f"    {fn:40s} {size:>10,} B")