403 lines
15 KiB
Python
403 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
大医网「症状→方剂/药材/穴位」多段桥接关联 及 关联融合总览统计
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import math
|
|
from collections import defaultdict
|
|
|
|
BASE = "/home/songyi/Documents/ai_agent_scraper_study/data/大医网/03_关联融合"
|
|
CROSS_DIR = os.path.join(BASE, "交叉关联分析")
|
|
OUTPUT_DIR = CROSS_DIR
|
|
|
|
# ============================================================
|
|
# PART 1: 症状→方剂/药材/穴位 桥接
|
|
# ============================================================
|
|
|
|
def load_symptom_disease():
|
|
"""Load 症状-疾病关联.json - format: {症状: [[疾病, 置信度(0-100), [标签]], ...]}"""
|
|
path = os.path.join(CROSS_DIR, "症状-疾病关联.json")
|
|
print(f" 加载 {path} ...")
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
# Normalize confidence: values 35-100 range, divide by 100 to get 0-1
|
|
normalized = {}
|
|
for symptom, diseases in data.items():
|
|
normalized[symptom] = []
|
|
for entry in diseases:
|
|
disease_name = entry[0]
|
|
conf = entry[1]
|
|
# Normalize to 0-1
|
|
if conf > 1:
|
|
conf = conf / 100.0
|
|
normalized[symptom].append((disease_name, conf))
|
|
return normalized
|
|
|
|
def load_disease_target(file_key, target_name):
|
|
"""Load disease->target mapping: {disease: [(target, conf_0_1), ...]}"""
|
|
fname_map = {
|
|
'formula': '疾病-方剂关联数据.json',
|
|
'herb': '疾病-药材关联数据.json',
|
|
'acupoint': '疾病-穴位关联数据.json'
|
|
}
|
|
topkey_map = {
|
|
'formula': 'disease_to_formulas',
|
|
'herb': 'disease_to_herbs',
|
|
'acupoint': 'disease_to_acupoints'
|
|
}
|
|
path = os.path.join(CROSS_DIR, fname_map[file_key])
|
|
print(f" 加载 {path} ...")
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
return data[topkey_map[file_key]]
|
|
|
|
def build_bridging(sym_disease, disease_target, target_label):
|
|
"""
|
|
Symptom -> Disease -> Target bridging.
|
|
sym_disease: {symptom: [(disease, conf_sd), ...]}
|
|
disease_target: {disease: [(target, conf_dt), ...]}
|
|
Returns: {symptom: [{target名: ..., 桥接路径: ..., 得分: ...}, ...]}
|
|
"""
|
|
print(f" 构建症状→{target_label}桥接 ...")
|
|
result = {}
|
|
|
|
for symptom, disease_list in sym_disease.items():
|
|
# For each symptom, collect all paths
|
|
target_scores = defaultdict(float)
|
|
target_paths = defaultdict(list)
|
|
|
|
for disease_name, conf_sd in disease_list:
|
|
if disease_name not in disease_target:
|
|
continue
|
|
for target_item in disease_target[disease_name]:
|
|
target_name = target_item[0]
|
|
conf_dt = target_item[1] if len(target_item) > 1 else 1.0
|
|
|
|
# Chain score = symptom-disease confidence * disease-target confidence
|
|
chain_score = conf_sd * conf_dt
|
|
|
|
path_str = f"{symptom}→{disease_name}→{target_name}"
|
|
|
|
if chain_score > target_scores[target_name]:
|
|
target_scores[target_name] = chain_score
|
|
target_paths[target_name] = path_str
|
|
|
|
if target_scores:
|
|
# Sort by score descending, take TOP20
|
|
sorted_targets = sorted(target_scores.items(), key=lambda x: -x[1])
|
|
top20 = sorted_targets[:20]
|
|
|
|
result[symptom] = [
|
|
{
|
|
target_label: tname,
|
|
"桥接路径": target_paths[tname],
|
|
"得分": round(score, 4)
|
|
}
|
|
for tname, score in top20
|
|
]
|
|
|
|
return result
|
|
|
|
def do_bridging():
|
|
print("=" * 60)
|
|
print("PART 1: 症状→方剂/药材/穴位 桥接关联")
|
|
print("=" * 60)
|
|
|
|
# Load symptoms->disease
|
|
sym_disease = load_symptom_disease()
|
|
print(f" 症状总数: {len(sym_disease)}")
|
|
|
|
# Load disease->formula/herb/acupoint
|
|
disease_formula = load_disease_target('formula', '方剂名')
|
|
disease_herb = load_disease_target('herb', '药材名')
|
|
disease_acupoint = load_disease_target('acupoint', '穴位名')
|
|
|
|
print(f" 疾病→方剂: {len(disease_formula)} 疾病")
|
|
print(f" 疾病→药材: {len(disease_herb)} 疾病")
|
|
print(f" 疾病→穴位: {len(disease_acupoint)} 疾病")
|
|
|
|
# Build bridges
|
|
sym_formula = build_bridging(sym_disease, disease_formula, "方剂名")
|
|
sym_herb = build_bridging(sym_disease, disease_herb, "药材名")
|
|
sym_acupoint = build_bridging(sym_disease, disease_acupoint, "穴位名")
|
|
|
|
print(f" 桥接症状→方剂: {len(sym_formula)} 症状")
|
|
print(f" 桥接症状→药材: {len(sym_herb)} 症状")
|
|
print(f" 桥接症状→穴位: {len(sym_acupoint)} 症状")
|
|
|
|
# Count unique items
|
|
all_formulas = set()
|
|
for symptom, items in sym_formula.items():
|
|
for item in items:
|
|
all_formulas.add(item['方剂名'])
|
|
|
|
all_herbs = set()
|
|
for symptom, items in sym_herb.items():
|
|
for item in items:
|
|
all_herbs.add(item['药材名'])
|
|
|
|
all_acupoints = set()
|
|
for symptom, items in sym_acupoint.items():
|
|
for item in items:
|
|
all_acupoints.add(item['穴位名'])
|
|
|
|
total_paths = sum(len(v) for v in sym_formula.values()) + \
|
|
sum(len(v) for v in sym_herb.values()) + \
|
|
sum(len(v) for v in sym_acupoint.values())
|
|
|
|
summary = {
|
|
"桥接症状数": len(set(list(sym_formula.keys()) + list(sym_herb.keys()) + list(sym_acupoint.keys()))),
|
|
"桥接方剂数": len(all_formulas),
|
|
"桥接药材数": len(all_herbs),
|
|
"桥接穴位数": len(all_acupoints),
|
|
"总桥接路径": total_paths
|
|
}
|
|
|
|
print(f" 统计摘要: {json.dumps(summary, ensure_ascii=False)}")
|
|
|
|
# Build output structure
|
|
output = {
|
|
"症状-方剂": sym_formula,
|
|
"症状-药材": sym_herb,
|
|
"症状-穴位": sym_acupoint,
|
|
"统计摘要": summary
|
|
}
|
|
|
|
output_path = os.path.join(OUTPUT_DIR, "症状-方剂药材穴位桥接.json")
|
|
print(f" 写入 {output_path} ...")
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
json.dump(output, f, ensure_ascii=False, indent=2)
|
|
|
|
return output
|
|
|
|
|
|
# ============================================================
|
|
# PART 2: 关联融合总览统计
|
|
# ============================================================
|
|
|
|
def count_records(obj):
|
|
"""Count 'records' in a JSON structure - heuristic for different formats"""
|
|
if isinstance(obj, list):
|
|
return len(obj)
|
|
elif isinstance(obj, dict):
|
|
# Check common patterns
|
|
# Pattern 1: values are arrays (like symptom->disease mapping)
|
|
array_values = [v for v in obj.values() if isinstance(v, (list, dict))]
|
|
if array_values:
|
|
# Sum of array lengths, or count of dict values
|
|
total = 0
|
|
for v in obj.values():
|
|
if isinstance(v, list):
|
|
total += len(v)
|
|
elif isinstance(v, dict):
|
|
total += len(v)
|
|
return total
|
|
# Pattern 2: top-level keys as records
|
|
return len(obj)
|
|
return 0
|
|
|
|
def get_content_type(filename):
|
|
"""Determine content description from filename"""
|
|
descriptions = {
|
|
'01_配方-药材关联表.json': '配方-药材映射',
|
|
'02_药材-配方关联表.json': '药材-配方映射',
|
|
'03_Top50高频药材.json': '高频药材统计',
|
|
'04_Top50大复方.json': '大复方统计',
|
|
'05_药材分类统计.json': '药材分类统计',
|
|
'06_总摘要.json': '关联分析摘要',
|
|
'症状-疾病关联.json': '症状→疾病关联(原始)',
|
|
'疾病-症状关联.json': '疾病→症状关联',
|
|
'症状-疾病_高置信关联.json': '症状→疾病高置信关联',
|
|
'疾病-方剂关联数据.json': '疾病→方剂关联',
|
|
'疾病-药材关联数据.json': '疾病→药材关联',
|
|
'疾病-穴位关联数据.json': '疾病→穴位关联',
|
|
'四库交叉关联数据.json': '四库交叉关联总览',
|
|
'三库关联总览.json': '三库关联统计',
|
|
'术语与症状深度关联分析.json': '术语-症状深度关联',
|
|
'症状_全部.json': '症状全集',
|
|
'症状_列表.json': '症状列表',
|
|
'症状_归档统计.json': '症状归档统计',
|
|
'症状->疾病推算模型.json': '症状→疾病推算模型',
|
|
'症状->疾病->四库桥接数据.json': '症状→疾病→四库桥接',
|
|
'疾病->增强方剂药材穴位反查数据.json': '疾病→方剂药材穴位反查',
|
|
'疾病-症状严谨关联数据.json': '疾病-症状严谨关联',
|
|
'体质-疾病症状挖掘.json': '体质-疾病症状挖掘',
|
|
'体质-方剂药材穴位挖掘.json': '体质-方剂药材穴位挖掘',
|
|
'体质-药膳推荐体系.json': '体质-药膳推荐体系',
|
|
'体质-知识索引关联.json': '体质-知识索引关联',
|
|
'咳嗽哮喘_关联数据.json': '咳嗽哮喘专题关联',
|
|
'心脑血管_关联数据.json': '心脑血管专题关联',
|
|
'脾胃调理_关联数据.json': '脾胃调理专题关联',
|
|
'失眠不寐_关联数据.json': '失眠不寐专题关联',
|
|
'女性补气血_关联数据.json': '女性补气血专题关联',
|
|
'症状-方剂药材穴位桥接.json': '症状→方剂/药材/穴位桥接',
|
|
'关联融合总览.json': '关联融合总览',
|
|
}
|
|
# Substring matching
|
|
for key, desc in descriptions.items():
|
|
if filename == key or filename.endswith(key):
|
|
return desc
|
|
|
|
# Generic detection
|
|
if '方剂' in filename and '药材' in filename:
|
|
return '配方-药材关联'
|
|
if '药材' in filename and '配方' in filename:
|
|
return '药材-配方关联'
|
|
if '高频' in filename:
|
|
return '高频统计'
|
|
if '分类' in filename:
|
|
return '分类统计'
|
|
if '摘要' in filename or '总汇' in filename or '汇总' in filename:
|
|
return '分析摘要'
|
|
return '关联数据'
|
|
|
|
def get_file_stats(filepath):
|
|
"""Get file size in KB and record count"""
|
|
size_kb = os.path.getsize(filepath) / 1024.0
|
|
size_kb = round(size_kb, 1)
|
|
|
|
try:
|
|
with open(filepath, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
records = count_records(data)
|
|
except:
|
|
records = 0
|
|
|
|
return size_kb, records
|
|
|
|
def scan_directory(dirpath, max_depth=1):
|
|
"""Scan a directory for JSON files and return stats"""
|
|
results = []
|
|
total_size_kb = 0
|
|
total_records = 0
|
|
|
|
for fname in sorted(os.listdir(dirpath)):
|
|
if not fname.endswith('.json'):
|
|
continue
|
|
fpath = os.path.join(dirpath, fname)
|
|
if os.path.isdir(fpath):
|
|
continue
|
|
|
|
size_kb, records = get_file_stats(fpath)
|
|
content = get_content_type(fname)
|
|
|
|
results.append({
|
|
"文件名": fname,
|
|
"大小KB": size_kb,
|
|
"记录数": records,
|
|
"内容": content
|
|
})
|
|
total_size_kb += size_kb
|
|
total_records += records
|
|
|
|
return results, total_size_kb, total_records
|
|
|
|
def scan_deep_mine():
|
|
"""Scan depth mining directory structure"""
|
|
deep_dir = os.path.join(BASE, "深度挖掘迭代")
|
|
|
|
versions = {}
|
|
total_files = 0
|
|
|
|
for item in sorted(os.listdir(deep_dir)):
|
|
item_path = os.path.join(deep_dir, item)
|
|
if not os.path.isdir(item_path):
|
|
continue
|
|
|
|
json_files = [f for f in os.listdir(item_path) if f.endswith('.json')]
|
|
count = len(json_files)
|
|
total_files += count
|
|
versions[item] = count
|
|
|
|
return total_files, versions
|
|
|
|
def build_overview():
|
|
print("\n" + "=" * 60)
|
|
print("PART 2: 关联融合总览统计")
|
|
print("=" * 60)
|
|
|
|
# 1. Scan 关联分析/
|
|
analysis_dir = os.path.join(BASE, "关联分析")
|
|
analysis_files, analysis_size, analysis_records = scan_directory(analysis_dir)
|
|
print(f" 关联分析: {len(analysis_files)} 文件, {analysis_size:.1f} KB")
|
|
|
|
# 2. Scan 交叉关联分析/
|
|
cross_files, cross_size, cross_records = scan_directory(CROSS_DIR)
|
|
print(f" 交叉关联分析: {len(cross_files)} 文件, {cross_size:.1f} KB")
|
|
|
|
# 3. Scan 深度挖掘迭代/
|
|
deep_total, versions = scan_deep_mine()
|
|
print(f" 深度挖掘迭代: {deep_total} 文件, {len(versions)} 版本")
|
|
|
|
# 4. 其他体质文件 in 交叉关联分析
|
|
tizhi_files = [f for f in cross_files if '体质' in f['文件名']]
|
|
print(f" 体质相关文件: {len(tizhi_files)}")
|
|
|
|
total_files = len(analysis_files) + len(cross_files) + deep_total
|
|
total_size_mb = (analysis_size + cross_size) / 1024.0
|
|
|
|
# Get deep_mine subdirectory sizes
|
|
deep_dir = os.path.join(BASE, "深度挖掘迭代")
|
|
total_deep_size_kb = 0
|
|
for root, dirs, files in os.walk(deep_dir):
|
|
for f in files:
|
|
if f.endswith('.json'):
|
|
total_deep_size_kb += os.path.getsize(os.path.join(root, f)) / 1024.0
|
|
total_size_mb = (analysis_size + cross_size + total_deep_size_kb) / 1024.0
|
|
|
|
overview = {
|
|
"总览": {
|
|
"子目录数": 3,
|
|
"总文件数": total_files,
|
|
"总数据量MB": round(total_size_mb, 2)
|
|
},
|
|
"关联分析": analysis_files,
|
|
"交叉关联分析": cross_files,
|
|
"深度挖掘迭代": {
|
|
"总文件数": deep_total,
|
|
"版本数": len(versions)
|
|
}
|
|
}
|
|
|
|
# Add version details
|
|
for ver_name, ver_count in sorted(versions.items()):
|
|
# Strip the '分析_' prefix since we already add '分析_' in the key
|
|
short_name = ver_name.replace('分析_', '', 1) if ver_name.startswith('分析_') else ver_name
|
|
overview["深度挖掘迭代"][f"分析_{short_name}文件数"] = ver_count
|
|
|
|
output_path = os.path.join(OUTPUT_DIR, "关联融合总览.json")
|
|
print(f" 写入 {output_path} ...")
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
json.dump(overview, f, ensure_ascii=False, indent=2)
|
|
|
|
return overview
|
|
|
|
|
|
# ============================================================
|
|
# MAIN
|
|
# ============================================================
|
|
|
|
if __name__ == "__main__":
|
|
# Part 1: bridging
|
|
bridge_result = do_bridging()
|
|
|
|
# Part 2: overview
|
|
overview_result = build_overview()
|
|
|
|
# Verify
|
|
print("\n" + "=" * 60)
|
|
print("验证输出文件")
|
|
print("=" * 60)
|
|
|
|
for fname in ["症状-方剂药材穴位桥接.json", "关联融合总览.json"]:
|
|
fpath = os.path.join(OUTPUT_DIR, fname)
|
|
if os.path.exists(fpath):
|
|
fsize = os.path.getsize(fpath)
|
|
print(f" ✓ {fname}: {fsize:,} bytes ({fsize/1024:.1f} KB)")
|
|
else:
|
|
print(f" ✗ {fname}: NOT FOUND")
|
|
|
|
print("\n完成!")
|