初始归档：健康管理资料库（大医网/药膳/体质监测/中医理论等12个资料集，7712个文件）
This commit is contained in:
512song committed
commit
80ae3811cf
7712 files changed
+4628547
No files matched your search
@@ -0,0 +1,402 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
大医网「症状→方剂/药材/穴位」多段桥接关联 及 关联融合总览统计
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import math
|
||||
from collections import defaultdict
|
||||
|
||||
BASE = "/home/songyi/Documents/ai_agent_scraper_study/data/大医网/03_关联融合"
|
||||
CROSS_DIR = os.path.join(BASE, "交叉关联分析")
|
||||
OUTPUT_DIR = CROSS_DIR
|
||||
|
||||
# ============================================================
|
||||
# PART 1: 症状→方剂/药材/穴位 桥接
|
||||
# ============================================================
|
||||
|
||||
def load_symptom_disease():
|
||||
"""Load 症状-疾病关联.json - format: {症状: [[疾病, 置信度(0-100), [标签]], ...]}"""
|
||||
path = os.path.join(CROSS_DIR, "症状-疾病关联.json")
|
||||
print(f" 加载 {path} ...")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
# Normalize confidence: values 35-100 range, divide by 100 to get 0-1
|
||||
normalized = {}
|
||||
for symptom, diseases in data.items():
|
||||
normalized[symptom] = []
|
||||
for entry in diseases:
|
||||
disease_name = entry[0]
|
||||
conf = entry[1]
|
||||
# Normalize to 0-1
|
||||
if conf > 1:
|
||||
conf = conf / 100.0
|
||||
normalized[symptom].append((disease_name, conf))
|
||||
return normalized
|
||||
|
||||
def load_disease_target(file_key, target_name):
|
||||
"""Load disease->target mapping: {disease: [(target, conf_0_1), ...]}"""
|
||||
fname_map = {
|
||||
'formula': '疾病-方剂关联数据.json',
|
||||
'herb': '疾病-药材关联数据.json',
|
||||
'acupoint': '疾病-穴位关联数据.json'
|
||||
}
|
||||
topkey_map = {
|
||||
'formula': 'disease_to_formulas',
|
||||
'herb': 'disease_to_herbs',
|
||||
'acupoint': 'disease_to_acupoints'
|
||||
}
|
||||
path = os.path.join(CROSS_DIR, fname_map[file_key])
|
||||
print(f" 加载 {path} ...")
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
return data[topkey_map[file_key]]
|
||||
|
||||
def build_bridging(sym_disease, disease_target, target_label):
|
||||
"""
|
||||
Symptom -> Disease -> Target bridging.
|
||||
sym_disease: {symptom: [(disease, conf_sd), ...]}
|
||||
disease_target: {disease: [(target, conf_dt), ...]}
|
||||
Returns: {symptom: [{target名: ..., 桥接路径: ..., 得分: ...}, ...]}
|
||||
"""
|
||||
print(f" 构建症状→{target_label}桥接 ...")
|
||||
result = {}
|
||||
|
||||
for symptom, disease_list in sym_disease.items():
|
||||
# For each symptom, collect all paths
|
||||
target_scores = defaultdict(float)
|
||||
target_paths = defaultdict(list)
|
||||
|
||||
for disease_name, conf_sd in disease_list:
|
||||
if disease_name not in disease_target:
|
||||
continue
|
||||
for target_item in disease_target[disease_name]:
|
||||
target_name = target_item[0]
|
||||
conf_dt = target_item[1] if len(target_item) > 1 else 1.0
|
||||
|
||||
# Chain score = symptom-disease confidence * disease-target confidence
|
||||
chain_score = conf_sd * conf_dt
|
||||
|
||||
path_str = f"{symptom}→{disease_name}→{target_name}"
|
||||
|
||||
if chain_score > target_scores[target_name]:
|
||||
target_scores[target_name] = chain_score
|
||||
target_paths[target_name] = path_str
|
||||
|
||||
if target_scores:
|
||||
# Sort by score descending, take TOP20
|
||||
sorted_targets = sorted(target_scores.items(), key=lambda x: -x[1])
|
||||
top20 = sorted_targets[:20]
|
||||
|
||||
result[symptom] = [
|
||||
{
|
||||
target_label: tname,
|
||||
"桥接路径": target_paths[tname],
|
||||
"得分": round(score, 4)
|
||||
}
|
||||
for tname, score in top20
|
||||
]
|
||||
|
||||
return result
|
||||
|
||||
def do_bridging():
|
||||
print("=" * 60)
|
||||
print("PART 1: 症状→方剂/药材/穴位 桥接关联")
|
||||
print("=" * 60)
|
||||
|
||||
# Load symptoms->disease
|
||||
sym_disease = load_symptom_disease()
|
||||
print(f" 症状总数: {len(sym_disease)}")
|
||||
|
||||
# Load disease->formula/herb/acupoint
|
||||
disease_formula = load_disease_target('formula', '方剂名')
|
||||
disease_herb = load_disease_target('herb', '药材名')
|
||||
disease_acupoint = load_disease_target('acupoint', '穴位名')
|
||||
|
||||
print(f" 疾病→方剂: {len(disease_formula)} 疾病")
|
||||
print(f" 疾病→药材: {len(disease_herb)} 疾病")
|
||||
print(f" 疾病→穴位: {len(disease_acupoint)} 疾病")
|
||||
|
||||
# Build bridges
|
||||
sym_formula = build_bridging(sym_disease, disease_formula, "方剂名")
|
||||
sym_herb = build_bridging(sym_disease, disease_herb, "药材名")
|
||||
sym_acupoint = build_bridging(sym_disease, disease_acupoint, "穴位名")
|
||||
|
||||
print(f" 桥接症状→方剂: {len(sym_formula)} 症状")
|
||||
print(f" 桥接症状→药材: {len(sym_herb)} 症状")
|
||||
print(f" 桥接症状→穴位: {len(sym_acupoint)} 症状")
|
||||
|
||||
# Count unique items
|
||||
all_formulas = set()
|
||||
for symptom, items in sym_formula.items():
|
||||
for item in items:
|
||||
all_formulas.add(item['方剂名'])
|
||||
|
||||
all_herbs = set()
|
||||
for symptom, items in sym_herb.items():
|
||||
for item in items:
|
||||
all_herbs.add(item['药材名'])
|
||||
|
||||
all_acupoints = set()
|
||||
for symptom, items in sym_acupoint.items():
|
||||
for item in items:
|
||||
all_acupoints.add(item['穴位名'])
|
||||
|
||||
total_paths = sum(len(v) for v in sym_formula.values()) + \
|
||||
sum(len(v) for v in sym_herb.values()) + \
|
||||
sum(len(v) for v in sym_acupoint.values())
|
||||
|
||||
summary = {
|
||||
"桥接症状数": len(set(list(sym_formula.keys()) + list(sym_herb.keys()) + list(sym_acupoint.keys()))),
|
||||
"桥接方剂数": len(all_formulas),
|
||||
"桥接药材数": len(all_herbs),
|
||||
"桥接穴位数": len(all_acupoints),
|
||||
"总桥接路径": total_paths
|
||||
}
|
||||
|
||||
print(f" 统计摘要: {json.dumps(summary, ensure_ascii=False)}")
|
||||
|
||||
# Build output structure
|
||||
output = {
|
||||
"症状-方剂": sym_formula,
|
||||
"症状-药材": sym_herb,
|
||||
"症状-穴位": sym_acupoint,
|
||||
"统计摘要": summary
|
||||
}
|
||||
|
||||
output_path = os.path.join(OUTPUT_DIR, "症状-方剂药材穴位桥接.json")
|
||||
print(f" 写入 {output_path} ...")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
# ============================================================
|
||||
# PART 2: 关联融合总览统计
|
||||
# ============================================================
|
||||
|
||||
def count_records(obj):
|
||||
"""Count 'records' in a JSON structure - heuristic for different formats"""
|
||||
if isinstance(obj, list):
|
||||
return len(obj)
|
||||
elif isinstance(obj, dict):
|
||||
# Check common patterns
|
||||
# Pattern 1: values are arrays (like symptom->disease mapping)
|
||||
array_values = [v for v in obj.values() if isinstance(v, (list, dict))]
|
||||
if array_values:
|
||||
# Sum of array lengths, or count of dict values
|
||||
total = 0
|
||||
for v in obj.values():
|
||||
if isinstance(v, list):
|
||||
total += len(v)
|
||||
elif isinstance(v, dict):
|
||||
total += len(v)
|
||||
return total
|
||||
# Pattern 2: top-level keys as records
|
||||
return len(obj)
|
||||
return 0
|
||||
|
||||
def get_content_type(filename):
|
||||
"""Determine content description from filename"""
|
||||
descriptions = {
|
||||
'01_配方-药材关联表.json': '配方-药材映射',
|
||||
'02_药材-配方关联表.json': '药材-配方映射',
|
||||
'03_Top50高频药材.json': '高频药材统计',
|
||||
'04_Top50大复方.json': '大复方统计',
|
||||
'05_药材分类统计.json': '药材分类统计',
|
||||
'06_总摘要.json': '关联分析摘要',
|
||||
'症状-疾病关联.json': '症状→疾病关联(原始)',
|
||||
'疾病-症状关联.json': '疾病→症状关联',
|
||||
'症状-疾病_高置信关联.json': '症状→疾病高置信关联',
|
||||
'疾病-方剂关联数据.json': '疾病→方剂关联',
|
||||
'疾病-药材关联数据.json': '疾病→药材关联',
|
||||
'疾病-穴位关联数据.json': '疾病→穴位关联',
|
||||
'四库交叉关联数据.json': '四库交叉关联总览',
|
||||
'三库关联总览.json': '三库关联统计',
|
||||
'术语与症状深度关联分析.json': '术语-症状深度关联',
|
||||
'症状_全部.json': '症状全集',
|
||||
'症状_列表.json': '症状列表',
|
||||
'症状_归档统计.json': '症状归档统计',
|
||||
'症状->疾病推算模型.json': '症状→疾病推算模型',
|
||||
'症状->疾病->四库桥接数据.json': '症状→疾病→四库桥接',
|
||||
'疾病->增强方剂药材穴位反查数据.json': '疾病→方剂药材穴位反查',
|
||||
'疾病-症状严谨关联数据.json': '疾病-症状严谨关联',
|
||||
'体质-疾病症状挖掘.json': '体质-疾病症状挖掘',
|
||||
'体质-方剂药材穴位挖掘.json': '体质-方剂药材穴位挖掘',
|
||||
'体质-药膳推荐体系.json': '体质-药膳推荐体系',
|
||||
'体质-知识索引关联.json': '体质-知识索引关联',
|
||||
'咳嗽哮喘_关联数据.json': '咳嗽哮喘专题关联',
|
||||
'心脑血管_关联数据.json': '心脑血管专题关联',
|
||||
'脾胃调理_关联数据.json': '脾胃调理专题关联',
|
||||
'失眠不寐_关联数据.json': '失眠不寐专题关联',
|
||||
'女性补气血_关联数据.json': '女性补气血专题关联',
|
||||
'症状-方剂药材穴位桥接.json': '症状→方剂/药材/穴位桥接',
|
||||
'关联融合总览.json': '关联融合总览',
|
||||
}
|
||||
# Substring matching
|
||||
for key, desc in descriptions.items():
|
||||
if filename == key or filename.endswith(key):
|
||||
return desc
|
||||
|
||||
# Generic detection
|
||||
if '方剂' in filename and '药材' in filename:
|
||||
return '配方-药材关联'
|
||||
if '药材' in filename and '配方' in filename:
|
||||
return '药材-配方关联'
|
||||
if '高频' in filename:
|
||||
return '高频统计'
|
||||
if '分类' in filename:
|
||||
return '分类统计'
|
||||
if '摘要' in filename or '总汇' in filename or '汇总' in filename:
|
||||
return '分析摘要'
|
||||
return '关联数据'
|
||||
|
||||
def get_file_stats(filepath):
|
||||
"""Get file size in KB and record count"""
|
||||
size_kb = os.path.getsize(filepath) / 1024.0
|
||||
size_kb = round(size_kb, 1)
|
||||
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
records = count_records(data)
|
||||
except:
|
||||
records = 0
|
||||
|
||||
return size_kb, records
|
||||
|
||||
def scan_directory(dirpath, max_depth=1):
|
||||
"""Scan a directory for JSON files and return stats"""
|
||||
results = []
|
||||
total_size_kb = 0
|
||||
total_records = 0
|
||||
|
||||
for fname in sorted(os.listdir(dirpath)):
|
||||
if not fname.endswith('.json'):
|
||||
continue
|
||||
fpath = os.path.join(dirpath, fname)
|
||||
if os.path.isdir(fpath):
|
||||
continue
|
||||
|
||||
size_kb, records = get_file_stats(fpath)
|
||||
content = get_content_type(fname)
|
||||
|
||||
results.append({
|
||||
"文件名": fname,
|
||||
"大小KB": size_kb,
|
||||
"记录数": records,
|
||||
"内容": content
|
||||
})
|
||||
total_size_kb += size_kb
|
||||
total_records += records
|
||||
|
||||
return results, total_size_kb, total_records
|
||||
|
||||
def scan_deep_mine():
|
||||
"""Scan depth mining directory structure"""
|
||||
deep_dir = os.path.join(BASE, "深度挖掘迭代")
|
||||
|
||||
versions = {}
|
||||
total_files = 0
|
||||
|
||||
for item in sorted(os.listdir(deep_dir)):
|
||||
item_path = os.path.join(deep_dir, item)
|
||||
if not os.path.isdir(item_path):
|
||||
continue
|
||||
|
||||
json_files = [f for f in os.listdir(item_path) if f.endswith('.json')]
|
||||
count = len(json_files)
|
||||
total_files += count
|
||||
versions[item] = count
|
||||
|
||||
return total_files, versions
|
||||
|
||||
def build_overview():
|
||||
print("\n" + "=" * 60)
|
||||
print("PART 2: 关联融合总览统计")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. Scan 关联分析/
|
||||
analysis_dir = os.path.join(BASE, "关联分析")
|
||||
analysis_files, analysis_size, analysis_records = scan_directory(analysis_dir)
|
||||
print(f" 关联分析: {len(analysis_files)} 文件, {analysis_size:.1f} KB")
|
||||
|
||||
# 2. Scan 交叉关联分析/
|
||||
cross_files, cross_size, cross_records = scan_directory(CROSS_DIR)
|
||||
print(f" 交叉关联分析: {len(cross_files)} 文件, {cross_size:.1f} KB")
|
||||
|
||||
# 3. Scan 深度挖掘迭代/
|
||||
deep_total, versions = scan_deep_mine()
|
||||
print(f" 深度挖掘迭代: {deep_total} 文件, {len(versions)} 版本")
|
||||
|
||||
# 4. 其他体质文件 in 交叉关联分析
|
||||
tizhi_files = [f for f in cross_files if '体质' in f['文件名']]
|
||||
print(f" 体质相关文件: {len(tizhi_files)}")
|
||||
|
||||
total_files = len(analysis_files) + len(cross_files) + deep_total
|
||||
total_size_mb = (analysis_size + cross_size) / 1024.0
|
||||
|
||||
# Get deep_mine subdirectory sizes
|
||||
deep_dir = os.path.join(BASE, "深度挖掘迭代")
|
||||
total_deep_size_kb = 0
|
||||
for root, dirs, files in os.walk(deep_dir):
|
||||
for f in files:
|
||||
if f.endswith('.json'):
|
||||
total_deep_size_kb += os.path.getsize(os.path.join(root, f)) / 1024.0
|
||||
total_size_mb = (analysis_size + cross_size + total_deep_size_kb) / 1024.0
|
||||
|
||||
overview = {
|
||||
"总览": {
|
||||
"子目录数": 3,
|
||||
"总文件数": total_files,
|
||||
"总数据量MB": round(total_size_mb, 2)
|
||||
},
|
||||
"关联分析": analysis_files,
|
||||
"交叉关联分析": cross_files,
|
||||
"深度挖掘迭代": {
|
||||
"总文件数": deep_total,
|
||||
"版本数": len(versions)
|
||||
}
|
||||
}
|
||||
|
||||
# Add version details
|
||||
for ver_name, ver_count in sorted(versions.items()):
|
||||
# Strip the '分析_' prefix since we already add '分析_' in the key
|
||||
short_name = ver_name.replace('分析_', '', 1) if ver_name.startswith('分析_') else ver_name
|
||||
overview["深度挖掘迭代"][f"分析_{short_name}文件数"] = ver_count
|
||||
|
||||
output_path = os.path.join(OUTPUT_DIR, "关联融合总览.json")
|
||||
print(f" 写入 {output_path} ...")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(overview, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return overview
|
||||
|
||||
|
||||
# ============================================================
|
||||
# MAIN
|
||||
# ============================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Part 1: bridging
|
||||
bridge_result = do_bridging()
|
||||
|
||||
# Part 2: overview
|
||||
overview_result = build_overview()
|
||||
|
||||
# Verify
|
||||
print("\n" + "=" * 60)
|
||||
print("验证输出文件")
|
||||
print("=" * 60)
|
||||
|
||||
for fname in ["症状-方剂药材穴位桥接.json", "关联融合总览.json"]:
|
||||
fpath = os.path.join(OUTPUT_DIR, fname)
|
||||
if os.path.exists(fpath):
|
||||
fsize = os.path.getsize(fpath)
|
||||
print(f" ✓ {fname}: {fsize:,} bytes ({fsize/1024:.1f} KB)")
|
||||
else:
|
||||
print(f" ✗ {fname}: NOT FOUND")
|
||||
|
||||
print("\n完成!")
|
||||
Reference in new issue
Block a user