Initial commit

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
lofyer
2026-07-13 15:38:41 +08:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
commit 71db82393a
180 changed files with 170640 additions and 0 deletions
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""
测试脚本:验证数据引擎能正确读取和解析Excel数据
"""
import sys
from pathlib import Path
# 添加项目路径
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
from app.engines.data_engine import DataEngine
from app.config import DIMENSION_FRAMEWORK, SCHOOL_TYPE_MAP
import json
def main():
print("=" * 80)
print("数据引擎测试")
print("=" * 80)
engine = DataEngine()
engine.load_all()
# 1. 基本信息
summary = engine.summary()
print(f"\n📊 数据摘要:")
print(f" 学校数量: {summary['school_count']}")
print(f" 学校列表: {summary['schools']}")
print(f" 基础信息行数: {summary['basic_info_rows']}")
print(f" 课程实施行数: {summary['course_impl_rows']}")
print(f" 学科课程行数: {summary['subject_impl_rows']}")
# 2. 测试单校数据
test_school = "延安中学"
print(f"\n🏫 {test_school} 基本信息:")
info = engine.get_school_basic_info(test_school)
for k, v in info.items():
print(f" {k}: {v}")
# 3. 测试课程数据
print(f"\n📚 {test_school} 课程实施数据:")
course_df = engine.get_school_course_data(test_school)
print(f" 总行数: {len(course_df)}")
print(f" 涉及学科: {sorted(course_df['学科'].unique().tolist())}")
print(f" 涉及题号: {sorted(course_df['题号'].unique().tolist())[:20]}...")
# 4. 测试学科数据
print(f"\n🔬 {test_school} 学科课程数据:")
subject_df = engine.get_school_subject_data(test_school)
print(f" 总行数: {len(subject_df)}")
print(f" 涉及学科: {sorted(subject_df['学科'].unique().tolist())}")
# 5. 测试周课时数据
print(f"\n{test_school} 周课时数据(前10行):")
hours = engine.get_weekly_hours(test_school)
if len(hours) > 0:
print(hours[["学科", "年级", "学期", "字段名称", "字段取值"]].head(10).to_string())
else:
print(" 无课时数据")
# 6. 测试课程规范数据
print(f"\n📋 {test_school} 课程规范数据(前5条):")
norms = engine.get_course_norms(test_school)
for n in norms[:5]:
print(f" {n['字段名称']}: {n['字段取值']}")
# 7. 所有学校类型
print(f"\n🏷️ 学校类型:")
for school in engine.schools:
info = SCHOOL_TYPE_MAP.get(school, {})
print(f" {school}: {info.get('type', '未知')} ({info.get('nature', '')}) [{info.get('feature', '')}]")
print("\n✅ 数据引擎测试完成!")
if __name__ == "__main__":
main()
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env python3
"""
测试脚本:完整Pipeline - 数据加载→赋分→PCA→标准化→水平判定→统计分析
"""
import sys
from pathlib import Path
import json
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
from app.engines.data_engine import DataEngine
from app.engines.scoring_engine import ScoringEngine
from app.engines.stats_engine import StatsEngine
from app.config import DIMENSION_FRAMEWORK, LEVEL_THRESHOLDS
import logging
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
def main():
print("=" * 80)
print("完整Pipeline测试:Excel → 赋分 → PCA → 标准化 → 水平判定")
print("=" * 80)
# Step 1: 加载数据
print("\n[Step 1] 加载Excel数据...")
data_engine = DataEngine()
data_engine.load_all()
# Step 2: 赋分
print("\n[Step 2] 对所有学校进行赋分...")
scoring_engine = ScoringEngine(data_engine)
raw_scores = scoring_engine.score_all_schools()
# 打印赋分结果概要
print("\n📊 赋分结果概要:")
for school in data_engine.schools:
scores = raw_scores[school]
print(f"\n 🏫 {school}:")
for dim, vals in scores.items():
avg = sum(vals) / len(vals) if vals else 0
print(f" {dim}: {len(vals)}个指标, 均值={avg:.2f}, 值={[round(v,1) for v in vals[:5]]}{'...' if len(vals) > 5 else ''}")
# Step 3: 统计分析
print("\n[Step 3] PCA合成 + 标准化...")
stats_engine = StatsEngine()
sub_scores = stats_engine.compute_dimension_scores(raw_scores)
print(f"\n三级维度标准化得分 (均值50, 标准差10):")
print(sub_scores.round(2).to_string())
# Step 4: 二级维度聚合
print("\n[Step 4] 二级维度聚合...")
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
print(f"\n二级维度得分:")
print(dim_scores.round(2).to_string())
# Step 5: 水平判定
print("\n[Step 5] 水平判定...")
levels = stats_engine.compute_levels(sub_scores)
print(f"\n各学校各维度水平:")
print(levels.to_string())
# Step 6: 聚类分析
print("\n[Step 6] 聚类分析...")
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
clusters = stats_engine.cluster_analysis(dim_scores[dim_cols])
print(f"\n学校聚类结果:")
for school, cluster in clusters["school_clusters"].items():
print(f" {school}: {cluster}")
# Step 7: 相关性
print("\n[Step 7] 维度间相关性:")
corr = stats_engine.correlation_analysis(dim_scores)
print(corr.round(3).to_string())
# Step 8: 生成一所学校的完整报告数据
test_school = "延安中学"
print(f"\n[Step 8] 生成 {test_school} 的完整报告数据包...")
report_data = stats_engine.compute_school_report_data(test_school, sub_scores, dim_scores)
print(f"\n📋 {test_school} 报告数据概要:")
print(f" 总体得分: {report_data['overall']['score']}")
print(f" 区内排名: {report_data['overall']['rank_in_district']}/{report_data['overall']['total_schools']}")
print(f" 聚类类型: {report_data['overall']['cluster']}")
print(f"\n 二级维度得分:")
for dim, data in report_data['dimensions'].items():
print(f" {dim}: {data['score']} (区均值{data['district_avg']}, 差异{data['diff_district']:+.2f})")
print(f"\n 三级维度水平:")
for sub_dim, data in report_data['sub_dimensions'].items():
print(f" {sub_dim}: 得分{data['score']}, 水平{data['level']}, {data['level_description'][:30]}")
# 保存报告数据为JSON
output_dir = Path(__file__).parent.parent / "output"
output_dir.mkdir(exist_ok=True)
# 清理numpy类型以便JSON序列化
def clean_for_json(obj):
import numpy as np
if isinstance(obj, dict):
return {k: clean_for_json(v) for k, v in obj.items()}
elif isinstance(obj, list):
return [clean_for_json(v) for v in obj]
elif isinstance(obj, (np.integer,)):
return int(obj)
elif isinstance(obj, (np.floating,)):
return round(float(obj), 4)
elif isinstance(obj, np.ndarray):
return obj.tolist()
elif isinstance(obj, float):
return round(obj, 4)
return obj
report_clean = clean_for_json(report_data)
with open(output_dir / f"{test_school}_report_data.json", "w", encoding="utf-8") as f:
json.dump(report_clean, f, ensure_ascii=False, indent=2)
print(f"\n✅ 报告数据已保存到 output/{test_school}_report_data.json")
# 保存所有学校的得分汇总
all_scores = {
"sub_dimension_scores": clean_for_json(sub_scores.to_dict()),
"dimension_scores": clean_for_json(dim_scores.to_dict()),
"levels": clean_for_json(levels.to_dict()),
"clusters": clean_for_json(clusters),
}
with open(output_dir / "all_schools_scores.json", "w", encoding="utf-8") as f:
json.dump(all_scores, f, ensure_ascii=False, indent=2)
print(f"✅ 全校得分汇总已保存到 output/all_schools_scores.json")
print("\n" + "=" * 80)
print("✅ 完整Pipeline测试完成!")
print("=" * 80)
if __name__ == "__main__":
main()
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""
完整报告生成脚本:Excel → 赋分 → 统计 → LLM并行生成 → HTML报告
"""
import sys
import time
import argparse
import logging
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
from app.engines.data_engine import DataEngine
from app.engines.scoring_engine import ScoringEngine
from app.engines.stats_engine import StatsEngine
from app.engines.llm_engine import LLMEngine
from app.engines.report_renderer import ReportRenderer
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
datefmt="%H:%M:%S",
)
logger = logging.getLogger(__name__)
def progress_callback(completed, total, segment_id):
"""进度回调"""
pct = completed / total * 100
bar = "" * int(pct / 5) + "" * (20 - int(pct / 5))
print(f"\r [{bar}] {pct:.0f}% ({completed}/{total}) {segment_id:<40}", end="", flush=True)
def main():
parser = argparse.ArgumentParser(description="生成学校课程实施监测报告")
parser.add_argument("--school", type=str, default="延安中学", help="学校名称")
parser.add_argument("--no-cache", action="store_true", help="不使用LLM缓存")
parser.add_argument("--no-llm", action="store_true", help="跳过LLM生成(仅图表+数据)")
parser.add_argument("--list-schools", action="store_true", help="列出所有可用学校")
args = parser.parse_args()
start_time = time.time()
# ===== Step 1: 加载数据 =====
print("=" * 70)
print("📊 课程实施监测报告生成系统")
print("=" * 70)
logger.info("[1/5] 加载Excel数据...")
data_engine = DataEngine()
data_engine.load_all()
if args.list_schools:
print("\n可用学校:")
for s in data_engine.schools:
from app.config import SCHOOL_TYPE_MAP
info = SCHOOL_TYPE_MAP.get(s, {})
print(f" {s} ({info.get('type', '')})")
return
school = args.school
if school not in data_engine.schools:
print(f"\n❌ 学校 '{school}' 不在数据中。可用学校:")
for s in data_engine.schools:
print(f" - {s}")
return
print(f"\n🏫 目标学校: {school}")
# ===== Step 2: 赋分 =====
logger.info("[2/5] 全区赋分计算...")
scoring_engine = ScoringEngine(data_engine)
raw_scores = scoring_engine.score_all_schools()
# ===== Step 3: 统计分析 =====
logger.info("[3/5] PCA合成 + 标准化 + 统计分析...")
stats_engine = StatsEngine()
sub_scores = stats_engine.compute_dimension_scores(raw_scores)
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
report_data = stats_engine.compute_school_report_data(school, sub_scores, dim_scores)
# ===== Step 4: LLM生成 =====
if args.no_llm:
logger.info("[4/5] 跳过LLM生成(--no-llm模式)")
llm_sections = {}
else:
logger.info("[4/5] LLM并行生成报告文字...")
llm_engine = LLMEngine()
llm_engine.set_progress_callback(progress_callback)
llm_sections = llm_engine.generate_report_segments(
report_data,
use_cache=not args.no_cache,
)
print() # 换行
# ===== Step 5: 渲染HTML =====
logger.info("[5/5] 渲染HTML报告...")
renderer = ReportRenderer()
output_dir = Path(__file__).parent.parent / "output"
output_dir.mkdir(parents=True, exist_ok=True)
output_path = output_dir / f"{school}_报告.html"
renderer.render_to_file(report_data, llm_sections, output_path)
elapsed = time.time() - start_time
print(f"\n{'=' * 70}")
print(f"✅ 报告生成完成!")
print(f" 学校: {school}")
print(f" 总体得分: {report_data['overall']['score']}分 (区内第{report_data['overall']['rank_in_district']}名)")
print(f" LLM段落: {len(llm_sections)}")
print(f" 耗时: {elapsed:.1f}")
print(f" 输出: {output_path}")
print(f"{'=' * 70}")
if __name__ == "__main__":
main()
+207
View File
@@ -0,0 +1,207 @@
#!/usr/bin/env python3
"""
批量报告生成脚本:为长宁区全部9所学校生成课程实施监测报告
数据加载和赋分/统计只做一次,LLM和渲染对每校独立执行
"""
import sys
import time
import argparse
import logging
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent / "backend"))
from app.engines.data_engine import DataEngine
from app.engines.scoring_engine import ScoringEngine
from app.engines.stats_engine import StatsEngine
from app.engines.llm_engine import LLMEngine
from app.engines.report_renderer import ReportRenderer
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
datefmt="%H:%M:%S",
)
logger = logging.getLogger(__name__)
def progress_callback(completed, total, segment_id):
"""进度回调"""
pct = completed / total * 100
bar = "" * int(pct / 5) + "" * (20 - int(pct / 5))
print(f"\r [{bar}] {pct:.0f}% ({completed}/{total}) {segment_id:<40}", end="", flush=True)
def main():
parser = argparse.ArgumentParser(description="批量生成全部学校课程实施监测报告")
parser.add_argument("--no-cache", action="store_true", help="不使用LLM缓存")
parser.add_argument("--no-llm", action="store_true", help="跳过LLM生成(仅图表+数据)")
parser.add_argument("--schools", nargs="*", help="指定学校列表(默认全部9所)")
args = parser.parse_args()
total_start = time.time()
print("=" * 70)
print("📊 课程实施监测报告 — 批量生成系统")
print("=" * 70)
# ===== Step 1: 加载数据(只做一次) =====
logger.info("[全局 1/3] 加载Excel数据...")
data_engine = DataEngine()
data_engine.load_all()
# ===== Step 2: 全区赋分(只做一次) =====
logger.info("[全局 2/3] 全区赋分计算...")
scoring_engine = ScoringEngine(data_engine)
raw_scores = scoring_engine.score_all_schools()
# ===== Step 3: 统计分析(只做一次) =====
logger.info("[全局 3/3] PCA合成 + 标准化...")
stats_engine = StatsEngine()
sub_scores = stats_engine.compute_dimension_scores(raw_scores)
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
# 确定生成哪些学校
all_schools = list(data_engine.schools)
if args.schools:
schools = []
for s in args.schools:
if s in all_schools:
schools.append(s)
else:
print(f" ⚠️ 学校 '{s}' 不在数据中,跳过")
if not schools:
print("❌ 没有有效的学校,退出")
return
else:
schools = all_schools
print(f"\n🏫 将为以下 {len(schools)} 所学校生成报告:")
for i, s in enumerate(schools, 1):
from app.config import SCHOOL_TYPE_MAP
info = SCHOOL_TYPE_MAP.get(s, {})
print(f" {i}. {s} ({info.get('type', '未知类型')})")
# ===== 初始化引擎 =====
renderer = ReportRenderer()
llm_engine = None
if not args.no_llm:
llm_engine = LLMEngine()
llm_engine.set_progress_callback(progress_callback)
output_dir = Path(__file__).parent.parent / "output"
output_dir.mkdir(parents=True, exist_ok=True)
# ===== 逐校生成 =====
results = []
for idx, school in enumerate(schools, 1):
school_start = time.time()
print(f"\n{'' * 70}")
print(f"🔄 [{idx}/{len(schools)}] 正在生成: {school}")
print(f"{'' * 70}")
try:
# 统计分析(学校专属数据包)
report_data = stats_engine.compute_school_report_data(school, sub_scores, dim_scores)
# LLM生成
if args.no_llm:
print(" ⏭️ 跳过LLM生成(--no-llm模式)")
llm_sections = {}
else:
print(" 🤖 LLM并行生成报告文字...")
llm_sections = llm_engine.generate_report_segments(
report_data,
use_cache=not args.no_cache,
)
print() # 换行(progress bar之后)
# 渲染HTML
output_path = output_dir / f"{school}_报告.html"
renderer.render_to_file(report_data, llm_sections, output_path)
# 保存数据JSON
import json
def _json_safe(obj):
"""处理numpy类型"""
import numpy as np
if isinstance(obj, (np.integer,)):
return int(obj)
if isinstance(obj, (np.floating,)):
return float(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
json_path = output_dir / f"{school}_report_data.json"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(report_data, f, ensure_ascii=False, indent=2, default=_json_safe)
elapsed = time.time() - school_start
score = report_data["overall"]["score"]
rank = report_data["overall"]["rank_in_district"]
cluster = report_data["overall"]["cluster"]
llm_count = len(llm_sections)
results.append({
"school": school,
"status": "",
"score": score,
"rank": rank,
"cluster": cluster,
"llm_segments": llm_count,
"time": elapsed,
"output": str(output_path),
})
print(f" ✅ 完成! 得分={score}分 排名={rank}/9 类型={cluster} LLM={llm_count}段 耗时={elapsed:.1f}s")
except Exception as e:
elapsed = time.time() - school_start
logger.error(f" ❌ 生成失败: {e}", exc_info=True)
results.append({
"school": school,
"status": "",
"error": str(e),
"time": elapsed,
})
# ===== 汇总报告 =====
total_elapsed = time.time() - total_start
print(f"\n{'=' * 70}")
print(f"📋 批量生成结果汇总")
print(f"{'=' * 70}")
print(f"{'学校':<10} {'状态':<4} {'得分':<8} {'排名':<8} {'类型':<8} {'LLM段':<8} {'耗时':<8}")
print(f"{'' * 62}")
success_count = 0
for r in results:
if r["status"] == "":
success_count += 1
print(f"{r['school']:<10} {r['status']:<4} {r['score']:<8.2f} {r['rank']}/9{'':<5} {r['cluster']:<8} {r['llm_segments']:<8} {r['time']:.1f}s")
else:
print(f"{r['school']:<10} {r['status']:<4} {'失败: ' + r.get('error', '未知')}")
print(f"{'' * 62}")
print(f"成功: {success_count}/{len(schools)} | 总耗时: {total_elapsed:.1f}s")
print(f"输出目录: {output_dir}")
print(f"{'=' * 70}")
# 保存汇总JSON
import json
summary_path = output_dir / "batch_summary.json"
with open(summary_path, "w", encoding="utf-8") as f:
json.dump({
"generated_at": time.strftime("%Y-%m-%d %H:%M:%S"),
"total_schools": len(schools),
"success_count": success_count,
"total_time": round(total_elapsed, 1),
"results": results,
}, f, ensure_ascii=False, indent=2)
print(f"汇总文件: {summary_path}")
if __name__ == "__main__":
main()