Initial commit
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
lofyer
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
commit
71db82393a
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
二期数据 ETL 脚本
|
||||
将原始平台导出格式（每题一个Sheet + 15个学科独立文件）
|
||||
转换为与一期相同的扁平表格式，供后续赋分/分析引擎使用
|
||||
|
||||
输入：二期/[整理前] 二期_课程实施+教材使用情况表/
|
||||
输出：report-admin/data/era2/ 下的3张扁平表
|
||||
- era2_基础信息表.xlsx
|
||||
- era2_课程实施情况表.xlsx
|
||||
- era2_学科课程实施情况表.xlsx
|
||||
|
||||
一期目标格式：
|
||||
基础信息表列: 题号, 题型, 题目ID, 题目内容, 字段ID, 字段名称, 字段值, 年级, 学期, 区, 学校名称, 办学性质, 学校类别, 学校等级, 地域类型, 特色类型, 等级B, 等级C
|
||||
课程实施情况表列: 题号, 题型, 字段ID, 字段名称, 字段取值, 选项文字, 学科, 学期, 年级, 学校简称, 所在区, 学校类别, 学校性质, 所处地区, 学校类型, 学校类型编号
|
||||
学科课程实施情况表列: 题号, 题型, 字段ID, 字段名称, 字段取值, 学科, 选项文字, 学校性质, 所在区, 学校类别, 学校特色, 学校简称, 所处地区, 学校类型, 学校类型编号
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import logging
|
||||
import time
|
||||
import glob
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ===== 路径配置 =====
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent # report-admin/
|
||||
DATA_ROOT = PROJECT_ROOT.parent # 20260301-邱老师-校长画像/
|
||||
ERA2_RAW_DIR = DATA_ROOT / "二期" / "[整理前] 二期_课程实施+教材使用情况表"
|
||||
ERA2_OUTPUT_DIR = PROJECT_ROOT / "data" / "era2"
|
||||
|
||||
|
||||
def load_raw_file(filepath: Path) -> Dict:
|
||||
"""
|
||||
加载一个二期原始 Excel 文件，返回：
|
||||
{
|
||||
'home': DataFrame, # 首页元信息
|
||||
'data_sheets': {题号: DataFrame}, # 每题的数据
|
||||
'field_map': DataFrame, # 字段映射关系
|
||||
'dim_map': DataFrame, # 维度映射关系
|
||||
'type_stats': DataFrame, # 题型统计
|
||||
}
|
||||
"""
|
||||
logger.info(f" 加载: {filepath.name}")
|
||||
xls = pd.ExcelFile(filepath)
|
||||
|
||||
result = {
|
||||
'home': pd.read_excel(xls, sheet_name='首页'),
|
||||
'data_sheets': {},
|
||||
'field_map': None,
|
||||
'dim_map': None,
|
||||
'type_stats': None,
|
||||
}
|
||||
|
||||
for sn in xls.sheet_names:
|
||||
if sn == '首页':
|
||||
continue
|
||||
elif sn == '字段映射关系':
|
||||
result['field_map'] = pd.read_excel(xls, sheet_name=sn)
|
||||
elif sn == '维度映射关系':
|
||||
result['dim_map'] = pd.read_excel(xls, sheet_name=sn)
|
||||
elif sn == '题型统计':
|
||||
result['type_stats'] = pd.read_excel(xls, sheet_name=sn)
|
||||
elif sn.isdigit():
|
||||
result['data_sheets'][int(sn)] = pd.read_excel(xls, sheet_name=sn)
|
||||
|
||||
xls.close()
|
||||
total_rows = sum(len(df) for df in result['data_sheets'].values())
|
||||
logger.info(f" → {len(result['data_sheets'])}个题Sheet, {total_rows}行数据")
|
||||
return result
|
||||
|
||||
|
||||
def build_type_map(type_stats: pd.DataFrame) -> Dict[int, str]:
|
||||
"""从题型统计构建 题号→题型 的映射"""
|
||||
return dict(zip(type_stats['题号'].astype(int), type_stats['题型']))
|
||||
|
||||
|
||||
def extract_school_short_name(full_name: str) -> str:
|
||||
"""
|
||||
从全称提取简称
|
||||
'上海市延安中学' → '延安中学'
|
||||
'华东政法大学附属中学' → '华政附中'
|
||||
'华东师范大学附属天山学校' → '天山学校'
|
||||
保留全称作为默认
|
||||
"""
|
||||
# 不做过度简化，保留全称让下游配置来处理映射
|
||||
return full_name
|
||||
|
||||
|
||||
def extract_district_name(region_str: str) -> str:
|
||||
"""
|
||||
从区域维度提取区名
|
||||
'上海市长宁区教育学院' → '长宁区'
|
||||
'上海市青浦区教师进修学院' → '青浦区'
|
||||
"""
|
||||
if pd.isna(region_str):
|
||||
return ""
|
||||
s = str(region_str)
|
||||
# 提取 "XX区" 部分
|
||||
for suffix in ['教育学院', '教师进修学院', '教育委员会']:
|
||||
s = s.replace(suffix, '')
|
||||
s = s.replace('上海市', '')
|
||||
return s.strip()
|
||||
|
||||
|
||||
# ===== 1. 基础信息表 ETL =====
|
||||
def transform_basic_info(raw: Dict) -> pd.DataFrame:
|
||||
"""
|
||||
将二期基础信息表转为一期格式
|
||||
一期列: 题号, 题型, 题目ID, 题目内容, 字段ID, 字段名称, 字段值, 年级, 学期, 区, 学校名称, ...
|
||||
二期列: 问题id, 字段id, 字段名称, 字段取值, 维度id, 维度名称, 学科维度, 年级维度, 学期维度, 学校维度, 区域维度, 用户维度, 状态, 问卷提交时间
|
||||
"""
|
||||
logger.info("转换基础信息表...")
|
||||
type_map = build_type_map(raw['type_stats'])
|
||||
rows = []
|
||||
|
||||
for q_num, df in sorted(raw['data_sheets'].items()):
|
||||
q_type = type_map.get(q_num, '未知')
|
||||
for _, row in df.iterrows():
|
||||
rows.append({
|
||||
'题号': q_num,
|
||||
'题型': q_type,
|
||||
'题目ID': row.get('问题id', ''),
|
||||
'题目内容': '', # 二期原始数据无此字段
|
||||
'字段ID': row.get('字段id', ''),
|
||||
'字段名称': row.get('字段名称', ''),
|
||||
'字段值': row.get('字段取值', ''),
|
||||
'年级': row.get('年级维度', '不分年级'),
|
||||
'学期': row.get('学期维度', ''),
|
||||
'区': extract_district_name(row.get('区域维度', '')),
|
||||
'学校名称': row.get('学校维度', ''),
|
||||
# 以下字段二期原始数据无，留空后续由配置补充
|
||||
'办学性质': '',
|
||||
'学校类别': '',
|
||||
'学校等级': '',
|
||||
'地域类型': '',
|
||||
'特色类型': '',
|
||||
'等级B': '',
|
||||
'等级C': '',
|
||||
})
|
||||
|
||||
result = pd.DataFrame(rows)
|
||||
# 强制所有列为字符串，避免混合类型导致 parquet 报错
|
||||
result = result.astype(str)
|
||||
logger.info(f" 基础信息表: {len(result)}行")
|
||||
return result
|
||||
|
||||
|
||||
# ===== 2. 课程实施情况表 ETL =====
|
||||
def transform_course_impl(raw: Dict) -> pd.DataFrame:
|
||||
"""
|
||||
将二期课程实施情况表转为一期格式
|
||||
一期列: 题号, 题型, 字段ID, 字段名称, 字段取值, 选项文字, 学科, 学期, 年级, 学校简称, 所在区, ...
|
||||
"""
|
||||
logger.info("转换课程实施情况表...")
|
||||
type_map = build_type_map(raw['type_stats'])
|
||||
rows = []
|
||||
|
||||
for q_num, df in sorted(raw['data_sheets'].items()):
|
||||
q_type = type_map.get(q_num, '未知')
|
||||
for _, row in df.iterrows():
|
||||
rows.append({
|
||||
'题号': str(q_num),
|
||||
'题型': q_type,
|
||||
'字段ID': row.get('字段id', ''),
|
||||
'字段名称': row.get('字段名称', ''),
|
||||
'字段取值': row.get('字段取值', ''),
|
||||
'选项文字': row.get('维度名称', ''), # 二期中维度名称对应选项文字
|
||||
'学科': row.get('学科维度', '不分学科'),
|
||||
'学期': row.get('学期维度', ''),
|
||||
'年级': row.get('年级维度', '不分年级'),
|
||||
'学校简称': row.get('学校维度', ''),
|
||||
'所在区': extract_district_name(row.get('区域维度', '')),
|
||||
# 以下字段二期原始数据无，留空
|
||||
'学校类别': '',
|
||||
'学校性质': '',
|
||||
'所处地区': '',
|
||||
'学校类型': '',
|
||||
'学校类型编号': '',
|
||||
})
|
||||
|
||||
result = pd.DataFrame(rows)
|
||||
result = result.astype(str)
|
||||
logger.info(f" 课程实施情况表: {len(result)}行")
|
||||
return result
|
||||
|
||||
|
||||
# ===== 3. 学科课程实施情况表 ETL =====
|
||||
def transform_subject_impl(raw_files: Dict[str, Dict]) -> pd.DataFrame:
|
||||
"""
|
||||
将二期的15个学科独立文件合并为一张表（一期格式）
|
||||
一期列: 题号, 题型, 字段ID, 字段名称, 字段取值, 学科, 选项文字, 学校性质, 所在区, ...
|
||||
"""
|
||||
logger.info("转换学科课程实施情况表（合并15个学科文件）...")
|
||||
all_rows = []
|
||||
|
||||
for subject, raw in sorted(raw_files.items()):
|
||||
type_map = build_type_map(raw['type_stats'])
|
||||
subject_rows = 0
|
||||
|
||||
for q_num, df in sorted(raw['data_sheets'].items()):
|
||||
q_type = type_map.get(q_num, '未知')
|
||||
for _, row in df.iterrows():
|
||||
all_rows.append({
|
||||
'题号': str(q_num),
|
||||
'题型': q_type,
|
||||
'字段ID': row.get('字段id', ''),
|
||||
'字段名称': row.get('字段名称', ''),
|
||||
'字段取值': row.get('字段取值', ''),
|
||||
'学科': subject,
|
||||
'选项文字': row.get('维度名称', ''),
|
||||
'学校性质': '',
|
||||
'所在区': extract_district_name(row.get('区域维度', '')),
|
||||
'学校类别': '',
|
||||
'学校特色': '',
|
||||
'学校简称': row.get('学校维度', ''),
|
||||
'所处地区': '',
|
||||
'学校类型': '',
|
||||
'学校类型编号': '',
|
||||
})
|
||||
subject_rows += 1
|
||||
|
||||
logger.info(f" {subject}: {subject_rows}行")
|
||||
|
||||
result = pd.DataFrame(all_rows)
|
||||
result = result.astype(str)
|
||||
logger.info(f" 学科课程实施情况表合计: {len(result)}行")
|
||||
return result
|
||||
|
||||
|
||||
# ===== 主流程 =====
|
||||
def main():
|
||||
start = time.time()
|
||||
|
||||
print("=" * 70)
|
||||
print("📊 二期数据 ETL 转换")
|
||||
print(f" 输入: {ERA2_RAW_DIR}")
|
||||
print(f" 输出: {ERA2_OUTPUT_DIR}")
|
||||
print("=" * 70)
|
||||
|
||||
# 检查输入目录
|
||||
if not ERA2_RAW_DIR.exists():
|
||||
logger.error(f"❌ 输入目录不存在: {ERA2_RAW_DIR}")
|
||||
return
|
||||
|
||||
# 创建输出目录
|
||||
ERA2_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ===== 1. 基础信息表 =====
|
||||
print("\n[1/3] 基础信息表")
|
||||
raw_basic = load_raw_file(ERA2_RAW_DIR / "第二期_学校基础信息表.xlsx")
|
||||
df_basic = transform_basic_info(raw_basic)
|
||||
out_basic = ERA2_OUTPUT_DIR / "era2_基础信息表.xlsx"
|
||||
out_basic_pq = ERA2_OUTPUT_DIR / "era2_基础信息表.parquet"
|
||||
df_basic.to_parquet(out_basic_pq, index=False)
|
||||
logger.info(f" ✅ 保存 → {out_basic_pq}")
|
||||
|
||||
# ===== 2. 课程实施情况表 =====
|
||||
print("\n[2/3] 课程实施情况表")
|
||||
raw_course = load_raw_file(ERA2_RAW_DIR / "第二期_学校课程实施情况表.xlsx")
|
||||
df_course = transform_course_impl(raw_course)
|
||||
out_course_pq = ERA2_OUTPUT_DIR / "era2_课程实施情况表.parquet"
|
||||
df_course.to_parquet(out_course_pq, index=False)
|
||||
logger.info(f" ✅ 保存 → {out_course_pq}")
|
||||
|
||||
# ===== 3. 学科课程实施情况表 =====
|
||||
print("\n[3/3] 学科课程实施情况表（15个学科文件）")
|
||||
subject_files = sorted(ERA2_RAW_DIR.glob("第二期_*学科课程实施情况表.xlsx"))
|
||||
raw_subjects = {}
|
||||
for f in subject_files:
|
||||
subject_name = f.name.replace("第二期_", "").replace("学科课程实施情况表.xlsx", "")
|
||||
raw_subjects[subject_name] = load_raw_file(f)
|
||||
|
||||
df_subject = transform_subject_impl(raw_subjects)
|
||||
|
||||
# 学科表超过Excel行数上限(1,048,576)，保存为parquet + 按区分片xlsx
|
||||
out_subject_parquet = ERA2_OUTPUT_DIR / "era2_学科课程实施情况表.parquet"
|
||||
df_subject.to_parquet(out_subject_parquet, index=False)
|
||||
logger.info(f" ✅ 保存 (parquet) → {out_subject_parquet}")
|
||||
|
||||
# 注：如需xlsx格式可按区分片，但parquet格式已满足分析需求
|
||||
|
||||
# ===== 汇总 =====
|
||||
elapsed = time.time() - start
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"✅ ETL 转换完成! 耗时 {elapsed:.1f}s")
|
||||
print(f" 基础信息表: {len(df_basic):>8,}行")
|
||||
print(f" 课程实施情况表: {len(df_course):>8,}行")
|
||||
print(f" 学科课程实施情况表: {len(df_subject):>8,}行")
|
||||
print(f" 学校数量: {df_course['学校简称'].nunique()}")
|
||||
print(f" 区域数量: {df_course['所在区'].nunique()}")
|
||||
print(f" 学科数量: {df_subject['学科'].nunique()}")
|
||||
print(f" 输出目录: {ERA2_OUTPUT_DIR}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
# 保存元信息
|
||||
meta = {
|
||||
'基础信息表行数': len(df_basic),
|
||||
'课程实施情况表行数': len(df_course),
|
||||
'学科课程实施情况表行数': len(df_subject),
|
||||
'学校列表': sorted(df_course['学校简称'].unique().tolist()),
|
||||
'区域列表': sorted(df_course['所在区'].unique().tolist()),
|
||||
'学科列表': sorted(df_subject['学科'].unique().tolist()),
|
||||
'课程实施题数': raw_course['type_stats']['题号'].max(),
|
||||
'各学科题数': {s: r['type_stats']['题号'].max() for s, r in raw_subjects.items()},
|
||||
}
|
||||
import json
|
||||
meta_path = ERA2_OUTPUT_DIR / "etl_meta.json"
|
||||
with open(meta_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(meta, f, ensure_ascii=False, indent=2, default=str)
|
||||
logger.info(f" 元信息 → {meta_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
二期数据验证脚本
|
||||
检查 ETL 产出的 parquet 文件是否符合预期，
|
||||
并与一期数据做结构和字段ID的对比
|
||||
"""
|
||||
import pandas as pd
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent # report-admin/
|
||||
ERA1_DATA = PROJECT_ROOT / "data"
|
||||
ERA2_DATA = PROJECT_ROOT / "data" / "era2"
|
||||
|
||||
|
||||
def section(title):
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f" {title}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
def main():
|
||||
print("📊 二期数据验证 & 一期对比")
|
||||
|
||||
# ===== 加载二期数据 =====
|
||||
section("1. 加载二期 ETL 输出")
|
||||
df_basic = pd.read_parquet(ERA2_DATA / "era2_基础信息表.parquet")
|
||||
df_course = pd.read_parquet(ERA2_DATA / "era2_课程实施情况表.parquet")
|
||||
df_subject = pd.read_parquet(ERA2_DATA / "era2_学科课程实施情况表.parquet")
|
||||
print(f" 基础信息表: {df_basic.shape}")
|
||||
print(f" 课程实施情况表: {df_course.shape}")
|
||||
print(f" 学科课程实施情况表: {df_subject.shape}")
|
||||
|
||||
# ===== 加载一期数据 =====
|
||||
section("2. 加载一期数据")
|
||||
df1_basic = pd.read_excel(ERA1_DATA / "长宁区_基础信息表.xlsx")
|
||||
df1_course = pd.read_excel(ERA1_DATA / "长宁区_课程实施情况表.xlsx")
|
||||
df1_subject = pd.read_excel(ERA1_DATA / "长宁区_学科课程实施情况表.xlsx")
|
||||
print(f" 基础信息表: {df1_basic.shape}")
|
||||
print(f" 课程实施情况表: {df1_course.shape}")
|
||||
print(f" 学科课程实施情况表: {df1_subject.shape}")
|
||||
|
||||
# ===== 列名对比 =====
|
||||
section("3. 列名对比")
|
||||
for name, (d1, d2) in {
|
||||
'基础信息表': (df1_basic, df_basic),
|
||||
'课程实施情况表': (df1_course, df_course),
|
||||
'学科课程实施情况表': (df1_subject, df_subject),
|
||||
}.items():
|
||||
c1 = set(d1.columns)
|
||||
c2 = set(d2.columns)
|
||||
print(f"\n [{name}]")
|
||||
print(f" 一期列: {sorted(c1)}")
|
||||
print(f" 二期列: {sorted(c2)}")
|
||||
print(f" 一期有/二期无: {sorted(c1 - c2) or '无'}")
|
||||
print(f" 二期有/一期无: {sorted(c2 - c1) or '无'}")
|
||||
print(f" 共有列: {sorted(c1 & c2)}")
|
||||
|
||||
# ===== 字段ID对比（课程实施情况表） =====
|
||||
section("4. 课程实施情况表 - 字段ID对比")
|
||||
# 一期用 "字段ID"，二期用 "字段ID"
|
||||
id_col_1 = '字段ID' if '字段ID' in df1_course.columns else None
|
||||
id_col_2 = '字段ID' if '字段ID' in df_course.columns else None
|
||||
|
||||
if id_col_1 and id_col_2:
|
||||
ids1 = set(df1_course[id_col_1].dropna().unique())
|
||||
ids2 = set(df_course[id_col_2].dropna().unique())
|
||||
print(f" 一期字段ID数: {len(ids1)}")
|
||||
print(f" 二期字段ID数: {len(ids2)}")
|
||||
common = ids1 & ids2
|
||||
print(f" 共有: {len(common)}")
|
||||
print(f" 一期有/二期无: {len(ids1 - ids2)}")
|
||||
if ids1 - ids2:
|
||||
print(f" 缺失: {sorted(ids1 - ids2)[:20]}{'...' if len(ids1 - ids2) > 20 else ''}")
|
||||
print(f" 二期有/一期无: {len(ids2 - ids1)}")
|
||||
if ids2 - ids1:
|
||||
print(f" 新增: {sorted(ids2 - ids1)[:20]}{'...' if len(ids2 - ids1) > 20 else ''}")
|
||||
else:
|
||||
print(f" ⚠️ 无法对比: 一期列={id_col_1}, 二期列={id_col_2}")
|
||||
# 尝试用字段名称对比
|
||||
names1 = set(df1_course['字段名称'].dropna().unique())
|
||||
names2 = set(df_course['字段名称'].dropna().unique())
|
||||
print(f" [改用字段名称对比]")
|
||||
print(f" 一期字段名称数: {len(names1)}")
|
||||
print(f" 二期字段名称数: {len(names2)}")
|
||||
common = names1 & names2
|
||||
print(f" 共有: {len(common)}")
|
||||
only1 = sorted(names1 - names2)
|
||||
only2 = sorted(names2 - names1)
|
||||
print(f" 一期有/二期无: {len(only1)}")
|
||||
if only1:
|
||||
for n in only1[:30]:
|
||||
print(f" - {n}")
|
||||
print(f" 二期有/一期无: {len(only2)}")
|
||||
if only2:
|
||||
for n in only2[:30]:
|
||||
print(f" + {n}")
|
||||
|
||||
# ===== 学科课程表 - 字段名称对比 =====
|
||||
section("5. 学科课程实施情况表 - 字段名称对比")
|
||||
names1 = set(df1_subject['字段名称'].dropna().unique())
|
||||
names2 = set(df_subject['字段名称'].dropna().unique())
|
||||
print(f" 一期字段名称数: {len(names1)}")
|
||||
print(f" 二期字段名称数: {len(names2)}")
|
||||
common = names1 & names2
|
||||
print(f" 共有: {len(common)}")
|
||||
only1 = sorted(names1 - names2)
|
||||
only2 = sorted(names2 - names1)
|
||||
print(f" 一期有/二期无: {len(only1)}")
|
||||
if only1:
|
||||
for n in only1[:20]:
|
||||
print(f" - {n}")
|
||||
if len(only1) > 20:
|
||||
print(f" ... (共{len(only1)}个)")
|
||||
print(f" 二期有/一期无: {len(only2)}")
|
||||
if only2:
|
||||
for n in only2[:20]:
|
||||
print(f" + {n}")
|
||||
if len(only2) > 20:
|
||||
print(f" ... (共{len(only2)}个)")
|
||||
|
||||
# ===== 长宁区数据提取验证 =====
|
||||
section("6. 长宁区数据提取")
|
||||
cn_course = df_course[df_course['所在区'] == '长宁区']
|
||||
cn_subject = df_subject[df_subject['所在区'] == '长宁区']
|
||||
print(f" 课程实施情况表(长宁区): {len(cn_course)}行")
|
||||
print(f" 学科课程实施情况表(长宁区): {len(cn_subject)}行")
|
||||
print(f" 长宁区学校: {sorted(cn_course['学校简称'].unique())}")
|
||||
print(f" 一期学校: {sorted(df1_course['学校简称'].unique()) if '学校简称' in df1_course.columns else sorted(df1_course['学校名称'].unique())}")
|
||||
|
||||
# ===== 数据质量 =====
|
||||
section("7. 数据质量检查")
|
||||
for name, df in [('课程实施情况表', df_course), ('学科课程实施情况表', df_subject)]:
|
||||
total = len(df)
|
||||
nulls = df['字段取值'].isna().sum() + (df['字段取值'] == '').sum() + (df['字段取值'] == 'nan').sum()
|
||||
print(f" [{name}]")
|
||||
print(f" 总行数: {total}")
|
||||
print(f" 字段取值为空/nan: {nulls} ({nulls/total*100:.1f}%)")
|
||||
print(f" 学校数: {df['学校简称'].nunique()}")
|
||||
|
||||
print(f"\n✅ 验证完成!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
二期 Pipeline 测试脚本
|
||||
数据加载 → PCA赋分(SPSS对齐) → 标准化 → 水平判定 → 聚类 → 统计分析
|
||||
不调用 LLM，不生成 HTML 报告
|
||||
|
||||
支持两种模式：
|
||||
--district 长宁区 只分析长宁区8所学校（与一期对比验证）
|
||||
--district all 分析全部13个区176所学校
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
# 添加当前目录到 path（加载 config_era2 和 data_engine_era2）
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from data_engine_era2 import DataEngineEra2
|
||||
from engines.pca_scoring_engine_era2 import PcaScoringEngineEra2
|
||||
from engines.stats_engine_era2 import StatsEngineEra2 as StatsEngine
|
||||
from config_era2 import DIMENSION_FRAMEWORK, LEVEL_THRESHOLDS
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def clean_for_json(obj):
|
||||
"""处理 numpy 类型"""
|
||||
if isinstance(obj, dict):
|
||||
return {k: clean_for_json(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [clean_for_json(v) for v in obj]
|
||||
elif isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
elif isinstance(obj, (np.floating,)):
|
||||
return round(float(obj), 4)
|
||||
elif isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
elif isinstance(obj, float):
|
||||
return round(obj, 4)
|
||||
return obj
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="二期数据Pipeline测试")
|
||||
parser.add_argument("--district", type=str, default="长宁区",
|
||||
help="区域筛选，'all'表示全部区域，默认'长宁区'")
|
||||
parser.add_argument("--school", type=str, default=None,
|
||||
help="指定一所学校查看详情")
|
||||
parser.add_argument("--list-schools", action="store_true",
|
||||
help="列出所有可用学校")
|
||||
args = parser.parse_args()
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
print("=" * 70)
|
||||
print("📊 二期数据 Pipeline 测试")
|
||||
print("=" * 70)
|
||||
|
||||
# ===== Step 1: 加载数据 =====
|
||||
district = None if args.district == "all" else args.district
|
||||
logger.info(f"[1/5] 加载二期数据... (区域: {args.district})")
|
||||
data_engine = DataEngineEra2(district_filter=district)
|
||||
data_engine.load_all()
|
||||
|
||||
summary = data_engine.summary()
|
||||
print(f"\n📋 数据摘要:")
|
||||
print(f" 学校数: {summary['school_count']}")
|
||||
print(f" 基础信息行数: {summary['basic_info_rows']}")
|
||||
print(f" 课程实施行数: {summary['course_impl_rows']}")
|
||||
print(f" 学科课程行数: {summary['subject_impl_rows']}")
|
||||
|
||||
if args.list_schools:
|
||||
print(f"\n可用学校 ({len(data_engine.schools)}所):")
|
||||
for i, s in enumerate(data_engine.schools, 1):
|
||||
print(f" {i:3d}. {s}")
|
||||
return
|
||||
|
||||
# ===== Step 2: PCA赋分（SPSS对齐） =====
|
||||
logger.info(f"[2/5] PCA赋分计算 ({len(data_engine.schools)}所学校, SPSS对齐)...")
|
||||
pca_engine = PcaScoringEngineEra2(data_engine)
|
||||
pca_sub_scores = pca_engine.compute_all()
|
||||
|
||||
# 打印PCA赋分摘要
|
||||
print(f"\n📊 PCA赋分结果摘要 (前5所学校):")
|
||||
for school in list(data_engine.schools)[:5]:
|
||||
if school in pca_sub_scores.index:
|
||||
row = pca_sub_scores.loc[school]
|
||||
vals = [f"{c}:{row[c]:.1f}" for c in row.index[:4] if not np.isnan(row[c])]
|
||||
print(f" {school}: {', '.join(vals)}...")
|
||||
|
||||
# ===== Step 3: 标准化 =====
|
||||
logger.info("[3/5] 全市基准标准化...")
|
||||
stats_engine = StatsEngine()
|
||||
sub_scores = stats_engine.compute_dimension_scores_pca(pca_sub_scores)
|
||||
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
|
||||
|
||||
print(f"\n📊 三级维度标准化得分 (均值50, 标准差10):")
|
||||
print(f" 形状: {sub_scores.shape}")
|
||||
print(f" 均值: {sub_scores.mean().mean():.2f}")
|
||||
print(f" 标准差: {sub_scores.std().mean():.2f}")
|
||||
|
||||
print(f"\n📊 二级维度得分 (前5所学校):")
|
||||
print(dim_scores.head().round(2).to_string())
|
||||
|
||||
# ===== Step 4: 水平判定 =====
|
||||
logger.info("[4/5] 水平判定...")
|
||||
levels = stats_engine.compute_levels(sub_scores)
|
||||
print(f"\n📊 水平分布统计:")
|
||||
for dim in levels.columns:
|
||||
dist = levels[dim].value_counts().sort_index()
|
||||
dist_str = " ".join([f"水平{k}:{v}所" for k, v in dist.items()])
|
||||
print(f" {dim}: {dist_str}")
|
||||
|
||||
# ===== Step 5: 聚类 + 相关性 =====
|
||||
logger.info("[5/5] 聚类分析 + 相关性...")
|
||||
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
|
||||
clusters = stats_engine.cluster_analysis(dim_scores[dim_cols])
|
||||
|
||||
cluster_dist = {}
|
||||
for school, cluster in clusters["school_clusters"].items():
|
||||
cluster_dist[cluster] = cluster_dist.get(cluster, 0) + 1
|
||||
print(f"\n📊 聚类分布: {cluster_dist}")
|
||||
|
||||
corr = stats_engine.correlation_analysis(dim_scores)
|
||||
print(f"\n📊 维度间相关性:")
|
||||
print(corr.round(3).to_string())
|
||||
|
||||
# ===== 单校详情 =====
|
||||
test_school = args.school or (data_engine.schools[0] if data_engine.schools else None)
|
||||
if test_school and test_school in data_engine.schools:
|
||||
print(f"\n{'─' * 70}")
|
||||
print(f"🏫 {test_school} 详细报告数据")
|
||||
print(f"{'─' * 70}")
|
||||
|
||||
report_data = stats_engine.compute_school_report_data(test_school, sub_scores, dim_scores)
|
||||
|
||||
print(f" 总体得分: {report_data['overall']['score']}")
|
||||
print(f" 排名: {report_data['overall']['rank_in_district']}/{report_data['overall']['total_schools']}")
|
||||
print(f" 聚类类型: {report_data['overall']['cluster']}")
|
||||
|
||||
print(f"\n 二级维度:")
|
||||
for dim, data in report_data['dimensions'].items():
|
||||
print(f" {dim}: {data['score']} (区均{data['district_avg']}, 差{data['diff_district']:+.2f}) [{data['cluster']}]")
|
||||
|
||||
print(f"\n 三级维度:")
|
||||
for sub_dim, data in report_data['sub_dimensions'].items():
|
||||
print(f" {sub_dim}: {data['score']} 水平{data['level']} (区均{data['district_avg']})")
|
||||
|
||||
# 保存报告数据
|
||||
output_dir = Path(__file__).parent.parent.parent / "output" / "era2"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report_clean = clean_for_json(report_data)
|
||||
json_path = output_dir / f"{test_school}_report_data.json"
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(report_clean, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n ✅ 报告数据 → {json_path}")
|
||||
|
||||
# ===== 保存全量得分 =====
|
||||
output_dir = Path(__file__).parent.parent.parent / "output" / "era2"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
all_data = {
|
||||
"district_filter": args.district,
|
||||
"school_count": len(data_engine.schools),
|
||||
"sub_dimension_scores": clean_for_json(sub_scores.to_dict()),
|
||||
"dimension_scores": clean_for_json(dim_scores.to_dict()),
|
||||
"levels": clean_for_json(levels.to_dict()),
|
||||
"clusters": clean_for_json(clusters),
|
||||
}
|
||||
scores_path = output_dir / f"all_scores_{args.district}.json"
|
||||
with open(scores_path, "w", encoding="utf-8") as f:
|
||||
json.dump(all_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"✅ Pipeline测试完成! 耗时 {elapsed:.1f}s")
|
||||
print(f" 区域: {args.district}")
|
||||
print(f" 学校数: {len(data_engine.schools)}")
|
||||
print(f" 得分文件: {scores_path}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
二期报告生成脚本框架
|
||||
数据 → PCA赋分(SPSS对齐) → 统计 → [LLM并行生成] → HTML报告
|
||||
|
||||
⚠️ 默认 --no-llm 模式，不会调用 LLM（避免产生费用）
|
||||
若需启用 LLM 生成，手动传 --enable-llm 参数
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from data_engine_era2 import DataEngineEra2
|
||||
from engines.pca_scoring_engine_era2 import PcaScoringEngineEra2
|
||||
from engines.stats_engine_era2 import StatsEngineEra2 as StatsEngine
|
||||
from config_era2 import DIMENSION_FRAMEWORK
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def clean_for_json(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {k: clean_for_json(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [clean_for_json(v) for v in obj]
|
||||
elif isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
elif isinstance(obj, (np.floating,)):
|
||||
return round(float(obj), 4)
|
||||
elif isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
elif isinstance(obj, float):
|
||||
return round(obj, 4)
|
||||
return obj
|
||||
|
||||
|
||||
def progress_callback(completed, total, segment_id):
|
||||
pct = completed / total * 100
|
||||
bar = "█" * int(pct / 5) + "░" * (20 - int(pct / 5))
|
||||
print(f"\r [{bar}] {pct:.0f}% ({completed}/{total}) {segment_id:<40}", end="", flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="二期报告生成")
|
||||
parser.add_argument("--school", type=str, required=True,
|
||||
help="学校名称（简称或全称均可）")
|
||||
parser.add_argument("--district", type=str, default="长宁区",
|
||||
help="区域筛选，'all'表示全部，默认'长宁区'")
|
||||
parser.add_argument("--enable-llm", action="store_true",
|
||||
help="启用LLM生成（会产生API调用费用！）")
|
||||
parser.add_argument("--no-cache", action="store_true",
|
||||
help="不使用LLM缓存")
|
||||
parser.add_argument("--no-scoring-cache", action="store_true",
|
||||
help="不使用赋分缓存（强制重新计算全市赋分）")
|
||||
parser.add_argument("--list-schools", action="store_true",
|
||||
help="列出所有可用学校")
|
||||
parser.add_argument("--enable-agent", action="store_true",
|
||||
help="在报告中嵌入AI对话助手（右下角浮动按钮）")
|
||||
parser.add_argument("--lang", type=str, default="zh", choices=["zh", "en", "both"],
|
||||
help="报告语言：zh（中文，默认）/ en（英文）/ both（同时生成两份）")
|
||||
args = parser.parse_args()
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
print("=" * 70)
|
||||
print("📊 二期课程实施监测报告生成系统")
|
||||
if not args.enable_llm:
|
||||
print("⚠️ LLM 已禁用（--no-llm 模式），仅生成数据+图表")
|
||||
if args.enable_agent:
|
||||
print("🤖 AI 对话助手已启用")
|
||||
print("=" * 70)
|
||||
|
||||
# ===== Step 1: 加载数据 =====
|
||||
district = None if args.district == "all" else args.district
|
||||
logger.info(f"[1/5] 加载二期数据... (区域: {args.district}, 全市基准)")
|
||||
# 加载全市数据，按区筛选报告范围
|
||||
data_engine = DataEngineEra2(district_filter=district)
|
||||
data_engine.load_all()
|
||||
district_schools = data_engine.schools # 本区学校列表
|
||||
|
||||
# 全市数据引擎（用于赋分全市学校）
|
||||
if data_engine.use_city_data and district:
|
||||
city_engine = DataEngineEra2(district_filter=None) # 不筛选区
|
||||
city_engine.load_all()
|
||||
all_schools_for_scoring = city_engine
|
||||
logger.info(f" 全市基准: {len(city_engine.schools)}校, 本区: {len(district_schools)}校")
|
||||
else:
|
||||
city_engine = None
|
||||
all_schools_for_scoring = data_engine
|
||||
|
||||
if args.list_schools:
|
||||
print(f"\n可用学校 ({len(district_schools)}所):")
|
||||
for i, s in enumerate(district_schools, 1):
|
||||
print(f" {i:3d}. {s}")
|
||||
return
|
||||
|
||||
school = args.school
|
||||
if school not in district_schools:
|
||||
# 尝试从映射查找
|
||||
from config_era2 import SCHOOL_NAME_SHORT_TO_FULL, SCHOOL_NAME_FULL_TO_SHORT
|
||||
if school in SCHOOL_NAME_SHORT_TO_FULL:
|
||||
pass
|
||||
elif school in SCHOOL_NAME_FULL_TO_SHORT:
|
||||
school = SCHOOL_NAME_FULL_TO_SHORT[school]
|
||||
|
||||
if school not in all_schools_for_scoring.schools:
|
||||
print(f"\n❌ 学校 '{args.school}' 不在数据中。本区可用学校:")
|
||||
for s in district_schools:
|
||||
print(f" - {s}")
|
||||
return
|
||||
|
||||
print(f"\n🏫 目标学校: {school}")
|
||||
print(f"📍 数据范围: {args.district} ({len(district_schools)}所学校)")
|
||||
if city_engine:
|
||||
print(f"📊 标准化基准: 全市{len(city_engine.schools)}所学校")
|
||||
|
||||
# ===== Step 2: PCA赋分（SPSS对齐，全市所有学校） =====
|
||||
logger.info(f"[2/5] PCA赋分计算 ({len(all_schools_for_scoring.schools)}校, SPSS对齐)...")
|
||||
pca_engine = PcaScoringEngineEra2(all_schools_for_scoring)
|
||||
pca_sub_scores = pca_engine.compute_all()
|
||||
|
||||
# ===== Step 3: 统计分析（全市基准标准化） =====
|
||||
logger.info("[3/5] 全市基准标准化 + 统计分析...")
|
||||
stats_engine = StatsEngine()
|
||||
sub_scores = stats_engine.compute_dimension_scores_pca(pca_sub_scores)
|
||||
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
|
||||
report_data = stats_engine.compute_school_report_data(
|
||||
school, sub_scores, dim_scores,
|
||||
district_schools=district_schools,
|
||||
)
|
||||
|
||||
# 注入区域信息（模板需要）
|
||||
report_data["district"] = args.district
|
||||
report_data["total_schools_in_district"] = len(district_schools)
|
||||
|
||||
# 决定要生成的语言列表
|
||||
langs = ["zh", "en"] if args.lang == "both" else [args.lang]
|
||||
|
||||
# ===== Step 4: LLM生成（按语言分别生成；缓存按 lang 隔离） =====
|
||||
llm_sections_by_lang = {}
|
||||
if args.enable_llm:
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "backend"))
|
||||
from app.engines.llm_engine import LLMEngine
|
||||
llm_engine = LLMEngine()
|
||||
llm_engine.set_progress_callback(progress_callback)
|
||||
for lg in langs:
|
||||
logger.info(f"[4/5] LLM并行生成报告文字 (lang={lg}) ...")
|
||||
llm_sections_by_lang[lg] = llm_engine.generate_report_segments(
|
||||
report_data,
|
||||
use_cache=not args.no_cache,
|
||||
lang=lg,
|
||||
)
|
||||
print()
|
||||
else:
|
||||
logger.info("[4/5] 跳过LLM生成（--no-llm 模式）")
|
||||
for lg in langs:
|
||||
llm_sections_by_lang[lg] = {}
|
||||
|
||||
# ===== Step 5: 渲染HTML =====
|
||||
from engines.report_renderer_era2 import ReportRendererEra2 as ReportRenderer
|
||||
renderer = ReportRenderer()
|
||||
|
||||
output_dir = Path(__file__).parent.parent.parent / "output" / "era2" / args.district
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_paths = {}
|
||||
for lg in langs:
|
||||
logger.info(f"[5/5] 渲染HTML报告 (lang={lg}) ...")
|
||||
suffix = "_report_en.html" if lg == "en" else "_报告.html"
|
||||
output_path = output_dir / f"{school}{suffix}"
|
||||
renderer.render_to_file(
|
||||
report_data, llm_sections_by_lang[lg], output_path,
|
||||
enable_agent=args.enable_agent, lang=lg,
|
||||
)
|
||||
output_paths[lg] = output_path
|
||||
|
||||
# 保存报告数据JSON（一份，与语言无关）
|
||||
json_path = output_dir / f"{school}_report_data.json"
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(clean_for_json(report_data), f, ensure_ascii=False, indent=2)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"✅ 报告生成完成!")
|
||||
print(f" 学校: {school}")
|
||||
print(f" 区域: {args.district} ({len(data_engine.schools)}所学校)")
|
||||
print(f" 总体得分: {report_data['overall']['score']}分 (第{report_data['overall']['rank_in_district']}名)")
|
||||
for lg in langs:
|
||||
n_sec = len(llm_sections_by_lang.get(lg, {}))
|
||||
print(f" LLM段落({lg}): {n_sec}个 {'(已禁用)' if not args.enable_llm else ''}")
|
||||
print(f" AI助手: {'✅ 已嵌入' if args.enable_agent else '❌ 未启用'}")
|
||||
print(f" 耗时: {elapsed:.1f}秒")
|
||||
for lg, p in output_paths.items():
|
||||
print(f" HTML({lg}): {p}")
|
||||
print(f" JSON: {json_path}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
二期批量报告生成脚本
|
||||
数据加载/PCA赋分(SPSS对齐)/统计只做一次，LLM和渲染对每校独立执行
|
||||
|
||||
⚠️ 默认 --no-llm 模式，不调用 LLM
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from data_engine_era2 import DataEngineEra2
|
||||
from engines.pca_scoring_engine_era2 import PcaScoringEngineEra2
|
||||
from engines.stats_engine_era2 import StatsEngineEra2 as StatsEngine
|
||||
from config_era2 import DIMENSION_FRAMEWORK
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def clean_for_json(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {k: clean_for_json(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [clean_for_json(v) for v in obj]
|
||||
elif isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
elif isinstance(obj, (np.floating,)):
|
||||
return round(float(obj), 4)
|
||||
elif isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
elif isinstance(obj, float):
|
||||
return round(obj, 4)
|
||||
return obj
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="二期批量报告生成")
|
||||
parser.add_argument("--district", type=str, default="长宁区",
|
||||
help="区域筛选，'all'表示全部")
|
||||
parser.add_argument("--schools", nargs="*",
|
||||
help="指定学校列表（默认该区全部）")
|
||||
parser.add_argument("--enable-llm", action="store_true",
|
||||
help="启用LLM（会产生API费用！）")
|
||||
parser.add_argument("--no-cache", action="store_true",
|
||||
help="不使用LLM缓存")
|
||||
parser.add_argument("--enable-agent", action="store_true",
|
||||
help="在报告中嵌入AI对话助手")
|
||||
args = parser.parse_args()
|
||||
|
||||
total_start = time.time()
|
||||
|
||||
print("=" * 70)
|
||||
print("📊 二期课程实施监测报告 — 批量生成")
|
||||
if not args.enable_llm:
|
||||
print("⚠️ LLM 已禁用，仅生成数据+图表报告")
|
||||
if args.enable_agent:
|
||||
print("🤖 AI 对话助手已启用")
|
||||
print("=" * 70)
|
||||
|
||||
# ===== 全局步骤（只做一次） =====
|
||||
district = None if args.district == "all" else args.district
|
||||
logger.info(f"[全局 1/3] 加载二期数据... (区域: {args.district}, 全市基准)")
|
||||
data_engine = DataEngineEra2(district_filter=district)
|
||||
data_engine.load_all()
|
||||
district_schools = data_engine.schools
|
||||
|
||||
# 全市数据引擎（用于赋分全市学校做基准）
|
||||
if data_engine.use_city_data and district:
|
||||
city_engine = DataEngineEra2(district_filter=None)
|
||||
city_engine.load_all()
|
||||
all_schools_for_scoring = city_engine
|
||||
logger.info(f" 全市基准: {len(city_engine.schools)}校, 本区: {len(district_schools)}校")
|
||||
else:
|
||||
city_engine = None
|
||||
all_schools_for_scoring = data_engine
|
||||
|
||||
logger.info(f"[全局 2/3] PCA赋分 ({len(all_schools_for_scoring.schools)}所学校, SPSS对齐)...")
|
||||
pca_engine = PcaScoringEngineEra2(all_schools_for_scoring)
|
||||
pca_sub_scores = pca_engine.compute_all()
|
||||
|
||||
logger.info("[全局 3/3] 全市基准标准化...")
|
||||
stats_engine = StatsEngine()
|
||||
sub_scores = stats_engine.compute_dimension_scores_pca(pca_sub_scores)
|
||||
dim_scores = stats_engine.compute_dimension_aggregates(sub_scores)
|
||||
|
||||
# 确定学校列表（本区）
|
||||
all_schools = district_schools
|
||||
if args.schools:
|
||||
schools = [s for s in args.schools if s in all_schools]
|
||||
skipped = [s for s in args.schools if s not in all_schools]
|
||||
for s in skipped:
|
||||
print(f" ⚠️ 学校 '{s}' 不在数据中，跳过")
|
||||
else:
|
||||
schools = all_schools
|
||||
|
||||
print(f"\n🏫 将为 {len(schools)} 所学校生成报告:")
|
||||
for i, s in enumerate(schools, 1):
|
||||
print(f" {i}. {s}")
|
||||
|
||||
# 初始化引擎
|
||||
from engines.report_renderer_era2 import ReportRendererEra2 as ReportRenderer
|
||||
renderer = ReportRenderer()
|
||||
|
||||
llm_engine = None
|
||||
if args.enable_llm:
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "backend"))
|
||||
from app.engines.llm_engine import LLMEngine
|
||||
llm_engine = LLMEngine()
|
||||
|
||||
output_dir = Path(__file__).parent.parent.parent / "output" / "era2" / args.district
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ===== 逐校生成 =====
|
||||
results = []
|
||||
for idx, school in enumerate(schools, 1):
|
||||
school_start = time.time()
|
||||
print(f"\n{'─' * 70}")
|
||||
print(f"🔄 [{idx}/{len(schools)}] {school}")
|
||||
|
||||
try:
|
||||
report_data = stats_engine.compute_school_report_data(
|
||||
school, sub_scores, dim_scores,
|
||||
district_schools=district_schools,
|
||||
)
|
||||
|
||||
# 注入区域信息（模板需要）
|
||||
report_data["district"] = args.district
|
||||
report_data["total_schools_in_district"] = len(district_schools)
|
||||
|
||||
if args.enable_llm and llm_engine:
|
||||
llm_sections = llm_engine.generate_report_segments(
|
||||
report_data, use_cache=not args.no_cache)
|
||||
else:
|
||||
llm_sections = {}
|
||||
|
||||
# 渲染
|
||||
html_path = output_dir / f"{school}_报告.html"
|
||||
renderer.render_to_file(report_data, llm_sections, html_path,
|
||||
enable_agent=args.enable_agent)
|
||||
|
||||
# 保存JSON
|
||||
json_path = output_dir / f"{school}_report_data.json"
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(clean_for_json(report_data), f, ensure_ascii=False, indent=2)
|
||||
|
||||
elapsed = time.time() - school_start
|
||||
score = report_data["overall"]["score"]
|
||||
rank = report_data["overall"]["rank_in_district"]
|
||||
total = report_data["overall"]["total_schools"]
|
||||
|
||||
results.append({
|
||||
"school": school, "status": "✅",
|
||||
"score": score, "rank": rank, "total": total,
|
||||
"time": elapsed,
|
||||
})
|
||||
print(f" ✅ 得分={score} 排名={rank}/{total} 耗时={elapsed:.1f}s")
|
||||
|
||||
except Exception as e:
|
||||
elapsed = time.time() - school_start
|
||||
logger.error(f" ❌ 失败: {e}", exc_info=True)
|
||||
results.append({"school": school, "status": "❌", "error": str(e), "time": elapsed})
|
||||
|
||||
# ===== 汇总 =====
|
||||
total_elapsed = time.time() - total_start
|
||||
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"📋 批量生成汇总")
|
||||
print(f"{'=' * 70}")
|
||||
print(f"{'学校':<15} {'状态':<4} {'得分':<8} {'排名':<10} {'耗时':<8}")
|
||||
print(f"{'─' * 50}")
|
||||
|
||||
success = 0
|
||||
for r in results:
|
||||
if r["status"] == "✅":
|
||||
success += 1
|
||||
print(f"{r['school']:<15} {r['status']:<4} {r['score']:<8.2f} {r['rank']}/{r['total']:<7} {r['time']:.1f}s")
|
||||
else:
|
||||
print(f"{r['school']:<15} {r['status']:<4} 失败: {r.get('error', '')[:30]}")
|
||||
|
||||
print(f"{'─' * 50}")
|
||||
print(f"成功: {success}/{len(schools)} | 总耗时: {total_elapsed:.1f}s")
|
||||
print(f"输出目录: {output_dir}")
|
||||
|
||||
# 保存汇总
|
||||
summary = {
|
||||
"generated_at": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"era": 2,
|
||||
"district": args.district,
|
||||
"total_schools": len(schools),
|
||||
"success": success,
|
||||
"total_time": round(total_elapsed, 1),
|
||||
"llm_enabled": args.enable_llm,
|
||||
"results": results,
|
||||
}
|
||||
with open(output_dir / "batch_summary.json", "w", encoding="utf-8") as f:
|
||||
json.dump(summary, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
覆盖率审计脚本
|
||||
检查：
|
||||
1. 二期ETL输出的字段名称，赋分引擎实际用到了多少
|
||||
2. 赋分引擎里按关键词匹配的字段，在二期数据中能否命中
|
||||
3. 每个维度的赋分数据源覆盖情况
|
||||
"""
|
||||
import sys
|
||||
import re
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "backend"))
|
||||
|
||||
from data_engine_era2 import DataEngineEra2
|
||||
from app.engines.scoring_engine import ScoringEngine
|
||||
from config_era2 import DIMENSION_FRAMEWORK, SUBJECTS
|
||||
|
||||
|
||||
def section(title):
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f" {title}")
|
||||
print(f"{'=' * 80}")
|
||||
|
||||
|
||||
def main():
|
||||
# ===== 加载二期全量数据（不分区，看全貌） =====
|
||||
section("1. 加载二期全量数据")
|
||||
engine = DataEngineEra2(district_filter=None)
|
||||
engine.load_all()
|
||||
print(f" 学校数: {len(engine.schools)}")
|
||||
|
||||
# 获取全量字段名称
|
||||
course_df = engine._course_impl
|
||||
subject_df = engine._subject_impl
|
||||
basic_df = engine._basic_info
|
||||
|
||||
course_fields = sorted(course_df["字段名称"].dropna().unique())
|
||||
subject_fields = sorted(subject_df["字段名称"].dropna().unique())
|
||||
basic_fields = sorted(basic_df["字段名称"].dropna().unique())
|
||||
|
||||
print(f" 课程实施表字段名称数: {len(course_fields)}")
|
||||
print(f" 学科课程表字段名称数: {len(subject_fields)}")
|
||||
print(f" 基础信息表字段名称数: {len(basic_fields)}")
|
||||
|
||||
# ===== 2. 提取赋分引擎中所有用到的字段名称和关键词 =====
|
||||
section("2. 赋分引擎字段引用分析")
|
||||
|
||||
# 从 ScoringEngine 源码中提取所有字符串常量（字段名称/关键词）
|
||||
src = inspect.getsource(ScoringEngine)
|
||||
|
||||
# 提取所有中文字符串（字段名称）
|
||||
# 匹配双引号和单引号中的中文字符串
|
||||
field_refs = set()
|
||||
keyword_refs = set()
|
||||
|
||||
# 精确字段引用：df["字段名称"] == "xxx" 或 字段名称 == field
|
||||
exact_patterns = re.findall(r'["\']([^"\']*[\u4e00-\u9fff][^"\']*)["\']', src)
|
||||
for p in exact_patterns:
|
||||
# 跳过注释性文字
|
||||
if len(p) > 30 or '：' in p or '。' in p or '赋分' in p:
|
||||
continue
|
||||
field_refs.add(p)
|
||||
|
||||
# 关键词引用：str.contains("xxx")
|
||||
contains_patterns = re.findall(r'\.str\.contains\(["\']([^"\']+)["\']', src)
|
||||
for p in contains_patterns:
|
||||
# 这些是用 | 分隔的关键词
|
||||
for kw in p.split('|'):
|
||||
keyword_refs.add(kw.strip())
|
||||
|
||||
# 其他关键词引用（in循环中的列表）
|
||||
keyword_lists = re.findall(r'for (?:keyword|field|hw_type|resource) in \[([^\]]+)\]', src)
|
||||
for kl in keyword_lists:
|
||||
items = re.findall(r'["\']([^"\']+)["\']', kl)
|
||||
for item in items:
|
||||
if any('\u4e00' <= c <= '\u9fff' for c in item):
|
||||
field_refs.add(item)
|
||||
|
||||
print(f"\n 赋分引擎中精确引用的字段名称: {len(field_refs)}个")
|
||||
print(f" 赋分引擎中关键词引用: {len(keyword_refs)}个")
|
||||
|
||||
# ===== 3. 逐一检查精确字段在二期数据中的命中情况 =====
|
||||
section("3. 精确字段匹配检查")
|
||||
|
||||
all_data_fields = set(course_fields) | set(subject_fields) | set(basic_fields)
|
||||
|
||||
matched = []
|
||||
missing = []
|
||||
for f in sorted(field_refs):
|
||||
if f in all_data_fields:
|
||||
matched.append(f)
|
||||
else:
|
||||
missing.append(f)
|
||||
|
||||
print(f"\n ✅ 命中: {len(matched)}/{len(field_refs)}")
|
||||
print(f" ❌ 未命中: {len(missing)}/{len(field_refs)}")
|
||||
|
||||
if missing:
|
||||
print(f"\n 未命中的字段（赋分引擎引用但二期数据中不存在）:")
|
||||
for f in missing:
|
||||
# 尝试模糊匹配
|
||||
fuzzy = [df for df in all_data_fields if f.replace('_', '') in df.replace('_', '') or df.replace('_', '') in f.replace('_', '')]
|
||||
if fuzzy:
|
||||
print(f" ❌ {f}")
|
||||
print(f" → 可能对应: {fuzzy[:3]}")
|
||||
else:
|
||||
print(f" ❌ {f} (无近似匹配)")
|
||||
|
||||
# ===== 4. 关键词匹配检查 =====
|
||||
section("4. 关键词匹配检查")
|
||||
|
||||
for kw in sorted(keyword_refs):
|
||||
course_hits = course_df[course_df["字段名称"].str.contains(kw, na=False)]["字段名称"].unique()
|
||||
subject_hits = subject_df[subject_df["字段名称"].str.contains(kw, na=False)]["字段名称"].unique()
|
||||
total = len(course_hits) + len(subject_hits)
|
||||
status = "✅" if total > 0 else "❌"
|
||||
print(f" {status} '{kw}': 课程表{len(course_hits)}个, 学科表{len(subject_hits)}个")
|
||||
if total == 0:
|
||||
# 看看有没有相近的
|
||||
all_names = list(course_fields) + list(subject_fields)
|
||||
similar = [n for n in all_names if kw[:2] in n][:3]
|
||||
if similar:
|
||||
print(f" → 近似: {similar}")
|
||||
|
||||
# ===== 5. 按维度逐一检查赋分数据覆盖 =====
|
||||
section("5. 按维度检查赋分数据覆盖（抽样一所学校）")
|
||||
|
||||
# 取一所数据较完整的学校
|
||||
test_school = engine.schools[0]
|
||||
print(f" 测试学校: {test_school}")
|
||||
|
||||
scoring = ScoringEngine(engine)
|
||||
scores = scoring._score_school(test_school)
|
||||
|
||||
print(f"\n {'维度':<20} {'赋分项数':>8} {'均值':>8} {'是否有效':>8}")
|
||||
print(f" {'─' * 50}")
|
||||
for dim, vals in scores.items():
|
||||
n = len(vals)
|
||||
avg = np.mean(vals) if vals else 0
|
||||
# 判断是否有效：是否全是默认值
|
||||
is_default = (n <= 1 and abs(avg - 1.0) < 0.01) or (n <= 1 and abs(avg - 1.5) < 0.01) or (n <= 1 and abs(avg - 2.0) < 0.01) or (n <= 1 and abs(avg - 0.5) < 0.01)
|
||||
status = "⚠️ 默认值" if is_default else "✅"
|
||||
print(f" {dim:<20} {n:>8} {avg:>8.2f} {status:>8}")
|
||||
|
||||
# ===== 6. 多校抽样统计 =====
|
||||
section("6. 多校统计：各维度赋分项数分布")
|
||||
|
||||
# 取前20所学校统计
|
||||
sample_schools = engine.schools[:20]
|
||||
dim_stats = defaultdict(list)
|
||||
|
||||
for school in sample_schools:
|
||||
s = scoring._score_school(school)
|
||||
for dim, vals in s.items():
|
||||
dim_stats[dim].append(len(vals))
|
||||
|
||||
print(f"\n 抽样学校数: {len(sample_schools)}")
|
||||
print(f"\n {'维度':<20} {'最小':>6} {'最大':>6} {'均值':>6} {'全为1':>8}")
|
||||
print(f" {'─' * 50}")
|
||||
for dim in DIMENSION_FRAMEWORK:
|
||||
for sub in DIMENSION_FRAMEWORK[dim]["sub_dimensions"]:
|
||||
vals = dim_stats.get(sub, [0])
|
||||
min_v = min(vals)
|
||||
max_v = max(vals)
|
||||
avg_v = np.mean(vals)
|
||||
all_one = sum(1 for v in vals if v <= 1)
|
||||
warn = "⚠️" if all_one > len(vals) * 0.5 else ""
|
||||
print(f" {sub:<20} {min_v:>6} {max_v:>6} {avg_v:>6.1f} {all_one:>4}/{len(vals)} {warn}")
|
||||
|
||||
# ===== 7. 二期新增但赋分引擎未使用的字段 =====
|
||||
section("7. 二期数据中存在但赋分引擎未引用的高频字段（Top 30）")
|
||||
|
||||
# 统计二期中每个字段名称出现的学校数
|
||||
course_field_school_count = course_df.groupby("字段名称")["学校名称"].nunique().sort_values(ascending=False)
|
||||
subject_field_school_count = subject_df.groupby("字段名称")["学校名称"].nunique().sort_values(ascending=False)
|
||||
|
||||
# 过滤掉已被赋分引擎引用的
|
||||
unused_course = course_field_school_count[~course_field_school_count.index.isin(field_refs)]
|
||||
unused_subject = subject_field_school_count[~subject_field_school_count.index.isin(field_refs)]
|
||||
|
||||
# 进一步过滤：去掉被关键词匹配可能命中的
|
||||
def is_keyword_matched(field_name):
|
||||
for kw in keyword_refs:
|
||||
if kw in str(field_name):
|
||||
return True
|
||||
return False
|
||||
|
||||
unused_course_strict = unused_course[~unused_course.index.map(is_keyword_matched)]
|
||||
unused_subject_strict = unused_subject[~unused_subject.index.map(is_keyword_matched)]
|
||||
|
||||
print(f"\n [课程实施表] 未被引用的字段 (按学校覆盖率排序, Top 20):")
|
||||
for field, cnt in unused_course_strict.head(20).items():
|
||||
print(f" {field}: {cnt}所学校有数据")
|
||||
|
||||
print(f"\n [学科课程表] 未被引用的字段 (Top 20):")
|
||||
for field, cnt in unused_subject_strict.head(20).items():
|
||||
print(f" {field}: {cnt}所学校有数据")
|
||||
|
||||
print(f"\n✅ 覆盖率审计完成!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
深度字段审计：逐方法 × 逐字段，精确检查赋分引擎每个赋分函数的数据覆盖。
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "backend"))
|
||||
|
||||
from data_engine_era2 import DataEngineEra2
|
||||
from config_era2 import SUBJECTS
|
||||
|
||||
|
||||
def section(title):
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f" {title}")
|
||||
print(f"{'=' * 80}")
|
||||
|
||||
|
||||
def check_exact_field(df, field_name, label=""):
|
||||
"""检查精确字段名是否存在"""
|
||||
hits = df[df["字段名称"] == field_name]
|
||||
n_schools = hits["学校名称"].nunique() if len(hits) > 0 else 0
|
||||
status = f"✅ {n_schools}校" if n_schools > 0 else "❌ 不存在"
|
||||
return n_schools, status
|
||||
|
||||
|
||||
def check_contains_field(df, keyword, label=""):
|
||||
"""检查关键词匹配的字段"""
|
||||
hits = df[df["字段名称"].str.contains(keyword, na=False)]
|
||||
n_fields = hits["字段名称"].nunique() if len(hits) > 0 else 0
|
||||
n_schools = hits["学校名称"].nunique() if len(hits) > 0 else 0
|
||||
return n_fields, n_schools
|
||||
|
||||
|
||||
def find_similar(all_fields, keyword, top=5):
|
||||
"""模糊搜索"""
|
||||
results = []
|
||||
for f in all_fields:
|
||||
if keyword in str(f):
|
||||
results.append(f)
|
||||
return results[:top]
|
||||
|
||||
|
||||
def main():
|
||||
engine = DataEngineEra2(district_filter=None)
|
||||
engine.load_all()
|
||||
|
||||
course_df = engine._course_impl
|
||||
subject_df = engine._subject_impl
|
||||
|
||||
all_course_fields = sorted(course_df["字段名称"].dropna().unique())
|
||||
all_subject_fields = sorted(subject_df["字段名称"].dropna().unique())
|
||||
all_fields = set(all_course_fields) | set(all_subject_fields)
|
||||
|
||||
total_schools = len(engine.schools)
|
||||
print(f"总学校数: {total_schools}")
|
||||
|
||||
# ===== 按赋分方法逐个检查 =====
|
||||
|
||||
results = {} # method -> {field: (status, detail)}
|
||||
|
||||
def audit_method(method_name, dim_name, checks):
|
||||
"""
|
||||
checks: list of (source, type, field_or_keyword, description)
|
||||
source: 'course' or 'subject'
|
||||
type: 'exact' or 'contains'
|
||||
"""
|
||||
results[method_name] = {"dim": dim_name, "fields": []}
|
||||
df_map = {"course": course_df, "subject": subject_df}
|
||||
for source, check_type, field, desc in checks:
|
||||
df = df_map[source]
|
||||
if check_type == "exact":
|
||||
n_schools, status = check_exact_field(df, field)
|
||||
similar = []
|
||||
if n_schools == 0:
|
||||
similar = find_similar(all_course_fields if source == "course" else all_subject_fields, field[:4] if len(field) > 4 else field)
|
||||
results[method_name]["fields"].append({
|
||||
"field": field, "source": source, "type": check_type,
|
||||
"desc": desc, "n_schools": n_schools, "status": status,
|
||||
"similar": similar,
|
||||
})
|
||||
else: # contains
|
||||
n_fields, n_schools = check_contains_field(df, field)
|
||||
results[method_name]["fields"].append({
|
||||
"field": field, "source": source, "type": check_type,
|
||||
"desc": desc, "n_schools": n_schools, "n_fields": n_fields,
|
||||
"status": f"✅ {n_fields}字段/{n_schools}校" if n_fields > 0 else "❌",
|
||||
})
|
||||
|
||||
# ===== 1. 国家标准遵循 =====
|
||||
audit_method("_score_national_standard", "国家标准遵循", [
|
||||
("subject", "exact", "学科必修课周课时", "必修课周课时"),
|
||||
("subject", "exact", "学科选择性必修课周课时", "选必课周课时"),
|
||||
("subject", "exact", "学科类选修课周课时", "选修课周课时"),
|
||||
])
|
||||
|
||||
# ===== 2. 课程结构建设 =====
|
||||
audit_method("_score_course_structure", "课程结构建设", [
|
||||
("course", "exact", "跨学科选修课门数", "跨学科选修课"),
|
||||
("course", "exact", "综合主题选修课门数", "综合主题选修课"),
|
||||
("course", "exact", "综合实践选修课门数", "综合实践选修课"),
|
||||
("course", "exact", "三年应完成的研究性学习数量", "研究性学习数量"),
|
||||
("course", "exact", "三年社会考察个数", "社会考察"),
|
||||
("course", "exact", "三年志愿服务时长", "志愿服务"),
|
||||
])
|
||||
|
||||
# ===== 3. 课程规范落实 =====
|
||||
audit_method("_score_course_norms", "课程规范落实", [
|
||||
("course", "contains", "建设规范文本", "规范文本"),
|
||||
("course", "contains", "档案", "档案规范"),
|
||||
("course", "contains", "已经建成并使用", "已建成使用"),
|
||||
("course", "contains", "已经建成尚未使用", "已建成未使用"),
|
||||
("course", "contains", "尚未建成", "尚未建成"),
|
||||
])
|
||||
|
||||
# ===== 4. 教学方式变革 =====
|
||||
audit_method("_score_teaching_reform", "教学方式变革", [
|
||||
("subject", "contains", "认识程度", "认识程度"),
|
||||
("subject", "contains", "落实程度", "落实程度"),
|
||||
("subject", "contains", "体现形式", "实施方式体现形式"),
|
||||
])
|
||||
|
||||
# ===== 5. 作业设计与管理变革 =====
|
||||
audit_method("_score_homework_reform", "作业设计与管理变革", [
|
||||
("subject", "exact", "实践类作业_有布置", "实践类作业"),
|
||||
("subject", "exact", "表现类作业_有布置", "表现类作业"),
|
||||
("subject", "exact", "跨学科作业_有布置", "跨学科作业"),
|
||||
("subject", "exact", "团队合作类作业_有布置", "团队合作类作业"),
|
||||
("subject", "contains", "作业属性标注", "作业属性标注"),
|
||||
("subject", "exact", "回家作业时长控制_学校控制", "时长控制-学校"),
|
||||
("subject", "exact", "回家作业时长控制_教研组负责", "时长控制-教研组"),
|
||||
("subject", "exact", "作业批改范围_全部批改", "全部批改"),
|
||||
("subject", "exact", "作业批改范围_部分练习", "部分批改"),
|
||||
])
|
||||
|
||||
# ===== 6. 个性化辅导 =====
|
||||
audit_method("_score_personalized_tutoring", "学科发展的个性化辅导", [
|
||||
("subject", "exact", "个别辅导时长_每周>2h", "辅导>2h"),
|
||||
("subject", "exact", "个别辅导时长_每周1~2h", "辅导1-2h"),
|
||||
("subject", "exact", "个别辅导时长_每周<1h", "辅导<1h"),
|
||||
("subject", "exact", "个别辅导时长_几乎无", "辅导-几乎无"),
|
||||
("subject", "exact", "个别辅导实施方式_分散辅导", "分散辅导"),
|
||||
("subject", "exact", "个别辅导实施方式_分组统一辅导", "分组辅导"),
|
||||
("subject", "exact", "个别辅导实施方式_班级统一辅导", "班级辅导"),
|
||||
])
|
||||
|
||||
# ===== 7. 生涯发展指导 =====
|
||||
audit_method("_score_career_guidance", "学生生涯发展指导", [
|
||||
("course", "exact", "生涯指导实施方式_专设课程", "专设课程"),
|
||||
("course", "exact", "生涯指导实施方式_社会考察和志愿服务", "社考志愿"),
|
||||
("course", "exact", "完成生涯指导的学生占比_90%+", "覆盖率90%+"),
|
||||
("course", "exact", "生涯指导教师构成_本校和外聘结合", "师资-结合"),
|
||||
("course", "exact", "生涯指导教师构成_本校为主", "师资-本校为主"),
|
||||
("course", "exact", "生涯指导的校外资源支持程度", "校外资源"),
|
||||
("course", "exact", "生涯指导的校内资源支持程度", "校内资源"),
|
||||
])
|
||||
|
||||
# ===== 8. 培训支持 =====
|
||||
audit_method("_score_training_support", "培训支持", [
|
||||
("subject", "exact", "提供外校培训的教师人数", "外校培训人数"),
|
||||
("subject", "exact", "区域培训指导人数", "区域培训人数"),
|
||||
])
|
||||
|
||||
# ===== 9. 教研支持 =====
|
||||
audit_method("_score_research_support", "教研支持", [
|
||||
("subject", "exact", "学科教研组每学期活动次数", "教研活动次数"),
|
||||
("subject", "exact", "学科教研组平均每次活动时长", "教研活动时长"),
|
||||
("subject", "exact", "学科教研工作计划_有", "教研计划"),
|
||||
("subject", "exact", "学科教研组校级展示_有", "校级展示"),
|
||||
("subject", "exact", "学科教研组区域展示_有", "区域展示"),
|
||||
("subject", "exact", "学科教研组成果发表_有", "成果发表"),
|
||||
])
|
||||
|
||||
# ===== 10. 项目支持 =====
|
||||
audit_method("_score_project_support", "项目支持", [
|
||||
("subject", "exact", "学科承担的市级教改项目个数", "学科市级项目"),
|
||||
("subject", "exact", "学科承担的区级教改项目个数", "学科区级项目"),
|
||||
("subject", "exact", "学科承担的校级教改项目个数", "学科校级项目"),
|
||||
("course", "exact", "学校负责的市级教改项目个数", "学校负责市级"),
|
||||
("course", "exact", "学校参与的市级教改项目个数", "学校参与市级"),
|
||||
("course", "exact", "学校负责的区级教改项目个数", "学校负责区级"),
|
||||
("course", "exact", "学校参与的区级教改项目个数", "学校参与区级"),
|
||||
("course", "exact", "校级教改项目个数", "校级项目"),
|
||||
])
|
||||
|
||||
# ===== 11. 科学评价观 =====
|
||||
audit_method("_score_scientific_evaluation", "科学评价观", [
|
||||
("subject", "contains", "作业评价关注点", "作业评价"),
|
||||
("subject", "contains", "课堂表现评价关注点", "课堂评价"),
|
||||
("subject", "contains", "学科实践活动评价关注点", "实践评价"),
|
||||
])
|
||||
|
||||
# ===== 12. 学业质量评估 =====
|
||||
audit_method("_score_academic_evaluation", "学业质量评估", [
|
||||
("subject", "exact", "作业评价工具_已经建成并使用", "作业工具-建成使用"),
|
||||
("subject", "exact", "课堂表现评价工具_已经建成并使用", "课堂工具-建成使用"),
|
||||
("subject", "exact", "作业评价工具_已经建成尚未使用", "作业工具-建成未用"),
|
||||
("subject", "exact", "作业评价工具_尚未建成和使用", "作业工具-未建成"),
|
||||
("subject", "exact", "学期考试分析_执行分析并存档", "考试分析-存档"),
|
||||
("subject", "exact", "学期考试分析_执行分析，不要求存档", "考试分析-不存档"),
|
||||
("subject", "exact", "学期考试分析_教师自己决定", "考试分析-自定"),
|
||||
("subject", "exact", "表现性评价应用程度_经常", "表现性-经常"),
|
||||
("subject", "exact", "表现性评价应用程度_有时", "表现性-有时"),
|
||||
("subject", "exact", "表现性评价应用程度_总是", "表现性-总是"),
|
||||
("subject", "exact", "表现性评价应用程度_从不", "表现性-从不"),
|
||||
])
|
||||
|
||||
# ===== 13. 综合素质评估 =====
|
||||
audit_method("_score_comprehensive_evaluation", "综合素质评估", [
|
||||
("course", "exact", "综评评价体系_已经建成并使用", "综评-建成使用"),
|
||||
("course", "exact", "综评评价体系_已经建成尚未使用", "综评-建成未用"),
|
||||
("course", "exact", "综评评价体系_未建成", "综评-未建成"),
|
||||
("course", "exact", "综评评价体系_不准备建设", "综评-不建设"),
|
||||
("course", "exact", "综评信息化实现_自建平台支持", "综评IT-自建"),
|
||||
("course", "exact", "综评信息化实现_借助第三方平台支持", "综评IT-三方"),
|
||||
("course", "contains", "综评结果使用", "综评结果使用"),
|
||||
("course", "contains", "综评应用", "综评应用"),
|
||||
])
|
||||
|
||||
# ===== 14. 实践活动评估 =====
|
||||
audit_method("_score_practice_evaluation", "实践活动评估", [
|
||||
("course", "exact", "研究性学习评价工具_已经建成并使用", "研学评价-建成使用"),
|
||||
("course", "exact", "研究性学习评价工具_尚未建成和使用", "研学评价-未建成"),
|
||||
("course", "exact", "社会考察评价工具_已经建成并使用", "社考评价-建成使用"),
|
||||
("course", "exact", "社会考察评价工具_尚未建成和使用", "社考评价-未建成"),
|
||||
("subject", "exact", "学科实践活动工具_已经建成并使用", "学科实践-建成使用"),
|
||||
("subject", "exact", "学科实践活动工具_已经建成尚未使用", "学科实践-建成未用"),
|
||||
("subject", "exact", "学科实践活动工具_尚未建成和使用", "学科实践-未建成"),
|
||||
])
|
||||
|
||||
# ===== 15. 区域推进 =====
|
||||
audit_method("_score_regional_promotion", "区域推进", [
|
||||
("course", "exact", "区域工作会参与_一月4次以上", "工作会-月4+"),
|
||||
("course", "exact", "区域工作会参与_一月1次", "工作会-月1"),
|
||||
("course", "exact", "区域工作会参与_二月1次", "工作会-两月1"),
|
||||
("course", "exact", "区域工作会参与_三月1次", "工作会-季1"),
|
||||
("course", "contains", "区域推进措施", "推进措施"),
|
||||
])
|
||||
|
||||
# ===== 16. 环境支持 =====
|
||||
audit_method("_score_environment_support", "环境支持", [
|
||||
("course", "exact", "场馆供给_能满足需要", "场馆-满足"),
|
||||
("course", "exact", "场馆供给_基本满足需要", "场馆-基本满足"),
|
||||
("course", "exact", "场馆供给_难以满足需要", "场馆-难满足"),
|
||||
("course", "exact", "专用教室供给_能满足需要", "专用教室-满足"),
|
||||
("course", "exact", "专用教室供给_基本满足需要", "专用教室-基本满足"),
|
||||
("course", "contains", "信息化平台功能", "信息化平台功能"),
|
||||
])
|
||||
|
||||
# ===== 17. 资源支持 =====
|
||||
audit_method("_score_resource_support", "资源支持", [
|
||||
("subject", "exact", "必修课校内资源支持程度", "必修校内资源"),
|
||||
("subject", "exact", "选择性必修课校内资源支持程度", "选必校内资源"),
|
||||
("subject", "exact", "必修课校外资源支持程度", "必修校外资源"),
|
||||
("subject", "exact", "选择性必修课校外资源支持程度", "选必校外资源"),
|
||||
("subject", "exact", "学科教研组总人数", "教研组总人数"),
|
||||
])
|
||||
|
||||
# ===== 18. 教学方式创新 =====
|
||||
audit_method("_score_digital_teaching", "教学方式创新", [
|
||||
("subject", "exact", "信息技术与教学融合的认识_所有人可做到", "IT融合-所有人"),
|
||||
("subject", "exact", "信息技术与教学融合的认识_个别人可做到", "IT融合-个别人"),
|
||||
("subject", "exact", "信息化终端使用比例_80%+", "终端-80%+"),
|
||||
("subject", "exact", "信息化终端使用比例_60~79%", "终端-60~79%"),
|
||||
("subject", "exact", "信息化终端使用比例_30~59%", "终端-30~59%"),
|
||||
("subject", "exact", "信息化终端使用比例_30%-", "终端-30%-"),
|
||||
])
|
||||
|
||||
# ===== 19. 评价精准化 =====
|
||||
audit_method("_score_digital_evaluation", "评价精准化与个性化", [
|
||||
("subject", "exact", "学业评价信息化实现_自建平台支持", "学评IT-自建"),
|
||||
("subject", "exact", "学业评价信息化实现_借助第三方平台支持", "学评IT-三方"),
|
||||
("subject", "exact", "学业评价信息化实现_没有平台支持", "学评IT-无"),
|
||||
("course", "exact", "学校信息系统对选课支持程度", "选课系统"),
|
||||
("course", "exact", "学校信息系统对排课支持程度", "排课系统"),
|
||||
])
|
||||
|
||||
# ===== 20. 课程迭代优化 =====
|
||||
audit_method("_score_digital_curriculum", "课程迭代优化", [
|
||||
("course", "exact", "数据连通_有数据能互通", "数据互通"),
|
||||
("course", "exact", "数据连通_有数据不互通", "数据不互通"),
|
||||
("course", "exact", "管理业务的信息化应用_绝大部分", "管理IT-绝大部分"),
|
||||
("course", "exact", "教学业务的信息化应用_绝大部分", "教学IT-绝大部分"),
|
||||
("course", "exact", "已完成的网络课程门数", "网络课程数"),
|
||||
])
|
||||
|
||||
# ===== 打印报告 =====
|
||||
section("完整字段覆盖审计报告")
|
||||
|
||||
total_fields = 0
|
||||
total_ok = 0
|
||||
total_missing = 0
|
||||
dim_summary = {}
|
||||
|
||||
for method, info in results.items():
|
||||
dim = info["dim"]
|
||||
ok = sum(1 for f in info["fields"] if f["n_schools"] > 0)
|
||||
fail = sum(1 for f in info["fields"] if f["n_schools"] == 0)
|
||||
total = len(info["fields"])
|
||||
total_fields += total
|
||||
total_ok += ok
|
||||
total_missing += fail
|
||||
|
||||
pct = ok / total * 100 if total > 0 else 0
|
||||
bar = "█" * int(pct / 5) + "░" * (20 - int(pct / 5))
|
||||
|
||||
print(f"\n{'─' * 80}")
|
||||
print(f" {dim} ({method})")
|
||||
print(f" [{bar}] {pct:.0f}% ({ok}/{total})")
|
||||
print(f"{'─' * 80}")
|
||||
|
||||
for f in info["fields"]:
|
||||
src_label = "课程表" if f["source"] == "course" else "学科表"
|
||||
type_label = "精确" if f["type"] == "exact" else "关键词"
|
||||
print(f" {f['status']:<16} [{src_label}/{type_label}] {f['field']}")
|
||||
if f["n_schools"] == 0 and f.get("similar"):
|
||||
print(f" → 近似: {f['similar'][:3]}")
|
||||
|
||||
dim_summary[dim] = {"ok": ok, "fail": fail, "total": total, "pct": pct}
|
||||
|
||||
# ===== 汇总 =====
|
||||
section("维度覆盖率汇总")
|
||||
print(f"\n{'维度':<20} {'覆盖':>5} {'缺失':>5} {'总数':>5} {'覆盖率':>8}")
|
||||
print(f"{'─' * 50}")
|
||||
for dim, s in dim_summary.items():
|
||||
marker = "⚠️" if s["pct"] < 60 else "✅" if s["pct"] >= 80 else "⚡"
|
||||
print(f"{dim:<20} {s['ok']:>5} {s['fail']:>5} {s['total']:>5} {s['pct']:>6.0f}% {marker}")
|
||||
print(f"{'─' * 50}")
|
||||
print(f"{'合计':<20} {total_ok:>5} {total_missing:>5} {total_fields:>5} {total_ok/total_fields*100:>6.0f}%")
|
||||
|
||||
# ===== ETL层面检查：一期有但二期没有的字段名模式 =====
|
||||
section("ETL字段名差异分析")
|
||||
|
||||
# 检查常见差异模式
|
||||
patterns_to_check = [
|
||||
("个别辅导时长", "辅导时长字段命名"),
|
||||
("信息技术与教学融合", "IT融合字段"),
|
||||
("信息化终端使用比例", "终端使用比例"),
|
||||
("信息化平台功能", "平台功能"),
|
||||
("教学业务的信息化", "教学信息化"),
|
||||
("数据连通", "数据连通"),
|
||||
("已完成的网络课程", "网络课程"),
|
||||
("作业批改范围", "作业批改"),
|
||||
("实践类作业", "实践作业"),
|
||||
("表现类作业", "表现作业"),
|
||||
("跨学科作业", "跨学科作业"),
|
||||
("团队合作类作业", "团队合作作业"),
|
||||
]
|
||||
|
||||
for keyword, label in patterns_to_check:
|
||||
course_hits = [f for f in all_course_fields if keyword in str(f)]
|
||||
subject_hits = [f for f in all_subject_fields if keyword in str(f)]
|
||||
if course_hits or subject_hits:
|
||||
print(f"\n ✅ '{keyword}' ({label}):")
|
||||
for h in course_hits:
|
||||
print(f" 课程表: {h}")
|
||||
for h in subject_hits:
|
||||
print(f" 学科表: {h}")
|
||||
else:
|
||||
# 更宽松的搜索
|
||||
kw_short = keyword[:4]
|
||||
c2 = [f for f in all_course_fields if kw_short in str(f)]
|
||||
s2 = [f for f in all_subject_fields if kw_short in str(f)]
|
||||
print(f"\n ❌ '{keyword}' ({label}): 不存在")
|
||||
if c2 or s2:
|
||||
print(f" 近似(课程表): {c2[:5]}")
|
||||
print(f" 近似(学科表): {s2[:5]}")
|
||||
|
||||
print(f"\n✅ 深度审计完成!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
全市数据ETL：将全市3个xlsx转为parquet
|
||||
输入：二期/A全市数据_基础信息表_sh_basic2.xlsx
|
||||
二期/B全市数据_学校课程实施情况表_sh_sch6.xlsx
|
||||
二期/C全市数据_学科课程实施情况表_sh_sub7(全)(1).xlsx
|
||||
输出：data/era2/city_基础信息表.parquet
|
||||
data/era2/city_课程实施情况表.parquet
|
||||
data/era2/city_学科课程实施情况表.parquet
|
||||
data/era2/city_school_meta.parquet (学校元数据：类型、性质等)
|
||||
"""
|
||||
import pandas as pd
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent # report-admin/
|
||||
RAW_DIR = PROJECT_ROOT.parent / "二期"
|
||||
OUTPUT_DIR = PROJECT_ROOT / "data" / "era2"
|
||||
|
||||
# 输入文件
|
||||
FILE_A = RAW_DIR / "A全市数据_基础信息表_sh_basic2.xlsx"
|
||||
FILE_B = RAW_DIR / "B全市数据_学校课程实施情况表_sh_sch6.xlsx"
|
||||
FILE_C = RAW_DIR / "C全市数据_学科课程实施情况表_sh_sub7(全)(1).xlsx"
|
||||
|
||||
|
||||
def etl_basic_info():
|
||||
"""A表：基础信息表"""
|
||||
logger.info("加载A表（基础信息）...")
|
||||
df = pd.read_excel(FILE_A)
|
||||
logger.info(f" 原始行数: {len(df)}, 学校数: {df['s_name'].nunique()}")
|
||||
|
||||
# 列名映射：与当前parquet格式对齐
|
||||
# A表列: item_no, item_type, item_id, alternative, field_id, field_name, field_value,
|
||||
# grade, sem, district, s_name, ownership, category, level, area, tese, levelb, levelc
|
||||
# 当前parquet列: 字段名称, 字段值, 学校名称, 区, ...
|
||||
df = df.rename(columns={
|
||||
"s_name": "学校名称",
|
||||
"district": "区",
|
||||
"field_name": "字段名称",
|
||||
"field_value": "字段值",
|
||||
"ownership": "学校性质",
|
||||
"category": "学校类别",
|
||||
"level": "学校类型_原始",
|
||||
"area": "所处地区",
|
||||
"tese": "学校特色",
|
||||
"levelb": "学校类型",
|
||||
"levelc": "学校类型编号",
|
||||
"grade": "年级",
|
||||
"sem": "学期",
|
||||
})
|
||||
|
||||
out = OUTPUT_DIR / "city_基础信息表.parquet"
|
||||
df.to_parquet(out, index=False)
|
||||
logger.info(f" → {out} ({len(df)}行, {df['学校名称'].nunique()}校)")
|
||||
return df
|
||||
|
||||
|
||||
def etl_course_impl():
|
||||
"""B表：课程实施情况表"""
|
||||
logger.info("加载B表（课程实施）...")
|
||||
df = pd.read_excel(FILE_B)
|
||||
logger.info(f" 原始行数: {len(df)}")
|
||||
|
||||
# 过滤表头行
|
||||
df = df[df["所在区"] != "district"].copy()
|
||||
logger.info(f" 过滤后行数: {len(df)}, 学校数: {df['学校简称'].nunique()}")
|
||||
|
||||
# 列名已与parquet一致，无需改
|
||||
out = OUTPUT_DIR / "city_课程实施情况表.parquet"
|
||||
df.to_parquet(out, index=False)
|
||||
logger.info(f" → {out} ({len(df)}行, {df['学校简称'].nunique()}校)")
|
||||
return df
|
||||
|
||||
|
||||
def etl_subject_impl():
|
||||
"""C表：学科课程实施情况表"""
|
||||
logger.info("加载C表（学科课程，约44MB，需要30-60秒）...")
|
||||
df = pd.read_excel(FILE_C)
|
||||
logger.info(f" 原始行数: {len(df)}")
|
||||
|
||||
# 过滤表头行
|
||||
df = df[df["所在区"] != "district"].copy()
|
||||
logger.info(f" 过滤后行数: {len(df)}, 学校数: {df['学校简称'].nunique()}")
|
||||
|
||||
out = OUTPUT_DIR / "city_学科课程实施情况表.parquet"
|
||||
df.to_parquet(out, index=False)
|
||||
logger.info(f" → {out} ({len(df)}行, {df['学校简称'].nunique()}校)")
|
||||
return df
|
||||
|
||||
|
||||
def build_school_meta(df_b: pd.DataFrame):
|
||||
"""从B表提取学校元数据（类型、性质等）"""
|
||||
logger.info("构建学校元数据表...")
|
||||
meta = df_b.drop_duplicates("学校简称")[
|
||||
["学校简称", "所在区", "学校类别", "学校性质", "所处地区", "学校类型", "学校类型编号"]
|
||||
].copy()
|
||||
meta = meta.sort_values(["所在区", "学校简称"]).reset_index(drop=True)
|
||||
|
||||
# 清理学校类型编号中的换行符
|
||||
meta["学校类型编号"] = meta["学校类型编号"].str.replace(r"\n", "", regex=True)
|
||||
|
||||
out = OUTPUT_DIR / "city_school_meta.parquet"
|
||||
meta.to_parquet(out, index=False)
|
||||
logger.info(f" → {out} ({len(meta)}校)")
|
||||
|
||||
# 同时输出JSON便于查看
|
||||
meta_json = OUTPUT_DIR / "city_school_meta.json"
|
||||
meta.to_json(meta_json, orient="records", force_ascii=False, indent=2)
|
||||
logger.info(f" → {meta_json}")
|
||||
|
||||
# 统计
|
||||
print(f"\n{'='*60}")
|
||||
print(f"学校元数据统计")
|
||||
print(f"{'='*60}")
|
||||
print(f"总学校数: {len(meta)}")
|
||||
print(f"区域数: {meta['所在区'].nunique()}")
|
||||
print(f"\n各区学校数:")
|
||||
for d in sorted(meta["所在区"].unique()):
|
||||
n = len(meta[meta["所在区"] == d])
|
||||
print(f" {d}: {n}所")
|
||||
print(f"\n学校类型分布:")
|
||||
print(meta["学校类型"].value_counts().to_string())
|
||||
print(f"\n学校性质分布:")
|
||||
print(meta["学校性质"].value_counts().to_string())
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
def main():
|
||||
total_start = time.time()
|
||||
print("=" * 60)
|
||||
print("📊 全市数据ETL — xlsx → parquet")
|
||||
print("=" * 60)
|
||||
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# A表
|
||||
t0 = time.time()
|
||||
df_a = etl_basic_info()
|
||||
logger.info(f" A表耗时: {time.time()-t0:.1f}s")
|
||||
|
||||
# B表
|
||||
t0 = time.time()
|
||||
df_b = etl_course_impl()
|
||||
logger.info(f" B表耗时: {time.time()-t0:.1f}s")
|
||||
|
||||
# C表
|
||||
t0 = time.time()
|
||||
df_c = etl_subject_impl()
|
||||
logger.info(f" C表耗时: {time.time()-t0:.1f}s")
|
||||
|
||||
# 学校元数据
|
||||
meta = build_school_meta(df_b)
|
||||
|
||||
total = time.time() - total_start
|
||||
print(f"\n{'='*60}")
|
||||
print(f"✅ ETL完成! 总耗时: {total:.1f}s")
|
||||
print(f" A表: {len(df_a)}行 → city_基础信息表.parquet")
|
||||
print(f" B表: {len(df_b)}行 → city_课程实施情况表.parquet")
|
||||
print(f" C表: {len(df_c)}行 → city_学科课程实施情况表.parquet")
|
||||
print(f" 元数据: {len(meta)}校 → city_school_meta.parquet/json")
|
||||
print(f"{'='*60}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
# 二期数据处理脚本
|
||||
|
||||
## 数据概况
|
||||
|
||||
| 项目 | 一期 | 二期 |
|
||||
|------|------|------|
|
||||
| 区域 | 仅长宁区 | **13个区** |
|
||||
| 学校数 | 9所 | **176~177所** |
|
||||
| 基础信息表 | 277行 | 5,557行 |
|
||||
| 课程实施情况表 | 5,506行 / 128题 | 145,484行 / **103题** |
|
||||
| 学科课程实施情况表 | 25,006行 / 131题 | **1,348,513行** / 87~131题 |
|
||||
|
||||
### 一期 vs 二期 关键差异
|
||||
|
||||
1. **格式不同**：一期是3张扁平Excel，二期是原始平台导出（每题一个Sheet + 15个学科独立文件）
|
||||
2. **题目数量**：课程实施表少了25题（128→103），学科表按学科不同题数也不同
|
||||
3. **学校名称**：一期用简称（"延安中学"），二期用全称（"上海市延安中学"）
|
||||
4. **长宁区**：二期只有8所（缺少"市三女中"），一期有9所
|
||||
5. **字段ID覆盖**：
|
||||
- 课程实施表：共有275个，一期独有74个，二期独有62个
|
||||
- 学科表字段名称：共有713个，一期独有150个，二期独有53个
|
||||
|
||||
## 脚本说明
|
||||
|
||||
### `01_etl_transform.py` — ETL转换
|
||||
|
||||
将二期原始数据转换为与一期相同列结构的3张扁平表（parquet格式）。
|
||||
|
||||
```bash
|
||||
cd report-admin
|
||||
python3 scripts/era2/01_etl_transform.py
|
||||
```
|
||||
|
||||
**输入**：`二期/[整理前] 二期_课程实施+教材使用情况表/` 下的17个xlsx文件
|
||||
**输出**：`data/era2/` 下的3个parquet文件 + etl_meta.json
|
||||
|
||||
- `era2_基础信息表.parquet` — 列与一期完全一致
|
||||
- `era2_课程实施情况表.parquet` — 列与一期完全一致
|
||||
- `era2_学科课程实施情况表.parquet` — 列与一期完全一致
|
||||
- `etl_meta.json` — 元信息（学校列表、区域列表等）
|
||||
|
||||
> ⚠️ 学科表有135万行，超过Excel行数上限(1,048,576)，因此使用parquet格式
|
||||
|
||||
### `02_verify_data.py` — 数据验证
|
||||
|
||||
对比一期和二期的列名、字段ID、学校名称等。
|
||||
|
||||
```bash
|
||||
cd report-admin
|
||||
python3 scripts/era2/02_verify_data.py
|
||||
```
|
||||
|
||||
## 完整脚本列表
|
||||
|
||||
| 脚本 | 功能 | 用法 |
|
||||
|------|------|------|
|
||||
| `01_etl_transform.py` | 原始格式 → 扁平表 parquet | `python3 scripts/era2/01_etl_transform.py` |
|
||||
| `02_verify_data.py` | 验证 ETL + 一期对比 | `python3 scripts/era2/02_verify_data.py` |
|
||||
| `03_test_pipeline.py` | 完整 Pipeline 测试（不调 LLM） | `python3 scripts/era2/03_test_pipeline.py --district 长宁区` |
|
||||
| `04_generate_report.py` | 单校报告生成 | `python3 scripts/era2/04_generate_report.py --school 延安中学` |
|
||||
| `05_batch_generate.py` | 批量报告生成 | `python3 scripts/era2/05_batch_generate.py --district 长宁区` |
|
||||
| `config_era2.py` | 二期配置（路径、学校映射等） | 被其他脚本导入 |
|
||||
| `data_engine_era2.py` | 二期数据引擎（兼容一期接口） | 被其他脚本导入 |
|
||||
|
||||
### 关键参数
|
||||
|
||||
- `--district 长宁区` — 只分析长宁区8所学校（默认）
|
||||
- `--district all` — 分析全部13个区176所学校
|
||||
- `--enable-llm` — 启用 LLM 文字生成（**会产生 API 费用**）
|
||||
- `--school 延安中学` — 指定学校（支持简称）
|
||||
- `--list-schools` — 列出可用学校
|
||||
|
||||
### 运行示例
|
||||
|
||||
```bash
|
||||
cd report-admin
|
||||
|
||||
# 1. ETL（只需运行一次）
|
||||
python3 scripts/era2/01_etl_transform.py
|
||||
|
||||
# 2. 测试 pipeline（长宁区）
|
||||
python3 scripts/era2/03_test_pipeline.py --district 长宁区
|
||||
|
||||
# 3. 全市 pipeline
|
||||
python3 scripts/era2/03_test_pipeline.py --district all
|
||||
|
||||
# 4. 生成延安中学报告（不调LLM）
|
||||
python3 scripts/era2/04_generate_report.py --school 延安中学
|
||||
|
||||
# 5. 批量生成长宁区全部报告（不调LLM）
|
||||
python3 scripts/era2/05_batch_generate.py --district 长宁区
|
||||
|
||||
# 6. 启用LLM生成（⚠️ 会产生费用）
|
||||
python3 scripts/era2/04_generate_report.py --school 延安中学 --enable-llm
|
||||
```
|
||||
|
||||
## 后续优化方向
|
||||
|
||||
1. **学校名称映射扩展**：目前只配了长宁区8所学校的全称→简称映射，全市176所需补充
|
||||
2. **学校类型信息补充**：办学性质、学校等级、特色类型等（一期在 config 手工配了9所）
|
||||
3. **题号映射精细核对**：确认二期103题与一期128题的对应关系，优化赋分覆盖率
|
||||
4. **赋分规则适配**：基于字段ID的赋分规则大部分可复用，但部分新增/缺失字段需处理
|
||||
5. **全市对照基准**：二期有176所学校，PCA和标准化终于有统计意义，可做真正的全市对照
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
二期配置文件
|
||||
学校名称映射、学校类型信息等
|
||||
"""
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
# ===== 路径 =====
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent # report-admin/
|
||||
DATA_DIR = PROJECT_ROOT / "data" / "era2"
|
||||
ERA1_DATA_DIR = PROJECT_ROOT / "data"
|
||||
|
||||
# --- 区级数据（原始ETL产出，用于兼容旧流程） ---
|
||||
PARQUET_BASIC_INFO = DATA_DIR / "era2_基础信息表.parquet"
|
||||
PARQUET_COURSE_IMPL = DATA_DIR / "era2_课程实施情况表.parquet"
|
||||
PARQUET_SUBJECT_IMPL = DATA_DIR / "era2_学科课程实施情况表.parquet"
|
||||
EXCEL_SCORING_RULES = ERA1_DATA_DIR / "赋分整理表.xlsx"
|
||||
|
||||
# --- 全市数据（08_etl_city_data.py 产出） ---
|
||||
CITY_PARQUET_BASIC_INFO = DATA_DIR / "city_基础信息表.parquet"
|
||||
CITY_PARQUET_COURSE_IMPL = DATA_DIR / "city_课程实施情况表.parquet"
|
||||
CITY_PARQUET_SUBJECT_IMPL = DATA_DIR / "city_学科课程实施情况表.parquet"
|
||||
CITY_SCHOOL_META = DATA_DIR / "city_school_meta.parquet"
|
||||
|
||||
# ===== PCA参数（与一期一致） =====
|
||||
PCA_MEAN = 50
|
||||
PCA_STD = 10
|
||||
|
||||
# ===== 学校全称→简称 映射（保留，兼容旧的区级数据） =====
|
||||
SCHOOL_NAME_FULL_TO_SHORT = {
|
||||
"上海市延安中学": "延安中学",
|
||||
"上海市复旦中学": "复旦中学",
|
||||
"上海市仙霞高级中学": "仙霞高中",
|
||||
"上海市建青实验学校": "建青实验",
|
||||
"华东政法大学附属中学": "华政附中",
|
||||
"上海市民办新虹桥中学": "民办新虹桥",
|
||||
"华东师范大学附属天山学校": "天山学校",
|
||||
"上海市西郊学校": "西郊学校",
|
||||
}
|
||||
SCHOOL_NAME_SHORT_TO_FULL = {v: k for k, v in SCHOOL_NAME_FULL_TO_SHORT.items()}
|
||||
|
||||
# ===== 全市学校类型映射（从 city_school_meta.parquet 自动加载） =====
|
||||
def _load_school_type_map() -> dict:
|
||||
"""从全市元数据parquet加载266校的类型信息"""
|
||||
if not CITY_SCHOOL_META.exists():
|
||||
return {}
|
||||
df = pd.read_parquet(CITY_SCHOOL_META)
|
||||
result = {}
|
||||
for _, row in df.iterrows():
|
||||
result[str(row["学校简称"]).strip()] = {
|
||||
"type": row.get("学校类型", ""),
|
||||
"code": row.get("学校类型编号", ""),
|
||||
"nature": row.get("学校性质", ""),
|
||||
"category": row.get("学校类别", ""),
|
||||
"area": row.get("所处地区", ""),
|
||||
"district": row.get("所在区", ""),
|
||||
}
|
||||
return result
|
||||
|
||||
SCHOOL_TYPE_MAP = _load_school_type_map()
|
||||
|
||||
# ===== 15个学科（与一期一致） =====
|
||||
SUBJECTS = [
|
||||
"语文", "数学", "英语", "物理", "化学", "生物学",
|
||||
"历史", "地理", "思想政治", "体育与健康",
|
||||
"信息技术", "通用技术", "艺术", "音乐", "美术"
|
||||
]
|
||||
|
||||
# ===== 七大维度体系（与一期完全一致） =====
|
||||
DIMENSION_FRAMEWORK = {
|
||||
"课程领导力": {
|
||||
"sub_dimensions": ["国家标准遵循", "课程结构建设", "课程规范落实"],
|
||||
},
|
||||
"教学变革力": {
|
||||
"sub_dimensions": ["教学方式变革", "作业设计与管理变革"],
|
||||
},
|
||||
"学生发展指导力": {
|
||||
"sub_dimensions": ["学科发展的个性化辅导", "学生生涯发展指导"],
|
||||
},
|
||||
"教师发展支持力": {
|
||||
"sub_dimensions": ["培训支持", "教研支持", "项目支持"],
|
||||
},
|
||||
"教育质量评估力": {
|
||||
"sub_dimensions": ["科学评价观", "学业质量评估", "综合素质评估", "实践活动评估"],
|
||||
},
|
||||
"教育条件保障力": {
|
||||
"sub_dimensions": ["区域推进", "环境支持", "资源支持"],
|
||||
},
|
||||
"数字化赋能力": {
|
||||
"sub_dimensions": ["教学方式创新", "评价精准化与个性化", "课程迭代优化"],
|
||||
},
|
||||
}
|
||||
|
||||
# ===== 聚类数配置（对齐SPSS） =====
|
||||
# SPSS中学生发展指导力/教师发展支持力/教育质量评估力使用3类聚类，其余使用2类
|
||||
CLUSTER_CONFIG = {
|
||||
"课程领导力": 2,
|
||||
"教学变革力": 2,
|
||||
"学生发展指导力": 3, # SPSS: 78:122:56
|
||||
"教师发展支持力": 3, # SPSS: 1:162:79
|
||||
"教育条件保障力": 2,
|
||||
"教育质量评估力": 3, # SPSS: 110:109:38
|
||||
"数字化赋能力": 2,
|
||||
"总体": 2,
|
||||
}
|
||||
|
||||
# 水平阈值（与一期完全一致，来自赋分整理表）
|
||||
LEVEL_THRESHOLDS = {
|
||||
"国家标准遵循": {"level4": 57, "level3": 53.5, "level2": 43},
|
||||
"课程结构建设": {"level4": 55.5, "level3": 49, "level2": 45},
|
||||
"课程规范落实": {"level4": 58, "level3": 50, "level2": 44},
|
||||
"教学方式变革": {"level4": 54, "level3": 49, "level2": 45},
|
||||
"作业设计与管理变革": {"level4": 54, "level3": 49, "level2": 45},
|
||||
"学科发展的个性化辅导": {"level4": 60, "level3": 50, "level2": 46},
|
||||
"学生生涯发展指导": {"level4": 56, "level3": 50, "level2": 42},
|
||||
"培训支持": {"level4": 51, "level3": 49, "level2": 47.5},
|
||||
"教研支持": {"level4": 55, "level3": 50, "level2": 47},
|
||||
"项目支持": {"level4": 56.5, "level3": 50, "level2": 44},
|
||||
"科学评价观": {"level4": 57, "level3": 50, "level2": 43},
|
||||
"学业质量评估": {"level4": 54, "level3": 46, "level2": 41},
|
||||
"综合素质评估": {"level4": 58.5, "level3": 50, "level2": 39},
|
||||
"实践活动评估": {"level4": 50, "level3": 46.5, "level2": 43},
|
||||
"区域推进": {"level4": 59, "level3": 52, "level2": 40},
|
||||
"环境支持": {"level4": 56, "level3": 49.5, "level2": 41},
|
||||
"资源支持": {"level4": 57.5, "level3": 49, "level2": 42},
|
||||
"教学方式创新": {"level4": 54, "level3": 45, "level2": 40},
|
||||
"评价精准化与个性化": {"level4": 60, "level3": 50, "level2": 40},
|
||||
"课程迭代优化": {"level4": 55, "level3": 50, "level2": 45},
|
||||
}
|
||||
|
||||
# 水平质性描述（与一期完全一致，完整版）
|
||||
LEVEL_DESCRIPTIONS = {
|
||||
"国家标准遵循": {4: "所有科目必修课程开齐开足，选必和选修满足要求", 3: "考试类科目必修开齐，选必和选修满足要求", 2: "必修未能开齐开足，选必和选修有一个满足要求", 1: "必修未能开齐开足，选必和选修均不满足要求"},
|
||||
"课程结构建设": {4: "学科类必修课程离差总和在30%以内，总体三类课程在150%以下，校本课程和综合实践较好", 3: "总体三类课程离差总和在250%以下，校本课程和综合实践高于平均水平", 2: "总体三类课程离差总和在300%以下，校本课程和综合实践较差", 1: "总体三类课程的离差和很高，校本课程和综合实践在末尾25%"},
|
||||
"课程规范落实": {4: "都有建设规范文本和档案记录", 3: "都有建设规范文本，但过程性档案记录已经全部建成，部分有尚未使用", 2: "部分有建设规范文本，档案大部分已经建成但未使用", 1: "基本没有建设规范文本，档案尚未建成"},
|
||||
"教学方式变革": {4: "所有老师有共识和研究，并能够系统设计、有效实施，至少有3种常态化落实形式", 3: "大部分老师有共识和研究，并能够落实教材中的相关要求，至少有2种常态化落实形式", 2: "个别老师有研究，并能够偶尔引导学生开展学习，至少有1种常态化落实形式", 1: "基本没有教学方法等相关研究，基本不组织相关学习，没有或仅有1种落实形式"},
|
||||
"作业设计与管理变革": {4: "在每类创新性作业中至少掌握3种类型，作业时长有统一控制管理，定期批改评价，有多元化属性标注", 3: "在每类创新性作业中至少掌握2种类型，偶有时长控制管理，面批为主，有至少两种属性标注", 2: "至少掌握1种类型或擅长某1-2种创新作业，学生自己控制时长，很少反馈，有属性标注", 1: "擅长某种创新作业，学生自己控制时长，几乎不反馈，无属性标注"},
|
||||
"学科发展的个性化辅导": {4: "各学科平均辅导时长每周2小时以上，教师根据学情确定内容，采取个别辅导方式", 3: "各学科平均辅导时长每周1-2小时，教师根据学情确定内容，主要个别分散辅导", 2: "各学科平均辅导时长每周1小时以内，学生提出需求后教师辅导，分组统一或分散辅导", 1: "辅导时长每周1小时以内或不辅导，学生提出需求后教师辅导，班级统一辅导"},
|
||||
"学生生涯发展指导": {4: "专设生涯指导课程，三年覆盖90%+学生，本校+外聘教师队伍，校内外资源足够支持", 3: "外请讲座实施，三年覆盖70-90%学生，外聘教师队伍，校内外资源有一些支持", 2: "与社会考察/志愿服务结合实施，三年覆盖50-70%学生，外聘教师，几乎无资源支持", 1: "与社会考察/志愿服务结合实施，三年覆盖50%以下，未形成稳定队伍，几乎无资源支持"},
|
||||
"培训支持": {4: "各学科教师平均外出培训人数≥2.5人", 3: "各学科教师平均外出培训人数≥1.5人", 2: "各学科教师平均外出培训人数≥1人", 1: "各学科教师几乎不进行外出培训"},
|
||||
"教研支持": {4: "教研的数量和质量均较高", 3: "有教研活动，质量较好", 2: "有教研活动但质量一般", 1: "活动数量和质量均较低"},
|
||||
"项目支持": {4: "各学科均有负责的校级以上项目至少一个", 3: "有部分学科负责校级以上项目至少一个", 2: "各学科没有负责的校级以上项目，部分学科有校级项目至少一个", 1: "各学科校级及校级以上的项目均没有"},
|
||||
"科学评价观": {4: "在所有方面均能至少关注两项素养发展", 3: "至少有三个方面关注两项素养发展", 2: "能较多关注学生发展", 1: "较少关注学生发展"},
|
||||
"学业质量评估": {4: "校本化评价工具已研制并使用，学期考试质量分析并标注属性较为全面", 3: "校本化评价工具已研制并使用，学期考试质量分析不做硬性要求", 2: "至少已研制一项校本化评价工具，学期考试质量分析不做硬性要求", 1: "未能重视学业质量评估，校本化评价工具均欠缺"},
|
||||
"综合素质评估": {4: "校本化综合素质评价体系均有建设和使用，有平台支持且评价结果表达科学", 3: "校本化综合素质评价体系均有建设和使用，支持平台和评价结果使用有待提高", 2: "校本化综合素质评价方案建成，但具体评价工具有待开发", 1: "未能重视校本化综合素质评价，方案、工具和结果使用等均欠缺"},
|
||||
"实践活动评估": {4: "研究性学习、社会考察和学科实践活动的校本化评价工具已研制并使用", 3: "学科实践活动的校本化评价工具已研制并使用，研究性学习/社会考察至少一项已研制但尚未使用", 2: "学科实践活动的校本化评价工具已研制", 1: "研究性学习、社会考察和学科实践活动的校本化评价工具均尚未研制和使用"},
|
||||
"区域推进": {4: "召开会议平均一个月2次及以上，区域管理文件数量和措施均为最大值", 3: "召开会议平均一个月1次及以上，区域管理文件3个以上，措施达3项", 2: "召开会议、文件和措施至少有一项建设较好", 1: "召开会议、文件和措施三项均建设较差"},
|
||||
"环境支持": {4: "信息化支持和硬件支持均处于较高水平", 3: "信息化支持和硬件在平均水平附近", 2: "信息化支持和硬件支持至少有一项建设较好", 1: "信息化支持和硬件支持均建设较差"},
|
||||
"资源支持": {4: "校内外资源和师资水平均处于较高水平", 3: "校内外资源至少有一项供给较好，师资水平较好", 2: "校内外资源至少有一项仅略低于平均水平，师资水平略低于平均水平", 1: "校内外资源和师资水平三项均建设较差"},
|
||||
"教学方式创新": {4: "信息技术与教学融合程度高，教师能常态化使用信息技术", 3: "信息技术与教学融合程度较高，教师能使用信息技术", 2: "信息技术与教学融合程度一般，学校有信息化平台建设", 1: "信息技术与教学融合程度较差，教师基本不使用信息技术"},
|
||||
"评价精准化与个性化": {4: "有自建信息技术平台支持学科诊断与综合素质评价", 3: "借助第三方平台支持学科诊断与综合素质评价", 2: "有信息技术平台支持学业评价", 1: "没有信息技术平台支持评价"},
|
||||
"课程迭代优化": {4: "学校对促进教学数字化转型有专项研修计划并开展相关活动，业务绝大部分使用信息化系统", 3: "学校尚未制定专项研修计划，业务绝大部分使用信息化系统", 2: "学校较少开展数字化转型活动，少数业务使用信息化系统", 1: "学校几乎不开展数字化转型活动，尚未建成信息化管理系统"},
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
"""
|
||||
二期数据引擎
|
||||
支持两种数据源：
|
||||
1. 全市数据（city_*.parquet） — 默认，266所学校，16个区
|
||||
2. 区级数据（era2_*.parquet） — 旧流程兼容，176所学校，13个区
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import logging
|
||||
|
||||
from config_era2 import (
|
||||
PARQUET_BASIC_INFO, PARQUET_COURSE_IMPL, PARQUET_SUBJECT_IMPL,
|
||||
CITY_PARQUET_BASIC_INFO, CITY_PARQUET_COURSE_IMPL, CITY_PARQUET_SUBJECT_IMPL,
|
||||
SCHOOL_NAME_FULL_TO_SHORT, SUBJECTS,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataEngineEra2:
|
||||
"""
|
||||
二期数据引擎
|
||||
|
||||
默认使用全市数据(city_*.parquet)，通过 district_filter 筛选特定区。
|
||||
同时提供全市基准数据接口供 StatsEngine 使用。
|
||||
"""
|
||||
|
||||
def __init__(self, district_filter: Optional[str] = None, use_city_data: bool = True):
|
||||
"""
|
||||
Args:
|
||||
district_filter: 可选，只加载某个区的数据（如 "杨浦区"）
|
||||
None 表示加载全部区域
|
||||
use_city_data: 是否使用全市数据（默认True）。False则用旧的区级parquet
|
||||
"""
|
||||
self.district_filter = district_filter
|
||||
self.use_city_data = use_city_data and CITY_PARQUET_COURSE_IMPL.exists()
|
||||
self._basic_info: Optional[pd.DataFrame] = None
|
||||
self._course_impl: Optional[pd.DataFrame] = None
|
||||
self._subject_impl: Optional[pd.DataFrame] = None
|
||||
# 全市数据（不受 district_filter 限制，用于全市基准）
|
||||
self._city_course_impl: Optional[pd.DataFrame] = None
|
||||
self._city_subject_impl: Optional[pd.DataFrame] = None
|
||||
|
||||
def load_all(self) -> None:
|
||||
"""加载所有数据"""
|
||||
src = "全市" if self.use_city_data else "区级"
|
||||
logger.info(f"开始加载二期parquet数据（{src}数据源）...")
|
||||
|
||||
if self.use_city_data:
|
||||
self._load_city_data()
|
||||
else:
|
||||
self._load_legacy_data()
|
||||
|
||||
logger.info(
|
||||
f"数据加载完成: 基础信息={len(self._basic_info)}行, "
|
||||
f"课程实施={len(self._course_impl)}行, "
|
||||
f"学科课程={len(self._subject_impl)}行, "
|
||||
f"当前区学校数={len(self.schools)}"
|
||||
)
|
||||
if self._city_course_impl is not None:
|
||||
logger.info(f"全市基准: {self._city_course_impl['学校名称'].nunique()}校")
|
||||
|
||||
@staticmethod
|
||||
def _strip_str_columns(df: pd.DataFrame, columns: List[str]) -> pd.DataFrame:
|
||||
"""对指定列做 strip()，去除空格和换行符"""
|
||||
for col in columns:
|
||||
if col in df.columns:
|
||||
df[col] = df[col].astype(str).str.strip()
|
||||
return df
|
||||
|
||||
def _load_city_data(self) -> None:
|
||||
"""从全市parquet加载（默认路径）"""
|
||||
# 全市课程实施（全量，用于基准）
|
||||
df_course_all = pd.read_parquet(CITY_PARQUET_COURSE_IMPL)
|
||||
df_course_all = df_course_all.rename(columns={"学校简称": "学校名称"})
|
||||
df_course_all = self._strip_str_columns(df_course_all, ["学校名称", "字段名称", "所在区"])
|
||||
self._city_course_impl = df_course_all
|
||||
|
||||
df_subject_all = pd.read_parquet(CITY_PARQUET_SUBJECT_IMPL)
|
||||
df_subject_all = df_subject_all.rename(columns={"学校简称": "学校名称"})
|
||||
df_subject_all = self._strip_str_columns(df_subject_all, ["学校名称", "字段名称", "所在区", "学科"])
|
||||
self._city_subject_impl = df_subject_all
|
||||
|
||||
# 按区筛选的数据（用于赋分和报告生成）
|
||||
if self.district_filter:
|
||||
self._course_impl = df_course_all[df_course_all["所在区"] == self.district_filter].copy()
|
||||
self._subject_impl = df_subject_all[df_subject_all["所在区"] == self.district_filter].copy()
|
||||
else:
|
||||
self._course_impl = df_course_all.copy()
|
||||
self._subject_impl = df_subject_all.copy()
|
||||
|
||||
# 基础信息表
|
||||
df_basic = pd.read_parquet(CITY_PARQUET_BASIC_INFO)
|
||||
df_basic = self._strip_str_columns(df_basic, ["学校名称", "字段名称", "区"])
|
||||
if self.district_filter:
|
||||
self._basic_info = df_basic[df_basic["区"] == self.district_filter].copy()
|
||||
else:
|
||||
self._basic_info = df_basic
|
||||
|
||||
def _load_legacy_data(self) -> None:
|
||||
"""从旧的区级parquet加载（兼容）"""
|
||||
self._basic_info = pd.read_parquet(PARQUET_BASIC_INFO)
|
||||
self._basic_info = self._strip_str_columns(self._basic_info, ["学校名称", "字段名称", "区"])
|
||||
if self.district_filter:
|
||||
self._basic_info = self._basic_info[self._basic_info["区"] == self.district_filter].copy()
|
||||
self._basic_info = self._standardize_school_name(self._basic_info, "学校名称")
|
||||
|
||||
df = pd.read_parquet(PARQUET_COURSE_IMPL)
|
||||
df = self._strip_str_columns(df, ["学校简称", "字段名称", "所在区"])
|
||||
if self.district_filter:
|
||||
df = df[df["所在区"] == self.district_filter].copy()
|
||||
df = self._standardize_school_name(df, "学校简称")
|
||||
df = df.rename(columns={"学校简称": "学校名称"})
|
||||
self._course_impl = df
|
||||
|
||||
df = pd.read_parquet(PARQUET_SUBJECT_IMPL)
|
||||
df = self._strip_str_columns(df, ["学校简称", "字段名称", "所在区", "学科"])
|
||||
if self.district_filter:
|
||||
df = df[df["所在区"] == self.district_filter].copy()
|
||||
df = self._standardize_school_name(df, "学校简称")
|
||||
df = df.rename(columns={"学校简称": "学校名称"})
|
||||
self._subject_impl = df
|
||||
|
||||
def _standardize_school_name(self, df: pd.DataFrame, col: str) -> pd.DataFrame:
|
||||
"""将学校全称映射为简称（旧数据兼容用）"""
|
||||
df[col] = df[col].map(lambda x: SCHOOL_NAME_FULL_TO_SHORT.get(x, x))
|
||||
return df
|
||||
|
||||
@property
|
||||
def city_schools(self) -> List[str]:
|
||||
"""获取全市学校列表（用于全市基准计算）"""
|
||||
if self._city_course_impl is not None:
|
||||
return sorted(self._city_course_impl["学校名称"].unique().tolist())
|
||||
return self.schools
|
||||
|
||||
@property
|
||||
def city_course_data(self) -> Optional[pd.DataFrame]:
|
||||
"""全市课程实施数据（不受district_filter限制）"""
|
||||
return self._city_course_impl
|
||||
|
||||
@property
|
||||
def city_subject_data(self) -> Optional[pd.DataFrame]:
|
||||
"""全市学科课程数据（不受district_filter限制）"""
|
||||
return self._city_subject_impl
|
||||
|
||||
# ===== 与一期 DataEngine 完全一致的接口 =====
|
||||
|
||||
@property
|
||||
def schools(self) -> List[str]:
|
||||
"""获取所有学校名称列表"""
|
||||
if self._course_impl is None:
|
||||
self.load_all()
|
||||
return sorted(self._course_impl["学校名称"].unique().tolist())
|
||||
|
||||
def get_school_basic_info(self, school: str) -> Dict:
|
||||
"""获取学校基本信息"""
|
||||
if self._basic_info is None:
|
||||
self.load_all()
|
||||
df = self._basic_info[self._basic_info["学校名称"] == school]
|
||||
|
||||
def _get_field(field_name):
|
||||
rows = df[df["字段名称"] == field_name]
|
||||
if len(rows) > 0:
|
||||
return rows.iloc[0]["字段值"]
|
||||
return None
|
||||
|
||||
return {
|
||||
"school_name": school,
|
||||
"建校年份": _get_field("建校年份"),
|
||||
"占地面积": _get_field("占地面积"),
|
||||
"建筑面积": _get_field("建筑面积"),
|
||||
}
|
||||
|
||||
def get_school_course_data(self, school: str) -> pd.DataFrame:
|
||||
"""获取某学校的课程实施情况数据"""
|
||||
if self._course_impl is None:
|
||||
self.load_all()
|
||||
return self._course_impl[self._course_impl["学校名称"] == school].copy()
|
||||
|
||||
def get_school_subject_data(self, school: str, subject: Optional[str] = None) -> pd.DataFrame:
|
||||
"""获取某学校的学科课程实施数据"""
|
||||
if self._subject_impl is None:
|
||||
self.load_all()
|
||||
df = self._subject_impl[self._subject_impl["学校名称"] == school].copy()
|
||||
if subject:
|
||||
df = df[df["学科"] == subject]
|
||||
return df
|
||||
|
||||
# ===== 课程领导力相关数据提取（与一期一致） =====
|
||||
|
||||
def get_weekly_hours(self, school: str) -> pd.DataFrame:
|
||||
"""获取学校各学科各年级各学期的周课时数据"""
|
||||
df = self.get_school_course_data(school)
|
||||
hours_fields = ["学科必修课周课时", "学科选择性必修课周课时", "学科类选修课周课时"]
|
||||
result = df[df["字段名称"].isin(hours_fields)].copy()
|
||||
result["字段取值"] = pd.to_numeric(result["字段取值"], errors="coerce")
|
||||
return result
|
||||
|
||||
def get_course_norms(self, school: str) -> List[Dict]:
|
||||
"""获取学校课程规范落实相关数据"""
|
||||
df = self.get_school_course_data(school)
|
||||
norm_keywords = ["建设规范文本", "档案", "已经建成并使用", "尚未建成"]
|
||||
mask = df["字段名称"].apply(
|
||||
lambda x: any(k in str(x) for k in norm_keywords) if pd.notna(x) else False
|
||||
)
|
||||
return df[mask][["字段名称", "字段取值"]].to_dict("records")
|
||||
|
||||
# ===== 教学变革力相关（与一期一致） =====
|
||||
|
||||
def get_teaching_reform_data(self, school: str) -> pd.DataFrame:
|
||||
df = self.get_school_subject_data(school)
|
||||
reform_keywords = [
|
||||
"认识程度", "落实程度", "认识", "落实",
|
||||
"理解式学习", "自主性学习", "实践性学习", "跨学科学习",
|
||||
"信息技术与教学融合", "信息融入教学",
|
||||
]
|
||||
mask = df["字段名称"].apply(
|
||||
lambda x: any(k in str(x) for k in reform_keywords) if pd.notna(x) else False
|
||||
)
|
||||
return df[mask]
|
||||
|
||||
def get_homework_data(self, school: str) -> pd.DataFrame:
|
||||
df = self.get_school_subject_data(school)
|
||||
hw_keywords = [
|
||||
"作业", "实践类", "表现类", "跨学科", "团队合作",
|
||||
"批改", "评价", "属性标注", "时长控制",
|
||||
]
|
||||
mask = df["字段名称"].apply(
|
||||
lambda x: any(k in str(x) for k in hw_keywords) if pd.notna(x) else False
|
||||
)
|
||||
return df[mask]
|
||||
|
||||
# ===== 通用方法 =====
|
||||
|
||||
def get_field_value(self, school: str, source: str, field_name: str,
|
||||
subject: Optional[str] = None) -> Optional[str]:
|
||||
if source == "basic":
|
||||
df = self._basic_info[self._basic_info["学校名称"] == school]
|
||||
col = "字段名称"
|
||||
val_col = "字段值"
|
||||
elif source == "course":
|
||||
df = self.get_school_course_data(school)
|
||||
col = "字段名称"
|
||||
val_col = "字段取值"
|
||||
elif source == "subject":
|
||||
df = self.get_school_subject_data(school, subject)
|
||||
col = "字段名称"
|
||||
val_col = "字段取值"
|
||||
else:
|
||||
return None
|
||||
rows = df[df[col] == field_name]
|
||||
if len(rows) > 0:
|
||||
return rows.iloc[0][val_col]
|
||||
return None
|
||||
|
||||
def summary(self) -> Dict:
|
||||
if self._basic_info is None:
|
||||
self.load_all()
|
||||
return {
|
||||
"era": 2,
|
||||
"district_filter": self.district_filter,
|
||||
"schools": self.schools,
|
||||
"school_count": len(self.schools),
|
||||
"basic_info_rows": len(self._basic_info),
|
||||
"course_impl_rows": len(self._course_impl),
|
||||
"subject_impl_rows": len(self._subject_impl),
|
||||
"subjects": SUBJECTS,
|
||||
"districts": sorted(self._course_impl["所在区"].unique().tolist()) if "所在区" in self._course_impl.columns else [],
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
# era2 engines package
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,929 @@
|
||||
"""
|
||||
二期报告渲染引擎 (era2)
|
||||
基于一期 report_renderer.py 拷贝，改了 import 路径
|
||||
支持中英双语（lang="zh" / "en"）
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import string
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import numpy as np
|
||||
import sys
|
||||
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
# era2 独立 config（不依赖 backend）
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from config_era2 import DIMENSION_FRAMEWORK, LEVEL_THRESHOLDS
|
||||
|
||||
# 项目根（用于添加 i18n 包到 sys.path）
|
||||
PROJECT_ROOT_FOR_I18N = Path(__file__).parent.parent.parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT_FOR_I18N))
|
||||
from i18n import get_translator, get_bundle # noqa: E402
|
||||
|
||||
# era2 模板目录（统一放在 report-admin/templates/era2）
|
||||
TEMPLATES_DIR = Path(__file__).parent.parent.parent.parent / "templates" / "era2"
|
||||
OUTPUT_DIR = Path(__file__).parent.parent.parent.parent / "output" / "era2"
|
||||
|
||||
# 二期暂无 SCHOOL_TYPE_MAP（176所学校没有逐校类型信息）
|
||||
SCHOOL_TYPE_MAP = {}
|
||||
|
||||
# 水平描述从 config 导入
|
||||
try:
|
||||
from config_era2 import LEVEL_DESCRIPTIONS
|
||||
except ImportError:
|
||||
LEVEL_DESCRIPTIONS = {}
|
||||
|
||||
|
||||
class ReportRendererEra2:
|
||||
"""二期 HTML 报告渲染引擎"""
|
||||
|
||||
def __init__(self):
|
||||
self.env = Environment(
|
||||
loader=FileSystemLoader(str(TEMPLATES_DIR)),
|
||||
autoescape=False,
|
||||
)
|
||||
|
||||
def render(self, report_data: Dict, llm_sections: Dict[str, str],
|
||||
enable_agent: bool = False, lang: str = "zh") -> str:
|
||||
"""
|
||||
渲染完整HTML报告
|
||||
|
||||
report_data: stats_engine.compute_school_report_data() 的输出
|
||||
llm_sections: llm_engine.generate_report_segments() 的输出
|
||||
enable_agent: 是否嵌入 AI 对话助手
|
||||
lang: 语言代码（"zh" / "en"）
|
||||
"""
|
||||
template = self.env.get_template("base.html")
|
||||
|
||||
# 翻译器
|
||||
t = get_translator(lang)
|
||||
|
||||
# 准备模板数据
|
||||
school = report_data["school"]
|
||||
district = report_data.get("district", "")
|
||||
|
||||
# 学校/区显示名（英文版区名翻译为 "Changning District" 等；学校名作为专有名词保留）
|
||||
school_display = school
|
||||
district_display = t.district(district) if district else district
|
||||
|
||||
# 用于模板/图表的两个翻译映射（中文 key → 当前 lang 显示）
|
||||
bundle = get_bundle(lang)
|
||||
dim_translation_map = bundle.get("DIMENSIONS", {})
|
||||
sub_translation_map = bundle.get("SUB_DIMENSIONS", {})
|
||||
|
||||
# ECharts JS 用 i18n bundle
|
||||
ec_i18n = dict(bundle.get("UI", {}))
|
||||
|
||||
# 子维度小节编号（中文：二、三..；英文：II. III..）
|
||||
if lang == "en":
|
||||
sub_section_labels = ["", "II.", "III.", "IV.", "V.", "VI.", "VII.", "VIII.", "IX."]
|
||||
ag_section_labels = ["I.", "II.", "III.", "IV.", "V.", "VI.", "VII.", "VIII.", "IX."]
|
||||
cn_subnums = sub_section_labels
|
||||
cn_nums = ag_section_labels
|
||||
html_lang = "en"
|
||||
else:
|
||||
sub_section_labels = ["", "二、", "三、", "四、", "五、", "六、", "七、", "八、", "九、"]
|
||||
ag_section_labels = ["一、", "二、", "三、", "四、", "五、", "六、", "七、", "八、", "九、"]
|
||||
cn_subnums = sub_section_labels
|
||||
cn_nums = ag_section_labels
|
||||
html_lang = "zh-CN"
|
||||
|
||||
# 生成日期（按语言）
|
||||
generation_date = t.date(datetime.now())
|
||||
|
||||
context = {
|
||||
"school": school,
|
||||
"school_display": school_display,
|
||||
"district": district,
|
||||
"district_display": district_display,
|
||||
"total_schools_in_district": report_data.get("total_schools_in_district", report_data["overall"].get("total_schools", 0)),
|
||||
"school_info": report_data["school_info"],
|
||||
"overall": self._to_namespace(report_data["overall"]),
|
||||
"dimensions": {
|
||||
name: self._to_namespace(data)
|
||||
for name, data in report_data["dimensions"].items()
|
||||
},
|
||||
"sub_dimensions": {
|
||||
name: self._to_namespace(data)
|
||||
for name, data in report_data["sub_dimensions"].items()
|
||||
},
|
||||
"framework": {
|
||||
name: self._to_namespace(info)
|
||||
for name, info in DIMENSION_FRAMEWORK.items()
|
||||
},
|
||||
"llm_sections": llm_sections,
|
||||
"level_descriptions": LEVEL_DESCRIPTIONS,
|
||||
"level_descriptions_keys": list(LEVEL_DESCRIPTIONS.keys()),
|
||||
"generation_date": generation_date,
|
||||
|
||||
# i18n 注入
|
||||
"t": t,
|
||||
"lang": lang,
|
||||
"html_lang": html_lang,
|
||||
"ec_i18n": ec_i18n,
|
||||
"dim_translation_map": dim_translation_map,
|
||||
"sub_translation_map": sub_translation_map,
|
||||
"cn_subnums": cn_subnums,
|
||||
"cn_nums": cn_nums,
|
||||
"ag_section_labels": ag_section_labels,
|
||||
|
||||
# ECharts数据 — 传原始dict，由模板的tojson过滤器序列化一次
|
||||
"radar_data": self._build_radar_data(report_data),
|
||||
"sub_dim_chart_data": self._build_sub_dim_charts(report_data),
|
||||
"score_compare_data": self._build_score_compare(report_data),
|
||||
"cluster_radar_data": self._build_cluster_radar(report_data),
|
||||
"correlation_data": self._build_correlation_heatmap(report_data),
|
||||
"level_dist_data": self._build_level_distribution(report_data),
|
||||
"school_ranking_data": self._build_school_ranking(report_data),
|
||||
# 新增图表数据
|
||||
"cluster_type_dist_data": self._build_cluster_type_distribution(report_data),
|
||||
"cluster_line_compare_data": self._build_cluster_line_compare(report_data),
|
||||
"dim_scatter_data": self._build_dim_scatter_charts(report_data),
|
||||
"dim_score_bar_data": self._build_dim_score_bars(report_data),
|
||||
"dim_sub_radar_data": self._build_dim_sub_radar_charts(report_data),
|
||||
# 创新图表
|
||||
"profile_card_data": self._build_profile_card(report_data),
|
||||
"quadrant_data": self._build_quadrant_chart(report_data),
|
||||
"thermometer_data": self._build_thermometer_data(report_data),
|
||||
"waterfall_data": self._build_waterfall_chart(report_data),
|
||||
}
|
||||
|
||||
# AI 对话助手（可选）
|
||||
if enable_agent:
|
||||
chat_config, report_json = self._build_chat_config(report_data)
|
||||
context["chat_config"] = chat_config
|
||||
context["report_data_json"] = report_json
|
||||
else:
|
||||
context["chat_config"] = None
|
||||
context["report_data_json"] = "{}"
|
||||
|
||||
return template.render(**context)
|
||||
|
||||
def render_to_file(self, report_data: Dict, llm_sections: Dict[str, str],
|
||||
output_path: Path = None, enable_agent: bool = False,
|
||||
lang: str = "zh") -> Path:
|
||||
"""渲染并保存到文件"""
|
||||
html = self.render(report_data, llm_sections, enable_agent=enable_agent, lang=lang)
|
||||
school = report_data["school"]
|
||||
|
||||
if output_path is None:
|
||||
output_dir = OUTPUT_DIR
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
suffix = "_report_en.html" if lang == "en" else "_报告.html"
|
||||
output_path = output_dir / f"{school}{suffix}"
|
||||
|
||||
output_path.write_text(html, encoding="utf-8")
|
||||
return output_path
|
||||
|
||||
def _build_radar_data(self, report_data: Dict) -> Dict:
|
||||
"""构建雷达图数据"""
|
||||
school = report_data["school"]
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
school_values = [report_data["dimensions"][d]["score"] for d in dims]
|
||||
avg_values = [report_data["dimensions"][d]["district_avg"] for d in dims]
|
||||
|
||||
return {
|
||||
"dimensions": dims,
|
||||
"legend": [school, "区均值"],
|
||||
"series": [
|
||||
{"name": school, "values": school_values},
|
||||
{"name": "区均值", "values": avg_values},
|
||||
],
|
||||
}
|
||||
|
||||
def _build_sub_dim_charts(self, report_data: Dict) -> Dict:
|
||||
"""构建各维度的子维度柱状图数据"""
|
||||
charts = {}
|
||||
part_names = {
|
||||
"课程领导力": "part3", "教学变革力": "part4", "学生发展指导力": "part5",
|
||||
"教师发展支持力": "part6", "教育质量评估力": "part7",
|
||||
"教育条件保障力": "part8", "数字化赋能力": "part9",
|
||||
}
|
||||
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
part_id = part_names[dim_name]
|
||||
sub_dims = info["sub_dimensions"]
|
||||
|
||||
categories = []
|
||||
school_values = []
|
||||
avg_values = []
|
||||
|
||||
for sd in sub_dims:
|
||||
sd_data = report_data["sub_dimensions"].get(sd, {})
|
||||
if sd_data:
|
||||
categories.append(sd)
|
||||
school_values.append(round(sd_data["score"], 2))
|
||||
avg_values.append(round(sd_data["district_avg"], 2))
|
||||
|
||||
charts[part_id] = {
|
||||
"categories": categories,
|
||||
"school_values": school_values,
|
||||
"avg_values": avg_values,
|
||||
}
|
||||
|
||||
return charts
|
||||
|
||||
def _build_score_compare(self, report_data: Dict) -> Dict:
|
||||
"""构建得分对比横向条形图数据：本校 vs 区均值 vs 同类学校均值"""
|
||||
school = report_data["school"]
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
return {
|
||||
"categories": dims,
|
||||
"school_values": [round(report_data["dimensions"][d]["score"], 2) for d in dims],
|
||||
"district_avg": [round(report_data["dimensions"][d]["district_avg"], 2) for d in dims],
|
||||
"same_type_avg": [round(report_data["dimensions"][d]["same_type_avg"], 2) for d in dims],
|
||||
"school_name": school,
|
||||
}
|
||||
|
||||
def _build_cluster_radar(self, report_data: Dict) -> Dict:
|
||||
"""构建聚类类型特征对比雷达图（较好类 vs 待提升类）"""
|
||||
all_dim_scores = report_data.get("all_schools_dim_scores", {})
|
||||
if not all_dim_scores:
|
||||
return {}
|
||||
|
||||
# 从 overall cluster info 中提取各学校的聚类标签
|
||||
school = report_data["school"]
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
|
||||
# 计算各学校的总体得分来判断聚类
|
||||
school_totals = {}
|
||||
for s in list(list(all_dim_scores.values())[0].keys()):
|
||||
total = 0
|
||||
for d in dims:
|
||||
total += all_dim_scores.get(d, {}).get(s, 50)
|
||||
school_totals[s] = total / len(dims)
|
||||
|
||||
# 二分聚类（简单按总分中位数分）
|
||||
median_score = sorted(school_totals.values())[len(school_totals) // 2]
|
||||
good_schools = [s for s, v in school_totals.items() if v >= median_score]
|
||||
weak_schools = [s for s, v in school_totals.items() if v < median_score]
|
||||
|
||||
good_avgs = []
|
||||
weak_avgs = []
|
||||
for d in dims:
|
||||
dim_data = all_dim_scores.get(d, {})
|
||||
good_avgs.append(round(sum(dim_data.get(s, 50) for s in good_schools) / max(len(good_schools), 1), 2))
|
||||
weak_avgs.append(round(sum(dim_data.get(s, 50) for s in weak_schools) / max(len(weak_schools), 1), 2))
|
||||
|
||||
return {
|
||||
"dimensions": dims,
|
||||
"legend": ["课程实施较好类", "课程实施待提升类", school],
|
||||
"series": [
|
||||
{"name": "课程实施较好类", "values": good_avgs},
|
||||
{"name": "课程实施待提升类", "values": weak_avgs},
|
||||
{"name": school, "values": [round(report_data["dimensions"][d]["score"], 2) for d in dims]},
|
||||
],
|
||||
"good_count": len(good_schools),
|
||||
"weak_count": len(weak_schools),
|
||||
}
|
||||
|
||||
def _build_correlation_heatmap(self, report_data: Dict) -> Dict:
|
||||
"""构建维度间相关性热力图"""
|
||||
correlation = report_data.get("correlation", {})
|
||||
if not correlation:
|
||||
return {}
|
||||
|
||||
dims = list(correlation.keys())
|
||||
# 构建二维数组 [x_index, y_index, value]
|
||||
data = []
|
||||
for i, d1 in enumerate(dims):
|
||||
for j, d2 in enumerate(dims):
|
||||
val = correlation.get(d1, {}).get(d2, 0)
|
||||
data.append([i, j, round(val, 3) if val is not None else 0])
|
||||
|
||||
# 短名
|
||||
short_names = [d.replace("力", "").replace("教育", "") for d in dims]
|
||||
|
||||
return {
|
||||
"dimensions": dims,
|
||||
"short_names": short_names,
|
||||
"data": data,
|
||||
}
|
||||
|
||||
def _build_level_distribution(self, report_data: Dict) -> Dict:
|
||||
"""构建各三级维度水平分布堆叠条形图"""
|
||||
charts = {}
|
||||
part_names = {
|
||||
"课程领导力": "part3", "教学变革力": "part4", "学生发展指导力": "part5",
|
||||
"教师发展支持力": "part6", "教育质量评估力": "part7",
|
||||
"教育条件保障力": "part8", "数字化赋能力": "part9",
|
||||
}
|
||||
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
part_id = part_names[dim_name]
|
||||
sub_dims = info["sub_dimensions"]
|
||||
|
||||
categories = []
|
||||
level1_pcts = []
|
||||
level2_pcts = []
|
||||
level3_pcts = []
|
||||
level4_pcts = []
|
||||
school_levels = []
|
||||
|
||||
for sd in sub_dims:
|
||||
sd_data = report_data["sub_dimensions"].get(sd, {})
|
||||
if not sd_data:
|
||||
continue
|
||||
dist = sd_data.get("level_distribution", {})
|
||||
total = sum(dist.values())
|
||||
if total == 0:
|
||||
continue
|
||||
|
||||
categories.append(sd)
|
||||
level1_pcts.append(round(dist.get("水平1", 0) / total * 100, 1))
|
||||
level2_pcts.append(round(dist.get("水平2", 0) / total * 100, 1))
|
||||
level3_pcts.append(round(dist.get("水平3", 0) / total * 100, 1))
|
||||
level4_pcts.append(round(dist.get("水平4", 0) / total * 100, 1))
|
||||
school_levels.append(sd_data.get("level", 0))
|
||||
|
||||
charts[part_id] = {
|
||||
"categories": categories,
|
||||
"level1": level1_pcts,
|
||||
"level2": level2_pcts,
|
||||
"level3": level3_pcts,
|
||||
"level4": level4_pcts,
|
||||
"school_levels": school_levels,
|
||||
}
|
||||
|
||||
return charts
|
||||
|
||||
def _build_school_ranking(self, report_data: Dict) -> Dict:
|
||||
"""构建区内各校维度排名对比图"""
|
||||
school = report_data["school"]
|
||||
all_dim_scores = report_data.get("all_schools_dim_scores", {})
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
|
||||
if not all_dim_scores:
|
||||
return {}
|
||||
|
||||
# 获取所有学校名
|
||||
first_dim = list(all_dim_scores.values())[0] if all_dim_scores else {}
|
||||
all_schools = list(first_dim.keys())
|
||||
|
||||
# 计算每校总体得分并排序
|
||||
school_totals = {}
|
||||
for s in all_schools:
|
||||
total = sum(all_dim_scores.get(d, {}).get(s, 50) for d in dims)
|
||||
school_totals[s] = round(total / len(dims), 2)
|
||||
|
||||
sorted_schools = sorted(school_totals.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
return {
|
||||
"schools": [s[0] for s in sorted_schools],
|
||||
"total_scores": [s[1] for s in sorted_schools],
|
||||
"current_school": school,
|
||||
"dimensions": dims,
|
||||
"dim_scores": {
|
||||
d: [round(all_dim_scores.get(d, {}).get(s[0], 50), 2) for s in sorted_schools]
|
||||
for d in dims
|
||||
},
|
||||
}
|
||||
|
||||
def _build_cluster_type_distribution(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
构建课程实施类型分布饼/条形图数据（如参考报告图2-2）
|
||||
展示 较好类 vs 待提升类 在区内各校的分布
|
||||
"""
|
||||
all_dim_scores = report_data.get("all_schools_dim_scores", {})
|
||||
if not all_dim_scores:
|
||||
return {}
|
||||
|
||||
school = report_data["school"]
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
|
||||
# 计算各学校总体得分并二分聚类
|
||||
school_totals = {}
|
||||
first_dim_data = list(all_dim_scores.values())[0] if all_dim_scores else {}
|
||||
all_schools = list(first_dim_data.keys())
|
||||
|
||||
for s in all_schools:
|
||||
total = sum(all_dim_scores.get(d, {}).get(s, 50) for d in dims)
|
||||
school_totals[s] = total / len(dims)
|
||||
|
||||
median_score = sorted(school_totals.values())[len(school_totals) // 2]
|
||||
good_schools = [s for s, v in school_totals.items() if v >= median_score]
|
||||
weak_schools = [s for s, v in school_totals.items() if v < median_score]
|
||||
|
||||
school_cluster = "较好" if school in good_schools else "待提升"
|
||||
|
||||
return {
|
||||
"good_count": len(good_schools),
|
||||
"weak_count": len(weak_schools),
|
||||
"total": len(all_schools),
|
||||
"school_cluster": school_cluster,
|
||||
"school_name": school,
|
||||
"good_schools": good_schools,
|
||||
"weak_schools": weak_schools,
|
||||
}
|
||||
|
||||
def _build_cluster_line_compare(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
构建两类学校特征折线对比图（如参考报告图2-3）
|
||||
两条折线：较好类 vs 待提升类在7个维度上的得分
|
||||
"""
|
||||
all_dim_scores = report_data.get("all_schools_dim_scores", {})
|
||||
if not all_dim_scores:
|
||||
return {}
|
||||
|
||||
school = report_data["school"]
|
||||
dims = list(report_data["dimensions"].keys())
|
||||
|
||||
# 计算聚类
|
||||
first_dim_data = list(all_dim_scores.values())[0] if all_dim_scores else {}
|
||||
all_schools = list(first_dim_data.keys())
|
||||
|
||||
school_totals = {}
|
||||
for s in all_schools:
|
||||
total = sum(all_dim_scores.get(d, {}).get(s, 50) for d in dims)
|
||||
school_totals[s] = total / len(dims)
|
||||
|
||||
median_score = sorted(school_totals.values())[len(school_totals) // 2]
|
||||
good_schools = [s for s, v in school_totals.items() if v >= median_score]
|
||||
weak_schools = [s for s, v in school_totals.items() if v < median_score]
|
||||
|
||||
good_avgs = []
|
||||
weak_avgs = []
|
||||
for d in dims:
|
||||
dim_data = all_dim_scores.get(d, {})
|
||||
good_avgs.append(round(sum(dim_data.get(s, 50) for s in good_schools) / max(len(good_schools), 1), 2))
|
||||
weak_avgs.append(round(sum(dim_data.get(s, 50) for s in weak_schools) / max(len(weak_schools), 1), 2))
|
||||
|
||||
return {
|
||||
"dimensions": dims,
|
||||
"good_values": good_avgs,
|
||||
"weak_values": weak_avgs,
|
||||
"good_count": len(good_schools),
|
||||
"weak_count": len(weak_schools),
|
||||
}
|
||||
|
||||
def _build_dim_scatter_charts(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
构建各二级维度的聚类散点图数据（2D / 3D）
|
||||
参考报告中每个维度都有一个散点图显示所有学校的聚类分布
|
||||
对于有2个子维度的 → 2D散点图
|
||||
对于有3个子维度的 → 3D散点图
|
||||
对于有4个子维度的 → 取前2个主成分的2D散点图
|
||||
"""
|
||||
all_sub_scores = report_data.get("all_schools_sub_scores", {})
|
||||
if not all_sub_scores:
|
||||
return {}
|
||||
|
||||
school = report_data["school"]
|
||||
charts = {}
|
||||
part_names = {
|
||||
"课程领导力": "part3", "教学变革力": "part4", "学生发展指导力": "part5",
|
||||
"教师发展支持力": "part6", "教育质量评估力": "part7",
|
||||
"教育条件保障力": "part8", "数字化赋能力": "part9",
|
||||
}
|
||||
|
||||
first_sub = list(all_sub_scores.values())[0] if all_sub_scores else {}
|
||||
all_schools = list(first_sub.keys())
|
||||
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
part_id = part_names[dim_name]
|
||||
sub_dims = info["sub_dimensions"]
|
||||
n_subs = len(sub_dims)
|
||||
|
||||
# 获取各学校在这些子维度上的得分
|
||||
school_scores = {}
|
||||
for s in all_schools:
|
||||
scores = []
|
||||
for sd in sub_dims:
|
||||
val = all_sub_scores.get(sd, {}).get(s, 50)
|
||||
scores.append(round(float(val), 2))
|
||||
school_scores[s] = scores
|
||||
|
||||
# 简单二分聚类
|
||||
totals = {s: sum(v) / len(v) for s, v in school_scores.items()}
|
||||
med = sorted(totals.values())[len(totals) // 2]
|
||||
clusters = {s: 0 if totals[s] >= med else 1 for s in all_schools}
|
||||
|
||||
if n_subs == 2:
|
||||
# 2D散点图
|
||||
data_good = []
|
||||
data_weak = []
|
||||
school_point = None
|
||||
|
||||
for s in all_schools:
|
||||
point = school_scores[s]
|
||||
if s == school:
|
||||
school_point = point
|
||||
elif clusters[s] == 0:
|
||||
data_good.append(point)
|
||||
else:
|
||||
data_weak.append(point)
|
||||
|
||||
charts[part_id] = {
|
||||
"type": "2d",
|
||||
"axes": sub_dims,
|
||||
"good_data": data_good,
|
||||
"weak_data": data_weak,
|
||||
"school_point": school_point,
|
||||
"school_name": school,
|
||||
"dim_name": dim_name,
|
||||
}
|
||||
|
||||
elif n_subs == 3:
|
||||
# 3D散点图
|
||||
data_good = []
|
||||
data_weak = []
|
||||
school_point = None
|
||||
|
||||
for s in all_schools:
|
||||
point = school_scores[s]
|
||||
if s == school:
|
||||
school_point = point
|
||||
elif clusters[s] == 0:
|
||||
data_good.append(point)
|
||||
else:
|
||||
data_weak.append(point)
|
||||
|
||||
charts[part_id] = {
|
||||
"type": "3d",
|
||||
"axes": sub_dims,
|
||||
"good_data": data_good,
|
||||
"weak_data": data_weak,
|
||||
"school_point": school_point,
|
||||
"school_name": school,
|
||||
"dim_name": dim_name,
|
||||
}
|
||||
|
||||
elif n_subs >= 4:
|
||||
# 取前两个子维度做2D散点
|
||||
axes = sub_dims[:2]
|
||||
data_good = []
|
||||
data_weak = []
|
||||
school_point = None
|
||||
|
||||
for s in all_schools:
|
||||
point = school_scores[s][:2]
|
||||
if s == school:
|
||||
school_point = point
|
||||
elif clusters[s] == 0:
|
||||
data_good.append(point)
|
||||
else:
|
||||
data_weak.append(point)
|
||||
|
||||
charts[part_id] = {
|
||||
"type": "2d",
|
||||
"axes": axes,
|
||||
"good_data": data_good,
|
||||
"weak_data": data_weak,
|
||||
"school_point": school_point,
|
||||
"school_name": school,
|
||||
"dim_name": dim_name,
|
||||
}
|
||||
|
||||
return charts
|
||||
|
||||
def _build_dim_score_bars(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
构建各维度独立得分柱状图数据（如参考报告图3-1、图4-1等）
|
||||
展示本校 vs 区均值 vs 同类学校均值 的对比
|
||||
"""
|
||||
school = report_data["school"]
|
||||
charts = {}
|
||||
part_names = {
|
||||
"课程领导力": "part3", "教学变革力": "part4", "学生发展指导力": "part5",
|
||||
"教师发展支持力": "part6", "教育质量评估力": "part7",
|
||||
"教育条件保障力": "part8", "数字化赋能力": "part9",
|
||||
}
|
||||
|
||||
for dim_name, dim_data in report_data["dimensions"].items():
|
||||
part_id = part_names.get(dim_name, "")
|
||||
if not part_id:
|
||||
continue
|
||||
|
||||
charts[part_id] = {
|
||||
"dim_name": dim_name,
|
||||
"school_name": school,
|
||||
"school_score": round(dim_data["score"], 2),
|
||||
"district_avg": round(dim_data["district_avg"], 2),
|
||||
"same_type_avg": round(dim_data["same_type_avg"], 2),
|
||||
}
|
||||
|
||||
return charts
|
||||
|
||||
def _build_dim_sub_radar_charts(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
构建各二级维度的子维度雷达图（如参考报告图3-2）
|
||||
多条线对比: 本校 vs 区均值 vs 同类学校均值
|
||||
"""
|
||||
school = report_data["school"]
|
||||
all_sub_scores = report_data.get("all_schools_sub_scores", {})
|
||||
charts = {}
|
||||
part_names = {
|
||||
"课程领导力": "part3", "教学变革力": "part4", "学生发展指导力": "part5",
|
||||
"教师发展支持力": "part6", "教育质量评估力": "part7",
|
||||
"教育条件保障力": "part8", "数字化赋能力": "part9",
|
||||
}
|
||||
|
||||
# Get same-type schools
|
||||
school_info = report_data.get("school_info", {})
|
||||
school_type = school_info.get("type", "")
|
||||
all_schools = list(list(all_sub_scores.values())[0].keys()) if all_sub_scores else []
|
||||
same_type_schools = [s for s in all_schools if SCHOOL_TYPE_MAP.get(s, {}).get("type") == school_type]
|
||||
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
part_id = part_names.get(dim_name, "")
|
||||
if not part_id:
|
||||
continue
|
||||
|
||||
sub_dims = info["sub_dimensions"]
|
||||
school_values = []
|
||||
district_avg = []
|
||||
same_type_avg = []
|
||||
|
||||
for sd in sub_dims:
|
||||
sd_data = report_data["sub_dimensions"].get(sd, {})
|
||||
if sd_data:
|
||||
school_values.append(round(sd_data["score"], 2))
|
||||
district_avg.append(round(sd_data["district_avg"], 2))
|
||||
# 计算同类学校均值
|
||||
if same_type_schools and all_sub_scores:
|
||||
st_vals = [all_sub_scores.get(sd, {}).get(s, 50) for s in same_type_schools]
|
||||
same_type_avg.append(round(sum(float(v) for v in st_vals) / len(st_vals), 2))
|
||||
else:
|
||||
same_type_avg.append(district_avg[-1])
|
||||
|
||||
if len(sub_dims) >= 3:
|
||||
charts[part_id] = {
|
||||
"type": "radar",
|
||||
"sub_dims": sub_dims,
|
||||
"school_name": school,
|
||||
"school_values": school_values,
|
||||
"district_avg": district_avg,
|
||||
"same_type_avg": same_type_avg,
|
||||
}
|
||||
else:
|
||||
charts[part_id] = {
|
||||
"type": "bar",
|
||||
"sub_dims": sub_dims,
|
||||
"school_name": school,
|
||||
"school_values": school_values,
|
||||
"district_avg": district_avg,
|
||||
"same_type_avg": same_type_avg,
|
||||
}
|
||||
|
||||
return charts
|
||||
|
||||
# ========== 创新图表数据构建 ==========
|
||||
|
||||
def _build_profile_card(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
A. 学校画像卡（综合仪表盘）
|
||||
- 总体得分环形仪表盘
|
||||
- 7个维度的红绿灯状态（基于各维度下子维度的最低水平）
|
||||
- 20个三级维度的水平分布概览
|
||||
"""
|
||||
school = report_data["school"]
|
||||
overall = report_data["overall"]
|
||||
|
||||
# 维度红绿灯：每个二级维度取其子维度的最低水平作为"短板"指示
|
||||
dim_signals = []
|
||||
for dim_name, dim_data in report_data["dimensions"].items():
|
||||
sub_dims = DIMENSION_FRAMEWORK[dim_name]["sub_dimensions"]
|
||||
levels = []
|
||||
for sd in sub_dims:
|
||||
sd_data = report_data["sub_dimensions"].get(sd, {})
|
||||
if sd_data:
|
||||
levels.append(sd_data.get("level", 0))
|
||||
min_level = min(levels) if levels else 0
|
||||
avg_level = round(sum(levels) / len(levels), 1) if levels else 0
|
||||
dim_signals.append({
|
||||
"name": dim_name,
|
||||
"score": round(dim_data["score"], 2),
|
||||
"min_level": min_level,
|
||||
"avg_level": avg_level,
|
||||
"rank": dim_data.get("rank_in_district", 0),
|
||||
})
|
||||
|
||||
# 20个三级维度的水平分布统计
|
||||
level_counts = {1: 0, 2: 0, 3: 0, 4: 0}
|
||||
for sd_name, sd_data in report_data["sub_dimensions"].items():
|
||||
lv = sd_data.get("level", 0)
|
||||
if lv in level_counts:
|
||||
level_counts[lv] += 1
|
||||
|
||||
return {
|
||||
"school_name": school,
|
||||
"school_type": report_data["school_info"].get("type", ""),
|
||||
"total_score": overall["score"],
|
||||
"district_avg": overall["district_avg"],
|
||||
"rank": overall["rank_in_district"],
|
||||
"total_schools": overall["total_schools"],
|
||||
"cluster": overall.get("cluster", ""),
|
||||
"dim_signals": dim_signals,
|
||||
"level_counts": level_counts,
|
||||
"total_sub_dims": sum(level_counts.values()),
|
||||
}
|
||||
|
||||
def _build_quadrant_chart(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
B. 优势-短板象限图（Gap Analysis）
|
||||
X轴 = 得分, Y轴 = 与区均值的差值
|
||||
四象限：右上=核心优势, 左下=急需改进, 右下=隐性风险, 左上=潜力项
|
||||
"""
|
||||
school = report_data["school"]
|
||||
items = []
|
||||
|
||||
for sd_name, sd_data in report_data["sub_dimensions"].items():
|
||||
parent_dim = sd_data.get("parent_dimension", "")
|
||||
items.append({
|
||||
"name": sd_name,
|
||||
"parent": parent_dim,
|
||||
"score": round(sd_data["score"], 2),
|
||||
"diff": round(sd_data["diff_district"], 2),
|
||||
"level": sd_data.get("level", 0),
|
||||
})
|
||||
|
||||
return {
|
||||
"school_name": school,
|
||||
"items": items,
|
||||
"x_center": 50, # 区均值（标准化后均值=50）
|
||||
"y_center": 0, # 差值=0 的参照线
|
||||
}
|
||||
|
||||
def _build_thermometer_data(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
C. 维度温度计条形图数据
|
||||
为每个三级维度构建横向温度计数据：
|
||||
- 得分范围20-80
|
||||
- 水平分界线
|
||||
- 本校位置、区均值位置、同类学校均值位置
|
||||
"""
|
||||
school = report_data["school"]
|
||||
school_type = report_data["school_info"].get("type", "")
|
||||
all_sub_scores = report_data.get("all_schools_sub_scores", {})
|
||||
same_type_schools = [s for s in SCHOOL_TYPE_MAP
|
||||
if SCHOOL_TYPE_MAP[s].get("type") == school_type]
|
||||
|
||||
thermometers = {}
|
||||
for sd_name, sd_data in report_data["sub_dimensions"].items():
|
||||
# 同类学校均值
|
||||
if same_type_schools and sd_name in all_sub_scores:
|
||||
st_vals = [float(all_sub_scores[sd_name].get(s, 50)) for s in same_type_schools]
|
||||
same_type_avg = round(sum(st_vals) / len(st_vals), 2)
|
||||
else:
|
||||
same_type_avg = round(sd_data["district_avg"], 2)
|
||||
|
||||
# 水平阈值
|
||||
thresholds = LEVEL_THRESHOLDS.get(sd_name, {})
|
||||
|
||||
thermometers[sd_name] = {
|
||||
"score": round(sd_data["score"], 2),
|
||||
"district_avg": round(sd_data["district_avg"], 2),
|
||||
"same_type_avg": same_type_avg,
|
||||
"level": sd_data.get("level", 0),
|
||||
"thresholds": {
|
||||
"level4": thresholds.get("level4", 57),
|
||||
"level3": thresholds.get("level3", 50),
|
||||
"level2": thresholds.get("level2", 43),
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
"school_name": school,
|
||||
"data": thermometers,
|
||||
}
|
||||
|
||||
def _build_waterfall_chart(self, report_data: Dict) -> Dict:
|
||||
"""
|
||||
D. 进步空间瀑布图
|
||||
展示：如果每个低于水平3的子维度提升到水平3的阈值，总分增加多少
|
||||
让校长看到"改哪几个点收益最大"
|
||||
"""
|
||||
school = report_data["school"]
|
||||
current_total = report_data["overall"]["score"]
|
||||
|
||||
# 找出所有低于水平3的子维度
|
||||
improvement_items = []
|
||||
for sd_name, sd_data in report_data["sub_dimensions"].items():
|
||||
level = sd_data.get("level", 0)
|
||||
if level < 3:
|
||||
current_score = sd_data["score"]
|
||||
# 提升到水平3的阈值
|
||||
target_score = LEVEL_THRESHOLDS.get(sd_name, {}).get("level3", 50)
|
||||
gap = round(target_score - current_score, 2)
|
||||
if gap > 0:
|
||||
improvement_items.append({
|
||||
"name": sd_name,
|
||||
"parent": sd_data.get("parent_dimension", ""),
|
||||
"current_score": round(current_score, 2),
|
||||
"target_score": round(target_score, 2),
|
||||
"gap": gap,
|
||||
"current_level": level,
|
||||
})
|
||||
|
||||
# 按收益从大到小排序
|
||||
improvement_items.sort(key=lambda x: x["gap"], reverse=True)
|
||||
|
||||
# 估算总分提升（简化：假设20个子维度等权重影响总分）
|
||||
total_sub_dims = len(report_data["sub_dimensions"])
|
||||
cumulative = current_total
|
||||
waterfall_steps = [{"name": "当前总分", "value": round(current_total, 2), "type": "current"}]
|
||||
|
||||
for item in improvement_items:
|
||||
# 粗略估算：子维度提升gap分 → 总分提升 gap / total_sub_dims * 权重
|
||||
# 实际PCA权重不同，此处用等权近似
|
||||
estimated_gain = round(item["gap"] / total_sub_dims, 2)
|
||||
cumulative += estimated_gain
|
||||
waterfall_steps.append({
|
||||
"name": item["name"],
|
||||
"value": round(estimated_gain, 2),
|
||||
"type": "gain",
|
||||
"detail": f"从水平{item['current_level']}→水平3 (+{item['gap']}分)",
|
||||
})
|
||||
|
||||
waterfall_steps.append({"name": "潜在总分", "value": round(cumulative, 2), "type": "potential"})
|
||||
|
||||
return {
|
||||
"school_name": school,
|
||||
"current_total": round(current_total, 2),
|
||||
"potential_total": round(cumulative, 2),
|
||||
"total_gain": round(cumulative - current_total, 2),
|
||||
"steps": waterfall_steps,
|
||||
"improvements": improvement_items,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _to_namespace(d: Dict) -> Dict:
|
||||
"""将dict转为可用点号访问的对象（Jinja2兼容）"""
|
||||
class Namespace(dict):
|
||||
def __getattr__(self, key):
|
||||
try:
|
||||
return self[key]
|
||||
except KeyError:
|
||||
return None
|
||||
return Namespace(d) if isinstance(d, dict) else d
|
||||
|
||||
# ========== AI 对话助手配置 ==========
|
||||
|
||||
@staticmethod
|
||||
def _xor_encode(plaintext: str, xor_key: str) -> str:
|
||||
"""XOR + Base64 编码（简单混淆，防止明文暴露）"""
|
||||
xor_bytes = bytearray(len(plaintext))
|
||||
for i, ch in enumerate(plaintext):
|
||||
xor_bytes[i] = ord(ch) ^ ord(xor_key[i % len(xor_key)])
|
||||
return base64.b64encode(xor_bytes).decode('ascii')
|
||||
|
||||
def _build_chat_config(self, report_data: Dict) -> tuple:
|
||||
"""
|
||||
构建 AI 对话助手的配置和上下文数据
|
||||
|
||||
Returns:
|
||||
(chat_config dict, report_data_json string)
|
||||
"""
|
||||
# 从 backend/app/config.py 读取 LLM 设置（全项目唯一真相源）
|
||||
_backend_path = str(Path(__file__).parent.parent.parent.parent / "backend")
|
||||
if _backend_path not in sys.path:
|
||||
sys.path.insert(0, _backend_path)
|
||||
from app.config import LLM_BASE_URL, LLM_API_KEY, LLM_MODEL
|
||||
|
||||
# 生成随机 XOR key（每次渲染不同）
|
||||
xor_key = ''.join(random.choices(string.ascii_letters + string.digits, k=16))
|
||||
|
||||
# XOR 编码 API Key
|
||||
api_key_encoded = self._xor_encode(LLM_API_KEY, xor_key)
|
||||
|
||||
chat_config = {
|
||||
"api_key_encoded": api_key_encoded,
|
||||
"xor_key": xor_key,
|
||||
"api_base_url": LLM_BASE_URL,
|
||||
"model": LLM_MODEL,
|
||||
"school_name": report_data["school"],
|
||||
}
|
||||
|
||||
# 构建精简版 report_data JSON（去掉超大的 all_schools 数据以节省体积）
|
||||
slim_data = {
|
||||
"school": report_data.get("school"),
|
||||
"school_info": report_data.get("school_info", {}),
|
||||
"overall": report_data.get("overall", {}),
|
||||
"dimensions": report_data.get("dimensions", {}),
|
||||
"sub_dimensions": report_data.get("sub_dimensions", {}),
|
||||
}
|
||||
|
||||
# 清理 numpy 类型
|
||||
def clean(obj):
|
||||
if isinstance(obj, dict):
|
||||
return {k: clean(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, list):
|
||||
return [clean(v) for v in obj]
|
||||
elif isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
elif isinstance(obj, (np.floating,)):
|
||||
return round(float(obj), 4)
|
||||
elif isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
elif isinstance(obj, float):
|
||||
return round(obj, 4)
|
||||
return obj
|
||||
|
||||
report_data_json = json.dumps(clean(slim_data), ensure_ascii=False)
|
||||
|
||||
return chat_config, report_data_json
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,374 @@
|
||||
"""
|
||||
二期统计引擎 (era2)
|
||||
支持全市基准标准化 + 同类学校均值计算
|
||||
"""
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sklearn.cluster import KMeans
|
||||
from scipy import stats
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from config_era2 import (
|
||||
PCA_MEAN, PCA_STD, LEVEL_THRESHOLDS, DIMENSION_FRAMEWORK, SUBJECTS,
|
||||
SCHOOL_TYPE_MAP,
|
||||
)
|
||||
try:
|
||||
from config_era2 import CLUSTER_CONFIG
|
||||
except ImportError:
|
||||
CLUSTER_CONFIG = {}
|
||||
try:
|
||||
from config_era2 import LEVEL_DESCRIPTIONS
|
||||
except ImportError:
|
||||
LEVEL_DESCRIPTIONS = {}
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StatsEngineEra2:
|
||||
"""二期统计引擎"""
|
||||
|
||||
def __init__(self):
|
||||
self._dimension_scores: Optional[pd.DataFrame] = None
|
||||
self._sub_dimension_scores: Optional[pd.DataFrame] = None
|
||||
|
||||
def standardize_scores(self, raw_scores: np.ndarray) -> np.ndarray:
|
||||
if len(raw_scores) < 2:
|
||||
return np.full_like(raw_scores, PCA_MEAN, dtype=float)
|
||||
mean = np.nanmean(raw_scores)
|
||||
std = np.nanstd(raw_scores, ddof=1)
|
||||
if std == 0 or np.isnan(std):
|
||||
return np.full_like(raw_scores, PCA_MEAN, dtype=float)
|
||||
return (raw_scores - mean) / std * PCA_STD + PCA_MEAN
|
||||
|
||||
def pca_compose(self, data_matrix: pd.DataFrame) -> np.ndarray:
|
||||
filled = data_matrix.fillna(data_matrix.mean())
|
||||
if filled.shape[1] == 0:
|
||||
return np.full(filled.shape[0], PCA_MEAN)
|
||||
if filled.shape[1] == 1:
|
||||
return self.standardize_scores(filled.iloc[:, 0].values)
|
||||
scaler = StandardScaler()
|
||||
scaled = scaler.fit_transform(filled)
|
||||
n_components = min(1, filled.shape[1], filled.shape[0])
|
||||
pca = PCA(n_components=n_components)
|
||||
scores = pca.fit_transform(scaled)[:, 0]
|
||||
loadings = pca.components_[0]
|
||||
if np.sum(loadings) < 0:
|
||||
scores = -scores
|
||||
return self.standardize_scores(scores)
|
||||
|
||||
def determine_level(self, score: float, dimension: str) -> int:
|
||||
thresholds = LEVEL_THRESHOLDS.get(dimension, {})
|
||||
if not thresholds:
|
||||
if score > 55: return 4
|
||||
elif score > 50: return 3
|
||||
elif score > 45: return 2
|
||||
else: return 1
|
||||
if score > thresholds["level4"]: return 4
|
||||
elif score > thresholds["level3"]: return 3
|
||||
elif score > thresholds["level2"]: return 2
|
||||
else: return 1
|
||||
|
||||
def get_level_description(self, dimension: str, level: int) -> str:
|
||||
descriptions = LEVEL_DESCRIPTIONS.get(dimension, {})
|
||||
return descriptions.get(level, f"水平{level}")
|
||||
|
||||
def compute_dimension_scores(self, school_raw_scores: Dict[str, Dict[str, List[float]]]) -> pd.DataFrame:
|
||||
"""原始赋分 → nanmean → z-score标准化（旧方法，兼容保留）"""
|
||||
schools = list(school_raw_scores.keys())
|
||||
all_sub_dims = []
|
||||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||||
all_sub_dims.extend(info["sub_dimensions"])
|
||||
|
||||
raw_matrix = {}
|
||||
for sub_dim in all_sub_dims:
|
||||
values = []
|
||||
for school in schools:
|
||||
scores = school_raw_scores.get(school, {}).get(sub_dim, [])
|
||||
values.append(np.nanmean(scores) if scores else np.nan)
|
||||
raw_matrix[sub_dim] = values
|
||||
|
||||
raw_df = pd.DataFrame(raw_matrix, index=schools)
|
||||
|
||||
result = pd.DataFrame(index=schools)
|
||||
for sub_dim in all_sub_dims:
|
||||
if sub_dim in raw_df.columns:
|
||||
result[sub_dim] = self.standardize_scores(raw_df[sub_dim].values)
|
||||
else:
|
||||
result[sub_dim] = PCA_MEAN
|
||||
|
||||
self._sub_dimension_scores = result
|
||||
return result
|
||||
|
||||
def compute_dimension_scores_pca(self, pca_sub_scores: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
接受PCA引擎的输出（已经是子维度得分的DataFrame），直接使用。
|
||||
PCA引擎内部已完成 Z标准化 → PCA → ×10+50 的全流程。
|
||||
|
||||
Args:
|
||||
pca_sub_scores: PcaScoringEngineEra2.compute_all() 的输出
|
||||
DataFrame, index=学校, columns=子维度名
|
||||
Returns:
|
||||
与 compute_dimension_scores 相同格式的 DataFrame
|
||||
"""
|
||||
all_sub_dims = []
|
||||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||||
all_sub_dims.extend(info["sub_dimensions"])
|
||||
|
||||
result = pd.DataFrame(index=pca_sub_scores.index)
|
||||
for sub_dim in all_sub_dims:
|
||||
if sub_dim in pca_sub_scores.columns:
|
||||
result[sub_dim] = pca_sub_scores[sub_dim]
|
||||
else:
|
||||
result[sub_dim] = PCA_MEAN
|
||||
|
||||
self._sub_dimension_scores = result
|
||||
return result
|
||||
|
||||
def compute_dimension_aggregates(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
|
||||
result = pd.DataFrame(index=sub_scores.index)
|
||||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||||
sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
|
||||
if sub_dims:
|
||||
result[dim] = sub_scores[sub_dims].mean(axis=1)
|
||||
else:
|
||||
result[dim] = PCA_MEAN
|
||||
result["总体得分"] = result[list(DIMENSION_FRAMEWORK.keys())].mean(axis=1)
|
||||
self._dimension_scores = result
|
||||
return result
|
||||
|
||||
def compute_levels(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
|
||||
levels = pd.DataFrame(index=sub_scores.index)
|
||||
for col in sub_scores.columns:
|
||||
levels[col] = sub_scores[col].apply(lambda x: self.determine_level(x, col))
|
||||
return levels
|
||||
|
||||
def cluster_analysis(self, scores: pd.DataFrame, n_clusters: int = 2,
|
||||
dimension_name: Optional[str] = None) -> Dict:
|
||||
"""
|
||||
K-means 聚类分析。
|
||||
|
||||
Args:
|
||||
scores: 学校×子维度 的得分 DataFrame
|
||||
n_clusters: 聚类数（默认2，可通过 CLUSTER_CONFIG 覆盖）
|
||||
dimension_name: 维度名称，用于从 CLUSTER_CONFIG 查询聚类数
|
||||
"""
|
||||
# 从配置覆盖聚类数
|
||||
if dimension_name and dimension_name in CLUSTER_CONFIG:
|
||||
n_clusters = CLUSTER_CONFIG[dimension_name]
|
||||
|
||||
scaler = StandardScaler()
|
||||
scaled = scaler.fit_transform(scores.fillna(PCA_MEAN))
|
||||
n_clusters = min(n_clusters, len(scores))
|
||||
if n_clusters < 2:
|
||||
return {"labels": [0] * len(scores), "centers": scores.values.tolist()}
|
||||
kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
|
||||
labels = kmeans.fit_predict(scaled)
|
||||
|
||||
cluster_means = {}
|
||||
for c in range(n_clusters):
|
||||
mask = labels == c
|
||||
cluster_means[c] = scores[mask].mean().to_dict()
|
||||
|
||||
avg_per_cluster = {c: np.mean(list(v.values())) for c, v in cluster_means.items()}
|
||||
sorted_clusters = sorted(avg_per_cluster.items(), key=lambda x: x[1], reverse=True)
|
||||
|
||||
# 命名策略：2类 → 较好/待提升，3类 → 较好/中等/待提升
|
||||
cluster_names = {}
|
||||
if n_clusters == 2:
|
||||
name_list = ["较好", "待提升"]
|
||||
elif n_clusters == 3:
|
||||
name_list = ["较好", "中等", "待提升"]
|
||||
else:
|
||||
name_list = [f"第{i+1}类" for i in range(n_clusters)]
|
||||
|
||||
for rank, (c, _) in enumerate(sorted_clusters):
|
||||
cluster_names[c] = name_list[rank] if rank < len(name_list) else f"第{rank+1}类"
|
||||
|
||||
return {
|
||||
"labels": labels.tolist(),
|
||||
"n_clusters": n_clusters,
|
||||
"school_clusters": {
|
||||
school: cluster_names[labels[i]]
|
||||
for i, school in enumerate(scores.index)
|
||||
},
|
||||
"cluster_means": cluster_means,
|
||||
"cluster_names": cluster_names,
|
||||
}
|
||||
|
||||
def t_test_vs_mean(self, school_scores: np.ndarray, ref_mean: float) -> Dict:
|
||||
scores = school_scores[~np.isnan(school_scores)]
|
||||
if len(scores) < 2:
|
||||
return {"t": np.nan, "p": np.nan, "significant": False, "n": len(scores)}
|
||||
t_stat, p_value = stats.ttest_1samp(scores, ref_mean)
|
||||
return {
|
||||
"t": round(float(t_stat), 3),
|
||||
"p": round(float(p_value), 4),
|
||||
"significant": float(p_value) < 0.05,
|
||||
"n": len(scores),
|
||||
}
|
||||
|
||||
def correlation_analysis(self, dim_scores: pd.DataFrame) -> pd.DataFrame:
|
||||
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
|
||||
return dim_scores[dim_cols].corr()
|
||||
|
||||
def compute_school_report_data(self, school: str,
|
||||
sub_scores: pd.DataFrame,
|
||||
dim_scores: pd.DataFrame,
|
||||
district_schools: Optional[List[str]] = None) -> Dict:
|
||||
"""
|
||||
生成单校报告数据。
|
||||
|
||||
sub_scores / dim_scores: 全市所有学校的标准化分数（全市基准）
|
||||
district_schools: 该学校所在区的学校列表（用于计算区均值和区内排名）
|
||||
如果为None，则用sub_scores中所有学校
|
||||
"""
|
||||
all_schools = list(sub_scores.index)
|
||||
if school not in all_schools:
|
||||
raise ValueError(f"学校 '{school}' 不在数据中")
|
||||
|
||||
# 确定区内学校列表
|
||||
if district_schools is None:
|
||||
district_schools = all_schools
|
||||
district_schools = [s for s in district_schools if s in all_schools]
|
||||
|
||||
# 区内分数切片
|
||||
dist_sub = sub_scores.loc[district_schools]
|
||||
dist_dim = dim_scores.loc[district_schools]
|
||||
|
||||
# 区均值
|
||||
district_avg_sub = dist_sub.mean()
|
||||
district_avg_dim = dist_dim.mean()
|
||||
|
||||
# 同类学校均值（从全市SCHOOL_TYPE_MAP中找同类型学校）
|
||||
school_info_data = SCHOOL_TYPE_MAP.get(school, {})
|
||||
school_type = school_info_data.get("type", "")
|
||||
same_type_schools = [
|
||||
s for s in all_schools
|
||||
if SCHOOL_TYPE_MAP.get(s, {}).get("type") == school_type and school_type
|
||||
]
|
||||
if len(same_type_schools) >= 2:
|
||||
same_type_avg_sub = sub_scores.loc[same_type_schools].mean()
|
||||
same_type_avg_dim = dim_scores.loc[same_type_schools].mean()
|
||||
else:
|
||||
same_type_avg_sub = district_avg_sub
|
||||
same_type_avg_dim = district_avg_dim
|
||||
|
||||
# 构建 school_info（从 SCHOOL_TYPE_MAP 获取）
|
||||
school_info = {}
|
||||
if school_info_data:
|
||||
school_info = {
|
||||
"type": school_info_data.get("type", ""),
|
||||
"code": school_info_data.get("code", ""),
|
||||
"nature": school_info_data.get("nature", ""),
|
||||
"feature": school_info_data.get("area", ""), # 所处地区作为feature
|
||||
}
|
||||
|
||||
levels = self.compute_levels(sub_scores)
|
||||
|
||||
# 聚类只在区内做
|
||||
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
|
||||
overall_cluster = self.cluster_analysis(dist_dim[dim_cols], dimension_name="总体")
|
||||
|
||||
dim_clusters = {}
|
||||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||||
sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
|
||||
if sub_dims:
|
||||
dim_clusters[dim] = self.cluster_analysis(dist_sub[sub_dims], dimension_name=dim)
|
||||
|
||||
correlation = self.correlation_analysis(dist_dim)
|
||||
|
||||
# 区内排名
|
||||
school_score = float(dist_dim.loc[school, "总体得分"])
|
||||
rank_in_district = int((dist_dim["总体得分"] >= school_score).sum())
|
||||
|
||||
# 全市排名（不分类型，所有参与监测的学校）
|
||||
rank_in_city = int((dim_scores["总体得分"] >= school_score).sum())
|
||||
total_schools_in_city = len(all_schools)
|
||||
|
||||
# 全市同类排名
|
||||
if len(same_type_schools) >= 2:
|
||||
st_dim = dim_scores.loc[same_type_schools]
|
||||
rank_in_same_type = int((st_dim["总体得分"] >= school_score).sum())
|
||||
total_same_type = len(same_type_schools)
|
||||
else:
|
||||
rank_in_same_type = rank_in_district
|
||||
total_same_type = len(district_schools)
|
||||
|
||||
report = {
|
||||
"school": school,
|
||||
"school_info": school_info,
|
||||
"overall": {
|
||||
"score": round(school_score, 2),
|
||||
"district_avg": round(float(district_avg_dim["总体得分"]), 2),
|
||||
"same_type_avg": round(float(same_type_avg_dim.get("总体得分", PCA_MEAN)), 2),
|
||||
"rank_in_district": rank_in_district,
|
||||
"total_schools": len(district_schools),
|
||||
"rank_in_city": rank_in_city,
|
||||
"total_schools_in_city": total_schools_in_city,
|
||||
"cluster": overall_cluster["school_clusters"].get(school, ""),
|
||||
"school_type": school_type,
|
||||
"same_type_count": total_same_type,
|
||||
"rank_in_same_type": rank_in_same_type,
|
||||
},
|
||||
"dimensions": {},
|
||||
"sub_dimensions": {},
|
||||
"correlation": correlation.to_dict(),
|
||||
"all_schools_dim_scores": dist_dim.to_dict(),
|
||||
"all_schools_sub_scores": dist_sub.to_dict(),
|
||||
}
|
||||
|
||||
for dim in DIMENSION_FRAMEWORK:
|
||||
score = float(dist_dim.loc[school, dim])
|
||||
d_avg = float(district_avg_dim[dim])
|
||||
st_avg = float(same_type_avg_dim.get(dim, PCA_MEAN))
|
||||
|
||||
sub_dims = DIMENSION_FRAMEWORK[dim]["sub_dimensions"]
|
||||
sub_vals = np.array([float(sub_scores.loc[school, s]) for s in sub_dims if s in sub_scores.columns])
|
||||
t_test = self.t_test_vs_mean(sub_vals, d_avg)
|
||||
|
||||
report["dimensions"][dim] = {
|
||||
"score": round(score, 2),
|
||||
"district_avg": round(d_avg, 2),
|
||||
"same_type_avg": round(st_avg, 2),
|
||||
"diff_district": round(score - d_avg, 2),
|
||||
"rank_in_district": int((dist_dim[dim] >= score).sum()),
|
||||
"rank_in_city": int((dim_scores[dim] >= score).sum()),
|
||||
"t_test_vs_district": t_test,
|
||||
"cluster": dim_clusters.get(dim, {}).get("school_clusters", {}).get(school, ""),
|
||||
}
|
||||
|
||||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||||
for sub_dim in info["sub_dimensions"]:
|
||||
if sub_dim not in sub_scores.columns:
|
||||
continue
|
||||
score = float(sub_scores.loc[school, sub_dim])
|
||||
d_avg = float(district_avg_sub[sub_dim])
|
||||
level = int(levels.loc[school, sub_dim])
|
||||
# 区内排名
|
||||
rank = int((dist_sub[sub_dim] >= score).sum())
|
||||
# 全市排名
|
||||
rank_city = int((sub_scores[sub_dim] >= score).sum())
|
||||
# 区内水平分布
|
||||
dist_levels = levels.loc[district_schools]
|
||||
dim_levels = dist_levels[sub_dim]
|
||||
level_dist = {f"水平{i}": int((dim_levels == i).sum()) for i in range(1, 5)}
|
||||
|
||||
report["sub_dimensions"][sub_dim] = {
|
||||
"parent_dimension": dim,
|
||||
"score": round(score, 2),
|
||||
"district_avg": round(d_avg, 2),
|
||||
"diff_district": round(score - d_avg, 2),
|
||||
"rank_in_district": rank,
|
||||
"rank_in_city": rank_city,
|
||||
"level": level,
|
||||
"level_description": self.get_level_description(sub_dim, level),
|
||||
"level_distribution": level_dist,
|
||||
}
|
||||
|
||||
return report
|
||||
@@ -0,0 +1,664 @@
|
||||
"""
|
||||
数据溯源引擎 — 生成单校从原始数据到最终得分的完整计算链路
|
||||
|
||||
设计原则：
|
||||
- 不侵入现有PCA引擎，独立读取数据并重建中间过程
|
||||
- 输出JSON结构，前端R3F组件直接消费
|
||||
- 支持缓存（同一学校的trace数据不会频繁变化）
|
||||
"""
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Any
|
||||
from sklearn.decomposition import PCA
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from config_era2 import (
|
||||
PCA_MEAN, PCA_STD, DIMENSION_FRAMEWORK, LEVEL_THRESHOLDS,
|
||||
LEVEL_DESCRIPTIONS, SUBJECTS, SCHOOL_TYPE_MAP,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 缓存目录
|
||||
PROJECT_ROOT = Path(__file__).parent.parent.parent.parent
|
||||
TRACE_CACHE_DIR = PROJECT_ROOT / "output" / "trace_cache"
|
||||
|
||||
|
||||
class TraceEngine:
|
||||
"""数据溯源引擎：为单校生成完整的6阶段计算链路"""
|
||||
|
||||
def __init__(self, data_engine, pca_engine, stats_engine,
|
||||
sub_scores: pd.DataFrame, dim_scores: pd.DataFrame):
|
||||
self.de = data_engine
|
||||
self.pca = pca_engine
|
||||
self.stats = stats_engine
|
||||
self.sub_scores = sub_scores
|
||||
self.dim_scores = dim_scores
|
||||
|
||||
def compute_trace(self, school: str, district_schools: List[str],
|
||||
use_cache: bool = True) -> Dict:
|
||||
"""
|
||||
主入口：生成单校的完整计算链路
|
||||
|
||||
Returns:
|
||||
{school, school_info, stages: {raw_data, scoring, pca_detail, standardized, levels, dimensions, overall}}
|
||||
"""
|
||||
# 缓存
|
||||
if use_cache:
|
||||
cached = self._load_cache(school)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
logger.info(f"[Trace] 生成 {school} 的计算链路...")
|
||||
|
||||
all_schools = list(self.sub_scores.index)
|
||||
dist_schools = [s for s in district_schools if s in all_schools]
|
||||
|
||||
# 阶段0：原始数据概览
|
||||
stage_raw = self._trace_raw_data(school)
|
||||
|
||||
# 阶段1：赋分过程 + 阶段2: PCA细节
|
||||
stage_scoring, stage_pca = self._trace_scoring_and_pca(school, all_schools)
|
||||
|
||||
# 阶段3：标准化后得分
|
||||
stage_std = self._trace_standardized(school, dist_schools)
|
||||
|
||||
# 阶段4：水平判定
|
||||
stage_levels = self._trace_levels(school)
|
||||
|
||||
# 阶段5：维度聚合 → 总分
|
||||
stage_dims, stage_overall = self._trace_dimensions(school, dist_schools)
|
||||
|
||||
# 全市对比数据（用于标准化分布可视化）
|
||||
all_schools_overall = {}
|
||||
for s in dist_schools:
|
||||
if s in self.dim_scores.index and "总体得分" in self.dim_scores.columns:
|
||||
all_schools_overall[s] = round(float(self.dim_scores.loc[s, "总体得分"]), 2)
|
||||
|
||||
result = {
|
||||
"school": school,
|
||||
"school_info": SCHOOL_TYPE_MAP.get(school, {}),
|
||||
"district_school_count": len(dist_schools),
|
||||
"city_school_count": len(all_schools),
|
||||
"stages": {
|
||||
"raw_data": stage_raw,
|
||||
"scoring": stage_scoring,
|
||||
"pca_detail": stage_pca,
|
||||
"standardized": stage_std,
|
||||
"levels": stage_levels,
|
||||
"dimensions": stage_dims,
|
||||
"overall": stage_overall,
|
||||
},
|
||||
"all_schools_overall": all_schools_overall,
|
||||
}
|
||||
|
||||
self._save_cache(school, result)
|
||||
return result
|
||||
|
||||
# ====================================================================
|
||||
# 阶段0：原始数据概览
|
||||
# ====================================================================
|
||||
|
||||
def _trace_raw_data(self, school: str) -> Dict:
|
||||
"""展示该校的原始数据概况"""
|
||||
# B表数据
|
||||
course_df = self._get_course(school)
|
||||
b_fields = len(course_df)
|
||||
b_sample = []
|
||||
if len(course_df) > 0:
|
||||
sample_rows = course_df.head(12)
|
||||
for _, r in sample_rows.iterrows():
|
||||
b_sample.append({
|
||||
"name": str(r.get("字段名称", "")),
|
||||
"value": _safe_value(r.get("字段取值", "")),
|
||||
"type": _guess_type(r.get("字段取值", "")),
|
||||
})
|
||||
|
||||
# C表数据
|
||||
c_total = 0
|
||||
c_subjects = []
|
||||
for subj in SUBJECTS:
|
||||
sub_df = self._get_subject(school, subj)
|
||||
count = len(sub_df)
|
||||
c_total += count
|
||||
if count > 0:
|
||||
c_subjects.append({"subject": subj, "field_count": count})
|
||||
|
||||
c_sample = []
|
||||
# 取第一个有数据的学科的前几行
|
||||
for subj in SUBJECTS:
|
||||
sub_df = self._get_subject(school, subj)
|
||||
if len(sub_df) > 0:
|
||||
for _, r in sub_df.head(8).iterrows():
|
||||
c_sample.append({
|
||||
"subject": subj,
|
||||
"name": str(r.get("字段名称", "")),
|
||||
"value": _safe_value(r.get("字段取值", "")),
|
||||
"type": _guess_type(r.get("字段取值", "")),
|
||||
})
|
||||
break
|
||||
|
||||
return {
|
||||
"b_table": {
|
||||
"field_count": b_fields,
|
||||
"sample_fields": b_sample,
|
||||
},
|
||||
"c_table": {
|
||||
"field_count": c_total,
|
||||
"subject_count": len(c_subjects),
|
||||
"subjects": c_subjects,
|
||||
"sample_fields": c_sample,
|
||||
},
|
||||
"total_fields": b_fields + c_total,
|
||||
}
|
||||
|
||||
# ====================================================================
|
||||
# 阶段1 & 2：赋分 + PCA
|
||||
# ====================================================================
|
||||
|
||||
def _trace_scoring_and_pca(self, school: str, all_schools: List[str]) -> tuple:
|
||||
"""
|
||||
为每个子维度重建赋分过程和PCA细节。
|
||||
|
||||
策略：对每个子维度，分别计算该子维度的赋分矩阵，
|
||||
记录该校的具体输入值、赋分规则和PCA参数。
|
||||
"""
|
||||
scoring = {}
|
||||
pca_detail = {}
|
||||
|
||||
# 按维度框架遍历每个子维度
|
||||
sub_dim_methods = {
|
||||
"国家标准遵循": self._trace_national_standard,
|
||||
"课程结构建设": self._trace_course_structure,
|
||||
"课程规范落实": self._trace_school_level_generic,
|
||||
"教学方式变革": self._trace_subject_level_generic,
|
||||
"作业设计与管理变革": self._trace_subject_level_generic,
|
||||
"学科发展的个性化辅导": self._trace_subject_level_generic,
|
||||
"学生生涯发展指导": self._trace_school_level_generic,
|
||||
"培训支持": self._trace_subject_level_generic,
|
||||
"教研支持": self._trace_subject_level_generic,
|
||||
"项目支持": self._trace_subject_level_generic,
|
||||
"科学评价观": self._trace_subject_level_generic,
|
||||
"学业质量评估": self._trace_subject_level_generic,
|
||||
"综合素质评估": self._trace_school_level_generic,
|
||||
"实践活动评估": self._trace_subject_level_generic,
|
||||
"区域推进": self._trace_school_level_generic,
|
||||
"环境支持": self._trace_school_level_generic,
|
||||
"资源支持": self._trace_school_level_generic,
|
||||
"教学方式创新": self._trace_subject_level_generic,
|
||||
"评价精准化与个性化": self._trace_school_level_generic,
|
||||
"课程迭代优化": self._trace_school_level_generic,
|
||||
}
|
||||
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
for sub_dim in info["sub_dimensions"]:
|
||||
method = sub_dim_methods.get(sub_dim, self._trace_generic_fallback)
|
||||
try:
|
||||
s_info, p_info = method(sub_dim, school, all_schools)
|
||||
except Exception as e:
|
||||
logger.warning(f"[Trace] {sub_dim} trace failed: {e}")
|
||||
s_info = {"method": "unknown", "error": str(e)}
|
||||
p_info = {}
|
||||
|
||||
# 追加最终得分
|
||||
final_score = float(self.sub_scores.loc[school, sub_dim]) if sub_dim in self.sub_scores.columns else None
|
||||
s_info["final_score"] = round(final_score, 2) if final_score is not None else None
|
||||
s_info["parent_dimension"] = dim_name
|
||||
|
||||
scoring[sub_dim] = s_info
|
||||
pca_detail[sub_dim] = p_info
|
||||
|
||||
return scoring, pca_detail
|
||||
|
||||
def _trace_national_standard(self, sub_dim: str, school: str,
|
||||
all_schools: List[str]) -> tuple:
|
||||
"""国家标准遵循的特殊trace"""
|
||||
BXIU_STD = {"语文": 8, "数学": 8, "英语": 6, "思想政治": 6, "历史": 4, "地理": 4,
|
||||
"物理": 6, "化学": 4, "生命科学": 4, "体育": 12, "技术": 6, "艺术": 6}
|
||||
MERGE_MAP = {
|
||||
"信息技术": "技术", "通用技术": "技术", "劳动技术": "技术",
|
||||
"音乐": "艺术", "美术": "艺术", "生物学": "生命科学", "体育与健康": "体育",
|
||||
}
|
||||
SPSS_12 = list(BXIU_STD.keys())
|
||||
|
||||
# 获取该校课时
|
||||
hours_df = self.pca._get_weekly_hours(school)
|
||||
merged = {s: {"必修": 0, "选必": 0, "选修": 0} for s in SPSS_12}
|
||||
if len(hours_df) > 0:
|
||||
for _, r in hours_df.iterrows():
|
||||
subj = r.get("学科", "")
|
||||
mapped = MERGE_MAP.get(subj, subj)
|
||||
if mapped not in merged:
|
||||
continue
|
||||
field = r["字段名称"]
|
||||
val = r["字段取值"]
|
||||
if pd.isna(val):
|
||||
continue
|
||||
if "必修课周课时" in field and "选择性" not in field:
|
||||
merged[mapped]["必修"] += val
|
||||
elif "选择性必修" in field:
|
||||
merged[mapped]["选必"] += val
|
||||
elif "选修课周课时" in field:
|
||||
merged[mapped]["选修"] += val
|
||||
|
||||
# 必修分档评分
|
||||
inputs = []
|
||||
for s in SPSS_12:
|
||||
actual = merged[s]["必修"]
|
||||
std = BXIU_STD[s]
|
||||
if actual == 0:
|
||||
score = 0.0
|
||||
rule = "低于标准->0"
|
||||
elif abs(actual - std) <= 1.0:
|
||||
score = 2.0
|
||||
rule = "一致->2"
|
||||
elif actual > std:
|
||||
score = 1.0
|
||||
rule = "高于标准->1"
|
||||
else:
|
||||
score = 0.0
|
||||
rule = "低于标准->0"
|
||||
inputs.append({
|
||||
"name": f"{s}必修课时",
|
||||
"raw": actual,
|
||||
"standard": std,
|
||||
"score": score,
|
||||
"rule": rule,
|
||||
})
|
||||
|
||||
total_xb = sum(merged[s]["选必"] for s in SPSS_12)
|
||||
total_xx = sum(merged[s]["选修"] for s in SPSS_12)
|
||||
|
||||
scoring_info = {
|
||||
"method": "PCA(12学科必修分档) + Z(选必达标) + Z(选修达标) -> 均值",
|
||||
"inputs": inputs,
|
||||
"sub_factors": [
|
||||
{"name": "必修PCA", "input_count": 12, "type": "PCA"},
|
||||
{"name": "选必达标", "raw": total_xb, "threshold": 42,
|
||||
"met": total_xb >= 42, "type": "Z-score"},
|
||||
{"name": "选修达标", "raw": total_xx, "threshold": 14,
|
||||
"met": total_xx >= 14, "type": "Z-score"},
|
||||
],
|
||||
}
|
||||
|
||||
# PCA细节：构建全市必修矩阵
|
||||
bx_rows = {}
|
||||
for s in all_schools:
|
||||
h_df = self.pca._get_weekly_hours(s)
|
||||
m = {subj: {"必修": 0} for subj in SPSS_12}
|
||||
if len(h_df) > 0:
|
||||
for _, r in h_df.iterrows():
|
||||
subj = r.get("学科", "")
|
||||
mapped = MERGE_MAP.get(subj, subj)
|
||||
if mapped not in m:
|
||||
continue
|
||||
field = r["字段名称"]
|
||||
val = r["字段取值"]
|
||||
if pd.isna(val):
|
||||
continue
|
||||
if "必修课周课时" in field and "选择性" not in field:
|
||||
m[mapped]["必修"] += val
|
||||
row = {}
|
||||
for subj in SPSS_12:
|
||||
actual = m[subj]["必修"]
|
||||
std_val = BXIU_STD[subj]
|
||||
if actual == 0:
|
||||
row[subj] = 0.0
|
||||
elif abs(actual - std_val) <= 1.0:
|
||||
row[subj] = 2.0
|
||||
elif actual > std_val:
|
||||
row[subj] = 1.0
|
||||
else:
|
||||
row[subj] = 0.0
|
||||
bx_rows[s] = row
|
||||
|
||||
matrix = pd.DataFrame(bx_rows).T
|
||||
pca_info = self._extract_pca_details(matrix, school, "必修课PCA")
|
||||
|
||||
return scoring_info, pca_info
|
||||
|
||||
def _trace_course_structure(self, sub_dim: str, school: str,
|
||||
all_schools: List[str]) -> tuple:
|
||||
"""课程结构建设的trace（3组PCA）"""
|
||||
scoring_info = {
|
||||
"method": "PCA(学科课程结构) + PCA(校本特色) + PCA(综合实践) -> 均值",
|
||||
"sub_factors": [
|
||||
{"name": "学科类课程结构PCA", "type": "PCA",
|
||||
"description": "必修/选必/选修课时比例偏离度"},
|
||||
{"name": "校本特色课程PCA", "type": "PCA",
|
||||
"description": "选修课数量+时长"},
|
||||
{"name": "综合实践PCA", "type": "PCA",
|
||||
"description": "党团次数+社考个数+志愿时长+劳动"},
|
||||
],
|
||||
}
|
||||
# 简化的PCA info
|
||||
pca_info = {
|
||||
"type": "multi_factor",
|
||||
"factor_count": 3,
|
||||
"school_count": len(all_schools),
|
||||
"note": "三组因子各自做PCA后取均值",
|
||||
}
|
||||
return scoring_info, pca_info
|
||||
|
||||
def _trace_subject_level_generic(self, sub_dim: str, school: str,
|
||||
all_schools: List[str]) -> tuple:
|
||||
"""C表学科级子维度的通用trace"""
|
||||
# 获取该校在该子维度的最终得分
|
||||
scoring_info = {
|
||||
"method": "学科级赋分 -> 全市Z标准化 -> PCA(第一主成分) -> x10+50 -> 学科均值 -> 学校均值",
|
||||
"data_source": "C表（学科课程实施情况表）",
|
||||
"subject_count": len(SUBJECTS),
|
||||
}
|
||||
|
||||
# 尝试获取该学科的一些原始数据作为示例
|
||||
sample_inputs = []
|
||||
for subj in SUBJECTS[:3]: # 取前3个学科作示例
|
||||
sub_df = self._get_subject(school, subj)
|
||||
if len(sub_df) > 0:
|
||||
for _, r in sub_df.head(3).iterrows():
|
||||
sample_inputs.append({
|
||||
"subject": subj,
|
||||
"name": str(r.get("字段名称", "")),
|
||||
"value": _safe_value(r.get("字段取值", "")),
|
||||
})
|
||||
scoring_info["sample_inputs"] = sample_inputs
|
||||
|
||||
pca_info = {
|
||||
"type": "subject_level",
|
||||
"school_count": len(all_schools),
|
||||
"subject_count": len(SUBJECTS),
|
||||
"pipeline": "题目赋分 -> Z标准化 -> PCA -> x10+50",
|
||||
}
|
||||
return scoring_info, pca_info
|
||||
|
||||
def _trace_school_level_generic(self, sub_dim: str, school: str,
|
||||
all_schools: List[str]) -> tuple:
|
||||
"""B表学校级子维度的通用trace"""
|
||||
scoring_info = {
|
||||
"method": "学校级赋分 -> 全市Z标准化 -> PCA(第一主成分) -> x10+50",
|
||||
"data_source": "B表（课程实施情况表）",
|
||||
}
|
||||
|
||||
# 取一些原始数据作示例
|
||||
course_df = self._get_course(school)
|
||||
sample_inputs = []
|
||||
if len(course_df) > 0:
|
||||
for _, r in course_df.head(6).iterrows():
|
||||
sample_inputs.append({
|
||||
"name": str(r.get("字段名称", "")),
|
||||
"value": _safe_value(r.get("字段取值", "")),
|
||||
})
|
||||
scoring_info["sample_inputs"] = sample_inputs
|
||||
|
||||
pca_info = {
|
||||
"type": "school_level",
|
||||
"school_count": len(all_schools),
|
||||
"pipeline": "赋分 -> Z标准化 -> PCA -> x10+50",
|
||||
}
|
||||
return scoring_info, pca_info
|
||||
|
||||
def _trace_generic_fallback(self, sub_dim: str, school: str,
|
||||
all_schools: List[str]) -> tuple:
|
||||
"""回退方法"""
|
||||
return {"method": "unknown"}, {}
|
||||
|
||||
# ====================================================================
|
||||
# PCA细节提取
|
||||
# ====================================================================
|
||||
|
||||
def _extract_pca_details(self, matrix: pd.DataFrame, school: str,
|
||||
label: str = "PCA") -> Dict:
|
||||
"""从赋分矩阵中提取PCA的详细参数"""
|
||||
matrix = matrix.dropna(axis=1, how="all")
|
||||
if matrix.shape[1] == 0:
|
||||
return {"label": label, "error": "empty_matrix"}
|
||||
|
||||
filled = matrix.copy().infer_objects(copy=False)
|
||||
for col in filled.columns:
|
||||
col_mean = filled[col].mean()
|
||||
if np.isnan(col_mean):
|
||||
col_mean = 0.0
|
||||
filled[col] = filled[col].fillna(col_mean)
|
||||
|
||||
if filled.shape[1] == 1:
|
||||
vals = filled.iloc[:, 0].values.astype(float)
|
||||
school_idx = list(filled.index).index(school) if school in filled.index else -1
|
||||
return {
|
||||
"label": label,
|
||||
"type": "single_variable_z",
|
||||
"variable": str(filled.columns[0]),
|
||||
"school_value": round(float(vals[school_idx]), 4) if school_idx >= 0 else None,
|
||||
"mean": round(float(np.nanmean(vals)), 4),
|
||||
"std": round(float(np.nanstd(vals, ddof=1)), 4),
|
||||
"n_schools": len(vals),
|
||||
}
|
||||
|
||||
# Z标准化
|
||||
scaler = StandardScaler()
|
||||
try:
|
||||
scaled = scaler.fit_transform(filled.values.astype(float))
|
||||
except ValueError:
|
||||
return {"label": label, "error": "scaling_failed"}
|
||||
|
||||
# PCA
|
||||
pca = PCA(n_components=min(1, filled.shape[1], filled.shape[0]))
|
||||
scores = pca.fit_transform(scaled)[:, 0]
|
||||
loadings = pca.components_[0]
|
||||
|
||||
if np.sum(loadings) < 0:
|
||||
scores = -scores
|
||||
loadings = -loadings
|
||||
|
||||
mean = np.mean(scores)
|
||||
std = np.std(scores, ddof=1)
|
||||
|
||||
school_idx = list(filled.index).index(school) if school in filled.index else -1
|
||||
school_raw_score = scores[school_idx] if school_idx >= 0 else None
|
||||
school_std_score = ((school_raw_score - mean) / std * PCA_STD + PCA_MEAN) if (school_raw_score is not None and std > 0) else None
|
||||
|
||||
# Loadings详情
|
||||
loading_details = []
|
||||
for i, col in enumerate(filled.columns):
|
||||
loading_details.append({
|
||||
"variable": str(col),
|
||||
"loading": round(float(loadings[i]), 4),
|
||||
})
|
||||
loading_details.sort(key=lambda x: abs(x["loading"]), reverse=True)
|
||||
|
||||
return {
|
||||
"label": label,
|
||||
"type": "pca",
|
||||
"n_schools": int(filled.shape[0]),
|
||||
"n_variables": int(filled.shape[1]),
|
||||
"explained_variance_ratio": round(float(pca.explained_variance_ratio_[0]), 4),
|
||||
"loadings": loading_details,
|
||||
"school_pca_score": round(float(school_raw_score), 4) if school_raw_score is not None else None,
|
||||
"school_standardized": round(float(school_std_score), 2) if school_std_score is not None else None,
|
||||
"pca_mean": round(float(mean), 4),
|
||||
"pca_std": round(float(std), 4),
|
||||
}
|
||||
|
||||
# ====================================================================
|
||||
# 阶段3：标准化后得分
|
||||
# ====================================================================
|
||||
|
||||
def _trace_standardized(self, school: str, dist_schools: List[str]) -> Dict:
|
||||
"""标准化后的20个子维度得分"""
|
||||
result = {}
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
for sub_dim in info["sub_dimensions"]:
|
||||
if sub_dim not in self.sub_scores.columns:
|
||||
continue
|
||||
score = float(self.sub_scores.loc[school, sub_dim])
|
||||
# 区内均值和全市均值
|
||||
dist_vals = self.sub_scores.loc[
|
||||
[s for s in dist_schools if s in self.sub_scores.index], sub_dim
|
||||
]
|
||||
all_vals = self.sub_scores[sub_dim]
|
||||
|
||||
result[sub_dim] = {
|
||||
"score": round(score, 2),
|
||||
"district_avg": round(float(dist_vals.mean()), 2),
|
||||
"city_avg": round(float(all_vals.mean()), 2),
|
||||
"city_std": round(float(all_vals.std()), 2),
|
||||
"diff_district": round(score - float(dist_vals.mean()), 2),
|
||||
"diff_city": round(score - float(all_vals.mean()), 2),
|
||||
"parent_dimension": dim_name,
|
||||
}
|
||||
return result
|
||||
|
||||
# ====================================================================
|
||||
# 阶段4：水平判定
|
||||
# ====================================================================
|
||||
|
||||
def _trace_levels(self, school: str) -> Dict:
|
||||
result = {}
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
for sub_dim in info["sub_dimensions"]:
|
||||
if sub_dim not in self.sub_scores.columns:
|
||||
continue
|
||||
score = float(self.sub_scores.loc[school, sub_dim])
|
||||
thresholds = LEVEL_THRESHOLDS.get(sub_dim, {})
|
||||
level = self.stats.determine_level(score, sub_dim)
|
||||
desc = LEVEL_DESCRIPTIONS.get(sub_dim, {}).get(level, "")
|
||||
|
||||
result[sub_dim] = {
|
||||
"score": round(score, 2),
|
||||
"thresholds": {
|
||||
"level2": thresholds.get("level2", 43),
|
||||
"level3": thresholds.get("level3", 50),
|
||||
"level4": thresholds.get("level4", 57),
|
||||
},
|
||||
"level": level,
|
||||
"description": desc,
|
||||
"parent_dimension": dim_name,
|
||||
}
|
||||
return result
|
||||
|
||||
# ====================================================================
|
||||
# 阶段5：维度聚合 → 总分
|
||||
# ====================================================================
|
||||
|
||||
def _trace_dimensions(self, school: str, dist_schools: List[str]) -> tuple:
|
||||
dims = {}
|
||||
for dim_name, info in DIMENSION_FRAMEWORK.items():
|
||||
sub_dims = info["sub_dimensions"]
|
||||
sub_scores = {}
|
||||
for sd in sub_dims:
|
||||
if sd in self.sub_scores.columns:
|
||||
sub_scores[sd] = round(float(self.sub_scores.loc[school, sd]), 2)
|
||||
|
||||
dim_score = float(self.dim_scores.loc[school, dim_name]) if dim_name in self.dim_scores.columns else None
|
||||
dist_dim_vals = self.dim_scores.loc[
|
||||
[s for s in dist_schools if s in self.dim_scores.index], dim_name
|
||||
] if dim_name in self.dim_scores.columns else pd.Series()
|
||||
|
||||
dims[dim_name] = {
|
||||
"sub_scores": sub_scores,
|
||||
"score": round(dim_score, 2) if dim_score is not None else None,
|
||||
"method": "mean(子维度标准化分)",
|
||||
"district_avg": round(float(dist_dim_vals.mean()), 2) if len(dist_dim_vals) > 0 else None,
|
||||
}
|
||||
|
||||
# 总分
|
||||
overall_score = float(self.dim_scores.loc[school, "总体得分"]) if "总体得分" in self.dim_scores.columns else None
|
||||
dist_overall = self.dim_scores.loc[
|
||||
[s for s in dist_schools if s in self.dim_scores.index], "总体得分"
|
||||
] if "总体得分" in self.dim_scores.columns else pd.Series()
|
||||
|
||||
rank = int((dist_overall >= overall_score).sum()) if overall_score is not None and len(dist_overall) > 0 else None
|
||||
|
||||
overall = {
|
||||
"score": round(overall_score, 2) if overall_score is not None else None,
|
||||
"method": "mean(7个维度分)",
|
||||
"district_avg": round(float(dist_overall.mean()), 2) if len(dist_overall) > 0 else None,
|
||||
"rank": rank,
|
||||
"total_schools": len(dist_schools),
|
||||
}
|
||||
|
||||
return dims, overall
|
||||
|
||||
# ====================================================================
|
||||
# 数据访问代理
|
||||
# ====================================================================
|
||||
|
||||
def _get_course(self, school: str) -> pd.DataFrame:
|
||||
return self.pca._get_course(school)
|
||||
|
||||
def _get_subject(self, school: str, subject: str) -> pd.DataFrame:
|
||||
return self.pca._get_subject(school, subject)
|
||||
|
||||
# ====================================================================
|
||||
# 缓存
|
||||
# ====================================================================
|
||||
|
||||
def _load_cache(self, school: str) -> Optional[Dict]:
|
||||
TRACE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_file = TRACE_CACHE_DIR / f"{school}_trace.json"
|
||||
if cache_file.exists():
|
||||
try:
|
||||
return json.loads(cache_file.read_text("utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _save_cache(self, school: str, data: Dict):
|
||||
TRACE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_file = TRACE_CACHE_DIR / f"{school}_trace.json"
|
||||
try:
|
||||
cache_file.write_text(json.dumps(data, ensure_ascii=False, indent=2, default=_json_default), "utf-8")
|
||||
except Exception as e:
|
||||
logger.warning(f"[Trace] Cache write failed for {school}: {e}")
|
||||
|
||||
|
||||
# ====================================================================
|
||||
# 工具函数
|
||||
# ====================================================================
|
||||
|
||||
def _safe_value(v) -> Any:
|
||||
"""将pandas值转为JSON安全类型"""
|
||||
if pd.isna(v):
|
||||
return None
|
||||
if isinstance(v, (np.integer,)):
|
||||
return int(v)
|
||||
if isinstance(v, (np.floating,)):
|
||||
return round(float(v), 4)
|
||||
return str(v)
|
||||
|
||||
|
||||
def _guess_type(v) -> str:
|
||||
"""猜测字段类型"""
|
||||
if pd.isna(v):
|
||||
return "null"
|
||||
s = str(v).strip()
|
||||
try:
|
||||
float(s)
|
||||
return "number"
|
||||
except ValueError:
|
||||
pass
|
||||
if s in ("0", "1", "有", "无", "是", "否"):
|
||||
return "binary"
|
||||
if s in ("已经建成并使用", "已经建成但未使用", "尚未建成", "已建成并使用", "已建成但未使用"):
|
||||
return "ordinal"
|
||||
return "text"
|
||||
|
||||
|
||||
def _json_default(obj):
|
||||
"""JSON序列化兜底"""
|
||||
if isinstance(obj, (np.integer,)):
|
||||
return int(obj)
|
||||
if isinstance(obj, (np.floating,)):
|
||||
return round(float(obj), 4)
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tolist()
|
||||
if isinstance(obj, pd.Timestamp):
|
||||
return str(obj)
|
||||
if isinstance(obj, float) and (np.isnan(obj) or np.isinf(obj)):
|
||||
return None
|
||||
return str(obj)
|
||||
@@ -0,0 +1,199 @@
|
||||
# 二期（Era2）全市数据升级 — 关键点说明
|
||||
|
||||
> 更新日期：2026-03-18
|
||||
>
|
||||
> 本文档用于向领导解释：为什么需要接入全市数据、接入后能带来什么价值、技术上需要注意哪些关键点。
|
||||
|
||||
---
|
||||
|
||||
## 一、背景：当前方案的局限
|
||||
|
||||
二期报告生成系统目前使用的数据来自**原始平台导出**（17个Excel分学科文件，经ETL合并），覆盖 **13个区、176所学校**。但存在三个核心局限：
|
||||
|
||||
### 1. "区均值恒等于50分"问题
|
||||
|
||||
当前系统对每个区**独立做标准化**（PCA + Z-score → 均值50，标准差10），导致：
|
||||
|
||||
- 任何一个区的"区均值"永远精确等于**50.00分**
|
||||
- 校长看到的"高于区均值X分"只能反映**区内相对位置**，无法回答"我们区在全市什么水平？"
|
||||
- 强区和弱区的50分含金量完全不同，但报告无法体现
|
||||
|
||||
### 2. 缺失学校类型信息
|
||||
|
||||
原始导出数据中**没有**学校类型字段（市实验性示范性高中、区实验性示范性高中、公办普通高中、民办高中、特色高中等），导致：
|
||||
|
||||
- 报告中"同类学校均值"被迫等于区均值，丧失了对标参考价值
|
||||
- LLM分析文字无法给出针对性的定位建议（如"作为市实验性示范性高中，应在全市发挥引领作用"）
|
||||
- 缺少A/B/C/D/T类学校的分类对比
|
||||
|
||||
### 3. 缺失3个区的数据
|
||||
|
||||
原始导出覆盖13个区、176所学校，缺少**嘉定区（11校）、普陀区（13校）、浦东新区（57校）**的数据，合计缺失约81所学校。
|
||||
|
||||
---
|
||||
|
||||
## 二、全市数据带来的价值
|
||||
|
||||
邱老师提供的全市数据（3个Excel文件）覆盖 **16个区、266所学校**，与当前数据相比：
|
||||
|
||||
### 价值1：全市统一基准线 ⭐⭐⭐
|
||||
|
||||
用全市266所学校做标准化基准后：
|
||||
|
||||
| 对比项 | 当前方案 | 升级后 |
|
||||
|--------|----------|--------|
|
||||
| 标准化基准 | 区内自己 → 区均值恒=50 | 全市266校 → 区均值有差异 |
|
||||
| 杨浦区均值 | 50.00（数学定义） | 可能 52-55（反映杨浦在全市偏上） |
|
||||
| 报告价值 | 只能看区内排名 | 能看到全市坐标、区域定位 |
|
||||
| 校长体感 | "高于均值8分"——均值是什么？ | "高于全市均值8分"——清晰有力 |
|
||||
|
||||
### 价值2：学校类型对标 ⭐⭐⭐
|
||||
|
||||
全市数据自带每所学校的完整元数据：
|
||||
|
||||
| 字段 | 示例 | 报告中的用途 |
|
||||
|------|------|-------------|
|
||||
| 学校类型 | 市实验性示范性高中 | "同类学校均值"计算、LLM定位建议 |
|
||||
| 学校类型编号 | A类/B类/C类/D类/T类 | 五类学校分组对比 |
|
||||
| 学校性质 | 公办/民办 | 办学性质维度分析 |
|
||||
| 所处地区 | 城区/城市郊区 | 城郊差异分析 |
|
||||
| 学校特色 | 科技类/艺术类/人文类 | 特色学校的个性化建议 |
|
||||
|
||||
**升级后的报告示例**：
|
||||
> 贵校课程实施总体得分58.75分，高于**杨浦区均值（53.2分）** 5.55分，高于**全市同类学校（市实验性示范性高中，74所）均值（54.8分）** 3.95分，在杨浦区14所学校中排名第1位。
|
||||
|
||||
### 价值3：覆盖完整16区 ⭐⭐
|
||||
|
||||
| 对比项 | 当前方案 | 升级后 |
|
||||
|--------|----------|--------|
|
||||
| 覆盖区域 | 13个区 | **16个区**（+嘉定、普陀、浦东新区） |
|
||||
| 学校数量 | 176所 | **266所** |
|
||||
| 浦东新区 | ❌ 缺失 | ✅ 57所（最大区） |
|
||||
|
||||
### 价值4：恢复被削弱的维度 ⭐⭐
|
||||
|
||||
全市数据比当前parquet **多出102个字段**，其中包括之前因原始导出格式问题被标记为"二期删除"的关键字段：
|
||||
|
||||
| 恢复的字段 | 归属维度 | 影响 |
|
||||
|-----------|---------|------|
|
||||
| `信息化平台功能_*`（18个） | 环境支持 | 该子维度从"降级评分"恢复为完整评分 |
|
||||
| `管理业务的信息化应用_*`（2个） | 课程迭代优化 | 该子维度从仅1个评分项恢复到3+个 |
|
||||
| `教学业务的信息化应用_*`（4个） | 课程迭代优化 | 同上 |
|
||||
| `已完成的网络课程门数`（1个） | 课程迭代优化 | 同上 |
|
||||
| 德育、劳动周、选课走班等（约75个） | 多个维度 | 增加评分区分度 |
|
||||
|
||||
> ⚠️ 唯一仍缺失的字段：`信息技术与教学融合的认识_*`（教学方式创新维度），已用`教学信息化应用程度_*`替代，不受影响。
|
||||
|
||||
---
|
||||
|
||||
## 三、技术实施方案
|
||||
|
||||
### 架构决策：在现有era2代码上扩展，不复制新分支
|
||||
|
||||
**原因**：赋分引擎、渲染引擎、模板、LLM引擎完全复用，只改数据加载层和统计基准层。复制会导致两套几乎相同的代码，后续维护成本翻倍。
|
||||
|
||||
### 改动范围
|
||||
|
||||
| 文件 | 改动内容 | 影响 |
|
||||
|------|---------|------|
|
||||
| `08_etl_city_data.py` | **新建** — 将全市3个xlsx转为parquet | 一次性ETL |
|
||||
| `config_era2.py` | 新增全市数据路径、自动构建`SCHOOL_TYPE_MAP`（266校） | 配置扩展 |
|
||||
| `data_engine_era2.py` | 支持加载全市parquet；提供全市基准数据接口 | 接口扩展 |
|
||||
| `stats_engine_era2.py` | 标准化基准改为全市；同类学校均值从全市同类型算 | 核心逻辑变化 |
|
||||
| `04_generate_report.py` | 新增`--city-baseline`参数（默认开启） | 参数扩展 |
|
||||
| `05_batch_generate.py` | 同上 | 参数扩展 |
|
||||
|
||||
赋分引擎、渲染引擎、LLM引擎、模板 → **零改动**。
|
||||
|
||||
---
|
||||
|
||||
## 四、需要注意的关键点
|
||||
|
||||
### 关键点1：学校名称映射
|
||||
|
||||
全市数据使用**简称**（如"同济一附"），当前系统使用**全称**（如"同济大学第一附属中学"）。
|
||||
|
||||
| 来源 | 杨浦区示例 |
|
||||
|------|-----------|
|
||||
| 全市数据 | 同济一附、复旦附中、交大附中、控江中学 |
|
||||
| 当前系统 | 同济大学第一附属中学、复旦大学附属中学… |
|
||||
|
||||
**处理方案**：
|
||||
- 统一使用全市数据的**简称**作为系统内部标识
|
||||
- 报告标题使用全称（从A表或映射表获取），正文用简称
|
||||
- 需构建266所学校的 简称↔全称 映射表
|
||||
|
||||
### 关键点2：学校数量差异
|
||||
|
||||
| 数据源 | 学校数 | 说明 |
|
||||
|--------|:------:|------|
|
||||
| A表（基础信息） | 270 | 含4所无课程数据的学校 |
|
||||
| B表（课程实施） | 266 | 以此为准 |
|
||||
| C表（学科课程） | 待确认 | 理论上应与B表一致 |
|
||||
| 当前parquet | 176 | 原始导出不完整 |
|
||||
|
||||
差异原因：当前parquet来自原始平台导出（17个分学科文件），部分区的数据未被导出。全市数据是整合后的完整版。
|
||||
|
||||
> **建议**：以全市B表的266所学校为准。A表多出的4所（宝山世外、协和高中、金瑞学校、青浦协和）无课程数据，不参与赋分。
|
||||
|
||||
### 关键点3：字段差异处理
|
||||
|
||||
| 类别 | 数量 | 说明 |
|
||||
|------|:----:|------|
|
||||
| 全市与parquet共有字段 | 331 | 赋分引擎已适配 |
|
||||
| 全市多出的字段 | 102 | 其中约25个可提升赋分精度，其余为新增题目 |
|
||||
| parquet独有的字段 | 5 | 其中`nan`为脏数据，其余4个为极少数学校的特殊选项 |
|
||||
|
||||
**原则**：赋分引擎基于字段名称匹配，多出的字段不会报错（被忽略），但可以主动利用恢复的字段提升评分精度。
|
||||
|
||||
### 关键点4：B表中的表头行污染
|
||||
|
||||
全市B表的`所在区`列中混有一行值为`district`的表头行，加载时需过滤：
|
||||
```python
|
||||
df = df[df['所在区'] != 'district']
|
||||
```
|
||||
|
||||
### 关键点5：标准化基准变化的影响
|
||||
|
||||
使用全市基准后，**所有已生成的报告得分都会变化**：
|
||||
|
||||
| 影响 | 说明 |
|
||||
|------|------|
|
||||
| 分数值变化 | 同一学校的得分可能从58.75→55.2或→62.3（取决于全市分布） |
|
||||
| 排名逻辑不变 | 区内排名仍按区内学校比较 |
|
||||
| 水平划分 | 阈值不变（仍用赋分整理表中的固定阈值），但学校可能跨水平 |
|
||||
| LLM缓存 | 所有旧缓存失效，需重新生成（约50秒/校） |
|
||||
|
||||
> **建议**：全市基准上线后，需一次性重新生成所有已交付区域的报告。
|
||||
|
||||
### 关键点6：C表文件较大
|
||||
|
||||
`C全市数据_学科课程实施情况表_sh_sub7(全)(1).xlsx` 约44MB，包含全市15个学科×266校的数据。ETL转parquet后约10-15MB，后续读取速度不受影响。
|
||||
|
||||
---
|
||||
|
||||
## 五、实施步骤（预估）
|
||||
|
||||
| 步骤 | 工作量 | 说明 |
|
||||
|------|--------|------|
|
||||
| 1. 全市数据ETL（xlsx→parquet） | 30分钟 | 新建脚本，处理名称映射和表头清洗 |
|
||||
| 2. 构建全量SCHOOL_TYPE_MAP | 15分钟 | 从A/B表提取266校元数据 |
|
||||
| 3. 改造data_engine支持全市数据 | 30分钟 | 加载全市parquet，提供基准接口 |
|
||||
| 4. 改造stats_engine标准化逻辑 | 45分钟 | 全市基准 + 同类学校均值 |
|
||||
| 5. 恢复C类削弱字段的赋分 | 30分钟 | 环境支持、课程迭代优化维度 |
|
||||
| 6. 测试验证 | 30分钟 | 对比前后得分变化，确认逻辑正确 |
|
||||
| 7. 重新生成已交付区域报告 | ~60分钟 | 杨浦14校+宝山19校+长宁9校 |
|
||||
|
||||
**合计：约 3-4 小时**
|
||||
|
||||
---
|
||||
|
||||
## 六、预期效果对比
|
||||
|
||||
### 报告第一段（Before）
|
||||
> 贵校课程实施总体得分为58.75分，高于**长宁区**均值（**50.00分**）8.75分，同时高于同类学校均值（**50.00分**）8.75分…
|
||||
|
||||
### 报告第一段（After）
|
||||
> 贵校课程实施总体得分为XX.XX分，高于**杨浦区**均值（**XX.XX分**）X.XX分，高于**全市同类学校（市实验性示范性高中，74所）** 均值（**XX.XX分**）X.XX分。在杨浦区14所学校中排名第1位，在全市同类学校中排名第X位…
|
||||
|
||||
信息密度和说服力显著提升。
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/bash
|
||||
# 批量生成7所学校报告
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
schools=(
|
||||
"同济一附:杨浦区"
|
||||
"上师二附:金山区"
|
||||
"嘉一实高:嘉定区"
|
||||
"上师闵分:闵行区"
|
||||
"上师大附中:浦东新区"
|
||||
"交大附中:杨浦区"
|
||||
"市西中学:静安区"
|
||||
)
|
||||
|
||||
echo "=============================="
|
||||
echo "批量生成 ${#schools[@]} 所学校报告"
|
||||
echo "=============================="
|
||||
|
||||
for item in "${schools[@]}"; do
|
||||
school="${item%%:*}"
|
||||
district="${item##*:}"
|
||||
echo ""
|
||||
echo ">>> 开始生成: ${school} (${district})"
|
||||
python3 04_generate_report.py --school "$school" --district "$district" --enable-llm 2>&1 | grep -E "✅|❌|总体得分|LLM段落|耗时|生成完成|ERROR|失败"
|
||||
echo "<<< 完成: ${school}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=============================="
|
||||
echo "全部完成!"
|
||||
echo "=============================="
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
报告本地服务器 — 自带 CORS 代理
|
||||
|
||||
功能：
|
||||
1. 以 HTTP 方式提供 output/era2/ 下的报告 HTML（解决 file:// CORS 问题）
|
||||
2. /proxy/chat/completions → 转发到真实 LLM API（支持流式 SSE）
|
||||
|
||||
用法：
|
||||
python3 serve_report.py # 默认端口 9380
|
||||
python3 serve_report.py --port 8080 # 指定端口
|
||||
|
||||
然后浏览器打开 http://localhost:9380/复旦大学附属中学_报告.html
|
||||
"""
|
||||
import argparse
|
||||
import http.server
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import webbrowser
|
||||
from pathlib import Path
|
||||
from functools import partial
|
||||
|
||||
# 自动定位 output/era2 目录
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
OUTPUT_DIR = SCRIPT_DIR.parent.parent / "output" / "era2"
|
||||
|
||||
# LLM API 配置 — 统一从 backend/app/config.py 读取（唯一真相源）
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
from config_era2 import * # noqa: F401,F403 era2 本地配置（路径/学校映射等）
|
||||
|
||||
_BACKEND_PATH = str(SCRIPT_DIR.parent.parent / "backend")
|
||||
if _BACKEND_PATH not in sys.path:
|
||||
sys.path.insert(0, _BACKEND_PATH)
|
||||
from app.config import LLM_BASE_URL, LLM_API_KEY # noqa: E402
|
||||
|
||||
|
||||
class ReportHandler(http.server.SimpleHTTPRequestHandler):
|
||||
"""扩展 SimpleHTTPRequestHandler，增加 CORS 代理路由"""
|
||||
|
||||
def __init__(self, *args, llm_base_url=None, llm_api_key=None, **kwargs):
|
||||
self.llm_base_url = llm_base_url or LLM_BASE_URL
|
||||
self.llm_api_key = llm_api_key or LLM_API_KEY
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def end_headers(self):
|
||||
"""所有响应都加 CORS 头"""
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization")
|
||||
super().end_headers()
|
||||
|
||||
def do_OPTIONS(self):
|
||||
"""处理 CORS 预检请求"""
|
||||
self.send_response(204)
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
"""代理 POST 请求到 LLM API"""
|
||||
if self.path == "/proxy/chat/completions":
|
||||
self._proxy_chat()
|
||||
else:
|
||||
self.send_error(404, "Not Found")
|
||||
|
||||
def _proxy_chat(self):
|
||||
"""转发聊天请求到 LLM API，支持流式 SSE"""
|
||||
try:
|
||||
# 读取请求体
|
||||
content_length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(content_length)
|
||||
|
||||
# 构造转发请求
|
||||
api_url = self.llm_base_url.rstrip("/") + "/chat/completions"
|
||||
req = urllib.request.Request(
|
||||
api_url,
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.llm_api_key}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
# 检查是否为流式请求
|
||||
try:
|
||||
req_json = json.loads(body)
|
||||
is_stream = req_json.get("stream", False)
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
is_stream = False
|
||||
|
||||
# 转发请求
|
||||
resp = urllib.request.urlopen(req, timeout=120)
|
||||
|
||||
# 发送响应头
|
||||
self.send_response(resp.status)
|
||||
# 传递关键响应头
|
||||
for header in ["Content-Type"]:
|
||||
val = resp.getheader(header)
|
||||
if val:
|
||||
self.send_header(header, val)
|
||||
if is_stream:
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("X-Accel-Buffering", "no")
|
||||
self.end_headers()
|
||||
|
||||
# 流式转发
|
||||
if is_stream:
|
||||
while True:
|
||||
chunk = resp.read(1024)
|
||||
if not chunk:
|
||||
break
|
||||
self.wfile.write(chunk)
|
||||
self.wfile.flush()
|
||||
else:
|
||||
self.wfile.write(resp.read())
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
error_body = e.read().decode("utf-8", errors="replace")
|
||||
self.send_response(e.code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({
|
||||
"error": {"message": f"LLM API error: {e.code}", "detail": error_body}
|
||||
}).encode())
|
||||
|
||||
except Exception as e:
|
||||
self.send_response(502)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({
|
||||
"error": {"message": f"Proxy error: {str(e)}"}
|
||||
}).encode())
|
||||
|
||||
def log_message(self, format, *args):
|
||||
"""美化日志"""
|
||||
msg = format % args
|
||||
if "/proxy/" in msg:
|
||||
sys.stderr.write(f" 🔄 PROXY {msg}\n")
|
||||
elif ".html" in msg:
|
||||
sys.stderr.write(f" 📄 {msg}\n")
|
||||
# 静默其他请求（JS/CSS/图片等）
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="报告本地服务器（带CORS代理）")
|
||||
parser.add_argument("--port", type=int, default=9380, help="端口号（默认9380）")
|
||||
parser.add_argument("--no-open", action="store_true", help="不自动打开浏览器")
|
||||
parser.add_argument("--dir", type=str, default=str(OUTPUT_DIR),
|
||||
help=f"报告目录（默认 {OUTPUT_DIR}）")
|
||||
args = parser.parse_args()
|
||||
|
||||
serve_dir = Path(args.dir)
|
||||
if not serve_dir.exists():
|
||||
print(f"❌ 目录不存在: {serve_dir}")
|
||||
sys.exit(1)
|
||||
|
||||
# 列出可用报告
|
||||
reports = sorted(serve_dir.glob("*_报告.html"))
|
||||
|
||||
print(f"{'=' * 60}")
|
||||
print(f"🌐 报告本地服务器")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" 目录: {serve_dir}")
|
||||
print(f" 地址: http://localhost:{args.port}")
|
||||
print(f" 代理: /proxy/chat/completions → {LLM_BASE_URL}")
|
||||
print(f" 报告: {len(reports)} 份")
|
||||
for r in reports:
|
||||
url = f"http://localhost:{args.port}/{r.name}"
|
||||
print(f" 📊 {url}")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" 按 Ctrl+C 停止\n")
|
||||
|
||||
# 创建 handler，绑定到报告目录
|
||||
handler = partial(
|
||||
ReportHandler,
|
||||
directory=str(serve_dir),
|
||||
llm_base_url=LLM_BASE_URL,
|
||||
llm_api_key=LLM_API_KEY,
|
||||
)
|
||||
|
||||
server = http.server.HTTPServer(("0.0.0.0", args.port), handler)
|
||||
|
||||
# 自动打开第一份报告
|
||||
if not args.no_open and reports:
|
||||
url = f"http://localhost:{args.port}/{reports[0].name}"
|
||||
threading.Timer(0.5, lambda: webbrowser.open(url)).start()
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n👋 服务器已停止")
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user