report-admin/scripts/era2/08_etl_city_data.py

#!/usr/bin/env python3
"""
全市数据ETL：将全市3个xlsx转为parquet
输入：二期/A全市数据_基础信息表_sh_basic2.xlsx
      二期/B全市数据_学校课程实施情况表_sh_sch6.xlsx
      二期/C全市数据_学科课程实施情况表_sh_sub7(全)(1).xlsx
输出：data/era2/city_基础信息表.parquet
      data/era2/city_课程实施情况表.parquet
      data/era2/city_学科课程实施情况表.parquet
      data/era2/city_school_meta.parquet  (学校元数据：类型、性质等)
"""
import pandas as pd
import json
import time
import logging
from pathlib import Path

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S")
logger = logging.getLogger(__name__)

PROJECT_ROOT = Path(__file__).parent.parent.parent  # report-admin/
RAW_DIR = PROJECT_ROOT.parent / "二期"
OUTPUT_DIR = PROJECT_ROOT / "data" / "era2"

# 输入文件
FILE_A = RAW_DIR / "A全市数据_基础信息表_sh_basic2.xlsx"
FILE_B = RAW_DIR / "B全市数据_学校课程实施情况表_sh_sch6.xlsx"
FILE_C = RAW_DIR / "C全市数据_学科课程实施情况表_sh_sub7(全)(1).xlsx"


def etl_basic_info():
    """A表：基础信息表"""
    logger.info("加载A表（基础信息）...")
    df = pd.read_excel(FILE_A)
    logger.info(f"  原始行数: {len(df)}, 学校数: {df['s_name'].nunique()}")

    # 列名映射：与当前parquet格式对齐
    # A表列: item_no, item_type, item_id, alternative, field_id, field_name, field_value,
    #         grade, sem, district, s_name, ownership, category, level, area, tese, levelb, levelc
    # 当前parquet列: 字段名称, 字段值, 学校名称, 区, ...
    df = df.rename(columns={
        "s_name": "学校名称",
        "district": "区",
        "field_name": "字段名称",
        "field_value": "字段值",
        "ownership": "学校性质",
        "category": "学校类别",
        "level": "学校类型_原始",
        "area": "所处地区",
        "tese": "学校特色",
        "levelb": "学校类型",
        "levelc": "学校类型编号",
        "grade": "年级",
        "sem": "学期",
    })

    out = OUTPUT_DIR / "city_基础信息表.parquet"
    df.to_parquet(out, index=False)
    logger.info(f"  → {out} ({len(df)}行, {df['学校名称'].nunique()}校)")
    return df


def etl_course_impl():
    """B表：课程实施情况表"""
    logger.info("加载B表（课程实施）...")
    df = pd.read_excel(FILE_B)
    logger.info(f"  原始行数: {len(df)}")

    # 过滤表头行
    df = df[df["所在区"] != "district"].copy()
    logger.info(f"  过滤后行数: {len(df)}, 学校数: {df['学校简称'].nunique()}")

    # 列名已与parquet一致，无需改
    out = OUTPUT_DIR / "city_课程实施情况表.parquet"
    df.to_parquet(out, index=False)
    logger.info(f"  → {out} ({len(df)}行, {df['学校简称'].nunique()}校)")
    return df


def etl_subject_impl():
    """C表：学科课程实施情况表"""
    logger.info("加载C表（学科课程，约44MB，需要30-60秒）...")
    df = pd.read_excel(FILE_C)
    logger.info(f"  原始行数: {len(df)}")

    # 过滤表头行
    df = df[df["所在区"] != "district"].copy()
    logger.info(f"  过滤后行数: {len(df)}, 学校数: {df['学校简称'].nunique()}")

    out = OUTPUT_DIR / "city_学科课程实施情况表.parquet"
    df.to_parquet(out, index=False)
    logger.info(f"  → {out} ({len(df)}行, {df['学校简称'].nunique()}校)")
    return df


def build_school_meta(df_b: pd.DataFrame):
    """从B表提取学校元数据（类型、性质等）"""
    logger.info("构建学校元数据表...")
    meta = df_b.drop_duplicates("学校简称")[
        ["学校简称", "所在区", "学校类别", "学校性质", "所处地区", "学校类型", "学校类型编号"]
    ].copy()
    meta = meta.sort_values(["所在区", "学校简称"]).reset_index(drop=True)

    # 清理学校类型编号中的换行符
    meta["学校类型编号"] = meta["学校类型编号"].str.replace(r"\n", "", regex=True)

    out = OUTPUT_DIR / "city_school_meta.parquet"
    meta.to_parquet(out, index=False)
    logger.info(f"  → {out} ({len(meta)}校)")

    # 同时输出JSON便于查看
    meta_json = OUTPUT_DIR / "city_school_meta.json"
    meta.to_json(meta_json, orient="records", force_ascii=False, indent=2)
    logger.info(f"  → {meta_json}")

    # 统计
    print(f"\n{'='*60}")
    print(f"学校元数据统计")
    print(f"{'='*60}")
    print(f"总学校数: {len(meta)}")
    print(f"区域数: {meta['所在区'].nunique()}")
    print(f"\n各区学校数:")
    for d in sorted(meta["所在区"].unique()):
        n = len(meta[meta["所在区"] == d])
        print(f"  {d}: {n}所")
    print(f"\n学校类型分布:")
    print(meta["学校类型"].value_counts().to_string())
    print(f"\n学校性质分布:")
    print(meta["学校性质"].value_counts().to_string())

    return meta


def main():
    total_start = time.time()
    print("=" * 60)
    print("📊 全市数据ETL — xlsx → parquet")
    print("=" * 60)

    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

    # A表
    t0 = time.time()
    df_a = etl_basic_info()
    logger.info(f"  A表耗时: {time.time()-t0:.1f}s")

    # B表
    t0 = time.time()
    df_b = etl_course_impl()
    logger.info(f"  B表耗时: {time.time()-t0:.1f}s")

    # C表
    t0 = time.time()
    df_c = etl_subject_impl()
    logger.info(f"  C表耗时: {time.time()-t0:.1f}s")

    # 学校元数据
    meta = build_school_meta(df_b)

    total = time.time() - total_start
    print(f"\n{'='*60}")
    print(f"✅ ETL完成! 总耗时: {total:.1f}s")
    print(f"  A表: {len(df_a)}行 → city_基础信息表.parquet")
    print(f"  B表: {len(df_b)}行 → city_课程实施情况表.parquet")
    print(f"  C表: {len(df_c)}行 → city_学科课程实施情况表.parquet")
    print(f"  元数据: {len(meta)}校 → city_school_meta.parquet/json")
    print(f"{'='*60}")


if __name__ == "__main__":
    main()