#!/usr/bin/env python3
"""
二期数据验证脚本
检查 ETL 产出的 parquet 文件是否符合预期，
并与一期数据做结构和字段ID的对比
"""
import pandas as pd
import json
from pathlib import Path

PROJECT_ROOT = Path(__file__).parent.parent.parent  # report-admin/
ERA1_DATA = PROJECT_ROOT / "data"
ERA2_DATA = PROJECT_ROOT / "data" / "era2"


def section(title):
    print(f"\n{'=' * 70}")
    print(f"  {title}")
    print(f"{'=' * 70}")


def main():
    print("📊 二期数据验证 & 一期对比")

    # ===== 加载二期数据 =====
    section("1. 加载二期 ETL 输出")
    df_basic = pd.read_parquet(ERA2_DATA / "era2_基础信息表.parquet")
    df_course = pd.read_parquet(ERA2_DATA / "era2_课程实施情况表.parquet")
    df_subject = pd.read_parquet(ERA2_DATA / "era2_学科课程实施情况表.parquet")
    print(f"  基础信息表: {df_basic.shape}")
    print(f"  课程实施情况表: {df_course.shape}")
    print(f"  学科课程实施情况表: {df_subject.shape}")

    # ===== 加载一期数据 =====
    section("2. 加载一期数据")
    df1_basic = pd.read_excel(ERA1_DATA / "长宁区_基础信息表.xlsx")
    df1_course = pd.read_excel(ERA1_DATA / "长宁区_课程实施情况表.xlsx")
    df1_subject = pd.read_excel(ERA1_DATA / "长宁区_学科课程实施情况表.xlsx")
    print(f"  基础信息表: {df1_basic.shape}")
    print(f"  课程实施情况表: {df1_course.shape}")
    print(f"  学科课程实施情况表: {df1_subject.shape}")

    # ===== 列名对比 =====
    section("3. 列名对比")
    for name, (d1, d2) in {
        '基础信息表': (df1_basic, df_basic),
        '课程实施情况表': (df1_course, df_course),
        '学科课程实施情况表': (df1_subject, df_subject),
    }.items():
        c1 = set(d1.columns)
        c2 = set(d2.columns)
        print(f"\n  [{name}]")
        print(f"    一期列: {sorted(c1)}")
        print(f"    二期列: {sorted(c2)}")
        print(f"    一期有/二期无: {sorted(c1 - c2) or '无'}")
        print(f"    二期有/一期无: {sorted(c2 - c1) or '无'}")
        print(f"    共有列: {sorted(c1 & c2)}")

    # ===== 字段ID对比（课程实施情况表） =====
    section("4. 课程实施情况表 - 字段ID对比")
    # 一期用 "字段ID"，二期用 "字段ID"
    id_col_1 = '字段ID' if '字段ID' in df1_course.columns else None
    id_col_2 = '字段ID' if '字段ID' in df_course.columns else None

    if id_col_1 and id_col_2:
        ids1 = set(df1_course[id_col_1].dropna().unique())
        ids2 = set(df_course[id_col_2].dropna().unique())
        print(f"  一期字段ID数: {len(ids1)}")
        print(f"  二期字段ID数: {len(ids2)}")
        common = ids1 & ids2
        print(f"  共有: {len(common)}")
        print(f"  一期有/二期无: {len(ids1 - ids2)}")
        if ids1 - ids2:
            print(f"    缺失: {sorted(ids1 - ids2)[:20]}{'...' if len(ids1 - ids2) > 20 else ''}")
        print(f"  二期有/一期无: {len(ids2 - ids1)}")
        if ids2 - ids1:
            print(f"    新增: {sorted(ids2 - ids1)[:20]}{'...' if len(ids2 - ids1) > 20 else ''}")
    else:
        print(f"  ⚠️ 无法对比: 一期列={id_col_1}, 二期列={id_col_2}")
        # 尝试用字段名称对比
        names1 = set(df1_course['字段名称'].dropna().unique())
        names2 = set(df_course['字段名称'].dropna().unique())
        print(f"  [改用字段名称对比]")
        print(f"  一期字段名称数: {len(names1)}")
        print(f"  二期字段名称数: {len(names2)}")
        common = names1 & names2
        print(f"  共有: {len(common)}")
        only1 = sorted(names1 - names2)
        only2 = sorted(names2 - names1)
        print(f"  一期有/二期无: {len(only1)}")
        if only1:
            for n in only1[:30]:
                print(f"    - {n}")
        print(f"  二期有/一期无: {len(only2)}")
        if only2:
            for n in only2[:30]:
                print(f"    + {n}")

    # ===== 学科课程表 - 字段名称对比 =====
    section("5. 学科课程实施情况表 - 字段名称对比")
    names1 = set(df1_subject['字段名称'].dropna().unique())
    names2 = set(df_subject['字段名称'].dropna().unique())
    print(f"  一期字段名称数: {len(names1)}")
    print(f"  二期字段名称数: {len(names2)}")
    common = names1 & names2
    print(f"  共有: {len(common)}")
    only1 = sorted(names1 - names2)
    only2 = sorted(names2 - names1)
    print(f"  一期有/二期无: {len(only1)}")
    if only1:
        for n in only1[:20]:
            print(f"    - {n}")
        if len(only1) > 20:
            print(f"    ... (共{len(only1)}个)")
    print(f"  二期有/一期无: {len(only2)}")
    if only2:
        for n in only2[:20]:
            print(f"    + {n}")
        if len(only2) > 20:
            print(f"    ... (共{len(only2)}个)")

    # ===== 长宁区数据提取验证 =====
    section("6. 长宁区数据提取")
    cn_course = df_course[df_course['所在区'] == '长宁区']
    cn_subject = df_subject[df_subject['所在区'] == '长宁区']
    print(f"  课程实施情况表(长宁区): {len(cn_course)}行")
    print(f"  学科课程实施情况表(长宁区): {len(cn_subject)}行")
    print(f"  长宁区学校: {sorted(cn_course['学校简称'].unique())}")
    print(f"  一期学校: {sorted(df1_course['学校简称'].unique()) if '学校简称' in df1_course.columns else sorted(df1_course['学校名称'].unique())}")

    # ===== 数据质量 =====
    section("7. 数据质量检查")
    for name, df in [('课程实施情况表', df_course), ('学科课程实施情况表', df_subject)]:
        total = len(df)
        nulls = df['字段取值'].isna().sum() + (df['字段取值'] == '').sum() + (df['字段取值'] == 'nan').sum()
        print(f"  [{name}]")
        print(f"    总行数: {total}")
        print(f"    字段取值为空/nan: {nulls} ({nulls/total*100:.1f}%)")
        print(f"    学校数: {df['学校简称'].nunique()}")

    print(f"\n✅ 验证完成!")


if __name__ == "__main__":
    main()