lofyerandfactory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
71db82393a
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
146 lines
6.0 KiB
Python
146 lines
6.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
二期数据验证脚本
|
|
检查 ETL 产出的 parquet 文件是否符合预期，
|
|
并与一期数据做结构和字段ID的对比
|
|
"""
|
|
import pandas as pd
|
|
import json
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent # report-admin/
|
|
ERA1_DATA = PROJECT_ROOT / "data"
|
|
ERA2_DATA = PROJECT_ROOT / "data" / "era2"
|
|
|
|
|
|
def section(title):
|
|
print(f"\n{'=' * 70}")
|
|
print(f" {title}")
|
|
print(f"{'=' * 70}")
|
|
|
|
|
|
def main():
|
|
print("📊 二期数据验证 & 一期对比")
|
|
|
|
# ===== 加载二期数据 =====
|
|
section("1. 加载二期 ETL 输出")
|
|
df_basic = pd.read_parquet(ERA2_DATA / "era2_基础信息表.parquet")
|
|
df_course = pd.read_parquet(ERA2_DATA / "era2_课程实施情况表.parquet")
|
|
df_subject = pd.read_parquet(ERA2_DATA / "era2_学科课程实施情况表.parquet")
|
|
print(f" 基础信息表: {df_basic.shape}")
|
|
print(f" 课程实施情况表: {df_course.shape}")
|
|
print(f" 学科课程实施情况表: {df_subject.shape}")
|
|
|
|
# ===== 加载一期数据 =====
|
|
section("2. 加载一期数据")
|
|
df1_basic = pd.read_excel(ERA1_DATA / "长宁区_基础信息表.xlsx")
|
|
df1_course = pd.read_excel(ERA1_DATA / "长宁区_课程实施情况表.xlsx")
|
|
df1_subject = pd.read_excel(ERA1_DATA / "长宁区_学科课程实施情况表.xlsx")
|
|
print(f" 基础信息表: {df1_basic.shape}")
|
|
print(f" 课程实施情况表: {df1_course.shape}")
|
|
print(f" 学科课程实施情况表: {df1_subject.shape}")
|
|
|
|
# ===== 列名对比 =====
|
|
section("3. 列名对比")
|
|
for name, (d1, d2) in {
|
|
'基础信息表': (df1_basic, df_basic),
|
|
'课程实施情况表': (df1_course, df_course),
|
|
'学科课程实施情况表': (df1_subject, df_subject),
|
|
}.items():
|
|
c1 = set(d1.columns)
|
|
c2 = set(d2.columns)
|
|
print(f"\n [{name}]")
|
|
print(f" 一期列: {sorted(c1)}")
|
|
print(f" 二期列: {sorted(c2)}")
|
|
print(f" 一期有/二期无: {sorted(c1 - c2) or '无'}")
|
|
print(f" 二期有/一期无: {sorted(c2 - c1) or '无'}")
|
|
print(f" 共有列: {sorted(c1 & c2)}")
|
|
|
|
# ===== 字段ID对比（课程实施情况表） =====
|
|
section("4. 课程实施情况表 - 字段ID对比")
|
|
# 一期用 "字段ID"，二期用 "字段ID"
|
|
id_col_1 = '字段ID' if '字段ID' in df1_course.columns else None
|
|
id_col_2 = '字段ID' if '字段ID' in df_course.columns else None
|
|
|
|
if id_col_1 and id_col_2:
|
|
ids1 = set(df1_course[id_col_1].dropna().unique())
|
|
ids2 = set(df_course[id_col_2].dropna().unique())
|
|
print(f" 一期字段ID数: {len(ids1)}")
|
|
print(f" 二期字段ID数: {len(ids2)}")
|
|
common = ids1 & ids2
|
|
print(f" 共有: {len(common)}")
|
|
print(f" 一期有/二期无: {len(ids1 - ids2)}")
|
|
if ids1 - ids2:
|
|
print(f" 缺失: {sorted(ids1 - ids2)[:20]}{'...' if len(ids1 - ids2) > 20 else ''}")
|
|
print(f" 二期有/一期无: {len(ids2 - ids1)}")
|
|
if ids2 - ids1:
|
|
print(f" 新增: {sorted(ids2 - ids1)[:20]}{'...' if len(ids2 - ids1) > 20 else ''}")
|
|
else:
|
|
print(f" ⚠️ 无法对比: 一期列={id_col_1}, 二期列={id_col_2}")
|
|
# 尝试用字段名称对比
|
|
names1 = set(df1_course['字段名称'].dropna().unique())
|
|
names2 = set(df_course['字段名称'].dropna().unique())
|
|
print(f" [改用字段名称对比]")
|
|
print(f" 一期字段名称数: {len(names1)}")
|
|
print(f" 二期字段名称数: {len(names2)}")
|
|
common = names1 & names2
|
|
print(f" 共有: {len(common)}")
|
|
only1 = sorted(names1 - names2)
|
|
only2 = sorted(names2 - names1)
|
|
print(f" 一期有/二期无: {len(only1)}")
|
|
if only1:
|
|
for n in only1[:30]:
|
|
print(f" - {n}")
|
|
print(f" 二期有/一期无: {len(only2)}")
|
|
if only2:
|
|
for n in only2[:30]:
|
|
print(f" + {n}")
|
|
|
|
# ===== 学科课程表 - 字段名称对比 =====
|
|
section("5. 学科课程实施情况表 - 字段名称对比")
|
|
names1 = set(df1_subject['字段名称'].dropna().unique())
|
|
names2 = set(df_subject['字段名称'].dropna().unique())
|
|
print(f" 一期字段名称数: {len(names1)}")
|
|
print(f" 二期字段名称数: {len(names2)}")
|
|
common = names1 & names2
|
|
print(f" 共有: {len(common)}")
|
|
only1 = sorted(names1 - names2)
|
|
only2 = sorted(names2 - names1)
|
|
print(f" 一期有/二期无: {len(only1)}")
|
|
if only1:
|
|
for n in only1[:20]:
|
|
print(f" - {n}")
|
|
if len(only1) > 20:
|
|
print(f" ... (共{len(only1)}个)")
|
|
print(f" 二期有/一期无: {len(only2)}")
|
|
if only2:
|
|
for n in only2[:20]:
|
|
print(f" + {n}")
|
|
if len(only2) > 20:
|
|
print(f" ... (共{len(only2)}个)")
|
|
|
|
# ===== 长宁区数据提取验证 =====
|
|
section("6. 长宁区数据提取")
|
|
cn_course = df_course[df_course['所在区'] == '长宁区']
|
|
cn_subject = df_subject[df_subject['所在区'] == '长宁区']
|
|
print(f" 课程实施情况表(长宁区): {len(cn_course)}行")
|
|
print(f" 学科课程实施情况表(长宁区): {len(cn_subject)}行")
|
|
print(f" 长宁区学校: {sorted(cn_course['学校简称'].unique())}")
|
|
print(f" 一期学校: {sorted(df1_course['学校简称'].unique()) if '学校简称' in df1_course.columns else sorted(df1_course['学校名称'].unique())}")
|
|
|
|
# ===== 数据质量 =====
|
|
section("7. 数据质量检查")
|
|
for name, df in [('课程实施情况表', df_course), ('学科课程实施情况表', df_subject)]:
|
|
total = len(df)
|
|
nulls = df['字段取值'].isna().sum() + (df['字段取值'] == '').sum() + (df['字段取值'] == 'nan').sum()
|
|
print(f" [{name}]")
|
|
print(f" 总行数: {total}")
|
|
print(f" 字段取值为空/nan: {nulls} ({nulls/total*100:.1f}%)")
|
|
print(f" 学校数: {df['学校简称'].nunique()}")
|
|
|
|
print(f"\n✅ 验证完成!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|