lofyerandfactory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
71db82393a
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
364 lines
13 KiB
Python
364 lines
13 KiB
Python
"""
|
||
统计引擎：PCA合成 + 标准化 + 水平判定 + 聚类 + T检验 + 相关性
|
||
将赋分后的原始数据合成为维度得分，并进行统计分析
|
||
"""
|
||
import pandas as pd
|
||
import numpy as np
|
||
from typing import Dict, List, Optional, Tuple
|
||
from sklearn.decomposition import PCA
|
||
from sklearn.preprocessing import StandardScaler
|
||
from sklearn.cluster import KMeans
|
||
from scipy import stats
|
||
import logging
|
||
|
||
from ..config import (
|
||
PCA_MEAN, PCA_STD, LEVEL_THRESHOLDS, LEVEL_DESCRIPTIONS,
|
||
DIMENSION_FRAMEWORK, SUBJECTS, SCHOOL_TYPE_MAP,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class StatsEngine:
|
||
"""统计引擎"""
|
||
|
||
def __init__(self):
|
||
self._dimension_scores: Optional[pd.DataFrame] = None
|
||
self._sub_dimension_scores: Optional[pd.DataFrame] = None
|
||
self._subject_dimension_scores: Optional[pd.DataFrame] = None
|
||
|
||
def standardize_scores(self, raw_scores: np.ndarray) -> np.ndarray:
|
||
"""标准化到均值50标准差10"""
|
||
if len(raw_scores) < 2:
|
||
return np.full_like(raw_scores, PCA_MEAN, dtype=float)
|
||
mean = np.nanmean(raw_scores)
|
||
std = np.nanstd(raw_scores, ddof=1)
|
||
if std == 0 or np.isnan(std):
|
||
return np.full_like(raw_scores, PCA_MEAN, dtype=float)
|
||
return (raw_scores - mean) / std * PCA_STD + PCA_MEAN
|
||
|
||
def pca_compose(self, data_matrix: pd.DataFrame) -> np.ndarray:
|
||
"""
|
||
PCA合成：将多个变量合成为一个主成分分数
|
||
data_matrix: 行=学校, 列=变量
|
||
返回：合成后的分数（已标准化到50/10）
|
||
"""
|
||
# 处理缺失值：均值填充
|
||
filled = data_matrix.fillna(data_matrix.mean())
|
||
if filled.shape[1] == 0:
|
||
return np.full(filled.shape[0], PCA_MEAN)
|
||
|
||
if filled.shape[1] == 1:
|
||
# 只有一个变量，直接标准化
|
||
return self.standardize_scores(filled.iloc[:, 0].values)
|
||
|
||
# 标准化
|
||
scaler = StandardScaler()
|
||
scaled = scaler.fit_transform(filled)
|
||
|
||
# PCA取第一主成分
|
||
n_components = min(1, filled.shape[1], filled.shape[0])
|
||
pca = PCA(n_components=n_components)
|
||
scores = pca.fit_transform(scaled)[:, 0]
|
||
|
||
# 如果载荷为负（方向反转），翻转
|
||
loadings = pca.components_[0]
|
||
if np.sum(loadings) < 0:
|
||
scores = -scores
|
||
|
||
# 标准化到50/10
|
||
return self.standardize_scores(scores)
|
||
|
||
def determine_level(self, score: float, dimension: str) -> int:
|
||
"""根据分数和维度确定水平（1-4）"""
|
||
thresholds = LEVEL_THRESHOLDS.get(dimension, {})
|
||
if not thresholds:
|
||
# 默认阈值
|
||
if score > 55:
|
||
return 4
|
||
elif score > 50:
|
||
return 3
|
||
elif score > 45:
|
||
return 2
|
||
else:
|
||
return 1
|
||
|
||
if score > thresholds["level4"]:
|
||
return 4
|
||
elif score > thresholds["level3"]:
|
||
return 3
|
||
elif score > thresholds["level2"]:
|
||
return 2
|
||
else:
|
||
return 1
|
||
|
||
def get_level_description(self, dimension: str, level: int) -> str:
|
||
"""获取水平的质性描述"""
|
||
descriptions = LEVEL_DESCRIPTIONS.get(dimension, {})
|
||
return descriptions.get(level, f"水平{level}")
|
||
|
||
def compute_dimension_scores(self, school_raw_scores: Dict[str, Dict[str, List[float]]]) -> pd.DataFrame:
|
||
"""
|
||
计算所有学校在各三级维度上的得分
|
||
|
||
school_raw_scores: {
|
||
school_name: {
|
||
sub_dimension_name: [score1, score2, ...] # 该维度下各题目的赋分
|
||
}
|
||
}
|
||
|
||
返回 DataFrame: 行=学校, 列=三级维度, 值=标准化得分
|
||
"""
|
||
schools = list(school_raw_scores.keys())
|
||
all_sub_dims = []
|
||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||
all_sub_dims.extend(info["sub_dimensions"])
|
||
|
||
# 构建原始矩阵
|
||
raw_matrix = {}
|
||
for sub_dim in all_sub_dims:
|
||
values = []
|
||
for school in schools:
|
||
scores = school_raw_scores.get(school, {}).get(sub_dim, [])
|
||
values.append(np.nanmean(scores) if scores else np.nan)
|
||
raw_matrix[sub_dim] = values
|
||
|
||
raw_df = pd.DataFrame(raw_matrix, index=schools)
|
||
|
||
# PCA合成并标准化各维度
|
||
result = pd.DataFrame(index=schools)
|
||
for sub_dim in all_sub_dims:
|
||
if sub_dim in raw_df.columns:
|
||
result[sub_dim] = self.standardize_scores(raw_df[sub_dim].values)
|
||
else:
|
||
result[sub_dim] = PCA_MEAN
|
||
|
||
self._sub_dimension_scores = result
|
||
return result
|
||
|
||
def compute_dimension_aggregates(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
|
||
"""
|
||
从三级维度分数聚合到二级维度（均值）
|
||
|
||
sub_scores: 行=学校, 列=三级维度
|
||
返回: 行=学校, 列=二级维度
|
||
"""
|
||
result = pd.DataFrame(index=sub_scores.index)
|
||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||
sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
|
||
if sub_dims:
|
||
result[dim] = sub_scores[sub_dims].mean(axis=1)
|
||
else:
|
||
result[dim] = PCA_MEAN
|
||
|
||
# 总体得分（七维度均值）
|
||
result["总体得分"] = result[list(DIMENSION_FRAMEWORK.keys())].mean(axis=1)
|
||
self._dimension_scores = result
|
||
return result
|
||
|
||
def compute_levels(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
|
||
"""计算各学校各维度的水平等级"""
|
||
levels = pd.DataFrame(index=sub_scores.index)
|
||
for col in sub_scores.columns:
|
||
levels[col] = sub_scores[col].apply(
|
||
lambda x: self.determine_level(x, col)
|
||
)
|
||
return levels
|
||
|
||
def cluster_analysis(self, scores: pd.DataFrame, n_clusters: int = 2) -> Dict:
|
||
"""
|
||
聚类分析
|
||
|
||
scores: 行=学校, 列=维度
|
||
返回: 聚类标签和各类特征
|
||
"""
|
||
# 标准化
|
||
scaler = StandardScaler()
|
||
scaled = scaler.fit_transform(scores.fillna(PCA_MEAN))
|
||
|
||
# KMeans聚类
|
||
n_clusters = min(n_clusters, len(scores))
|
||
if n_clusters < 2:
|
||
return {"labels": [0] * len(scores), "centers": scores.values.tolist()}
|
||
|
||
kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
|
||
labels = kmeans.fit_predict(scaled)
|
||
|
||
# 计算各类均值
|
||
cluster_means = {}
|
||
for c in range(n_clusters):
|
||
mask = labels == c
|
||
cluster_means[c] = scores[mask].mean().to_dict()
|
||
|
||
# 确定哪个是"较好类"（总均值更高的）
|
||
avg_per_cluster = {c: np.mean(list(v.values())) for c, v in cluster_means.items()}
|
||
sorted_clusters = sorted(avg_per_cluster.items(), key=lambda x: x[1], reverse=True)
|
||
|
||
cluster_names = {}
|
||
for rank, (c, _) in enumerate(sorted_clusters):
|
||
if rank == 0:
|
||
cluster_names[c] = "较好"
|
||
else:
|
||
cluster_names[c] = "待提升"
|
||
|
||
return {
|
||
"labels": labels.tolist(),
|
||
"school_clusters": {
|
||
school: cluster_names[labels[i]]
|
||
for i, school in enumerate(scores.index)
|
||
},
|
||
"cluster_means": cluster_means,
|
||
"cluster_names": cluster_names,
|
||
}
|
||
|
||
def t_test_vs_mean(self, school_scores: np.ndarray, ref_mean: float) -> Dict:
|
||
"""
|
||
单样本T检验：学校各学科得分 vs 参考均值
|
||
|
||
school_scores: 该学校在某维度各学科的得分
|
||
ref_mean: 参考均值（如区均值、全市均值）
|
||
"""
|
||
scores = school_scores[~np.isnan(school_scores)]
|
||
if len(scores) < 2:
|
||
return {"t": np.nan, "p": np.nan, "significant": False, "n": len(scores)}
|
||
|
||
t_stat, p_value = stats.ttest_1samp(scores, ref_mean)
|
||
return {
|
||
"t": round(float(t_stat), 3),
|
||
"p": round(float(p_value), 4),
|
||
"significant": float(p_value) < 0.05,
|
||
"n": len(scores),
|
||
}
|
||
|
||
def correlation_analysis(self, dim_scores: pd.DataFrame) -> pd.DataFrame:
|
||
"""维度间相关性分析"""
|
||
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
|
||
return dim_scores[dim_cols].corr()
|
||
|
||
def compute_school_report_data(self, school: str,
|
||
sub_scores: pd.DataFrame,
|
||
dim_scores: pd.DataFrame) -> Dict:
|
||
"""
|
||
为某一所学校生成完整的报告数据包
|
||
|
||
返回包含所有统计分析结果的结构化数据
|
||
"""
|
||
schools = list(sub_scores.index)
|
||
if school not in schools:
|
||
raise ValueError(f"学校 '{school}' 不在数据中")
|
||
|
||
school_info = SCHOOL_TYPE_MAP.get(school, {})
|
||
school_type = school_info.get("type", "")
|
||
|
||
# 区均值
|
||
district_avg_sub = sub_scores.mean()
|
||
district_avg_dim = dim_scores.mean()
|
||
|
||
# 同类学校均值
|
||
same_type_schools = [s for s in schools if SCHOOL_TYPE_MAP.get(s, {}).get("type") == school_type]
|
||
if same_type_schools:
|
||
same_type_avg_sub = sub_scores.loc[same_type_schools].mean()
|
||
same_type_avg_dim = dim_scores.loc[same_type_schools].mean()
|
||
else:
|
||
same_type_avg_sub = district_avg_sub
|
||
same_type_avg_dim = district_avg_dim
|
||
|
||
# 水平判定
|
||
levels = self.compute_levels(sub_scores)
|
||
|
||
# 聚类（二级维度）
|
||
dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
|
||
overall_cluster = self.cluster_analysis(dim_scores[dim_cols])
|
||
|
||
# 各二级维度聚类
|
||
dim_clusters = {}
|
||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||
sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
|
||
if sub_dims:
|
||
dim_clusters[dim] = self.cluster_analysis(sub_scores[sub_dims])
|
||
|
||
# 相关性
|
||
correlation = self.correlation_analysis(dim_scores)
|
||
|
||
# 构建报告数据
|
||
report = {
|
||
"school": school,
|
||
"school_info": school_info,
|
||
|
||
# 总体得分
|
||
"overall": {
|
||
"score": round(float(dim_scores.loc[school, "总体得分"]), 2),
|
||
"district_avg": round(float(district_avg_dim["总体得分"]), 2),
|
||
"same_type_avg": round(float(same_type_avg_dim.get("总体得分", PCA_MEAN)), 2),
|
||
"rank_in_district": int((dim_scores["总体得分"] >= dim_scores.loc[school, "总体得分"]).sum()),
|
||
"total_schools": len(schools),
|
||
"cluster": overall_cluster["school_clusters"].get(school, ""),
|
||
},
|
||
|
||
# 二级维度
|
||
"dimensions": {},
|
||
|
||
# 三级维度
|
||
"sub_dimensions": {},
|
||
|
||
# 相关性矩阵
|
||
"correlation": correlation.to_dict(),
|
||
|
||
# 所有学校得分（用于对比）
|
||
"all_schools_dim_scores": dim_scores.to_dict(),
|
||
"all_schools_sub_scores": sub_scores.to_dict(),
|
||
}
|
||
|
||
# 填充二级维度数据
|
||
for dim in DIMENSION_FRAMEWORK:
|
||
score = float(dim_scores.loc[school, dim])
|
||
d_avg = float(district_avg_dim[dim])
|
||
st_avg = float(same_type_avg_dim.get(dim, PCA_MEAN))
|
||
|
||
# T检验：该学校在该维度下各三级维度得分 vs 区均值
|
||
sub_dims = DIMENSION_FRAMEWORK[dim]["sub_dimensions"]
|
||
sub_vals = np.array([float(sub_scores.loc[school, s]) for s in sub_dims if s in sub_scores.columns])
|
||
t_test = self.t_test_vs_mean(sub_vals, d_avg)
|
||
|
||
report["dimensions"][dim] = {
|
||
"score": round(score, 2),
|
||
"district_avg": round(d_avg, 2),
|
||
"same_type_avg": round(st_avg, 2),
|
||
"diff_district": round(score - d_avg, 2),
|
||
"rank_in_district": int((dim_scores[dim] >= score).sum()),
|
||
"t_test_vs_district": t_test,
|
||
"cluster": dim_clusters.get(dim, {}).get("school_clusters", {}).get(school, ""),
|
||
}
|
||
|
||
# 填充三级维度数据
|
||
for dim, info in DIMENSION_FRAMEWORK.items():
|
||
for sub_dim in info["sub_dimensions"]:
|
||
if sub_dim not in sub_scores.columns:
|
||
continue
|
||
score = float(sub_scores.loc[school, sub_dim])
|
||
d_avg = float(district_avg_sub[sub_dim])
|
||
level = int(levels.loc[school, sub_dim])
|
||
|
||
# 各学校在该维度的排名
|
||
rank = int((sub_scores[sub_dim] >= score).sum())
|
||
|
||
# 水平分布统计
|
||
dim_levels = levels[sub_dim]
|
||
level_dist = {
|
||
f"水平{i}": int((dim_levels == i).sum())
|
||
for i in range(1, 5)
|
||
}
|
||
|
||
report["sub_dimensions"][sub_dim] = {
|
||
"parent_dimension": dim,
|
||
"score": round(score, 2),
|
||
"district_avg": round(d_avg, 2),
|
||
"diff_district": round(score - d_avg, 2),
|
||
"rank_in_district": rank,
|
||
"level": level,
|
||
"level_description": self.get_level_description(sub_dim, level),
|
||
"level_distribution": level_dist,
|
||
}
|
||
|
||
return report
|