Initial commit

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>

backend/app/engines/stats_engine.py

@@ -0,0 +1,363 @@
+"""
+统计引擎：PCA合成 + 标准化 + 水平判定 + 聚类 + T检验 + 相关性
+将赋分后的原始数据合成为维度得分，并进行统计分析
+"""
+import pandas as pd
+import numpy as np
+from typing import Dict, List, Optional, Tuple
+from sklearn.decomposition import PCA
+from sklearn.preprocessing import StandardScaler
+from sklearn.cluster import KMeans
+from scipy import stats
+import logging
+
+from ..config import (
+    PCA_MEAN, PCA_STD, LEVEL_THRESHOLDS, LEVEL_DESCRIPTIONS,
+    DIMENSION_FRAMEWORK, SUBJECTS, SCHOOL_TYPE_MAP,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class StatsEngine:
+    """统计引擎"""
+
+    def __init__(self):
+        self._dimension_scores: Optional[pd.DataFrame] = None
+        self._sub_dimension_scores: Optional[pd.DataFrame] = None
+        self._subject_dimension_scores: Optional[pd.DataFrame] = None
+
+    def standardize_scores(self, raw_scores: np.ndarray) -> np.ndarray:
+        """标准化到均值50标准差10"""
+        if len(raw_scores) < 2:
+            return np.full_like(raw_scores, PCA_MEAN, dtype=float)
+        mean = np.nanmean(raw_scores)
+        std = np.nanstd(raw_scores, ddof=1)
+        if std == 0 or np.isnan(std):
+            return np.full_like(raw_scores, PCA_MEAN, dtype=float)
+        return (raw_scores - mean) / std * PCA_STD + PCA_MEAN
+
+    def pca_compose(self, data_matrix: pd.DataFrame) -> np.ndarray:
+        """
+        PCA合成：将多个变量合成为一个主成分分数
+        data_matrix: 行=学校, 列=变量
+        返回：合成后的分数（已标准化到50/10）
+        """
+        # 处理缺失值：均值填充
+        filled = data_matrix.fillna(data_matrix.mean())
+        if filled.shape[1] == 0:
+            return np.full(filled.shape[0], PCA_MEAN)
+
+        if filled.shape[1] == 1:
+            # 只有一个变量，直接标准化
+            return self.standardize_scores(filled.iloc[:, 0].values)
+
+        # 标准化
+        scaler = StandardScaler()
+        scaled = scaler.fit_transform(filled)
+
+        # PCA取第一主成分
+        n_components = min(1, filled.shape[1], filled.shape[0])
+        pca = PCA(n_components=n_components)
+        scores = pca.fit_transform(scaled)[:, 0]
+
+        # 如果载荷为负（方向反转），翻转
+        loadings = pca.components_[0]
+        if np.sum(loadings) < 0:
+            scores = -scores
+
+        # 标准化到50/10
+        return self.standardize_scores(scores)
+
+    def determine_level(self, score: float, dimension: str) -> int:
+        """根据分数和维度确定水平（1-4）"""
+        thresholds = LEVEL_THRESHOLDS.get(dimension, {})
+        if not thresholds:
+            # 默认阈值
+            if score > 55:
+                return 4
+            elif score > 50:
+                return 3
+            elif score > 45:
+                return 2
+            else:
+                return 1
+
+        if score > thresholds["level4"]:
+            return 4
+        elif score > thresholds["level3"]:
+            return 3
+        elif score > thresholds["level2"]:
+            return 2
+        else:
+            return 1
+
+    def get_level_description(self, dimension: str, level: int) -> str:
+        """获取水平的质性描述"""
+        descriptions = LEVEL_DESCRIPTIONS.get(dimension, {})
+        return descriptions.get(level, f"水平{level}")
+
+    def compute_dimension_scores(self, school_raw_scores: Dict[str, Dict[str, List[float]]]) -> pd.DataFrame:
+        """
+        计算所有学校在各三级维度上的得分
+
+        school_raw_scores: {
+            school_name: {
+                sub_dimension_name: [score1, score2, ...]  # 该维度下各题目的赋分
+            }
+        }
+
+        返回 DataFrame: 行=学校, 列=三级维度, 值=标准化得分
+        """
+        schools = list(school_raw_scores.keys())
+        all_sub_dims = []
+        for dim, info in DIMENSION_FRAMEWORK.items():
+            all_sub_dims.extend(info["sub_dimensions"])
+
+        # 构建原始矩阵
+        raw_matrix = {}
+        for sub_dim in all_sub_dims:
+            values = []
+            for school in schools:
+                scores = school_raw_scores.get(school, {}).get(sub_dim, [])
+                values.append(np.nanmean(scores) if scores else np.nan)
+            raw_matrix[sub_dim] = values
+
+        raw_df = pd.DataFrame(raw_matrix, index=schools)
+
+        # PCA合成并标准化各维度
+        result = pd.DataFrame(index=schools)
+        for sub_dim in all_sub_dims:
+            if sub_dim in raw_df.columns:
+                result[sub_dim] = self.standardize_scores(raw_df[sub_dim].values)
+            else:
+                result[sub_dim] = PCA_MEAN
+
+        self._sub_dimension_scores = result
+        return result
+
+    def compute_dimension_aggregates(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
+        """
+        从三级维度分数聚合到二级维度（均值）
+
+        sub_scores: 行=学校, 列=三级维度
+        返回: 行=学校, 列=二级维度
+        """
+        result = pd.DataFrame(index=sub_scores.index)
+        for dim, info in DIMENSION_FRAMEWORK.items():
+            sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
+            if sub_dims:
+                result[dim] = sub_scores[sub_dims].mean(axis=1)
+            else:
+                result[dim] = PCA_MEAN
+
+        # 总体得分（七维度均值）
+        result["总体得分"] = result[list(DIMENSION_FRAMEWORK.keys())].mean(axis=1)
+        self._dimension_scores = result
+        return result
+
+    def compute_levels(self, sub_scores: pd.DataFrame) -> pd.DataFrame:
+        """计算各学校各维度的水平等级"""
+        levels = pd.DataFrame(index=sub_scores.index)
+        for col in sub_scores.columns:
+            levels[col] = sub_scores[col].apply(
+                lambda x: self.determine_level(x, col)
+            )
+        return levels
+
+    def cluster_analysis(self, scores: pd.DataFrame, n_clusters: int = 2) -> Dict:
+        """
+        聚类分析
+
+        scores: 行=学校, 列=维度
+        返回: 聚类标签和各类特征
+        """
+        # 标准化
+        scaler = StandardScaler()
+        scaled = scaler.fit_transform(scores.fillna(PCA_MEAN))
+
+        # KMeans聚类
+        n_clusters = min(n_clusters, len(scores))
+        if n_clusters < 2:
+            return {"labels": [0] * len(scores), "centers": scores.values.tolist()}
+
+        kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10)
+        labels = kmeans.fit_predict(scaled)
+
+        # 计算各类均值
+        cluster_means = {}
+        for c in range(n_clusters):
+            mask = labels == c
+            cluster_means[c] = scores[mask].mean().to_dict()
+
+        # 确定哪个是"较好类"（总均值更高的）
+        avg_per_cluster = {c: np.mean(list(v.values())) for c, v in cluster_means.items()}
+        sorted_clusters = sorted(avg_per_cluster.items(), key=lambda x: x[1], reverse=True)
+
+        cluster_names = {}
+        for rank, (c, _) in enumerate(sorted_clusters):
+            if rank == 0:
+                cluster_names[c] = "较好"
+            else:
+                cluster_names[c] = "待提升"
+
+        return {
+            "labels": labels.tolist(),
+            "school_clusters": {
+                school: cluster_names[labels[i]]
+                for i, school in enumerate(scores.index)
+            },
+            "cluster_means": cluster_means,
+            "cluster_names": cluster_names,
+        }
+
+    def t_test_vs_mean(self, school_scores: np.ndarray, ref_mean: float) -> Dict:
+        """
+        单样本T检验：学校各学科得分 vs 参考均值
+
+        school_scores: 该学校在某维度各学科的得分
+        ref_mean: 参考均值（如区均值、全市均值）
+        """
+        scores = school_scores[~np.isnan(school_scores)]
+        if len(scores) < 2:
+            return {"t": np.nan, "p": np.nan, "significant": False, "n": len(scores)}
+
+        t_stat, p_value = stats.ttest_1samp(scores, ref_mean)
+        return {
+            "t": round(float(t_stat), 3),
+            "p": round(float(p_value), 4),
+            "significant": float(p_value) < 0.05,
+            "n": len(scores),
+        }
+
+    def correlation_analysis(self, dim_scores: pd.DataFrame) -> pd.DataFrame:
+        """维度间相关性分析"""
+        dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
+        return dim_scores[dim_cols].corr()
+
+    def compute_school_report_data(self, school: str,
+                                    sub_scores: pd.DataFrame,
+                                    dim_scores: pd.DataFrame) -> Dict:
+        """
+        为某一所学校生成完整的报告数据包
+
+        返回包含所有统计分析结果的结构化数据
+        """
+        schools = list(sub_scores.index)
+        if school not in schools:
+            raise ValueError(f"学校 '{school}' 不在数据中")
+
+        school_info = SCHOOL_TYPE_MAP.get(school, {})
+        school_type = school_info.get("type", "")
+
+        # 区均值
+        district_avg_sub = sub_scores.mean()
+        district_avg_dim = dim_scores.mean()
+
+        # 同类学校均值
+        same_type_schools = [s for s in schools if SCHOOL_TYPE_MAP.get(s, {}).get("type") == school_type]
+        if same_type_schools:
+            same_type_avg_sub = sub_scores.loc[same_type_schools].mean()
+            same_type_avg_dim = dim_scores.loc[same_type_schools].mean()
+        else:
+            same_type_avg_sub = district_avg_sub
+            same_type_avg_dim = district_avg_dim
+
+        # 水平判定
+        levels = self.compute_levels(sub_scores)
+
+        # 聚类（二级维度）
+        dim_cols = [c for c in dim_scores.columns if c in DIMENSION_FRAMEWORK]
+        overall_cluster = self.cluster_analysis(dim_scores[dim_cols])
+
+        # 各二级维度聚类
+        dim_clusters = {}
+        for dim, info in DIMENSION_FRAMEWORK.items():
+            sub_dims = [s for s in info["sub_dimensions"] if s in sub_scores.columns]
+            if sub_dims:
+                dim_clusters[dim] = self.cluster_analysis(sub_scores[sub_dims])
+
+        # 相关性
+        correlation = self.correlation_analysis(dim_scores)
+
+        # 构建报告数据
+        report = {
+            "school": school,
+            "school_info": school_info,
+
+            # 总体得分
+            "overall": {
+                "score": round(float(dim_scores.loc[school, "总体得分"]), 2),
+                "district_avg": round(float(district_avg_dim["总体得分"]), 2),
+                "same_type_avg": round(float(same_type_avg_dim.get("总体得分", PCA_MEAN)), 2),
+                "rank_in_district": int((dim_scores["总体得分"] >= dim_scores.loc[school, "总体得分"]).sum()),
+                "total_schools": len(schools),
+                "cluster": overall_cluster["school_clusters"].get(school, ""),
+            },
+
+            # 二级维度
+            "dimensions": {},
+
+            # 三级维度
+            "sub_dimensions": {},
+
+            # 相关性矩阵
+            "correlation": correlation.to_dict(),
+
+            # 所有学校得分（用于对比）
+            "all_schools_dim_scores": dim_scores.to_dict(),
+            "all_schools_sub_scores": sub_scores.to_dict(),
+        }
+
+        # 填充二级维度数据
+        for dim in DIMENSION_FRAMEWORK:
+            score = float(dim_scores.loc[school, dim])
+            d_avg = float(district_avg_dim[dim])
+            st_avg = float(same_type_avg_dim.get(dim, PCA_MEAN))
+
+            # T检验：该学校在该维度下各三级维度得分 vs 区均值
+            sub_dims = DIMENSION_FRAMEWORK[dim]["sub_dimensions"]
+            sub_vals = np.array([float(sub_scores.loc[school, s]) for s in sub_dims if s in sub_scores.columns])
+            t_test = self.t_test_vs_mean(sub_vals, d_avg)
+
+            report["dimensions"][dim] = {
+                "score": round(score, 2),
+                "district_avg": round(d_avg, 2),
+                "same_type_avg": round(st_avg, 2),
+                "diff_district": round(score - d_avg, 2),
+                "rank_in_district": int((dim_scores[dim] >= score).sum()),
+                "t_test_vs_district": t_test,
+                "cluster": dim_clusters.get(dim, {}).get("school_clusters", {}).get(school, ""),
+            }
+
+        # 填充三级维度数据
+        for dim, info in DIMENSION_FRAMEWORK.items():
+            for sub_dim in info["sub_dimensions"]:
+                if sub_dim not in sub_scores.columns:
+                    continue
+                score = float(sub_scores.loc[school, sub_dim])
+                d_avg = float(district_avg_sub[sub_dim])
+                level = int(levels.loc[school, sub_dim])
+
+                # 各学校在该维度的排名
+                rank = int((sub_scores[sub_dim] >= score).sum())
+
+                # 水平分布统计
+                dim_levels = levels[sub_dim]
+                level_dist = {
+                    f"水平{i}": int((dim_levels == i).sum())
+                    for i in range(1, 5)
+                }
+
+                report["sub_dimensions"][sub_dim] = {
+                    "parent_dimension": dim,
+                    "score": round(score, 2),
+                    "district_avg": round(d_avg, 2),
+                    "diff_district": round(score - d_avg, 2),
+                    "rank_in_district": rank,
+                    "level": level,
+                    "level_description": self.get_level_description(sub_dim, level),
+                    "level_distribution": level_dist,
+                }
+
+        return report