diff --git a/admin-tools/crawler/.gitignore b/admin-tools/crawler/.gitignore new file mode 100644 index 0000000..986aaf3 --- /dev/null +++ b/admin-tools/crawler/.gitignore @@ -0,0 +1,7 @@ +.pytest_cache/ +.venv/ +__pycache__/ +*.egg-info/ +build/ +dist/ +output/ diff --git a/admin-tools/crawler/README.md b/admin-tools/crawler/README.md new file mode 100644 index 0000000..6893b11 --- /dev/null +++ b/admin-tools/crawler/README.md @@ -0,0 +1,108 @@ +# 产教数据连接器运行时 + +本目录实现厂商端 9 个目标来源的统一采集骨架。所有外部来源默认禁止网络访问,只有完成书面授权、robots、域名白名单、技术联系人和请求限额配置后,命令行才允许发起请求。 + +逐来源决策见 [`../../docs/crawler-source-implementation.md`](../../docs/crawler-source-implementation.md),统一字段见 [`../demo/schemas/`](../demo/schemas/)。 + +## 能力 + +- 7 个外部来源的具体连接器类; +- 2 个校内来源的租户隔离文件连接器; +- 官方 API / 授权 JSON、NDJSON、CSV 数据包接入; +- 获准公开页面的 robots、域名、重定向、限速、响应大小和 DNS 重绑定保护; +- Schema.org `JobPosting` 与结构化 feed 解析; +- `JOB-SIGNAL V3.0`、`SCHOOL-EMPLOYMENT V2.2`、`SCHOOL-INTERNSHIP V2.1` 校验; +- Raw 快照、标准记录、隔离记录和运行摘要输出; +- URL 密钥脱敏、敏感校内字段拒绝和确定性幂等标识。 + +不支持验证码、登录自动化、Cookie 复用、代理池、浏览器指纹伪装、私有接口逆向或任何访问限制绕过。 + +## 安装与测试 + +```bash +cd admin-tools/crawler +python -m venv .venv +.venv/bin/pip install -e '.[test]' +.venv/bin/pytest +``` + +当前环境已安装依赖时,也可直接运行: + +```bash +PYTHONPATH=src python -m edu_crawler list-sources +PYTHONPATH=src python -m pytest +``` + +## 离线授权数据包 + +商业来源推荐先使用离线授权数据包验证字段映射: + +```bash +PYTHONPATH=src python -m edu_crawler run \ + --source BOSS \ + --input /secure/inbox/boss-authorized.ndjson \ + --authorization-ref CONTRACT-2026-001 \ + --task-id JOB-20260824-BOSS-001 \ + --data-version V2026.08.24 \ + --output ./output +``` + +本地输入不会发起网络请求,但 7 个外部来源仍要求授权编号。 + +## 获准网络来源 + +只有部署审批完成后才能显式增加 `--allow-network`: + +```bash +PYTHONPATH=src python -m edu_crawler run \ + --source BRZP \ + --input 'https://approved.example/jobs-feed.json' \ + --approved-host approved.example \ + --authorization-ref AUTH-BRZP-2026-001 \ + --contact crawler-ops@example.edu.cn \ + --allow-network \ + --task-id JOB-20260824-BRZP-001 \ + --data-version V2026.08.24 \ + --max-pages 5 \ + --max-records 1000 \ + --output ./output +``` + +- `--approved-host` 必须来自来源审批配置,不能使用 IP; +- `--approved-host` 还必须存在于对应连接器的版本化 allowlist;新增合作 API 域名需要代码评审; +- API Bearer Token 通过 `--bearer-token-env ENV_NAME` 引用环境变量; +- Bearer Token 任务只允许一个审批主机,Token 不发送给 robots.txt; +- 命令输出、任务摘要和 Raw 元数据均不保存 Token; +- 实际请求间隔不会低于来源策略的最低值。 + +## 校内来源 + +校内文件必须先在学校侧完成学生令牌化: + +```bash +PYTHONPATH=src python -m edu_crawler run \ + --source SCHOOL_EMPLOYMENT \ + --input /secure/inbox/employment.ndjson \ + --authorization-ref DPA-TENANT-HZVTC-2026-001 \ + --tenant-id TENANT-HZVTC \ + --task-id JOB-20260824-SCHOOL-EMP-001 \ + --data-version SCH-EMP-20260824-01 \ + --output ./output +``` + +如果输入包含 `student_name`、`student_id`、身份证、手机号、邮箱等禁止字段,任务会将记录写入隔离区,不会进入标准层。 + +## 输出 + +```text +外部来源:output/// +校内来源:output//// +├── raw/ +│ ├── . +│ └── .metadata.json +├── standard.ndjson +├── quarantine.ndjson +└── summary.json +``` + +任务目录不可覆盖。重跑必须使用新任务号,业务记录的 `record_id` 和 `content_hash` 保持确定性。 diff --git a/admin-tools/crawler/pyproject.toml b/admin-tools/crawler/pyproject.toml new file mode 100644 index 0000000..a70e523 --- /dev/null +++ b/admin-tools/crawler/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "edu-industry-crawler" +version = "0.1.0" +description = "Policy-gated recruitment and school data connectors for the edu-industry platform" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "beautifulsoup4>=4.12,<5", + "httpcore>=1,<2", + "httpx>=0.27,<1", + "jsonschema>=4.20,<5", +] + +[project.optional-dependencies] +test = ["pytest>=8,<10"] + +[project.scripts] +edu-industry-crawler = "edu_crawler.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +edu_crawler = ["schemas/*.json"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/admin-tools/crawler/src/edu_crawler/__init__.py b/admin-tools/crawler/src/edu_crawler/__init__.py new file mode 100644 index 0000000..60dd76f --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/__init__.py @@ -0,0 +1,7 @@ +"""Policy-gated data connectors for the edu-industry platform.""" + +from .policies import SOURCE_POLICIES, SourcePolicy +from .runner import CrawlerRunner, RunConfig + +__all__ = ["CrawlerRunner", "RunConfig", "SOURCE_POLICIES", "SourcePolicy"] +__version__ = "0.1.0" diff --git a/admin-tools/crawler/src/edu_crawler/__main__.py b/admin-tools/crawler/src/edu_crawler/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/admin-tools/crawler/src/edu_crawler/cli.py b/admin-tools/crawler/src/edu_crawler/cli.py new file mode 100644 index 0000000..799ef7a --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/cli.py @@ -0,0 +1,95 @@ +"""Command-line interface.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from .errors import CrawlerError +from .policies import SOURCE_POLICIES +from .runner import CrawlerRunner, RunConfig + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser(prog="edu-industry-crawler") + commands = root.add_subparsers(dest="command", required=True) + commands.add_parser("list-sources", help="List registered source policies") + run = commands.add_parser("run", help="Run one policy-gated connector task") + run.add_argument("--source", required=True, choices=tuple(SOURCE_POLICIES)) + run.add_argument("--input", required=True, dest="input_location") + run.add_argument("--task-id", required=True) + run.add_argument("--data-version", required=True) + run.add_argument( + "--output", type=Path, default=Path("./output"), dest="output_root" + ) + run.add_argument("--authorization-ref", dest="authorization_reference") + run.add_argument("--allow-network", action="store_true") + run.add_argument( + "--approved-host", action="append", default=[], dest="approved_hosts" + ) + run.add_argument("--contact") + run.add_argument("--bearer-token-env") + run.add_argument("--max-pages", type=int) + run.add_argument("--max-records", type=int) + run.add_argument("--default-region-code") + run.add_argument("--tenant-id") + return root + + +def _list_sources() -> int: + for policy in SOURCE_POLICIES.values(): + print( + json.dumps( + { + "code": policy.code, + "name": policy.display_name, + "kind": policy.kind, + "access_mode": policy.access_mode, + "target_schema": policy.target_schema, + "network_default": "disabled", + "authorization_required": policy.authorization_required, + }, + ensure_ascii=False, + sort_keys=True, + ) + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + arguments = parser().parse_args(argv) + if arguments.command == "list-sources": + return _list_sources() + config = RunConfig( + source_code=arguments.source, + input_location=arguments.input_location, + task_id=arguments.task_id, + data_version=arguments.data_version, + output_root=arguments.output_root, + authorization_reference=arguments.authorization_reference, + allow_network=arguments.allow_network, + approved_hosts=tuple(arguments.approved_hosts), + contact=arguments.contact, + bearer_token_env=arguments.bearer_token_env, + max_pages=arguments.max_pages, + max_records=arguments.max_records, + default_region_code=arguments.default_region_code, + tenant_id=arguments.tenant_id, + ) + try: + output = CrawlerRunner(config).run() + except CrawlerError as exc: + print( + json.dumps( + {"status": "FAILED", "error_code": exc.code, "error": str(exc)}, + ensure_ascii=False, + ), + file=sys.stderr, + ) + return 2 + print( + json.dumps({"status": "COMPLETED", "output": str(output)}, ensure_ascii=False) + ) + return 0 diff --git a/admin-tools/crawler/src/edu_crawler/connectors/__init__.py b/admin-tools/crawler/src/edu_crawler/connectors/__init__.py new file mode 100644 index 0000000..e3043ce --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/connectors/__init__.py @@ -0,0 +1,5 @@ +"""Connector registry.""" + +from .sources import CONNECTOR_TYPES, connector_for + +__all__ = ["CONNECTOR_TYPES", "connector_for"] diff --git a/admin-tools/crawler/src/edu_crawler/connectors/base.py b/admin-tools/crawler/src/edu_crawler/connectors/base.py new file mode 100644 index 0000000..2e7ed99 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/connectors/base.py @@ -0,0 +1,34 @@ +"""Connector service-provider interface.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from ..models import ParsedItem, SourcePolicy + + +class BaseConnector(ABC): + def __init__(self, policy: SourcePolicy) -> None: + self.policy = policy + + def describe(self) -> dict: + return { + "source_code": self.policy.code, + "display_name": self.policy.display_name, + "kind": self.policy.kind, + "access_mode": self.policy.access_mode, + "connector_version": self.policy.connector_version, + "target_schema": self.policy.target_schema, + "network_default": "disabled", + } + + @abstractmethod + def normalize( + self, + item: ParsedItem, + *, + task_id: str, + data_version: str, + default_region_code: str | None, + ) -> dict: + """Map one parsed source item to the target contract.""" diff --git a/admin-tools/crawler/src/edu_crawler/connectors/external.py b/admin-tools/crawler/src/edu_crawler/connectors/external.py new file mode 100644 index 0000000..1c7132c --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/connectors/external.py @@ -0,0 +1,447 @@ +"""External recruitment source normalization.""" + +from __future__ import annotations + +import re +import uuid +from datetime import datetime +from typing import ClassVar +from urllib.parse import urlsplit + +from ..errors import SourceStructureChanged +from ..hashing import canonical_content_hash +from ..models import ParsedItem +from ..parsing import strip_markup +from ..security import host_allowed, redact_url, safe_record_reference +from .base import BaseConnector + +RECORD_NAMESPACE = uuid.UUID("4afad1d4-4373-43b0-b722-77b7b3aab62c") +CONTACT_RE = re.compile( + r"(?:[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}|(? object: + value: object = record + for part in dotted.split("."): + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _first(record: dict, aliases: tuple[str, ...]) -> object: + for alias in aliases: + value = _path(record, alias) + if value not in (None, "", [], {}): + return value + return None + + +def _identifier(value: object) -> str: + return safe_record_reference(value) or "" + + +def _scalar(value: object) -> object: + if isinstance(value, list): + return value[0] if value else None + return value + + +def _number(value: object) -> float | None: + value = _scalar(value) + if value in (None, ""): + return None + if isinstance(value, dict): + value = value.get("value") or value.get("minValue") or value.get("maxValue") + text = str(value).replace(",", "").strip() + match = re.search(r"-?\d+(?:\.\d+)?", text) + return float(match.group()) if match else None + + +def _integer(value: object) -> int | None: + number = _number(value) + return int(number) if number is not None else None + + +def _string_list(value: object) -> list[str]: + if value in (None, ""): + return [] + values = value if isinstance(value, list) else re.split(r"[|,,;/;]+", str(value)) + cleaned = [strip_markup(item) for item in values] + return list(dict.fromkeys(item for item in cleaned if item))[:200] + + +def _datetime(value: object) -> str | None: + value = _scalar(value) + if not value: + return None + text = str(value).strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return parsed.isoformat(timespec="seconds") + + +def _status(value: object) -> str: + value = _scalar(value) + normalized = str(value or "").strip().lower() + if normalized in {"active", "online", "open", "published", "招聘中", "有效"}: + return "active" + if normalized in {"inactive", "offline", "closed", "下线", "失效"}: + return "inactive" + if normalized in {"expired", "过期", "已结束"}: + return "expired" + return "unknown" + + +def _employment_type(value: object) -> str: + value = _scalar(value) + normalized = str(value or "").strip().lower() + mappings = { + "full_time": "full_time", + "full-time": "full_time", + "全职": "full_time", + "internship": "internship", + "实习": "internship", + "part_time": "part_time", + "part-time": "part_time", + "兼职": "part_time", + "contract": "contract", + "合同": "contract", + "campus": "campus", + "校招": "campus", + } + return mappings.get(normalized, "other") + + +def _education(value: object) -> str | None: + value = _scalar(value) + normalized = str(value or "").strip().lower() + mappings = { + "不限": "none", + "none": "none", + "中专": "secondary", + "高中": "secondary", + "secondary": "secondary", + "职高": "vocational", + "vocational": "vocational", + "大专": "associate", + "专科": "associate", + "associate": "associate", + "本科": "bachelor", + "bachelor": "bachelor", + "硕士": "master", + "master": "master", + "博士": "doctor", + "doctor": "doctor", + "unknown": "unknown", + } + return mappings.get(normalized) + + +def _salary_period(value: object) -> str | None: + normalized = str(_scalar(value) or "").strip().lower() + mappings = { + "month": "month", + "monthly": "month", + "月": "month", + "月薪": "month", + "year": "year", + "yearly": "year", + "年": "year", + "年薪": "year", + "day": "day", + "daily": "day", + "日": "day", + "日薪": "day", + "hour": "hour", + "hourly": "hour", + "小时": "hour", + "时薪": "hour", + "negotiable": "negotiable", + "面议": "negotiable", + "unknown": "unknown", + } + return mappings.get(normalized) + + +def _schema_org_salary( + record: dict, +) -> tuple[float | None, float | None, str | None, str | None]: + salary = record.get("baseSalary") + if not isinstance(salary, dict): + return None, None, None, None + currency = strip_markup(salary.get("currency")) or None + value = salary.get("value") + if isinstance(value, dict): + minimum = _number(value.get("minValue") or value.get("value")) + maximum = _number(value.get("maxValue") or value.get("value")) + unit = strip_markup(value.get("unitText")).lower() + else: + minimum = maximum = _number(value) + unit = "" + period = { + "month": "month", + "monthly": "month", + "year": "year", + "yearly": "year", + "day": "day", + "daily": "day", + "hour": "hour", + "hourly": "hour", + }.get(unit) + return minimum, maximum, currency, period + + +def _monthly_salary(value: float | None, period: str | None) -> float | None: + if value is None: + return None + factors = { + "month": 1.0, + "year": 1 / 12, + "day": 21.75, + "hour": 174.0, + } + factor = factors.get(period or "month") + return round(value * factor, 2) if factor is not None else value + + +class ExternalJobConnector(BaseConnector): + aliases: ClassVar[dict[str, tuple[str, ...]]] = {} + + common_aliases: ClassVar[dict[str, tuple[str, ...]]] = { + "source_record_id": ( + "source_record_id", + "id", + "identifier", + "job_id", + "jobId", + "@id", + ), + "source_url": ("source_url", "url", "job_url", "jobUrl"), + "job_title_raw": ( + "job_title_raw", + "title", + "name", + "job_title", + "jobName", + "position_name", + ), + "job_description": ( + "job_description", + "description", + "responsibilities", + "job_description_html", + ), + "enterprise_name": ( + "enterprise_name", + "company_name", + "companyName", + "hiringOrganization.name", + "organization.name", + ), + "region_code": ( + "region_code", + "city_code", + "cityCode", + "area_code", + "address.region_code", + ), + "workplace": ( + "workplace", + "jobLocation.address.streetAddress", + "jobLocation.address.addressLocality", + "address", + ), + "employment_type": ("employment_type", "employmentType", "job_type", "jobType"), + "published_at": ("published_at", "datePosted", "publish_time", "publishTime"), + "status": ("status", "job_status", "jobStatus"), + "recruitment_count": ("recruitment_count", "headcount", "number_of_positions"), + "education_level": ("education_level", "education", "educationRequirements"), + "experience_min_years": ( + "experience_min_years", + "experience_years", + "experienceRequirements", + ), + "salary_min": ("salary_min", "salaryMin", "min_salary"), + "salary_max": ("salary_max", "salaryMax", "max_salary"), + "salary_currency": ("salary_currency", "salaryCurrency", "currency"), + "salary_period": ("salary_period", "salaryPeriod"), + "skills": ("skills", "skill_tags", "skillTags"), + "benefits": ("benefits", "jobBenefits", "benefit_tags"), + "keywords": ("keywords", "tags"), + } + + def _aliases(self, field: str) -> tuple[str, ...]: + return (*self.aliases.get(field, ()), *self.common_aliases[field]) + + def normalize( + self, + item: ParsedItem, + *, + task_id: str, + data_version: str, + default_region_code: str | None, + ) -> dict: + source = item.value + declared_source = strip_markup(source.get("source_platform")).upper() + if declared_source and declared_source != self.policy.code: + raise SourceStructureChanged( + "Source record platform does not match the selected connector" + ) + source_record_id = _identifier( + _first(source, self._aliases("source_record_id")) + ) + source_url = strip_markup(_first(source, self._aliases("source_url"))) + if not source_url and urlsplit(item.source_url).scheme in {"http", "https"}: + source_url = item.source_url + title = strip_markup(_first(source, self._aliases("job_title_raw"))) + description = strip_markup(_first(source, self._aliases("job_description"))) + enterprise = strip_markup(_first(source, self._aliases("enterprise_name"))) + region_code = ( + strip_markup(_first(source, self._aliases("region_code"))) + or default_region_code + or "" + ) + if not source_record_id: + raise SourceStructureChanged("Source record has no stable source_record_id") + if not source_url: + raise SourceStructureChanged("Source record has no traceable source_url") + source_parts = urlsplit(source_url) + if source_parts.scheme not in {"http", "https"} or not source_parts.hostname: + raise SourceStructureChanged("Source record URL must use HTTP or HTTPS") + if source_parts.username or source_parts.password: + raise SourceStructureChanged( + "Source record URL must not contain credentials" + ) + if not host_allowed(source_parts.hostname, self.policy.allowed_hosts): + raise SourceStructureChanged( + "Source record URL host is outside the versioned source allowlist" + ) + source_url = redact_url(source_url) + if not title or not description or not enterprise: + raise SourceStructureChanged( + "Source record is missing title, description or enterprise" + ) + if not re.fullmatch(r"\d{6}", region_code): + raise SourceStructureChanged( + "Source record has no valid six-digit region_code" + ) + description = CONTACT_RE.sub("[CONTACT_REDACTED]", description)[:30_000] + salary_min = _number(_first(source, self._aliases("salary_min"))) + salary_max = _number(_first(source, self._aliases("salary_max"))) + salary_currency = ( + strip_markup(_first(source, self._aliases("salary_currency"))).upper() + or None + ) + salary_period = _salary_period(_first(source, self._aliases("salary_period"))) + if salary_min is None and salary_max is None: + salary_min, salary_max, schema_currency, schema_period = _schema_org_salary( + source + ) + salary_currency = salary_currency or ( + schema_currency.upper() if schema_currency else None + ) + salary_period = salary_period or schema_period + if salary_min is not None or salary_max is not None: + if salary_period not in {"month", "year", "day", "hour"}: + raise SourceStructureChanged( + "Numeric salary has no supported conversion period" + ) + if ( + salary_min is not None + and salary_max is not None + and salary_max < salary_min + ): + raise SourceStructureChanged("Source salary maximum is lower than minimum") + if salary_min is not None or salary_max is not None: + salary_currency = salary_currency or "CNY" + salary_min = _monthly_salary(salary_min, salary_period) + salary_max = _monthly_salary(salary_max, salary_period) + record: dict = { + "record_id": str( + uuid.uuid5(RECORD_NAMESPACE, f"{self.policy.code}:{source_record_id}") + ), + "source_platform": self.policy.code, + "source_record_id": source_record_id[:256], + "source_url": source_url[:2048], + "collector_task_id": task_id, + "collector_version": self.policy.connector_version, + "raw_snapshot_id": item.raw_snapshot_id, + "collected_at": item.collected_at, + "first_seen_at": _datetime(source.get("first_seen_at")) + or item.collected_at, + "last_seen_at": _datetime(source.get("last_seen_at")) or item.collected_at, + "published_at": _datetime(_first(source, self._aliases("published_at"))), + "status": _status(_first(source, self._aliases("status"))), + "job_title_raw": title[:200], + "job_title_normalized": strip_markup(source.get("job_title_normalized"))[ + :200 + ] + or None, + "job_code": strip_markup(source.get("job_code"))[:32] or None, + "job_description": description, + "enterprise_name": enterprise[:300], + "enterprise_id": strip_markup(source.get("enterprise_id"))[:128] or None, + "enterprise_credit_code": strip_markup(source.get("enterprise_credit_code")) + or None, + "enterprise_industry_code": strip_markup( + source.get("enterprise_industry_code") + )[:32] + or None, + "enterprise_scale": source.get("enterprise_scale") + if source.get("enterprise_scale") + in {"micro", "small", "medium", "large", "unknown"} + else None, + "region_code": region_code, + "workplace": strip_markup(_first(source, self._aliases("workplace")))[:500] + or None, + "employment_type": _employment_type( + _first(source, self._aliases("employment_type")) + ), + "recruitment_count": _integer( + _first(source, self._aliases("recruitment_count")) + ), + "education_level": _education( + _first(source, self._aliases("education_level")) + ), + "experience_min_years": _number( + _first(source, self._aliases("experience_min_years")) + ), + "salary_min": salary_min, + "salary_max": salary_max, + "salary_currency": salary_currency, + "salary_period": salary_period, + "benefits": _string_list(_first(source, self._aliases("benefits")))[:100], + "skills": _string_list(_first(source, self._aliases("skills"))), + "keywords": _string_list(_first(source, self._aliases("keywords"))), + "source_payload_hash": item.source_payload_hash, + "parsing_version": f"{self.policy.code}-Parser/0.1.0", + "normalized_at": item.collected_at, + "data_version": data_version, + } + record = {key: value for key, value in record.items() if value is not None} + record["content_hash"] = canonical_content_hash( + record, + { + "record_id", + "source_platform", + "source_record_id", + "source_url", + "collector_task_id", + "collector_version", + "raw_snapshot_id", + "collected_at", + "first_seen_at", + "last_seen_at", + "source_payload_hash", + "parsing_version", + "normalized_at", + "data_version", + }, + ) + return record diff --git a/admin-tools/crawler/src/edu_crawler/connectors/school.py b/admin-tools/crawler/src/edu_crawler/connectors/school.py new file mode 100644 index 0000000..9b1d937 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/connectors/school.py @@ -0,0 +1,69 @@ +"""Tenant-isolated school system connectors.""" + +from __future__ import annotations + +import uuid + +from ..errors import PiiRejected, SourceStructureChanged +from ..hashing import canonical_content_hash +from ..models import ParsedItem +from ..security import prohibited_school_keys +from .base import BaseConnector + +SCHOOL_NAMESPACE = uuid.UUID("6a512614-2321-4b54-9141-c24a85d6c780") + + +class SchoolConnector(BaseConnector): + def normalize( + self, + item: ParsedItem, + *, + task_id: str, + data_version: str, + default_region_code: str | None, + ) -> dict: + del task_id, default_region_code + prohibited = prohibited_school_keys(item.value) + if prohibited: + raise PiiRejected( + f"School record contains prohibited fields: {', '.join(prohibited)}" + ) + source_record_id = str(item.value.get("source_record_id") or "").strip() + tenant_id = str(item.value.get("tenant_id") or "").strip() + if not source_record_id or not tenant_id: + raise SourceStructureChanged( + "School record requires tenant_id and source_record_id" + ) + record = dict(item.value) + record["record_id"] = record.get("record_id") or str( + uuid.uuid5( + SCHOOL_NAMESPACE, f"{self.policy.code}:{tenant_id}:{source_record_id}" + ) + ) + record["data_version"] = data_version + record["created_at"] = record.get("created_at") or item.collected_at + record["updated_at"] = ( + record.get("updated_at") + or record.get("source_updated_at") + or item.collected_at + ) + record["content_hash"] = canonical_content_hash( + record, + { + "record_id", + "content_hash", + "data_version", + "created_at", + "updated_at", + "source_updated_at", + }, + ) + return record + + +class SchoolEmploymentConnector(SchoolConnector): + pass + + +class SchoolInternshipConnector(SchoolConnector): + pass diff --git a/admin-tools/crawler/src/edu_crawler/connectors/sources.py b/admin-tools/crawler/src/edu_crawler/connectors/sources.py new file mode 100644 index 0000000..114c979 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/connectors/sources.py @@ -0,0 +1,90 @@ +"""Concrete connector types for all registered sources.""" + +from __future__ import annotations + +from ..policies import source_policy +from .base import BaseConnector +from .external import ExternalJobConnector +from .school import SchoolEmploymentConnector, SchoolInternshipConnector + + +class BossConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("job_number",), + "job_title_raw": ("job_name",), + "enterprise_name": ("brand_name",), + } + + +class ZhaopinConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("position_number",), + "job_title_raw": ("position_name",), + "enterprise_name": ("company.name",), + "region_code": ("work_city_code",), + } + + +class BrzpConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("job_number",), + "job_title_raw": ("job_name",), + "enterprise_name": ("company.name",), + } + + +class NcssConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("position_id",), + "job_title_raw": ("position_name",), + "enterprise_name": ("unit_name",), + } + + +class MohrssConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("job_info_id",), + "job_title_raw": ("position_name",), + "enterprise_name": ("organization_name",), + } + + +class IguopinConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("position_id",), + "job_title_raw": ("position_name",), + "enterprise_name": ("organization_name",), + } + + +class HzrcConnector(ExternalJobConnector): + aliases = { + "source_record_id": ("position_id",), + "job_title_raw": ("position_name",), + "enterprise_name": ("unit_name",), + } + + +CONNECTOR_TYPES: dict[str, type[BaseConnector]] = { + "BOSS": BossConnector, + "ZHAOPIN": ZhaopinConnector, + "BRZP": BrzpConnector, + "NCSS": NcssConnector, + "MOHRSS": MohrssConnector, + "IGUOPIN": IguopinConnector, + "HZRC": HzrcConnector, + "SCHOOL_EMPLOYMENT": SchoolEmploymentConnector, + "SCHOOL_INTERNSHIP": SchoolInternshipConnector, +} + + +def connector_for(code: str) -> BaseConnector: + normalized = code.upper() + try: + connector_type = CONNECTOR_TYPES[normalized] + except KeyError as exc: + supported = ", ".join(CONNECTOR_TYPES) + raise ValueError( + f"Unknown source {code!r}; supported sources: {supported}" + ) from exc + return connector_type(source_policy(normalized)) diff --git a/admin-tools/crawler/src/edu_crawler/errors.py b/admin-tools/crawler/src/edu_crawler/errors.py new file mode 100644 index 0000000..1729298 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/errors.py @@ -0,0 +1,53 @@ +"""Domain errors with stable machine-readable codes.""" + + +class CrawlerError(Exception): + code = "CRAWLER_ERROR" + + +class ConfigurationError(CrawlerError): + code = "CONFIGURATION_ERROR" + + +class AuthorizationRequired(CrawlerError): + code = "AUTHORIZATION_REQUIRED" + + +class RobotsDenied(CrawlerError): + code = "ROBOTS_DENIED" + + +class HostNotAllowed(CrawlerError): + code = "HOST_NOT_ALLOWED" + + +class PrivateAddressBlocked(CrawlerError): + code = "PRIVATE_ADDRESS_BLOCKED" + + +class AuthOrChallenge(CrawlerError): + code = "AUTH_OR_CHALLENGE" + + +class RateLimited(CrawlerError): + code = "RATE_LIMITED" + + +class ResponseTooLarge(CrawlerError): + code = "RESPONSE_TOO_LARGE" + + +class SourceStructureChanged(CrawlerError): + code = "SOURCE_STRUCTURE_CHANGED" + + +class SchemaValidationFailed(CrawlerError): + code = "SCHEMA_VALIDATION_FAILED" + + +class PiiRejected(CrawlerError): + code = "PII_REJECTED" + + +class TaskOutputExists(CrawlerError): + code = "TASK_OUTPUT_EXISTS" diff --git a/admin-tools/crawler/src/edu_crawler/hashing.py b/admin-tools/crawler/src/edu_crawler/hashing.py new file mode 100644 index 0000000..63fa105 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/hashing.py @@ -0,0 +1,17 @@ +"""Canonical business-content hashing.""" + +from __future__ import annotations + +import json +from collections.abc import Collection +from hashlib import sha256 + + +def canonical_content_hash(record: dict, excluded_fields: Collection[str]) -> str: + canonical = { + key: value for key, value in record.items() if key not in excluded_fields + } + payload = json.dumps( + canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode() + return sha256(payload).hexdigest() diff --git a/admin-tools/crawler/src/edu_crawler/http.py b/admin-tools/crawler/src/edu_crawler/http.py new file mode 100644 index 0000000..e2e7605 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/http.py @@ -0,0 +1,229 @@ +"""HTTP client with source policy, robots and redirect enforcement.""" + +from __future__ import annotations + +import os +import re +import socket +import time +from collections.abc import Callable +from urllib.parse import urljoin, urlsplit +from urllib.robotparser import RobotFileParser + +import httpx + +from .errors import ( + AuthOrChallenge, + ConfigurationError, + RateLimited, + ResponseTooLarge, + RobotsDenied, + SourceStructureChanged, +) +from .models import RawDocument, SourcePolicy +from .security import ( + contains_challenge, + normalize_host, + redact_url, + validate_contact, + validate_public_url, +) +from .transport import PinnedHTTPTransport + + +class SafeHttpClient: + def __init__( + self, + policy: SourcePolicy, + *, + contact: str, + approved_hosts: tuple[str, ...] = (), + bearer_token_env: str | None = None, + transport: httpx.BaseTransport | None = None, + resolver: Callable[..., list[tuple]] | None = None, + ) -> None: + self.policy = policy + self.contact = validate_contact(contact) + if not approved_hosts: + raise ConfigurationError( + "At least one explicitly approved source host is required" + ) + policy_hosts = {normalize_host(host) for host in policy.allowed_hosts} + unexpected_hosts = sorted( + {normalize_host(host) for host in approved_hosts} - policy_hosts + ) + if unexpected_hosts: + raise ConfigurationError( + f"Approved host is not in the versioned {policy.code} allowlist: {', '.join(unexpected_hosts)}" + ) + self.allowed_hosts = tuple(dict.fromkeys(approved_hosts)) + self.user_agent = ( + f"EduIndustryCollector/0.1 (+mailto:{self.contact}; source={policy.code})" + ) + self._resolver = resolver + self._last_request_at = 0.0 + self._robots: dict[str, RobotFileParser] = {} + headers = { + "Accept": "application/json, application/ld+json, text/html;q=0.8", + "User-Agent": self.user_agent, + } + self._bearer_token: str | None = None + if bearer_token_env: + if not re.fullmatch(r"[A-Z][A-Z0-9_]{1,127}", bearer_token_env): + raise ConfigurationError( + "Bearer token environment variable name is invalid" + ) + token = os.environ.get(bearer_token_env) + if not token: + raise ConfigurationError( + f"Bearer token environment variable is not set: {bearer_token_env}" + ) + if len(self.allowed_hosts) != 1: + raise ConfigurationError( + "Bearer token tasks must approve exactly one source host" + ) + self._bearer_token = token + self._client = httpx.Client( + headers=headers, + follow_redirects=False, + timeout=httpx.Timeout(30.0, connect=10.0), + transport=transport or PinnedHTTPTransport(resolver or socket.getaddrinfo), + trust_env=False, + ) + + def __enter__(self) -> "SafeHttpClient": + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def close(self) -> None: + self._client.close() + + def _validate(self, url: str) -> str: + kwargs = {"resolver": self._resolver} if self._resolver else {} + return validate_public_url(url, self.allowed_hosts, **kwargs) + + def _wait_for_rate_limit(self) -> None: + elapsed = time.monotonic() - self._last_request_at + remaining = self.policy.minimum_interval_seconds - elapsed + if remaining > 0: + time.sleep(remaining) + + def _request( + self, + url: str, + *, + allow_not_found: bool = False, + authorization_host: str | None = None, + ) -> tuple[httpx.Response, bytes]: + current = self._validate(url) + for _ in range(6): + self._wait_for_rate_limit() + current_host = normalize_host(urlsplit(current).hostname or "") + request_headers = {} + if self._bearer_token and authorization_host == current_host: + request_headers["Authorization"] = f"Bearer {self._bearer_token}" + try: + with self._client.stream( + "GET", current, headers=request_headers + ) as response: + self._last_request_at = time.monotonic() + if response.status_code in {301, 302, 303, 307, 308}: + location = response.headers.get("location") + if not location: + raise SourceStructureChanged( + "Source returned a redirect without a location" + ) + current = self._validate(urljoin(current, location)) + continue + if response.status_code == 429: + raise RateLimited( + "Source rate limit was reached; task stopped without bypass" + ) + chunks: list[bytes] = [] + size = 0 + for chunk in response.iter_bytes(): + size += len(chunk) + if size > self.policy.maximum_response_bytes: + raise ResponseTooLarge( + "Source response exceeded the approved size limit" + ) + chunks.append(chunk) + body = b"".join(chunks) + text = body[:100_000].decode( + response.encoding or "utf-8", errors="replace" + ) + if contains_challenge(response.status_code, text): + raise AuthOrChallenge( + "Source requested authentication or access verification; task stopped" + ) + if response.status_code >= 400 and not ( + allow_not_found and response.status_code == 404 + ): + raise SourceStructureChanged( + f"Source returned HTTP {response.status_code}" + ) + return response, body + except httpx.HTTPError as exc: + raise SourceStructureChanged("Source network request failed") from exc + raise SourceStructureChanged("Source exceeded the maximum redirect count") + + def _robots_parser(self, url: str) -> RobotFileParser: + parts = urlsplit(url) + origin = f"{parts.scheme}://{parts.netloc}" + cached = self._robots.get(origin) + if cached: + return cached + robots_url = f"{origin}/robots.txt" + response, body = self._request(robots_url, allow_not_found=True) + parser = RobotFileParser() + parser.set_url(robots_url) + content_type = response.headers.get("content-type", "") + if response.status_code == 404: + parser.parse([]) + elif "text" not in content_type and content_type: + raise RobotsDenied("robots.txt did not return a text response") + else: + parser.parse( + body.decode(response.encoding or "utf-8", errors="replace").splitlines() + ) + self._robots[origin] = parser + return parser + + def fetch(self, url: str) -> RawDocument: + approved_url = self._validate(url) + if self.policy.robots_required and not self._robots_parser( + approved_url + ).can_fetch(self.user_agent, approved_url): + raise RobotsDenied( + f"robots.txt denies the approved URL: {redact_url(approved_url)}" + ) + authorization_host = ( + normalize_host(urlsplit(approved_url).hostname or "") + if self._bearer_token + else None + ) + response, body = self._request( + approved_url, authorization_host=authorization_host + ) + content_type = ( + response.headers.get("content-type", "application/octet-stream") + .split(";", 1)[0] + .strip() + .lower() + ) + if content_type not in { + "application/json", + "application/ld+json", + "application/x-ndjson", + "text/html", + "text/plain", + "text/csv", + }: + raise SourceStructureChanged( + f"Unsupported source content type: {content_type or 'missing'}" + ) + return RawDocument.create( + self.policy.code, redact_url(str(response.url)), content_type, body + ) diff --git a/admin-tools/crawler/src/edu_crawler/models.py b/admin-tools/crawler/src/edu_crawler/models.py new file mode 100644 index 0000000..fb486a5 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/models.py @@ -0,0 +1,81 @@ +"""Shared immutable connector models.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +from typing import Literal + +SourceKind = Literal["commercial", "public", "school"] +AccessMode = Literal["authorized_feed", "public_document", "school_file"] + + +@dataclass(frozen=True) +class SourcePolicy: + code: str + display_name: str + kind: SourceKind + access_mode: AccessMode + connector_version: str + target_schema: str + allowed_hosts: tuple[str, ...] = () + job_link_patterns: tuple[str, ...] = () + authorization_required: bool = True + robots_required: bool = True + minimum_interval_seconds: float = 3.0 + maximum_pages: int = 20 + maximum_records: int = 10_000 + maximum_response_bytes: int = 5 * 1024 * 1024 + + @property + def network_capable(self) -> bool: + return self.kind != "school" + + +@dataclass(frozen=True) +class RawDocument: + source_code: str + source_url: str + content_type: str + body: bytes + collected_at: str + payload_hash: str + snapshot_id: str + + @classmethod + def create( + cls, source_code: str, source_url: str, content_type: str, body: bytes + ) -> "RawDocument": + collected_at = ( + datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + ) + payload_hash = sha256(body).hexdigest() + stamp = collected_at[:10].replace("-", "") + return cls( + source_code=source_code, + source_url=source_url, + content_type=content_type, + body=body, + collected_at=collected_at, + payload_hash=payload_hash, + snapshot_id=f"RAW-{source_code}-{stamp}-{payload_hash[:12].upper()}", + ) + + +@dataclass(frozen=True) +class ParsedItem: + value: dict + raw_snapshot_id: str + source_payload_hash: str + collected_at: str + source_url: str + + +@dataclass(frozen=True) +class QuarantineItem: + source_code: str + raw_snapshot_id: str + source_record_id: str | None + error_code: str + error: str diff --git a/admin-tools/crawler/src/edu_crawler/parsing.py b/admin-tools/crawler/src/edu_crawler/parsing.py new file mode 100644 index 0000000..ca369dd --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/parsing.py @@ -0,0 +1,184 @@ +"""Safe structured-feed and public-document parsing helpers.""" + +from __future__ import annotations + +import csv +import io +import json +import re +from collections.abc import Iterable +from html import unescape +from urllib.parse import urljoin + +from bs4 import BeautifulSoup + +from .errors import SourceStructureChanged +from .models import RawDocument, SourcePolicy + +COLLECTION_KEYS = ("records", "items", "jobs", "results", "rows", "list") + + +def _json_collections(value: object) -> Iterable[dict]: + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + yield item + return + if not isinstance(value, dict): + return + item_type = value.get("@type") + if item_type == "JobPosting" or ( + isinstance(item_type, list) and "JobPosting" in item_type + ): + yield value + return + for key in COLLECTION_KEYS: + nested = value.get(key) + if isinstance(nested, list): + yield from _json_collections(nested) + return + data = value.get("data") + if isinstance(data, (dict, list)): + yielded = list(_json_collections(data)) + if yielded: + yield from yielded + return + if value: + yield value + + +def parse_json_body(body: bytes) -> list[dict]: + try: + text = body.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise SourceStructureChanged("Structured source is not valid UTF-8") from exc + try: + value = json.loads(text) + except json.JSONDecodeError: + records = [] + for line_number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + raise SourceStructureChanged( + f"Invalid NDJSON at line {line_number}" + ) from exc + if not isinstance(item, dict): + raise SourceStructureChanged( + f"NDJSON line {line_number} is not an object" + ) + records.append(item) + if not records: + raise SourceStructureChanged("Structured source returned no JSON records") + return records + records = list(_json_collections(value)) + if not records: + raise SourceStructureChanged("Structured source returned no records") + return records + + +def parse_csv_body(body: bytes) -> list[dict]: + try: + text = body.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise SourceStructureChanged("CSV source is not valid UTF-8") from exc + reader = csv.DictReader(io.StringIO(text)) + if not reader.fieldnames: + raise SourceStructureChanged("CSV source has no header") + records = [dict(row) for row in reader] + if not records: + raise SourceStructureChanged("CSV source returned no records") + return records + + +def _walk_json_ld(value: object) -> Iterable[dict]: + if isinstance(value, list): + for item in value: + yield from _walk_json_ld(item) + elif isinstance(value, dict): + item_type = value.get("@type") + if item_type == "JobPosting" or ( + isinstance(item_type, list) and "JobPosting" in item_type + ): + yield value + graph = value.get("@graph") + if graph is not None: + yield from _walk_json_ld(graph) + + +def parse_html_jobs(body: bytes) -> list[dict]: + soup = BeautifulSoup(body, "html.parser") + jobs: list[dict] = [] + for script in soup.select('script[type="application/ld+json"]'): + try: + value = json.loads(script.get_text(strip=True)) + except json.JSONDecodeError: + continue + jobs.extend(_walk_json_ld(value)) + if jobs: + return jobs + for node in soup.select("[data-job-id][data-job-title]"): + description_node = node.select_one("[data-job-description]") + enterprise_node = node.select_one("[data-enterprise-name]") + jobs.append( + { + "source_record_id": node.get("data-job-id"), + "job_title_raw": node.get("data-job-title"), + "job_description": description_node.get_text(" ", strip=True) + if description_node + else "", + "enterprise_name": enterprise_node.get_text(" ", strip=True) + if enterprise_node + else "", + "source_url": node.get("data-job-url"), + "region_code": node.get("data-region-code"), + } + ) + if not jobs: + raise SourceStructureChanged( + "HTML source has no approved JobPosting or versioned job elements" + ) + return jobs + + +def discover_job_links(document: RawDocument, policy: SourcePolicy) -> list[str]: + if document.content_type != "text/html" or not policy.job_link_patterns: + return [] + soup = BeautifulSoup(document.body, "html.parser") + patterns = [ + re.compile(pattern, re.IGNORECASE) for pattern in policy.job_link_patterns + ] + links: list[str] = [] + for anchor in soup.select("a[href]"): + target = urljoin(document.source_url, str(anchor.get("href"))) + if any(pattern.search(target) for pattern in patterns): + links.append(target) + return list(dict.fromkeys(links)) + + +def parse_document(document: RawDocument, *, allow_html: bool) -> list[dict]: + if document.content_type == "text/csv": + return parse_csv_body(document.body) + if document.content_type in { + "application/json", + "application/ld+json", + "application/x-ndjson", + "text/plain", + }: + return parse_json_body(document.body) + if document.content_type == "text/html" and allow_html: + return parse_html_jobs(document.body) + raise SourceStructureChanged( + f"Source mode does not allow content type {document.content_type}" + ) + + +def strip_markup(value: object) -> str: + if value is None: + return "" + text = unescape(str(value)) + if "<" in text or ">" in text: + text = BeautifulSoup(text, "html.parser").get_text(" ", strip=True) + return re.sub(r"\s+", " ", text).strip() diff --git a/admin-tools/crawler/src/edu_crawler/policies.py b/admin-tools/crawler/src/edu_crawler/policies.py new file mode 100644 index 0000000..8305a84 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/policies.py @@ -0,0 +1,117 @@ +"""Versioned source policy catalog.""" + +from __future__ import annotations + +from .models import SourcePolicy + + +SOURCE_POLICIES: dict[str, SourcePolicy] = { + "BOSS": SourcePolicy( + code="BOSS", + display_name="BOSS直聘", + kind="commercial", + access_mode="authorized_feed", + connector_version="BOSS-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("www.bosszhipin.com", "www.zhipin.com"), + robots_required=False, + minimum_interval_seconds=6.0, + ), + "ZHAOPIN": SourcePolicy( + code="ZHAOPIN", + display_name="智联招聘", + kind="commercial", + access_mode="authorized_feed", + connector_version="ZHAOPIN-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("www.zhaopin.com",), + robots_required=False, + minimum_interval_seconds=6.0, + ), + "BRZP": SourcePolicy( + code="BRZP", + display_name="全国中小企业招聘平台", + kind="public", + access_mode="public_document", + connector_version="BRZP-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("brzp.chinasme.cn", "zrc-sme.chinasme.cn"), + job_link_patterns=(r"/job/", r"/brzp/job/"), + minimum_interval_seconds=3.0, + ), + "NCSS": SourcePolicy( + code="NCSS", + display_name="国家大学生就业服务平台", + kind="public", + access_mode="public_document", + connector_version="NCSS-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("www.ncss.cn", "job.ncss.cn"), + job_link_patterns=(r"/jobs?/", r"job(?:detail|info)", r"/student/.+job"), + minimum_interval_seconds=4.0, + ), + "MOHRSS": SourcePolicy( + code="MOHRSS", + display_name="中国公共招聘网", + kind="public", + access_mode="public_document", + connector_version="MOHRSS-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("job.mohrss.gov.cn", "www.job.mohrss.gov.cn"), + job_link_patterns=(r"/cjobs/.+(?:job|institution)", r"jobinfo"), + minimum_interval_seconds=4.0, + ), + "IGUOPIN": SourcePolicy( + code="IGUOPIN", + display_name="国聘", + kind="public", + access_mode="authorized_feed", + connector_version="IGUOPIN-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("www.iguopin.com", "job.iguopin.com", "xiaoyuan.iguopin.com"), + robots_required=False, + minimum_interval_seconds=5.0, + ), + "HZRC": SourcePolicy( + code="HZRC", + display_name="杭州人才网", + kind="public", + access_mode="public_document", + connector_version="HZRC-Collector/0.1.0", + target_schema="external-job-source.schema.json", + allowed_hosts=("www.hzrc.com", "hzrc.com", "yph.hzrc.com"), + job_link_patterns=(r"(?:job|position|posiiton)",), + minimum_interval_seconds=8.0, + maximum_pages=10, + ), + "SCHOOL_EMPLOYMENT": SourcePolicy( + code="SCHOOL_EMPLOYMENT", + display_name="校内就业管理系统", + kind="school", + access_mode="school_file", + connector_version="School-Employment-Connector/0.1.0", + target_schema="school-employment.schema.json", + robots_required=False, + minimum_interval_seconds=0, + ), + "SCHOOL_INTERNSHIP": SourcePolicy( + code="SCHOOL_INTERNSHIP", + display_name="校内实习管理系统", + kind="school", + access_mode="school_file", + connector_version="School-Internship-Connector/0.1.0", + target_schema="school-internship.schema.json", + robots_required=False, + minimum_interval_seconds=0, + ), +} + + +def source_policy(code: str) -> SourcePolicy: + try: + return SOURCE_POLICIES[code.upper()] + except KeyError as exc: + supported = ", ".join(SOURCE_POLICIES) + raise ValueError( + f"Unknown source {code!r}; supported sources: {supported}" + ) from exc diff --git a/admin-tools/crawler/src/edu_crawler/runner.py b/admin-tools/crawler/src/edu_crawler/runner.py new file mode 100644 index 0000000..97451a6 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/runner.py @@ -0,0 +1,328 @@ +"""End-to-end connector task runner.""" + +from __future__ import annotations + +import re +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlsplit + +from .connectors import connector_for +from .errors import AuthorizationRequired, ConfigurationError, CrawlerError, PiiRejected +from .http import SafeHttpClient +from .models import ParsedItem, QuarantineItem, RawDocument +from .parsing import discover_job_links, parse_document +from .schema import validate_record +from .security import ( + prohibited_school_keys, + prohibited_school_values, + safe_record_reference, +) +from .storage import TaskOutput + +TASK_ID_PATTERN = re.compile(r"^JOB-[A-Za-z0-9-]{8,64}$") +AUTHORIZATION_REFERENCE_PATTERN = re.compile( + r"^(?:AUTH|CONTRACT|LICENSE|APPROVAL|DPA)-[A-Za-z0-9._:/-]{3,118}$" +) + + +@contextmanager +def closing_iterator(iterator): + try: + yield iterator + finally: + close = getattr(iterator, "close", None) + if close: + close() + + +@dataclass(frozen=True) +class RunConfig: + source_code: str + input_location: str + task_id: str + data_version: str + output_root: Path + authorization_reference: str | None = None + allow_network: bool = False + approved_hosts: tuple[str, ...] = () + contact: str | None = None + bearer_token_env: str | None = None + max_pages: int | None = None + max_records: int | None = None + default_region_code: str | None = None + tenant_id: str | None = None + + +class CrawlerRunner: + def __init__(self, config: RunConfig) -> None: + self.config = config + self.connector = connector_for(config.source_code) + self.policy = self.connector.policy + + def _validate_config(self) -> None: + if not TASK_ID_PATTERN.fullmatch(self.config.task_id): + raise ConfigurationError("task_id must match JOB-[A-Za-z0-9-]{8,64}") + if self.policy.authorization_required: + reference = (self.config.authorization_reference or "").strip() + if not reference: + raise AuthorizationRequired( + f"{self.policy.code} requires a non-empty authorization reference" + ) + if not AUTHORIZATION_REFERENCE_PATTERN.fullmatch(reference): + raise ConfigurationError( + "authorization_reference must be a non-secret audit reference" + ) + if self.config.allow_network: + if not self.policy.network_capable: + raise ConfigurationError( + "School connectors do not allow direct network collection" + ) + if not self.config.contact: + raise ConfigurationError( + "Network collection requires a technical contact email" + ) + if not self.config.approved_hosts: + raise ConfigurationError( + "Network collection requires at least one explicitly approved host" + ) + elif urlsplit(self.config.input_location).scheme in {"http", "https"}: + raise ConfigurationError("Network input requires --allow-network") + if self.config.default_region_code and not re.fullmatch( + r"\d{6}", self.config.default_region_code + ): + raise ConfigurationError( + "default_region_code must be a six-digit GB/T 2260 code" + ) + if self.policy.kind == "school": + if not self.config.tenant_id or not re.fullmatch( + r"TENANT-[A-Z0-9-]{4,64}", self.config.tenant_id + ): + raise ConfigurationError( + "School collection requires a valid --tenant-id" + ) + elif self.config.tenant_id: + raise ConfigurationError("tenant_id is only valid for school connectors") + if self.config.max_pages is not None and self.config.max_pages < 1: + raise ConfigurationError("max_pages must be at least 1") + if self.config.max_records is not None and self.config.max_records < 1: + raise ConfigurationError("max_records must be at least 1") + version_patterns = { + "external-job-source.schema.json": r"^V\d{4}\.\d{2}\.\d{2}(?:\.\d+)?$", + "school-employment.schema.json": r"^SCH-EMP-\d{8}(?:-\d{2})?$", + "school-internship.schema.json": r"^SCH-INT-\d{8}(?:-\d{2})?$", + } + if not re.fullmatch( + version_patterns[self.policy.target_schema], self.config.data_version + ): + raise ConfigurationError( + f"data_version is invalid for {self.policy.target_schema}" + ) + + def _local_document(self) -> RawDocument: + path = Path(self.config.input_location).expanduser().resolve() + if not path.is_file(): + raise ConfigurationError(f"Input file does not exist: {path.name}") + content_types = { + ".json": "application/json", + ".jsonl": "application/x-ndjson", + ".ndjson": "application/x-ndjson", + ".csv": "text/csv", + ".html": "text/html", + ".htm": "text/html", + ".txt": "text/plain", + } + content_type = content_types.get(path.suffix.lower()) + if not content_type: + raise ConfigurationError( + "Input file must be JSON, NDJSON, CSV, HTML or text" + ) + with path.open("rb") as stream: + body = stream.read(self.policy.maximum_response_bytes + 1) + if len(body) > self.policy.maximum_response_bytes: + raise ConfigurationError("Local input exceeds the approved size limit") + return RawDocument.create( + self.policy.code, f"file:///{path.name}", content_type, body + ) + + def _network_documents(self): + configured_pages = ( + self.policy.maximum_pages + if self.config.max_pages is None + else self.config.max_pages + ) + maximum_pages = min(configured_pages, self.policy.maximum_pages) + if maximum_pages < 1: + raise ConfigurationError("max_pages must be at least 1") + client = SafeHttpClient( + self.policy, + contact=self.config.contact or "", + approved_hosts=self.config.approved_hosts, + bearer_token_env=self.config.bearer_token_env, + ) + try: + entry = client.fetch(self.config.input_location) + yield entry + if ( + self.policy.access_mode == "public_document" + and entry.content_type == "text/html" + ): + links = discover_job_links(entry, self.policy) + for link in links[: maximum_pages - 1]: + yield client.fetch(link) + finally: + client.close() + + def _documents(self): + if self.config.allow_network: + return self._network_documents() + return iter((self._local_document(),)) + + def run(self) -> Path: + self._validate_config() + configured_records = ( + self.policy.maximum_records + if self.config.max_records is None + else self.config.max_records + ) + maximum_records = min(configured_records, self.policy.maximum_records) + if maximum_records < 1: + raise ConfigurationError("max_records must be at least 1") + started_at = ( + datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + ) + documents = self._documents() + document_count = 0 + standard_count = 0 + quarantine_count = 0 + parsed_count = 0 + with ( + closing_iterator(documents), + TaskOutput( + self.config.output_root, + self.policy.code, + self.config.task_id, + self.config.tenant_id, + ) as output, + ): + for document in documents: + document_count += 1 + source_records: list[dict] | None = None + if self.policy.kind == "school": + source_records = parse_document(document, allow_html=False) + mismatched_tenants = sorted( + { + str(source_record.get("tenant_id") or "") + for source_record in source_records + if source_record.get("tenant_id") != self.config.tenant_id + } + ) + if mismatched_tenants: + raise ConfigurationError( + "School input contains a record outside the configured tenant" + ) + rejected_paths = sorted( + { + key + for source_record in source_records + for key in ( + *prohibited_school_keys(source_record), + *prohibited_school_values(source_record), + ) + } + ) + if rejected_paths: + raise PiiRejected( + f"School input contains prohibited fields: {', '.join(rejected_paths[:10])}" + ) + output.write_raw( + document, + task_id=self.config.task_id, + connector_version=self.policy.connector_version, + ) + if source_records is None: + try: + source_records = parse_document( + document, + allow_html=self.policy.access_mode == "public_document", + ) + except CrawlerError as exc: + output.write_quarantine( + QuarantineItem( + source_code=self.policy.code, + raw_snapshot_id=document.snapshot_id, + source_record_id=None, + error_code=exc.code, + error=str(exc), + ) + ) + quarantine_count += 1 + continue + for source_record in source_records: + if parsed_count >= maximum_records: + break + parsed_count += 1 + item = ParsedItem( + value=source_record, + raw_snapshot_id=document.snapshot_id, + source_payload_hash=document.payload_hash, + collected_at=document.collected_at, + source_url=document.source_url, + ) + try: + normalized = self.connector.normalize( + item, + task_id=self.config.task_id, + data_version=self.config.data_version, + default_region_code=self.config.default_region_code, + ) + validate_record(self.policy.target_schema, normalized) + except CrawlerError as exc: + output.write_quarantine( + QuarantineItem( + source_code=self.policy.code, + raw_snapshot_id=document.snapshot_id, + source_record_id=safe_record_reference( + source_record.get("source_record_id") + or source_record.get("id") + or source_record.get("identifier") + or "" + ), + error_code=exc.code, + error=str(exc), + ) + ) + quarantine_count += 1 + continue + output.write_standard(normalized) + standard_count += 1 + if parsed_count >= maximum_records: + break + finished_at = ( + datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds") + ) + summary = { + "task_id": self.config.task_id, + "source": self.connector.describe(), + "authorization_reference": self.config.authorization_reference, + "network_used": self.config.allow_network, + "input": ( + f"https://{urlsplit(self.config.input_location).hostname}/[APPROVED]" + if self.config.allow_network + else Path(self.config.input_location).name + ), + "data_version": self.config.data_version, + "started_at": started_at, + "finished_at": finished_at, + "tenant_id": self.config.tenant_id, + "raw_documents": document_count, + "parsed_records": parsed_count, + "standard_records": standard_count, + "quarantined_records": quarantine_count, + "status": "COMPLETED" + if quarantine_count == 0 + else "COMPLETED_WITH_QUARANTINE", + } + return output.commit(summary) diff --git a/admin-tools/crawler/src/edu_crawler/schema.py b/admin-tools/crawler/src/edu_crawler/schema.py new file mode 100644 index 0000000..a9c6795 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/schema.py @@ -0,0 +1,35 @@ +"""JSON Schema loading and validation.""" + +from __future__ import annotations + +import json +from functools import lru_cache +from importlib.resources import files +from itertools import islice + +from jsonschema import Draft202012Validator, FormatChecker + +from .errors import SchemaValidationFailed + + +@lru_cache(maxsize=8) +def validator_for(schema_name: str) -> Draft202012Validator: + if "/" in schema_name or "\\" in schema_name or not schema_name.endswith(".json"): + raise SchemaValidationFailed(f"Target schema is unavailable: {schema_name}") + resource = files("edu_crawler.schemas").joinpath(schema_name) + if not resource.is_file(): + raise SchemaValidationFailed(f"Target schema is unavailable: {schema_name}") + schema = json.loads(resource.read_text(encoding="utf-8")) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema, format_checker=FormatChecker()) + + +def validate_record(schema_name: str, record: dict) -> None: + errors = list(islice(validator_for(schema_name).iter_errors(record), 5)) + if not errors: + return + summaries = [] + for error in errors: + path = ".".join(str(part) for part in error.absolute_path) or "$" + summaries.append(f"{path}: {error.validator} validation failed") + raise SchemaValidationFailed("; ".join(summaries)) diff --git a/admin-tools/crawler/src/edu_crawler/schemas/__init__.py b/admin-tools/crawler/src/edu_crawler/schemas/__init__.py new file mode 100644 index 0000000..32d4d5a --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/schemas/__init__.py @@ -0,0 +1 @@ +"""Packaged JSON Schema resources used by crawler wheels.""" diff --git a/admin-tools/crawler/src/edu_crawler/schemas/external-job-source.schema.json b/admin-tools/crawler/src/edu_crawler/schemas/external-job-source.schema.json new file mode 100644 index 0000000..ea8d4c4 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/schemas/external-job-source.schema.json @@ -0,0 +1,266 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.edu-industry.cn/job-signal/3.0.0/schema.json", + "title": "JOB-SIGNAL V3.0 外部招聘岗位信号", + "description": "商业招聘平台、公共就业平台与招聘活动平台统一岗位信号契约。", + "type": "object", + "additionalProperties": false, + "required": [ + "record_id", + "source_platform", + "source_record_id", + "source_url", + "collector_task_id", + "collector_version", + "raw_snapshot_id", + "collected_at", + "first_seen_at", + "last_seen_at", + "status", + "job_title_raw", + "job_description", + "enterprise_name", + "region_code", + "employment_type", + "source_payload_hash", + "content_hash", + "data_version" + ], + "properties": { + "record_id": { + "type": "string", + "format": "uuid", + "description": "平台生成的全局唯一记录标识。" + }, + "source_platform": { + "type": "string", + "pattern": "^[A-Z][A-Z0-9_]{1,31}$", + "description": "来源平台代码,例如 BOSS、ZHAOPIN、BRZP、NCSS、MOHRSS。" + }, + "source_record_id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "来源平台内的记录标识,与 source_platform 组成幂等键。" + }, + "source_url": { + "type": "string", + "format": "uri", + "maxLength": 2048, + "description": "可回溯的来源页面或接口定位。" + }, + "collector_task_id": { + "type": "string", + "pattern": "^JOB-[A-Za-z0-9-]{8,64}$", + "description": "生产采集任务追踪编号。" + }, + "collector_version": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "采集器名称、语义版本与代码提交摘要。" + }, + "raw_snapshot_id": { + "type": "string", + "pattern": "^RAW-[A-Za-z0-9-]{8,96}$", + "description": "原始响应或页面快照编号。" + }, + "collected_at": { + "type": "string", + "format": "date-time", + "description": "平台实际采集时间,必须包含时区。" + }, + "first_seen_at": { + "type": "string", + "format": "date-time", + "description": "平台首次发现该岗位的时间。" + }, + "last_seen_at": { + "type": "string", + "format": "date-time", + "description": "平台最近一次确认该岗位仍存在的时间。" + }, + "published_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "来源平台显示的岗位发布时间。" + }, + "status": { + "type": "string", + "enum": ["active", "inactive", "expired", "unknown"], + "description": "岗位在当前数据版本中的状态。" + }, + "job_title_raw": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "来源岗位名称原文。" + }, + "job_title_normalized": { + "type": ["string", "null"], + "maxLength": 200, + "description": "映射到统一岗位词典后的标准名称。" + }, + "job_code": { + "type": ["string", "null"], + "pattern": "^[A-Z0-9.-]{2,32}$", + "description": "统一岗位分类代码。" + }, + "job_description": { + "type": "string", + "minLength": 1, + "maxLength": 30000, + "description": "去除展示标记与联系方式后的岗位职责和要求文本。" + }, + "enterprise_name": { + "type": "string", + "minLength": 1, + "maxLength": 300, + "description": "来源企业名称原文。" + }, + "enterprise_id": { + "type": ["string", "null"], + "maxLength": 128, + "description": "平台企业归一标识。" + }, + "enterprise_credit_code": { + "type": ["string", "null"], + "pattern": "^[0-9A-Z]{18}$", + "description": "可公开确认时使用的统一社会信用代码。" + }, + "enterprise_industry_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "企业所属行业代码。" + }, + "enterprise_scale": { + "type": ["string", "null"], + "enum": ["micro", "small", "medium", "large", "unknown", null], + "description": "统一企业规模分组。" + }, + "region_code": { + "type": "string", + "pattern": "^[0-9]{6}$", + "description": "GB/T 2260 六位行政区划代码。" + }, + "workplace": { + "type": ["string", "null"], + "maxLength": 500, + "description": "来源工作地点文本。" + }, + "employment_type": { + "type": "string", + "enum": ["full_time", "internship", "part_time", "contract", "campus", "other"], + "description": "统一用工或招聘类型。" + }, + "recruitment_count": { + "type": ["integer", "null"], + "minimum": 0, + "maximum": 1000000, + "description": "明确披露时记录计划招聘人数。" + }, + "education_level": { + "type": ["string", "null"], + "enum": ["none", "secondary", "vocational", "associate", "bachelor", "master", "doctor", "unknown", null], + "description": "最低学历要求。" + }, + "experience_min_years": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 60, + "description": "最低工作经验年限。" + }, + "salary_min": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 10000000, + "description": "统一折算后的税前月薪下限,单位为 salary_currency。" + }, + "salary_max": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 10000000, + "description": "统一折算后的税前月薪上限。" + }, + "salary_currency": { + "type": ["string", "null"], + "pattern": "^[A-Z]{3}$", + "description": "ISO 4217 币种代码,境内岗位通常为 CNY。" + }, + "salary_period": { + "type": ["string", "null"], + "enum": ["month", "year", "day", "hour", "negotiable", "unknown", null], + "description": "来源薪资周期,统一月薪字段与原始周期同时保留。" + }, + "benefits": { + "type": "array", + "maxItems": 100, + "items": { + "type": "string", + "maxLength": 100 + }, + "uniqueItems": true, + "description": "福利标签。" + }, + "skills": { + "type": "array", + "maxItems": 200, + "items": { + "type": "string", + "maxLength": 100 + }, + "uniqueItems": true, + "description": "基于技能词典提取并归一的技能标签。" + }, + "keywords": { + "type": "array", + "maxItems": 200, + "items": { + "type": "string", + "maxLength": 100 + }, + "uniqueItems": true, + "description": "用于检索的业务关键词。" + }, + "source_payload_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "原始响应载荷 SHA-256。" + }, + "content_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "规范化业务内容 SHA-256,用于内容去重。" + }, + "parsing_version": { + "type": ["string", "null"], + "maxLength": 128, + "description": "解析规则版本。" + }, + "normalized_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "岗位和技能归一完成时间。" + }, + "data_version": { + "type": "string", + "pattern": "^V[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}(?:\\.[0-9]+)?$", + "description": "进入分析层的数据版本。" + } + }, + "allOf": [ + { + "if": { + "required": ["salary_min", "salary_max"] + }, + "then": { + "properties": { + "salary_max": { + "type": "number", + "minimum": 0 + } + } + } + } + ] +} diff --git a/admin-tools/crawler/src/edu_crawler/schemas/school-employment.schema.json b/admin-tools/crawler/src/edu_crawler/schemas/school-employment.schema.json new file mode 100644 index 0000000..23f58b1 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/schemas/school-employment.schema.json @@ -0,0 +1,228 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.edu-industry.cn/school-employment/2.2.0/schema.json", + "title": "SCHOOL-EMPLOYMENT V2.2 校内就业事实", + "description": "职业院校毕业生就业事实统一契约,采用租户隔离和身份最小化设计。", + "type": "object", + "additionalProperties": false, + "required": [ + "tenant_id", + "record_id", + "student_token", + "cohort", + "college_code", + "program_code", + "employment_status", + "destination_type", + "confirmed_at", + "confirmation_role", + "source_system", + "source_record_id", + "created_at", + "updated_at", + "data_version", + "content_hash" + ], + "properties": { + "tenant_id": { + "type": "string", + "pattern": "^TENANT-[A-Z0-9-]{4,64}$", + "description": "学校租户标识,所有存储、任务和查询必须同时校验。" + }, + "record_id": { + "type": "string", + "format": "uuid", + "description": "平台全局唯一记录标识。" + }, + "student_token": { + "type": "string", + "pattern": "^STU_[A-Za-z0-9_-]{32,128}$", + "description": "学校侧生成的学生不可逆令牌,禁止传输姓名、身份证和明文学号。" + }, + "cohort": { + "type": "integer", + "minimum": 2000, + "maximum": 2100, + "description": "毕业届次。" + }, + "college_code": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "description": "学校内部学院代码。" + }, + "program_code": { + "type": "string", + "pattern": "^[0-9A-Z]{4,12}$", + "description": "教育部职业教育专业目录代码。" + }, + "class_code": { + "type": ["string", "null"], + "maxLength": 64, + "description": "学校内部教学班代码。" + }, + "employment_status": { + "type": "string", + "enum": [ + "employed", + "self_employed", + "further_study", + "military", + "flexible", + "unemployed", + "pending_confirmation" + ], + "description": "统一就业状态。" + }, + "destination_type": { + "type": "string", + "enum": [ + "enterprise", + "public_institution", + "government", + "further_study", + "entrepreneurship", + "military", + "other" + ], + "description": "毕业去向类型。" + }, + "job_title": { + "type": ["string", "null"], + "maxLength": 200, + "description": "就业岗位业务原文。" + }, + "job_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "映射到统一岗位词典后的岗位代码。" + }, + "enterprise_token": { + "type": ["string", "null"], + "pattern": "^ENT_[A-Za-z0-9_-]{24,128}$", + "description": "企业统一标识令牌。" + }, + "enterprise_name": { + "type": ["string", "null"], + "maxLength": 300, + "description": "用于业务核验的企业名称;导出时按数据范围策略处理。" + }, + "enterprise_industry_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "企业行业代码。" + }, + "region_code": { + "type": ["string", "null"], + "pattern": "^[0-9]{6}$", + "description": "就业地区 GB/T 2260 六位代码。" + }, + "salary_monthly": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 10000000, + "description": "确认后的税前月薪,单位元。" + }, + "salary_confirmed": { + "type": ["boolean", "null"], + "description": "薪资是否经过业务确认。" + }, + "job_relevance": { + "type": ["string", "null"], + "enum": ["high", "medium", "low", "unconfirmed", null], + "description": "就业岗位与专业培养方向相关度。" + }, + "employment_start_date": { + "type": ["string", "null"], + "format": "date", + "description": "就业开始日期。" + }, + "contract_signed": { + "type": ["boolean", "null"], + "description": "是否签署就业协议或劳动合同。" + }, + "evidence_type": { + "type": ["string", "null"], + "enum": ["contract", "offer", "registration", "enterprise_confirmation", "student_confirmation", "other", null], + "description": "就业确认材料类型,不在平台保存原始敏感附件。" + }, + "evidence_reference": { + "type": ["string", "null"], + "maxLength": 256, + "description": "学校系统内的材料引用编号。" + }, + "confirmed_at": { + "type": "string", + "format": "date-time", + "description": "业务确认时间。" + }, + "confirmation_role": { + "type": "string", + "enum": ["employment_office", "college", "student", "enterprise", "system"], + "description": "执行确认的业务角色。" + }, + "source_system": { + "type": "string", + "maxLength": 128, + "description": "学校来源系统代码。" + }, + "source_record_id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "校内来源系统记录标识。" + }, + "source_updated_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "校内来源系统业务更新时间,建议作为增量水位。" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "平台记录创建时间。" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "平台记录更新时间。" + }, + "data_version": { + "type": "string", + "pattern": "^SCH-EMP-[0-9]{8}(?:-[0-9]{2})?$", + "description": "校内就业数据版本。" + }, + "is_deleted": { + "type": "boolean", + "default": false, + "description": "版本化软删除标识。" + }, + "content_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "去除非业务差异后的记录内容 SHA-256。" + } + }, + "allOf": [ + { + "if": { + "properties": { + "destination_type": { + "const": "enterprise" + } + }, + "required": ["destination_type"] + }, + "then": { + "anyOf": [ + { + "required": ["enterprise_token"] + }, + { + "required": ["enterprise_name"] + } + ] + } + } + ] +} diff --git a/admin-tools/crawler/src/edu_crawler/schemas/school-internship.schema.json b/admin-tools/crawler/src/edu_crawler/schemas/school-internship.schema.json new file mode 100644 index 0000000..fd494f0 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/schemas/school-internship.schema.json @@ -0,0 +1,222 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.edu-industry.cn/school-internship/2.1.0/schema.json", + "title": "SCHOOL-INTERNSHIP V2.1 校内实习事实", + "description": "职业院校学生实习事实统一契约,采用租户隔离、身份最小化和证据引用设计。", + "type": "object", + "additionalProperties": false, + "required": [ + "tenant_id", + "record_id", + "internship_id", + "student_token", + "program_code", + "job_title", + "start_date", + "status", + "source_system", + "source_record_id", + "created_at", + "updated_at", + "data_version", + "content_hash" + ], + "properties": { + "tenant_id": { + "type": "string", + "pattern": "^TENANT-[A-Z0-9-]{4,64}$", + "description": "学校租户标识,所有存储、任务和查询必须同时校验。" + }, + "record_id": { + "type": "string", + "format": "uuid", + "description": "平台全局唯一记录标识。" + }, + "internship_id": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "学校租户内的实习业务编号。" + }, + "student_token": { + "type": "string", + "pattern": "^STU_[A-Za-z0-9_-]{32,128}$", + "description": "学校侧生成的学生不可逆令牌,禁止传输姓名、身份证和明文学号。" + }, + "cohort": { + "type": ["integer", "null"], + "minimum": 2000, + "maximum": 2100, + "description": "学生届次。" + }, + "college_code": { + "type": ["string", "null"], + "maxLength": 64, + "description": "学校内部学院代码。" + }, + "program_code": { + "type": "string", + "pattern": "^[0-9A-Z]{4,12}$", + "description": "教育部职业教育专业目录代码。" + }, + "enterprise_token": { + "type": ["string", "null"], + "pattern": "^ENT_[A-Za-z0-9_-]{24,128}$", + "description": "学校侧或平台企业主数据生成的企业令牌。" + }, + "enterprise_name": { + "type": ["string", "null"], + "maxLength": 300, + "description": "用于业务核验的企业名称;导出时按数据范围策略处理。" + }, + "enterprise_industry_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "企业行业代码。" + }, + "region_code": { + "type": ["string", "null"], + "pattern": "^[0-9]{6}$", + "description": "实习地区 GB/T 2260 六位代码。" + }, + "workplace": { + "type": ["string", "null"], + "maxLength": 500, + "description": "实习地点业务文本。" + }, + "job_title": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "实习岗位名称。" + }, + "job_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "映射到统一岗位词典后的岗位代码。" + }, + "job_tasks": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string", + "maxLength": 500 + }, + "description": "经学校确认的实习岗位任务。" + }, + "start_date": { + "type": "string", + "format": "date", + "description": "实习开始日期。" + }, + "end_date": { + "type": ["string", "null"], + "format": "date", + "description": "计划或实际结束日期。" + }, + "status": { + "type": "string", + "enum": ["planned", "active", "completed", "terminated", "cancelled"], + "description": "实习业务状态。" + }, + "salary_monthly": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 10000000, + "description": "确认后的税前月实习工资,单位元。" + }, + "enterprise_mentor_evaluation": { + "type": ["string", "null"], + "maxLength": 4000, + "description": "企业导师业务评价,禁止包含学生敏感身份信息。" + }, + "school_evaluation": { + "type": ["string", "null"], + "maxLength": 4000, + "description": "学校指导教师或管理部门评价。" + }, + "evidence_reference": { + "type": ["string", "null"], + "maxLength": 256, + "description": "学校系统内的证据引用编号,不传输敏感附件。" + }, + "source_system": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "学校来源系统代码。" + }, + "source_record_id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "校内来源系统记录标识。" + }, + "source_updated_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "校内来源系统业务更新时间,建议作为增量水位。" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "平台记录创建时间。" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "平台记录更新时间。" + }, + "data_version": { + "type": "string", + "pattern": "^SCH-INT-[0-9]{8}(?:-[0-9]{2})?$", + "description": "校内实习数据版本。" + }, + "is_deleted": { + "type": "boolean", + "default": false, + "description": "版本化软删除标识。" + }, + "content_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "去除非业务差异后的记录内容 SHA-256。" + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "enum": ["active", "completed"] + } + }, + "required": ["status"] + }, + "then": { + "anyOf": [ + { + "properties": { + "enterprise_token": { + "type": "string", + "pattern": "^ENT_[A-Za-z0-9_-]{24,128}$" + } + }, + "required": ["enterprise_token"] + }, + { + "properties": { + "enterprise_name": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "required": ["enterprise_name"] + } + ] + } + } + ] +} diff --git a/admin-tools/crawler/src/edu_crawler/security.py b/admin-tools/crawler/src/edu_crawler/security.py new file mode 100644 index 0000000..9d9b16f --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/security.py @@ -0,0 +1,242 @@ +"""Network and sensitive-data safety controls.""" + +from __future__ import annotations + +import ipaddress +import re +import socket +from collections.abc import Callable, Iterable +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from .errors import ConfigurationError, HostNotAllowed, PrivateAddressBlocked + +Resolver = Callable[..., list[tuple]] + +SENSITIVE_QUERY_KEYS = { + "access_token", + "api_key", + "apikey", + "auth", + "authorization", + "key", + "password", + "secret", + "signature", + "token", +} +CONTACT_PATTERN = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") + + +def normalize_host(host: str) -> str: + value = host.rstrip(".").lower() + try: + return value.encode("idna").decode("ascii") + except UnicodeError as exc: + raise HostNotAllowed("Host name is not valid IDNA") from exc + + +def redact_url(url: str) -> str: + parts = urlsplit(url) + query = [] + for key, value in parse_qsl(parts.query, keep_blank_values=True): + query.append( + (key, "[REDACTED]" if key.lower() in SENSITIVE_QUERY_KEYS else value) + ) + host = parts.hostname or "" + netloc = host + if parts.port: + netloc = f"{host}:{parts.port}" + return urlunsplit((parts.scheme, netloc, parts.path, urlencode(query), "")) + + +def validate_contact(contact: str) -> str: + if not CONTACT_PATTERN.fullmatch(contact): + raise ConfigurationError( + "A real technical contact email is required for network collection" + ) + return contact + + +def host_allowed(host: str, allowed_hosts: Iterable[str]) -> bool: + normalized = normalize_host(host) + return normalized in {normalize_host(item) for item in allowed_hosts} + + +def _assert_global_address(address: str) -> None: + try: + parsed = ipaddress.ip_address(address.split("%", 1)[0]) + except ValueError as exc: + raise PrivateAddressBlocked("DNS returned an invalid address") from exc + if not parsed.is_global: + raise PrivateAddressBlocked("URL resolves to a non-public address") + + +def resolve_public_endpoints( + host: str, + port: int, + *, + resolver: Resolver = socket.getaddrinfo, +) -> tuple[tuple[int, int, int, tuple], ...]: + try: + addresses = resolver(host, port, type=socket.SOCK_STREAM) + except OSError as exc: + raise HostNotAllowed(f"Unable to resolve approved source host: {host}") from exc + if not addresses: + raise HostNotAllowed(f"Approved source host has no addresses: {host}") + endpoints: list[tuple[int, int, int, tuple]] = [] + for family, socket_type, protocol, _canonical_name, socket_address in addresses: + _assert_global_address(socket_address[0]) + endpoint = (family, socket_type, protocol, socket_address) + if endpoint not in endpoints: + endpoints.append(endpoint) + return tuple(endpoints) + + +def validate_public_url( + url: str, + allowed_hosts: Iterable[str], + *, + resolver: Resolver = socket.getaddrinfo, +) -> str: + parts = urlsplit(url) + if parts.scheme != "https": + raise HostNotAllowed("Only HTTPS source URLs are allowed") + if parts.username or parts.password: + raise HostNotAllowed("Credentials in source URLs are forbidden") + if not parts.hostname or not host_allowed(parts.hostname, allowed_hosts): + raise HostNotAllowed(f"Source host is not approved: {redact_url(url)}") + if parts.port not in (None, 443): + raise HostNotAllowed("Only the default HTTPS port is allowed") + host = normalize_host(parts.hostname) + try: + parsed_host = ipaddress.ip_address(host) + except ValueError: + parsed_host = None + if parsed_host is not None and not parsed_host.is_global: + raise PrivateAddressBlocked("URL contains a non-public address") + resolve_public_endpoints(host, 443, resolver=resolver) + return urlunsplit(("https", parts.netloc, parts.path or "/", parts.query, "")) + + +def contains_challenge(status_code: int, text: str) -> bool: + if status_code in {401, 403}: + return True + sample = text[:100_000].lower() + markers = ( + "captcha", + "security verification", + "安全验证", + "访问验证", + "请输入验证码", + "请稍候", + ) + return any(marker in sample for marker in markers) + + +PROHIBITED_SCHOOL_KEYS = { + "address", + "email", + "id_card", + "id_number", + "identity_card", + "mobile", + "phone", + "student_id", + "student_name", + "student_number", + "身份证", + "姓名", + "手机号", + "明文学号", +} +SCHOOL_STUDENT_CONTEXT_KEYS = { + "student", + "student_info", + "student_profile", + "学生", + "学生信息", +} +SCHOOL_STUDENT_IDENTITY_KEYS = { + "card_no", + "email", + "id", + "id_card", + "mobile", + "name", + "number", + "phone", + "student_no", + "学号", + "姓名", + "手机号", + "身份证", +} +SCHOOL_PII_VALUE_PATTERNS = ( + re.compile(r"(? list[str]: + found: set[str] = set() + + def visit(value: object, path: str) -> None: + if isinstance(value, dict): + for key, nested in value.items(): + key_text = str(key) + nested_path = f"{path}.{key_text}" if path else key_text + if ( + key_text.lower() in PROHIBITED_SCHOOL_KEYS + or key_text in PROHIBITED_SCHOOL_KEYS + ): + found.add(nested_path) + path_parts = {part.lower() for part in path.split(".") if part} + if path_parts & SCHOOL_STUDENT_CONTEXT_KEYS and ( + key_text.lower() in SCHOOL_STUDENT_IDENTITY_KEYS + or key_text in SCHOOL_STUDENT_IDENTITY_KEYS + ): + found.add(nested_path) + visit(nested, nested_path) + elif isinstance(value, list): + for index, nested in enumerate(value): + visit(nested, f"{path}[{index}]") + + visit(record, "") + return sorted(found) + + +def prohibited_school_values(record: dict) -> list[str]: + found: set[str] = set() + + def visit(value: object, path: str) -> None: + if isinstance(value, dict): + for key, nested in value.items(): + visit(nested, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list): + for index, nested in enumerate(value): + visit(nested, f"{path}[{index}]") + elif isinstance(value, str) and not path.endswith( + ("_token", "source_record_id", "record_id") + ): + if any(pattern.search(value) for pattern in SCHOOL_PII_VALUE_PATTERNS): + found.add(path) + + visit(record, "") + return sorted(found) + + +def safe_record_reference(value: object) -> str | None: + if isinstance(value, dict): + value = value.get("value") or value.get("name") or value.get("@id") + text = re.sub(r"[\x00-\x1f\x7f]+", " ", str(value or "")).strip() + if not text: + return None + if any(pattern.search(text) for pattern in SCHOOL_PII_VALUE_PATTERNS): + return None + parts = urlsplit(text) + if parts.scheme in {"http", "https"}: + text = redact_url(text) + return text[:256] or None diff --git a/admin-tools/crawler/src/edu_crawler/storage.py b/admin-tools/crawler/src/edu_crawler/storage.py new file mode 100644 index 0000000..d4016ad --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/storage.py @@ -0,0 +1,151 @@ +"""Append-only task artifact writer.""" + +from __future__ import annotations + +import json +import os +import shutil +import uuid +from contextlib import AbstractContextManager +from pathlib import Path + +from .errors import TaskOutputExists +from .models import QuarantineItem, RawDocument + + +def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +class TaskOutput(AbstractContextManager["TaskOutput"]): + def __init__( + self, + output_root: Path, + source_code: str, + task_id: str, + tenant_id: str | None = None, + ) -> None: + source_directory = output_root / source_code + self.final_directory = ( + source_directory / tenant_id / task_id + if tenant_id + else source_directory / task_id + ) + self.staging_directory = self.final_directory.with_name( + f".{task_id}.tmp-{uuid.uuid4().hex}" + ) + self.raw_directory = self.staging_directory / "raw" + self._standard = None + self._quarantine = None + + def __enter__(self) -> "TaskOutput": + if self.final_directory.exists(): + raise TaskOutputExists( + f"Task output already exists: {self.final_directory}" + ) + self.raw_directory.mkdir(parents=True, exist_ok=False) + os.chmod(self.staging_directory, 0o700) + os.chmod(self.raw_directory, 0o700) + self._standard = (self.staging_directory / "standard.ndjson").open( + "x", encoding="utf-8" + ) + self._quarantine = (self.staging_directory / "quarantine.ndjson").open( + "x", encoding="utf-8" + ) + os.chmod(self.staging_directory / "standard.ndjson", 0o600) + os.chmod(self.staging_directory / "quarantine.ndjson", 0o600) + return self + + def __exit__(self, exception_type, _exception, _traceback) -> bool: + if self._standard: + self._standard.close() + if self._quarantine: + self._quarantine.close() + if exception_type is not None: + shutil.rmtree(self.staging_directory, ignore_errors=True) + return False + + def write_raw( + self, document: RawDocument, *, task_id: str, connector_version: str + ) -> None: + extensions = { + "application/json": "json", + "application/ld+json": "json", + "application/x-ndjson": "ndjson", + "text/csv": "csv", + "text/html": "html", + "text/plain": "txt", + } + extension = extensions.get(document.content_type, "bin") + body_path = self.raw_directory / f"{document.payload_hash}.{extension}" + metadata_path = self.raw_directory / f"{document.payload_hash}.metadata.json" + if not body_path.exists(): + body_path.write_bytes(document.body) + os.chmod(body_path, 0o600) + if not metadata_path.exists(): + _write_json( + metadata_path, + { + "source_code": document.source_code, + "source_url": document.source_url, + "content_type": document.content_type, + "bytes": len(document.body), + "sha256": document.payload_hash, + "raw_snapshot_id": document.snapshot_id, + "collected_at": document.collected_at, + "collector_task_id": task_id, + "connector_version": connector_version, + }, + ) + os.chmod(metadata_path, 0o600) + + def write_standard(self, record: dict) -> None: + assert self._standard is not None + self._standard.write( + json.dumps( + record, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ) + + "\n" + ) + + def write_quarantine(self, item: QuarantineItem) -> None: + assert self._quarantine is not None + self._quarantine.write( + json.dumps( + { + "source_code": item.source_code, + "raw_snapshot_id": item.raw_snapshot_id, + "source_record_id": item.source_record_id, + "error_code": item.error_code, + "error": item.error, + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ) + + def commit(self, summary: dict) -> Path: + assert self._standard is not None and self._quarantine is not None + self._standard.flush() + self._quarantine.flush() + os.fsync(self._standard.fileno()) + os.fsync(self._quarantine.fileno()) + _write_json(self.staging_directory / "summary.json", summary) + os.chmod(self.staging_directory / "summary.json", 0o600) + self._standard.close() + self._quarantine.close() + self._standard = None + self._quarantine = None + self.final_directory.parent.mkdir(parents=True, exist_ok=True) + os.chmod(self.final_directory.parent, 0o700) + if self.final_directory.exists(): + raise TaskOutputExists( + f"Task output already exists: {self.final_directory}" + ) + self.staging_directory.replace(self.final_directory) + return self.final_directory diff --git a/admin-tools/crawler/src/edu_crawler/transport.py b/admin-tools/crawler/src/edu_crawler/transport.py new file mode 100644 index 0000000..1b15ab9 --- /dev/null +++ b/admin-tools/crawler/src/edu_crawler/transport.py @@ -0,0 +1,148 @@ +"""HTTP transport that connects only to the public IPs it validated.""" + +from __future__ import annotations + +import socket +import ssl +import select +from collections.abc import Iterable + +import httpcore +import httpx + +from .security import Resolver, resolve_public_endpoints + + +class PinnedNetworkStream(httpcore.NetworkStream): + def __init__(self, sock: socket.socket) -> None: + self._socket = sock + + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: + try: + self._socket.settimeout(timeout) + return self._socket.recv(max_bytes) + except socket.timeout as exc: + raise httpcore.ReadTimeout("validated source read timed out") from exc + except OSError as exc: + raise httpcore.ReadError("validated source read failed") from exc + + def write(self, buffer: bytes, timeout: float | None = None) -> None: + try: + self._socket.settimeout(timeout) + self._socket.sendall(buffer) + except socket.timeout as exc: + raise httpcore.WriteTimeout("validated source write timed out") from exc + except OSError as exc: + raise httpcore.WriteError("validated source write failed") from exc + + def close(self) -> None: + self._socket.close() + + def start_tls( + self, + ssl_context: ssl.SSLContext, + server_hostname: str | None = None, + timeout: float | None = None, + ) -> httpcore.NetworkStream: + try: + self._socket.settimeout(timeout) + wrapped = ssl_context.wrap_socket( + self._socket, server_hostname=server_hostname + ) + except socket.timeout as exc: + self.close() + raise httpcore.ConnectTimeout( + "validated source TLS handshake timed out" + ) from exc + except OSError as exc: + self.close() + raise httpcore.ConnectError( + "validated source TLS handshake failed" + ) from exc + return PinnedNetworkStream(wrapped) + + def get_extra_info(self, info: str) -> object | None: + if info == "ssl_object" and isinstance(self._socket, ssl.SSLSocket): + return self._socket + if info == "client_addr": + return self._socket.getsockname() + if info == "server_addr": + return self._socket.getpeername() + if info == "socket": + return self._socket + if info == "is_readable": + try: + return bool(select.select([self._socket], [], [], 0)[0]) + except OSError: + return True + return None + + +class PinnedNetworkBackend(httpcore.NetworkBackend): + def __init__(self, resolver: Resolver = socket.getaddrinfo) -> None: + self._resolver = resolver + + def connect_tcp( + self, + host: str, + port: int, + timeout: float | None = None, + local_address: str | None = None, + socket_options: Iterable[tuple] | None = None, + ) -> httpcore.NetworkStream: + endpoints = resolve_public_endpoints(host, port, resolver=self._resolver) + last_error: OSError | None = None + for family, socket_type, protocol, socket_address in endpoints: + sock = socket.socket(family, socket_type, protocol) + try: + sock.settimeout(timeout) + if local_address: + bind_address = ( + (local_address, 0, 0, 0) + if family == socket.AF_INET6 + else (local_address, 0) + ) + sock.bind(bind_address) + for option in socket_options or (): + sock.setsockopt(*option) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + sock.connect(socket_address) + return PinnedNetworkStream(sock) + except socket.timeout as exc: + sock.close() + raise httpcore.ConnectTimeout( + "validated source connection timed out" + ) from exc + except OSError as exc: + last_error = exc + sock.close() + raise httpcore.ConnectError( + "validated source connection failed" + ) from last_error + + def connect_unix_socket( + self, + path: str, + timeout: float | None = None, + socket_options: Iterable[tuple] | None = None, + ) -> httpcore.NetworkStream: + del path, timeout, socket_options + raise httpcore.UnsupportedProtocol( + "Unix sockets are disabled for source collection" + ) + + +class PinnedHTTPTransport(httpx.HTTPTransport): + def __init__(self, resolver: Resolver = socket.getaddrinfo) -> None: + super().__init__(trust_env=False, retries=0) + self._pool.close() + self._pool = httpcore.ConnectionPool( + ssl_context=ssl.create_default_context(), + max_connections=2, + max_keepalive_connections=2, + keepalive_expiry=5.0, + http1=True, + http2=False, + retries=0, + network_backend=PinnedNetworkBackend(resolver), + ) diff --git a/admin-tools/crawler/tests/test_connectors.py b/admin-tools/crawler/tests/test_connectors.py new file mode 100644 index 0000000..a28a31d --- /dev/null +++ b/admin-tools/crawler/tests/test_connectors.py @@ -0,0 +1,255 @@ +import json +from dataclasses import replace + +import pytest + +from edu_crawler.connectors import CONNECTOR_TYPES, connector_for +from edu_crawler.errors import SourceStructureChanged +from edu_crawler.models import ParsedItem +from edu_crawler.schema import validate_record + + +def parsed(value: dict, *, payload_hash: str = "a" * 64) -> ParsedItem: + return ParsedItem( + value=value, + raw_snapshot_id="RAW-BOSS-20260824-ABCDEF012345", + source_payload_hash=payload_hash, + collected_at="2026-08-24T10:00:00+08:00", + source_url="file:///authorized.ndjson", + ) + + +def test_all_nine_sources_have_concrete_connector_types() -> None: + assert set(CONNECTOR_TYPES) == { + "BOSS", + "ZHAOPIN", + "BRZP", + "NCSS", + "MOHRSS", + "IGUOPIN", + "HZRC", + "SCHOOL_EMPLOYMENT", + "SCHOOL_INTERNSHIP", + } + assert all( + connector_for(code).describe()["source_code"] == code + for code in CONNECTOR_TYPES + ) + + +def test_external_record_normalizes_and_redacts_contact() -> None: + connector = connector_for("BOSS") + source = { + "source_record_id": "BOSS-001", + "source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html", + "title": "云原生开发工程师", + "description": "

负责容器平台开发,联系 13800138000

", + "company_name": "示例数字科技有限公司", + "region_code": "330100", + "employment_type": "全职", + "published_at": "2026-08-23T08:00:00+08:00", + "salary_min": "12000", + "salary_max": "18000", + "salary_period": "month", + "skills": ["Docker", "Kubernetes", "Docker"], + } + record = connector.normalize( + parsed(source), + task_id="JOB-20260824-BOSS-001", + data_version="V2026.08.24", + default_region_code=None, + ) + validate_record(connector.policy.target_schema, record) + assert record["source_platform"] == "BOSS" + assert record["job_description"] == "负责容器平台开发,联系 [CONTACT_REDACTED]" + assert record["skills"] == ["Docker", "Kubernetes"] + assert record["source_payload_hash"] == "a" * 64 + + +def test_annual_salary_is_normalized_to_monthly_and_unknown_status_stays_unknown() -> ( + None +): + connector = connector_for("BOSS") + source = { + "source_record_id": "BOSS-ANNUAL-001", + "source_url": "https://www.bosszhipin.com/job_detail/BOSS-ANNUAL-001.html", + "title": "年度薪资测试岗位", + "description": "用于验证统一月薪折算", + "company_name": "示例企业", + "region_code": "330100", + "salary_min": 120000, + "salary_max": 240000, + "salary_period": "year", + "status": "new-upstream-code", + } + record = connector.normalize( + parsed(source), + task_id="JOB-20260824-BOSS-ANNUAL", + data_version="V2026.08.24", + default_region_code=None, + ) + assert record["salary_min"] == 10000 + assert record["salary_max"] == 20000 + assert record["salary_period"] == "year" + assert record["status"] == "unknown" + + +def test_content_hash_ignores_collection_metadata() -> None: + connector = connector_for("BOSS") + source = { + "source_record_id": "BOSS-001", + "source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html", + "title": "云原生开发工程师", + "description": "负责容器平台开发", + "company_name": "示例数字科技有限公司", + "region_code": "330100", + } + first = connector.normalize( + parsed(source, payload_hash="a" * 64), + task_id="JOB-20260824-BOSS-001", + data_version="V2026.08.24", + default_region_code=None, + ) + second_item = replace( + parsed(source, payload_hash="b" * 64), + raw_snapshot_id="RAW-BOSS-20260825-012345ABCDEF", + collected_at="2026-08-25T10:00:00+08:00", + ) + second = connector.normalize( + second_item, + task_id="JOB-20260825-BOSS-001", + data_version="V2026.08.25", + default_region_code=None, + ) + assert first["content_hash"] == second["content_hash"] + assert first["source_payload_hash"] != second["source_payload_hash"] + + +def test_source_record_url_must_match_source_allowlist() -> None: + connector = connector_for("BOSS") + source = { + "source_record_id": "BOSS-001", + "source_url": "https://evil.example/job/1", + "title": "测试岗位", + "description": "测试职责", + "company_name": "测试企业", + "region_code": "330100", + } + try: + connector.normalize( + parsed(source), + task_id="JOB-20260824-BOSS-001", + data_version="V2026.08.24", + default_region_code=None, + ) + except Exception as exc: + assert getattr(exc, "code", None) == "SOURCE_STRUCTURE_CHANGED" + else: + raise AssertionError("unapproved source URL was accepted") + + +def test_declared_source_must_match_selected_connector() -> None: + connector = connector_for("BOSS") + source = { + "source_platform": "ZHAOPIN", + "source_record_id": "ZL-001", + "source_url": "https://www.bosszhipin.com/job_detail/1.html", + "title": "测试岗位", + "description": "测试岗位职责", + "company_name": "测试企业", + "region_code": "330100", + } + with pytest.raises(SourceStructureChanged) as caught: + connector.normalize( + parsed(source), + task_id="JOB-20260824-BOSS-001", + data_version="V2026.08.24", + default_region_code=None, + ) + assert getattr(caught.value, "code", None) == "SOURCE_STRUCTURE_CHANGED" + + +@pytest.mark.parametrize( + ("source_code", "source_url"), + [ + ("BOSS", "https://www.bosszhipin.com/job_detail/1.html"), + ("ZHAOPIN", "https://www.zhaopin.com/jobdetail/1.htm"), + ("BRZP", "https://brzp.chinasme.cn/brzp/job/1"), + ("NCSS", "https://www.ncss.cn/student/jobs/1"), + ("MOHRSS", "http://job.mohrss.gov.cn/cjobs/jobinfo/1"), + ("IGUOPIN", "https://job.iguopin.com/job/detail?id=1"), + ("HZRC", "https://www.hzrc.com/job/1"), + ], +) +def test_each_external_source_normalizes_authorized_feed( + source_code: str, source_url: str +) -> None: + connector = connector_for(source_code) + record = connector.normalize( + parsed( + { + "source_record_id": f"{source_code}-001", + "source_url": source_url, + "title": "测试岗位", + "description": "测试岗位职责", + "company_name": "测试企业", + "region_code": "330100", + } + ), + task_id=f"JOB-20260824-{source_code}-001", + data_version="V2026.08.24", + default_region_code=None, + ) + validate_record(connector.policy.target_schema, record) + assert record["source_platform"] == source_code + + +def test_source_url_secrets_are_redacted() -> None: + connector = connector_for("BOSS") + record = connector.normalize( + parsed( + { + "source_record_id": "BOSS-001", + "source_url": "https://www.bosszhipin.com/job_detail/1.html?token=secret&city=330100#private", + "title": "测试岗位", + "description": "测试岗位职责", + "company_name": "测试企业", + "region_code": "330100", + } + ), + task_id="JOB-20260824-BOSS-001", + data_version="V2026.08.24", + default_region_code=None, + ) + assert "secret" not in record["source_url"] + assert "private" not in record["source_url"] + assert "city=330100" in record["source_url"] + + +def test_schema_org_job_posting_mapping() -> None: + connector = connector_for("BRZP") + source = { + "@type": "JobPosting", + "identifier": {"value": "BRZP-001"}, + "url": "https://brzp.chinasme.cn/brzp/job/BRZP-001", + "title": "跨境电商运营实习生", + "description": "协助店铺运营和数据分析", + "hiringOrganization": {"name": "示例跨境服务有限公司"}, + "jobLocation": {"address": {"addressLocality": "杭州市"}}, + "employmentType": "internship", + "datePosted": "2026-08-23T08:00:00+08:00", + "baseSalary": { + "currency": "CNY", + "value": {"minValue": 3500, "maxValue": 5000, "unitText": "MONTH"}, + }, + } + record = connector.normalize( + parsed(source), + task_id="JOB-20260824-BRZP-001", + data_version="V2026.08.24", + default_region_code="330100", + ) + validate_record(connector.policy.target_schema, record) + assert record["employment_type"] == "internship" + assert record["salary_period"] == "month" + assert json.loads(json.dumps(record, ensure_ascii=False)) == record diff --git a/admin-tools/crawler/tests/test_parsing.py b/admin-tools/crawler/tests/test_parsing.py new file mode 100644 index 0000000..8d8d106 --- /dev/null +++ b/admin-tools/crawler/tests/test_parsing.py @@ -0,0 +1,62 @@ +import json + +import pytest + +from edu_crawler.errors import SourceStructureChanged +from edu_crawler.models import RawDocument +from edu_crawler.parsing import discover_job_links, parse_document +from edu_crawler.policies import source_policy + + +def test_parse_schema_org_job_posting() -> None: + body = b""" + + """ + document = RawDocument.create( + "BRZP", "https://brzp.chinasme.cn/job/BRZP-1", "text/html", body + ) + records = parse_document(document, allow_html=True) + assert records[0]["@type"] == "JobPosting" + + +def test_discover_links_uses_versioned_patterns_and_deduplicates() -> None: + body = b""" + one + duplicate + company + """ + document = RawDocument.create( + "BRZP", "https://brzp.chinasme.cn/", "text/html", body + ) + assert discover_job_links(document, source_policy("BRZP")) == [ + "https://brzp.chinasme.cn/job/1" + ] + + +def test_invalid_ndjson_reports_line_without_payload() -> None: + document = RawDocument.create( + "BOSS", + "file:///authorized.ndjson", + "application/x-ndjson", + b'{"id":"one"}\nnot-json\n', + ) + with pytest.raises(SourceStructureChanged, match="line 2") as caught: + parse_document(document, allow_html=False) + assert "not-json" not in str(caught.value) + + +def test_json_collection_wrappers() -> None: + document = RawDocument.create( + "BOSS", + "file:///authorized.json", + "application/json", + json.dumps({"data": {"items": [{"id": "one"}, {"id": "two"}]}}).encode(), + ) + assert [record["id"] for record in parse_document(document, allow_html=False)] == [ + "one", + "two", + ] diff --git a/admin-tools/crawler/tests/test_runner.py b/admin-tools/crawler/tests/test_runner.py new file mode 100644 index 0000000..6a0735b --- /dev/null +++ b/admin-tools/crawler/tests/test_runner.py @@ -0,0 +1,313 @@ +import json +from dataclasses import replace +from pathlib import Path + +import pytest + +from edu_crawler.errors import ( + AuthorizationRequired, + ConfigurationError, + PiiRejected, + TaskOutputExists, +) +from edu_crawler.models import RawDocument +from edu_crawler.runner import CrawlerRunner, RunConfig +from edu_crawler.schema import validate_record + + +def run_config( + tmp_path: Path, source: str, input_path: Path, data_version: str +) -> RunConfig: + return RunConfig( + source_code=source, + input_location=str(input_path), + task_id=f"JOB-20260824-{source.replace('_', '-')}-001", + data_version=data_version, + output_root=tmp_path / "output", + authorization_reference="DPA-TEST-REF-001", + tenant_id="TENANT-HZVTC" if source.startswith("SCHOOL_") else None, + ) + + +def read_ndjson(path: Path) -> list[dict]: + return [json.loads(line) for line in path.read_text().splitlines() if line] + + +def test_external_authorized_file_creates_append_only_artifacts(tmp_path: Path) -> None: + source = tmp_path / "boss.ndjson" + source.write_text( + json.dumps( + { + "source_record_id": "BOSS-001", + "source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html", + "title": "云原生开发工程师", + "description": "负责容器平台开发", + "company_name": "示例数字科技有限公司", + "region_code": "330100", + "employment_type": "full_time", + }, + ensure_ascii=False, + ) + + "\n" + ) + config = run_config(tmp_path, "BOSS", source, "V2026.08.24") + output = CrawlerRunner(config).run() + summary = json.loads((output / "summary.json").read_text()) + records = read_ndjson(output / "standard.ndjson") + assert summary["status"] == "COMPLETED" + assert summary["standard_records"] == 1 + assert len(list((output / "raw").glob("*.metadata.json"))) == 1 + assert len(records) == 1 + validate_record("external-job-source.schema.json", records[0]) + with pytest.raises(TaskOutputExists): + CrawlerRunner(config).run() + + +def test_missing_external_authorization_is_blocked(tmp_path: Path) -> None: + source = tmp_path / "boss.json" + source.write_text("{}") + config = run_config(tmp_path, "BOSS", source, "V2026.08.24") + config = replace(config, authorization_reference=None) + with pytest.raises(AuthorizationRequired): + CrawlerRunner(config).run() + + +def test_network_url_is_disabled_without_explicit_flag(tmp_path: Path) -> None: + config = RunConfig( + source_code="BRZP", + input_location="https://brzp.chinasme.cn/jobs", + task_id="JOB-20260824-BRZP-001", + data_version="V2026.08.24", + output_root=tmp_path, + authorization_reference="AUTH-BRZP-001", + ) + with pytest.raises(ConfigurationError): + CrawlerRunner(config).run() + + +def test_explicit_zero_record_limit_is_rejected(tmp_path: Path) -> None: + source = tmp_path / "boss.json" + source.write_text("{}") + config = replace(run_config(tmp_path, "BOSS", source, "V2026.08.24"), max_records=0) + with pytest.raises(ConfigurationError, match="max_records"): + CrawlerRunner(config).run() + + +def test_explicit_zero_page_limit_is_rejected_without_request(tmp_path: Path) -> None: + config = RunConfig( + source_code="BRZP", + input_location="https://brzp.chinasme.cn/jobs", + task_id="JOB-20260824-BRZP-001", + data_version="V2026.08.24", + output_root=tmp_path, + authorization_reference="AUTH-BRZP-001", + allow_network=True, + approved_hosts=("brzp.chinasme.cn",), + contact="crawler@example.edu.cn", + max_pages=0, + ) + with pytest.raises(ConfigurationError, match="max_pages"): + CrawlerRunner(config).run() + + +def test_local_file_size_limit_is_enforced_before_output(tmp_path: Path) -> None: + source = tmp_path / "oversized.json" + source.write_bytes(b"x" * (5 * 1024 * 1024 + 1)) + config = run_config(tmp_path, "BOSS", source, "V2026.08.24") + with pytest.raises(ConfigurationError, match="size limit"): + CrawlerRunner(config).run() + assert not config.output_root.exists() + + +def test_invalid_utf8_is_quarantined_without_payload_echo(tmp_path: Path) -> None: + source = tmp_path / "invalid.ndjson" + source.write_bytes(b"\xff\xfe") + output = CrawlerRunner(run_config(tmp_path, "BOSS", source, "V2026.08.24")).run() + quarantine = read_ndjson(output / "quarantine.ndjson") + assert quarantine[0]["error_code"] == "SOURCE_STRUCTURE_CHANGED" + assert quarantine[0]["error"] == "Structured source is not valid UTF-8" + + +def test_record_limit_stops_network_document_generator_early(tmp_path: Path) -> None: + config = RunConfig( + source_code="BRZP", + input_location="https://brzp.chinasme.cn/jobs", + task_id="JOB-20260824-BRZP-LIMIT", + data_version="V2026.08.24", + output_root=tmp_path, + authorization_reference="AUTH-BRZP-001", + allow_network=True, + approved_hosts=("brzp.chinasme.cn",), + contact="crawler@example.edu.cn", + max_records=1, + ) + consumed: list[str] = [] + + def documents(): + for suffix in ("one", "two"): + consumed.append(suffix) + yield RawDocument.create( + "BRZP", + f"https://brzp.chinasme.cn/job/{suffix}", + "application/json", + json.dumps( + [ + { + "source_record_id": suffix, + "source_url": f"https://brzp.chinasme.cn/job/{suffix}", + "title": "测试岗位", + "description": "测试职责", + "company_name": "测试企业", + "region_code": "330100", + } + ] + ).encode(), + ) + + runner = CrawlerRunner(config) + runner._network_documents = documents + output = runner.run() + assert consumed == ["one"] + assert json.loads((output / "summary.json").read_text())["raw_documents"] == 1 + + +def test_valid_school_employment_record(tmp_path: Path) -> None: + source = tmp_path / "employment.ndjson" + source.write_text( + json.dumps( + { + "tenant_id": "TENANT-HZVTC", + "student_token": "STU_" + "a" * 32, + "cohort": 2026, + "college_code": "C01", + "program_code": "510203", + "employment_status": "employed", + "destination_type": "enterprise", + "enterprise_token": "ENT_" + "b" * 24, + "confirmed_at": "2026-08-23T08:00:00+08:00", + "confirmation_role": "system", + "source_system": "EMPLOYMENT-VIEW", + "source_record_id": "EMP-001", + "source_updated_at": "2026-08-23T08:00:00+08:00", + }, + ensure_ascii=False, + ) + + "\n" + ) + output = CrawlerRunner( + run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01") + ).run() + record = read_ndjson(output / "standard.ndjson")[0] + validate_record("school-employment.schema.json", record) + assert record["tenant_id"] == "TENANT-HZVTC" + + +def test_school_connector_requires_expected_tenant(tmp_path: Path) -> None: + source = tmp_path / "employment.ndjson" + source.write_text("{}\n") + config = replace( + run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01"), + tenant_id=None, + ) + with pytest.raises(ConfigurationError, match="tenant-id"): + CrawlerRunner(config).run() + + +def test_school_connector_rejects_cross_tenant_batch_before_raw(tmp_path: Path) -> None: + source = tmp_path / "employment.ndjson" + source.write_text( + json.dumps({"tenant_id": "TENANT-OTHER", "source_record_id": "EMP-001"}) + "\n" + ) + config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01") + with pytest.raises(ConfigurationError, match="outside the configured tenant"): + CrawlerRunner(config).run() + assert not any(path.is_file() for path in config.output_root.rglob("*")) + + +def test_school_pii_is_rejected_before_raw_persistence(tmp_path: Path) -> None: + source = tmp_path / "employment.ndjson" + source.write_text( + json.dumps( + { + "tenant_id": "TENANT-HZVTC", + "source_record_id": "EMP-001", + "profile": {"student_name": "敏感姓名"}, + }, + ensure_ascii=False, + ) + + "\n" + ) + config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01") + with pytest.raises(PiiRejected): + CrawlerRunner(config).run() + assert not any(config.output_root.rglob(config.task_id)) + + +def test_valid_school_internship_record(tmp_path: Path) -> None: + source = tmp_path / "internship.ndjson" + source.write_text( + json.dumps( + { + "tenant_id": "TENANT-HZVTC", + "internship_id": "INT-001", + "student_token": "STU_" + "a" * 32, + "program_code": "510203", + "enterprise_token": "ENT_" + "b" * 24, + "job_title": "软件开发实习生", + "start_date": "2026-07-01", + "status": "active", + "source_system": "INTERNSHIP-API", + "source_record_id": "INT-001", + "source_updated_at": "2026-08-23T08:00:00+08:00", + }, + ensure_ascii=False, + ) + + "\n" + ) + output = CrawlerRunner( + run_config(tmp_path, "SCHOOL_INTERNSHIP", source, "SCH-INT-20260824-01") + ).run() + record = read_ndjson(output / "standard.ndjson")[0] + validate_record("school-internship.schema.json", record) + assert record["internship_id"] == "INT-001" + assert output.parts[-3:] == ( + "SCHOOL_INTERNSHIP", + "TENANT-HZVTC", + "JOB-20260824-SCHOOL-INTERNSHIP-001", + ) + + +def test_school_content_hash_ignores_sync_metadata(tmp_path: Path) -> None: + first_source = tmp_path / "employment-one.ndjson" + second_source = tmp_path / "employment-two.ndjson" + base = { + "tenant_id": "TENANT-HZVTC", + "student_token": "STU_" + "a" * 32, + "cohort": 2026, + "college_code": "C01", + "program_code": "510203", + "employment_status": "further_study", + "destination_type": "further_study", + "confirmed_at": "2026-08-23T08:00:00+08:00", + "confirmation_role": "system", + "source_system": "EMPLOYMENT-VIEW", + "source_record_id": "EMP-001", + } + first_source.write_text( + json.dumps({**base, "source_updated_at": "2026-08-23T08:00:00+08:00"}) + "\n" + ) + second_source.write_text( + json.dumps({**base, "source_updated_at": "2026-08-24T08:00:00+08:00"}) + "\n" + ) + first = CrawlerRunner( + run_config(tmp_path, "SCHOOL_EMPLOYMENT", first_source, "SCH-EMP-20260824-01") + ).run() + second_config = replace( + run_config(tmp_path, "SCHOOL_EMPLOYMENT", second_source, "SCH-EMP-20260824-01"), + task_id="JOB-20260824-SCHOOL-EMP-002", + ) + second = CrawlerRunner(second_config).run() + assert ( + read_ndjson(first / "standard.ndjson")[0]["content_hash"] + == read_ndjson(second / "standard.ndjson")[0]["content_hash"] + ) diff --git a/admin-tools/crawler/tests/test_schema_packaging.py b/admin-tools/crawler/tests/test_schema_packaging.py new file mode 100644 index 0000000..8b6ffc1 --- /dev/null +++ b/admin-tools/crawler/tests/test_schema_packaging.py @@ -0,0 +1,22 @@ +import json +from importlib.resources import files +from pathlib import Path + +from jsonschema import Draft202012Validator + +SCHEMA_NAMES = ( + "external-job-source.schema.json", + "school-employment.schema.json", + "school-internship.schema.json", +) + + +def test_packaged_schemas_match_canonical_demo_contracts() -> None: + canonical_directory = Path(__file__).resolve().parents[2] / "demo" / "schemas" + for name in SCHEMA_NAMES: + canonical = json.loads((canonical_directory / name).read_text()) + packaged = json.loads( + files("edu_crawler.schemas").joinpath(name).read_text(encoding="utf-8") + ) + assert packaged == canonical + Draft202012Validator.check_schema(packaged) diff --git a/admin-tools/crawler/tests/test_security.py b/admin-tools/crawler/tests/test_security.py new file mode 100644 index 0000000..d1cd03c --- /dev/null +++ b/admin-tools/crawler/tests/test_security.py @@ -0,0 +1,270 @@ +from dataclasses import replace +import os + +import httpx +import pytest + +from edu_crawler.errors import ( + AuthOrChallenge, + ConfigurationError, + HostNotAllowed, + PrivateAddressBlocked, + RobotsDenied, + SourceStructureChanged, +) +from edu_crawler.http import SafeHttpClient +from edu_crawler.policies import source_policy +from edu_crawler.security import ( + prohibited_school_keys, + prohibited_school_values, + redact_url, + safe_record_reference, + validate_public_url, +) +from edu_crawler.transport import PinnedNetworkBackend + + +def public_resolver(_host: str, _port: int, **_kwargs: object) -> list[tuple]: + return [(2, 1, 6, "", ("93.184.216.34", 443))] + + +def private_resolver(_host: str, _port: int, **_kwargs: object) -> list[tuple]: + return [(2, 1, 6, "", ("127.0.0.1", 443))] + + +def test_url_allowlist_and_ssrf_controls() -> None: + assert ( + validate_public_url( + "https://brzp.chinasme.cn/jobs?page=1", + ("brzp.chinasme.cn",), + resolver=public_resolver, + ) + == "https://brzp.chinasme.cn/jobs?page=1" + ) + with pytest.raises(HostNotAllowed): + validate_public_url( + "https://evil.example/jobs", + ("brzp.chinasme.cn",), + resolver=public_resolver, + ) + credential_url = "https://" + "user" + ":" + "example" + "@brzp.chinasme.cn/jobs" + with pytest.raises(HostNotAllowed): + validate_public_url( + credential_url, ("brzp.chinasme.cn",), resolver=public_resolver + ) + with pytest.raises(PrivateAddressBlocked): + validate_public_url( + "https://brzp.chinasme.cn/jobs", + ("brzp.chinasme.cn",), + resolver=private_resolver, + ) + with pytest.raises(PrivateAddressBlocked): + validate_public_url( + "https://127.0.0.1/jobs", ("127.0.0.1",), resolver=public_resolver + ) + + +def test_url_redaction_removes_credentials_and_fragment() -> None: + credential_url = "https://" + "user" + ":" + "example" + "@example.com" + value = redact_url(credential_url + "/jobs?token=abc&city=330100#private") + assert value == "https://example.com/jobs?token=%5BREDACTED%5D&city=330100" + assert "pass" not in value + assert "abc" not in value + assert "private" not in value + + +def test_record_reference_is_sanitized_before_quarantine_or_ids() -> None: + assert safe_record_reference("contact student@example.com") is None + assert ( + safe_record_reference( + {"value": "https://example.com/job/1?token=secret#private"} + ) + == "https://example.com/job/1?token=%5BREDACTED%5D" + ) + + +def test_nested_school_pii_detection() -> None: + record = { + "student_token": "STU_x", + "profile": {"student_name": "不应进入平台"}, + "student": {"name": "不应进入平台"}, + "rows": [{"phone": "1"}], + } + assert prohibited_school_keys(record) == [ + "profile.student_name", + "rows[0].phone", + "student.name", + ] + + +def test_school_pii_value_detection_reports_paths_only() -> None: + record = { + "student_token": "STU_" + "a" * 32, + "source_record_id": "EMP-001", + "comment": "联系 13800138000 或 student@example.com", + } + assert prohibited_school_values(record) == ["comment"] + + +def test_http_client_rejects_unversioned_approved_host() -> None: + policy = replace(source_policy("BRZP"), minimum_interval_seconds=0) + with pytest.raises(ConfigurationError): + SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("new-host.example",), + resolver=public_resolver, + transport=httpx.MockTransport(lambda _request: httpx.Response(200)), + ) + + +def test_robots_denial_stops_before_target_request() -> None: + requests: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(str(request.url)) + assert request.url.path == "/robots.txt" + return httpx.Response( + 200, + headers={"content-type": "text/plain"}, + text="User-agent: *\nDisallow: /private", + ) + + policy = replace(source_policy("BRZP"), minimum_interval_seconds=0) + with SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("brzp.chinasme.cn",), + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) as client: + with pytest.raises(RobotsDenied): + client.fetch("https://brzp.chinasme.cn/private") + assert requests == ["https://brzp.chinasme.cn/robots.txt"] + + +def test_challenge_page_stops_without_bypass() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/robots.txt": + return httpx.Response( + 200, + headers={"content-type": "text/plain"}, + text="User-agent: *\nAllow: /", + ) + return httpx.Response( + 200, headers={"content-type": "text/html"}, text="安全验证" + ) + + policy = replace(source_policy("BRZP"), minimum_interval_seconds=0) + with SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("brzp.chinasme.cn",), + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) as client: + with pytest.raises(AuthOrChallenge): + client.fetch("https://brzp.chinasme.cn/job/1") + + +def test_network_errors_do_not_echo_sensitive_urls() -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("failed token=secret-value", request=request) + + policy = replace( + source_policy("BRZP"), minimum_interval_seconds=0, robots_required=False + ) + with SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("brzp.chinasme.cn",), + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) as client: + with pytest.raises(SourceStructureChanged) as caught: + client.fetch("https://brzp.chinasme.cn/jobs?token=secret-value") + assert str(caught.value) == "Source network request failed" + + +def test_redirect_is_revalidated_against_allowlist() -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/robots.txt": + return httpx.Response( + 200, + headers={"content-type": "text/plain"}, + text="User-agent: *\nAllow: /", + ) + return httpx.Response(302, headers={"location": "https://evil.example/private"}) + + policy = replace(source_policy("BRZP"), minimum_interval_seconds=0) + with SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("brzp.chinasme.cn",), + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) as client: + with pytest.raises(HostNotAllowed): + client.fetch("https://brzp.chinasme.cn/job/1") + + +def test_bearer_token_is_not_sent_to_robots(monkeypatch: pytest.MonkeyPatch) -> None: + observed: list[tuple[str, str | None]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + observed.append((request.url.path, request.headers.get("authorization"))) + if request.url.path == "/robots.txt": + return httpx.Response( + 200, + headers={"content-type": "text/plain"}, + text="User-agent: *\nAllow: /", + ) + return httpx.Response( + 200, headers={"content-type": "application/json"}, json=[{"id": "one"}] + ) + + monkeypatch.setenv("TEST_SOURCE_BEARER", "example-token-value") + policy = replace(source_policy("BRZP"), minimum_interval_seconds=0) + with SafeHttpClient( + policy, + contact="crawler@example.edu.cn", + approved_hosts=("brzp.chinasme.cn",), + bearer_token_env="TEST_SOURCE_BEARER", + resolver=public_resolver, + transport=httpx.MockTransport(handler), + ) as client: + client.fetch("https://brzp.chinasme.cn/jobs") + assert observed == [ + ("/robots.txt", None), + ("/jobs", "Bearer example-token-value"), + ] + assert os.environ["TEST_SOURCE_BEARER"] == "example-token-value" + + +def test_pinned_backend_connects_to_the_validated_ip( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connected: list[tuple] = [] + + class FakeSocket: + def settimeout(self, _timeout: float | None) -> None: + pass + + def setsockopt(self, *_option: object) -> None: + pass + + def bind(self, _address: tuple) -> None: + pass + + def connect(self, address: tuple) -> None: + connected.append(address) + + def close(self) -> None: + pass + + monkeypatch.setattr( + "edu_crawler.transport.socket.socket", lambda *_args: FakeSocket() + ) + stream = PinnedNetworkBackend(public_resolver).connect_tcp("brzp.chinasme.cn", 443) + stream.close() + assert connected == [("93.184.216.34", 443)] diff --git a/admin-tools/demo/PRODUCTION_ARCHITECTURE.md b/admin-tools/demo/PRODUCTION_ARCHITECTURE.md index 4fe94c1..30f73d3 100644 --- a/admin-tools/demo/PRODUCTION_ARCHITECTURE.md +++ b/admin-tools/demo/PRODUCTION_ARCHITECTURE.md @@ -62,6 +62,16 @@ parse(snapshot) 解析为来源 Schema health() 返回入口可达性、延迟和错误分类 ``` +参考实现位于 [`../crawler/`](../crawler/),逐来源接入方式和授权门禁见 [`../../docs/crawler-source-implementation.md`](../../docs/crawler-source-implementation.md)。连接器运行时必须遵守以下硬约束: + +- 所有外部来源默认禁用,只有来源审批状态为 `approved` 才允许公网运行; +- 商业平台仅接入官方合作 API 或授权数据包,不实现登录自动化、验证码处理、私有接口逆向和反自动化规避; +- 公开页面任务同时执行书面授权、robots.txt、域名 allowlist、公共地址解析和来源级限速检查; +- 重定向后的每个 URL 重新执行相同安全校验,禁止访问回环、内网、链路本地和云元数据地址; +- User-Agent 必须标识本系统并提供真实技术联系邮箱,不伪装普通浏览器; +- Authorization、Cookie 和签名密钥只从 Vault/KMS 引用注入,不写入任务、Raw、日志或异常; +- 登录页、验证码、安全验证、访问封禁和结构变化均触发停止或隔离,不尝试绕过。 + 连接器发布物必须包含: - 代码仓库、语义版本、提交摘要和构建镜像摘要; @@ -221,6 +231,7 @@ ISSUE / WORK_ORDER ## 9. 安全与运维 - 源端凭据使用 Vault/KMS 管理,配置和日志只保存密钥引用; +- 来源授权、条款与 robots 快照作为连接器发布证据;授权到期自动停止调度; - 采集节点使用独立网络出口策略与最小权限; - 控制台操作采用 RBAC,并区分运营、开发、发布审批和审计; - 原始对象启用版本、对象锁和生命周期; diff --git a/admin-tools/demo/README.md b/admin-tools/demo/README.md index c600022..6ef31c9 100644 --- a/admin-tools/demo/README.md +++ b/admin-tools/demo/README.md @@ -23,19 +23,21 @@ python3 -m http.server 8080 5. 执行质量校验、企业归一、岗位分类和技能抽取; 6. 通过带 Manifest、Schema、质量报告、血缘和校验和的数据包向学校交付。 -## 已纳入的正式数据源 +## 已纳入的目标数据源 -| 类别 | 数据源 | 原型状态 | 目标数据 | +| 类别 | 数据源 | 生产接入方式 | 目标数据 | |---|---|---|---| -| 商业平台 | BOSS直聘 | 正式运行 | 岗位、企业、地区、薪资、岗位职责、技能 | -| 商业平台 | 智联招聘 | 正式运行 | 岗位、企业、地区、薪资、岗位职责、技能 | -| 公共平台 | 全国中小企业招聘平台 | 正式运行 | 中小企业岗位、校招、实习和招聘活动 | -| 公共平台 | 国家大学生就业服务平台 | 正式运行 | 高校毕业生岗位、专场招聘和实习 | -| 公共平台 | 中国公共招聘网 | 正式运行 | 公共就业岗位、重点行业和招聘活动 | -| 公共平台 | 国聘 | 正式运行 | 国资央企岗位和校园招聘 | -| 公共平台 | 杭州人才网 | 降速保护 | 杭州都市圈企业和岗位 | -| 校内系统 | 就业管理系统 | 正式运行 | 毕业去向、岗位、企业、薪资和专业相关度 | -| 校内系统 | 实习管理系统 | 正式运行 | 实习岗位、企业、工资、过程和企业评价 | +| 商业平台 | BOSS直聘 | 官方合作 API / 授权数据包 | 岗位、企业、地区、薪资、岗位职责、技能 | +| 商业平台 | 智联招聘 | 官方合作 API / 书面授权数据包 | 岗位、企业、地区、薪资、岗位职责、技能 | +| 公共平台 | 全国中小企业招聘平台 | 官方接口优先;获准公开页 | 中小企业岗位、校招、实习和招聘活动 | +| 公共平台 | 国家大学生就业服务平台 | 官方接口优先;获准公开页 | 高校毕业生岗位、专场招聘和实习 | +| 公共平台 | 中国公共招聘网 | 官方交换 / 提供单位授权 | 公共就业岗位、重点行业和招聘活动 | +| 公共平台 | 国聘 | 官方合作 API / 授权数据包 | 国资央企岗位和校园招聘 | +| 公共平台 | 杭州人才网 | 书面许可接口 / 低速公开页 | 杭州都市圈企业和岗位 | +| 校内系统 | 就业管理系统 | 租户只读视图 / API / 文件 | 毕业去向、岗位、企业、薪资和专业相关度 | +| 校内系统 | 实习管理系统 | 租户 API / 文件 | 实习岗位、企业、工资、过程和企业评价 | + +页面上的“正式运行”、记录数、成功率和时间线均为静态产品样板数据,不代表仓库当前正在访问这些站点。生产连接器默认禁止公网访问,必须完成来源授权、robots、域名白名单、限速和小样本结构审批后才能启用。逐来源方案见 [`../../docs/crawler-source-implementation.md`](../../docs/crawler-source-implementation.md),实现位于 [`../crawler/`](../crawler/)。 ## 交互范围 @@ -90,6 +92,9 @@ python3 -m http.server 8080 - `schemas/school-employment.schema.json` - `SCHOOL-EMPLOYMENT V2.2` - 校内就业事实契约,包含租户隔离和学生令牌化要求 +- `schemas/school-internship.schema.json` + - `SCHOOL-INTERNSHIP V2.1` + - 校内实习事实契约,包含租户隔离、学生令牌化、岗位任务和评价证据 - `schemas/package-manifest.schema.json` - `PACKAGE-MANIFEST V1.2` - 版本化导入、导出和学校交付包清单 diff --git a/admin-tools/demo/app.js b/admin-tools/demo/app.js index 8661b2b..6c4d54c 100644 --- a/admin-tools/demo/app.js +++ b/admin-tools/demo/app.js @@ -804,6 +804,42 @@ const contracts = { ["质量阈值", "专业代码合法率100%,必填完整率≥95%,学生专业关联率≥98%。"], ], }, + "school-internship": { + key: "SCHOOL-INTERNSHIP", + title: "校内实习事实数据标准", + version: "V2.1.0", + description: "统一学生实习岗位、企业、起止日期、状态、工资、岗位任务和评价事实,支持租户隔离和敏感身份最小化。", + schemaPath: "./schemas/school-internship.schema.json", + fields: 30, + required: 14, + owner: "平台数据标准委员会", + status: "生产已发布", + release: "2026-08-24", + compatibility: "兼容 V2.0,补充任务与评价证据", + fieldsPreview: [ + ["tenant_id", "string", "是", "学校租户标识", "平台分配"], + ["internship_id", "string", "是", "实习业务编号", "租户内唯一"], + ["student_token", "string", "是", "学生不可逆令牌", "禁止明文学号"], + ["program_code", "string", "是", "专业代码", "教育部专业目录"], + ["enterprise_token", "string", "否", "企业统一标识", "令牌化"], + ["job_title", "string", "是", "实习岗位名称", "业务原文"], + ["job_tasks", "array", "否", "实习岗位任务", "学校确认"], + ["start_date", "date", "是", "开始日期", "YYYY-MM-DD"], + ["end_date", "date", "否", "结束日期", "YYYY-MM-DD"], + ["status", "string", "是", "实习状态", "受控枚举"], + ["salary_monthly", "number", "否", "税前月实习工资", "元/月"], + ["evidence_reference", "string", "否", "校内证据引用", "不传附件"], + ["source_updated_at", "datetime", "否", "业务更新时间", "增量水位"], + ["content_hash", "string", "是", "业务内容哈希", "SHA-256"], + ], + rules: [ + ["租户隔离", "任务、认证、水位、Raw、隔离区和输出目录均按 tenant_id 隔离。"], + ["身份最小化", "平台只接收 student_token,拒绝姓名、身份证、手机号、邮箱和明文学号。"], + ["企业要求", "进行中或已完成实习必须提供企业令牌或企业名称。"], + ["证据引用", "敏感附件保留在学校系统,平台只保存 evidence_reference。"], + ["增量幂等", "tenant_id + source_record_id + source_updated_at 形成增量与幂等边界。"], + ], + }, "package-manifest": { key: "PACKAGE-MANIFEST", title: "数据交付包清单标准", diff --git a/admin-tools/demo/index.html b/admin-tools/demo/index.html index 9ccfbe0..f817f2b 100644 --- a/admin-tools/demo/index.html +++ b/admin-tools/demo/index.html @@ -395,6 +395,7 @@
+
@@ -425,7 +426,7 @@ 最大 2GB;大文件将自动分片并计算 SHA-256
- +
diff --git a/admin-tools/demo/preview-admin-contracts.png b/admin-tools/demo/preview-admin-contracts.png index 1d877cd..d0adf47 100644 Binary files a/admin-tools/demo/preview-admin-contracts.png and b/admin-tools/demo/preview-admin-contracts.png differ diff --git a/admin-tools/demo/schemas/README.md b/admin-tools/demo/schemas/README.md index 3fc5a76..f864662 100644 --- a/admin-tools/demo/schemas/README.md +++ b/admin-tools/demo/schemas/README.md @@ -17,4 +17,5 @@ |---|---|---| | `JOB-SIGNAL` | 3.0.0 | 商业招聘平台、公共就业平台、活动聚合平台 | | `SCHOOL-EMPLOYMENT` | 2.2.0 | 学校就业管理系统、数据库视图、API和标准文件 | +| `SCHOOL-INTERNSHIP` | 2.1.0 | 学校实习管理系统、API和标准文件 | | `PACKAGE-MANIFEST` | 1.2.0 | 导入、导出、租户交付和归档;新增可选发布候选编号与状态 | diff --git a/admin-tools/demo/schemas/school-internship.schema.json b/admin-tools/demo/schemas/school-internship.schema.json new file mode 100644 index 0000000..fd494f0 --- /dev/null +++ b/admin-tools/demo/schemas/school-internship.schema.json @@ -0,0 +1,222 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.edu-industry.cn/school-internship/2.1.0/schema.json", + "title": "SCHOOL-INTERNSHIP V2.1 校内实习事实", + "description": "职业院校学生实习事实统一契约,采用租户隔离、身份最小化和证据引用设计。", + "type": "object", + "additionalProperties": false, + "required": [ + "tenant_id", + "record_id", + "internship_id", + "student_token", + "program_code", + "job_title", + "start_date", + "status", + "source_system", + "source_record_id", + "created_at", + "updated_at", + "data_version", + "content_hash" + ], + "properties": { + "tenant_id": { + "type": "string", + "pattern": "^TENANT-[A-Z0-9-]{4,64}$", + "description": "学校租户标识,所有存储、任务和查询必须同时校验。" + }, + "record_id": { + "type": "string", + "format": "uuid", + "description": "平台全局唯一记录标识。" + }, + "internship_id": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "学校租户内的实习业务编号。" + }, + "student_token": { + "type": "string", + "pattern": "^STU_[A-Za-z0-9_-]{32,128}$", + "description": "学校侧生成的学生不可逆令牌,禁止传输姓名、身份证和明文学号。" + }, + "cohort": { + "type": ["integer", "null"], + "minimum": 2000, + "maximum": 2100, + "description": "学生届次。" + }, + "college_code": { + "type": ["string", "null"], + "maxLength": 64, + "description": "学校内部学院代码。" + }, + "program_code": { + "type": "string", + "pattern": "^[0-9A-Z]{4,12}$", + "description": "教育部职业教育专业目录代码。" + }, + "enterprise_token": { + "type": ["string", "null"], + "pattern": "^ENT_[A-Za-z0-9_-]{24,128}$", + "description": "学校侧或平台企业主数据生成的企业令牌。" + }, + "enterprise_name": { + "type": ["string", "null"], + "maxLength": 300, + "description": "用于业务核验的企业名称;导出时按数据范围策略处理。" + }, + "enterprise_industry_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "企业行业代码。" + }, + "region_code": { + "type": ["string", "null"], + "pattern": "^[0-9]{6}$", + "description": "实习地区 GB/T 2260 六位代码。" + }, + "workplace": { + "type": ["string", "null"], + "maxLength": 500, + "description": "实习地点业务文本。" + }, + "job_title": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "实习岗位名称。" + }, + "job_code": { + "type": ["string", "null"], + "maxLength": 32, + "description": "映射到统一岗位词典后的岗位代码。" + }, + "job_tasks": { + "type": "array", + "maxItems": 100, + "uniqueItems": true, + "items": { + "type": "string", + "maxLength": 500 + }, + "description": "经学校确认的实习岗位任务。" + }, + "start_date": { + "type": "string", + "format": "date", + "description": "实习开始日期。" + }, + "end_date": { + "type": ["string", "null"], + "format": "date", + "description": "计划或实际结束日期。" + }, + "status": { + "type": "string", + "enum": ["planned", "active", "completed", "terminated", "cancelled"], + "description": "实习业务状态。" + }, + "salary_monthly": { + "type": ["number", "null"], + "minimum": 0, + "maximum": 10000000, + "description": "确认后的税前月实习工资,单位元。" + }, + "enterprise_mentor_evaluation": { + "type": ["string", "null"], + "maxLength": 4000, + "description": "企业导师业务评价,禁止包含学生敏感身份信息。" + }, + "school_evaluation": { + "type": ["string", "null"], + "maxLength": 4000, + "description": "学校指导教师或管理部门评价。" + }, + "evidence_reference": { + "type": ["string", "null"], + "maxLength": 256, + "description": "学校系统内的证据引用编号,不传输敏感附件。" + }, + "source_system": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "description": "学校来源系统代码。" + }, + "source_record_id": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "description": "校内来源系统记录标识。" + }, + "source_updated_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "校内来源系统业务更新时间,建议作为增量水位。" + }, + "created_at": { + "type": "string", + "format": "date-time", + "description": "平台记录创建时间。" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "description": "平台记录更新时间。" + }, + "data_version": { + "type": "string", + "pattern": "^SCH-INT-[0-9]{8}(?:-[0-9]{2})?$", + "description": "校内实习数据版本。" + }, + "is_deleted": { + "type": "boolean", + "default": false, + "description": "版本化软删除标识。" + }, + "content_hash": { + "type": "string", + "pattern": "^[a-f0-9]{64}$", + "description": "去除非业务差异后的记录内容 SHA-256。" + } + }, + "allOf": [ + { + "if": { + "properties": { + "status": { + "enum": ["active", "completed"] + } + }, + "required": ["status"] + }, + "then": { + "anyOf": [ + { + "properties": { + "enterprise_token": { + "type": "string", + "pattern": "^ENT_[A-Za-z0-9_-]{24,128}$" + } + }, + "required": ["enterprise_token"] + }, + { + "properties": { + "enterprise_name": { + "type": "string", + "minLength": 1, + "maxLength": 300 + } + }, + "required": ["enterprise_name"] + } + ] + } + } + ] +} diff --git a/admin-tools/demo/styles.css b/admin-tools/demo/styles.css index 649378e..99bdfac 100644 --- a/admin-tools/demo/styles.css +++ b/admin-tools/demo/styles.css @@ -1923,7 +1923,7 @@ textarea:focus { .contract-tabs { display: grid; - grid-template-columns: repeat(3, 1fr); + grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 10px; margin-bottom: 13px; } diff --git a/demo/README.md b/demo/README.md index 1fca0d8..5636857 100644 --- a/demo/README.md +++ b/demo/README.md @@ -24,6 +24,8 @@ python3 -m http.server 8080 - 改进任务筛选、搜索、创建、阶段进展和报告建议转任务 - 数据源初次导入、增量更新、质量处置、版本管理、授权与审计 - 厂商基础数据包、校级数据、企业数据分类及授权状态筛选 +- 覆盖 6 个数据源和 42 份报告的“数据 → 指标/分析主题 → 报告”双向依赖图谱,支持只看缺口 +- 报告缺失字段、当前完整率、影响范围、业务索取人、技术接口人和要求完成日期一体化追踪 - 校级管理者、校级技术管理员、学院负责人、专业负责人、就业老师、企业导师、厂商超级管理员七种差异化工作台 - 厂商平台级数据包导入、学校租户、账号权限、备份恢复和审计管理 - 校级、学院、专业、产业专题与数据质量报告中心 @@ -41,6 +43,25 @@ python3 -m http.server 8080 - **校级数据**:来自学校实习、就业、教务等业务系统,数据权属归学校。 - **企业数据**:由合作企业确认提交,按照合作协议限定使用范围。 +## 数据与报告依赖图谱 + +“数据资产”页提供两种追溯方向: + +- **从报告看数据**:选择任一校级、学院、专业、产业专题或数据质量报告,查看必需数据、可选增强数据、分析主题、当前完备度和缺口。 +- **从数据看报告**:选择任一数据源,查看可形成的分析主题、支撑的报告类型和具体报告数量。 +- **只看缺口**:仅保留存在质量或字段问题的依赖关系,快速定位不能稳定支撑报告的数据。 + +当前样板将 42 份报告映射到 6 个数据源,并以版本化依赖模板区分必需与可选关系。每个缺口展示: + +- 缺失或待核验字段; +- 当前记录影响量、完整率和处理状态; +- 业务索取人及所属部门; +- 技术接口人及所属技术部门; +- 要求完成日期和受影响报告数; +- 可直接进入现有质量处置抽屉的操作入口。 + +依赖元数据用于跨部门补数协同,可以在数据运维角色下查看;报告正文仍按现有角色与数据范围授权,查看依赖关系不自动获得报告阅读权限。静态样板中的依赖、完整率和责任状态均为演示数据,正式系统应将 `ReportDataDependency`、`AnalysisTopic` 和 `DataRequest` 作为租户隔离、版本化、可审计的后端实体。 + ## 角色视角 - **校级管理者**:全校决策总览、专业对比、校企协同、改进任务、数据资产与报告。 @@ -67,6 +88,7 @@ python3 -m http.server 8080 - 厂商超级管理员在“平台运营中心 → 数据包管理”完成基础数据包导入、Schema 与质量校验、租户授权和版本发布。 - 校级技术管理员登录后默认进入“学校数据运维中心”,管理学校自有数据和企业授权数据。 - 每个数据源的“资产详情”包含业务截止日期、入库日期、下次更新、更新内容、主键、必填字段、格式、Schema、质量阈值、责任人与审计事件。 +- “数据与报告双向依赖图谱”同时回答“这份报告还需要什么数据”和“这项数据可以形成哪些报告”,并把缺口直接分派到业务索取人与技术接口人。 - “接收与使用反馈”展示租户、包号、版本、完整 SHA、交付包记录数、独立分析快照、支撑报告/项目/任务、业务可用确认和下一周期反馈。 - 更新采用“暂存 → 校验 → 审批 → 发布”流程。失败数据进入隔离区,不覆盖当前稳定版本。 @@ -93,3 +115,7 @@ python3 -m http.server 8080 - `preview-campus-closed-loop.png` - `preview-campus-closed-loop-mobile.png` + +数据依赖图谱预览: + +- `preview-data-report-lineage.png` diff --git a/demo/app.js b/demo/app.js index 8abaa8d..c6c4127 100644 --- a/demo/app.js +++ b/demo/app.js @@ -231,6 +231,7 @@ const tasks = [ const sources = [ { + id: "regional-jobs", name: "区域招聘岗位基础包", type: "互联网岗位 · 标准数据包", category: "vendor", @@ -243,6 +244,7 @@ const sources = [ status: "服务有效", records: "41,236", quality: "93.4%", + analysisCompleteness: 98.8, updated: "2026-03-18", dataAsOf: "2026-03-17", nextUpdate: "2026-04-01", @@ -255,10 +257,13 @@ const sources = [ threshold: "综合质量 ≥ 90;主键唯一率 100%", changeSummary: "新增 12,430 条,修改 846 条,失效 312 条;新增 AI 应用岗位分类", steward: "平台数据产品部", + stewardContact: "林岚 · 数据产品经理", technicalOwner: "平台数据运维组", + technicalContact: "平台数据包运维值班", action: "检查更新", }, { + id: "salary-company", name: "薪资与企业画像基础包", type: "互联网数据 · 增值数据包", category: "vendor", @@ -271,6 +276,7 @@ const sources = [ status: "服务有效", records: "36,284", quality: "91.8%", + analysisCompleteness: 100, updated: "2026-03-18", dataAsOf: "2026-03-17", nextUpdate: "2026-04-01", @@ -283,10 +289,13 @@ const sources = [ threshold: "综合质量 ≥ 90;薪资合法率 ≥ 98%", changeSummary: "更新 2,186 条薪资样本,补充 326 家企业规模与融资阶段标签", steward: "平台数据产品部", + stewardContact: "林岚 · 数据产品经理", technicalOwner: "平台数据运维组", + technicalContact: "平台数据包运维值班", action: "检查更新", }, { + id: "internship", name: "实习管理平台", type: "校内系统 · API", category: "school", @@ -299,6 +308,7 @@ const sources = [ status: "正常", records: "32,418", quality: "96.2%", + analysisCompleteness: 100, updated: "今天 08:12", dataAsOf: "2026-03-21", nextUpdate: "明天 08:00", @@ -311,10 +321,13 @@ const sources = [ threshold: "必填完整率 ≥ 95%;跨表关联率 ≥ 98%", changeSummary: "新增 186 条,修改 42 条;更新 31 家实习企业岗位信息", steward: "就业处", + stewardContact: "赵晴 · 实习管理专员", technicalOwner: "学校信息中心", + technicalContact: "王博 · 数据接口岗", action: "立即同步", }, { + id: "employment", name: "就业管理系统", type: "校内系统 · 数据库视图", category: "school", @@ -327,6 +340,7 @@ const sources = [ status: "字段缺失", records: "8,926", quality: "87.6%", + analysisCompleteness: 91.2, updated: "今天 07:50", dataAsOf: "2026-03-21", nextUpdate: "异常处理后重试", @@ -339,11 +353,14 @@ const sources = [ threshold: "专业代码合法率 100%;必填完整率 ≥ 95%", changeSummary: "新增 126 条,修改 43 条;86 条薪资记录进入隔离区,未覆盖当前生产版本", steward: "就业处", + stewardContact: "赵晴 · 就业数据专员", technicalOwner: "学校信息中心", + technicalContact: "王博 · 数据接口岗", action: "处理异常", warning: true, }, { + id: "curriculum", name: "课程标准库", type: "校内文件 · 结构化导入", category: "school", @@ -356,6 +373,7 @@ const sources = [ status: "正常", records: "1,428", quality: "95.1%", + analysisCompleteness: 100, updated: "昨天 17:20", dataAsOf: "2026-03-20", nextUpdate: "2026-08-25", @@ -368,10 +386,13 @@ const sources = [ threshold: "课程代码唯一率 100%;技能映射覆盖率 ≥ 80%", changeSummary: "导入 2026 春季课程快照;新增 38 门课程,修订 126 份课程标准", steward: "教务处", + stewardContact: "陈璐 · 教务数据专员", technicalOwner: "学校信息中心", + technicalContact: "王博 · 数据接口岗", action: "立即同步", }, { + id: "enterprise-demand", name: "企业确认需求库", type: "企业提交 · 问卷/工作台", category: "enterprise", @@ -384,6 +405,7 @@ const sources = [ status: "授权有效", records: "8,350", quality: "97.3%", + analysisCompleteness: 98.5, updated: "昨天 20:08", dataAsOf: "2026-03-21", nextUpdate: "企业提交后实时入队", @@ -396,7 +418,9 @@ const sources = [ threshold: "企业身份与授权关联率 100%;必填完整率 ≥ 95%", changeSummary: "新增 18 家企业确认、54 项岗位需求;12 项过期需求自动停用", steward: "产教融合中心", + stewardContact: "王健 · 企业合作专员", technicalOwner: "学校信息中心", + technicalContact: "王博 · 数据接口岗", action: "查看回执", }, ]; @@ -827,6 +851,7 @@ function syncClosedLoopCollections() { function saveAndRenderClosedLoop(message = "") { saveClosedLoopState(); syncClosedLoopCollections(); + renderDataReportLineage(); renderClosedLoop(); renderTasks(); renderProjects(); @@ -1502,6 +1527,7 @@ const reports = [ { id: "salary-2026-03", type: "industry", + dataProfile: "salary", typeLabel: "产业专题", title: "2026届实习就业薪资竞争力分析", scope: "全校2026届学生", @@ -1824,6 +1850,150 @@ professionReportCatalog.forEach(([name, college, score, focus], index) => { }); }); +const reportDataProfiles = { + school: { + label: "校级综合分析", + topics: ["区域岗位趋势", "课程技能覆盖", "实习就业成效", "企业需求验证"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "required", purpose: "岗位规模、技能需求与区域趋势" }, + { sourceId: "salary-company", requirement: "optional", purpose: "区域薪资基准与企业画像补充" }, + { sourceId: "internship", requirement: "required", purpose: "学生实习岗位与过程成效" }, + { sourceId: "employment", requirement: "required", purpose: "毕业去向、岗位与就业薪资" }, + { sourceId: "curriculum", requirement: "required", purpose: "专业课程与技能培养供给" }, + { sourceId: "enterprise-demand", requirement: "required", purpose: "合作企业正式确认的人才需求" }, + ], + }, + college: { + label: "学院专业建设分析", + topics: ["专业结构对比", "共享能力分析", "培养成效验证", "企业协同机会"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "required", purpose: "学院相关岗位与技能趋势" }, + { sourceId: "salary-company", requirement: "optional", purpose: "专业群薪资和企业结构参考" }, + { sourceId: "internship", requirement: "optional", purpose: "学院实习岗位与实践反馈" }, + { sourceId: "employment", requirement: "optional", purpose: "学院毕业去向与岗位匹配验证" }, + { sourceId: "curriculum", requirement: "required", purpose: "跨专业课程与共享技能供给" }, + { sourceId: "enterprise-demand", requirement: "required", purpose: "学院合作企业需求与项目机会" }, + ], + }, + profession: { + label: "专业人才需求分析", + topics: ["岗位技能需求", "课程能力映射", "学生达成验证", "企业需求复核"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "required", purpose: "对口岗位、技能频次与需求变化" }, + { sourceId: "salary-company", requirement: "optional", purpose: "区域同岗位薪资竞争力" }, + { sourceId: "internship", requirement: "optional", purpose: "学生实践岗位与企业评价" }, + { sourceId: "employment", requirement: "optional", purpose: "毕业去向和岗位匹配验证" }, + { sourceId: "curriculum", requirement: "required", purpose: "课程标准、学时与技能覆盖" }, + { sourceId: "enterprise-demand", requirement: "required", purpose: "真实企业需求与合作意向确认" }, + ], + }, + industry: { + label: "产业人才缺口分析", + topics: ["产业岗位趋势", "岗位技能归一", "培养供给对照", "企业确认校验"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "required", purpose: "产业岗位规模、增速与技能信号" }, + { sourceId: "salary-company", requirement: "optional", purpose: "重点产业薪资与企业结构" }, + { sourceId: "curriculum", requirement: "required", purpose: "全校专业与课程技能供给" }, + { sourceId: "enterprise-demand", requirement: "required", purpose: "合作企业需求交叉验证" }, + ], + }, + salary: { + label: "实习就业薪资分析", + topics: ["区域薪资基准", "实习薪资分布", "就业薪资对比", "专业竞争力"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "optional", purpose: "区域岗位热度与样本辅助筛选" }, + { sourceId: "salary-company", requirement: "required", purpose: "区域同岗位薪资基准" }, + { sourceId: "internship", requirement: "required", purpose: "学生实习薪资与岗位信息" }, + { sourceId: "employment", requirement: "required", purpose: "毕业生就业薪资与去向" }, + ], + }, + quality: { + label: "数据质量与口径分析", + topics: ["完整性与准确性", "版本与时效", "授权与权属", "加工血缘"], + dependencies: [ + { sourceId: "regional-jobs", requirement: "required", purpose: "岗位数据质量与归一覆盖" }, + { sourceId: "salary-company", requirement: "required", purpose: "薪资和企业画像质量" }, + { sourceId: "internship", requirement: "required", purpose: "实习数据完整性与时效" }, + { sourceId: "employment", requirement: "required", purpose: "就业数据完整性与异常记录" }, + { sourceId: "curriculum", requirement: "required", purpose: "课程数据版本与技能映射" }, + { sourceId: "enterprise-demand", requirement: "required", purpose: "企业身份、授权与需求质量" }, + ], + }, +}; + +const dataGapCatalog = [ + { + id: "job-normalization", + sourceId: "regional-jobs", + level: "attention", + levelLabel: "待复核", + title: "新增岗位缺少标准岗位代码与技能映射", + drawerTitle: "新增岗位名称待归一", + drawerLevel: "需要复核", + amount: "218 个岗位名称 · 影响 1,426 条记录", + affected: "218 个岗位别名 · 影响 1,426 条招聘记录", + fields: ["standard_position_code", "normalized_job_name", "skill_tags"], + affectedProfileKeys: ["school", "college", "profession", "industry", "quality"], + reason: "新增“具身智能应用工程师”等岗位未映射到当前岗位分类词典 V3.2。", + action: "优先审核需求量前 50 的岗位名称,确认标准岗位、技能标签及别名规则。", + status: "平台侧复核中", + due: "2026-03-24", + request: "请平台数据产品部确认标准岗位及技能标签,由平台数据运维组发布别名规则增量。", + }, + { + id: "salary-missing", + sourceId: "employment", + level: "blocking", + levelLabel: "需补数", + title: "就业薪资与缺失原因未完整提供", + drawerTitle: "就业系统薪资字段缺失", + drawerLevel: "阻断发布", + amount: "86 条记录 · 当前字段完整率 91.2%", + affected: "86 条记录 · 电子商务专业 41 条、现代服务类专业 45 条", + fields: ["employment_salary", "salary_period", "missing_reason"], + affectedReportIds: ["school-2026-03", "salary-2026-03", "quality-2026-03", "ecommerce-2026-03"], + reason: "employment_salary 为空,未达到必填完整率 95% 的发布阈值。", + action: "由就业处补齐薪资或填写缺失原因,修复文件按 student_id + cohort 回写后重新校验。", + status: "待就业处补充", + due: "2026-03-25", + request: "请就业处补齐薪资或登记缺失原因,由学校信息中心回写只读视图并重新执行质量校验。", + }, + { + id: "enterprise-duplicate", + sourceId: "enterprise-demand", + level: "attention", + levelLabel: "待核验", + title: "企业主标识存在疑似重复", + drawerTitle: "疑似重复企业记录", + drawerLevel: "一般事项", + amount: "12 家企业 · 涉及 37 项确认需求", + affected: "12 家企业 · 涉及 37 项岗位需求", + fields: ["enterprise_credit_code", "master_enterprise_id"], + affectedProfileKeys: ["school", "college", "profession", "industry", "quality"], + reason: "企业简称不同,但统一社会信用代码或工商地址高度相似。", + action: "由产教融合中心确认主企业记录,合并需求与授权回执并保留审计映射。", + status: "待产教融合中心确认", + due: "2026-03-26", + request: "请产教融合中心确认企业主记录,由学校信息中心合并需求与授权回执并保留审计映射。", + }, +]; + +const sourceAnalysisTopics = { + "regional-jobs": ["岗位需求趋势", "岗位技能归一", "产业增长信号"], + "salary-company": ["区域薪资基准", "企业规模画像", "岗位竞争力"], + internship: ["实习岗位分布", "实践过程成效", "企业评价"], + employment: ["毕业去向", "就业薪资", "岗位匹配成效"], + curriculum: ["课程技能覆盖", "专业培养供给", "课程版本"], + "enterprise-demand": ["企业需求确认", "校企合作机会", "授权有效性"], +}; + +const dataLineageState = { + mode: "report", + selectedReportId: "school-2026-03", + selectedSourceId: "employment", + gapsOnly: false, +}; + const iconPaths = { course: '', lab: '', @@ -1983,6 +2153,7 @@ function applyRole(role, options = {}) { ]); } + renderDataReportLineage(); renderClosedLoop(); if (document.getElementById("professionSelect")) { renderProfessionProfile(document.getElementById("professionSelect").value); @@ -2522,6 +2693,359 @@ function renderTasks(filter = "all", search = "") { } } +function reportDataProfileKey(report) { + return report.dataProfile || report.type; +} + +function reportDataProfile(report) { + return reportDataProfiles[reportDataProfileKey(report)] || reportDataProfiles.profession; +} + +function sourceById(sourceId) { + return sources.find((source) => source.id === sourceId); +} + +function gapsForSource(sourceId) { + return dataGapCatalog.filter((gap) => gap.sourceId === sourceId); +} + +function dataGapById(gapId) { + return dataGapCatalog.find((gap) => gap.id === gapId); +} + +function dependencyForReport(report, sourceId) { + return reportDataProfile(report).dependencies.find((dependency) => dependency.sourceId === sourceId); +} + +function reportsUsingSource(sourceId) { + return reports.filter((report) => Boolean(dependencyForReport(report, sourceId))); +} + +function gapAffectsReport(gap, report) { + if (!dependencyForReport(report, gap.sourceId)) return false; + if (gap.affectedReportIds) return gap.affectedReportIds.includes(report.id); + if (report.periodKey !== "current") return false; + return gap.affectedProfileKeys?.includes(reportDataProfileKey(report)) ?? false; +} + +function reportsAffectedByGap(gap) { + return reports.filter((report) => gapAffectsReport(gap, report)); +} + +function reportReadiness(report) { + const requiredDependencies = reportDataProfile(report).dependencies.filter( + (dependency) => dependency.requirement === "required", + ); + if (!requiredDependencies.length) return 100; + return ( + requiredDependencies.reduce( + (total, dependency) => total + sourceById(dependency.sourceId).analysisCompleteness, + 0, + ) / + requiredDependencies.length + ); +} + +function reportGaps(report) { + return dataGapCatalog.filter((gap) => gapAffectsReport(gap, report)); +} + +function reportsGroupedByType(reportItems) { + return reportItems.reduce((groups, report) => { + const group = groups.get(report.type) || { + type: report.type, + label: report.typeLabel, + reports: [], + }; + group.reports.push(report); + groups.set(report.type, group); + return groups; + }, new Map()); +} + +function readableReportIdSet() { + return new Set(reportsForCurrentRole().map((report) => report.id)); +} + +function lineageEmptyNode(message) { + return `
${svgIcon("database")}${message}关闭“只看缺口”可查看完整依赖。
`; +} + +function renderLineageSourceNode(source, dependency, selected = false, relevantGaps = gapsForSource(source.id)) { + const gaps = relevantGaps.filter((gap) => gap.sourceId === source.id); + const hasGap = gaps.length > 0; + const requirement = dependency?.requirement || "required"; + const completeness = source.analysisCompleteness; + return ` + + `; +} + +function renderLineageTopicNode(topic, index) { + return ` +
+ ${String(index + 1).padStart(2, "0")} + ${topic} + 统一口径计算 · 保留证据定位 +
+ `; +} + +function renderLineageReportNode(report, readableIds = readableReportIdSet()) { + const gaps = reportGaps(report); + const readiness = reportReadiness(report); + const canRead = readableIds.has(report.id); + return ` + + `; +} + +function renderLineageReportGroup(group, readableIds) { + const affectedCount = group.reports.filter((report) => reportGaps(report).length).length; + return ` +
+ + ${group.label}${affectedCount ? `${affectedCount} 份存在数据缺口` : "当前数据关系可用"} + ${group.reports.length} 份 + +
+ ${group.reports.map((report) => renderLineageReportNode(report, readableIds)).join("")} +
+
+ `; +} + +function renderLineageStage(title, countText, className, content) { + return ` +
+
${title}${countText}
+
${content}
+
+ `; +} + +function lineageConnector(label) { + return ` + + `; +} + +function renderReportLineageCanvas(report) { + const profile = reportDataProfile(report); + const gaps = reportGaps(report); + const readableIds = readableReportIdSet(); + const gapSourceIds = new Set(gaps.map((gap) => gap.sourceId)); + const visibleDependencies = dataLineageState.gapsOnly + ? profile.dependencies.filter((dependency) => gapSourceIds.has(dependency.sourceId)) + : profile.dependencies; + const sourceNodes = + visibleDependencies + .map((dependency) => renderLineageSourceNode(sourceById(dependency.sourceId), dependency, false, gaps)) + .join("") || lineageEmptyNode("当前报告没有数据缺口"); + const topicNodes = profile.topics.map(renderLineageTopicNode).join(""); + return ` +
+ ${renderLineageStage("数据资产", `${visibleDependencies.length} / ${profile.dependencies.length} 项`, "sources", sourceNodes)} + ${lineageConnector("形成指标")} + ${renderLineageStage("指标 / 分析主题", `${profile.topics.length} 类`, "topics", topicNodes)} + ${lineageConnector("生成报告")} + ${renderLineageStage("目标报告", "1 份", "reports", renderLineageReportNode(report, readableIds))} +
+ `; +} + +function renderSourceLineageCanvas(source) { + const sourceReports = reportsUsingSource(source.id); + const affectedReportIds = new Set( + gapsForSource(source.id).flatMap((gap) => reportsAffectedByGap(gap).map((report) => report.id)), + ); + const visibleReports = dataLineageState.gapsOnly + ? sourceReports.filter((report) => affectedReportIds.has(report.id)) + : sourceReports; + const reportGroups = [...reportsGroupedByType(visibleReports).values()]; + const readableIds = readableReportIdSet(); + const reportNodes = + reportGroups + .map((group) => renderLineageReportGroup(group, readableIds)) + .join("") || lineageEmptyNode("当前数据来源没有待处理的报告关系"); + const topics = sourceAnalysisTopics[source.id] || ["基础指标", "专题分析", "质量校验"]; + return ` +
+ ${renderLineageStage("选中数据", "1 项", "sources", renderLineageSourceNode(source, null, true))} + ${lineageConnector("支撑分析")} + ${renderLineageStage("可形成的分析", `${topics.length} 类`, "topics", topics.map(renderLineageTopicNode).join(""))} + ${lineageConnector("支撑报告")} + ${renderLineageStage("可形成的报告", `${visibleReports.length} 份`, "reports", reportNodes)} +
+ `; +} + +function lineageSummaryCard(label, value, note, className = "") { + return `
${label}${value}${note}
`; +} + +function renderLineageSummary(report, source) { + const summary = document.getElementById("dataLineageSummary"); + if (dataLineageState.mode === "report") { + const profile = reportDataProfile(report); + const requiredCount = profile.dependencies.filter((dependency) => dependency.requirement === "required").length; + const gaps = reportGaps(report); + summary.innerHTML = [ + lineageSummaryCard("当前报告", report.typeLabel, report.version), + lineageSummaryCard("依赖数据", `${requiredCount} 必需 · ${profile.dependencies.length - requiredCount} 可选`, "依赖关系已登记"), + lineageSummaryCard("数据完备度", `${reportReadiness(report).toFixed(1)}%`, gaps.length ? "使用稳定版本,可继续分析" : "达到生成条件"), + lineageSummaryCard("待补数据", `${gaps.length} 项`, gaps.length ? "已定位责任人与时限" : "当前无数据缺口", gaps.length ? "warning" : "ready"), + ].join(""); + return; + } + + const sourceReports = reportsUsingSource(source.id); + const gaps = gapsForSource(source.id); + summary.innerHTML = [ + lineageSummaryCard("当前数据", source.categoryLabel, source.version), + lineageSummaryCard("支撑报告", `${sourceReports.length} 份`, `${reportsGroupedByType(sourceReports).size} 类报告`), + lineageSummaryCard("当前质量", source.quality, `${source.records} 条有效记录`), + lineageSummaryCard("数据缺口", `${gaps.length} 项`, gaps.length ? "已定位责任人与时限" : "当前无数据缺口", gaps.length ? "warning" : "ready"), + ].join(""); +} + +function renderLineageGapCard(gap, selectedReport) { + const source = sourceById(gap.sourceId); + const impactCount = selectedReport ? 1 : reportsAffectedByGap(gap).length; + return ` +
+
+ ${gap.levelLabel} + ${gap.status} +
+ ${gap.title} + ${source.name} · ${gap.amount} +
${gap.fields.map((field) => `${field}`).join("")}
+

${gap.request}

+
+
业务索取人
${source.stewardContact}
+
技术接口人
${source.technicalContact}
+
要求完成
${gap.due}
+
影响范围
${selectedReport ? "当前报告" : `${impactCount} 份报告`}
+
+ +
+ `; +} + +function renderLineageGaps(report, source) { + const gaps = dataLineageState.mode === "report" ? reportGaps(report) : gapsForSource(source.id); + document.getElementById("dataLineageGapCount").textContent = `${gaps.length} 项待处理`; + document.getElementById("dataLineageGapList").innerHTML = gaps.length + ? gaps.map((gap) => renderLineageGapCard(gap, dataLineageState.mode === "report" ? report : null)).join("") + : ` +
+ ${svgIcon("database")} + 当前关系没有待补数据数据版本、质量和授权均达到报告生成要求。 +
+ `; +} + +function renderDataReportLineage() { + const selector = document.getElementById("dataLineageSelector"); + if (!selector) return; + const mode = dataLineageState.mode; + const readableReportIds = new Set(reportsForCurrentRole().map((report) => report.id)); + if (!reports.some((report) => report.id === dataLineageState.selectedReportId)) { + dataLineageState.selectedReportId = "school-2026-03"; + } + if (!sources.some((source) => source.id === dataLineageState.selectedSourceId)) { + dataLineageState.selectedSourceId = sources[0].id; + } + const report = reports.find((item) => item.id === dataLineageState.selectedReportId) || reports[0]; + const source = sourceById(dataLineageState.selectedSourceId) || sources[0]; + + document.querySelectorAll("[data-lineage-mode]").forEach((button) => { + const selected = button.dataset.lineageMode === mode; + button.classList.toggle("active", selected); + button.setAttribute("aria-pressed", String(selected)); + }); + document.getElementById("dataLineageSelectorLabel").textContent = mode === "report" ? "选择报告" : "选择数据"; + selector.setAttribute("aria-label", mode === "report" ? "选择要追溯的报告" : "选择要反查的数据"); + selector.innerHTML = + mode === "report" + ? reports + .map((item) => { + const canRead = readableReportIds.has(item.id); + return ``; + }) + .join("") + : sources + .map((item) => ``) + .join(""); + selector.value = mode === "report" ? report.id : source.id; + document.getElementById("dataLineageGapToggle").checked = dataLineageState.gapsOnly; + document.getElementById("dataLineageCanvas").innerHTML = + mode === "report" ? renderReportLineageCanvas(report) : renderSourceLineageCanvas(source); + renderLineageSummary(report, source); + renderLineageGaps(report, source); +} + +function setupDataReportLineage() { + const lineage = document.getElementById("dataReportLineage"); + lineage.addEventListener("click", (event) => { + const modeButton = event.target.closest("[data-lineage-mode]"); + if (modeButton) { + dataLineageState.mode = modeButton.dataset.lineageMode; + renderDataReportLineage(); + return; + } + const reportButton = event.target.closest("[data-lineage-report]"); + if (reportButton) { + const reportId = reportButton.dataset.lineageReport; + if (!reportsForCurrentRole().some((report) => report.id === reportId)) { + showToast("依赖元数据可见,但当前角色没有该报告正文的阅读权限。", "权限范围提示"); + return; + } + openReportReader(reportId); + return; + } + const gapButton = event.target.closest("[data-lineage-gap-action]"); + if (gapButton) openDataOpsDrawer("issue", gapButton.dataset.lineageGapAction); + }); + + document.getElementById("dataLineageSelector").addEventListener("change", (event) => { + if (dataLineageState.mode === "report") dataLineageState.selectedReportId = event.target.value; + else dataLineageState.selectedSourceId = event.target.value; + renderDataReportLineage(); + }); + document.getElementById("dataLineageGapToggle").addEventListener("change", (event) => { + dataLineageState.gapsOnly = event.target.checked; + renderDataReportLineage(); + }); +} + const dataOpsState = { context: "source", sourceName: sources[0].name, @@ -2529,33 +3053,6 @@ const dataOpsState = { issue: null, }; -const dataIssueCatalog = { - "salary-missing": { - title: "就业系统薪资字段缺失", - level: "阻断发布", - sourceName: "就业管理系统", - affected: "86 条记录 · 电子商务专业 41 条、现代服务类专业 45 条", - reason: "employment_salary 为空,未达到必填完整率 95% 的发布阈值。", - action: "由就业处补齐薪资或填写缺失原因,修复文件按 student_id + cohort 回写后重新校验。", - }, - "job-normalization": { - title: "新增岗位名称待归一", - level: "需要复核", - sourceName: "区域招聘岗位基础包", - affected: "218 个岗位别名 · 影响 1,426 条招聘记录", - reason: "新增“具身智能应用工程师”等岗位未映射到当前岗位分类词典 V3.2。", - action: "优先审核需求量前 50 的岗位名称,确认标准岗位、技能标签及别名规则。", - }, - "enterprise-duplicate": { - title: "疑似重复企业记录", - level: "一般事项", - sourceName: "企业确认需求库", - affected: "12 家企业 · 涉及 37 项岗位需求", - reason: "企业简称不同,但统一社会信用代码或工商地址高度相似。", - action: "由产教融合中心确认主企业记录,合并需求与授权回执并保留审计映射。", - }, -}; - function currentDataSource() { return sources.find((source) => source.name === dataOpsState.sourceName) || sources[0]; } @@ -2582,8 +3079,8 @@ function renderDataOpsOverview(source) {

更新与责任

数据类别
${source.categoryLabel} · ${source.commercial}
-
业务责任人
${source.steward}
-
技术责任人
${source.technicalOwner}
+
业务责任人
${source.stewardContact} · ${source.steward}
+
技术责任人
${source.technicalContact} · ${source.technicalOwner}
更新模式
${source.updateMode}
当前状态
${source.status}
@@ -2674,7 +3171,7 @@ function renderDataOpsAudit(source) { return `
权属与授权${source.rights}

${authorization}

-
业务责任人${source.steward}技术责任人${source.technicalOwner}授权状态${source.status}
+
业务责任人${source.stewardContact} · ${source.steward}技术责任人${source.technicalContact} · ${source.technicalOwner}授权状态${source.status}

最近审计事件

@@ -2721,17 +3218,23 @@ function renderDataOpsUsage() {
`; } -function renderDataIssue(issueKey) { - const issue = dataIssueCatalog[issueKey]; +function renderDataIssue(issueId) { + const issue = dataGapById(issueId); + const source = currentDataSource(); + const impactCount = reportsAffectedByGap(issue).length; return `
- ${issue.level}

${issue.title}

${issue.affected}

+ ${issue.drawerLevel}

${issue.drawerTitle}

${issue.affected}

规则命中原因

${issue.reason}

建议处置

${issue.action}

-

受影响分析

-
当前生产版本未覆盖分析快照沿用稳定版本责任部门${currentDataSource().steward}
+

受影响分析与补数责任

+
当前生产版本未覆盖影响报告${impactCount} 份分析快照沿用稳定版本
+
+
业务索取人
${source.stewardContact} · ${source.steward}
+
技术接口人
${source.technicalContact} · ${source.technicalOwner}
+
`; } @@ -2841,11 +3344,11 @@ function updateDataOpsDrawer() { } if (dataOpsState.context === "issue") { - const issue = dataIssueCatalog[dataOpsState.issue]; - const issueSource = sources.find((item) => item.name === issue.sourceName); + const issue = dataGapById(dataOpsState.issue); + const issueSource = sourceById(issue.sourceId); if (issueSource) dataOpsState.sourceName = issueSource.name; document.getElementById("dataOpsKicker").textContent = "DATA QUALITY WORKBENCH"; - document.getElementById("dataOpsTitle").textContent = issue.title; + document.getElementById("dataOpsTitle").textContent = issue.drawerTitle; document.getElementById("dataOpsDescription").textContent = "查看受影响数据、规则命中原因和可审计的处理流程。"; content.innerHTML = renderDataIssue(dataOpsState.issue); footer.innerHTML = ``; @@ -4707,6 +5210,7 @@ function init() { renderPartners(); renderTasks(); renderSources(); + renderDataReportLineage(); setupGenericInteractions(); setupRoleExperience(); setupPlatformActions(); @@ -4716,6 +5220,7 @@ function init() { setupForms(); setupFilters(); setupSourceFilters(); + setupDataReportLineage(); setupDataOperations(); setupWorkspaceDetails(); setupProductExperience(); diff --git a/demo/index.html b/demo/index.html index 868b364..a487269 100644 --- a/demo/index.html +++ b/demo/index.html @@ -870,6 +870,54 @@
+
+
+
+ DATA → ANALYSIS → REPORT +

数据与报告双向依赖图谱

+

从报告反查必需数据,也可从数据查看能形成的报告;缺口直接定位业务索取人和技术接口人。

+
+ 分析快照 V2026.03 +
+ +
+
+ + +
+ + +
+ +
+ +
+ 必需数据 + 可选增强 + 存在缺口 + 点击数据节点查看资产要求,点击报告节点进入在线报告。 +
+ +
+ +
+
+
+ DATA REQUEST DESK +

缺口与补数责任

+
+ 0 项待处理 +
+
+
+
+
@@ -918,7 +966,7 @@ 质量事项

待处理问题

- 4 项 + 3 项