edu-industry/admin-tools/crawler/tests/test_runner.py

import json
from dataclasses import replace
from pathlib import Path

import pytest

from edu_crawler.errors import (
    AuthorizationRequired,
    ConfigurationError,
    PiiRejected,
    TaskOutputExists,
)
from edu_crawler.models import RawDocument
from edu_crawler.runner import CrawlerRunner, RunConfig
from edu_crawler.schema import validate_record


def run_config(
    tmp_path: Path, source: str, input_path: Path, data_version: str
) -> RunConfig:
    return RunConfig(
        source_code=source,
        input_location=str(input_path),
        task_id=f"JOB-20260824-{source.replace('_', '-')}-001",
        data_version=data_version,
        output_root=tmp_path / "output",
        authorization_reference="DPA-TEST-REF-001",
        tenant_id="TENANT-HZVTC" if source.startswith("SCHOOL_") else None,
    )


def read_ndjson(path: Path) -> list[dict]:
    return [json.loads(line) for line in path.read_text().splitlines() if line]


def test_external_authorized_file_creates_append_only_artifacts(tmp_path: Path) -> None:
    source = tmp_path / "boss.ndjson"
    source.write_text(
        json.dumps(
            {
                "source_record_id": "BOSS-001",
                "source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html",
                "title": "云原生开发工程师",
                "description": "负责容器平台开发",
                "company_name": "示例数字科技有限公司",
                "region_code": "330100",
                "employment_type": "full_time",
            },
            ensure_ascii=False,
        )
        + "\n"
    )
    config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
    output = CrawlerRunner(config).run()
    summary = json.loads((output / "summary.json").read_text())
    records = read_ndjson(output / "standard.ndjson")
    assert summary["status"] == "COMPLETED"
    assert summary["standard_records"] == 1
    assert len(list((output / "raw").glob("*.metadata.json"))) == 1
    assert len(records) == 1
    validate_record("external-job-source.schema.json", records[0])
    with pytest.raises(TaskOutputExists):
        CrawlerRunner(config).run()


def test_missing_external_authorization_is_blocked(tmp_path: Path) -> None:
    source = tmp_path / "boss.json"
    source.write_text("{}")
    config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
    config = replace(config, authorization_reference=None)
    with pytest.raises(AuthorizationRequired):
        CrawlerRunner(config).run()


def test_network_url_is_disabled_without_explicit_flag(tmp_path: Path) -> None:
    config = RunConfig(
        source_code="BRZP",
        input_location="https://brzp.chinasme.cn/jobs",
        task_id="JOB-20260824-BRZP-001",
        data_version="V2026.08.24",
        output_root=tmp_path,
        authorization_reference="AUTH-BRZP-001",
    )
    with pytest.raises(ConfigurationError):
        CrawlerRunner(config).run()


def test_explicit_zero_record_limit_is_rejected(tmp_path: Path) -> None:
    source = tmp_path / "boss.json"
    source.write_text("{}")
    config = replace(run_config(tmp_path, "BOSS", source, "V2026.08.24"), max_records=0)
    with pytest.raises(ConfigurationError, match="max_records"):
        CrawlerRunner(config).run()


def test_explicit_zero_page_limit_is_rejected_without_request(tmp_path: Path) -> None:
    config = RunConfig(
        source_code="BRZP",
        input_location="https://brzp.chinasme.cn/jobs",
        task_id="JOB-20260824-BRZP-001",
        data_version="V2026.08.24",
        output_root=tmp_path,
        authorization_reference="AUTH-BRZP-001",
        allow_network=True,
        approved_hosts=("brzp.chinasme.cn",),
        contact="crawler@example.edu.cn",
        max_pages=0,
    )
    with pytest.raises(ConfigurationError, match="max_pages"):
        CrawlerRunner(config).run()


def test_local_file_size_limit_is_enforced_before_output(tmp_path: Path) -> None:
    source = tmp_path / "oversized.json"
    source.write_bytes(b"x" * (5 * 1024 * 1024 + 1))
    config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
    with pytest.raises(ConfigurationError, match="size limit"):
        CrawlerRunner(config).run()
    assert not config.output_root.exists()


def test_invalid_utf8_is_quarantined_without_payload_echo(tmp_path: Path) -> None:
    source = tmp_path / "invalid.ndjson"
    source.write_bytes(b"\xff\xfe")
    output = CrawlerRunner(run_config(tmp_path, "BOSS", source, "V2026.08.24")).run()
    quarantine = read_ndjson(output / "quarantine.ndjson")
    assert quarantine[0]["error_code"] == "SOURCE_STRUCTURE_CHANGED"
    assert quarantine[0]["error"] == "Structured source is not valid UTF-8"


def test_record_limit_stops_network_document_generator_early(tmp_path: Path) -> None:
    config = RunConfig(
        source_code="BRZP",
        input_location="https://brzp.chinasme.cn/jobs",
        task_id="JOB-20260824-BRZP-LIMIT",
        data_version="V2026.08.24",
        output_root=tmp_path,
        authorization_reference="AUTH-BRZP-001",
        allow_network=True,
        approved_hosts=("brzp.chinasme.cn",),
        contact="crawler@example.edu.cn",
        max_records=1,
    )
    consumed: list[str] = []

    def documents():
        for suffix in ("one", "two"):
            consumed.append(suffix)
            yield RawDocument.create(
                "BRZP",
                f"https://brzp.chinasme.cn/job/{suffix}",
                "application/json",
                json.dumps(
                    [
                        {
                            "source_record_id": suffix,
                            "source_url": f"https://brzp.chinasme.cn/job/{suffix}",
                            "title": "测试岗位",
                            "description": "测试职责",
                            "company_name": "测试企业",
                            "region_code": "330100",
                        }
                    ]
                ).encode(),
            )

    runner = CrawlerRunner(config)
    runner._network_documents = documents
    output = runner.run()
    assert consumed == ["one"]
    assert json.loads((output / "summary.json").read_text())["raw_documents"] == 1


def test_valid_school_employment_record(tmp_path: Path) -> None:
    source = tmp_path / "employment.ndjson"
    source.write_text(
        json.dumps(
            {
                "tenant_id": "TENANT-HZVTC",
                "student_token": "STU_" + "a" * 32,
                "cohort": 2026,
                "college_code": "C01",
                "program_code": "510203",
                "employment_status": "employed",
                "destination_type": "enterprise",
                "enterprise_token": "ENT_" + "b" * 24,
                "confirmed_at": "2026-08-23T08:00:00+08:00",
                "confirmation_role": "system",
                "source_system": "EMPLOYMENT-VIEW",
                "source_record_id": "EMP-001",
                "source_updated_at": "2026-08-23T08:00:00+08:00",
            },
            ensure_ascii=False,
        )
        + "\n"
    )
    output = CrawlerRunner(
        run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
    ).run()
    record = read_ndjson(output / "standard.ndjson")[0]
    validate_record("school-employment.schema.json", record)
    assert record["tenant_id"] == "TENANT-HZVTC"


def test_school_connector_requires_expected_tenant(tmp_path: Path) -> None:
    source = tmp_path / "employment.ndjson"
    source.write_text("{}\n")
    config = replace(
        run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01"),
        tenant_id=None,
    )
    with pytest.raises(ConfigurationError, match="tenant-id"):
        CrawlerRunner(config).run()


def test_school_connector_rejects_cross_tenant_batch_before_raw(tmp_path: Path) -> None:
    source = tmp_path / "employment.ndjson"
    source.write_text(
        json.dumps({"tenant_id": "TENANT-OTHER", "source_record_id": "EMP-001"}) + "\n"
    )
    config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
    with pytest.raises(ConfigurationError, match="outside the configured tenant"):
        CrawlerRunner(config).run()
    assert not any(path.is_file() for path in config.output_root.rglob("*"))


def test_school_pii_is_rejected_before_raw_persistence(tmp_path: Path) -> None:
    source = tmp_path / "employment.ndjson"
    source.write_text(
        json.dumps(
            {
                "tenant_id": "TENANT-HZVTC",
                "source_record_id": "EMP-001",
                "profile": {"student_name": "敏感姓名"},
            },
            ensure_ascii=False,
        )
        + "\n"
    )
    config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
    with pytest.raises(PiiRejected):
        CrawlerRunner(config).run()
    assert not any(config.output_root.rglob(config.task_id))


def test_valid_school_internship_record(tmp_path: Path) -> None:
    source = tmp_path / "internship.ndjson"
    source.write_text(
        json.dumps(
            {
                "tenant_id": "TENANT-HZVTC",
                "internship_id": "INT-001",
                "student_token": "STU_" + "a" * 32,
                "program_code": "510203",
                "enterprise_token": "ENT_" + "b" * 24,
                "job_title": "软件开发实习生",
                "start_date": "2026-07-01",
                "status": "active",
                "source_system": "INTERNSHIP-API",
                "source_record_id": "INT-001",
                "source_updated_at": "2026-08-23T08:00:00+08:00",
            },
            ensure_ascii=False,
        )
        + "\n"
    )
    output = CrawlerRunner(
        run_config(tmp_path, "SCHOOL_INTERNSHIP", source, "SCH-INT-20260824-01")
    ).run()
    record = read_ndjson(output / "standard.ndjson")[0]
    validate_record("school-internship.schema.json", record)
    assert record["internship_id"] == "INT-001"
    assert output.parts[-3:] == (
        "SCHOOL_INTERNSHIP",
        "TENANT-HZVTC",
        "JOB-20260824-SCHOOL-INTERNSHIP-001",
    )


def test_school_content_hash_ignores_sync_metadata(tmp_path: Path) -> None:
    first_source = tmp_path / "employment-one.ndjson"
    second_source = tmp_path / "employment-two.ndjson"
    base = {
        "tenant_id": "TENANT-HZVTC",
        "student_token": "STU_" + "a" * 32,
        "cohort": 2026,
        "college_code": "C01",
        "program_code": "510203",
        "employment_status": "further_study",
        "destination_type": "further_study",
        "confirmed_at": "2026-08-23T08:00:00+08:00",
        "confirmation_role": "system",
        "source_system": "EMPLOYMENT-VIEW",
        "source_record_id": "EMP-001",
    }
    first_source.write_text(
        json.dumps({**base, "source_updated_at": "2026-08-23T08:00:00+08:00"}) + "\n"
    )
    second_source.write_text(
        json.dumps({**base, "source_updated_at": "2026-08-24T08:00:00+08:00"}) + "\n"
    )
    first = CrawlerRunner(
        run_config(tmp_path, "SCHOOL_EMPLOYMENT", first_source, "SCH-EMP-20260824-01")
    ).run()
    second_config = replace(
        run_config(tmp_path, "SCHOOL_EMPLOYMENT", second_source, "SCH-EMP-20260824-01"),
        task_id="JOB-20260824-SCHOOL-EMP-002",
    )
    second = CrawlerRunner(second_config).run()
    assert (
        read_ndjson(first / "standard.ndjson")[0]["content_hash"]
        == read_ndjson(second / "standard.ndjson")[0]["content_hash"]
    )