feat: add compliant data connectors and lineage
- add authorized nine-source crawler connectors with security controls and packaged schemas - add the school internship contract to the vendor data standard center - add bidirectional campus data-to-report lineage with gap ownership and remediation Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
lofyer
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
92515928d9
commit
8dff9847e3
@@ -0,0 +1,313 @@
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from edu_crawler.errors import (
|
||||
AuthorizationRequired,
|
||||
ConfigurationError,
|
||||
PiiRejected,
|
||||
TaskOutputExists,
|
||||
)
|
||||
from edu_crawler.models import RawDocument
|
||||
from edu_crawler.runner import CrawlerRunner, RunConfig
|
||||
from edu_crawler.schema import validate_record
|
||||
|
||||
|
||||
def run_config(
|
||||
tmp_path: Path, source: str, input_path: Path, data_version: str
|
||||
) -> RunConfig:
|
||||
return RunConfig(
|
||||
source_code=source,
|
||||
input_location=str(input_path),
|
||||
task_id=f"JOB-20260824-{source.replace('_', '-')}-001",
|
||||
data_version=data_version,
|
||||
output_root=tmp_path / "output",
|
||||
authorization_reference="DPA-TEST-REF-001",
|
||||
tenant_id="TENANT-HZVTC" if source.startswith("SCHOOL_") else None,
|
||||
)
|
||||
|
||||
|
||||
def read_ndjson(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if line]
|
||||
|
||||
|
||||
def test_external_authorized_file_creates_append_only_artifacts(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html",
|
||||
"title": "云原生开发工程师",
|
||||
"description": "负责容器平台开发",
|
||||
"company_name": "示例数字科技有限公司",
|
||||
"region_code": "330100",
|
||||
"employment_type": "full_time",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
output = CrawlerRunner(config).run()
|
||||
summary = json.loads((output / "summary.json").read_text())
|
||||
records = read_ndjson(output / "standard.ndjson")
|
||||
assert summary["status"] == "COMPLETED"
|
||||
assert summary["standard_records"] == 1
|
||||
assert len(list((output / "raw").glob("*.metadata.json"))) == 1
|
||||
assert len(records) == 1
|
||||
validate_record("external-job-source.schema.json", records[0])
|
||||
with pytest.raises(TaskOutputExists):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_missing_external_authorization_is_blocked(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.json"
|
||||
source.write_text("{}")
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
config = replace(config, authorization_reference=None)
|
||||
with pytest.raises(AuthorizationRequired):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_network_url_is_disabled_without_explicit_flag(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-001",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
)
|
||||
with pytest.raises(ConfigurationError):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_explicit_zero_record_limit_is_rejected(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.json"
|
||||
source.write_text("{}")
|
||||
config = replace(run_config(tmp_path, "BOSS", source, "V2026.08.24"), max_records=0)
|
||||
with pytest.raises(ConfigurationError, match="max_records"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_explicit_zero_page_limit_is_rejected_without_request(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-001",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
allow_network=True,
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
contact="crawler@example.edu.cn",
|
||||
max_pages=0,
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="max_pages"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_local_file_size_limit_is_enforced_before_output(tmp_path: Path) -> None:
|
||||
source = tmp_path / "oversized.json"
|
||||
source.write_bytes(b"x" * (5 * 1024 * 1024 + 1))
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
with pytest.raises(ConfigurationError, match="size limit"):
|
||||
CrawlerRunner(config).run()
|
||||
assert not config.output_root.exists()
|
||||
|
||||
|
||||
def test_invalid_utf8_is_quarantined_without_payload_echo(tmp_path: Path) -> None:
|
||||
source = tmp_path / "invalid.ndjson"
|
||||
source.write_bytes(b"\xff\xfe")
|
||||
output = CrawlerRunner(run_config(tmp_path, "BOSS", source, "V2026.08.24")).run()
|
||||
quarantine = read_ndjson(output / "quarantine.ndjson")
|
||||
assert quarantine[0]["error_code"] == "SOURCE_STRUCTURE_CHANGED"
|
||||
assert quarantine[0]["error"] == "Structured source is not valid UTF-8"
|
||||
|
||||
|
||||
def test_record_limit_stops_network_document_generator_early(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-LIMIT",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
allow_network=True,
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
contact="crawler@example.edu.cn",
|
||||
max_records=1,
|
||||
)
|
||||
consumed: list[str] = []
|
||||
|
||||
def documents():
|
||||
for suffix in ("one", "two"):
|
||||
consumed.append(suffix)
|
||||
yield RawDocument.create(
|
||||
"BRZP",
|
||||
f"https://brzp.chinasme.cn/job/{suffix}",
|
||||
"application/json",
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"source_record_id": suffix,
|
||||
"source_url": f"https://brzp.chinasme.cn/job/{suffix}",
|
||||
"title": "测试岗位",
|
||||
"description": "测试职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
]
|
||||
).encode(),
|
||||
)
|
||||
|
||||
runner = CrawlerRunner(config)
|
||||
runner._network_documents = documents
|
||||
output = runner.run()
|
||||
assert consumed == ["one"]
|
||||
assert json.loads((output / "summary.json").read_text())["raw_documents"] == 1
|
||||
|
||||
|
||||
def test_valid_school_employment_record(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"cohort": 2026,
|
||||
"college_code": "C01",
|
||||
"program_code": "510203",
|
||||
"employment_status": "employed",
|
||||
"destination_type": "enterprise",
|
||||
"enterprise_token": "ENT_" + "b" * 24,
|
||||
"confirmed_at": "2026-08-23T08:00:00+08:00",
|
||||
"confirmation_role": "system",
|
||||
"source_system": "EMPLOYMENT-VIEW",
|
||||
"source_record_id": "EMP-001",
|
||||
"source_updated_at": "2026-08-23T08:00:00+08:00",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
output = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
).run()
|
||||
record = read_ndjson(output / "standard.ndjson")[0]
|
||||
validate_record("school-employment.schema.json", record)
|
||||
assert record["tenant_id"] == "TENANT-HZVTC"
|
||||
|
||||
|
||||
def test_school_connector_requires_expected_tenant(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text("{}\n")
|
||||
config = replace(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01"),
|
||||
tenant_id=None,
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="tenant-id"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_school_connector_rejects_cross_tenant_batch_before_raw(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps({"tenant_id": "TENANT-OTHER", "source_record_id": "EMP-001"}) + "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
with pytest.raises(ConfigurationError, match="outside the configured tenant"):
|
||||
CrawlerRunner(config).run()
|
||||
assert not any(path.is_file() for path in config.output_root.rglob("*"))
|
||||
|
||||
|
||||
def test_school_pii_is_rejected_before_raw_persistence(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"source_record_id": "EMP-001",
|
||||
"profile": {"student_name": "敏感姓名"},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
with pytest.raises(PiiRejected):
|
||||
CrawlerRunner(config).run()
|
||||
assert not any(config.output_root.rglob(config.task_id))
|
||||
|
||||
|
||||
def test_valid_school_internship_record(tmp_path: Path) -> None:
|
||||
source = tmp_path / "internship.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"internship_id": "INT-001",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"program_code": "510203",
|
||||
"enterprise_token": "ENT_" + "b" * 24,
|
||||
"job_title": "软件开发实习生",
|
||||
"start_date": "2026-07-01",
|
||||
"status": "active",
|
||||
"source_system": "INTERNSHIP-API",
|
||||
"source_record_id": "INT-001",
|
||||
"source_updated_at": "2026-08-23T08:00:00+08:00",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
output = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_INTERNSHIP", source, "SCH-INT-20260824-01")
|
||||
).run()
|
||||
record = read_ndjson(output / "standard.ndjson")[0]
|
||||
validate_record("school-internship.schema.json", record)
|
||||
assert record["internship_id"] == "INT-001"
|
||||
assert output.parts[-3:] == (
|
||||
"SCHOOL_INTERNSHIP",
|
||||
"TENANT-HZVTC",
|
||||
"JOB-20260824-SCHOOL-INTERNSHIP-001",
|
||||
)
|
||||
|
||||
|
||||
def test_school_content_hash_ignores_sync_metadata(tmp_path: Path) -> None:
|
||||
first_source = tmp_path / "employment-one.ndjson"
|
||||
second_source = tmp_path / "employment-two.ndjson"
|
||||
base = {
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"cohort": 2026,
|
||||
"college_code": "C01",
|
||||
"program_code": "510203",
|
||||
"employment_status": "further_study",
|
||||
"destination_type": "further_study",
|
||||
"confirmed_at": "2026-08-23T08:00:00+08:00",
|
||||
"confirmation_role": "system",
|
||||
"source_system": "EMPLOYMENT-VIEW",
|
||||
"source_record_id": "EMP-001",
|
||||
}
|
||||
first_source.write_text(
|
||||
json.dumps({**base, "source_updated_at": "2026-08-23T08:00:00+08:00"}) + "\n"
|
||||
)
|
||||
second_source.write_text(
|
||||
json.dumps({**base, "source_updated_at": "2026-08-24T08:00:00+08:00"}) + "\n"
|
||||
)
|
||||
first = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", first_source, "SCH-EMP-20260824-01")
|
||||
).run()
|
||||
second_config = replace(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", second_source, "SCH-EMP-20260824-01"),
|
||||
task_id="JOB-20260824-SCHOOL-EMP-002",
|
||||
)
|
||||
second = CrawlerRunner(second_config).run()
|
||||
assert (
|
||||
read_ndjson(first / "standard.ndjson")[0]["content_hash"]
|
||||
== read_ndjson(second / "standard.ndjson")[0]["content_hash"]
|
||||
)
|
||||
Reference in New Issue
Block a user