feat: add compliant data connectors and lineage
- add authorized nine-source crawler connectors with security controls and packaged schemas - add the school internship contract to the vendor data standard center - add bidirectional campus data-to-report lineage with gap ownership and remediation Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
lofyer
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
92515928d9
commit
8dff9847e3
@@ -0,0 +1,255 @@
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from edu_crawler.connectors import CONNECTOR_TYPES, connector_for
|
||||
from edu_crawler.errors import SourceStructureChanged
|
||||
from edu_crawler.models import ParsedItem
|
||||
from edu_crawler.schema import validate_record
|
||||
|
||||
|
||||
def parsed(value: dict, *, payload_hash: str = "a" * 64) -> ParsedItem:
|
||||
return ParsedItem(
|
||||
value=value,
|
||||
raw_snapshot_id="RAW-BOSS-20260824-ABCDEF012345",
|
||||
source_payload_hash=payload_hash,
|
||||
collected_at="2026-08-24T10:00:00+08:00",
|
||||
source_url="file:///authorized.ndjson",
|
||||
)
|
||||
|
||||
|
||||
def test_all_nine_sources_have_concrete_connector_types() -> None:
|
||||
assert set(CONNECTOR_TYPES) == {
|
||||
"BOSS",
|
||||
"ZHAOPIN",
|
||||
"BRZP",
|
||||
"NCSS",
|
||||
"MOHRSS",
|
||||
"IGUOPIN",
|
||||
"HZRC",
|
||||
"SCHOOL_EMPLOYMENT",
|
||||
"SCHOOL_INTERNSHIP",
|
||||
}
|
||||
assert all(
|
||||
connector_for(code).describe()["source_code"] == code
|
||||
for code in CONNECTOR_TYPES
|
||||
)
|
||||
|
||||
|
||||
def test_external_record_normalizes_and_redacts_contact() -> None:
|
||||
connector = connector_for("BOSS")
|
||||
source = {
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html",
|
||||
"title": "云原生开发工程师",
|
||||
"description": "<p>负责容器平台开发，联系 13800138000</p>",
|
||||
"company_name": "示例数字科技有限公司",
|
||||
"region_code": "330100",
|
||||
"employment_type": "全职",
|
||||
"published_at": "2026-08-23T08:00:00+08:00",
|
||||
"salary_min": "12000",
|
||||
"salary_max": "18000",
|
||||
"salary_period": "month",
|
||||
"skills": ["Docker", "Kubernetes", "Docker"],
|
||||
}
|
||||
record = connector.normalize(
|
||||
parsed(source),
|
||||
task_id="JOB-20260824-BOSS-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
validate_record(connector.policy.target_schema, record)
|
||||
assert record["source_platform"] == "BOSS"
|
||||
assert record["job_description"] == "负责容器平台开发，联系 [CONTACT_REDACTED]"
|
||||
assert record["skills"] == ["Docker", "Kubernetes"]
|
||||
assert record["source_payload_hash"] == "a" * 64
|
||||
|
||||
|
||||
def test_annual_salary_is_normalized_to_monthly_and_unknown_status_stays_unknown() -> (
|
||||
None
|
||||
):
|
||||
connector = connector_for("BOSS")
|
||||
source = {
|
||||
"source_record_id": "BOSS-ANNUAL-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/BOSS-ANNUAL-001.html",
|
||||
"title": "年度薪资测试岗位",
|
||||
"description": "用于验证统一月薪折算",
|
||||
"company_name": "示例企业",
|
||||
"region_code": "330100",
|
||||
"salary_min": 120000,
|
||||
"salary_max": 240000,
|
||||
"salary_period": "year",
|
||||
"status": "new-upstream-code",
|
||||
}
|
||||
record = connector.normalize(
|
||||
parsed(source),
|
||||
task_id="JOB-20260824-BOSS-ANNUAL",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
assert record["salary_min"] == 10000
|
||||
assert record["salary_max"] == 20000
|
||||
assert record["salary_period"] == "year"
|
||||
assert record["status"] == "unknown"
|
||||
|
||||
|
||||
def test_content_hash_ignores_collection_metadata() -> None:
|
||||
connector = connector_for("BOSS")
|
||||
source = {
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html",
|
||||
"title": "云原生开发工程师",
|
||||
"description": "负责容器平台开发",
|
||||
"company_name": "示例数字科技有限公司",
|
||||
"region_code": "330100",
|
||||
}
|
||||
first = connector.normalize(
|
||||
parsed(source, payload_hash="a" * 64),
|
||||
task_id="JOB-20260824-BOSS-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
second_item = replace(
|
||||
parsed(source, payload_hash="b" * 64),
|
||||
raw_snapshot_id="RAW-BOSS-20260825-012345ABCDEF",
|
||||
collected_at="2026-08-25T10:00:00+08:00",
|
||||
)
|
||||
second = connector.normalize(
|
||||
second_item,
|
||||
task_id="JOB-20260825-BOSS-001",
|
||||
data_version="V2026.08.25",
|
||||
default_region_code=None,
|
||||
)
|
||||
assert first["content_hash"] == second["content_hash"]
|
||||
assert first["source_payload_hash"] != second["source_payload_hash"]
|
||||
|
||||
|
||||
def test_source_record_url_must_match_source_allowlist() -> None:
|
||||
connector = connector_for("BOSS")
|
||||
source = {
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://evil.example/job/1",
|
||||
"title": "测试岗位",
|
||||
"description": "测试职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
try:
|
||||
connector.normalize(
|
||||
parsed(source),
|
||||
task_id="JOB-20260824-BOSS-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
except Exception as exc:
|
||||
assert getattr(exc, "code", None) == "SOURCE_STRUCTURE_CHANGED"
|
||||
else:
|
||||
raise AssertionError("unapproved source URL was accepted")
|
||||
|
||||
|
||||
def test_declared_source_must_match_selected_connector() -> None:
|
||||
connector = connector_for("BOSS")
|
||||
source = {
|
||||
"source_platform": "ZHAOPIN",
|
||||
"source_record_id": "ZL-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/1.html",
|
||||
"title": "测试岗位",
|
||||
"description": "测试岗位职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
with pytest.raises(SourceStructureChanged) as caught:
|
||||
connector.normalize(
|
||||
parsed(source),
|
||||
task_id="JOB-20260824-BOSS-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
assert getattr(caught.value, "code", None) == "SOURCE_STRUCTURE_CHANGED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_code", "source_url"),
|
||||
[
|
||||
("BOSS", "https://www.bosszhipin.com/job_detail/1.html"),
|
||||
("ZHAOPIN", "https://www.zhaopin.com/jobdetail/1.htm"),
|
||||
("BRZP", "https://brzp.chinasme.cn/brzp/job/1"),
|
||||
("NCSS", "https://www.ncss.cn/student/jobs/1"),
|
||||
("MOHRSS", "http://job.mohrss.gov.cn/cjobs/jobinfo/1"),
|
||||
("IGUOPIN", "https://job.iguopin.com/job/detail?id=1"),
|
||||
("HZRC", "https://www.hzrc.com/job/1"),
|
||||
],
|
||||
)
|
||||
def test_each_external_source_normalizes_authorized_feed(
|
||||
source_code: str, source_url: str
|
||||
) -> None:
|
||||
connector = connector_for(source_code)
|
||||
record = connector.normalize(
|
||||
parsed(
|
||||
{
|
||||
"source_record_id": f"{source_code}-001",
|
||||
"source_url": source_url,
|
||||
"title": "测试岗位",
|
||||
"description": "测试岗位职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
),
|
||||
task_id=f"JOB-20260824-{source_code}-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
validate_record(connector.policy.target_schema, record)
|
||||
assert record["source_platform"] == source_code
|
||||
|
||||
|
||||
def test_source_url_secrets_are_redacted() -> None:
|
||||
connector = connector_for("BOSS")
|
||||
record = connector.normalize(
|
||||
parsed(
|
||||
{
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/1.html?token=secret&city=330100#private",
|
||||
"title": "测试岗位",
|
||||
"description": "测试岗位职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
),
|
||||
task_id="JOB-20260824-BOSS-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code=None,
|
||||
)
|
||||
assert "secret" not in record["source_url"]
|
||||
assert "private" not in record["source_url"]
|
||||
assert "city=330100" in record["source_url"]
|
||||
|
||||
|
||||
def test_schema_org_job_posting_mapping() -> None:
|
||||
connector = connector_for("BRZP")
|
||||
source = {
|
||||
"@type": "JobPosting",
|
||||
"identifier": {"value": "BRZP-001"},
|
||||
"url": "https://brzp.chinasme.cn/brzp/job/BRZP-001",
|
||||
"title": "跨境电商运营实习生",
|
||||
"description": "协助店铺运营和数据分析",
|
||||
"hiringOrganization": {"name": "示例跨境服务有限公司"},
|
||||
"jobLocation": {"address": {"addressLocality": "杭州市"}},
|
||||
"employmentType": "internship",
|
||||
"datePosted": "2026-08-23T08:00:00+08:00",
|
||||
"baseSalary": {
|
||||
"currency": "CNY",
|
||||
"value": {"minValue": 3500, "maxValue": 5000, "unitText": "MONTH"},
|
||||
},
|
||||
}
|
||||
record = connector.normalize(
|
||||
parsed(source),
|
||||
task_id="JOB-20260824-BRZP-001",
|
||||
data_version="V2026.08.24",
|
||||
default_region_code="330100",
|
||||
)
|
||||
validate_record(connector.policy.target_schema, record)
|
||||
assert record["employment_type"] == "internship"
|
||||
assert record["salary_period"] == "month"
|
||||
assert json.loads(json.dumps(record, ensure_ascii=False)) == record
|
||||
@@ -0,0 +1,62 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from edu_crawler.errors import SourceStructureChanged
|
||||
from edu_crawler.models import RawDocument
|
||||
from edu_crawler.parsing import discover_job_links, parse_document
|
||||
from edu_crawler.policies import source_policy
|
||||
|
||||
|
||||
def test_parse_schema_org_job_posting() -> None:
|
||||
body = b"""
|
||||
<html><head><script type="application/ld+json">
|
||||
{"@context":"https://schema.org","@type":"JobPosting","identifier":{"value":"BRZP-1"},
|
||||
"url":"https://brzp.chinasme.cn/job/BRZP-1","title":"Test","description":"Description",
|
||||
"hiringOrganization":{"name":"Example"}}
|
||||
</script></head></html>
|
||||
"""
|
||||
document = RawDocument.create(
|
||||
"BRZP", "https://brzp.chinasme.cn/job/BRZP-1", "text/html", body
|
||||
)
|
||||
records = parse_document(document, allow_html=True)
|
||||
assert records[0]["@type"] == "JobPosting"
|
||||
|
||||
|
||||
def test_discover_links_uses_versioned_patterns_and_deduplicates() -> None:
|
||||
body = b"""
|
||||
<a href="/job/1">one</a>
|
||||
<a href="/job/1">duplicate</a>
|
||||
<a href="/company/1">company</a>
|
||||
"""
|
||||
document = RawDocument.create(
|
||||
"BRZP", "https://brzp.chinasme.cn/", "text/html", body
|
||||
)
|
||||
assert discover_job_links(document, source_policy("BRZP")) == [
|
||||
"https://brzp.chinasme.cn/job/1"
|
||||
]
|
||||
|
||||
|
||||
def test_invalid_ndjson_reports_line_without_payload() -> None:
|
||||
document = RawDocument.create(
|
||||
"BOSS",
|
||||
"file:///authorized.ndjson",
|
||||
"application/x-ndjson",
|
||||
b'{"id":"one"}\nnot-json\n',
|
||||
)
|
||||
with pytest.raises(SourceStructureChanged, match="line 2") as caught:
|
||||
parse_document(document, allow_html=False)
|
||||
assert "not-json" not in str(caught.value)
|
||||
|
||||
|
||||
def test_json_collection_wrappers() -> None:
|
||||
document = RawDocument.create(
|
||||
"BOSS",
|
||||
"file:///authorized.json",
|
||||
"application/json",
|
||||
json.dumps({"data": {"items": [{"id": "one"}, {"id": "two"}]}}).encode(),
|
||||
)
|
||||
assert [record["id"] for record in parse_document(document, allow_html=False)] == [
|
||||
"one",
|
||||
"two",
|
||||
]
|
||||
@@ -0,0 +1,313 @@
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from edu_crawler.errors import (
|
||||
AuthorizationRequired,
|
||||
ConfigurationError,
|
||||
PiiRejected,
|
||||
TaskOutputExists,
|
||||
)
|
||||
from edu_crawler.models import RawDocument
|
||||
from edu_crawler.runner import CrawlerRunner, RunConfig
|
||||
from edu_crawler.schema import validate_record
|
||||
|
||||
|
||||
def run_config(
|
||||
tmp_path: Path, source: str, input_path: Path, data_version: str
|
||||
) -> RunConfig:
|
||||
return RunConfig(
|
||||
source_code=source,
|
||||
input_location=str(input_path),
|
||||
task_id=f"JOB-20260824-{source.replace('_', '-')}-001",
|
||||
data_version=data_version,
|
||||
output_root=tmp_path / "output",
|
||||
authorization_reference="DPA-TEST-REF-001",
|
||||
tenant_id="TENANT-HZVTC" if source.startswith("SCHOOL_") else None,
|
||||
)
|
||||
|
||||
|
||||
def read_ndjson(path: Path) -> list[dict]:
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if line]
|
||||
|
||||
|
||||
def test_external_authorized_file_creates_append_only_artifacts(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"source_record_id": "BOSS-001",
|
||||
"source_url": "https://www.bosszhipin.com/job_detail/BOSS-001.html",
|
||||
"title": "云原生开发工程师",
|
||||
"description": "负责容器平台开发",
|
||||
"company_name": "示例数字科技有限公司",
|
||||
"region_code": "330100",
|
||||
"employment_type": "full_time",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
output = CrawlerRunner(config).run()
|
||||
summary = json.loads((output / "summary.json").read_text())
|
||||
records = read_ndjson(output / "standard.ndjson")
|
||||
assert summary["status"] == "COMPLETED"
|
||||
assert summary["standard_records"] == 1
|
||||
assert len(list((output / "raw").glob("*.metadata.json"))) == 1
|
||||
assert len(records) == 1
|
||||
validate_record("external-job-source.schema.json", records[0])
|
||||
with pytest.raises(TaskOutputExists):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_missing_external_authorization_is_blocked(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.json"
|
||||
source.write_text("{}")
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
config = replace(config, authorization_reference=None)
|
||||
with pytest.raises(AuthorizationRequired):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_network_url_is_disabled_without_explicit_flag(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-001",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
)
|
||||
with pytest.raises(ConfigurationError):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_explicit_zero_record_limit_is_rejected(tmp_path: Path) -> None:
|
||||
source = tmp_path / "boss.json"
|
||||
source.write_text("{}")
|
||||
config = replace(run_config(tmp_path, "BOSS", source, "V2026.08.24"), max_records=0)
|
||||
with pytest.raises(ConfigurationError, match="max_records"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_explicit_zero_page_limit_is_rejected_without_request(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-001",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
allow_network=True,
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
contact="crawler@example.edu.cn",
|
||||
max_pages=0,
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="max_pages"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_local_file_size_limit_is_enforced_before_output(tmp_path: Path) -> None:
|
||||
source = tmp_path / "oversized.json"
|
||||
source.write_bytes(b"x" * (5 * 1024 * 1024 + 1))
|
||||
config = run_config(tmp_path, "BOSS", source, "V2026.08.24")
|
||||
with pytest.raises(ConfigurationError, match="size limit"):
|
||||
CrawlerRunner(config).run()
|
||||
assert not config.output_root.exists()
|
||||
|
||||
|
||||
def test_invalid_utf8_is_quarantined_without_payload_echo(tmp_path: Path) -> None:
|
||||
source = tmp_path / "invalid.ndjson"
|
||||
source.write_bytes(b"\xff\xfe")
|
||||
output = CrawlerRunner(run_config(tmp_path, "BOSS", source, "V2026.08.24")).run()
|
||||
quarantine = read_ndjson(output / "quarantine.ndjson")
|
||||
assert quarantine[0]["error_code"] == "SOURCE_STRUCTURE_CHANGED"
|
||||
assert quarantine[0]["error"] == "Structured source is not valid UTF-8"
|
||||
|
||||
|
||||
def test_record_limit_stops_network_document_generator_early(tmp_path: Path) -> None:
|
||||
config = RunConfig(
|
||||
source_code="BRZP",
|
||||
input_location="https://brzp.chinasme.cn/jobs",
|
||||
task_id="JOB-20260824-BRZP-LIMIT",
|
||||
data_version="V2026.08.24",
|
||||
output_root=tmp_path,
|
||||
authorization_reference="AUTH-BRZP-001",
|
||||
allow_network=True,
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
contact="crawler@example.edu.cn",
|
||||
max_records=1,
|
||||
)
|
||||
consumed: list[str] = []
|
||||
|
||||
def documents():
|
||||
for suffix in ("one", "two"):
|
||||
consumed.append(suffix)
|
||||
yield RawDocument.create(
|
||||
"BRZP",
|
||||
f"https://brzp.chinasme.cn/job/{suffix}",
|
||||
"application/json",
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"source_record_id": suffix,
|
||||
"source_url": f"https://brzp.chinasme.cn/job/{suffix}",
|
||||
"title": "测试岗位",
|
||||
"description": "测试职责",
|
||||
"company_name": "测试企业",
|
||||
"region_code": "330100",
|
||||
}
|
||||
]
|
||||
).encode(),
|
||||
)
|
||||
|
||||
runner = CrawlerRunner(config)
|
||||
runner._network_documents = documents
|
||||
output = runner.run()
|
||||
assert consumed == ["one"]
|
||||
assert json.loads((output / "summary.json").read_text())["raw_documents"] == 1
|
||||
|
||||
|
||||
def test_valid_school_employment_record(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"cohort": 2026,
|
||||
"college_code": "C01",
|
||||
"program_code": "510203",
|
||||
"employment_status": "employed",
|
||||
"destination_type": "enterprise",
|
||||
"enterprise_token": "ENT_" + "b" * 24,
|
||||
"confirmed_at": "2026-08-23T08:00:00+08:00",
|
||||
"confirmation_role": "system",
|
||||
"source_system": "EMPLOYMENT-VIEW",
|
||||
"source_record_id": "EMP-001",
|
||||
"source_updated_at": "2026-08-23T08:00:00+08:00",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
output = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
).run()
|
||||
record = read_ndjson(output / "standard.ndjson")[0]
|
||||
validate_record("school-employment.schema.json", record)
|
||||
assert record["tenant_id"] == "TENANT-HZVTC"
|
||||
|
||||
|
||||
def test_school_connector_requires_expected_tenant(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text("{}\n")
|
||||
config = replace(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01"),
|
||||
tenant_id=None,
|
||||
)
|
||||
with pytest.raises(ConfigurationError, match="tenant-id"):
|
||||
CrawlerRunner(config).run()
|
||||
|
||||
|
||||
def test_school_connector_rejects_cross_tenant_batch_before_raw(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps({"tenant_id": "TENANT-OTHER", "source_record_id": "EMP-001"}) + "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
with pytest.raises(ConfigurationError, match="outside the configured tenant"):
|
||||
CrawlerRunner(config).run()
|
||||
assert not any(path.is_file() for path in config.output_root.rglob("*"))
|
||||
|
||||
|
||||
def test_school_pii_is_rejected_before_raw_persistence(tmp_path: Path) -> None:
|
||||
source = tmp_path / "employment.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"source_record_id": "EMP-001",
|
||||
"profile": {"student_name": "敏感姓名"},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
config = run_config(tmp_path, "SCHOOL_EMPLOYMENT", source, "SCH-EMP-20260824-01")
|
||||
with pytest.raises(PiiRejected):
|
||||
CrawlerRunner(config).run()
|
||||
assert not any(config.output_root.rglob(config.task_id))
|
||||
|
||||
|
||||
def test_valid_school_internship_record(tmp_path: Path) -> None:
|
||||
source = tmp_path / "internship.ndjson"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"internship_id": "INT-001",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"program_code": "510203",
|
||||
"enterprise_token": "ENT_" + "b" * 24,
|
||||
"job_title": "软件开发实习生",
|
||||
"start_date": "2026-07-01",
|
||||
"status": "active",
|
||||
"source_system": "INTERNSHIP-API",
|
||||
"source_record_id": "INT-001",
|
||||
"source_updated_at": "2026-08-23T08:00:00+08:00",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
output = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_INTERNSHIP", source, "SCH-INT-20260824-01")
|
||||
).run()
|
||||
record = read_ndjson(output / "standard.ndjson")[0]
|
||||
validate_record("school-internship.schema.json", record)
|
||||
assert record["internship_id"] == "INT-001"
|
||||
assert output.parts[-3:] == (
|
||||
"SCHOOL_INTERNSHIP",
|
||||
"TENANT-HZVTC",
|
||||
"JOB-20260824-SCHOOL-INTERNSHIP-001",
|
||||
)
|
||||
|
||||
|
||||
def test_school_content_hash_ignores_sync_metadata(tmp_path: Path) -> None:
|
||||
first_source = tmp_path / "employment-one.ndjson"
|
||||
second_source = tmp_path / "employment-two.ndjson"
|
||||
base = {
|
||||
"tenant_id": "TENANT-HZVTC",
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"cohort": 2026,
|
||||
"college_code": "C01",
|
||||
"program_code": "510203",
|
||||
"employment_status": "further_study",
|
||||
"destination_type": "further_study",
|
||||
"confirmed_at": "2026-08-23T08:00:00+08:00",
|
||||
"confirmation_role": "system",
|
||||
"source_system": "EMPLOYMENT-VIEW",
|
||||
"source_record_id": "EMP-001",
|
||||
}
|
||||
first_source.write_text(
|
||||
json.dumps({**base, "source_updated_at": "2026-08-23T08:00:00+08:00"}) + "\n"
|
||||
)
|
||||
second_source.write_text(
|
||||
json.dumps({**base, "source_updated_at": "2026-08-24T08:00:00+08:00"}) + "\n"
|
||||
)
|
||||
first = CrawlerRunner(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", first_source, "SCH-EMP-20260824-01")
|
||||
).run()
|
||||
second_config = replace(
|
||||
run_config(tmp_path, "SCHOOL_EMPLOYMENT", second_source, "SCH-EMP-20260824-01"),
|
||||
task_id="JOB-20260824-SCHOOL-EMP-002",
|
||||
)
|
||||
second = CrawlerRunner(second_config).run()
|
||||
assert (
|
||||
read_ndjson(first / "standard.ndjson")[0]["content_hash"]
|
||||
== read_ndjson(second / "standard.ndjson")[0]["content_hash"]
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
import json
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
SCHEMA_NAMES = (
|
||||
"external-job-source.schema.json",
|
||||
"school-employment.schema.json",
|
||||
"school-internship.schema.json",
|
||||
)
|
||||
|
||||
|
||||
def test_packaged_schemas_match_canonical_demo_contracts() -> None:
|
||||
canonical_directory = Path(__file__).resolve().parents[2] / "demo" / "schemas"
|
||||
for name in SCHEMA_NAMES:
|
||||
canonical = json.loads((canonical_directory / name).read_text())
|
||||
packaged = json.loads(
|
||||
files("edu_crawler.schemas").joinpath(name).read_text(encoding="utf-8")
|
||||
)
|
||||
assert packaged == canonical
|
||||
Draft202012Validator.check_schema(packaged)
|
||||
@@ -0,0 +1,270 @@
|
||||
from dataclasses import replace
|
||||
import os
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from edu_crawler.errors import (
|
||||
AuthOrChallenge,
|
||||
ConfigurationError,
|
||||
HostNotAllowed,
|
||||
PrivateAddressBlocked,
|
||||
RobotsDenied,
|
||||
SourceStructureChanged,
|
||||
)
|
||||
from edu_crawler.http import SafeHttpClient
|
||||
from edu_crawler.policies import source_policy
|
||||
from edu_crawler.security import (
|
||||
prohibited_school_keys,
|
||||
prohibited_school_values,
|
||||
redact_url,
|
||||
safe_record_reference,
|
||||
validate_public_url,
|
||||
)
|
||||
from edu_crawler.transport import PinnedNetworkBackend
|
||||
|
||||
|
||||
def public_resolver(_host: str, _port: int, **_kwargs: object) -> list[tuple]:
|
||||
return [(2, 1, 6, "", ("93.184.216.34", 443))]
|
||||
|
||||
|
||||
def private_resolver(_host: str, _port: int, **_kwargs: object) -> list[tuple]:
|
||||
return [(2, 1, 6, "", ("127.0.0.1", 443))]
|
||||
|
||||
|
||||
def test_url_allowlist_and_ssrf_controls() -> None:
|
||||
assert (
|
||||
validate_public_url(
|
||||
"https://brzp.chinasme.cn/jobs?page=1",
|
||||
("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
)
|
||||
== "https://brzp.chinasme.cn/jobs?page=1"
|
||||
)
|
||||
with pytest.raises(HostNotAllowed):
|
||||
validate_public_url(
|
||||
"https://evil.example/jobs",
|
||||
("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
)
|
||||
credential_url = "https://" + "user" + ":" + "example" + "@brzp.chinasme.cn/jobs"
|
||||
with pytest.raises(HostNotAllowed):
|
||||
validate_public_url(
|
||||
credential_url, ("brzp.chinasme.cn",), resolver=public_resolver
|
||||
)
|
||||
with pytest.raises(PrivateAddressBlocked):
|
||||
validate_public_url(
|
||||
"https://brzp.chinasme.cn/jobs",
|
||||
("brzp.chinasme.cn",),
|
||||
resolver=private_resolver,
|
||||
)
|
||||
with pytest.raises(PrivateAddressBlocked):
|
||||
validate_public_url(
|
||||
"https://127.0.0.1/jobs", ("127.0.0.1",), resolver=public_resolver
|
||||
)
|
||||
|
||||
|
||||
def test_url_redaction_removes_credentials_and_fragment() -> None:
|
||||
credential_url = "https://" + "user" + ":" + "example" + "@example.com"
|
||||
value = redact_url(credential_url + "/jobs?token=abc&city=330100#private")
|
||||
assert value == "https://example.com/jobs?token=%5BREDACTED%5D&city=330100"
|
||||
assert "pass" not in value
|
||||
assert "abc" not in value
|
||||
assert "private" not in value
|
||||
|
||||
|
||||
def test_record_reference_is_sanitized_before_quarantine_or_ids() -> None:
|
||||
assert safe_record_reference("contact student@example.com") is None
|
||||
assert (
|
||||
safe_record_reference(
|
||||
{"value": "https://example.com/job/1?token=secret#private"}
|
||||
)
|
||||
== "https://example.com/job/1?token=%5BREDACTED%5D"
|
||||
)
|
||||
|
||||
|
||||
def test_nested_school_pii_detection() -> None:
|
||||
record = {
|
||||
"student_token": "STU_x",
|
||||
"profile": {"student_name": "不应进入平台"},
|
||||
"student": {"name": "不应进入平台"},
|
||||
"rows": [{"phone": "1"}],
|
||||
}
|
||||
assert prohibited_school_keys(record) == [
|
||||
"profile.student_name",
|
||||
"rows[0].phone",
|
||||
"student.name",
|
||||
]
|
||||
|
||||
|
||||
def test_school_pii_value_detection_reports_paths_only() -> None:
|
||||
record = {
|
||||
"student_token": "STU_" + "a" * 32,
|
||||
"source_record_id": "EMP-001",
|
||||
"comment": "联系 13800138000 或 student@example.com",
|
||||
}
|
||||
assert prohibited_school_values(record) == ["comment"]
|
||||
|
||||
|
||||
def test_http_client_rejects_unversioned_approved_host() -> None:
|
||||
policy = replace(source_policy("BRZP"), minimum_interval_seconds=0)
|
||||
with pytest.raises(ConfigurationError):
|
||||
SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("new-host.example",),
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(lambda _request: httpx.Response(200)),
|
||||
)
|
||||
|
||||
|
||||
def test_robots_denial_stops_before_target_request() -> None:
|
||||
requests: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(str(request.url))
|
||||
assert request.url.path == "/robots.txt"
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/plain"},
|
||||
text="User-agent: *\nDisallow: /private",
|
||||
)
|
||||
|
||||
policy = replace(source_policy("BRZP"), minimum_interval_seconds=0)
|
||||
with SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as client:
|
||||
with pytest.raises(RobotsDenied):
|
||||
client.fetch("https://brzp.chinasme.cn/private")
|
||||
assert requests == ["https://brzp.chinasme.cn/robots.txt"]
|
||||
|
||||
|
||||
def test_challenge_page_stops_without_bypass() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/plain"},
|
||||
text="User-agent: *\nAllow: /",
|
||||
)
|
||||
return httpx.Response(
|
||||
200, headers={"content-type": "text/html"}, text="<title>安全验证</title>"
|
||||
)
|
||||
|
||||
policy = replace(source_policy("BRZP"), minimum_interval_seconds=0)
|
||||
with SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as client:
|
||||
with pytest.raises(AuthOrChallenge):
|
||||
client.fetch("https://brzp.chinasme.cn/job/1")
|
||||
|
||||
|
||||
def test_network_errors_do_not_echo_sensitive_urls() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("failed token=secret-value", request=request)
|
||||
|
||||
policy = replace(
|
||||
source_policy("BRZP"), minimum_interval_seconds=0, robots_required=False
|
||||
)
|
||||
with SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as client:
|
||||
with pytest.raises(SourceStructureChanged) as caught:
|
||||
client.fetch("https://brzp.chinasme.cn/jobs?token=secret-value")
|
||||
assert str(caught.value) == "Source network request failed"
|
||||
|
||||
|
||||
def test_redirect_is_revalidated_against_allowlist() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/plain"},
|
||||
text="User-agent: *\nAllow: /",
|
||||
)
|
||||
return httpx.Response(302, headers={"location": "https://evil.example/private"})
|
||||
|
||||
policy = replace(source_policy("BRZP"), minimum_interval_seconds=0)
|
||||
with SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as client:
|
||||
with pytest.raises(HostNotAllowed):
|
||||
client.fetch("https://brzp.chinasme.cn/job/1")
|
||||
|
||||
|
||||
def test_bearer_token_is_not_sent_to_robots(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
observed: list[tuple[str, str | None]] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
observed.append((request.url.path, request.headers.get("authorization")))
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/plain"},
|
||||
text="User-agent: *\nAllow: /",
|
||||
)
|
||||
return httpx.Response(
|
||||
200, headers={"content-type": "application/json"}, json=[{"id": "one"}]
|
||||
)
|
||||
|
||||
monkeypatch.setenv("TEST_SOURCE_BEARER", "example-token-value")
|
||||
policy = replace(source_policy("BRZP"), minimum_interval_seconds=0)
|
||||
with SafeHttpClient(
|
||||
policy,
|
||||
contact="crawler@example.edu.cn",
|
||||
approved_hosts=("brzp.chinasme.cn",),
|
||||
bearer_token_env="TEST_SOURCE_BEARER",
|
||||
resolver=public_resolver,
|
||||
transport=httpx.MockTransport(handler),
|
||||
) as client:
|
||||
client.fetch("https://brzp.chinasme.cn/jobs")
|
||||
assert observed == [
|
||||
("/robots.txt", None),
|
||||
("/jobs", "Bearer example-token-value"),
|
||||
]
|
||||
assert os.environ["TEST_SOURCE_BEARER"] == "example-token-value"
|
||||
|
||||
|
||||
def test_pinned_backend_connects_to_the_validated_ip(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
connected: list[tuple] = []
|
||||
|
||||
class FakeSocket:
|
||||
def settimeout(self, _timeout: float | None) -> None:
|
||||
pass
|
||||
|
||||
def setsockopt(self, *_option: object) -> None:
|
||||
pass
|
||||
|
||||
def bind(self, _address: tuple) -> None:
|
||||
pass
|
||||
|
||||
def connect(self, address: tuple) -> None:
|
||||
connected.append(address)
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
"edu_crawler.transport.socket.socket", lambda *_args: FakeSocket()
|
||||
)
|
||||
stream = PinnedNetworkBackend(public_resolver).connect_tcp("brzp.chinasme.cn", 443)
|
||||
stream.close()
|
||||
assert connected == [("93.184.216.34", 443)]
|
||||
Reference in New Issue
Block a user