lofyerandfactory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
8dff9847e3
- add authorized nine-source crawler connectors with security controls and packaged schemas - add the school internship contract to the vendor data standard center - add bidirectional campus data-to-report lineage with gap ownership and remediation Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
63 lines
2.0 KiB
Python
63 lines
2.0 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from edu_crawler.errors import SourceStructureChanged
|
|
from edu_crawler.models import RawDocument
|
|
from edu_crawler.parsing import discover_job_links, parse_document
|
|
from edu_crawler.policies import source_policy
|
|
|
|
|
|
def test_parse_schema_org_job_posting() -> None:
|
|
body = b"""
|
|
<html><head><script type="application/ld+json">
|
|
{"@context":"https://schema.org","@type":"JobPosting","identifier":{"value":"BRZP-1"},
|
|
"url":"https://brzp.chinasme.cn/job/BRZP-1","title":"Test","description":"Description",
|
|
"hiringOrganization":{"name":"Example"}}
|
|
</script></head></html>
|
|
"""
|
|
document = RawDocument.create(
|
|
"BRZP", "https://brzp.chinasme.cn/job/BRZP-1", "text/html", body
|
|
)
|
|
records = parse_document(document, allow_html=True)
|
|
assert records[0]["@type"] == "JobPosting"
|
|
|
|
|
|
def test_discover_links_uses_versioned_patterns_and_deduplicates() -> None:
|
|
body = b"""
|
|
<a href="/job/1">one</a>
|
|
<a href="/job/1">duplicate</a>
|
|
<a href="/company/1">company</a>
|
|
"""
|
|
document = RawDocument.create(
|
|
"BRZP", "https://brzp.chinasme.cn/", "text/html", body
|
|
)
|
|
assert discover_job_links(document, source_policy("BRZP")) == [
|
|
"https://brzp.chinasme.cn/job/1"
|
|
]
|
|
|
|
|
|
def test_invalid_ndjson_reports_line_without_payload() -> None:
|
|
document = RawDocument.create(
|
|
"BOSS",
|
|
"file:///authorized.ndjson",
|
|
"application/x-ndjson",
|
|
b'{"id":"one"}\nnot-json\n',
|
|
)
|
|
with pytest.raises(SourceStructureChanged, match="line 2") as caught:
|
|
parse_document(document, allow_html=False)
|
|
assert "not-json" not in str(caught.value)
|
|
|
|
|
|
def test_json_collection_wrappers() -> None:
|
|
document = RawDocument.create(
|
|
"BOSS",
|
|
"file:///authorized.json",
|
|
"application/json",
|
|
json.dumps({"data": {"items": [{"id": "one"}, {"id": "two"}]}}).encode(),
|
|
)
|
|
assert [record["id"] for record in parse_document(document, allow_html=False)] == [
|
|
"one",
|
|
"two",
|
|
]
|