feat: add compliant data connectors and lineage
- add authorized nine-source crawler connectors with security controls and packaged schemas - add the school internship contract to the vendor data standard center - add bidirectional campus data-to-report lineage with gap ownership and remediation Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
92515928d9
commit
8dff9847e3
@@ -0,0 +1,62 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from edu_crawler.errors import SourceStructureChanged
|
||||
from edu_crawler.models import RawDocument
|
||||
from edu_crawler.parsing import discover_job_links, parse_document
|
||||
from edu_crawler.policies import source_policy
|
||||
|
||||
|
||||
def test_parse_schema_org_job_posting() -> None:
|
||||
body = b"""
|
||||
<html><head><script type="application/ld+json">
|
||||
{"@context":"https://schema.org","@type":"JobPosting","identifier":{"value":"BRZP-1"},
|
||||
"url":"https://brzp.chinasme.cn/job/BRZP-1","title":"Test","description":"Description",
|
||||
"hiringOrganization":{"name":"Example"}}
|
||||
</script></head></html>
|
||||
"""
|
||||
document = RawDocument.create(
|
||||
"BRZP", "https://brzp.chinasme.cn/job/BRZP-1", "text/html", body
|
||||
)
|
||||
records = parse_document(document, allow_html=True)
|
||||
assert records[0]["@type"] == "JobPosting"
|
||||
|
||||
|
||||
def test_discover_links_uses_versioned_patterns_and_deduplicates() -> None:
|
||||
body = b"""
|
||||
<a href="/job/1">one</a>
|
||||
<a href="/job/1">duplicate</a>
|
||||
<a href="/company/1">company</a>
|
||||
"""
|
||||
document = RawDocument.create(
|
||||
"BRZP", "https://brzp.chinasme.cn/", "text/html", body
|
||||
)
|
||||
assert discover_job_links(document, source_policy("BRZP")) == [
|
||||
"https://brzp.chinasme.cn/job/1"
|
||||
]
|
||||
|
||||
|
||||
def test_invalid_ndjson_reports_line_without_payload() -> None:
|
||||
document = RawDocument.create(
|
||||
"BOSS",
|
||||
"file:///authorized.ndjson",
|
||||
"application/x-ndjson",
|
||||
b'{"id":"one"}\nnot-json\n',
|
||||
)
|
||||
with pytest.raises(SourceStructureChanged, match="line 2") as caught:
|
||||
parse_document(document, allow_html=False)
|
||||
assert "not-json" not in str(caught.value)
|
||||
|
||||
|
||||
def test_json_collection_wrappers() -> None:
|
||||
document = RawDocument.create(
|
||||
"BOSS",
|
||||
"file:///authorized.json",
|
||||
"application/json",
|
||||
json.dumps({"data": {"items": [{"id": "one"}, {"id": "two"}]}}).encode(),
|
||||
)
|
||||
assert [record["id"] for record in parse_document(document, allow_html=False)] == [
|
||||
"one",
|
||||
"two",
|
||||
]
|
||||
Reference in New Issue
Block a user