From 571ab3cf4fb54ae10f40d7d05b446c1cf69197b9 Mon Sep 17 00:00:00 2001 From: Mohammed Taha Khan Date: Mon, 20 Jul 2026 16:46:52 +0200 Subject: [PATCH] fix(harvester): normalize malformed URN identifiers --- .../transform/mappers/identifiers.py | 71 +++++++++++++------ .../inspire_harvester/test_transformer.py | 53 ++++++++++++++ 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py index 387fb211..f0f40919 100644 --- a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py +++ b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py @@ -11,12 +11,55 @@ from dataclasses import dataclass from flask import current_app -from idutils.normalizers import normalize_isbn -from idutils.validators import is_doi +from idutils.normalizers import normalize_isbn, normalize_urn +from idutils.validators import is_doi, is_urn from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase +def _coerce_urn(value): + """Return a CDS-valid URN, or ``None`` if it cannot be safely corrected. + + INSPIRE sometimes stores URNs with an ``http(s)://`` prefix, e.g. + ``http://nbnurn:nbn:de:...``. CDS validates with ``idutils.is_urn``, which + requires the ``urn:`` scheme. Keep the value from that scheme onward only + when the result is a valid URN. + """ + if not isinstance(value, str): + return None + + if is_urn(value): + return normalize_urn(value) + + urn_idx = value.lower().find("urn:") + candidate = value[urn_idx:] if urn_idx != -1 else None + return normalize_urn(candidate) if candidate and is_urn(candidate) else None + + +def _related_identifier(schema, value, ctx): + """Build a related-identifier dict, or ``None`` if the value is skipped. + + URN values are coerced/validated before they reach CDS record validation + (same pattern as ISBN via ``normalize_isbn``). + """ + if schema == "urn": + original = value + value = _coerce_urn(value) + if not value: + ctx.errors.append(f"Invalid URN. | details: value={original}") + return None + + related = { + "identifier": value, + "scheme": schema, + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-other"}, + } + if schema == "doi": + related["relation_type"] = {"id": "isversionof"} + return related + + @dataclass(frozen=True) class DOIMapper(MapperBase): """Mapper for DOI identifiers.""" @@ -140,15 +183,9 @@ def map_value(self, src_record, ctx, logger): if schema in RDM_RECORDS_IDENTIFIERS_SCHEMES.keys(): continue elif schema in RDM_RECORDS_RELATED_IDENTIFIERS_SCHEMES.keys(): - new_id = { - "identifier": value, - "scheme": schema, - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-other"}, - } - if schema == "doi": - new_id["relation_type"] = {"id": "isversionof"} - identifiers.append(new_id) + related = _related_identifier(schema, value, ctx) + if related: + identifiers.append(related) else: ctx.errors.append( "Unexpected schema in persistent_identifiers. " @@ -167,15 +204,9 @@ def map_value(self, src_record, ctx, logger): if schema in RDM_RECORDS_IDENTIFIERS_SCHEMES.keys(): continue elif schema in RDM_RECORDS_RELATED_IDENTIFIERS_SCHEMES.keys(): - new_id = { - "identifier": value, - "scheme": schema, - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-other"}, - } - if schema == "doi": - new_id["relation_type"] = {"id": "isversionof"} - identifiers.append(new_id) + related = _related_identifier(schema, value, ctx) + if related: + identifiers.append(related) else: # Already reported by IdentifiersMapper with a stable message. diff --git a/site/tests/inspire_harvester/test_transformer.py b/site/tests/inspire_harvester/test_transformer.py index 1aae69fa..cd21ea54 100644 --- a/site/tests/inspire_harvester/test_transformer.py +++ b/site/tests/inspire_harvester/test_transformer.py @@ -82,6 +82,59 @@ def test_transform_related_identifiers(mock_normalize_isbn, running_app): "relation_type": {"id": "isvariantformof"}, "resource_type": {"id": "publication-book"}, } in result + assert { + "identifier": "urn:nbn:de:hebis:77-25439", + "scheme": "urn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-other"}, + } in result + + +def test_transform_related_identifiers_coerces_http_prefixed_urn(running_app): + """INSPIRE#1714668-style URNs with an http(s) prefix are corrected.""" + # Real INSPIRE value that previously failed CDS validation: + # metadata.related_identifiers.N.identifier: Invalid URN identifier. + raw = "http://nbnurn:nbn:de:bvb:91-diss-20170808-1368097-1-4" + src_record = { + "metadata": { + "persistent_identifiers": [{"schema": "URN", "value": raw}], + }, + "created": "2023-01-01", + } + ctx = MetadataSerializationContext( + resource_type=ResourceType.OTHER, inspire_id="1714668" + ) + logger = Logger(inspire_id="1714668") + + result = RelatedIdentifiersMapper().map_value(src_record, ctx, logger) + + assert { + "identifier": "urn:nbn:de:bvb:91-diss-20170808-1368097-1-4", + "scheme": "urn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-other"}, + } in result + assert not any("Invalid URN" in error for error in ctx.errors) + + +def test_transform_related_identifiers_skips_invalid_urn(running_app): + """Uncorrectable URNs are omitted and reported as transformation errors.""" + raw = "http://example.com/not-a-urn" + src_record = { + "metadata": { + "persistent_identifiers": [{"schema": "URN", "value": raw}], + }, + "created": "2023-01-01", + } + ctx = MetadataSerializationContext( + resource_type=ResourceType.OTHER, inspire_id="12345" + ) + logger = Logger(inspire_id="12345") + + result = RelatedIdentifiersMapper().map_value(src_record, ctx, logger) + + assert all(item.get("scheme") != "urn" for item in result) + assert f"Invalid URN. | details: value={raw}" in ctx.errors def test_transform_identifiers(running_app):