From f106ae2103d9656b34c231d1fe2071071a98badb Mon Sep 17 00:00:00 2001 From: Wei Xu Date: Thu, 9 Jul 2026 07:05:35 +0800 Subject: [PATCH] fix: preserve non-ASCII entity names in ontology URI normalization (#1036) The entity URI normalizer stripped every non-ASCII character via the ASCII-only patterns `[^a-z0-9\-.]` and `[^a-z0-9\-]`. For non-Latin entity names (e.g. Chinese), this deleted the entire name, so distinct entities collapsed onto the same URI ("...//-"), silently merging unrelated nodes and breaking OntoRAG for non-English content. Switch both filters to `[^\w\-.]` / `[^\w\-]`. In Python 3, `\w` is Unicode-aware for str patterns, so non-ASCII letters (CJK, etc.) are preserved while punctuation and symbols are still removed. ASCII behaviour is unchanged: names are lowercased and underscores converted to hyphens before the filter runs, so for ASCII input the patterns are equivalent to the originals. Adds tests covering ASCII regression guards and non-ASCII names/types, including the core case that two distinct Chinese entities must not collapse onto the same URI. Co-authored-by: arthurxuwei <5179840+arthurxuwei@users.noreply.github.com> --- .../test_ontology/test_entity_normalizer.py | 79 +++++++++++++++++++ .../extract/kg/ontology/entity_normalizer.py | 13 ++- 2 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_extract/test_ontology/test_entity_normalizer.py diff --git a/tests/unit/test_extract/test_ontology/test_entity_normalizer.py b/tests/unit/test_extract/test_ontology/test_entity_normalizer.py new file mode 100644 index 00000000..290a1808 --- /dev/null +++ b/tests/unit/test_extract/test_ontology/test_entity_normalizer.py @@ -0,0 +1,79 @@ +""" +Unit tests for entity URI normalization. + +Covers ASCII behaviour and, critically, non-ASCII (e.g. Chinese) entity +names and types, which must be preserved rather than stripped so that +distinct entities do not collapse onto a single URI. +""" + +import pytest +from trustgraph.extract.kg.ontology.entity_normalizer import ( + normalize_entity_name, + normalize_type_identifier, + build_entity_uri, + EntityRegistry, +) + + +class TestNormalizeEntityNameAscii: + """ASCII normalization must keep working as before.""" + + def test_lowercases_and_hyphenates(self): + assert normalize_entity_name("Cornish pasty") == "cornish-pasty" + + def test_strips_punctuation(self): + assert normalize_entity_name("beef!!!") == "beef" + + def test_collapses_and_trims_hyphens(self): + assert normalize_entity_name(" beef stew ") == "beef-stew" + + +class TestNormalizeEntityNameUnicode: + """Non-ASCII names must be preserved, not stripped to empty.""" + + def test_chinese_name_preserved(self): + # Previously the ASCII-only filter deleted every CJK character, + # collapsing the name to "". + assert normalize_entity_name("苹果") == "苹果" + + def test_chinese_name_with_space(self): + assert normalize_entity_name("有机 苹果") == "有机-苹果" + + def test_distinct_chinese_names_stay_distinct(self): + assert normalize_entity_name("苹果") != normalize_entity_name("香蕉") + + def test_mixed_ascii_and_chinese(self): + assert normalize_entity_name("iPhone 手机") == "iphone-手机" + + +class TestNormalizeTypeIdentifierUnicode: + def test_ascii_prefixed_type(self): + assert normalize_type_identifier("fo/Recipe") == "fo-recipe" + + def test_chinese_type_preserved(self): + assert normalize_type_identifier("食物") == "食物" + + +class TestBuildEntityUriUnicode: + def test_distinct_chinese_entities_get_distinct_uris(self): + # This is the core regression: with ASCII-only normalization both + # names collapsed to the same URI, merging unrelated entities. + uri_apple = build_entity_uri("苹果", "水果", "food") + uri_banana = build_entity_uri("香蕉", "水果", "food") + assert uri_apple != uri_banana + assert uri_apple.endswith("苹果") + assert uri_banana.endswith("香蕉") + + def test_uri_is_not_bare_prefix(self): + uri = build_entity_uri("苹果", "水果", "food") + # Must not collapse to ".../food/水果-" or ".../food/-" + assert not uri.endswith("-") + + +class TestEntityRegistryUnicode: + def test_distinct_chinese_entities_not_deduplicated(self): + registry = EntityRegistry("food") + uri_apple = registry.get_or_create_uri("苹果", "水果") + uri_banana = registry.get_or_create_uri("香蕉", "水果") + assert uri_apple != uri_banana + assert registry.size() == 2 diff --git a/trustgraph-flow/trustgraph/extract/kg/ontology/entity_normalizer.py b/trustgraph-flow/trustgraph/extract/kg/ontology/entity_normalizer.py index 093302a9..b47c50c3 100644 --- a/trustgraph-flow/trustgraph/extract/kg/ontology/entity_normalizer.py +++ b/trustgraph-flow/trustgraph/extract/kg/ontology/entity_normalizer.py @@ -23,8 +23,11 @@ def normalize_entity_name(entity_name: str) -> str: # Replace spaces and underscores with hyphens normalized = re.sub(r'[\s_]+', '-', normalized) - # Remove any characters that aren't alphanumeric, hyphens, or periods - normalized = re.sub(r'[^a-z0-9\-.]', '', normalized) + # Remove any characters that aren't word characters (incl. non-ASCII + # letters such as CJK), hyphens, or periods. \w is Unicode-aware for + # str patterns in Python 3, so non-ASCII entity names are preserved + # rather than stripped (which would collapse them onto a single URI). + normalized = re.sub(r'[^\w\-.]', '', normalized) # Remove leading/trailing hyphens normalized = normalized.strip('-') @@ -52,8 +55,10 @@ def normalize_type_identifier(type_id: str) -> str: # Replace slashes, colons, and spaces with hyphens normalized = re.sub(r'[/:.\s_]+', '-', normalized) - # Remove any remaining non-alphanumeric characters except hyphens - normalized = re.sub(r'[^a-z0-9\-]', '', normalized) + # Remove any remaining characters that aren't word characters (incl. + # non-ASCII letters such as CJK) or hyphens. \w is Unicode-aware for + # str patterns in Python 3, preserving non-ASCII type identifiers. + normalized = re.sub(r'[^\w\-]', '', normalized) # Remove leading/trailing hyphens normalized = normalized.strip('-')