fix: preserve non-ASCII entity names in ontology URI normalization (#1036)

The entity URI normalizer stripped every non-ASCII character via the
ASCII-only patterns `[^a-z0-9\-.]` and `[^a-z0-9\-]`. For non-Latin
entity names (e.g. Chinese), this deleted the entire name, so distinct
entities collapsed onto the same URI (".../<ontology>/-"), silently
merging unrelated nodes and breaking OntoRAG for non-English content.

Switch both filters to `[^\w\-.]` / `[^\w\-]`. In Python 3, `\w` is
Unicode-aware for str patterns, so non-ASCII letters (CJK, etc.) are
preserved while punctuation and symbols are still removed. ASCII
behaviour is unchanged: names are lowercased and underscores converted
to hyphens before the filter runs, so for ASCII input the patterns are
equivalent to the originals.

Adds tests covering ASCII regression guards and non-ASCII names/types,
including the core case that two distinct Chinese entities must not
collapse onto the same URI.

Co-authored-by: arthurxuwei <5179840+arthurxuwei@users.noreply.github.com>

tests/unit/test_extract/test_ontology/test_entity_normalizer.py

@@ -0,0 +1,79 @@
+"""
+Unit tests for entity URI normalization.
+
+Covers ASCII behaviour and, critically, non-ASCII (e.g. Chinese) entity
+names and types, which must be preserved rather than stripped so that
+distinct entities do not collapse onto a single URI.
+"""
+
+import pytest
+from trustgraph.extract.kg.ontology.entity_normalizer import (
+    normalize_entity_name,
+    normalize_type_identifier,
+    build_entity_uri,
+    EntityRegistry,
+)
+
+
+class TestNormalizeEntityNameAscii:
+    """ASCII normalization must keep working as before."""
+
+    def test_lowercases_and_hyphenates(self):
+        assert normalize_entity_name("Cornish pasty") == "cornish-pasty"
+
+    def test_strips_punctuation(self):
+        assert normalize_entity_name("beef!!!") == "beef"
+
+    def test_collapses_and_trims_hyphens(self):
+        assert normalize_entity_name("  beef   stew  ") == "beef-stew"
+
+
+class TestNormalizeEntityNameUnicode:
+    """Non-ASCII names must be preserved, not stripped to empty."""
+
+    def test_chinese_name_preserved(self):
+        # Previously the ASCII-only filter deleted every CJK character,
+        # collapsing the name to "".
+        assert normalize_entity_name("苹果") == "苹果"
+
+    def test_chinese_name_with_space(self):
+        assert normalize_entity_name("有机 苹果") == "有机-苹果"
+
+    def test_distinct_chinese_names_stay_distinct(self):
+        assert normalize_entity_name("苹果") != normalize_entity_name("香蕉")
+
+    def test_mixed_ascii_and_chinese(self):
+        assert normalize_entity_name("iPhone 手机") == "iphone-手机"
+
+
+class TestNormalizeTypeIdentifierUnicode:
+    def test_ascii_prefixed_type(self):
+        assert normalize_type_identifier("fo/Recipe") == "fo-recipe"
+
+    def test_chinese_type_preserved(self):
+        assert normalize_type_identifier("食物") == "食物"
+
+
+class TestBuildEntityUriUnicode:
+    def test_distinct_chinese_entities_get_distinct_uris(self):
+        # This is the core regression: with ASCII-only normalization both
+        # names collapsed to the same URI, merging unrelated entities.
+        uri_apple = build_entity_uri("苹果", "水果", "food")
+        uri_banana = build_entity_uri("香蕉", "水果", "food")
+        assert uri_apple != uri_banana
+        assert uri_apple.endswith("苹果")
+        assert uri_banana.endswith("香蕉")
+
+    def test_uri_is_not_bare_prefix(self):
+        uri = build_entity_uri("苹果", "水果", "food")
+        # Must not collapse to ".../food/水果-" or ".../food/-"
+        assert not uri.endswith("-")
+
+
+class TestEntityRegistryUnicode:
+    def test_distinct_chinese_entities_not_deduplicated(self):
+        registry = EntityRegistry("food")
+        uri_apple = registry.get_or_create_uri("苹果", "水果")
+        uri_banana = registry.get_or_create_uri("香蕉", "水果")
+        assert uri_apple != uri_banana
+        assert registry.size() == 2

trustgraph-flow/trustgraph/extract/kg/ontology/entity_normalizer.py

@@ -23,8 +23,11 @@ def normalize_entity_name(entity_name: str) -> str:
     # Replace spaces and underscores with hyphens
     normalized = re.sub(r'[\s_]+', '-', normalized)
 
-    # Remove any characters that aren't alphanumeric, hyphens, or periods
+    # Remove any characters that aren't word characters (incl. non-ASCII
-    normalized = re.sub(r'[^a-z0-9\-.]', '', normalized)
+    # letters such as CJK), hyphens, or periods. \w is Unicode-aware for
+    # str patterns in Python 3, so non-ASCII entity names are preserved
+    # rather than stripped (which would collapse them onto a single URI).
+    normalized = re.sub(r'[^\w\-.]', '', normalized)
 
     # Remove leading/trailing hyphens
     normalized = normalized.strip('-')
@@ -52,8 +55,10 @@ def normalize_type_identifier(type_id: str) -> str:
     # Replace slashes, colons, and spaces with hyphens
     normalized = re.sub(r'[/:.\s_]+', '-', normalized)
 
-    # Remove any remaining non-alphanumeric characters except hyphens
+    # Remove any remaining characters that aren't word characters (incl.
-    normalized = re.sub(r'[^a-z0-9\-]', '', normalized)
+    # non-ASCII letters such as CJK) or hyphens. \w is Unicode-aware for
+    # str patterns in Python 3, preserving non-ASCII type identifiers.
+    normalized = re.sub(r'[^\w\-]', '', normalized)
 
     # Remove leading/trailing hyphens
     normalized = normalized.strip('-')