keep break indicators [:-] during normalisation

All punctuation will be converted to '-'. Soft breaks : may be
added by preprocessors. The break signs are only used during
query analysis and are ignored during import token analysis.
This commit is contained in:
Sarah Hoffmann
2025-01-07 21:32:32 +01:00
parent d984100e23
commit 86ad9efa8a
3 changed files with 8 additions and 5 deletions

View File

@@ -25,6 +25,8 @@ class ICUTokenAnalysis:
def __init__(self, norm_rules: str, trans_rules: str,
analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
# additional break signs are not relevant during name analysis
norm_rules += ";[[:Space:][-:]]+ > ' ';"
self.normalizer = Transliterator.createFromRules("icu_normalization",
norm_rules)
trans_rules += ";[:Space:]+ > ' '"