add japanese phrase preprocessing

Code adapted from GSOC code by @miku.
2025-01-08 19:43:25 +01:00
parent 86ad9efa8a
commit efc09a5cfc
3 changed files with 96 additions and 0 deletions
--- a/settings/icu_tokenizer.yaml
+++ b/settings/icu_tokenizer.yaml
@@ -1,4 +1,5 @@
 query-preprocessing:
+    - step: split_japanese_phrases
    - step: normalize
 normalization:
    - ":: lower ()"