add Python part for new ICU-based tokenizer

2021-05-02 17:52:45 +02:00
parent 3c67bae868
commit f44af49df9
6 changed files with 1626 additions and 1 deletions
--- a/lib-sql/tokenizer/legacy_tokenizer_tables.sql
+++ b/lib-sql/tokenizer/legacy_tokenizer_tables.sql
@@ -12,6 +12,8 @@ CREATE TABLE word (

 CREATE INDEX idx_word_word_token ON word
    USING BTREE (word_token) {{db.tablespace.search_index}};
+CREATE INDEX idx_word_word ON word
+    USING BTREE (word) {{db.tablespace.search_index}} WHERE word is not null;
 GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}";

 DROP SEQUENCE IF EXISTS seq_word;