move index creation for word table to tokenizer

This introduces a finalization routing for the tokenizer
where it can post-process the import if necessary.

test/python/dummy_tokenizer.py

@@ -26,6 +26,10 @@ class DummyTokenizer:
         self.init_state = "loaded"
 
 
+    def finalize_import(self, _):
+        pass
+
+
     def name_analyzer(self):
         return DummyNameAnalyzer(self.analyser_cache)