remove support for unindexed tokens

This was a special feature of the legacy tokenizer who would not
index very frequent tokens.
This commit is contained in:
Sarah Hoffmann
2024-09-22 10:39:10 +02:00
parent 290c22a153
commit a690605a96
6 changed files with 23 additions and 30 deletions

View File

@@ -101,7 +101,6 @@ class Token(ABC):
count: int
addr_count: int
lookup_word: str
is_indexed: bool
@abstractmethod