remove support for unindexed tokens

This was a special feature of the legacy tokenizer who would not
index very frequent tokens.

test/python/api/search/test_token_assignment.py

@@ -20,7 +20,7 @@ class MyToken(Token):
 def make_query(*args):
     q = QueryStruct([Phrase(args[0][1], '')])
     dummy = MyToken(penalty=3.0, token=45, count=1, addr_count=1,
-                    lookup_word='foo', is_indexed=True)
+                    lookup_word='foo')
 
     for btype, ptype, _ in args[1:]:
         q.add_node(btype, ptype)