hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-12 05:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

show token begin and end in debug output

Browse Source
This commit is contained in:
Sarah Hoffmann
2025-02-28 17:19:37 +01:00
parent fc1c6261ed
commit e67ae701ac
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/nominatim_api/search/icu_tokenizer.py
View File

@@ -298,12 +298,12 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer):
def _dump_word_tokens(query: qmod.QueryStruct) -> Iterator[List[Any]]: def _dump_word_tokens(query: qmod.QueryStruct) -> Iterator[List[Any]]:
yield ['type', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info'] yield ['type', 'from', 'to', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info']
for node in query.nodes: for i, node in enumerate(query.nodes):
for tlist in node.starting: for tlist in node.starting:
for token in tlist.tokens: for token in tlist.tokens:
t = cast(ICUToken, token) t = cast(ICUToken, token)
yield [tlist.ttype, t.token, t.word_token or '', yield [tlist.ttype, str(i), str(tlist.end), t.token, t.word_token or '',
t.lookup_word or '', t.penalty, t.count, t.info] t.lookup_word or '', t.penalty, t.count, t.info]