add normalised country name to word table

Country tokens now follow the usual convetion of having the
normalized version in the word column and the extra info about the
country code in the info column.
This commit is contained in:
Sarah Hoffmann
2025-11-27 12:00:47 +01:00
parent f2a122c5c0
commit 81c6cb72e6
7 changed files with 67 additions and 32 deletions

View File

@@ -127,6 +127,12 @@ class Token(ABC):
category objects.
"""
@abstractmethod
def get_country(self) -> str:
""" Return the country code this tojen is associated with
(currently for country tokens only).
"""
@dataclasses.dataclass
class TokenRange: