more unit tests for tokenizers

This commit is contained in:
Sarah Hoffmann
2021-12-02 15:46:36 +01:00
parent 7617a9316e
commit 14a78f55cd
5 changed files with 333 additions and 31 deletions

View File

@@ -17,6 +17,14 @@ class MockIcuWordTable:
conn.commit()
def add_full_word(self, word_id, word, word_token=None):
with self.conn.cursor() as cur:
cur.execute("""INSERT INTO word (word_id, word_token, type, word, info)
VALUES(%s, %s, 'W', %s, '{}'::jsonb)""",
(word_id, word or word_token, word))
self.conn.commit()
def add_special(self, word_token, word, cls, typ, oper):
with self.conn.cursor() as cur:
cur.execute("""INSERT INTO word (word_token, type, word, info)