Fixed setting tokenizer property

This commit is contained in:
Robbe Haesendonck
2023-09-27 16:32:28 +02:00
committed by Sarah Hoffmann
parent 59ae63e6f5
commit c06f902398

View File

@@ -208,7 +208,7 @@ class SetupAll:
""" """
from ..tokenizer import factory as tokenizer_factory from ..tokenizer import factory as tokenizer_factory
if continue_at is None or continue_at == 'load-data': if continue_at in ('import-from-file', 'load-data', None):
# (re)initialise the tokenizer data # (re)initialise the tokenizer data
return tokenizer_factory.create_tokenizer(config) return tokenizer_factory.create_tokenizer(config)