recreate word table when refreshing counts

The counting touches a large part of the word table, leaving
bloated tables and indexes. Thus recreate the table instead and
swap it in.
This commit is contained in:
Sarah Hoffmann
2024-02-04 16:43:33 +01:00
parent 33c0f249b1
commit 81eed0680c
10 changed files with 130 additions and 82 deletions

View File

@@ -110,7 +110,7 @@ class UpdateRefresh:
if args.word_counts:
LOG.warning('Recompute word statistics')
self._get_tokenizer(args.config).update_statistics()
self._get_tokenizer(args.config).update_statistics(args.config)
if args.address_levels:
LOG.warning('Updating address levels')

View File

@@ -169,7 +169,7 @@ class SetupAll:
tokenizer.finalize_import(args.config)
LOG.warning('Recompute word counts')
tokenizer.update_statistics()
tokenizer.update_statistics(args.config)
webdir = args.project_dir / 'website'
LOG.warning('Setup website at %s', webdir)