2021-04-21 10:57:17 +03:00
|
|
|
"""
|
|
|
|
Tokenizer for testing.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def create(dsn, data_dir):
|
|
|
|
""" Create a new instance of the tokenizer provided by this module.
|
|
|
|
"""
|
|
|
|
return DummyTokenizer(dsn, data_dir)
|
|
|
|
|
|
|
|
class DummyTokenizer:
|
|
|
|
|
|
|
|
def __init__(self, dsn, data_dir):
|
|
|
|
self.dsn = dsn
|
|
|
|
self.data_dir = data_dir
|
|
|
|
self.init_state = None
|
2021-04-27 12:37:18 +03:00
|
|
|
self.analyser_cache = {}
|
2021-04-21 10:57:17 +03:00
|
|
|
|
|
|
|
|
2021-04-28 11:59:07 +03:00
|
|
|
def init_new_db(self, *args, **kwargs):
|
2021-04-21 10:57:17 +03:00
|
|
|
assert self.init_state == None
|
|
|
|
self.init_state = "new"
|
|
|
|
|
|
|
|
|
|
|
|
def init_from_project(self):
|
|
|
|
assert self.init_state == None
|
|
|
|
self.init_state = "loaded"
|
2021-04-24 23:35:46 +03:00
|
|
|
|
|
|
|
|
|
|
|
def name_analyzer(self):
|
2021-04-27 12:37:18 +03:00
|
|
|
return DummyNameAnalyzer(self.analyser_cache)
|
2021-04-24 23:35:46 +03:00
|
|
|
|
|
|
|
|
|
|
|
class DummyNameAnalyzer:
|
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
|
|
self.close()
|
|
|
|
|
|
|
|
|
2021-04-27 12:37:18 +03:00
|
|
|
def __init__(self, cache):
|
|
|
|
self.analyser_cache = cache
|
|
|
|
cache['countries'] = []
|
|
|
|
|
|
|
|
|
2021-04-24 23:35:46 +03:00
|
|
|
def close(self):
|
|
|
|
pass
|
|
|
|
|
2021-04-25 19:26:36 +03:00
|
|
|
def add_postcodes_from_db(self):
|
|
|
|
pass
|
|
|
|
|
2021-04-27 22:50:35 +03:00
|
|
|
def update_special_phrases(self, phrases):
|
|
|
|
self.analyser_cache['special_phrases'] = phrases
|
2021-04-24 23:35:46 +03:00
|
|
|
|
2021-04-27 12:37:18 +03:00
|
|
|
def add_country_names(self, code, names):
|
|
|
|
self.analyser_cache['countries'].append((code, names))
|
|
|
|
|
|
|
|
def process_place(self, place):
|
2021-04-24 23:35:46 +03:00
|
|
|
return {}
|