Code cleaning and SPLoader deleted

This commit is contained in:
AntoJvlt 2021-05-16 16:59:12 +02:00
parent fb0ebb5bf0
commit 06aab389ed
6 changed files with 24 additions and 45 deletions

View File

@ -46,7 +46,7 @@ class ImportSpecialPhrases:
def start_import(args, loader):
"""
Create the SPImporter object containing the right
SPLoader and then start the import of special phrases.
sp loader and then start the import of special phrases.
"""
from ..tokenizer import factory as tokenizer_factory

View File

@ -5,14 +5,13 @@
"""
import csv
import os
from collections.abc import Iterator
from nominatim.tools.special_phrases.special_phrase import SpecialPhrase
from nominatim.tools.special_phrases.sp_loader import SPLoader
from nominatim.errors import UsageError
class SPCsvLoader(SPLoader):
class SPCsvLoader(Iterator):
"""
Base class for special phrases loaders.
Handle the loading of special phrases from external sources.
Handles loading of special phrases from external csv file.
"""
def __init__(self, csv_path):
super().__init__()
@ -24,18 +23,17 @@ class SPCsvLoader(SPLoader):
raise StopIteration()
self.has_been_read = True
SPCsvLoader.check_csv_validity(self.csv_path)
return SPCsvLoader.parse_csv(self.csv_path)
self.check_csv_validity()
return self.parse_csv()
@staticmethod
def parse_csv(csv_path):
def parse_csv(self):
"""
Open and parse the given csv file.
Create the corresponding SpecialPhrases.
"""
phrases = set()
with open(csv_path) as file:
with open(self.csv_path) as file:
reader = csv.DictReader(file, delimiter=',')
for row in reader:
phrases.add(
@ -43,12 +41,11 @@ class SPCsvLoader(SPLoader):
)
return phrases
@staticmethod
def check_csv_validity(csv_path):
def check_csv_validity(self):
"""
Check that the csv file has the right extension.
"""
_, extension = os.path.splitext(csv_path)
_, extension = os.path.splitext(self.csv_path)
if extension != '.csv':
raise UsageError('The file {} is not a csv file.'.format(csv_path))
raise UsageError('The file {} is not a csv file.'.format(self.csv_path))

View File

@ -25,7 +25,7 @@ class SPImporter():
"""
Class handling the process of special phrases importations into the database.
Take a SPLoader which load the phrases from an external source.
Take a sp loader which load the phrases from an external source.
"""
def __init__(self, config, phplib_dir, db_connection, sp_loader) -> None:
self.config = config
@ -121,16 +121,14 @@ class SPImporter():
"""
#blacklisting: disallow certain class/type combinations
if (
phrase.p_class in self.black_list.keys() and
phrase.p_type in self.black_list[phrase.p_class]
): return None
if phrase.p_class in self.black_list.keys() \
and phrase.p_type in self.black_list[phrase.p_class]:
return None
#whitelisting: if class is in whitelist, allow only tags in the list
if (
phrase.p_class in self.white_list.keys() and
phrase.p_type not in self.white_list[phrase.p_class]
): return None
if phrase.p_class in self.white_list.keys() \
and phrase.p_type not in self.white_list[phrase.p_class]:
return None
#sanity check, in case somebody added garbage in the wiki
if not self._check_sanity(phrase):

View File

@ -1,16 +0,0 @@
"""
Module containing the SPLoader class.
"""
from abc import ABC, abstractmethod
class SPLoader(ABC):
"""
Base class for special phrases loaders.
Handle the loading of special phrases from external sources.
"""
def __iter__(self):
return self
@abstractmethod
def __next__(self):
pass

View File

@ -3,12 +3,12 @@
"""
import re
import logging
from collections.abc import Iterator
from nominatim.tools.special_phrases.special_phrase import SpecialPhrase
from nominatim.tools.special_phrases.sp_loader import SPLoader
from nominatim.tools.exec_utils import get_url
LOG = logging.getLogger()
class SPWikiLoader(SPLoader):
class SPWikiLoader(Iterator):
"""
Handles loading of special phrases from the wiki.
"""
@ -28,7 +28,7 @@ class SPWikiLoader(SPLoader):
raise StopIteration
lang = self.languages.pop(0)
loaded_xml = SPWikiLoader._get_wiki_content(lang)
loaded_xml = self._get_wiki_content(lang)
LOG.warning('Importing phrases for lang: %s...', lang)
return self.parse_xml(loaded_xml)

View File

@ -13,7 +13,7 @@ def test_parse_csv(sp_csv_loader):
Test method parse_csv()
Should return the right SpecialPhrase objects.
"""
phrases = sp_csv_loader.parse_csv(sp_csv_loader.csv_path)
phrases = sp_csv_loader.parse_csv()
assert check_phrases_content(phrases)
@ -33,10 +33,10 @@ def test_check_csv_validity(sp_csv_loader):
different exception than .csv is given.
"""
sp_csv_loader.csv_path = 'test.csv'
sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
sp_csv_loader.check_csv_validity()
sp_csv_loader.csv_path = 'test.wrong'
with pytest.raises(UsageError):
assert sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
assert sp_csv_loader.check_csv_validity()
def check_phrases_content(phrases):
"""