Code cleaning and SPLoader deleted

nominatim/clicmd/special_phrases.py

@@ -46,7 +46,7 @@ class ImportSpecialPhrases:
     def start_import(args, loader):
         """
             Create the SPImporter object containing the right
-            SPLoader and then start the import of special phrases.
+            sp loader and then start the import of special phrases.
         """
         from ..tokenizer import factory as tokenizer_factory
 

nominatim/tools/special_phrases/sp_csv_loader.py

@@ -5,14 +5,13 @@
 """
 import csv
 import os
+from collections.abc import Iterator
 from nominatim.tools.special_phrases.special_phrase import SpecialPhrase
-from nominatim.tools.special_phrases.sp_loader import SPLoader
 from nominatim.errors import UsageError
 
-class SPCsvLoader(SPLoader):
+class SPCsvLoader(Iterator):
     """
-        Base class for special phrases loaders.
+        Handles loading of special phrases from external csv file.
-        Handle the loading of special phrases from external sources.
     """
     def __init__(self, csv_path):
         super().__init__()
@@ -24,18 +23,17 @@ class SPCsvLoader(SPLoader):
             raise StopIteration()
 
         self.has_been_read = True
-        SPCsvLoader.check_csv_validity(self.csv_path)
+        self.check_csv_validity()
-        return SPCsvLoader.parse_csv(self.csv_path)
+        return self.parse_csv()
 
-    @staticmethod
+    def parse_csv(self):
-    def parse_csv(csv_path):
         """
             Open and parse the given csv file.
             Create the corresponding SpecialPhrases.
         """
         phrases = set()
 
-        with open(csv_path) as file:
+        with open(self.csv_path) as file:
             reader = csv.DictReader(file, delimiter=',')
             for row in reader:
                 phrases.add(
@@ -43,12 +41,11 @@ class SPCsvLoader(SPLoader):
                 )
         return phrases
 
-    @staticmethod
+    def check_csv_validity(self):
-    def check_csv_validity(csv_path):
         """
             Check that the csv file has the right extension.
         """
-        _, extension = os.path.splitext(csv_path)
+        _, extension = os.path.splitext(self.csv_path)
 
         if extension != '.csv':
-            raise UsageError('The file {} is not a csv file.'.format(csv_path))
+            raise UsageError('The file {} is not a csv file.'.format(self.csv_path))

nominatim/tools/special_phrases/sp_importer.py

@@ -25,7 +25,7 @@ class SPImporter():
     """
         Class handling the process of special phrases importations into the database.
 
-        Take a SPLoader which load the phrases from an external source.
+        Take a sp loader which load the phrases from an external source.
     """
     def __init__(self, config, phplib_dir, db_connection, sp_loader) -> None:
         self.config = config
@@ -121,16 +121,14 @@ class SPImporter():
         """
 
         #blacklisting: disallow certain class/type combinations
-        if (
+        if phrase.p_class in self.black_list.keys() \
-            phrase.p_class in self.black_list.keys() and
+           and phrase.p_type in self.black_list[phrase.p_class]:
-            phrase.p_type in self.black_list[phrase.p_class]
+            return None
-        ): return None
 
         #whitelisting: if class is in whitelist, allow only tags in the list
-        if (
+        if phrase.p_class in self.white_list.keys() \
-            phrase.p_class in self.white_list.keys() and
+           and phrase.p_type not in self.white_list[phrase.p_class]:
-            phrase.p_type not in self.white_list[phrase.p_class]
+            return None
-        ): return None
 
         #sanity check, in case somebody added garbage in the wiki
         if not self._check_sanity(phrase):

nominatim/tools/special_phrases/sp_loader.py

@@ -1,16 +0,0 @@
-"""
-    Module containing the SPLoader class.
-"""
-from abc import ABC, abstractmethod
-
-class SPLoader(ABC):
-    """
-        Base class for special phrases loaders.
-        Handle the loading of special phrases from external sources.
-    """
-    def __iter__(self):
-        return self
-
-    @abstractmethod
-    def __next__(self):
-        pass

nominatim/tools/special_phrases/sp_wiki_loader.py

@@ -3,12 +3,12 @@
 """
 import re
 import logging
+from collections.abc import Iterator
 from nominatim.tools.special_phrases.special_phrase import SpecialPhrase
-from nominatim.tools.special_phrases.sp_loader import SPLoader
 from nominatim.tools.exec_utils import get_url
 
 LOG = logging.getLogger()
-class SPWikiLoader(SPLoader):
+class SPWikiLoader(Iterator):
     """
         Handles loading of special phrases from the wiki.
     """
@@ -28,7 +28,7 @@ class SPWikiLoader(SPLoader):
             raise StopIteration
 
         lang = self.languages.pop(0)
-        loaded_xml = SPWikiLoader._get_wiki_content(lang)
+        loaded_xml = self._get_wiki_content(lang)
         LOG.warning('Importing phrases for lang: %s...', lang)
         return self.parse_xml(loaded_xml)
 

test/python/test_tools_sp_csv_loader.py

@@ -13,7 +13,7 @@ def test_parse_csv(sp_csv_loader):
         Test method parse_csv()
         Should return the right SpecialPhrase objects.
     """
-    phrases = sp_csv_loader.parse_csv(sp_csv_loader.csv_path)
+    phrases = sp_csv_loader.parse_csv()
     assert check_phrases_content(phrases)
 
 
@@ -33,10 +33,10 @@ def test_check_csv_validity(sp_csv_loader):
         different exception than .csv is given.
     """
     sp_csv_loader.csv_path = 'test.csv'
-    sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+    sp_csv_loader.check_csv_validity()
     sp_csv_loader.csv_path = 'test.wrong'
     with pytest.raises(UsageError):
-        assert sp_csv_loader.check_csv_validity(sp_csv_loader.csv_path)
+        assert sp_csv_loader.check_csv_validity()
 
 def check_phrases_content(phrases):
     """