219af24 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2318 from dalf/add-pylint

Add pylint

commit219af243a20899d0c8a677700c36e339d7cdcc74 authorAlexandre Flament <alex@al-f.net> dateTue 1 Dec 2020, 15:31 (6 years ago) parenta1e6bc4 8aa2a75

29 files changed+74−94

compare with master

unifiedsplit

Makefile+13−1

@@ -13,6 +13,8 @@ include utils/makefile.include

1313PYOBJECTS = searx

1414DOC = docs

1515PY_SETUP_EXTRAS ?= \[test\]

16+PYLINT_SEARX_DISABLE_OPTION := I,C,R,W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401

17+PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES := supported_languages,language_aliases

1618

1719include utils/makefile.python

1820include utils/makefile.sphinx

@@ -210,8 +212,8 @@ gecko.driver:

210212PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot

211213test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot

212214

213−# TODO: balance linting with pylint

214215

216+# TODO: balance linting with pylint

215217test.pylint: pyenvinstall

216218 $(call cmd,pylint,\

217219 searx/preferences.py \

@@ -219,6 +221,16 @@ test.pylint: pyenvinstall

219221 searx/engines/gigablast.py \

220222 searx/engines/deviantart.py \

221223 )

224+ $(call cmd,pylint,\

225+ --disable=$(PYLINT_SEARX_DISABLE_OPTION) \

226+ --additional-builtins=$(PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES) \

227+ searx/engines \

228+ )

229+ $(call cmd,pylint,\

230+ --disable=$(PYLINT_SEARX_DISABLE_OPTION) \

231+ --ignore=searx/engines \

232+ searx tests \

233+ )

222234

223235# ignored rules:

224236# E402 module level import not at top of file

searx/engines/__init__.py+2−2

@@ -72,7 +72,7 @@ def load_engine(engine_data):

7272

7373 try:

7474 engine = load_module(engine_module + '.py', engine_dir)

75− except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError) as e:

75+ except (SyntaxError, KeyboardInterrupt, SystemExit, SystemError, ImportError, RuntimeError):

7676 logger.exception('Fatal exception in engine "{}"'.format(engine_module))

7777 sys.exit(1)

7878 except:

@@ -234,7 +234,7 @@ def get_engines_stats(preferences):

234234 results = to_percentage(results, max_results)

235235 scores = to_percentage(scores, max_score)

236236 scores_per_result = to_percentage(scores_per_result, max_score_per_result)

237− erros = to_percentage(errors, max_errors)

237+ errors = to_percentage(errors, max_errors)

238238

239239 return [

240240 (

searx/engines/acgsou.py−1

@@ -41,7 +41,6 @@ def response(resp):

4141 # defaults

4242 filesize = 0

4343 magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"

44− torrent_link = ""

4544

4645 try:

4746 category = extract_text(result.xpath(xpath_category)[0])

searx/engines/arxiv.py+1−1

@@ -61,7 +61,7 @@ def response(resp):

6161 content = content_string.format(doi_content="", abstract_content=abstract)

6262

6363 if len(content) > 300:

64− content = content[0:300] + "..."

64+ content = content[0:300] + "..."

6565 # TODO: center snippet on query term

6666

6767 publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ')

searx/engines/base.py+1−4

@@ -80,10 +80,7 @@ def response(resp):

8080

8181 date = datetime.now() # needed in case no dcdate is available for an item

8282 for item in entry:

83− if item.attrib["name"] == "dchdate":

84− harvestDate = item.text

85−

86− elif item.attrib["name"] == "dcdate":

83+ if item.attrib["name"] == "dcdate":

8784 date = item.text

8885

8986 elif item.attrib["name"] == "dctitle":

searx/engines/bing_images.py+1−1

@@ -18,7 +18,7 @@ from json import loads

1818from searx.utils import match_language

1919

2020from searx.engines.bing import language_aliases

21−from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA

21+from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2222

2323# engine dependent config

2424categories = ['images']

searx/engines/bing_news.py+1−1

@@ -17,7 +17,7 @@ from urllib.parse import urlencode, urlparse, parse_qsl

1717from lxml import etree

1818from searx.utils import list_get, match_language

1919from searx.engines.bing import language_aliases

20−from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA

20+from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2121

2222# engine dependent config

2323categories = ['news']

searx/engines/bing_videos.py+1−1

@@ -16,7 +16,7 @@ from urllib.parse import urlencode

1616from searx.utils import match_language

1717

1818from searx.engines.bing import language_aliases

19−from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA

19+from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2020

2121categories = ['videos']

2222paging = True

searx/engines/currency_convert.py−2

@@ -41,8 +41,6 @@ def request(query, params):

4141 from_currency = name_to_iso4217(from_currency.strip())

4242 to_currency = name_to_iso4217(to_currency.strip())

4343

44− q = (from_currency + to_currency).upper()

45−

4644 params['url'] = url.format(from_currency, to_currency)

4745 params['amount'] = amount

4846 params['from'] = from_currency

searx/engines/duckduckgo.py+2−2

@@ -49,11 +49,11 @@ correction_xpath = '//div[@id="did_you_mean"]//a'

4949

5050

5151# match query's language to a region code that duckduckgo will accept

52−def get_region_code(lang, lang_list=[]):

52+def get_region_code(lang, lang_list=None):

5353 if lang == 'all':

5454 return None

5555

56− lang_code = match_language(lang, lang_list, language_aliases, 'wt-WT')

56+ lang_code = match_language(lang, lang_list or [], language_aliases, 'wt-WT')

5757 lang_parts = lang_code.split('-')

5858

5959 # country code goes first

searx/engines/duckduckgo_definitions.py+1−1

@@ -16,7 +16,7 @@ from lxml import html

1616from searx import logger

1717from searx.data import WIKIDATA_UNITS

1818from searx.engines.duckduckgo import language_aliases

19−from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA

19+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2020from searx.utils import extract_text, html_to_text, match_language, get_string_replaces_function

2121from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom

2222

searx/engines/duckduckgo_images.py+1−1

@@ -16,7 +16,7 @@

1616from json import loads

1717from urllib.parse import urlencode

1818from searx.engines.duckduckgo import get_region_code

19−from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA

19+from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2020from searx.poolrequests import get

2121

2222# engine dependent config

searx/engines/filecrop.py+1−1

@@ -8,7 +8,7 @@ search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&eng

88paging = True

1010

11−class FilecropResultParser(HTMLParser):

11+class FilecropResultParser(HTMLParser): # pylint: disable=W0223 # (see https://bugs.python.org/issue31844)

1212

1313 def __init__(self):

1414 HTMLParser.__init__(self)

searx/engines/google_images.py+1−4

@@ -29,10 +29,7 @@ from lxml import html

2929from flask_babel import gettext

3030from searx import logger

3131from searx.utils import extract_text, eval_xpath

32−from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA

33−

34−# pylint: disable=unused-import

35−# pylint: enable=unused-import

32+from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

3633

3734from searx.engines.google import (

3835 get_lang_country,

searx/engines/google_news.py+1−1

@@ -13,7 +13,7 @@

1313from urllib.parse import urlencode

1414from lxml import html

1515from searx.utils import match_language

16−from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA

16+from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

1717

1818# search-url

1919categories = ['news']

searx/engines/peertube.py−1

@@ -90,6 +90,5 @@ def response(resp):

9090

9191

9292def _fetch_supported_languages(resp):

93− ret_val = {}

9493 peertube_languages = list(loads(resp.text).keys())

9594 return peertube_languages

searx/engines/pubmed.py+1−1

@@ -81,7 +81,7 @@ def response(resp):

8181 pass

8282

8383 if len(content) > 300:

84− content = content[0:300] + "..."

84+ content = content[0:300] + "..."

8585 # TODO: center snippet on query term

8686

8787 res_dict = {'url': url,

searx/engines/wikidata.py+10−3

@@ -23,7 +23,7 @@ from searx.data import WIKIDATA_UNITS

2323from searx.poolrequests import post, get

2424from searx.utils import match_language, searx_useragent, get_string_replaces_function

2525from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom

26−from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url # NOQA

26+from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2727

2828logger = logger.getChild('wikidata')

2929

@@ -228,7 +228,7 @@ def get_results(attribute_result, attributes, language):

228228 # Should use normalized value p:P2046/psn:P2046/wikibase:quantityAmount

229229 area = attribute_result.get('P2046')

230230 osm_zoom = area_to_osm_zoom(area) if area else 19

231− url = attribute.get_str(attribute_result, language, osm_zoom=osm_zoom)

231+ url = attribute.get_geo_url(attribute_result, osm_zoom=osm_zoom)

232232 if url:

233233 infobox_urls.append({'title': attribute.get_label(language),

234234 'url': url,

@@ -546,7 +546,14 @@ class WDGeoAttribute(WDAttribute):

546546 def get_group_by(self):

547547 return self.get_select()

548548

549− def get_str(self, result, language, osm_zoom=19):

549+ def get_str(self, result, language):

550+ latitude = result.get(self.name + 'Lat')

551+ longitude = result.get(self.name + 'Long')

552+ if latitude and longitude:

553+ return latitude + ' ' + longitude

554+ return None

555+

556+ def get_geo_url(self, result, osm_zoom=19):

550557 latitude = result.get(self.name + 'Lat')

551558 longitude = result.get(self.name + 'Long')

552559 if latitude and longitude:

searx/engines/yahoo_news.py+1−1

@@ -14,7 +14,7 @@ from datetime import datetime, timedelta

1414from urllib.parse import urlencode

1515from lxml import html

1616from searx.engines.yahoo import parse_url, language_aliases

17−from searx.engines.yahoo import _fetch_supported_languages, supported_languages_url # NOQA

17+from searx.engines.yahoo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

1818from dateutil import parser

1919from searx.utils import extract_text, extract_url, match_language

2020

searx/plugins/__init__.py+2−2

@@ -158,8 +158,8 @@ def prepare_package_resources(pkg, name):

158158

159159def sha_sum(filename):

160160 with open(filename, "rb") as f:

161− bytes = f.read()

162− return sha256(bytes).hexdigest()

161+ file_content_bytes = f.read()

162+ return sha256(file_content_bytes).hexdigest()

163163

164164

165165plugins = PluginStore()

searx/query.py+9−9

@@ -92,15 +92,15 @@ class RawTextQuery:

9292 or lang == english_name

9393 or lang.replace('-', ' ') == country)\

9494 and lang not in self.languages:

95− searx_query_part = True

96− lang_parts = lang_id.split('-')

97− if len(lang_parts) == 2:

98− self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())

99− else:

100− self.languages.append(lang_id)

101− # to ensure best match (first match is not necessarily the best one)

102− if lang == lang_id:

103− break

95+ searx_query_part = True

96+ lang_parts = lang_id.split('-')

97+ if len(lang_parts) == 2:

98+ self.languages.append(lang_parts[0] + '-' + lang_parts[1].upper())

99+ else:

100+ self.languages.append(lang_id)

101+ # to ensure best match (first match is not necessarily the best one)

102+ if lang == lang_id:

103+ break

104104

105105 # user may set a valid, yet not selectable language

106106 if VALID_LANGUAGE_CODE.match(lang):

searx/results.py+1−1

@@ -298,7 +298,7 @@ class ResultContainer:

298298 gresults = []

299299 categoryPositions = {}

300300

301− for i, res in enumerate(results):

301+ for res in results:

302302 # FIXME : handle more than one category per engine

303303 res['category'] = engines[res['engine']].categories[0]

304304

searx/search.py+2−3

@@ -43,9 +43,8 @@ else:

4343 logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))

4444 else:

4545 logger.critical('outgoing.max_request_timeout if defined has to be float')

46− from sys import exit

47−

48− exit(1)

46+ import sys

47+ sys.exit(1)

4948

5049

5150class EngineRef:

searx/utils.py+9−9

@@ -52,7 +52,7 @@ class HTMLTextExtractorException(Exception):

5252 pass

5353

5454

55−class HTMLTextExtractor(HTMLParser):

55+class HTMLTextExtractor(HTMLParser): # pylint: disable=W0223 # (see https://bugs.python.org/issue31844)

5656

5757 def __init__(self):

5858 HTMLParser.__init__(self)

@@ -74,18 +74,18 @@ class HTMLTextExtractor(HTMLParser):

7474 def is_valid_tag(self):

7575 return not self.tags or self.tags[-1] not in blocked_tags

7676

77− def handle_data(self, d):

77+ def handle_data(self, data):

7878 if not self.is_valid_tag():

7979 return

80− self.result.append(d)

80+ self.result.append(data)

8181

82− def handle_charref(self, number):

82+ def handle_charref(self, name):

8383 if not self.is_valid_tag():

8484 return

85− if number[0] in ('x', 'X'):

86− codepoint = int(number[1:], 16)

85+ if name[0] in ('x', 'X'):

86+ codepoint = int(name[1:], 16)

8787 else:

88− codepoint = int(number)

88+ codepoint = int(name)

8989 self.result.append(chr(codepoint))

9090

9191 def handle_entityref(self, name):

@@ -380,7 +380,7 @@ def _get_lang_to_lc_dict(lang_list):

380380 return value

381381

382382

383−def _match_language(lang_code, lang_list=[], custom_aliases={}):

383+def _match_language(lang_code, lang_list=[], custom_aliases={}): # pylint: disable=W0102

384384 """auxiliary function to match lang_code in lang_list"""

385385 # replace language code with a custom alias if necessary

386386 if lang_code in custom_aliases:

@@ -403,7 +403,7 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):

403403 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)

404404

405405

406−def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):

406+def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'): # pylint: disable=W0102

407407 """get the language code from lang_list that best matches locale_code"""

408408 # try to get language from given locale_code

409409 language = _match_language(locale_code, lang_list, custom_aliases)

searx/webapp.py+1−2

@@ -466,7 +466,7 @@ def pre_request():

466466 else:

467467 try:

468468 preferences.parse_dict(request.form)

469− except Exception as e:

469+ except Exception:

470470 logger.exception('invalid settings')

471471 request.errors.append(gettext('Invalid settings'))

472472

@@ -819,7 +819,6 @@ def preferences():

819819

820820 # render preferences

821821 image_proxy = request.preferences.get_value('image_proxy')

822− lang = request.preferences.get_value('language')

823822 disabled_engines = request.preferences.engines.get_disabled()

824823 allowed_plugins = request.preferences.plugins.get_enabled()

825824

tests/test_robot.pydeleted−23

@@ -1,23 +0,0 @@

1−# -*- coding: utf-8 -*-

2−

3−import os

4−import unittest2 as unittest

5−from plone.testing import layered

6−from robotsuite import RobotTestSuite

7−from searx.testing import SEARXROBOTLAYER

8−

9−

10−def test_suite():

11− suite = unittest.TestSuite()

12− current_dir = os.path.abspath(os.path.dirname(__file__))

13− robot_dir = os.path.join(current_dir, 'robot')

14− tests = [

15− os.path.join('robot', f) for f in

16− os.listdir(robot_dir) if f.endswith('.robot') and

17− f.startswith('test_')

18− ]

19− for test in tests:

20− suite.addTests([

21− layered(RobotTestSuite(test), layer=SEARXROBOTLAYER),

22− ])

23− return suite

tests/unit/test_plugins.py+2−2

@@ -31,10 +31,10 @@ class PluginStoreTest(SearxTestCase):

3131 request = Mock()

3232 store.call([], 'asdf', request, Mock())

3333

34− self.assertFalse(testplugin.asdf.called)

34+ self.assertFalse(testplugin.asdf.called) # pylint: disable=E1101

3535

3636 store.call([testplugin], 'asdf', request, Mock())

37− self.assertTrue(testplugin.asdf.called)

37+ self.assertTrue(testplugin.asdf.called) # pylint: disable=E1101

3838

3939

4040class SelfIPTest(SearxTestCase):

tests/unit/test_preferences.py+8−12

@@ -5,8 +5,8 @@ from searx.testing import SearxTestCase

66class PluginStub:

8− def __init__(self, id, default_on):

9− self.id = id

8+ def __init__(self, plugin_id, default_on):

9+ self.id = plugin_id

1010 self.default_on = default_on

1111

1212

@@ -15,11 +15,11 @@ class TestSettings(SearxTestCase):

1515

1616 def test_map_setting_invalid_initialization(self):

1717 with self.assertRaises(MissingArgumentException):

18− setting = MapSetting(3, wrong_argument={'0': 0})

18+ MapSetting(3, wrong_argument={'0': 0})

1919

2020 def test_map_setting_invalid_default_value(self):

2121 with self.assertRaises(ValidationException):

22− setting = MapSetting(3, map={'dog': 1, 'bat': 2})

22+ MapSetting(3, map={'dog': 1, 'bat': 2})

2323

2424 def test_map_setting_invalid_choice(self):

2525 setting = MapSetting(2, map={'dog': 1, 'bat': 2})

@@ -36,18 +36,14 @@ class TestSettings(SearxTestCase):

3636 setting.parse('bat')

3737 self.assertEqual(setting.get_value(), 2)

3838

39− def test_enum_setting_invalid_initialization(self):

40− with self.assertRaises(MissingArgumentException):

41− setting = EnumStringSetting('cat', wrong_argument=[0, 1, 2])

42−

4339 # enum settings

4440 def test_enum_setting_invalid_initialization(self):

4541 with self.assertRaises(MissingArgumentException):

46− setting = EnumStringSetting('cat', wrong_argument=[0, 1, 2])

42+ EnumStringSetting('cat', wrong_argument=[0, 1, 2])

4743

4844 def test_enum_setting_invalid_default_value(self):

4945 with self.assertRaises(ValidationException):

50− setting = EnumStringSetting(3, choices=[0, 1, 2])

46+ EnumStringSetting(3, choices=[0, 1, 2])

5147

5248 def test_enum_setting_invalid_choice(self):

5349 setting = EnumStringSetting(0, choices=[0, 1, 2])

@@ -67,11 +63,11 @@ class TestSettings(SearxTestCase):

6763 # multiple choice settings

6864 def test_multiple_setting_invalid_initialization(self):

6965 with self.assertRaises(MissingArgumentException):

70− setting = MultipleChoiceSetting(['2'], wrong_argument=['0', '1', '2'])

66+ MultipleChoiceSetting(['2'], wrong_argument=['0', '1', '2'])

7167

7268 def test_multiple_setting_invalid_default_value(self):

7369 with self.assertRaises(ValidationException):

74− setting = MultipleChoiceSetting(['3', '4'], choices=['0', '1', '2'])

70+ MultipleChoiceSetting(['3', '4'], choices=['0', '1', '2'])

7571

7672 def test_multiple_setting_invalid_choice(self):

7773 setting = MultipleChoiceSetting(['1', '2'], choices=['0', '1', '2'])

tests/unit/test_webapp.py+1−1

@@ -145,7 +145,7 @@ class ViewsTestCase(SearxTestCase):

145145 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

146146 self.assertEqual(result.status_code, 308)

147147

148− def test_index_rss(self):

148+ def test_search_rss(self):

149149 result = self.app.post('/search', data={'q': 'test', 'format': 'rss'})

150150

151151 self.assertIn(