git.iko.soy/searx

on master help sign in

Makefile−5

@@ -213,10 +213,6 @@ gecko.driver:

213213PHONY += test test.sh test.pylint test.pep8 test.unit test.coverage test.robot

214214test: buildenv test.pylint test.pep8 test.unit gecko.driver test.robot

215215

216−ifeq ($(PY),2)

217−test.pylint:

218− @echo "LINT skip liniting py2"

219−else

220216# TODO: balance linting with pylint

221217

222218test.pylint: pyenvinstall

@@ -225,7 +221,6 @@ test.pylint: pyenvinstall

225221 searx/testing.py \

226222 searx/engines/gigablast.py \

227223 )

228−endif

229224

230225# ignored rules:

231226# E402 module level import not at top of file

manage.sh+2−2

@@ -39,7 +39,7 @@ install_geckodriver() {

3939 return

4040 fi

4141 GECKODRIVER_VERSION="v0.24.0"

42− PLATFORM="`python -c "import six; import platform; six.print_(platform.system().lower(), platform.architecture()[0])"`"

42+ PLATFORM="`python3 -c "import platform; print(platform.system().lower(), platform.architecture()[0])"`"

4343 case "$PLATFORM" in

4444 "linux 32bit" | "linux2 32bit") ARCH="linux32";;

4545 "linux 64bit" | "linux2 64bit") ARCH="linux64";;

@@ -136,7 +136,7 @@ docker_build() {

136136 # Check consistency between the git tag and the searx/version.py file

137137 # /!\ HACK : parse Python file with bash /!\

138138 # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py )

139− # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")

139+ # SEARX_PYTHON_VERSION=$(python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)")

140140 SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -)

141141 if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then

142142 echo "Inconsistency between the last git tag and the searx/version.py file"

searx/__init__.py+2−6

@@ -21,12 +21,8 @@ from os import environ

2121from os.path import realpath, dirname, join, abspath, isfile

2222from io import open

2323from ssl import OPENSSL_VERSION_INFO, OPENSSL_VERSION

24−try:

25− from yaml import safe_load

26−except:

27− from sys import exit, stderr

28− stderr.write('[E] install pyyaml\n')

29− exit(2)

24+from yaml import safe_load

25+

3026

3127searx_dir = abspath(dirname(__file__))

3228engine_dir = dirname(realpath(__file__))

searx/answerers/__init__.py+2−6

@@ -1,12 +1,8 @@

11from os import listdir

22from os.path import realpath, dirname, join, isdir

3−from sys import version_info

43from searx.utils import load_module

54from collections import defaultdict

65

7−if version_info[0] == 3:

8− unicode = str

9−

106

117answerers_dir = dirname(realpath(__file__))

128

@@ -36,10 +32,10 @@ def ask(query):

3632 results = []

3733 query_parts = list(filter(None, query.query.split()))

3834

39− if query_parts[0].decode('utf-8') not in answerers_by_keywords:

35+ if query_parts[0] not in answerers_by_keywords:

4036 return results

4137

42− for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:

38+ for answerer in answerers_by_keywords[query_parts[0]]:

4339 result = answerer(query)

4440 if result:

4541 results.append(result)

searx/answerers/random/answerer.py+12−18

@@ -1,7 +1,6 @@

11import hashlib

22import random

33import string

4−import sys

54import uuid

65from flask_babel import gettext

76

@@ -10,12 +9,7 @@ from flask_babel import gettext

109keywords = ('random',)

1110

1211random_int_max = 2**31

13−

14−if sys.version_info[0] == 2:

15− random_string_letters = string.lowercase + string.digits + string.uppercase

16−else:

17− unicode = str

18− random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

12+random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

1913

2014

2115def random_characters():

@@ -24,32 +18,32 @@ def random_characters():

2418

2519

2620def random_string():

27− return u''.join(random_characters())

21+ return ''.join(random_characters())

2822

2923

3024def random_float():

31− return unicode(random.random())

25+ return str(random.random())

3226

3327

3428def random_int():

35− return unicode(random.randint(-random_int_max, random_int_max))

29+ return str(random.randint(-random_int_max, random_int_max))

3630

3731

3832def random_sha256():

3933 m = hashlib.sha256()

4034 m.update(''.join(random_characters()).encode())

41− return unicode(m.hexdigest())

35+ return str(m.hexdigest())

4236

4337

4438def random_uuid():

45− return unicode(uuid.uuid4())

39+ return str(uuid.uuid4())

4640

4741

48−random_types = {b'string': random_string,

49− b'int': random_int,

50− b'float': random_float,

51− b'sha256': random_sha256,

52− b'uuid': random_uuid}

42+random_types = {'string': random_string,

43+ 'int': random_int,

44+ 'float': random_float,

45+ 'sha256': random_sha256,

46+ 'uuid': random_uuid}

5347

5448

5549# required answerer function

@@ -70,4 +64,4 @@ def answer(query):

7064def self_info():

7165 return {'name': gettext('Random value generator'),

7266 'description': gettext('Generate different random values'),

73− 'examples': [u'random {}'.format(x.decode('utf-8')) for x in random_types]}

67+ 'examples': ['random {}'.format(x) for x in random_types]}

searx/answerers/statistics/answerer.py+6−9

@@ -1,11 +1,8 @@

1−from sys import version_info

21from functools import reduce

32from operator import mul

43

54from flask_babel import gettext

65

7−if version_info[0] == 3:

8− unicode = str

96

107keywords = ('min',

118 'max',

@@ -30,21 +27,21 @@ def answer(query):

3027 func = parts[0]

3128 answer = None

3229

33− if func == b'min':

30+ if func == 'min':

3431 answer = min(args)

35− elif func == b'max':

32+ elif func == 'max':

3633 answer = max(args)

37− elif func == b'avg':

34+ elif func == 'avg':

3835 answer = sum(args) / len(args)

39− elif func == b'sum':

36+ elif func == 'sum':

4037 answer = sum(args)

41− elif func == b'prod':

38+ elif func == 'prod':

4239 answer = reduce(mul, args, 1)

4340

4441 if answer is None:

4542 return []

4643

47− return [{'answer': unicode(answer)}]

44+ return [{'answer': str(answer)}]

4845

4946

5047# required answerer function

searx/autocomplete.py+7−10

@@ -16,19 +16,16 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1616'''

1717

1818

19−import sys

2019from lxml import etree

2120from json import loads

21+from urllib.parse import urlencode

22+

2223from searx import settings

2324from searx.languages import language_codes

2425from searx.engines import (

2526 categories, engines, engine_shortcuts

2627)

2728from searx.poolrequests import get as http_get

28−from searx.url_utils import urlencode

29−

30−if sys.version_info[0] == 3:

31− unicode = str

3229

3330

3431def get(*args, **kwargs):

@@ -85,22 +82,22 @@ def searx_bang(full_query):

8582 engine_query = full_query.getSearchQuery()[1:]

8683

8784 for lc in language_codes:

88− lang_id, lang_name, country, english_name = map(unicode.lower, lc)

85+ lang_id, lang_name, country, english_name = map(str.lower, lc)

8986

9087 # check if query starts with language-id

9188 if lang_id.startswith(engine_query):

9289 if len(engine_query) <= 2:

93− results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0]))

90+ results.append(':{lang_id}'.format(lang_id=lang_id.split('-')[0]))

9491 else:

95− results.append(u':{lang_id}'.format(lang_id=lang_id))

92+ results.append(':{lang_id}'.format(lang_id=lang_id))

9693

9794 # check if query starts with language name

9895 if lang_name.startswith(engine_query) or english_name.startswith(engine_query):

99− results.append(u':{lang_name}'.format(lang_name=lang_name))

96+ results.append(':{lang_name}'.format(lang_name=lang_name))

10097

10198 # check if query starts with country

10299 if country.startswith(engine_query.replace('_', ' ')):

103− results.append(u':{country}'.format(country=country.replace(' ', '_')))

100+ results.append(':{country}'.format(country=country.replace(' ', '_')))

104101

105102 # remove duplicates

106103 result_set = set(results)

searx/engines/1337x.py+2−1

@@ -1,7 +1,8 @@

1+from urllib.parse import quote, urljoin

12from lxml import html

23from searx.engines.xpath import extract_text

34from searx.utils import get_torrent_size

4−from searx.url_utils import quote, urljoin

5+

56

67url = 'https://1337x.to/'

78search_url = url + 'search/{search_term}/{pageno}/'

searx/engines/acgsou.py+2−2

@@ -9,9 +9,9 @@

99 @parse url, title, content, seed, leech, torrentfile

1010"""

1111

12+from urllib.parse import urlencode

1213from lxml import html

1314from searx.engines.xpath import extract_text

14−from searx.url_utils import urlencode

1515from searx.utils import get_torrent_size, int_or_zero

1616

1717# engine dependent config

@@ -63,7 +63,7 @@ def response(resp):

6363 except:

6464 pass

6565 # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime

66− content = u'Category: "{category}".'

66+ content = 'Category: "{category}".'

6767 content = content.format(category=category)

6868

6969 results.append({'url': href,

searx/engines/apkmirror.py+2−1

@@ -9,9 +9,10 @@

99 @parse url, title, thumbnail_src

1010"""

1111

12+from urllib.parse import urlencode

1213from lxml import html

1314from searx.engines.xpath import extract_text

14−from searx.url_utils import urlencode

15+

1516

1617# engine dependent config

1718categories = ['it']

searx/engines/archlinux.py+2−2

@@ -11,9 +11,9 @@

1111 @parse url, title

1212"""

1313

14+from urllib.parse import urlencode, urljoin

1415from lxml import html

1516from searx.engines.xpath import extract_text

16−from searx.url_utils import urlencode, urljoin

1717

1818# engine dependent config

1919categories = ['it']

@@ -105,7 +105,7 @@ def request(query, params):

105105 # if our language is hosted on the main site, we need to add its name

106106 # to the query in order to narrow the results to that language

107107 if language in main_langs:

108− query += b' (' + main_langs[language] + b')'

108+ query += ' (' + main_langs[language] + ')'

109109

110110 # prepare the request parameters

111111 query = urlencode({'search': query})

searx/engines/arxiv.py+2−2

@@ -11,9 +11,9 @@

1111 More info on api: https://arxiv.org/help/api/user-manual

1212"""

1313

14+from urllib.parse import urlencode

1415from lxml import html

1516from datetime import datetime

16−from searx.url_utils import urlencode

1717

1818

1919categories = ['science']

@@ -30,7 +30,7 @@ def request(query, params):

3030 # basic search

3131 offset = (params['pageno'] - 1) * number_of_results

3232

33− string_args = dict(query=query.decode('utf-8'),

33+ string_args = dict(query=query,

3434 offset=offset,

3535 number_of_results=number_of_results)

3636

searx/engines/base.py+2−2

@@ -13,10 +13,10 @@

1313 More info on api: http://base-search.net/about/download/base_interface.pdf

1414"""

1515

16+from urllib.parse import urlencode

1617from lxml import etree

1718from datetime import datetime

1819import re

19−from searx.url_utils import urlencode

2020from searx.utils import searx_useragent

2121

2222

@@ -55,7 +55,7 @@ shorcut_dict = {

5555def request(query, params):

5656 # replace shortcuts with API advanced search keywords

5757 for key in shorcut_dict.keys():

58− query = re.sub(key, shorcut_dict[key], str(query))

58+ query = re.sub(key, shorcut_dict[key], query)

5959

6060 # basic search

6161 offset = (params['pageno'] - 1) * number_of_results

searx/engines/bing.py+2−2

@@ -14,10 +14,10 @@

1414"""

1515

1616import re

17+from urllib.parse import urlencode

1718from lxml import html

1819from searx import logger, utils

1920from searx.engines.xpath import extract_text

20−from searx.url_utils import urlencode

2121from searx.utils import match_language, gen_useragent, eval_xpath

2222

2323logger = logger.getChild('bing engine')

@@ -47,7 +47,7 @@ def request(query, params):

4747 else:

4848 lang = match_language(params['language'], supported_languages, language_aliases)

4949

50− query = u'language:{} {}'.format(lang.split('-')[0].upper(), query.decode('utf-8')).encode('utf-8')

50+ query = 'language:{} {}'.format(lang.split('-')[0].upper(), query)

5151

5252 search_path = search_string.format(

5353 query=urlencode({'q': query}),

searx/engines/bing_images.py+2−2

@@ -12,10 +12,10 @@

1212

1313"""

1414

15+from urllib.parse import urlencode

1516from lxml import html

1617from json import loads

1718import re

18−from searx.url_utils import urlencode

1919from searx.utils import match_language

2020

2121from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases

@@ -91,7 +91,7 @@ def response(resp):

9191

9292 # strip 'Unicode private use area' highlighting, they render to Tux

9393 # the Linux penguin and a standing diamond on my machine...

94− title = m.get('t', '').replace(u'\ue000', '').replace(u'\ue001', '')

94+ title = m.get('t', '').replace('\ue000', '').replace('\ue001', '')

9595 results.append({'template': 'images.html',

9696 'url': m['purl'],

9797 'thumbnail_src': m['turl'],

searx/engines/bing_news.py+1−2

@@ -13,10 +13,9 @@

1313

1414from datetime import datetime

1515from dateutil import parser

16+from urllib.parse import urlencode, urlparse, parse_qsl

1617from lxml import etree

1718from searx.utils import list_get, match_language

18−from searx.url_utils import urlencode, urlparse, parse_qsl

19−

2019from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases

2120

2221# engine dependent config

searx/engines/bing_videos.py+1−1

@@ -12,7 +12,7 @@

1212

1313from json import loads

1414from lxml import html

15−from searx.url_utils import urlencode

15+from urllib.parse import urlencode

1616from searx.utils import match_language

1717

1818from searx.engines.bing import _fetch_supported_languages, supported_languages_url, language_aliases

searx/engines/btdigg.py+1−1

@@ -12,8 +12,8 @@

1212

1313from lxml import html

1414from operator import itemgetter

15+from urllib.parse import quote, urljoin

1516from searx.engines.xpath import extract_text

16−from searx.url_utils import quote, urljoin

1717from searx.utils import get_torrent_size

1818

1919# engine dependent config

searx/engines/currency_convert.py+2−5

@@ -1,26 +1,23 @@

11import json

22import re

33import os

4−import sys

54import unicodedata

65

76from io import open

87from datetime import datetime

98

10−if sys.version_info[0] == 3:

11− unicode = str

129

1310categories = []

1411url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'

1512weight = 100

1613

17−parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

14+parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

1815

1916db = 1

2017

2118

2219def normalize_name(name):

23− name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')

20+ name = name.lower().replace('-', ' ').rstrip('s')

2421 name = re.sub(' +', ' ', name)

2522 return unicodedata.normalize('NFKD', name).lower()

2623

searx/engines/dailymotion.py+1−1

@@ -14,7 +14,7 @@

1414

1515from json import loads

1616from datetime import datetime

17−from searx.url_utils import urlencode

17+from urllib.parse import urlencode

1818from searx.utils import match_language, html_to_text

1919

2020# engine dependent config

searx/engines/deezer.py+2−2

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515

1616# engine dependent config

1717categories = ['music']

@@ -50,7 +50,7 @@ def response(resp):

5050 if url.startswith('http://'):

5151 url = 'https' + url[4:]

5252

53− content = u'{} - {} - {}'.format(

53+ content = '{} - {} - {}'.format(

5454 result['artist']['name'],

5555 result['album']['title'],

5656 result['title'])

searx/engines/deviantart.py+2−1

@@ -14,8 +14,9 @@

1414

1515from lxml import html

1616import re

17+from urllib.parse import urlencode

1718from searx.engines.xpath import extract_text

18−from searx.url_utils import urlencode

19+

1920

2021# engine dependent config

2122categories = ['images']

searx/engines/dictzone.py+4−4

@@ -10,15 +10,15 @@

1010"""

1111

1212import re

13+from urllib.parse import urljoin

1314from lxml import html

1415from searx.utils import is_valid_lang, eval_xpath

15−from searx.url_utils import urljoin

1616

1717categories = ['general']

18−url = u'https://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

18+url = 'https://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

1919weight = 100

2020

21−parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

21+parser_re = re.compile('.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

2222results_xpath = './/table[@id="r"]/tr'

2323

2424

@@ -37,7 +37,7 @@ def request(query, params):

3737

3838 params['url'] = url.format(from_lang=from_lang[2],

3939 to_lang=to_lang[2],

40− query=query.decode('utf-8'))

40+ query=query)

4141

4242 return params

4343

searx/engines/digbt.py+1−4

@@ -10,14 +10,11 @@

1010 @parse url, title, content, magnetlink

1111"""

1212

13−from sys import version_info

13+from urllib.parse import urljoin

1414from lxml import html

1515from searx.engines.xpath import extract_text

1616from searx.utils import get_torrent_size

17−from searx.url_utils import urljoin

1817

19−if version_info[0] == 3:

20− unicode = str

2118

2219categories = ['videos', 'music', 'files']

2320paging = True

searx/engines/digg.py+1−1

@@ -14,8 +14,8 @@ import random

1414import string

1515from dateutil import parser

1616from json import loads

17+from urllib.parse import urlencode

1718from lxml import html

18−from searx.url_utils import urlencode

1919from datetime import datetime

2020

2121# engine dependent config

searx/engines/doku.py+1−1

@@ -9,10 +9,10 @@

99# @stable yes

1010# @parse (general) url, title, content

1111

12+from urllib.parse import urlencode

1213from lxml.html import fromstring

1314from searx.engines.xpath import extract_text

1415from searx.utils import eval_xpath

15−from searx.url_utils import urlencode

1616

1717# engine dependent config

1818categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py+1−1

@@ -15,9 +15,9 @@

1515

1616from lxml.html import fromstring

1717from json import loads

18+from urllib.parse import urlencode

1819from searx.engines.xpath import extract_text

1920from searx.poolrequests import get

20−from searx.url_utils import urlencode

2121from searx.utils import match_language, eval_xpath

2222

2323# engine dependent config

searx/engines/duckduckgo_definitions.py+1−1

@@ -10,11 +10,11 @@ DuckDuckGo (definitions)

1010"""

1111

1212import json

13+from urllib.parse import urlencode

1314from lxml import html

1415from re import compile

1516from searx.engines.xpath import extract_text

1617from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases

17−from searx.url_utils import urlencode

1818from searx.utils import html_to_text, match_language

1919

2020url = 'https://api.duckduckgo.com/'\

searx/engines/duckduckgo_images.py+1−1

@@ -14,13 +14,13 @@

1414"""

1515

1616from json import loads

17+from urllib.parse import urlencode

1718from searx.engines.xpath import extract_text

1819from searx.engines.duckduckgo import (

1920 _fetch_supported_languages, supported_languages_url,

2021 get_region_code, language_aliases

2122)

2223from searx.poolrequests import get

23−from searx.url_utils import urlencode

2424

2525# engine dependent config

2626categories = ['images']

searx/engines/duden.py+1−1

@@ -10,9 +10,9 @@

1010

1111from lxml import html, etree

1212import re

13+from urllib.parse import quote, urljoin

1314from searx.engines.xpath import extract_text

1415from searx.utils import eval_xpath

15−from searx.url_utils import quote, urljoin

1616from searx import logger

1717

1818categories = ['general']

searx/engines/etools.py+1−1

@@ -10,8 +10,8 @@

1010"""

1111

1212from lxml import html

13+from urllib.parse import quote

1314from searx.engines.xpath import extract_text

14−from searx.url_utils import quote

1515from searx.utils import eval_xpath

1616

1717categories = ['general']

searx/engines/fdroid.py+1−1

@@ -9,9 +9,9 @@

99 @parse url, title, content

1010"""

1111

12+from urllib.parse import urlencode

1213from lxml import html

1314from searx.engines.xpath import extract_text

14−from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['files']

searx/engines/filecrop.py+2−5

@@ -1,9 +1,6 @@

1−from searx.url_utils import urlencode

1+from html.parser import HTMLParser

2+from urllib.parse import urlencode

23

3−try:

4− from HTMLParser import HTMLParser

5−except:

6− from html.parser import HTMLParser

74

85url = 'http://www.filecrop.com/'

96search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa

searx/engines/flickr.py+1−1

@@ -14,7 +14,7 @@

1414"""

1515

1616from json import loads

17−from searx.url_utils import urlencode

17+from urllib.parse import urlencode

1818

1919categories = ['images']

2020

searx/engines/flickr_noapi.py+5−5

@@ -15,8 +15,8 @@

1515from json import loads

1616from time import time

1717import re

18+from urllib.parse import urlencode

1819from searx.engines import logger

19−from searx.url_utils import urlencode

2020from searx.utils import ecma_unescape, html_to_text

2121

2222logger = logger.getChild('flickr-noapi')

@@ -117,10 +117,10 @@ def response(resp):

117117 'img_format': img_format,

118118 'template': 'images.html'

119119 }

120− result['author'] = author.encode('utf-8', 'ignore').decode('utf-8')

121− result['source'] = source.encode('utf-8', 'ignore').decode('utf-8')

122− result['title'] = title.encode('utf-8', 'ignore').decode('utf-8')

123− result['content'] = content.encode('utf-8', 'ignore').decode('utf-8')

120+ result['author'] = author.encode(errors='ignore').decode()

121+ result['source'] = source.encode(errors='ignore').decode()

122+ result['title'] = title.encode(errors='ignore').decode()

123+ result['content'] = content.encode(errors='ignore').decode()

124124 results.append(result)

125125

126126 return results

searx/engines/framalibre.py+2−5

@@ -10,13 +10,10 @@

1010 @parse url, title, content, thumbnail, img_src

1111"""

1212

13−try:

14− from cgi import escape

15−except:

16− from html import escape

13+from html import escape

14+from urllib.parse import urljoin, urlencode

1715from lxml import html

1816from searx.engines.xpath import extract_text

19−from searx.url_utils import urljoin, urlencode

2017

2118# engine dependent config

2219categories = ['it']

searx/engines/frinkiac.py+1−1

@@ -10,7 +10,7 @@ Frinkiac (Images)

1010"""

1111

1212from json import loads

13−from searx.url_utils import urlencode

13+from urllib.parse import urlencode

1414

1515categories = ['images']

1616

searx/engines/genius.py+1−1

@@ -11,7 +11,7 @@ Genius

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515from datetime import datetime

1616

1717# engine dependent config

searx/engines/gentoo.py+2−2

@@ -11,9 +11,9 @@

1111 @parse url, title

1212"""

1313

14+from urllib.parse import urlencode, urljoin

1415from lxml import html

1516from searx.engines.xpath import extract_text

16−from searx.url_utils import urlencode, urljoin

1717

1818# engine dependent config

1919categories = ['it']

@@ -90,7 +90,7 @@ def request(query, params):

9090 # if our language is hosted on the main site, we need to add its name

9191 # to the query in order to narrow the results to that language

9292 if language in main_langs:

93− query += b' (' + (main_langs[language]).encode('utf-8') + b')'

93+ query += ' (' + main_langs[language] + ')'

9494

9595 # prepare the request parameters

9696 query = urlencode({'search': query})

searx/engines/gigablast.py+1−1

@@ -14,8 +14,8 @@

1414

1515import re

1616from json import loads

17+from urllib.parse import urlencode

1718# from searx import logger

18−from searx.url_utils import urlencode

1919from searx.poolrequests import get

2020

2121# engine dependent config

searx/engines/github.py+1−1

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515

1616# engine dependent config

1717categories = ['it']

searx/engines/google.py+1−1

@@ -18,11 +18,11 @@ Definitions`_.

1818

1919# pylint: disable=invalid-name, missing-function-docstring

2020

21+from urllib.parse import urlencode, urlparse

2122from lxml import html

2223from flask_babel import gettext

2324from searx.engines.xpath import extract_text

2425from searx import logger

25−from searx.url_utils import urlencode, urlparse

2626from searx.utils import match_language, eval_xpath

2727

2828logger = logger.getChild('google engine')

searx/engines/google_images.py+2−3

@@ -24,11 +24,10 @@ Definitions`_.

2424

2525"""

2626

27−import urllib

27+from urllib.parse import urlencode, urlparse, unquote

2828from lxml import html

2929from flask_babel import gettext

3030from searx import logger

31−from searx.url_utils import urlencode, urlparse

3231from searx.utils import eval_xpath

3332from searx.engines.xpath import extract_text

3433

@@ -87,7 +86,7 @@ def scrap_img_by_id(script, data_id):

8786 if 'gstatic.com/images' in line and data_id in line:

8887 url_line = _script[i + 1]

8988 img_url = url_line.split('"')[1]

90− img_url = urllib.parse.unquote(img_url.replace(r'\u00', r'%'))

89+ img_url = unquote(img_url.replace(r'\u00', r'%'))

9190 return img_url

9291

9392

searx/engines/google_news.py+1−1

@@ -10,9 +10,9 @@

1010 @parse url, title, content, publishedDate

1111"""

1212

13+from urllib.parse import urlencode

1314from lxml import html

1415from searx.engines.google import _fetch_supported_languages, supported_languages_url

15−from searx.url_utils import urlencode

1616from searx.utils import match_language

1717

1818# search-url

searx/engines/google_videos.py+1−1

@@ -12,9 +12,9 @@

1212

1313from datetime import date, timedelta

1414from json import loads

15+from urllib.parse import urlencode

1516from lxml import html

1617from searx.engines.xpath import extract_text

17−from searx.url_utils import urlencode

1818import re

1919

2020# engine dependent config

searx/engines/ina.py+2−5

@@ -12,15 +12,12 @@

1212# @todo embedded (needs some md5 from video page)

1313

1414from json import loads

15+from urllib.parse import urlencode

1516from lxml import html

1617from dateutil import parser

18+from html.parser import HTMLParser

1719from searx.engines.xpath import extract_text

18−from searx.url_utils import urlencode

1920

20−try:

21− from HTMLParser import HTMLParser

22−except:

23− from html.parser import HTMLParser

2421

2522# engine dependent config

2623categories = ['videos']

searx/engines/invidious.py+1−1

@@ -8,7 +8,7 @@

88# @stable yes

99# @parse url, title, content, publishedDate, thumbnail, embedded, author, length

1010

11−from searx.url_utils import quote_plus

11+from urllib.parse import quote_plus

1212from dateutil import parser

1313import time

1414

searx/engines/json_engine.py+1−6

@@ -1,11 +1,8 @@

11from collections import Iterable

22from json import loads

3−from sys import version_info

4−from searx.url_utils import urlencode

3+from urllib.parse import urlencode

54from searx.utils import to_string

65

7−if version_info[0] == 3:

8− unicode = str

96

107search_url = None

118url_query = None

@@ -37,8 +34,6 @@ def iterate(iterable):

3734def is_iterable(obj):

3835 if type(obj) == str:

3936 return False

40− if type(obj) == unicode:

41− return False

4237 return isinstance(obj, Iterable)

4338

4439

searx/engines/kickass.py+1−1

@@ -12,9 +12,9 @@

1212

1313from lxml import html

1414from operator import itemgetter

15+from urllib.parse import quote, urljoin

1516from searx.engines.xpath import extract_text

1617from searx.utils import get_torrent_size, convert_str_to_int

17−from searx.url_utils import quote, urljoin

1818

1919# engine dependent config

2020categories = ['videos', 'music', 'files']

searx/engines/mediawiki.py+2−2

@@ -14,7 +14,7 @@

1414

1515from json import loads

1616from string import Formatter

17−from searx.url_utils import urlencode, quote

17+from urllib.parse import urlencode, quote

1818

1919# engine dependent config

2020categories = ['general']

@@ -79,7 +79,7 @@ def response(resp):

7979 if result.get('snippet', '').startswith('#REDIRECT'):

8080 continue

8181 url = base_url.format(language=resp.search_params['language']) +\

82− 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8'))

82+ 'wiki/' + quote(result['title'].replace(' ', '_').encode())

8383

8484 # append result

8585 results.append({'url': url,

searx/engines/microsoft_academic.py+1−2

@@ -12,8 +12,7 @@ Microsoft Academic (Science)

1212from datetime import datetime

1313from json import loads

1414from uuid import uuid4

15−

16−from searx.url_utils import urlencode

15+from urllib.parse import urlencode

1716from searx.utils import html_to_text

1817

1918categories = ['images']

searx/engines/mixcloud.py+1−1

@@ -12,7 +12,7 @@

1212

1313from json import loads

1414from dateutil import parser

15−from searx.url_utils import urlencode

15+from urllib.parse import urlencode

1616

1717# engine dependent config

1818categories = ['music']

searx/engines/nyaa.py+1−1

@@ -10,8 +10,8 @@

1010"""

1111

1212from lxml import html

13+from urllib.parse import urlencode

1314from searx.engines.xpath import extract_text

14−from searx.url_utils import urlencode

1515from searx.utils import get_torrent_size, int_or_zero

1616

1717# engine dependent config

searx/engines/openstreetmap.py+4−4

@@ -30,8 +30,8 @@ route_re = re.compile('(?:from )?(.+) to (.+)')

3030# do search-request

3131def request(query, params):

3232

33− params['url'] = base_url + search_string.format(query=query.decode('utf-8'))

34− params['route'] = route_re.match(query.decode('utf-8'))

33+ params['url'] = base_url + search_string.format(query=query)

34+ params['route'] = route_re.match(query)

3535

3636 return params

3737

@@ -52,7 +52,7 @@ def response(resp):

5252 if 'display_name' not in r:

5353 continue

5454

55− title = r['display_name'] or u''

55+ title = r['display_name'] or ''

5656 osm_type = r.get('osm_type', r.get('type'))

5757 url = result_base_url.format(osm_type=osm_type,

5858 osm_id=r['osm_id'])

@@ -64,7 +64,7 @@ def response(resp):

6464

6565 # if no geojson is found and osm_type is a node, add geojson Point

6666 if not geojson and osm_type == 'node':

67− geojson = {u'type': u'Point', u'coordinates': [r['lon'], r['lat']]}

67+ geojson = {'type': 'Point', 'coordinates': [r['lon'], r['lat']]}

6868

6969 address_raw = r.get('address')

7070 address = {}

searx/engines/peertube.py+1−1

@@ -14,7 +14,7 @@

1414

1515from json import loads

1616from datetime import datetime

17−from searx.url_utils import urlencode

17+from urllib.parse import urlencode

1818from searx.utils import html_to_text

1919

2020# engine dependent config

searx/engines/photon.py+1−1

@@ -11,8 +11,8 @@

1111"""

1212

1313from json import loads

14+from urllib.parse import urlencode

1415from searx.utils import searx_useragent

15−from searx.url_utils import urlencode

1616

1717# engine dependent config

1818categories = ['map']

searx/engines/piratebay.py+5−3

@@ -11,7 +11,9 @@

1111from json import loads

1212from datetime import datetime

1313from operator import itemgetter

14−from searx.url_utils import quote

14+

15+from urllib.parse import quote, urljoin

16+from searx.engines.xpath import extract_text

1517from searx.utils import get_torrent_size

1618

1719# engine dependent config

@@ -62,8 +64,8 @@ def response(resp):

6264 # parse results

6365 for result in search_res:

6466 link = url + "description.php?id=" + result["id"]

65− magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + \

66− "&dn=" + result["name"] + "&tr=" + "&tr=".join(trackers)

67+ magnetlink = "magnet:?xt=urn:btih:" + result["info_hash"] + "&dn=" + result["name"]\

68+ + "&tr=" + "&tr=".join(trackers)

6769

6870 params = {

6971 "url": link,

searx/engines/pubmed.py+1−1

@@ -14,7 +14,7 @@

1414from flask_babel import gettext

1515from lxml import etree

1616from datetime import datetime

17−from searx.url_utils import urlencode

17+from urllib.parse import urlencode

1818from searx.poolrequests import get

1919

2020

searx/engines/qwant.py+3−3

@@ -12,9 +12,9 @@

1212

1313from datetime import datetime

1414from json import loads

15−from searx.utils import html_to_text

16−from searx.url_utils import urlencode

17−from searx.utils import match_language

15+from urllib.parse import urlencode

16+from searx.utils import html_to_text, match_language

17+

1818

1919# engine dependent config

2020categories = None

searx/engines/reddit.py+1−1

@@ -12,7 +12,7 @@

1212

1313import json

1414from datetime import datetime

15−from searx.url_utils import urlencode, urljoin, urlparse

15+from urllib.parse import urlencode, urljoin, urlparse

1616

1717# engine dependent config

1818categories = ['general', 'images', 'news', 'social media']

searx/engines/scanr_structures.py+2−2

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads, dumps

14−from searx.utils import html_to_text

14+from urllib.parse import html_to_text

1515

1616# engine dependent config

1717categories = ['science']

@@ -29,7 +29,7 @@ def request(query, params):

2929 params['url'] = search_url

3030 params['method'] = 'POST'

3131 params['headers']['Content-type'] = "application/json"

32− params['data'] = dumps({"query": query.decode('utf-8'),

32+ params['data'] = dumps({"query": query,

3333 "searchField": "ALL",

3434 "sortDirection": "ASC",

3535 "sortOrder": "RELEVANCY",

searx/engines/searchcode_code.py+1−1

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515

1616

1717# engine dependent config

searx/engines/searchcode_doc.py+1−1

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515

1616# engine dependent config

1717categories = ['it']

searx/engines/seedpeer.py+1−1

@@ -11,7 +11,7 @@

1111from lxml import html

1212from json import loads

1313from operator import itemgetter

14−from searx.url_utils import quote, urljoin

14+from urllib.parse import quote, urljoin

1515from searx.engines.xpath import extract_text

1616

1717

searx/engines/soundcloud.py+3−6

@@ -14,14 +14,11 @@ import re

1414from json import loads

1515from lxml import html

1616from dateutil import parser

17+from io import StringIO

18+from urllib.parse import quote_plus, urlencode

1719from searx import logger

1820from searx.poolrequests import get as http_get

19−from searx.url_utils import quote_plus, urlencode

2021

21−try:

22− from cStringIO import StringIO

23−except:

24− from io import StringIO

2522

2623# engine dependent config

2724categories = ['music']

@@ -61,7 +58,7 @@ def get_client_id():

6158 # gets app_js and searches for the clientid

6259 response = http_get(app_js_url)

6360 if response.ok:

64− cids = cid_re.search(response.content.decode("utf-8"))

61+ cids = cid_re.search(response.content.decode())

6562 if cids is not None and len(cids.groups()):

6663 return cids.groups()[0]

6764 logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

searx/engines/spotify.py+4−4

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from searx.url_utils import urlencode

14+from urllib.parse import urlencode

1515import requests

1616import base64

1717

@@ -39,8 +39,8 @@ def request(query, params):

3939 'https://accounts.spotify.com/api/token',

4040 data={'grant_type': 'client_credentials'},

4141 headers={'Authorization': 'Basic ' + base64.b64encode(

42− "{}:{}".format(api_client_id, api_client_secret).encode('utf-8')

43− ).decode('utf-8')}

42+ "{}:{}".format(api_client_id, api_client_secret).encode()

43+ ).decode()}

4444 )

4545 j = loads(r.text)

4646 params['headers'] = {'Authorization': 'Bearer {}'.format(j.get('access_token'))}

@@ -59,7 +59,7 @@ def response(resp):

5959 if result['type'] == 'track':

6060 title = result['name']

6161 url = result['external_urls']['spotify']

62− content = u'{} - {} - {}'.format(

62+ content = '{} - {} - {}'.format(

6363 result['artists'][0]['name'],

6464 result['album']['name'],

6565 result['name'])

searx/engines/stackoverflow.py+1−1

@@ -10,9 +10,9 @@

1010 @parse url, title, content

1111"""

1212

13+from urllib.parse import urlencode, urljoin

1314from lxml import html

1415from searx.engines.xpath import extract_text

15−from searx.url_utils import urlencode, urljoin

1616

1717# engine dependent config

1818categories = ['it']

searx/engines/tokyotoshokan.py+1−1

@@ -11,10 +11,10 @@

1111"""

1212

1313import re

14+from urllib.parse import urlencode

1415from lxml import html

1516from searx.engines.xpath import extract_text

1617from datetime import datetime

17−from searx.url_utils import urlencode

1818from searx.utils import get_torrent_size, int_or_zero

1919

2020# engine dependent config

searx/engines/torrentz.py+1−1

@@ -12,10 +12,10 @@

1212"""

1313

1414import re

15+from urllib.parse import urlencode

1516from lxml import html

1617from datetime import datetime

1718from searx.engines.xpath import extract_text

18−from searx.url_utils import urlencode

1919from searx.utils import get_torrent_size

2020

2121# engine dependent config

searx/engines/translated.py+5−5

@@ -12,11 +12,11 @@ import re

1212from searx.utils import is_valid_lang

1313

1414categories = ['general']

15−url = u'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

16−web_url = u'https://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

15+url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

16+web_url = 'https://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

1717weight = 100

1818

19−parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) (.{2,})$', re.I)

19+parser_re = re.compile('.*?([a-z]+)-([a-z]+) (.{2,})$', re.I)

2020api_key = ''

2121

2222

@@ -39,9 +39,9 @@ def request(query, params):

3939 key_form = ''

4040 params['url'] = url.format(from_lang=from_lang[1],

4141 to_lang=to_lang[1],

42− query=query.decode('utf-8'),

42+ query=query,

4343 key=key_form)

44− params['query'] = query.decode('utf-8')

44+ params['query'] = query

4545 params['from_lang'] = from_lang

4646 params['to_lang'] = to_lang

4747

searx/engines/twitter.py+1−1

@@ -12,10 +12,10 @@

1212 @todo publishedDate

1313"""

1414

15+from urllib.parse import urlencode, urljoin

1516from lxml import html

1617from datetime import datetime

1718from searx.engines.xpath import extract_text

18−from searx.url_utils import urlencode, urljoin

1919

2020# engine dependent config

2121categories = ['social media']

searx/engines/unsplash.py+1−1

@@ -10,7 +10,7 @@

1010 @parse url, title, img_src, thumbnail_src

1111"""

1212

13−from searx.url_utils import urlencode, urlparse, urlunparse, parse_qsl

13+from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl

1414from json import loads

1515

1616url = 'https://unsplash.com/'

searx/engines/vimeo.py+1−1

@@ -12,9 +12,9 @@

1212# @todo rewrite to api

1313# @todo set content-parameter with correct data

1414

15+from urllib.parse import urlencode

1516from json import loads

1617from dateutil import parser

17−from searx.url_utils import urlencode

1818

1919# engine dependent config

2020categories = ['videos']

searx/engines/wikidata.py+7−7

@@ -15,9 +15,9 @@ from searx import logger

1515from searx.poolrequests import get

1616from searx.engines.xpath import extract_text

1717from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

18−from searx.url_utils import urlencode

1918from searx.utils import match_language, eval_xpath

2019

20+from urllib.parse import urlencode

2121from json import loads

2222from lxml.html import fromstring

2323from lxml import etree

@@ -76,7 +76,7 @@ def request(query, params):

7676def response(resp):

7777 results = []

7878 htmlparser = etree.HTMLParser()

79− html = fromstring(resp.content.decode("utf-8"), parser=htmlparser)

79+ html = fromstring(resp.content.decode(), parser=htmlparser)

8080 search_results = eval_xpath(html, wikidata_ids_xpath)

8181

8282 if resp.search_params['language'].split('-')[0] == 'all':

@@ -89,7 +89,7 @@ def response(resp):

8989 wikidata_id = search_result.split('/')[-1]

9090 url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))

9191 htmlresponse = get(url)

92− jsonresponse = loads(htmlresponse.content.decode("utf-8"))

92+ jsonresponse = loads(htmlresponse.content.decode())

9393 results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser)

9494

9595 return results

@@ -453,16 +453,16 @@ def get_geolink(result):

453453 latitude, longitude = coordinates.split(',')

454454

455455 # convert to decimal

456− lat = int(latitude[:latitude.find(u'°')])

456+ lat = int(latitude[:latitude.find('°')])

457457 if latitude.find('\'') >= 0:

458− lat += int(latitude[latitude.find(u'°') + 1:latitude.find('\'')] or 0) / 60.0

458+ lat += int(latitude[latitude.find('°') + 1:latitude.find('\'')] or 0) / 60.0

459459 if latitude.find('"') >= 0:

460460 lat += float(latitude[latitude.find('\'') + 1:latitude.find('"')] or 0) / 3600.0

461461 if latitude.find('S') >= 0:

462462 lat *= -1

463− lon = int(longitude[:longitude.find(u'°')])

463+ lon = int(longitude[:longitude.find('°')])

464464 if longitude.find('\'') >= 0:

465− lon += int(longitude[longitude.find(u'°') + 1:longitude.find('\'')] or 0) / 60.0

465+ lon += int(longitude[longitude.find('°') + 1:longitude.find('\'')] or 0) / 60.0

466466 if longitude.find('"') >= 0:

467467 lon += float(longitude[longitude.find('\'') + 1:longitude.find('"')] or 0) / 3600.0

468468 if longitude.find('W') >= 0:

searx/engines/wikipedia.py+2−2

@@ -10,13 +10,13 @@

1010 @parse url, infobox

1111"""

1212

13+from urllib.parse import quote

1314from json import loads

1415from lxml.html import fromstring

15−from searx.url_utils import quote

1616from searx.utils import match_language, searx_useragent

1717

1818# search-url

19−search_url = u'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

19+search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

2020supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

2121

2222

searx/engines/wolframalpha_api.py+10−10

@@ -9,7 +9,7 @@

99# @parse url, infobox

1010

1111from lxml import etree

12−from searx.url_utils import urlencode

12+from urllib.parse import urlencode

1313

1414# search-url

1515search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'

@@ -45,15 +45,15 @@ def request(query, params):

4545

4646# replace private user area characters to make text legible

4747def replace_pua_chars(text):

48− pua_chars = {u'\uf522': u'\u2192', # rigth arrow

49− u'\uf7b1': u'\u2115', # set of natural numbers

50− u'\uf7b4': u'\u211a', # set of rational numbers

51− u'\uf7b5': u'\u211d', # set of real numbers

52− u'\uf7bd': u'\u2124', # set of integer numbers

53− u'\uf74c': 'd', # differential

54− u'\uf74d': u'\u212f', # euler's number

55− u'\uf74e': 'i', # imaginary number

56− u'\uf7d9': '='} # equals sign

48+ pua_chars = {'\uf522': '\u2192', # rigth arrow

49+ '\uf7b1': '\u2115', # set of natural numbers

50+ '\uf7b4': '\u211a', # set of rational numbers

51+ '\uf7b5': '\u211d', # set of real numbers

52+ '\uf7bd': '\u2124', # set of integer numbers

53+ '\uf74c': 'd', # differential

54+ '\uf74d': '\u212f', # euler's number

55+ '\uf74e': 'i', # imaginary number

56+ '\uf7d9': '='} # equals sign

5757

5858 for k, v in pua_chars.items():

5959 text = text.replace(k, v)

searx/engines/wolframalpha_noapi.py+1−1

@@ -10,9 +10,9 @@

1010

1111from json import loads

1212from time import time

13+from urllib.parse import urlencode

1314

1415from searx.poolrequests import get as http_get

15−from searx.url_utils import urlencode

1616

1717# search-url

1818url = 'https://www.wolframalpha.com/'

searx/engines/www1x.py+1−1

@@ -11,7 +11,7 @@

1111"""

1212

1313from lxml import html

14−from searx.url_utils import urlencode, urljoin

14+from urllib.parse import urlencode, urljoin

1515from searx.engines.xpath import extract_text

1616

1717# engine dependent config

searx/engines/xpath.py+3−3

@@ -1,7 +1,7 @@

1+from urllib.parse import unquote, urlencode, urljoin, urlparse

12from lxml import html

23from lxml.etree import _ElementStringResult, _ElementUnicodeResult

34from searx.utils import html_to_text, eval_xpath

4−from searx.url_utils import unquote, urlencode, urljoin, urlparse

55

66search_url = None

77url_xpath = None

@@ -56,7 +56,7 @@ def extract_url(xpath_results, search_url):

5656 if url.startswith('//'):

5757 # add http or https to this kind of url //example.com/

5858 parsed_search_url = urlparse(search_url)

59− url = u'{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

59+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

6060 elif url.startswith('/'):

6161 # fix relative url to the search engine

6262 url = urljoin(search_url, url)

@@ -86,7 +86,7 @@ def normalize_url(url):

8686 p = parsed_url.path

8787 mark = p.find('/**')

8888 if mark != -1:

89− return unquote(p[mark + 3:]).decode('utf-8')

89+ return unquote(p[mark + 3:]).decode()

9090

9191 return url

9292

searx/engines/yacy.py+1−1

@@ -14,7 +14,7 @@

1414

1515from json import loads

1616from dateutil import parser

17−from searx.url_utils import urlencode

17+from urllib.parse import urlencode

1818

1919from searx.utils import html_to_text

2020

searx/engines/yahoo.py+1−1

@@ -11,9 +11,9 @@

1111 @parse url, title, content, suggestion

1212"""

1313

14+from urllib.parse import unquote, urlencode

1415from lxml import html

1516from searx.engines.xpath import extract_text, extract_url

16−from searx.url_utils import unquote, urlencode

1717from searx.utils import match_language, eval_xpath

1818

1919# engine dependent config

searx/engines/yahoo_news.py+2−2

@@ -11,13 +11,13 @@

1111

1212import re

1313from datetime import datetime, timedelta

14+from urllib.parse import urlencode

1415from lxml import html

1516from searx.engines.xpath import extract_text, extract_url

1617from searx.engines.yahoo import (

1718 parse_url, _fetch_supported_languages, supported_languages_url, language_aliases

1819)

1920from dateutil import parser

20−from searx.url_utils import urlencode

2121from searx.utils import match_language

2222

2323# engine dependent config

@@ -58,7 +58,7 @@ def request(query, params):

5858

5959def sanitize_url(url):

6060 if ".yahoo.com/" in url:

61− return re.sub(u"\\;\\_ylt\\=.+$", "", url)

61+ return re.sub("\\;\\_ylt\\=.+$", "", url)

6262 else:

6363 return url

6464

searx/engines/yandex.py+1−1

@@ -9,9 +9,9 @@

99 @parse url, title, content

1010"""

1111

12+from urllib.parse import urlencode

1213from lxml import html

1314from searx import logger

14−from searx.url_utils import urlencode

1515

1616logger = logger.getChild('yandex engine')

1717

searx/engines/yggtorrent.py+1−1

@@ -11,8 +11,8 @@

1111from lxml import html

1212from operator import itemgetter

1313from datetime import datetime

14+from urllib.parse import quote

1415from searx.engines.xpath import extract_text

15−from searx.url_utils import quote

1616from searx.utils import get_torrent_size

1717from searx.poolrequests import get as http_get

1818

searx/engines/youtube_api.py+1−1

@@ -10,7 +10,7 @@

1010

1111from json import loads

1212from dateutil import parser

13−from searx.url_utils import urlencode

13+from urllib.parse import urlencode

1414

1515# engine dependent config

1616categories = ['videos', 'music']

searx/engines/youtube_noapi.py+1−1

@@ -10,9 +10,9 @@

1010

1111from functools import reduce

1212from json import loads

13+from urllib.parse import quote_plus

1314from searx.engines.xpath import extract_text

1415from searx.utils import list_get

15−from searx.url_utils import quote_plus

1616

1717# engine dependent config

1818categories = ['videos', 'music']

searx/exceptions.py+1−1

@@ -27,7 +27,7 @@ class SearxParameterException(SearxException):

2727 message = 'Empty ' + name + ' parameter'

2828 else:

2929 message = 'Invalid value "' + value + '" for parameter ' + name

30− super(SearxParameterException, self).__init__(message)

30+ super().__init__(message)

3131 self.message = message

3232 self.parameter_name = name

3333 self.parameter_value = value

searx/external_bang.py+1−1

@@ -23,7 +23,7 @@ def get_bang_url(search_query):

2323 """

2424

2525 if search_query.external_bang:

26− query = search_query.query.decode('utf-8', 'ignore')

26+ query = search_query.query

2727 bang = _get_bang(search_query.external_bang)

2828

2929 if bang and query:

searx/languages.py+69−69

@@ -3,73 +3,73 @@

33# this file is generated automatically by utils/update_search_languages.py

44

55language_codes = (

6− (u"af-NA", u"Afrikaans", u"", u"Afrikaans"),

7− (u"ar-SA", u"العربية", u"", u"Arabic"),

8− (u"be-BY", u"Беларуская", u"", u"Belarusian"),

9− (u"bg-BG", u"Български", u"", u"Bulgarian"),

10− (u"ca-AD", u"Català", u"", u"Catalan"),

11− (u"cs-CZ", u"Čeština", u"", u"Czech"),

12− (u"da-DK", u"Dansk", u"", u"Danish"),

13− (u"de", u"Deutsch", u"", u"German"),

14− (u"de-AT", u"Deutsch", u"Österreich", u"German"),

15− (u"de-CH", u"Deutsch", u"Schweiz", u"German"),

16− (u"de-DE", u"Deutsch", u"Deutschland", u"German"),

17− (u"el-GR", u"Ελληνικά", u"", u"Greek"),

18− (u"en", u"English", u"", u"English"),

19− (u"en-AU", u"English", u"Australia", u"English"),

20− (u"en-CA", u"English", u"Canada", u"English"),

21− (u"en-GB", u"English", u"United Kingdom", u"English"),

22− (u"en-IE", u"English", u"Ireland", u"English"),

23− (u"en-IN", u"English", u"India", u"English"),

24− (u"en-NZ", u"English", u"New Zealand", u"English"),

25− (u"en-PH", u"English", u"Philippines", u"English"),

26− (u"en-SG", u"English", u"Singapore", u"English"),

27− (u"en-US", u"English", u"United States", u"English"),

28− (u"es", u"Español", u"", u"Spanish"),

29− (u"es-AR", u"Español", u"Argentina", u"Spanish"),

30− (u"es-CL", u"Español", u"Chile", u"Spanish"),

31− (u"es-ES", u"Español", u"España", u"Spanish"),

32− (u"es-MX", u"Español", u"México", u"Spanish"),

33− (u"et-EE", u"Eesti", u"", u"Estonian"),

34− (u"fa-IR", u"فارسی", u"", u"Persian"),

35− (u"fi-FI", u"Suomi", u"", u"Finnish"),

36− (u"fr", u"Français", u"", u"French"),

37− (u"fr-BE", u"Français", u"Belgique", u"French"),

38− (u"fr-CA", u"Français", u"Canada", u"French"),

39− (u"fr-CH", u"Français", u"Suisse", u"French"),

40− (u"fr-FR", u"Français", u"France", u"French"),

41− (u"he-IL", u"עברית", u"", u"Hebrew"),

42− (u"hr-HR", u"Hrvatski", u"", u"Croatian"),

43− (u"hu-HU", u"Magyar", u"", u"Hungarian"),

44− (u"hy-AM", u"Հայերեն", u"", u"Armenian"),

45− (u"id-ID", u"Indonesia", u"", u"Indonesian"),

46− (u"is-IS", u"Íslenska", u"", u"Icelandic"),

47− (u"it-IT", u"Italiano", u"", u"Italian"),

48− (u"ja-JP", u"日本語", u"", u"Japanese"),

49− (u"ko-KR", u"한국어", u"", u"Korean"),

50− (u"lt-LT", u"Lietuvių", u"", u"Lithuanian"),

51− (u"lv-LV", u"Latviešu", u"", u"Latvian"),

52− (u"ms-MY", u"Melayu", u"", u"Malay"),

53− (u"nb-NO", u"Norsk Bokmål", u"", u"Norwegian Bokmål"),

54− (u"nl", u"Nederlands", u"", u"Dutch"),

55− (u"nl-BE", u"Nederlands", u"België", u"Dutch"),

56− (u"nl-NL", u"Nederlands", u"Nederland", u"Dutch"),

57− (u"pl-PL", u"Polski", u"", u"Polish"),

58− (u"pt", u"Português", u"", u"Portuguese"),

59− (u"pt-BR", u"Português", u"Brasil", u"Portuguese"),

60− (u"pt-PT", u"Português", u"Portugal", u"Portuguese"),

61− (u"ro-RO", u"Română", u"", u"Romanian"),

62− (u"ru-RU", u"Русский", u"", u"Russian"),

63− (u"sk-SK", u"Slovenčina", u"", u"Slovak"),

64− (u"sl-SI", u"Slovenščina", u"", u"Slovenian"),

65− (u"sr-RS", u"Srpski", u"", u"Serbian"),

66− (u"sv-SE", u"Svenska", u"", u"Swedish"),

67− (u"sw-KE", u"Kiswahili", u"", u"Swahili"),

68− (u"th-TH", u"ไทย", u"", u"Thai"),

69− (u"tr-TR", u"Türkçe", u"", u"Turkish"),

70− (u"uk-UA", u"Українська", u"", u"Ukrainian"),

71− (u"vi-VN", u"Tiếng Việt", u"", u"Vietnamese"),

72− (u"zh", u"中文", u"", u"Chinese"),

73− (u"zh-CN", u"中文", u"中国", u"Chinese"),

74− (u"zh-TW", u"中文", u"台灣", u"Chinese")

6+ ("af-NA", "Afrikaans", "", "Afrikaans"),

7+ ("ar-SA", "العربية", "", "Arabic"),

8+ ("be-BY", "Беларуская", "", "Belarusian"),

9+ ("bg-BG", "Български", "", "Bulgarian"),

10+ ("ca-AD", "Català", "", "Catalan"),

11+ ("cs-CZ", "Čeština", "", "Czech"),

12+ ("da-DK", "Dansk", "", "Danish"),

13+ ("de", "Deutsch", "", "German"),

14+ ("de-AT", "Deutsch", "Österreich", "German"),

15+ ("de-CH", "Deutsch", "Schweiz", "German"),

16+ ("de-DE", "Deutsch", "Deutschland", "German"),

17+ ("el-GR", "Ελληνικά", "", "Greek"),

18+ ("en", "English", "", "English"),

19+ ("en-AU", "English", "Australia", "English"),

20+ ("en-CA", "English", "Canada", "English"),

21+ ("en-GB", "English", "United Kingdom", "English"),

22+ ("en-IE", "English", "Ireland", "English"),

23+ ("en-IN", "English", "India", "English"),

24+ ("en-NZ", "English", "New Zealand", "English"),

25+ ("en-PH", "English", "Philippines", "English"),

26+ ("en-SG", "English", "Singapore", "English"),

27+ ("en-US", "English", "United States", "English"),

28+ ("es", "Español", "", "Spanish"),

29+ ("es-AR", "Español", "Argentina", "Spanish"),

30+ ("es-CL", "Español", "Chile", "Spanish"),

31+ ("es-ES", "Español", "España", "Spanish"),

32+ ("es-MX", "Español", "México", "Spanish"),

33+ ("et-EE", "Eesti", "", "Estonian"),

34+ ("fa-IR", "فارسی", "", "Persian"),

35+ ("fi-FI", "Suomi", "", "Finnish"),

36+ ("fr", "Français", "", "French"),

37+ ("fr-BE", "Français", "Belgique", "French"),

38+ ("fr-CA", "Français", "Canada", "French"),

39+ ("fr-CH", "Français", "Suisse", "French"),

40+ ("fr-FR", "Français", "France", "French"),

41+ ("he-IL", "עברית", "", "Hebrew"),

42+ ("hr-HR", "Hrvatski", "", "Croatian"),

43+ ("hu-HU", "Magyar", "", "Hungarian"),

44+ ("hy-AM", "Հայերեն", "", "Armenian"),

45+ ("id-ID", "Indonesia", "", "Indonesian"),

46+ ("is-IS", "Íslenska", "", "Icelandic"),

47+ ("it-IT", "Italiano", "", "Italian"),

48+ ("ja-JP", "日本語", "", "Japanese"),

49+ ("ko-KR", "한국어", "", "Korean"),

50+ ("lt-LT", "Lietuvių", "", "Lithuanian"),

51+ ("lv-LV", "Latviešu", "", "Latvian"),

52+ ("ms-MY", "Melayu", "", "Malay"),

53+ ("nb-NO", "Norsk Bokmål", "", "Norwegian Bokmål"),

54+ ("nl", "Nederlands", "", "Dutch"),

55+ ("nl-BE", "Nederlands", "België", "Dutch"),

56+ ("nl-NL", "Nederlands", "Nederland", "Dutch"),

57+ ("pl-PL", "Polski", "", "Polish"),

58+ ("pt", "Português", "", "Portuguese"),

59+ ("pt-BR", "Português", "Brasil", "Portuguese"),

60+ ("pt-PT", "Português", "Portugal", "Portuguese"),

61+ ("ro-RO", "Română", "", "Romanian"),

62+ ("ru-RU", "Русский", "", "Russian"),

63+ ("sk-SK", "Slovenčina", "", "Slovak"),

64+ ("sl-SI", "Slovenščina", "", "Slovenian"),

65+ ("sr-RS", "Srpski", "", "Serbian"),

66+ ("sv-SE", "Svenska", "", "Swedish"),

67+ ("sw-KE", "Kiswahili", "", "Swahili"),

68+ ("th-TH", "ไทย", "", "Thai"),

69+ ("tr-TR", "Türkçe", "", "Turkish"),

70+ ("uk-UA", "Українська", "", "Ukrainian"),

71+ ("vi-VN", "Tiếng Việt", "", "Vietnamese"),

72+ ("zh", "中文", "", "Chinese"),

73+ ("zh-CN", "中文", "中国", "Chinese"),

74+ ("zh-TW", "中文", "台灣", "Chinese")

7575)

searx/plugins/__init__.py+2−5

@@ -20,13 +20,10 @@ from importlib import import_module

2020from os import listdir, makedirs, remove, stat, utime

2121from os.path import abspath, basename, dirname, exists, join

2222from shutil import copyfile

23−from sys import version_info

2423from traceback import print_exc

2524

2625from searx import logger, settings, static_path

2726

28−if version_info[0] == 3:

29− unicode = str

3027

3128logger = logger.getChild('plugins')

3229

@@ -38,8 +35,8 @@ from searx.plugins import (oa_doi_rewrite,

3835 tracker_url_remover,

3936 vim_hotkeys)

4037

41−required_attrs = (('name', (str, unicode)),

42− ('description', (str, unicode)),

38+required_attrs = (('name', str),

39+ ('description', str),

4340 ('default_on', bool))

4441

4542optional_attrs = (('js_dependencies', tuple),

searx/plugins/https_rewrite.py+1−4

@@ -16,17 +16,14 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1616'''

1717

1818import re

19−import sys

19+from urllib.parse import urlparse

2020from lxml import etree

2121from os import listdir, environ

2222from os.path import isfile, isdir, join

2323from searx.plugins import logger

2424from flask_babel import gettext

2525from searx import searx_dir

26−from searx.url_utils import urlparse

2726

28−if sys.version_info[0] == 3:

29− unicode = str

3027

3128name = "HTTPS rewrite"

3229description = gettext('Rewrite HTTP links to HTTPS if possible')

searx/plugins/oa_doi_rewrite.py+1−1

@@ -1,6 +1,6 @@

1+from urllib.parse import urlparse, parse_qsl

12from flask_babel import gettext

23import re

3−from searx.url_utils import urlparse, parse_qsl

44from searx import settings

55

66

searx/plugins/self_info.py+2−2

@@ -22,7 +22,7 @@ default_on = True

2222

2323

2424# Self User Agent regex

25−p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)

25+p = re.compile('.*user[ -]agent.*', re.IGNORECASE)

2626

2727

2828# attach callback to the post search hook

@@ -31,7 +31,7 @@ p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)

3131def post_search(request, search):

3232 if search.search_query.pageno > 1:

3333 return True

34− if search.search_query.query == b'ip':

34+ if search.search_query.query == 'ip':

3535 x_forwarded_for = request.headers.getlist("X-Forwarded-For")

3636 if x_forwarded_for:

3737 ip = x_forwarded_for[0]

searx/plugins/tracker_url_remover.py+1−1

@@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1717

1818from flask_babel import gettext

1919import re

20−from searx.url_utils import urlunparse, parse_qsl, urlencode

20+from urllib.parse import urlunparse, parse_qsl, urlencode

2121

2222regexes = {re.compile(r'utm_[^&]+'),

2323 re.compile(r'(wkey|wemail)[^&]*'),

searx/poolrequests.py+3−3

@@ -20,7 +20,7 @@ class HTTPAdapterWithConnParams(requests.adapters.HTTPAdapter):

2020 self.config = {}

2121 self.proxy_manager = {}

2222

23− super(requests.adapters.HTTPAdapter, self).__init__()

23+ super().__init__()

2424

2525 self._pool_connections = pool_connections

2626 self._pool_maxsize = pool_maxsize

@@ -60,7 +60,7 @@ else:

6060class SessionSinglePool(requests.Session):

6161

6262 def __init__(self):

63− super(SessionSinglePool, self).__init__()

63+ super().__init__()

6464

6565 # reuse the same adapters

6666 with RLock():

@@ -71,7 +71,7 @@ class SessionSinglePool(requests.Session):

7171 def close(self):

7272 """Call super, but clear adapters since there are managed globaly"""

7373 self.adapters.clear()

74− super(SessionSinglePool, self).close()

74+ super().close()

7575

7676

7777def set_timeout_for_thread(timeout, start_time=None):

searx/preferences.py+6−11

@@ -6,16 +6,11 @@

66

77from base64 import urlsafe_b64encode, urlsafe_b64decode

88from zlib import compress, decompress

9−from sys import version

9+from urllib.parse import parse_qs, urlencode

1010

1111from searx import settings, autocomplete

1212from searx.languages import language_codes as languages

1313from searx.utils import match_language

14−from searx.url_utils import parse_qs, urlencode

15−

16−if version[0] == '3':

17− # pylint: disable=invalid-name

18− unicode = str

1914

2015

2116COOKIE_MAX_AGE = 60 * 60 * 24 * 365 * 5 # 5 years

@@ -37,7 +32,7 @@ class ValidationException(Exception):

3732 """

3833

3934

40−class Setting(object):

35+class Setting:

4136 """Base class of user settings"""

4237

4338 def __init__(self, default_value, **kwargs):

@@ -315,7 +310,7 @@ class PluginsSetting(SwitchableSetting):

315310 return [item[len('plugin_'):] for item in items]

316311

317312

318−class Preferences(object):

313+class Preferences:

319314 """Validates and saves preferences to cookies"""

320315

321316 def __init__(self, themes, categories, engines, plugins):

@@ -402,14 +397,14 @@ class Preferences(object):

402397

403398 settings_kv['tokens'] = ','.join(self.tokens.values)

404399

405− return urlsafe_b64encode(compress(urlencode(settings_kv).encode('utf-8'))).decode('utf-8')

400+ return urlsafe_b64encode(compress(urlencode(settings_kv).encode())).decode()

406401

407402 def parse_encoded_data(self, input_data):

408403 """parse (base64) preferences from request (``flask.request.form['preferences']``)"""

409− decoded_data = decompress(urlsafe_b64decode(input_data.encode('utf-8')))

404+ decoded_data = decompress(urlsafe_b64decode(input_data.encode()))

410405 dict_data = {}

411406 for x, y in parse_qs(decoded_data).items():

412− dict_data[x.decode('utf8')] = y[0].decode('utf8')

407+ dict_data[x.decode()] = y[0].decode()

413408 self.parse_dict(dict_data)

414409

415410 def parse_dict(self, input_data):

searx/query.py+10−11

@@ -17,23 +17,22 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1717(C) 2014 by Thomas Pointhuber, <thomas.pointhuber@gmx.at>

1818'''

1919

20+import re

21+

2022from searx.languages import language_codes

2123from searx.engines import (

2224 categories, engines, engine_shortcuts

2325)

24−import re

25−import sys

2626

27−if sys.version_info[0] == 3:

28− unicode = str

2927

3028VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')

3129

3230

33−class RawTextQuery(object):

31+class RawTextQuery:

3432 """parse raw text query (the value from the html input)"""

3533

3634 def __init__(self, query, disabled_engines):

35+ assert isinstance(query, str)

3736 self.query = query

3837 self.disabled_engines = []

3938

@@ -53,7 +52,7 @@ class RawTextQuery(object):

5352 self.query_parts = []

5453

5554 # split query, including whitespaces

56− raw_query_parts = re.split(r'(\s+)' if isinstance(self.query, str) else b'(\s+)', self.query)

55+ raw_query_parts = re.split(r'(\s+)', self.query)

5756

5857 parse_next = True

5958

@@ -93,7 +92,7 @@ class RawTextQuery(object):

9392 # check if any language-code is equal with

9493 # declared language-codes

9594 for lc in language_codes:

96− lang_id, lang_name, country, english_name = map(unicode.lower, lc)

95+ lang_id, lang_name, country, english_name = map(str.lower, lc)

9796

9897 # if correct language-code is found

9998 # set it as new search-language

@@ -177,15 +176,15 @@ class RawTextQuery(object):

177176

178177 def getFullQuery(self):

179178 # get full querry including whitespaces

180− return u''.join(self.query_parts)

179+ return ''.join(self.query_parts)

181180

182181

183−class SearchQuery(object):

182+class SearchQuery:

184183 """container for all the search parameters (query, language, etc...)"""

185184

186185 def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range,

187186 timeout_limit=None, preferences=None, external_bang=None):

188− self.query = query.encode('utf-8')

187+ self.query = query

189188 self.engines = engines

190189 self.categories = categories

191190 self.lang = lang

@@ -197,4 +196,4 @@ class SearchQuery(object):

197196 self.external_bang = external_bang

198197

199198 def __str__(self):

200− return str(self.query) + ";" + str(self.engines)

199+ return self.query + ";" + str(self.engines)

searx/results.py+7−10

@@ -1,14 +1,11 @@

11import re

2−import sys

32from collections import defaultdict

43from operator import itemgetter

54from threading import RLock

5+from urllib.parse import urlparse, unquote

66from searx import logger

77from searx.engines import engines

8−from searx.url_utils import urlparse, unquote

98

10−if sys.version_info[0] == 3:

11− basestring = str

129

1310CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

1411WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

@@ -16,7 +13,7 @@ WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

1613

1714# return the meaningful length of the content for a result

1815def result_content_len(content):

19− if isinstance(content, basestring):

16+ if isinstance(content, str):

2017 return len(CONTENT_LEN_IGNORED_CHARS_REGEX.sub('', content))

2118 else:

2219 return 0

@@ -125,14 +122,14 @@ def result_score(result):

125122 return sum((occurences * weight) / position for position in result['positions'])

126123

127124

128−class ResultContainer(object):

125+class ResultContainer:

129126 """docstring for ResultContainer"""

130127

131128 __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\

132129 '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url'

133130

134131 def __init__(self):

135− super(ResultContainer, self).__init__()

132+ super().__init__()

136133 self._merged_results = []

137134 self.infoboxes = []

138135 self.suggestions = set()

@@ -161,11 +158,11 @@ class ResultContainer(object):

161158 self._number_of_results.append(result['number_of_results'])

162159 else:

163160 # standard result (url, title, content)

164− if 'url' in result and not isinstance(result['url'], basestring):

161+ if 'url' in result and not isinstance(result['url'], str):

165162 logger.debug('result: invalid URL: %s', str(result))

166− elif 'title' in result and not isinstance(result['title'], basestring):

163+ elif 'title' in result and not isinstance(result['title'], str):

167164 logger.debug('result: invalid title: %s', str(result))

168− elif 'content' in result and not isinstance(result['content'], basestring):

165+ elif 'content' in result and not isinstance(result['content'], str):

169166 logger.debug('result: invalid content: %s', str(result))

170167 else:

171168 self._merge_result(result, standard_result_count + 1)

searx/search.py+8−15

@@ -20,8 +20,8 @@ import sys

2020import threading

2121from time import time

2222from uuid import uuid4

23+from _thread import start_new_thread

2324

24−import six

2525from flask_babel import gettext

2626import requests.exceptions

2727import searx.poolrequests as requests_lib

@@ -37,13 +37,6 @@ from searx import logger

3737from searx.plugins import plugins

3838from searx.exceptions import SearxParameterException

3939

40−try:

41− from thread import start_new_thread

42−except:

43− from _thread import start_new_thread

44−

45−if sys.version_info[0] == 3:

46− unicode = str

4740

4841logger = logger.getChild('search')

4942

@@ -355,11 +348,11 @@ def get_search_query_from_webapp(preferences, form):

355348 load_default_categories = True

356349 for pd_name, pd in form.items():

357350 if pd_name == 'categories':

358− query_categories.extend(categ for categ in map(unicode.strip, pd.split(',')) if categ in categories)

351+ query_categories.extend(categ for categ in map(str.strip, pd.split(',')) if categ in categories)

359352 elif pd_name == 'engines':

360353 pd_engines = [{'category': engines[engine].categories[0],

361354 'name': engine}

362− for engine in map(unicode.strip, pd.split(',')) if engine in engines]

355+ for engine in map(str.strip, pd.split(',')) if engine in engines]

363356 if pd_engines:

364357 query_engines.extend(pd_engines)

365358 load_default_categories = False

@@ -414,12 +407,12 @@ def get_search_query_from_webapp(preferences, form):

414407 raw_text_query)

415408

416409

417−class Search(object):

410+class Search:

418411 """Search information container"""

419412

420413 def __init__(self, search_query):

421414 # init vars

422− super(Search, self).__init__()

415+ super().__init__()

423416 self.search_query = search_query

424417 self.result_container = ResultContainer()

425418 self.actual_timeout = None

@@ -434,7 +427,7 @@ class Search(object):

434427

435428 # This means there was a valid bang and the

436429 # rest of the search does not need to be continued

437− if isinstance(self.result_container.redirect_url, six.string_types):

430+ if isinstance(self.result_container.redirect_url, str):

438431 return self.result_container

439432 # start time

440433 start_time = time()

@@ -541,13 +534,13 @@ class SearchWithPlugins(Search):

541534 """Similar to the Search class but call the plugins."""

542535

543536 def __init__(self, search_query, ordered_plugin_list, request):

544− super(SearchWithPlugins, self).__init__(search_query)

537+ super().__init__(search_query)

545538 self.ordered_plugin_list = ordered_plugin_list

546539 self.request = request

547540

548541 def search(self):

549542 if plugins.call(self.ordered_plugin_list, 'pre_search', self.request, self):

550− super(SearchWithPlugins, self).search()

543+ super().search()

551544

552545 plugins.call(self.ordered_plugin_list, 'post_search', self.request, self)

553546

searx/templates/courgette/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/legacy/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/oscar/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="text-center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/simple/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/testing.py+2−2

@@ -17,7 +17,7 @@ from unittest2 import TestCase

1717class SearxTestLayer:

1818 """Base layer for non-robot tests."""

1919

20− __name__ = u'SearxTestLayer'

20+ __name__ = 'SearxTestLayer'

2121

2222 @classmethod

2323 def setUp(cls):

@@ -66,7 +66,7 @@ class SearxRobotLayer():

6666 stderr=subprocess.STDOUT

6767 )

6868 if hasattr(self.server.stdout, 'read1'):

69− print(self.server.stdout.read1(1024).decode('utf-8'))

69+ print(self.server.stdout.read1(1024).decode())

7070

7171 def tearDown(self):

7272 os.kill(self.server.pid, 9)

searx/url_utils.pydeleted−30

@@ -1,30 +0,0 @@

1−from sys import version_info

2−

3−if version_info[0] == 2:

4− from urllib import quote, quote_plus, unquote, urlencode

5− from urlparse import parse_qs, parse_qsl, urljoin, urlparse, urlunparse, ParseResult

6−else:

7− from urllib.parse import (

8− parse_qs,

9− parse_qsl,

10− quote,

11− quote_plus,

12− unquote,

13− urlencode,

14− urljoin,

15− urlparse,

16− urlunparse,

17− ParseResult

18− )

19−

20−

21−__export__ = (parse_qs,

22− parse_qsl,

23− quote,

24− quote_plus,

25− unquote,

26− urlencode,

27− urljoin,

28− urlparse,

29− urlunparse,

30− ParseResult)

searx/utils.py+26−52

@@ -1,21 +1,22 @@

11# -*- coding: utf-8 -*-

2+import os

3+import sys

24import csv

35import hashlib

46import hmac

5−import os

67import re

8+import json

79

8−from babel.core import get_global

9−from babel.dates import format_date

1010from codecs import getincrementalencoder

1111from imp import load_source

1212from numbers import Number

1313from os.path import splitext, join

14−from io import open

14+from io import open, StringIO

1515from random import choice

16+from html.parser import HTMLParser

1617from lxml.etree import XPath

17−import sys

18−import json

18+from babel.core import get_global

19+from babel.dates import format_date

1920

2021from searx import settings

2122from searx.version import VERSION_STRING

@@ -23,23 +24,6 @@ from searx.languages import language_codes

2324from searx import settings

2425from searx import logger

2526

26−try:

27− from cStringIO import StringIO

28−except:

29− from io import StringIO

30−

31−try:

32− from HTMLParser import HTMLParser

33−except:

34− from html.parser import HTMLParser

35−

36−if sys.version_info[0] == 3:

37− unichr = chr

38− unicode = str

39− IS_PY2 = False

40− basestring = str

41−else:

42− IS_PY2 = True

4327

4428logger = logger.getChild('utils')

4529

@@ -75,19 +59,18 @@ def highlight_content(content, query):

7559 if content.find('<') != -1:

7660 return content

7761

78− query = query.decode('utf-8')

7962 if content.lower().find(query.lower()) > -1:

80− query_regex = u'({0})'.format(re.escape(query))

63+ query_regex = '({0})'.format(re.escape(query))

8164 content = re.sub(query_regex, '<span class="highlight">\\1</span>',

8265 content, flags=re.I | re.U)

8366 else:

8467 regex_parts = []

8568 for chunk in query.split():

8669 if len(chunk) == 1:

87− regex_parts.append(u'\\W+{0}\\W+'.format(re.escape(chunk)))

70+ regex_parts.append('\\W+{0}\\W+'.format(re.escape(chunk)))

8871 else:

89− regex_parts.append(u'{0}'.format(re.escape(chunk)))

90− query_regex = u'({0})'.format('|'.join(regex_parts))

72+ regex_parts.append('{0}'.format(re.escape(chunk)))

73+ query_regex = '({0})'.format('|'.join(regex_parts))

9174 content = re.sub(query_regex, '<span class="highlight">\\1</span>',

9275 content, flags=re.I | re.U)

9376

@@ -124,21 +107,21 @@ class HTMLTextExtractor(HTMLParser):

124107 def handle_charref(self, number):

125108 if not self.is_valid_tag():

126109 return

127− if number[0] in (u'x', u'X'):

110+ if number[0] in ('x', 'X'):

128111 codepoint = int(number[1:], 16)

129112 else:

130113 codepoint = int(number)

131− self.result.append(unichr(codepoint))

114+ self.result.append(chr(codepoint))

132115

133116 def handle_entityref(self, name):

134117 if not self.is_valid_tag():

135118 return

136119 # codepoint = htmlentitydefs.name2codepoint[name]

137− # self.result.append(unichr(codepoint))

120+ # self.result.append(chr(codepoint))

138121 self.result.append(name)

139122

140123 def get_text(self):

141− return u''.join(self.result).strip()

124+ return ''.join(self.result).strip()

142125

143126

144127def html_to_text(html):

@@ -163,22 +146,14 @@ class UnicodeWriter:

163146 self.encoder = getincrementalencoder(encoding)()

164147

165148 def writerow(self, row):

166− if IS_PY2:

167− row = [s.encode("utf-8") if hasattr(s, 'encode') else s for s in row]

168149 self.writer.writerow(row)

169150 # Fetch UTF-8 output from the queue ...

170151 data = self.queue.getvalue()

171− if IS_PY2:

172− data = data.decode("utf-8")

173− else:

174− data = data.strip('\x00')

152+ data = data.strip('\x00')

175153 # ... and reencode it into the target encoding

176154 data = self.encoder.encode(data)

177155 # write to the target stream

178− if IS_PY2:

179− self.stream.write(data)

180− else:

181− self.stream.write(data.decode("utf-8"))

156+ self.stream.write(data.decode())

182157 # empty queue

183158 self.queue.truncate(0)

184159

@@ -253,7 +228,7 @@ def dict_subset(d, properties):

253228def prettify_url(url, max_length=74):

254229 if len(url) > max_length:

255230 chunk_len = int(max_length / 2 + 1)

256− return u'{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])

231+ return '{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])

257232 else:

258233 return url

259234

@@ -309,8 +284,10 @@ def int_or_zero(num):

309284

310285

311286def is_valid_lang(lang):

287+ if isinstance(lang, bytes):

288+ lang = lang.decode()

312289 is_abbr = (len(lang) == 2)

313− lang = lang.lower().decode('utf-8')

290+ lang = lang.lower()

314291 if is_abbr:

315292 for l in language_codes:

316293 if l[0][:2] == lang:

@@ -407,17 +384,14 @@ def new_hmac(secret_key, url):

407384 secret_key_bytes = secret_key

408385 else:

409386 raise err

410− if sys.version_info[0] == 2:

411− return hmac.new(bytes(secret_key), url, hashlib.sha256).hexdigest()

412− else:

413− return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest()

387+ return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest()

414388

415389

416390def to_string(obj):

417− if isinstance(obj, basestring):

391+ if isinstance(obj, str):

418392 return obj

419393 if isinstance(obj, Number):

420− return unicode(obj)

394+ return str(obj)

421395 if hasattr(obj, '__str__'):

422396 return obj.__str__()

423397 if hasattr(obj, '__repr__'):

@@ -433,9 +407,9 @@ def ecma_unescape(s):

433407 """

434408 # s = unicode(s)

435409 # "%u5409" becomes "吉"

436− s = ecma_unescape4_re.sub(lambda e: unichr(int(e.group(1), 16)), s)

410+ s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)

437411 # "%20" becomes " ", "%F3" becomes "ó"

438− s = ecma_unescape2_re.sub(lambda e: unichr(int(e.group(1), 16)), s)

412+ s = ecma_unescape2_re.sub(lambda e: chr(int(e.group(1), 16)), s)

439413 return s

440414

441415

searx/webapp.py+26−47

@@ -17,37 +17,35 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1717(C) 2013- by Adam Tauber, <asciimoo@gmail.com>

1818'''

1919

20+import sys

21+if sys.version_info[0] < 3:

22+ print('\033[1;31m Python2 is no longer supported\033[0m')

23+ exit(1)

24+

2025if __name__ == '__main__':

21− from sys import path

2226 from os.path import realpath, dirname

23− path.append(realpath(dirname(realpath(__file__)) + '/../'))

27+ sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

2428

2529import hashlib

2630import hmac

2731import json

2832import os

29−import sys

3033

3134import requests

3235

3336from searx import logger

3437logger = logger.getChild('webapp')

3538

36−try:

37− from pygments import highlight

38− from pygments.lexers import get_lexer_by_name

39− from pygments.formatters import HtmlFormatter

40−except:

41− logger.critical("cannot import dependency: pygments")

42− from sys import exit

43− exit(1)

44−try:

45− from cgi import escape

46−except:

47− from html import escape

48−from six import next

4939from datetime import datetime, timedelta

5040from time import time

41+from html import escape

42+from io import StringIO

43+from urllib.parse import urlencode, urlparse, urljoin

44+

45+from pygments import highlight

46+from pygments.lexers import get_lexer_by_name

47+from pygments.formatters import HtmlFormatter

48+

5149from werkzeug.middleware.proxy_fix import ProxyFix

5250from flask import (

5351 Flask, request, render_template, url_for, Response, make_response,

@@ -78,7 +76,6 @@ from searx.plugins import plugins

7876from searx.plugins.oa_doi_rewrite import get_doi_resolver

7977from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES

8078from searx.answerers import answerers

81−from searx.url_utils import urlencode, urlparse, urljoin

8279from searx.utils import new_hmac

8380

8481# check if the pyopenssl package is installed.

@@ -89,19 +86,6 @@ except ImportError:

8986 logger.critical("The pyopenssl package has to be installed.\n"

9087 "Some HTTPS connections will fail")

9188

92−try:

93− from cStringIO import StringIO

94−except:

95− from io import StringIO

96−

97−

98−if sys.version_info[0] == 3:

99− unicode = str

100− PY3 = True

101−else:

102− logger.warning('\033[1;31m Python2 is no longer supported\033[0m')

103− exit(1)

104−

10589# serve pages with HTTP/1.1

10690from werkzeug.serving import WSGIRequestHandler

10791WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))

@@ -315,11 +299,11 @@ def proxify(url):

315299 if not settings.get('result_proxy'):

316300 return url

317301

318− url_params = dict(mortyurl=url.encode('utf-8'))

302+ url_params = dict(mortyurl=url.encode())

319303

320304 if settings['result_proxy'].get('key'):

321305 url_params['mortyhash'] = hmac.new(settings['result_proxy']['key'],

322− url.encode('utf-8'),

306+ url.encode(),

323307 hashlib.sha256).hexdigest()

324308

325309 return '{0}?{1}'.format(settings['result_proxy']['url'],

@@ -347,10 +331,10 @@ def image_proxify(url):

347331 if settings.get('result_proxy'):

348332 return proxify(url)

349333

350− h = new_hmac(settings['server']['secret_key'], url.encode('utf-8'))

334+ h = new_hmac(settings['server']['secret_key'], url.encode())

351335

352336 return '{0}?{1}'.format(url_for('image_proxy'),

353− urlencode(dict(url=url.encode('utf-8'), h=h)))

337+ urlencode(dict(url=url.encode(), h=h)))

354338

355339

356340def render(template_name, override_theme=None, **kwargs):

@@ -424,8 +408,6 @@ def render(template_name, override_theme=None, **kwargs):

424408

425409 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

426410

427− kwargs['unicode'] = unicode

428−

429411 kwargs['preferences'] = request.preferences

430412

431413 kwargs['brand'] = brand

@@ -612,7 +594,7 @@ def index():

612594 if 'content' in result and result['content']:

613595 result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)

614596 if 'title' in result and result['title']:

615− result['title'] = highlight_content(escape(result['title'] or u''), search_query.query)

597+ result['title'] = highlight_content(escape(result['title'] or ''), search_query.query)

616598 else:

617599 if result.get('content'):

618600 result['content'] = html_to_text(result['content']).strip()

@@ -634,14 +616,14 @@ def index():

634616 minutes = int((timedifference.seconds / 60) % 60)

635617 hours = int(timedifference.seconds / 60 / 60)

636618 if hours == 0:

637− result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)

619+ result['publishedDate'] = gettext('{minutes} minute(s) ago').format(minutes=minutes)

638620 else:

639− result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa

621+ result['publishedDate'] = gettext('{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa

640622 else:

641623 result['publishedDate'] = format_date(result['publishedDate'])

642624

643625 if output_format == 'json':

644− return Response(json.dumps({'query': search_query.query.decode('utf-8'),

626+ return Response(json.dumps({'query': search_query.query,

645627 'number_of_results': number_of_results,

646628 'results': results,

647629 'answers': list(result_container.answers),

@@ -670,7 +652,7 @@ def index():

670652 csv.writerow([row.get(key, '') for key in keys])

671653 csv.stream.seek(0)

672654 response = Response(csv.stream.read(), mimetype='application/csv')

673− cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query.decode('utf-8'))

655+ cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)

674656 response.headers.add('Content-Disposition', cont_disp)

675657 return response

676658

@@ -754,10 +736,7 @@ def autocompleter():

754736 disabled_engines = request.preferences.engines.get_disabled()

755737

756738 # parse query

757− if PY3:

758− raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines)

759− else:

760− raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)

739+ raw_text_query = RawTextQuery(str(request.form.get('q', b'')), disabled_engines)

761740 raw_text_query.parse_query()

762741

763742 # check if search query is set

@@ -879,7 +858,7 @@ def _is_selected_language_supported(engine, preferences):

879858

880859@app.route('/image_proxy', methods=['GET'])

881860def image_proxy():

882− url = request.args.get('url').encode('utf-8')

861+ url = request.args.get('url').encode()

883862

884863 if not url:

885864 return '', 400

@@ -1061,7 +1040,7 @@ def run():

10611040 )

10621041

10631042

1064−class ReverseProxyPathFix(object):

1043+class ReverseProxyPathFix:

10651044 '''Wrap the application in this middleware and configure the

10661045 front-end server to add these headers, to let you quietly bind

10671046 this to a URL other than / and to an HTTP scheme that is

tests/unit/test_answerers.py+2−2

@@ -10,7 +10,7 @@ class AnswererTest(SearxTestCase):

1010

1111 def test_unicode_input(self):

1212 query = Mock()

13− unicode_payload = u'árvíztűrő tükörfúrógép'

13+ unicode_payload = 'árvíztűrő tükörfúrógép'

1414 for answerer in answerers:

15− query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)

15+ query.query = '{} {}'.format(answerer.keywords[0], unicode_payload)

1616 self.assertTrue(isinstance(answerer.answer(query), list))

tests/unit/test_plugins.py+8−8

@@ -48,11 +48,11 @@ class SelfIPTest(SearxTestCase):

4848 # IP test

4949 request = Mock(remote_addr='127.0.0.1')

5050 request.headers.getlist.return_value = []

51− search = get_search_mock(query=b'ip', pageno=1)

51+ search = get_search_mock(query='ip', pageno=1)

5252 store.call(store.plugins, 'post_search', request, search)

5353 self.assertTrue('127.0.0.1' in search.result_container.answers["ip"]["answer"])

5454

55− search = get_search_mock(query=b'ip', pageno=2)

55+ search = get_search_mock(query='ip', pageno=2)

5656 store.call(store.plugins, 'post_search', request, search)

5757 self.assertFalse('ip' in search.result_container.answers)

5858

@@ -60,26 +60,26 @@ class SelfIPTest(SearxTestCase):

6060 request = Mock(user_agent='Mock')

6161 request.headers.getlist.return_value = []

6262

63− search = get_search_mock(query=b'user-agent', pageno=1)

63+ search = get_search_mock(query='user-agent', pageno=1)

6464 store.call(store.plugins, 'post_search', request, search)

6565 self.assertTrue('Mock' in search.result_container.answers["user-agent"]["answer"])

6666

67− search = get_search_mock(query=b'user-agent', pageno=2)

67+ search = get_search_mock(query='user-agent', pageno=2)

6868 store.call(store.plugins, 'post_search', request, search)

6969 self.assertFalse('user-agent' in search.result_container.answers)

7070

71− search = get_search_mock(query=b'user-agent', pageno=1)

71+ search = get_search_mock(query='user-agent', pageno=1)

7272 store.call(store.plugins, 'post_search', request, search)

7373 self.assertTrue('Mock' in search.result_container.answers["user-agent"]["answer"])

7474

75− search = get_search_mock(query=b'user-agent', pageno=2)

75+ search = get_search_mock(query='user-agent', pageno=2)

7676 store.call(store.plugins, 'post_search', request, search)

7777 self.assertFalse('user-agent' in search.result_container.answers)

7878

79− search = get_search_mock(query=b'What is my User-Agent?', pageno=1)

79+ search = get_search_mock(query='What is my User-Agent?', pageno=1)

8080 store.call(store.plugins, 'post_search', request, search)

8181 self.assertTrue('Mock' in search.result_container.answers["user-agent"]["answer"])

8282

83− search = get_search_mock(query=b'What is my User-Agent?', pageno=2)

83+ search = get_search_mock(query='What is my User-Agent?', pageno=2)

8484 store.call(store.plugins, 'post_search', request, search)

8585 self.assertFalse('user-agent' in search.result_container.answers)

tests/unit/test_preferences.py+16−16

@@ -3,7 +3,7 @@ from searx.preferences import (EnumStringSetting, MapSetting, MissingArgumentExc

33from searx.testing import SearxTestCase

44

55

6−class PluginStub(object):

6+class PluginStub:

77

88 def __init__(self, id, default_on):

99 self.id = id

@@ -28,13 +28,13 @@ class TestSettings(SearxTestCase):

2828

2929 def test_map_setting_valid_default(self):

3030 setting = MapSetting(3, map={'dog': 1, 'bat': 2, 'cat': 3})

31− self.assertEquals(setting.get_value(), 3)

31+ self.assertEqual(setting.get_value(), 3)

3232

3333 def test_map_setting_valid_choice(self):

3434 setting = MapSetting(3, map={'dog': 1, 'bat': 2, 'cat': 3})

35− self.assertEquals(setting.get_value(), 3)

35+ self.assertEqual(setting.get_value(), 3)

3636 setting.parse('bat')

37− self.assertEquals(setting.get_value(), 2)

37+ self.assertEqual(setting.get_value(), 2)

3838

3939 def test_enum_setting_invalid_initialization(self):

4040 with self.assertRaises(MissingArgumentException):

@@ -56,13 +56,13 @@ class TestSettings(SearxTestCase):

5656

5757 def test_enum_setting_valid_default(self):

5858 setting = EnumStringSetting(3, choices=[1, 2, 3])

59− self.assertEquals(setting.get_value(), 3)

59+ self.assertEqual(setting.get_value(), 3)

6060

6161 def test_enum_setting_valid_choice(self):

6262 setting = EnumStringSetting(3, choices=[1, 2, 3])

63− self.assertEquals(setting.get_value(), 3)

63+ self.assertEqual(setting.get_value(), 3)

6464 setting.parse(2)

65− self.assertEquals(setting.get_value(), 2)

65+ self.assertEqual(setting.get_value(), 2)

6666

6767 # multiple choice settings

6868 def test_multiple_setting_invalid_initialization(self):

@@ -80,48 +80,48 @@ class TestSettings(SearxTestCase):

8080

8181 def test_multiple_setting_valid_default(self):

8282 setting = MultipleChoiceSetting(['3'], choices=['1', '2', '3'])

83− self.assertEquals(setting.get_value(), ['3'])

83+ self.assertEqual(setting.get_value(), ['3'])

8484

8585 def test_multiple_setting_valid_choice(self):

8686 setting = MultipleChoiceSetting(['3'], choices=['1', '2', '3'])

87− self.assertEquals(setting.get_value(), ['3'])

87+ self.assertEqual(setting.get_value(), ['3'])

8888 setting.parse('2')

89− self.assertEquals(setting.get_value(), ['2'])

89+ self.assertEqual(setting.get_value(), ['2'])

9090

9191 # search language settings

9292 def test_lang_setting_valid_choice(self):

9393 setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

9494 setting.parse('de')

95− self.assertEquals(setting.get_value(), 'de')

95+ self.assertEqual(setting.get_value(), 'de')

9696

9797 def test_lang_setting_invalid_choice(self):

9898 setting = SearchLanguageSetting('all', choices=['all', 'de', 'en'])

9999 setting.parse('xx')

100− self.assertEquals(setting.get_value(), 'all')

100+ self.assertEqual(setting.get_value(), 'all')

101101

102102 def test_lang_setting_old_cookie_choice(self):

103103 setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

104104 setting.parse('es_XA')

105− self.assertEquals(setting.get_value(), 'es')

105+ self.assertEqual(setting.get_value(), 'es')

106106

107107 def test_lang_setting_old_cookie_format(self):

108108 setting = SearchLanguageSetting('all', choices=['all', 'es', 'es-ES'])

109109 setting.parse('es_ES')

110− self.assertEquals(setting.get_value(), 'es-ES')

110+ self.assertEqual(setting.get_value(), 'es-ES')

111111

112112 # plugins settings

113113 def test_plugins_setting_all_default_enabled(self):

114114 plugin1 = PluginStub('plugin1', True)

115115 plugin2 = PluginStub('plugin2', True)

116116 setting = PluginsSetting(['3'], choices=[plugin1, plugin2])

117− self.assertEquals(setting.get_enabled(), set(['plugin1', 'plugin2']))

117+ self.assertEqual(setting.get_enabled(), set(['plugin1', 'plugin2']))

118118

119119 def test_plugins_setting_few_default_enabled(self):

120120 plugin1 = PluginStub('plugin1', True)

121121 plugin2 = PluginStub('plugin2', False)

122122 plugin3 = PluginStub('plugin3', True)

123123 setting = PluginsSetting('name', choices=[plugin1, plugin2, plugin3])

124− self.assertEquals(setting.get_enabled(), set(['plugin1', 'plugin3']))

124+ self.assertEqual(setting.get_enabled(), set(['plugin1', 'plugin3']))

125125

126126

127127class TestPreferences(SearxTestCase):

tests/unit/test_query.py+26−26

@@ -9,9 +9,9 @@ class TestQuery(SearxTestCase):

99 query = RawTextQuery(query_text, [])

1010 query.parse_query()

1111

12− self.assertEquals(query.getFullQuery(), query_text)

13− self.assertEquals(len(query.query_parts), 1)

14− self.assertEquals(len(query.languages), 0)

12+ self.assertEqual(query.getFullQuery(), query_text)

13+ self.assertEqual(len(query.query_parts), 1)

14+ self.assertEqual(len(query.languages), 0)

1515 self.assertFalse(query.specific)

1616

1717 def test_language_code(self):

@@ -21,9 +21,9 @@ class TestQuery(SearxTestCase):

2121 query = RawTextQuery(full_query, [])

2222 query.parse_query()

2323

24− self.assertEquals(query.getFullQuery(), full_query)

25− self.assertEquals(len(query.query_parts), 3)

26− self.assertEquals(len(query.languages), 1)

24+ self.assertEqual(query.getFullQuery(), full_query)

25+ self.assertEqual(len(query.query_parts), 3)

26+ self.assertEqual(len(query.languages), 1)

2727 self.assertIn(language, query.languages)

2828 self.assertFalse(query.specific)

2929

@@ -34,8 +34,8 @@ class TestQuery(SearxTestCase):

3434 query = RawTextQuery(full_query, [])

3535 query.parse_query()

3636

37− self.assertEquals(query.getFullQuery(), full_query)

38− self.assertEquals(len(query.query_parts), 3)

37+ self.assertEqual(query.getFullQuery(), full_query)

38+ self.assertEqual(len(query.query_parts), 3)

3939 self.assertIn('en', query.languages)

4040 self.assertFalse(query.specific)

4141

@@ -46,8 +46,8 @@ class TestQuery(SearxTestCase):

4646 query = RawTextQuery(full_query, [])

4747 query.parse_query()

4848

49− self.assertEquals(query.getFullQuery(), full_query)

50− self.assertEquals(len(query.query_parts), 3)

49+ self.assertEqual(query.getFullQuery(), full_query)

50+ self.assertEqual(len(query.query_parts), 3)

5151 self.assertIn('all', query.languages)

5252 self.assertFalse(query.specific)

5353

@@ -58,9 +58,9 @@ class TestQuery(SearxTestCase):

5858 query = RawTextQuery(full_query, [])

5959 query.parse_query()

6060

61− self.assertEquals(query.getFullQuery(), full_query)

62− self.assertEquals(len(query.query_parts), 1)

63− self.assertEquals(len(query.languages), 0)

61+ self.assertEqual(query.getFullQuery(), full_query)

62+ self.assertEqual(len(query.query_parts), 1)

63+ self.assertEqual(len(query.languages), 0)

6464 self.assertFalse(query.specific)

6565

6666 def test_timeout_below100(self):

@@ -68,9 +68,9 @@ class TestQuery(SearxTestCase):

6868 query = RawTextQuery(query_text, [])

6969 query.parse_query()

7070

71− self.assertEquals(query.getFullQuery(), query_text)

72− self.assertEquals(len(query.query_parts), 3)

73− self.assertEquals(query.timeout_limit, 3)

71+ self.assertEqual(query.getFullQuery(), query_text)

72+ self.assertEqual(len(query.query_parts), 3)

73+ self.assertEqual(query.timeout_limit, 3)

7474 self.assertFalse(query.specific)

7575

7676 def test_timeout_above100(self):

@@ -78,9 +78,9 @@ class TestQuery(SearxTestCase):

7878 query = RawTextQuery(query_text, [])

7979 query.parse_query()

8080

81− self.assertEquals(query.getFullQuery(), query_text)

82− self.assertEquals(len(query.query_parts), 3)

83− self.assertEquals(query.timeout_limit, 0.35)

81+ self.assertEqual(query.getFullQuery(), query_text)

82+ self.assertEqual(len(query.query_parts), 3)

83+ self.assertEqual(query.timeout_limit, 0.35)

8484 self.assertFalse(query.specific)

8585

8686 def test_timeout_above1000(self):

@@ -88,9 +88,9 @@ class TestQuery(SearxTestCase):

8888 query = RawTextQuery(query_text, [])

8989 query.parse_query()

9090

91− self.assertEquals(query.getFullQuery(), query_text)

92− self.assertEquals(len(query.query_parts), 3)

93− self.assertEquals(query.timeout_limit, 3.5)

91+ self.assertEqual(query.getFullQuery(), query_text)

92+ self.assertEqual(len(query.query_parts), 3)

93+ self.assertEqual(query.timeout_limit, 3.5)

9494 self.assertFalse(query.specific)

9595

9696 def test_timeout_invalid(self):

@@ -99,8 +99,8 @@ class TestQuery(SearxTestCase):

9999 query = RawTextQuery(query_text, [])

100100 query.parse_query()

101101

102− self.assertEquals(query.getFullQuery(), query_text)

103− self.assertEquals(len(query.query_parts), 1)

104− self.assertEquals(query.query_parts[0], query_text)

105− self.assertEquals(query.timeout_limit, None)

102+ self.assertEqual(query.getFullQuery(), query_text)

103+ self.assertEqual(len(query.query_parts), 1)

104+ self.assertEqual(query.query_parts[0], query_text)

105+ self.assertEqual(query.timeout_limit, None)

106106 self.assertFalse(query.specific)

tests/unit/test_search.py+8−8

@@ -45,7 +45,7 @@ class SearchTestCase(SearxTestCase):

4545 preferences=Preferences(['oscar'], ['general'], engines, []))

4646 search = searx.search.Search(search_query)

4747 search.search()

48− self.assertEquals(search.actual_timeout, 3.0)

48+ self.assertEqual(search.actual_timeout, 3.0)

4949

5050 def test_timeout_query_above_default_nomax(self):

5151 searx.search.max_request_timeout = None

@@ -54,7 +54,7 @@ class SearchTestCase(SearxTestCase):

5454 preferences=Preferences(['oscar'], ['general'], engines, []))

5555 search = searx.search.Search(search_query)

5656 search.search()

57− self.assertEquals(search.actual_timeout, 3.0)

57+ self.assertEqual(search.actual_timeout, 3.0)

5858

5959 def test_timeout_query_below_default_nomax(self):

6060 searx.search.max_request_timeout = None

@@ -63,7 +63,7 @@ class SearchTestCase(SearxTestCase):

6363 preferences=Preferences(['oscar'], ['general'], engines, []))

6464 search = searx.search.Search(search_query)

6565 search.search()

66− self.assertEquals(search.actual_timeout, 1.0)

66+ self.assertEqual(search.actual_timeout, 1.0)

6767

6868 def test_timeout_query_below_max(self):

6969 searx.search.max_request_timeout = 10.0

@@ -72,7 +72,7 @@ class SearchTestCase(SearxTestCase):

7272 preferences=Preferences(['oscar'], ['general'], engines, []))

7373 search = searx.search.Search(search_query)

7474 search.search()

75− self.assertEquals(search.actual_timeout, 5.0)

75+ self.assertEqual(search.actual_timeout, 5.0)

7676

7777 def test_timeout_query_above_max(self):

7878 searx.search.max_request_timeout = 10.0

@@ -81,7 +81,7 @@ class SearchTestCase(SearxTestCase):

8181 preferences=Preferences(['oscar'], ['general'], engines, []))

8282 search = searx.search.Search(search_query)

8383 search.search()

84− self.assertEquals(search.actual_timeout, 10.0)

84+ self.assertEqual(search.actual_timeout, 10.0)

8585

8686 def test_query_private_engine_without_token(self):

8787 search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': PRIVATE_ENGINE_NAME}],

@@ -89,7 +89,7 @@ class SearchTestCase(SearxTestCase):

8989 preferences=Preferences(['oscar'], ['general'], engines, []))

9090 search = searx.search.Search(search_query)

9191 results = search.search()

92− self.assertEquals(results.results_length(), 0)

92+ self.assertEqual(results.results_length(), 0)

9393

9494 def test_query_private_engine_with_incorrect_token(self):

9595 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

@@ -99,7 +99,7 @@ class SearchTestCase(SearxTestCase):

9999 preferences=preferences_with_tokens)

100100 search = searx.search.Search(search_query)

101101 results = search.search()

102− self.assertEquals(results.results_length(), 0)

102+ self.assertEqual(results.results_length(), 0)

103103

104104 def test_query_private_engine_with_correct_token(self):

105105 preferences_with_tokens = Preferences(['oscar'], ['general'], engines, [])

@@ -109,7 +109,7 @@ class SearchTestCase(SearxTestCase):

109109 preferences=preferences_with_tokens)

110110 search = searx.search.Search(search_query)

111111 results = search.search()

112− self.assertEquals(results.results_length(), 1)

112+ self.assertEqual(results.results_length(), 1)

113113

114114 def test_external_bang(self):

115115 search_query = searx.query.SearchQuery('yes yes',

tests/unit/test_utils.py+8−12

@@ -1,12 +1,8 @@

11# -*- coding: utf-8 -*-

22import mock

3−import sys

43from searx.testing import SearxTestCase

54from searx import utils

65

7−if sys.version_info[0] == 3:

8− unicode = str

9−

106

117class TestUtils(SearxTestCase):

128

@@ -34,9 +30,9 @@ class TestUtils(SearxTestCase):

3430 self.assertEqual(utils.highlight_content(content, None), content)

3531

3632 content = 'a'

37− query = b'test'

33+ query = 'test'

3834 self.assertEqual(utils.highlight_content(content, query), content)

39− query = b'a test'

35+ query = 'a test'

4036 self.assertEqual(utils.highlight_content(content, query), content)

4137

4238 def test_html_to_text(self):

@@ -52,15 +48,15 @@ class TestUtils(SearxTestCase):

5248 </span>

5349 </a>

5450 """

55− self.assertIsInstance(utils.html_to_text(html), unicode)

51+ self.assertIsInstance(utils.html_to_text(html), str)

5652 self.assertIsNotNone(utils.html_to_text(html))

5753 self.assertEqual(utils.html_to_text(html), "Test text")

5854

5955 def test_prettify_url(self):

6056 data = (('https://searx.me/', 'https://searx.me/'),

61− (u'https://searx.me/ű', u'https://searx.me/ű'),

57+ ('https://searx.me/ű', 'https://searx.me/ű'),

6258 ('https://searx.me/' + (100 * 'a'), 'https://searx.me/[...]aaaaaaaaaaaaaaaaa'),

63− (u'https://searx.me/' + (100 * u'ű'), u'https://searx.me/[...]űűűűűűűűűűűűűűűűű'))

59+ ('https://searx.me/' + (100 * 'ű'), 'https://searx.me/[...]űűűűűűűűűűűűűűűűű'))

6460

6561 for test_url, expected in data:

6662 self.assertEqual(utils.prettify_url(test_url, max_length=32), expected)

@@ -108,12 +104,12 @@ class TestHTMLTextExtractor(SearxTestCase):

108104

109105 def test_handle_charref(self):

110106 self.html_text_extractor.handle_charref('xF')

111− self.assertIn(u'\x0f', self.html_text_extractor.result)

107+ self.assertIn('\x0f', self.html_text_extractor.result)

112108 self.html_text_extractor.handle_charref('XF')

113− self.assertIn(u'\x0f', self.html_text_extractor.result)

109+ self.assertIn('\x0f', self.html_text_extractor.result)

114110

115111 self.html_text_extractor.handle_charref('97')

116− self.assertIn(u'a', self.html_text_extractor.result)

112+ self.assertIn('a', self.html_text_extractor.result)

117113

118114 def test_handle_entityref(self):

119115 entity = 'test'

tests/unit/test_webapp.py+2−2

@@ -1,11 +1,11 @@

11# -*- coding: utf-8 -*-

22

33import json

4+from urllib.parse import ParseResult

45from mock import Mock

56from searx import webapp

67from searx.testing import SearxTestCase

78from searx.search import Search

8−from searx.url_utils import ParseResult

99

1010

1111class ViewsTestCase(SearxTestCase):

@@ -89,7 +89,7 @@ class ViewsTestCase(SearxTestCase):

8989 def test_index_json(self):

9090 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

9191

92− result_dict = json.loads(result.data.decode('utf-8'))

92+ result_dict = json.loads(result.data.decode())

9393

9494 self.assertEqual('test', result_dict['query'])

9595 self.assertEqual(len(result_dict['results']), 2)

utils/fabfile.py+1−1

@@ -1,5 +1,5 @@

11from fabric.api import cd, run, sudo, put

2−from cStringIO import StringIO

2+from io import StringIO

33

44

55base_dir = '/usr/local'

utils/fetch_currencies.py+10−10

@@ -1,11 +1,11 @@

11# -*- coding: utf-8 -*-

2−from __future__ import print_function

2+

33

44import json

55import re

66import unicodedata

77import string

8−from urllib import urlencode

8+from urllib.parse import urlencode

99from requests import get

1010

1111languages = {'de', 'en', 'es', 'fr', 'hu', 'it', 'nl', 'jp'}

@@ -39,7 +39,7 @@ def add_currency_name(name, iso4217):

3939

4040 db_names = db['names']

4141

42− if not isinstance(iso4217, basestring):

42+ if not isinstance(iso4217, str):

4343 print("problem", name, iso4217)

4444 return

4545

@@ -126,7 +126,7 @@ def wdq_query(query):

126126 url = url_wmflabs_template + query

127127 htmlresponse = get(url)

128128 jsonresponse = json.loads(htmlresponse.content)

129− qlist = map(add_q, jsonresponse.get('items', {}))

129+ qlist = list(map(add_q, jsonresponse.get('items', {})))

130130 error = jsonresponse.get('status', {}).get('error', None)

131131 if error is not None and error != 'OK':

132132 print("error for query '" + query + "' :" + error)

@@ -150,12 +150,12 @@ for q in wmflabs_queries:

150150 wdq_query(q)

151151

152152# static

153−add_currency_name(u"euro", 'EUR')

154−add_currency_name(u"euros", 'EUR')

155−add_currency_name(u"dollar", 'USD')

156−add_currency_name(u"dollars", 'USD')

157−add_currency_name(u"peso", 'MXN')

158−add_currency_name(u"pesos", 'MXN')

153+add_currency_name("euro", 'EUR')

154+add_currency_name("euros", 'EUR')

155+add_currency_name("dollar", 'USD')

156+add_currency_name("dollars", 'USD')

157+add_currency_name("peso", 'MXN')

158+add_currency_name("pesos", 'MXN')

159159

160160# write

161161f = open("currencies.json", "wb")

utils/fetch_firefox_version.py+1−1

@@ -9,9 +9,9 @@ path.append(realpath(dirname(realpath(__file__)) + '/../'))

99import json

1010import requests

1111import re

12+from urllib.parse import urlparse, urljoin

1213from distutils.version import LooseVersion, StrictVersion

1314from lxml import html

14−from searx.url_utils import urlparse, urljoin

1515from searx import searx_dir

1616

1717URL = 'https://ftp.mozilla.org/pub/firefox/releases/'

utils/fetch_languages.py+8−5

@@ -174,14 +174,17 @@ def write_languages_file(languages):

174174 + '# this file is generated automatically by utils/update_search_languages.py\n'\

175175 + '\nlanguage_codes = ('

176176 for code in sorted(languages):

177− file_content += '\n (u"' + code + '"'\

178− + ', u"' + languages[code]['name'].split(' (')[0] + '"'\

179− + ', u"' + languages[code].get('country', '') + '"'\

180− + ', u"' + languages[code].get('english_name', '').split(' (')[0] + '"),'

177+ if 'name' in languages[code]:

178+ file_content += '\n ("' + code + '"'\

179+ + ', "' + languages[code]['name'].split(' (')[0] + '"'\

180+ + ', "' + languages[code].get('country', '') + '"'\

181+ + ', "' + languages[code].get('english_name', '').split(' (')[0] + '"),'

182+ else:

183+ print('ignore ',languages[code])

181184 # remove last comma

182185 file_content = file_content[:-1]

183186 file_content += '\n)\n'

184− new_file.write(file_content.encode('utf8'))

187+ new_file.write(file_content.encode())

185188 new_file.close()

186189

187190

utils/makefile.python+5−5

@@ -69,11 +69,11 @@ python-help::

6969 @echo ' py[un]install - [un]install python objects in editable mode'

7070 @echo ' upload-pypi - upload $(PYDIST)/* files to PyPi'

7171 @echo 'options:'

72− @echo ' make PY=2 [targets] => to eval targets with python 2 ($(PY))'

73− @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))'

74− @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)'

75− @echo ' make DEBUG= => target "debug": do not invoke PDB on errors'

76− @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]'

72+ @echo ' make PY=3.7 [targets] => to eval targets with python 3.7 ($(PY))'

73+ @echo ' make PIP_INST= => to set/unset pip install options ($(PIP_INST))'

74+ @echo ' make TEST=. => choose test from $(TEST_FOLDER) (default "." runs all)'

75+ @echo ' make DEBUG= => target "debug": do not invoke PDB on errors'

76+ @echo ' make PY_SETUP_EXTRAS => also install extras_require from setup.py \[develop,test\]'

7777 @echo ' when using target "pydebug", set breakpoints within py-source by adding::'

7878 @echo ' DEBUG()'

7979

utils/standalone_searx.py+1−2

@@ -56,7 +56,7 @@ args = parser.parse_args()

5656# search results for the query

5757form = {

5858 "q":args.query,

59− "categories":args.category.decode('utf-8'),

59+ "categories":args.category.decode(),

6060 "pageno":str(args.pageno),

6161 "language":args.lang,

6262 "time_range":args.timerange

@@ -101,4 +101,3 @@ result_container_json = {

101101}

102102sys.stdout = codecs.getwriter("UTF-8")(sys.stdout)

103103sys.stdout.write(dumps(result_container_json, sort_keys=True, indent=4, ensure_ascii=False, encoding="utf-8", default=json_serial))

104−