52e615d · searx · git.iko.soy

.travis.yml+3−2

compare with master

unifiedsplit

@@ -9,6 +9,7 @@ addons:

9language: python

10python:

11 - "2.7"

12+ - "3.6"

12before_install:

13before_install:

13 - "export DISPLAY=:99.0"

14 - "export DISPLAY=:99.0"

14 - "sh -e /etc/init.d/xvfb start"

15 - "sh -e /etc/init.d/xvfb start"

@@ -24,9 +25,9 @@ script:

24 - ./manage.sh styles

25 - ./manage.sh styles

25 - ./manage.sh grunt_build

26 - ./manage.sh grunt_build

26 - ./manage.sh tests

27 - ./manage.sh tests

27− - ./manage.sh py_test_coverage

28after_success:

29+ - ./manage.sh py_test_coverage

29− coveralls

30+ - coveralls

30notifications:

31notifications:

31 irc:

32 irc:

32 channels:

33 channels:

requirements-dev.txt+1−2

@@ -3,8 +3,7 @@ mock==2.0.0

3nose2[coverage-plugin]

4pep8==1.7.0

5plone.testing==5.0.0

6−robotframework-selenium2library==1.8.0

6+splinter==0.7.5

7−robotsuite==1.7.0

8transifex-client==0.12.2

7transifex-client==0.12.2

9unittest2==1.1.0

8unittest2==1.1.0

10zope.testrunner==4.5.1

9zope.testrunner==4.5.1

searx/answerers/__init__.py+8−4

@@ -1,8 +1,12 @@

1from os import listdir

2from os.path import realpath, dirname, join, isdir

3+from sys import version_info

3from searx.utils import load_module

4from searx.utils import load_module

4from collections import defaultdict

5from collections import defaultdict

5

6

7+if version_info[0] == 3:

8+ unicode = str

9+

6

10

7answerers_dir = dirname(realpath(__file__))

11answerers_dir = dirname(realpath(__file__))

8

12

@@ -10,7 +14,7 @@ answerers_dir = dirname(realpath(__file__))

10def load_answerers():

14def load_answerers():

11 answerers = []

15 answerers = []

12 for filename in listdir(answerers_dir):

16 for filename in listdir(answerers_dir):

13− if not isdir(join(answerers_dir, filename)):

17+ if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):

14 continue

18 continue

15 module = load_module('answerer.py', join(answerers_dir, filename))

19 module = load_module('answerer.py', join(answerers_dir, filename))

16 if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):

20 if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):

@@ -30,12 +34,12 @@ def get_answerers_by_keywords(answerers):

30

34

31def ask(query):

35def ask(query):

32 results = []

36 results = []

33− query_parts = filter(None, query.query.split())

37+ query_parts = list(filter(None, query.query.split()))

34

38

35− if query_parts[0] not in answerers_by_keywords:

39+ if query_parts[0].decode('utf-8') not in answerers_by_keywords:

36 return results

40 return results

37

41

38− for answerer in answerers_by_keywords[query_parts[0]]:

42+ for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:

39 result = answerer(query)

43 result = answerer(query)

40 if result:

44 if result:

41 results.append(result)

45 results.append(result)

searx/answerers/random/answerer.py+9−4

@@ -1,5 +1,6 @@

1import random

2import string

3+import sys

3from flask_babel import gettext

4from flask_babel import gettext

4

5

5# required answerer attribute

6# required answerer attribute

@@ -8,7 +9,11 @@ keywords = ('random',)

8

9

9random_int_max = 2**31

10random_int_max = 2**31

10

11

12+if sys.version_info[0] == 2:

11−random_string_letters = string.lowercase + string.digits + string.uppercase

13+ random_string_letters = string.lowercase + string.digits + string.uppercase

14+else:

15+ unicode = str

16+ random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

12

17

13

18

14def random_string():

19def random_string():

@@ -24,9 +29,9 @@ def random_int():

24 return unicode(random.randint(-random_int_max, random_int_max))

29 return unicode(random.randint(-random_int_max, random_int_max))

25

30

26

31

27−random_types = {u'string': random_string,

32+random_types = {b'string': random_string,

28− u'int': random_int,

33+ b'int': random_int,

29− u'float': random_float}

34+ b'float': random_float}

30

35

31

36

32# required answerer function

37# required answerer function

searx/answerers/statistics/answerer.py+10−6

@@ -1,8 +1,12 @@

1+from sys import version_info

1from functools import reduce

2from functools import reduce

2from operator import mul

3from operator import mul

3

4

4from flask_babel import gettext

5from flask_babel import gettext

5

6

7+if version_info[0] == 3:

8+ unicode = str

9+

6keywords = ('min',

10keywords = ('min',

7 'max',

11 'max',

8 'avg',

12 'avg',

@@ -19,22 +23,22 @@ def answer(query):

19 return []

23 return []

20

24

21 try:

25 try:

22− args = map(float, parts[1:])

26+ args = list(map(float, parts[1:]))

23 except:

27 except:

24 return []

28 return []

25

29

26 func = parts[0]

30 func = parts[0]

27 answer = None

31 answer = None

28

32

29− if func == 'min':

33+ if func == b'min':

30 answer = min(args)

34 answer = min(args)

31− elif func == 'max':

35+ elif func == b'max':

32 answer = max(args)

36 answer = max(args)

33− elif func == 'avg':

37+ elif func == b'avg':

34 answer = sum(args) / len(args)

38 answer = sum(args) / len(args)

35− elif func == 'sum':

39+ elif func == b'sum':

36 answer = sum(args)

40 answer = sum(args)

37− elif func == 'prod':

41+ elif func == b'prod':

38 answer = reduce(mul, args, 1)

42 answer = reduce(mul, args, 1)

39

43

40 if answer is None:

44 if answer is None:

searx/autocomplete.py+5−1

@@ -18,7 +18,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

18

19from lxml import etree

20from json import loads

21−from urllib import urlencode

22from searx import settings

21from searx import settings

23from searx.languages import language_codes

22from searx.languages import language_codes

24from searx.engines import (

23from searx.engines import (

@@ -26,6 +25,11 @@ from searx.engines import (

26)

25)

27from searx.poolrequests import get as http_get

26from searx.poolrequests import get as http_get

28

27

28+try:

29+ from urllib import urlencode

30+except:

31+ from urllib.parse import urlencode

32+

29

33

30def get(*args, **kwargs):

34def get(*args, **kwargs):

31 if 'timeout' not in kwargs:

35 if 'timeout' not in kwargs:

searx/engines/1337x.py+1−2

@@ -1,8 +1,7 @@

1−from urllib import quote

2from lxml import html

1from lxml import html

3from searx.engines.xpath import extract_text

2from searx.engines.xpath import extract_text

4from searx.utils import get_torrent_size

3from searx.utils import get_torrent_size

5−from urlparse import urljoin

4+from searx.url_utils import quote, urljoin

6

5

7url = 'https://1337x.to/'

6url = 'https://1337x.to/'

8search_url = url + 'search/{search_term}/{pageno}/'

7search_url = url + 'search/{search_term}/{pageno}/'

searx/engines/__init__.py+2−3

@@ -72,12 +72,11 @@ def load_engine(engine_data):

72 if engine_data['categories'] == 'none':

73 engine.categories = []

74 else:

75− engine.categories = map(

76− str.strip, engine_data['categories'].split(','))

75+ engine.categories = list(map(str.strip, engine_data['categories'].split(',')))

77 continue

76 continue

78 setattr(engine, param_name, engine_data[param_name])

77 setattr(engine, param_name, engine_data[param_name])

79

78

80− for arg_name, arg_value in engine_default_args.iteritems():

79+ for arg_name, arg_value in engine_default_args.items():

81 if not hasattr(engine, arg_name):

80 if not hasattr(engine, arg_name):

82 setattr(engine, arg_name, arg_value)

81 setattr(engine, arg_name, arg_value)

83

82

searx/engines/archlinux.py+1−2

@@ -11,10 +11,9 @@

11 @parse url, title

12"""

13

14−from urlparse import urljoin

15−from urllib import urlencode

16from lxml import html

14from lxml import html

17from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

16+from searx.url_utils import urlencode, urljoin

18

17

19# engine dependent config

18# engine dependent config

20categories = ['it']

19categories = ['it']

searx/engines/base.py+3−3

@@ -14,10 +14,10 @@

14"""

15

16from lxml import etree

17−from urllib import urlencode

18−from searx.utils import searx_useragent

19from datetime import datetime

17from datetime import datetime

20import re

18import re

19+from searx.url_utils import urlencode

20+from searx.utils import searx_useragent

21

22

23categories = ['science']

@@ -73,7 +73,7 @@ def request(query, params):

73def response(resp):

74 results = []

75

76− search_results = etree.XML(resp.content)

76+ search_results = etree.XML(resp.text)

77

78 for entry in search_results.xpath('./result/doc'):

79 content = "No description available"

searx/engines/bing.py+1−1

@@ -13,9 +13,9 @@

13 @todo publishedDate

14"""

15

16−from urllib import urlencode

17from lxml import html

16from lxml import html

18from searx.engines.xpath import extract_text

17from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

19

20# engine dependent config

21categories = ['general']

searx/engines/bing_images.py+1−1

@@ -15,11 +15,11 @@

15 limited response to 10 images

16"""

17

18−from urllib import urlencode

19from lxml import html

18from lxml import html

20from json import loads

19from json import loads

21import re

20import re

22from searx.engines.bing import _fetch_supported_languages, supported_languages_url

21from searx.engines.bing import _fetch_supported_languages, supported_languages_url

22+from searx.url_utils import urlencode

23

24# engine dependent config

25categories = ['images']

searx/engines/bing_news.py+2−3

@@ -11,13 +11,12 @@

11 @parse url, title, content, publishedDate, thumbnail

12"""

13

14−from urllib import urlencode

15−from urlparse import urlparse, parse_qsl

16from datetime import datetime

14from datetime import datetime

17from dateutil import parser

15from dateutil import parser

18from lxml import etree

16from lxml import etree

19from searx.utils import list_get

17from searx.utils import list_get

20from searx.engines.bing import _fetch_supported_languages, supported_languages_url

18from searx.engines.bing import _fetch_supported_languages, supported_languages_url

19+from searx.url_utils import urlencode, urlparse, parse_qsl

21

20

22# engine dependent config

21# engine dependent config

23categories = ['news']

22categories = ['news']

@@ -86,7 +85,7 @@ def request(query, params):

86def response(resp):

85def response(resp):

87 results = []

86 results = []

88

87

89− rss = etree.fromstring(resp.content)

88+ rss = etree.fromstring(resp.text)

90

89

91 ns = rss.nsmap

90 ns = rss.nsmap

92

91

searx/engines/blekko_images.py+1−1

@@ -11,7 +11,7 @@

11"""

12

13from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['images']

searx/engines/btdigg.py+2−3

@@ -10,11 +10,10 @@

10 @parse url, title, content, seed, leech, magnetlink

11"""

12

13−from urlparse import urljoin

14−from urllib import quote

15from lxml import html

13from lxml import html

16from operator import itemgetter

14from operator import itemgetter

17from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

16+from searx.url_utils import quote, urljoin

18from searx.utils import get_torrent_size

17from searx.utils import get_torrent_size

19

18

20# engine dependent config

19# engine dependent config

@@ -38,7 +37,7 @@ def request(query, params):

38def response(resp):

37def response(resp):

39 results = []

38 results = []

40

39

41− dom = html.fromstring(resp.content)

40+ dom = html.fromstring(resp.text)

42

41

43 search_res = dom.xpath('//div[@id="search_res"]/table/tr')

42 search_res = dom.xpath('//div[@id="search_res"]/table/tr')

44

43

searx/engines/currency_convert.py+9−5

@@ -1,21 +1,25 @@

1−from datetime import datetime

1+import json

2import re

3import os

4−import json

4+import sys

5import unicodedata

6

7+from datetime import datetime

8+

9+if sys.version_info[0] == 3:

10+ unicode = str

7

11

8categories = []

12categories = []

9url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'

13url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'

10weight = 100

14weight = 100

11

15

12−parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) # noqa

16+parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

13

17

14db = 1

18db = 1

15

19

16

20

17def normalize_name(name):

21def normalize_name(name):

18− name = name.lower().replace('-', ' ').rstrip('s')

22+ name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')

19 name = re.sub(' +', ' ', name)

23 name = re.sub(' +', ' ', name)

20 return unicodedata.normalize('NFKD', name).lower()

24 return unicodedata.normalize('NFKD', name).lower()

21

25

@@ -35,7 +39,7 @@ def iso4217_to_name(iso4217, language):

35

39

36

40

37def request(query, params):

41def request(query, params):

38− m = parser_re.match(unicode(query, 'utf8'))

42+ m = parser_re.match(query)

39 if not m:

43 if not m:

40 # wrong query

44 # wrong query

41 return params

45 return params

searx/engines/dailymotion.py+1−2

@@ -12,10 +12,9 @@

12 @todo set content-parameter with correct data

13"""

14

15−from urllib import urlencode

16from json import loads

15from json import loads

17from datetime import datetime

16from datetime import datetime

18−from requests import get

17+from searx.url_utils import urlencode

19

18

20# engine dependent config

19# engine dependent config

21categories = ['videos']

20categories = ['videos']

searx/engines/deezer.py+2−3

@@ -11,7 +11,7 @@

11"""

12

13from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['music']

@@ -30,8 +30,7 @@ embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true"

30def request(query, params):

31 offset = (params['pageno'] - 1) * 25

32

33− params['url'] = search_url.format(query=urlencode({'q': query}),

33+ params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)

34− offset=offset)

35

34

36 return params

35 return params

37

36

searx/engines/deviantart.py+1−1

@@ -12,10 +12,10 @@

12 @todo rewrite to api

13"""

14

15−from urllib import urlencode

16from lxml import html

15from lxml import html

17import re

16import re

18from searx.engines.xpath import extract_text

17from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

19

20# engine dependent config

21categories = ['images']

searx/engines/dictzone.py+3−3

@@ -10,20 +10,20 @@

10"""

11

12import re

13−from urlparse import urljoin

14from lxml import html

13from lxml import html

15from searx.utils import is_valid_lang

14from searx.utils import is_valid_lang

15+from searx.url_utils import urljoin

16

17categories = ['general']

18url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

19weight = 100

20

21−parser_re = re.compile(u'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

21+parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

22results_xpath = './/table[@id="r"]/tr'

23

24

25def request(query, params):

26− m = parser_re.match(unicode(query, 'utf8'))

26+ m = parser_re.match(query)

27 if not m:

28 return params

29

searx/engines/digbt.py+6−2

@@ -10,10 +10,14 @@

10 @parse url, title, content, magnetlink

11"""

12

13−from urlparse import urljoin

13+from sys import version_info

14from lxml import html

15from searx.engines.xpath import extract_text

16from searx.utils import get_torrent_size

17+from searx.url_utils import urljoin

18+

19+if version_info[0] == 3:

20+ unicode = str

17

21

18categories = ['videos', 'music', 'files']

22categories = ['videos', 'music', 'files']

19paging = True

23paging = True

@@ -31,7 +35,7 @@ def request(query, params):

31

35

32

36

33def response(resp):

37def response(resp):

34− dom = html.fromstring(resp.content)

38+ dom = html.fromstring(resp.text)

35 search_res = dom.xpath('.//td[@class="x-item"]')

39 search_res = dom.xpath('.//td[@class="x-item"]')

36

40

37 if not search_res:

41 if not search_res:

searx/engines/digg.py+2−2

@@ -10,10 +10,10 @@

10 @parse url, title, content, publishedDate, thumbnail

11"""

12

13−from urllib import quote_plus

13+from dateutil import parser

14from json import loads

15from lxml import html

16−from dateutil import parser

16+from searx.url_utils import quote_plus

17

18# engine dependent config

19categories = ['news', 'social media']

searx/engines/doku.py+1−1

@@ -9,9 +9,9 @@

9# @stable yes

10# @parse (general) url, title, content

11

12−from urllib import urlencode

13from lxml.html import fromstring

12from lxml.html import fromstring

14from searx.engines.xpath import extract_text

13from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py+1−1

@@ -13,11 +13,11 @@

13 @todo rewrite to api

14"""

15

16−from urllib import urlencode

17from lxml.html import fromstring

16from lxml.html import fromstring

18from requests import get

17from requests import get

19from json import loads

18from json import loads

20from searx.engines.xpath import extract_text

19from searx.engines.xpath import extract_text

20+from searx.url_utils import urlencode

21

22# engine dependent config

23categories = ['general']

searx/engines/duckduckgo_definitions.py+3−3

@@ -1,10 +1,10 @@

1import json

2−from urllib import urlencode

3−from re import compile, sub

4from lxml import html

2from lxml import html

5−from searx.utils import html_to_text

3+from re import compile

6from searx.engines.xpath import extract_text

4from searx.engines.xpath import extract_text

7from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url

5from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url

6+from searx.url_utils import urlencode

7+from searx.utils import html_to_text

8

9url = 'https://api.duckduckgo.com/'\

10 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

searx/engines/faroo.py+1−1

@@ -10,10 +10,10 @@

10 @parse url, title, content, publishedDate, img_src

11"""

12

13−from urllib import urlencode

14from json import loads

13from json import loads

15import datetime

14import datetime

16from searx.utils import searx_useragent

15from searx.utils import searx_useragent

16+from searx.url_utils import urlencode

17

18# engine dependent config

19categories = ['general', 'news']

searx/engines/fdroid.py+3−4

@@ -9,9 +9,9 @@

9 @parse url, title, content

10"""

11

12−from urllib import urlencode

13−from searx.engines.xpath import extract_text

14from lxml import html

12from lxml import html

13+from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['files']

@@ -24,8 +24,7 @@ search_url = base_url + 'repository/browse/?{query}'

24

25# do search-request

26def request(query, params):

27− query = urlencode({'fdfilter': query,

27+ query = urlencode({'fdfilter': query, 'fdpage': params['pageno']})

28− 'fdpage': params['pageno']})

29 params['url'] = search_url.format(query=query)

28 params['url'] = search_url.format(query=query)

30 return params

29 return params

31

30

searx/engines/filecrop.py+7−4

@@ -1,5 +1,9 @@

1−from urllib import urlencode

1+from searx.url_utils import urlencode

2+

3+try:

2−from HTMLParser import HTMLParser

4+ from HTMLParser import HTMLParser

5+except:

6+ from html.parser import HTMLParser

3

7

4url = 'http://www.filecrop.com/'

8url = 'http://www.filecrop.com/'

5search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa

9search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa

@@ -73,8 +77,7 @@ class FilecropResultParser(HTMLParser):

73

77

74def request(query, params):

78def request(query, params):

75 index = 1 + (params['pageno'] - 1) * 30

79 index = 1 + (params['pageno'] - 1) * 30

76− params['url'] = search_url.format(query=urlencode({'w': query}),

80+ params['url'] = search_url.format(query=urlencode({'w': query}), index=index)

77− index=index)

78 return params

81 return params

79

82

80

83

searx/engines/flickr.py+1−1

@@ -13,8 +13,8 @@

13 More info on api-key : https://www.flickr.com/services/apps/create/

14"""

15

16−from urllib import urlencode

17from json import loads

16from json import loads

17+from searx.url_utils import urlencode

18

19categories = ['images']

20

searx/engines/flickr_noapi.py+1−1

@@ -12,11 +12,11 @@

12 @parse url, title, thumbnail, img_src

13"""

14

15−from urllib import urlencode

16from json import loads

15from json import loads

17from time import time

16from time import time

18import re

17import re

19from searx.engines import logger

18from searx.engines import logger

19+from searx.url_utils import urlencode

20

21

22logger = logger.getChild('flickr-noapi')

searx/engines/framalibre.py+1−3

@@ -10,12 +10,10 @@

10 @parse url, title, content, thumbnail, img_src

11"""

12

13−from urlparse import urljoin

14from cgi import escape

13from cgi import escape

15−from urllib import urlencode

16from lxml import html

14from lxml import html

17from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

18−from dateutil import parser

16+from searx.url_utils import urljoin, urlencode

19

17

20# engine dependent config

18# engine dependent config

21categories = ['it']

19categories = ['it']

searx/engines/frinkiac.py+1−1

@@ -10,7 +10,7 @@ Frinkiac (Images)

10"""

11

12from json import loads

13−from urllib import urlencode

13+from searx.url_utils import urlencode

14

15categories = ['images']

16

searx/engines/gigablast.py+1−2

@@ -11,10 +11,9 @@

11"""

12

13from json import loads

14−from random import randint

15from time import time

14from time import time

16−from urllib import urlencode

17from lxml.html import fromstring

15from lxml.html import fromstring

16+from searx.url_utils import urlencode

18

17

19# engine dependent config

18# engine dependent config

20categories = ['general']

19categories = ['general']

searx/engines/github.py+1−1

@@ -10,8 +10,8 @@

10 @parse url, title, content

11"""

12

13−from urllib import urlencode

14from json import loads

13from json import loads

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['it']

searx/engines/google.py+2−3

@@ -9,11 +9,10 @@

9# @parse url, title, content, suggestion

10

11import re

12−from urllib import urlencode

13−from urlparse import urlparse, parse_qsl

14from lxml import html, etree

12from lxml import html, etree

15from searx.engines.xpath import extract_text, extract_url

13from searx.engines.xpath import extract_text, extract_url

16−from searx.search import logger

14+from searx import logger

15+from searx.url_utils import urlencode, urlparse, parse_qsl

17

16

18logger = logger.getChild('google engine')

17logger = logger.getChild('google engine')

19

18

searx/engines/google_images.py+1−1

@@ -11,9 +11,9 @@

11"""

12

13from datetime import date, timedelta

14−from urllib import urlencode

15from json import loads

14from json import loads

16from lxml import html

15from lxml import html

16+from searx.url_utils import urlencode

17

18

19# engine dependent config

searx/engines/google_news.py+1−2

@@ -11,9 +11,8 @@

11"""

12

13from lxml import html

14−from urllib import urlencode

15−from json import loads

16from searx.engines.google import _fetch_supported_languages, supported_languages_url

14from searx.engines.google import _fetch_supported_languages, supported_languages_url

15+from searx.url_utils import urlencode

17

16

18# search-url

17# search-url

19categories = ['news']

18categories = ['news']

searx/engines/ina.py+7−3

@@ -12,11 +12,15 @@

12# @todo embedded (needs some md5 from video page)

13

14from json import loads

15−from urllib import urlencode

16from lxml import html

15from lxml import html

17−from HTMLParser import HTMLParser

18−from searx.engines.xpath import extract_text

19from dateutil import parser

16from dateutil import parser

17+from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

19+

20+try:

21+ from HTMLParser import HTMLParser

22+except:

23+ from html.parser import HTMLParser

20

24

21# engine dependent config

25# engine dependent config

22categories = ['videos']

26categories = ['videos']

searx/engines/json_engine.py+8−3

@@ -1,11 +1,16 @@

1−from urllib import urlencode

2−from json import loads

3from collections import Iterable

1from collections import Iterable

2+from json import loads

3+from sys import version_info

4+from searx.url_utils import urlencode

5+

6+if version_info[0] == 3:

7+ unicode = str

4

8

5search_url = None

9search_url = None

6url_query = None

10url_query = None

7content_query = None

11content_query = None

8title_query = None

12title_query = None

13+paging = False

9suggestion_query = ''

14suggestion_query = ''

10results_query = ''

15results_query = ''

11

16

@@ -20,7 +25,7 @@ first_page_num = 1

20

25

21def iterate(iterable):

26def iterate(iterable):

22 if type(iterable) == dict:

27 if type(iterable) == dict:

23− it = iterable.iteritems()

28+ it = iterable.items()

24

29

25 else:

30 else:

26 it = enumerate(iterable)

31 it = enumerate(iterable)

searx/engines/kickass.py+1−2

@@ -10,12 +10,11 @@

10 @parse url, title, content, seed, leech, magnetlink

11"""

12

13−from urlparse import urljoin

14−from urllib import quote

15from lxml import html

13from lxml import html

16from operator import itemgetter

14from operator import itemgetter

17from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

18from searx.utils import get_torrent_size, convert_str_to_int

16from searx.utils import get_torrent_size, convert_str_to_int

17+from searx.url_utils import quote, urljoin

19

18

20# engine dependent config

19# engine dependent config

21categories = ['videos', 'music', 'files']

20categories = ['videos', 'music', 'files']

searx/engines/mediawiki.py+1−1

@@ -14,7 +14,7 @@

14

15from json import loads

16from string import Formatter

17−from urllib import urlencode, quote

17+from searx.url_utils import urlencode, quote

18

19# engine dependent config

20categories = ['general']

searx/engines/mixcloud.py+1−1

@@ -11,8 +11,8 @@

11"""

12

13from json import loads

14−from urllib import urlencode

15from dateutil import parser

14from dateutil import parser

15+from searx.url_utils import urlencode

16

17# engine dependent config

18categories = ['music']

searx/engines/nyaa.py+1−1

@@ -9,9 +9,9 @@

9 @parse url, title, content, seed, leech, torrentfile

10"""

11

12−from urllib import urlencode

13from lxml import html

12from lxml import html

14from searx.engines.xpath import extract_text

13from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['files', 'images', 'videos', 'music']

searx/engines/openstreetmap.py−4

@@ -11,7 +11,6 @@

11"""

12

13from json import loads

14−from searx.utils import searx_useragent

15

14

16# engine dependent config

15# engine dependent config

17categories = ['map']

16categories = ['map']

@@ -27,9 +26,6 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'

27def request(query, params):

26def request(query, params):

28 params['url'] = base_url + search_string.format(query=query)

27 params['url'] = base_url + search_string.format(query=query)

29

28

30− # using searx User-Agent

31− params['headers']['User-Agent'] = searx_useragent()

32−

33 return params

29 return params

34

30

35

31

searx/engines/photon.py+1−1

@@ -10,9 +10,9 @@

10 @parse url, title

11"""

12

13−from urllib import urlencode

14from json import loads

13from json import loads

15from searx.utils import searx_useragent

14from searx.utils import searx_useragent

15+from searx.url_utils import urlencode

16

17# engine dependent config

18categories = ['map']

searx/engines/piratebay.py+1−2

@@ -8,11 +8,10 @@

8# @stable yes (HTML can change)

9# @parse url, title, content, seed, leech, magnetlink

10

11−from urlparse import urljoin

12−from urllib import quote

13from lxml import html

11from lxml import html

14from operator import itemgetter

12from operator import itemgetter

15from searx.engines.xpath import extract_text

13from searx.engines.xpath import extract_text

14+from searx.url_utils import quote, urljoin

16

15

17# engine dependent config

16# engine dependent config

18categories = ['videos', 'music', 'files']

17categories = ['videos', 'music', 'files']

searx/engines/qwant.py+1−2

@@ -12,9 +12,8 @@

12

13from datetime import datetime

14from json import loads

15−from urllib import urlencode

16−

17from searx.utils import html_to_text

15from searx.utils import html_to_text

16+from searx.url_utils import urlencode

18

17

19# engine dependent config

18# engine dependent config

20categories = None

19categories = None

searx/engines/reddit.py+2−4

@@ -11,9 +11,8 @@

11"""

12

13import json

14−from urllib import urlencode

15−from urlparse import urlparse, urljoin

16from datetime import datetime

14from datetime import datetime

15+from searx.url_utils import urlencode, urljoin, urlparse

17

16

18# engine dependent config

17# engine dependent config

19categories = ['general', 'images', 'news', 'social media']

18categories = ['general', 'images', 'news', 'social media']

@@ -26,8 +25,7 @@ search_url = base_url + 'search.json?{query}'

26

25

27# do search-request

26# do search-request

28def request(query, params):

27def request(query, params):

29− query = urlencode({'q': query,

28+ query = urlencode({'q': query, 'limit': page_size})

30− 'limit': page_size})

31 params['url'] = search_url.format(query=query)

29 params['url'] = search_url.format(query=query)

32

30

33 return params

31 return params

searx/engines/scanr_structures.py+1−3

@@ -10,9 +10,7 @@

10 @parse url, title, content, img_src

11"""

12

13−from urllib import urlencode

14from json import loads, dumps

13from json import loads, dumps

15−from dateutil import parser

16from searx.utils import html_to_text

14from searx.utils import html_to_text

17

15

18# engine dependent config

16# engine dependent config

@@ -48,7 +46,7 @@ def response(resp):

48 search_res = loads(resp.text)

46 search_res = loads(resp.text)

49

47

50 # return empty array if there are no results

48 # return empty array if there are no results

51− if search_res.get('total') < 1:

49+ if search_res.get('total', 0) < 1:

52 return []

50 return []

53

51

54 # parse results

52 # parse results

searx/engines/searchcode_code.py+2−3

@@ -10,8 +10,8 @@

10 @parse url, title, content

11"""

12

13−from urllib import urlencode

14from json import loads

13from json import loads

14+from searx.url_utils import urlencode

15

16

17# engine dependent config

@@ -31,8 +31,7 @@ code_endings = {'cs': 'c#',

31

32# do search-request

33def request(query, params):

34− params['url'] = search_url.format(query=urlencode({'q': query}),

34+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)

35− pageno=params['pageno'] - 1)

36

35

37 return params

36 return params

38

37

searx/engines/searchcode_doc.py+2−3

@@ -10,8 +10,8 @@

10 @parse url, title, content

11"""

12

13−from urllib import urlencode

14from json import loads

13from json import loads

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['it']

@@ -24,8 +24,7 @@ search_url = url + 'api/search_IV/?{query}&p={pageno}'

24

25# do search-request

26def request(query, params):

27− params['url'] = search_url.format(query=urlencode({'q': query}),

27+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)

28− pageno=params['pageno'] - 1)

29

28

30 return params

29 return params

31

30

searx/engines/seedpeer.py+1−3

@@ -8,11 +8,9 @@

8# @stable yes (HTML can change)

9# @parse url, title, content, seed, leech, magnetlink

10

11−from urlparse import urljoin

12−from urllib import quote

13from lxml import html

11from lxml import html

14from operator import itemgetter

12from operator import itemgetter

15−from searx.engines.xpath import extract_text

13+from searx.url_utils import quote, urljoin

16

14

17

15

18url = 'http://www.seedpeer.eu/'

16url = 'http://www.seedpeer.eu/'

searx/engines/soundcloud.py+12−7

@@ -11,13 +11,17 @@

11"""

12

13import re

14−from StringIO import StringIO

15from json import loads

14from json import loads

16−from lxml import etree

15+from lxml import html

17−from urllib import urlencode, quote_plus

18from dateutil import parser

16from dateutil import parser

19from searx import logger

17from searx import logger

20from searx.poolrequests import get as http_get

18from searx.poolrequests import get as http_get

19+from searx.url_utils import quote_plus, urlencode

20+

21+try:

22+ from cStringIO import StringIO

23+except:

24+ from io import StringIO

21

25

22# engine dependent config

26# engine dependent config

23categories = ['music']

27categories = ['music']

@@ -36,14 +40,15 @@ embedded_url = '<iframe width="100%" height="166" ' +\

36 'scrolling="no" frameborder="no" ' +\

40 'scrolling="no" frameborder="no" ' +\

37 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'

41 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'

38

42

43+cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)

44+

39

45

40def get_client_id():

46def get_client_id():

41 response = http_get("https://soundcloud.com")

47 response = http_get("https://soundcloud.com")

42− rx_namespace = {"re": "http://exslt.org/regular-expressions"}

43

48

44 if response.ok:

49 if response.ok:

45− tree = etree.parse(StringIO(response.content), etree.HTMLParser())

50+ tree = html.fromstring(response.content)

46− script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)

51+ script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")

47 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

52 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

48

53

49 # extracts valid app_js urls from soundcloud.com content

54 # extracts valid app_js urls from soundcloud.com content

@@ -51,7 +56,7 @@ def get_client_id():

51 # gets app_js and searches for the clientid

56 # gets app_js and searches for the clientid

52 response = http_get(app_js_url)

57 response = http_get(app_js_url)

53 if response.ok:

58 if response.ok:

54− cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)

59+ cids = cid_re.search(response.text)

55 if cids is not None and len(cids.groups()):

60 if cids is not None and len(cids.groups()):

56 return cids.groups()[0]

61 return cids.groups()[0]

57 logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

62 logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

searx/engines/spotify.py+2−3

@@ -11,7 +11,7 @@

11"""

12

13from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

15

16# engine dependent config

17categories = ['music']

@@ -29,8 +29,7 @@ embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{

29def request(query, params):

30 offset = (params['pageno'] - 1) * 20

31

32− params['url'] = search_url.format(query=urlencode({'q': query}),

32+ params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)

33− offset=offset)

34

33

35 return params

34 return params

36

35

searx/engines/stackoverflow.py+2−4

@@ -10,10 +10,9 @@

10 @parse url, title, content

11"""

12

13−from urlparse import urljoin

14−from urllib import urlencode

15from lxml import html

13from lxml import html

16from searx.engines.xpath import extract_text

14from searx.engines.xpath import extract_text

15+from searx.url_utils import urlencode, urljoin

17

16

18# engine dependent config

17# engine dependent config

19categories = ['it']

18categories = ['it']

@@ -31,8 +30,7 @@ content_xpath = './/div[@class="excerpt"]'

31

30

32# do search-request

31# do search-request

33def request(query, params):

32def request(query, params):

34− params['url'] = search_url.format(query=urlencode({'q': query}),

33+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'])

35− pageno=params['pageno'])

36

34

37 return params

35 return params

38

36

searx/engines/startpage.py+1−1

@@ -56,7 +56,7 @@ def request(query, params):

56def response(resp):

57 results = []

58

59− dom = html.fromstring(resp.content)

59+ dom = html.fromstring(resp.text)

60

61 # parse results

62 for result in dom.xpath(results_xpath):

searx/engines/subtitleseeker.py+1−1

@@ -10,10 +10,10 @@

10 @parse url, title, content

11"""

12

13−from urllib import quote_plus

14from lxml import html

13from lxml import html

15from searx.languages import language_codes

14from searx.languages import language_codes

16from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

16+from searx.url_utils import quote_plus

17

18# engine dependent config

19categories = ['videos']

searx/engines/swisscows.py+13−14

@@ -11,9 +11,9 @@

11"""

12

13from json import loads

14−from urllib import urlencode, unquote

15import re

14import re

16from lxml.html import fromstring

15from lxml.html import fromstring

16+from searx.url_utils import unquote, urlencode

17

18# engine dependent config

19categories = ['general', 'images']

@@ -27,10 +27,10 @@ search_string = '?{query}&page={page}'

27supported_languages_url = base_url

28

29# regex

30−regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')

30+regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')

31−regex_json_remove_start = re.compile(r'^initialData:\s*')

31+regex_json_remove_start = re.compile(b'^initialData:\s*')

32−regex_json_remove_end = re.compile(r',\s*environment$')

32+regex_json_remove_end = re.compile(b',\s*environment$')

33−regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')

33+regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

34

35

36# do search-request

@@ -45,10 +45,9 @@ def request(query, params):

45 ui_language = params['language'].split('-')[0]

46

47 search_path = search_string.format(

48− query=urlencode({'query': query,

48+ query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),

49− 'uiLanguage': ui_language,

50− 'region': region}),

51− page=params['pageno'])

49+ page=params['pageno']

50+ )

52

51

53 # image search query is something like 'image?{query}&page={page}'

52 # image search query is something like 'image?{query}&page={page}'

54 if params['category'] == 'images':

53 if params['category'] == 'images':

@@ -63,14 +62,14 @@ def request(query, params):

63def response(resp):

62def response(resp):

64 results = []

63 results = []

65

64

66− json_regex = regex_json.search(resp.content)

65+ json_regex = regex_json.search(resp.text)

67

66

68 # check if results are returned

67 # check if results are returned

69 if not json_regex:

68 if not json_regex:

70 return []

69 return []

71

70

72− json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))

71+ json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))

73− json = loads(json_raw)

72+ json = loads(json_raw.decode('utf-8'))

74

73

75 # parse results

74 # parse results

76 for result in json['Results'].get('items', []):

75 for result in json['Results'].get('items', []):

@@ -78,7 +77,7 @@ def response(resp):

78

77

79 # parse image results

78 # parse image results

80 if result.get('ContentType', '').startswith('image'):

79 if result.get('ContentType', '').startswith('image'):

81− img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

80+ img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))

82

81

83 # append result

82 # append result

84 results.append({'url': result['SourceUrl'],

83 results.append({'url': result['SourceUrl'],

@@ -100,7 +99,7 @@ def response(resp):

100 # parse images

99 # parse images

101 for result in json.get('Images', []):

100 for result in json.get('Images', []):

102 # decode image url

101 # decode image url

103− img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

102+ img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))

104

103

105 # append result

104 # append result

106 results.append({'url': result['SourceUrl'],

105 results.append({'url': result['SourceUrl'],

searx/engines/tokyotoshokan.py+5−6

@@ -11,11 +11,11 @@

11"""

12

13import re

14−from urllib import urlencode

15from lxml import html

14from lxml import html

16from searx.engines.xpath import extract_text

15from searx.engines.xpath import extract_text

17from datetime import datetime

16from datetime import datetime

18from searx.engines.nyaa import int_or_zero, get_filesize_mul

17from searx.engines.nyaa import int_or_zero, get_filesize_mul

18+from searx.url_utils import urlencode

19

20# engine dependent config

21categories = ['files', 'videos', 'music']

@@ -28,8 +28,7 @@ search_url = base_url + 'search.php?{query}'

28

29# do search-request

30def request(query, params):

31− query = urlencode({'page': params['pageno'],

31+ query = urlencode({'page': params['pageno'], 'terms': query})

32− 'terms': query})

33 params['url'] = search_url.format(query=query)

32 params['url'] = search_url.format(query=query)

34 return params

33 return params

35

34

@@ -50,7 +49,7 @@ def response(resp):

50 size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)

49 size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)

51

50

52 # processing the results, two rows at a time

51 # processing the results, two rows at a time

53− for i in xrange(0, len(rows), 2):

52+ for i in range(0, len(rows), 2):

54 # parse the first row

53 # parse the first row

55 name_row = rows[i]

54 name_row = rows[i]

56

55

@@ -79,14 +78,14 @@ def response(resp):

79 groups = size_re.match(item).groups()

78 groups = size_re.match(item).groups()

80 multiplier = get_filesize_mul(groups[1])

79 multiplier = get_filesize_mul(groups[1])

81 params['filesize'] = int(multiplier * float(groups[0]))

80 params['filesize'] = int(multiplier * float(groups[0]))

82− except Exception as e:

81+ except:

83 pass

82 pass

84 elif item.startswith('Date:'):

83 elif item.startswith('Date:'):

85 try:

84 try:

86 # Date: 2016-02-21 21:44 UTC

85 # Date: 2016-02-21 21:44 UTC

87 date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')

86 date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')

88 params['publishedDate'] = date

87 params['publishedDate'] = date

89− except Exception as e:

88+ except:

90 pass

89 pass

91 elif item.startswith('Comment:'):

90 elif item.startswith('Comment:'):

92 params['content'] = item

91 params['content'] = item

searx/engines/torrentz.py+4−4

@@ -12,11 +12,11 @@

12"""

13

14import re

15−from urllib import urlencode

16from lxml import html

15from lxml import html

17−from searx.engines.xpath import extract_text

18from datetime import datetime

16from datetime import datetime

19from searx.engines.nyaa import int_or_zero, get_filesize_mul

17from searx.engines.nyaa import int_or_zero, get_filesize_mul

18+from searx.engines.xpath import extract_text

19+from searx.url_utils import urlencode

20

21# engine dependent config

22categories = ['files', 'videos', 'music']

@@ -70,7 +70,7 @@ def response(resp):

70 size_str = result.xpath('./dd/span[@class="s"]/text()')[0]

71 size, suffix = size_str.split()

72 params['filesize'] = int(size) * get_filesize_mul(suffix)

73− except Exception as e:

73+ except:

74 pass

75

76 # does our link contain a valid SHA1 sum?

@@ -84,7 +84,7 @@ def response(resp):

84 # Fri, 25 Mar 2016 16:29:01

85 date = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')

86 params['publishedDate'] = date

87− except Exception as e:

87+ except:

88 pass

89

90 results.append(params)

searx/engines/translated.py+4

@@ -9,8 +9,12 @@

9 @parse url, title, content

10"""

11import re

12+from sys import version_info

12from searx.utils import is_valid_lang

13from searx.utils import is_valid_lang

13

14

15+if version_info[0] == 3:

16+ unicode = str

17+

14categories = ['general']

18categories = ['general']

15url = u'http://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

19url = u'http://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

16web_url = u'http://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

20web_url = u'http://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

searx/engines/twitter.py+1−2

@@ -12,11 +12,10 @@

12 @todo publishedDate

13"""

14

15−from urlparse import urljoin

16−from urllib import urlencode

17from lxml import html

15from lxml import html

18from datetime import datetime

16from datetime import datetime

19from searx.engines.xpath import extract_text

17from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode, urljoin

20

19

21# engine dependent config

20# engine dependent config

22categories = ['social media']

21categories = ['social media']

searx/engines/vimeo.py+1−1

@@ -13,8 +13,8 @@

13# @todo set content-parameter with correct data

14

15from json import loads

16−from urllib import urlencode

17from dateutil import parser

16from dateutil import parser

17+from searx.url_utils import urlencode

18

19# engine dependent config

20categories = ['videos']

searx/engines/wikidata.py+5−8

@@ -14,12 +14,11 @@

14from searx import logger

15from searx.poolrequests import get

16from searx.engines.xpath import extract_text

17−from searx.utils import format_date_by_locale

18from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

18+from searx.url_utils import urlencode

19

20from json import loads

21from lxml.html import fromstring

22−from urllib import urlencode

23

22

24logger = logger.getChild('wikidata')

23logger = logger.getChild('wikidata')

25result_count = 1

24result_count = 1

@@ -62,14 +61,13 @@ def request(query, params):

62 language = 'en'

61 language = 'en'

63

62

64 params['url'] = url_search.format(

63 params['url'] = url_search.format(

65− query=urlencode({'label': query,

64+ query=urlencode({'label': query, 'language': language}))

66− 'language': language}))

67 return params

65 return params

68

66

69

67

70def response(resp):

68def response(resp):

71 results = []

69 results = []

72− html = fromstring(resp.content)

70+ html = fromstring(resp.text)

73 wikidata_ids = html.xpath(wikidata_ids_xpath)

71 wikidata_ids = html.xpath(wikidata_ids_xpath)

74

72

75 language = resp.search_params['language'].split('-')[0]

73 language = resp.search_params['language'].split('-')[0]

@@ -78,10 +76,9 @@ def response(resp):

78

76

79 # TODO: make requests asynchronous to avoid timeout when result_count > 1

77 # TODO: make requests asynchronous to avoid timeout when result_count > 1

80 for wikidata_id in wikidata_ids[:result_count]:

78 for wikidata_id in wikidata_ids[:result_count]:

81− url = url_detail.format(query=urlencode({'page': wikidata_id,

79+ url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))

82− 'uselang': language}))

83 htmlresponse = get(url)

80 htmlresponse = get(url)

84− jsonresponse = loads(htmlresponse.content)

81+ jsonresponse = loads(htmlresponse.text)

85 results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])

82 results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])

86

83

87 return results

84 return results

searx/engines/wikipedia.py+9−12

@@ -11,13 +11,12 @@

11"""

12

13from json import loads

14−from urllib import urlencode, quote

15from lxml.html import fromstring

14from lxml.html import fromstring

16−

15+from searx.url_utils import quote, urlencode

17

16

18# search-url

17# search-url

19−base_url = 'https://{language}.wikipedia.org/'

18+base_url = u'https://{language}.wikipedia.org/'

20−search_postfix = 'w/api.php?'\

19+search_url = base_url + u'w/api.php?'\

21 'action=query'\

20 'action=query'\

22 '&format=json'\

21 '&format=json'\

23 '&{query}'\

22 '&{query}'\

@@ -37,16 +36,16 @@ def url_lang(lang):

37 else:

36 else:

38 language = lang

37 language = lang

39

38

40− return base_url.format(language=language)

39+ return language

41

40

42

41

43# do search-request

42# do search-request

44def request(query, params):

43def request(query, params):

45 if query.islower():

44 if query.islower():

46− query += '|' + query.title()

45+ query = u'{0}|{1}'.format(query.decode('utf-8'), query.decode('utf-8').title()).encode('utf-8')

47

46

48− params['url'] = url_lang(params['language']) \

49− + search_postfix.format(query=urlencode({'titles': query}))

47+ params['url'] = search_url.format(query=urlencode({'titles': query}),

48+ language=url_lang(params['language']))

50

49

51 return params

50 return params

52

51

@@ -78,7 +77,7 @@ def extract_first_paragraph(content, title, image):

78def response(resp):

77def response(resp):

79 results = []

78 results = []

80

79

81− search_result = loads(resp.content)

80+ search_result = loads(resp.text)

82

81

83 # wikipedia article's unique id

82 # wikipedia article's unique id

84 # first valid id is assumed to be the requested article

83 # first valid id is assumed to be the requested article

@@ -99,11 +98,9 @@ def response(resp):

99 extract = page.get('extract')

98 extract = page.get('extract')

100

99

101 summary = extract_first_paragraph(extract, title, image)

100 summary = extract_first_paragraph(extract, title, image)

102− if not summary:

103− return []

104

101

105 # link to wikipedia article

102 # link to wikipedia article

106− wikipedia_link = url_lang(resp.search_params['language']) \

103+ wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \

107 + 'wiki/' + quote(title.replace(' ', '_').encode('utf8'))

104 + 'wiki/' + quote(title.replace(' ', '_').encode('utf8'))

108

105

109 results.append({'url': wikipedia_link, 'title': title})

106 results.append({'url': wikipedia_link, 'title': title})

searx/engines/wolframalpha_api.py+6−7

@@ -8,8 +8,8 @@

8# @stable yes

9# @parse url, infobox

10

11−from urllib import urlencode

12from lxml import etree

11from lxml import etree

12+from searx.url_utils import urlencode

13

14# search-url

15search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'

@@ -37,8 +37,7 @@ image_pods = {'VisualRepresentation',

37

38# do search-request

39def request(query, params):

40− params['url'] = search_url.format(query=urlencode({'input': query}),

40+ params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key)

41− api_key=api_key)

42 params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))

41 params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))

43

42

44 return params

43 return params

@@ -56,7 +55,7 @@ def replace_pua_chars(text):

56 u'\uf74e': 'i', # imaginary number

55 u'\uf74e': 'i', # imaginary number

57 u'\uf7d9': '='} # equals sign

56 u'\uf7d9': '='} # equals sign

58

57

59− for k, v in pua_chars.iteritems():

58+ for k, v in pua_chars.items():

60 text = text.replace(k, v)

59 text = text.replace(k, v)

61

60

62 return text

61 return text

@@ -66,7 +65,7 @@ def replace_pua_chars(text):

66def response(resp):

65def response(resp):

67 results = []

66 results = []

68

67

69− search_results = etree.XML(resp.content)

68+ search_results = etree.XML(resp.text)

70

69

71 # return empty array if there are no results

70 # return empty array if there are no results

72 if search_results.xpath(failure_xpath):

71 if search_results.xpath(failure_xpath):

@@ -120,10 +119,10 @@ def response(resp):

120 # append infobox

119 # append infobox

121 results.append({'infobox': infobox_title,

120 results.append({'infobox': infobox_title,

122 'attributes': result_chunks,

121 'attributes': result_chunks,

123− 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})

122+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})

124

123

125 # append link to site

124 # append link to site

126− results.append({'url': resp.request.headers['Referer'].decode('utf8'),

125+ results.append({'url': resp.request.headers['Referer'],

127 'title': title,

126 'title': title,

128 'content': result_content})

127 'content': result_content})

129

128

searx/engines/wolframalpha_noapi.py+4−5

@@ -10,10 +10,9 @@

10

11from json import loads

12from time import time

13−from urllib import urlencode

14−from lxml.etree import XML

15

13

16from searx.poolrequests import get as http_get

14from searx.poolrequests import get as http_get

15+from searx.url_utils import urlencode

17

16

18# search-url

17# search-url

19url = 'https://www.wolframalpha.com/'

18url = 'https://www.wolframalpha.com/'

@@ -62,7 +61,7 @@ obtain_token()

62# do search-request

61# do search-request

63def request(query, params):

62def request(query, params):

64 # obtain token if last update was more than an hour

63 # obtain token if last update was more than an hour

65− if time() - token['last_updated'] > 3600:

64+ if time() - (token['last_updated'] or 0) > 3600:

66 obtain_token()

65 obtain_token()

67 params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])

66 params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])

68 params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))

67 params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))

@@ -112,9 +111,9 @@ def response(resp):

112

111

113 results.append({'infobox': infobox_title,

112 results.append({'infobox': infobox_title,

114 'attributes': result_chunks,

113 'attributes': result_chunks,

115− 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})

114+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})

116

115

117− results.append({'url': resp.request.headers['Referer'].decode('utf8'),

116+ results.append({'url': resp.request.headers['Referer'],

118 'title': 'Wolfram|Alpha (' + infobox_title + ')',

117 'title': 'Wolfram|Alpha (' + infobox_title + ')',

119 'content': result_content})

118 'content': result_content})

120

119

searx/engines/www1x.py+2−4

@@ -10,11 +10,9 @@

10 @parse url, title, thumbnail, img_src, content

11"""

12

13−from urllib import urlencode

14−from urlparse import urljoin

15from lxml import html

13from lxml import html

16−import string

17import re

14import re

15+from searx.url_utils import urlencode, urljoin

18

16

19# engine dependent config

17# engine dependent config

20categories = ['images']

18categories = ['images']

@@ -55,7 +53,7 @@ def response(resp):

55 cur_element += result_part

53 cur_element += result_part

56

54

57 # fix xml-error

55 # fix xml-error

58− cur_element = string.replace(cur_element, '"></a>', '"/></a>')

56+ cur_element = cur_element.replace('"></a>', '"/></a>')

59

57

60 dom = html.fromstring(cur_element)

58 dom = html.fromstring(cur_element)

61 link = dom.xpath('//a')[0]

59 link = dom.xpath('//a')[0]

searx/engines/www500px.py+1−2

@@ -13,8 +13,7 @@

13"""

14

15from json import loads

16−from urllib import urlencode

16+from searx.url_utils import urlencode, urljoin

17−from urlparse import urljoin

18

17

19# engine dependent config

18# engine dependent config

20categories = ['images']

19categories = ['images']

searx/engines/xpath.py+2−2

@@ -1,13 +1,13 @@

1from lxml import html

2−from urllib import urlencode, unquote

3−from urlparse import urlparse, urljoin

4from lxml.etree import _ElementStringResult, _ElementUnicodeResult

2from lxml.etree import _ElementStringResult, _ElementUnicodeResult

5from searx.utils import html_to_text

3from searx.utils import html_to_text

4+from searx.url_utils import unquote, urlencode, urljoin, urlparse

6

5

7search_url = None

6search_url = None

8url_xpath = None

7url_xpath = None

9content_xpath = None

8content_xpath = None

10title_xpath = None

9title_xpath = None

10+paging = False

11suggestion_xpath = ''

12results_xpath = ''

13

searx/engines/yacy.py+1−1

@@ -13,8 +13,8 @@

13# @todo parse video, audio and file results

14

15from json import loads

16−from urllib import urlencode

17from dateutil import parser

16from dateutil import parser

17+from searx.url_utils import urlencode

18

19from searx.utils import html_to_text

20

searx/engines/yahoo.py+1−2

@@ -11,10 +11,9 @@

11 @parse url, title, content, suggestion

12"""

13

14−from urllib import urlencode

15−from urlparse import unquote

16from lxml import html

14from lxml import html

17from searx.engines.xpath import extract_text, extract_url

15from searx.engines.xpath import extract_text, extract_url

16+from searx.url_utils import unquote, urlencode

18

17

19# engine dependent config

18# engine dependent config

20categories = ['general']

19categories = ['general']

searx/engines/yahoo_news.py+3−3

@@ -9,13 +9,13 @@

9# @stable no (HTML can change)

10# @parse url, title, content, publishedDate

11

12−from urllib import urlencode

12+import re

13+from datetime import datetime, timedelta

13from lxml import html

14from lxml import html

14from searx.engines.xpath import extract_text, extract_url

15from searx.engines.xpath import extract_text, extract_url

15from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url

16from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url

16−from datetime import datetime, timedelta

17−import re

18from dateutil import parser

17from dateutil import parser

18+from searx.url_utils import urlencode

19

20# engine dependent config

21categories = ['news']

searx/engines/yandex.py+2−2

@@ -9,9 +9,9 @@

9 @parse url, title, content

10"""

11

12−from urllib import urlencode

13from lxml import html

12from lxml import html

14−from searx.search import logger

13+from searx import logger

14+from searx.url_utils import urlencode

15

16logger = logger.getChild('yandex engine')

17

searx/engines/youtube_api.py+1−1

@@ -9,8 +9,8 @@

9# @parse url, title, content, publishedDate, thumbnail, embedded

10

11from json import loads

12−from urllib import urlencode

13from dateutil import parser

12from dateutil import parser

13+from searx.url_utils import urlencode

14

15# engine dependent config

16categories = ['videos', 'music']

searx/engines/youtube_noapi.py+1−1

@@ -8,10 +8,10 @@

8# @stable no

9# @parse url, title, content, publishedDate, thumbnail, embedded

10

11−from urllib import quote_plus

12from lxml import html

11from lxml import html

13from searx.engines.xpath import extract_text

12from searx.engines.xpath import extract_text

14from searx.utils import list_get

13from searx.utils import list_get

14+from searx.url_utils import quote_plus

15

16# engine dependent config

17categories = ['videos', 'music']

searx/plugins/__init__.py+4−1

@@ -14,9 +14,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

14

16'''

17−from sys import exit

17+from sys import exit, version_info

18from searx import logger

19

20+if version_info[0] == 3:

21+ unicode = str

22+

20logger = logger.getChild('plugins')

23logger = logger.getChild('plugins')

21

24

22from searx.plugins import (doai_rewrite,

25from searx.plugins import (doai_rewrite,

searx/plugins/doai_rewrite.py+1−1

@@ -1,6 +1,6 @@

1from flask_babel import gettext

2import re

3−from urlparse import urlparse, parse_qsl

3+from searx.url_utils import urlparse, parse_qsl

4

5regex = re.compile(r'10\.\d{4,9}/[^\s]+')

6

searx/plugins/https_rewrite.py+4−1

@@ -16,14 +16,17 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

16'''

17

18import re

19−from urlparse import urlparse

19+import sys

20from lxml import etree

21from os import listdir, environ

22from os.path import isfile, isdir, join

23from searx.plugins import logger

24from flask_babel import gettext

25from searx import searx_dir

26+from searx.url_utils import urlparse

26

27

28+if sys.version_info[0] == 3:

29+ unicode = str

27

30

28name = "HTTPS rewrite"

31name = "HTTPS rewrite"

29description = gettext('Rewrite HTTP links to HTTPS if possible')

32description = gettext('Rewrite HTTP links to HTTPS if possible')

searx/plugins/self_info.py+2−2

@@ -22,7 +22,7 @@ default_on = True

22

23

24# Self User Agent regex

25−p = re.compile('.*user[ -]agent.*', re.IGNORECASE)

25+p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)

26

27

28# attach callback to the post search hook

@@ -31,7 +31,7 @@ p = re.compile('.*user[ -]agent.*', re.IGNORECASE)

31def post_search(request, search):

32 if search.search_query.pageno > 1:

33 return True

34− if search.search_query.query == 'ip':

34+ if search.search_query.query == b'ip':

35 x_forwarded_for = request.headers.getlist("X-Forwarded-For")

36 if x_forwarded_for:

37 ip = x_forwarded_for[0]

searx/plugins/tracker_url_remover.py+1−1

@@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

17

18from flask_babel import gettext

19import re

20−from urlparse import urlunparse

20+from searx.url_utils import urlunparse

21

22regexes = {re.compile(r'utm_[^&]+&?'),

23 re.compile(r'(wkey|wemail)[^&]+&?'),

searx/preferences.py+9−9

@@ -23,7 +23,7 @@ class Setting(object):

23 def __init__(self, default_value, **kwargs):

24 super(Setting, self).__init__()

25 self.value = default_value

26− for key, value in kwargs.iteritems():

26+ for key, value in kwargs.items():

27 setattr(self, key, value)

28

29 self._post_init()

@@ -38,7 +38,7 @@ class Setting(object):

38 return self.value

39

40 def save(self, name, resp):

41− resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)

41+ resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE)

42

43

44class StringSetting(Setting):

@@ -133,7 +133,7 @@ class MapSetting(Setting):

133

134 def save(self, name, resp):

135 if hasattr(self, 'key'):

136− resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)

136+ resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)

137

138

139class SwitchableSetting(Setting):

@@ -194,7 +194,7 @@ class EnginesSetting(SwitchableSetting):

194 def _post_init(self):

195 super(EnginesSetting, self)._post_init()

196 transformed_choices = []

197− for engine_name, engine in self.choices.iteritems():

197+ for engine_name, engine in self.choices.items():

198 for category in engine.categories:

199 transformed_choice = dict()

200 transformed_choice['default_on'] = not engine.disabled

@@ -241,9 +241,9 @@ class Preferences(object):

241 'language': SearchLanguageSetting(settings['search']['language'],

242 choices=LANGUAGE_CODES),

243 'locale': EnumStringSetting(settings['ui']['default_locale'],

244− choices=settings['locales'].keys() + ['']),

244+ choices=list(settings['locales'].keys()) + ['']),

245 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],

246− choices=autocomplete.backends.keys() + ['']),

246+ choices=list(autocomplete.backends.keys()) + ['']),

247 'image_proxy': MapSetting(settings['server']['image_proxy'],

248 map={'': settings['server']['image_proxy'],

249 '0': False,

@@ -260,7 +260,7 @@ class Preferences(object):

260 self.unknown_params = {}

261

262 def parse_cookies(self, input_data):

263− for user_setting_name, user_setting in input_data.iteritems():

263+ for user_setting_name, user_setting in input_data.items():

264 if user_setting_name in self.key_value_settings:

265 self.key_value_settings[user_setting_name].parse(user_setting)

266 elif user_setting_name == 'disabled_engines':

@@ -274,7 +274,7 @@ class Preferences(object):

274 disabled_engines = []

275 enabled_categories = []

276 disabled_plugins = []

277− for user_setting_name, user_setting in input_data.iteritems():

277+ for user_setting_name, user_setting in input_data.items():

278 if user_setting_name in self.key_value_settings:

279 self.key_value_settings[user_setting_name].parse(user_setting)

280 elif user_setting_name.startswith('engine_'):

@@ -295,7 +295,7 @@ class Preferences(object):

295 return self.key_value_settings[user_setting_name].get_value()

296

297 def save(self, resp):

298− for user_setting_name, user_setting in self.key_value_settings.iteritems():

298+ for user_setting_name, user_setting in self.key_value_settings.items():

299 user_setting.save(user_setting_name, resp)

300 self.engines.save(resp)

301 self.plugins.save(resp)

searx/query.py+6−2

@@ -21,8 +21,12 @@ from searx.languages import language_codes

21from searx.engines import (

22 categories, engines, engine_shortcuts

23)

24−import string

25import re

24import re

25+import string

26+import sys

27+

28+if sys.version_info[0] == 3:

29+ unicode = str

26

30

27VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')

31VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')

28

32

@@ -146,7 +150,7 @@ class SearchQuery(object):

146 """container for all the search parameters (query, language, etc...)"""

150 """container for all the search parameters (query, language, etc...)"""

147

151

148 def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):

152 def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):

149− self.query = query

153+ self.query = query.encode('utf-8')

150 self.engines = engines

154 self.engines = engines

151 self.categories = categories

155 self.categories = categories

152 self.lang = lang

156 self.lang = lang

searx/results.py+5−1

@@ -1,9 +1,13 @@

1import re

2+import sys

2from collections import defaultdict

3from collections import defaultdict

3from operator import itemgetter

4from operator import itemgetter

4from threading import RLock

5from threading import RLock

5−from urlparse import urlparse, unquote

6from searx.engines import engines

7+from searx.url_utils import urlparse, unquote

8+

9+if sys.version_info[0] == 3:

10+ basestring = str

7

11

8CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

12CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

9WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

13WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

searx/search.py+10−2

@@ -16,8 +16,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

16'''

17

18import gc

19+import sys

19import threading

20import threading

20−from thread import start_new_thread

21from time import time

22from uuid import uuid4

23import requests.exceptions

@@ -33,6 +33,14 @@ from searx import logger

33from searx.plugins import plugins

34from searx.exceptions import SearxParameterException

35

36+try:

37+ from thread import start_new_thread

38+except:

39+ from _thread import start_new_thread

40+

41+if sys.version_info[0] == 3:

42+ unicode = str

43+

36logger = logger.getChild('search')

44logger = logger.getChild('search')

37

45

38number_of_searches = 0

46number_of_searches = 0

@@ -387,7 +395,7 @@ class Search(object):

387 request_params['time_range'] = search_query.time_range

395 request_params['time_range'] = search_query.time_range

388

396

389 # append request to list

397 # append request to list

390− requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))

398+ requests.append((selected_engine['name'], search_query.query, request_params))

391

399

392 # update timeout_limit

400 # update timeout_limit

393 timeout_limit = max(timeout_limit, engine.timeout)

401 timeout_limit = max(timeout_limit, engine.timeout)

searx/settings_robot.yml+1−1

@@ -17,7 +17,7 @@ server:

17

18ui:

19 themes_path : ""

20− default_theme : legacy

20+ default_theme : oscar

21 default_locale : ""

22

23outgoing:

searx/templates/courgette/404.html+1−1

@@ -3,7 +3,7 @@

3<div class="center">

4 <h1>{{ _('Page not found') }}</h1>

5 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

7 {% endautoescape %}

8</div>

9{% endblock %}

searx/templates/legacy/404.html+1−1

@@ -3,7 +3,7 @@

3<div class="center">

4 <h1>{{ _('Page not found') }}</h1>

5 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

7 {% endautoescape %}

8</div>

9{% endblock %}

searx/templates/oscar/404.html+1−1

@@ -3,7 +3,7 @@

3<div class="text-center">

4 <h1>{{ _('Page not found') }}</h1>

5 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

7 {% endautoescape %}

8</div>

9{% endblock %}

searx/templates/pix-art/404.html+1−1

@@ -3,7 +3,7 @@

3<div class="center">

4 <h1>{{ _('Page not found') }}</h1>

5 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

7 {% endautoescape %}

8</div>

9{% endblock %}

searx/testing.py+26−16

@@ -1,13 +1,16 @@

1# -*- coding: utf-8 -*-

2"""Shared testing code."""

3

4−from plone.testing import Layer

5−from unittest2 import TestCase

6−from os.path import dirname, join, abspath

7−

8

4

9import os

5import os

10import subprocess

6import subprocess

7+import traceback

8+

9+

10+from os.path import dirname, join, abspath

11+

12+from splinter import Browser

13+from unittest2 import TestCase

11

14

12

15

13class SearxTestLayer:

16class SearxTestLayer:

@@ -32,7 +35,7 @@ class SearxTestLayer:

32 testTearDown = classmethod(testTearDown)

35 testTearDown = classmethod(testTearDown)

33

36

34

37

35−class SearxRobotLayer(Layer):

38+class SearxRobotLayer():

36 """Searx Robot Test Layer"""

39 """Searx Robot Test Layer"""

37

40

38 def setUp(self):

41 def setUp(self):

@@ -62,7 +65,12 @@ class SearxRobotLayer(Layer):

62 del os.environ['SEARX_SETTINGS_PATH']

65 del os.environ['SEARX_SETTINGS_PATH']

63

66

64

67

65−SEARXROBOTLAYER = SearxRobotLayer()

68+# SEARXROBOTLAYER = SearxRobotLayer()

69+def run_robot_tests(tests):

70+ print('Running {0} tests'.format(len(tests)))

71+ for test in tests:

72+ with Browser() as browser:

73+ test(browser)

66

74

67

75

68class SearxTestCase(TestCase):

76class SearxTestCase(TestCase):

@@ -72,17 +80,19 @@ class SearxTestCase(TestCase):

72

80

73

81

74if __name__ == '__main__':

82if __name__ == '__main__':

75− from tests.test_robot import test_suite

76 import sys

83 import sys

77− from zope.testrunner.runner import Runner

84+ # test cases

85+ from tests import robot

78

86

79 base_dir = abspath(join(dirname(__file__), '../tests'))

87 base_dir = abspath(join(dirname(__file__), '../tests'))

80 if sys.argv[1] == 'robot':

88 if sys.argv[1] == 'robot':

81− r = Runner(['--color',

89+ test_layer = SearxRobotLayer()

82− '--auto-progress',

90+ errors = False

83− '--stop-on-error',

91+ try:

84− '--path',

92+ test_layer.setUp()

85− base_dir],

93+ run_robot_tests([getattr(robot, x) for x in dir(robot) if x.startswith('test_')])

86− found_suites=[test_suite()])

94+ except Exception:

87− r.run()

95+ errors = True

88− sys.exit(int(r.failed))

96+ print('Error occured: {0}'.format(traceback.format_exc()))

97+ test_layer.tearDown()

98+ sys.exit(1 if errors else 0)

searx/url_utils.pynew file+28

@@ -0,0 +1,28 @@

1+from sys import version_info

2+

3+if version_info[0] == 2:

4+ from urllib import quote, quote_plus, unquote, urlencode

5+ from urlparse import parse_qsl, urljoin, urlparse, urlunparse, ParseResult

6+else:

7+ from urllib.parse import (

8+ parse_qsl,

9+ quote,

10+ quote_plus,

11+ unquote,

12+ urlencode,

13+ urljoin,

14+ urlparse,

15+ urlunparse,

16+ ParseResult

17+ )

18+

19+

20+__export__ = (parse_qsl,

21+ quote,

22+ quote_plus,

23+ unquote,

24+ urlencode,

25+ urljoin,

26+ urlparse,

27+ urlunparse,

28+ ParseResult)

searx/utils.py+18−8

@@ -1,11 +1,9 @@

1−import cStringIO

2import csv

1import csv

3import os

2import os

4import re

3import re

5

4

6from babel.dates import format_date

5from babel.dates import format_date

7from codecs import getincrementalencoder

6from codecs import getincrementalencoder

8−from HTMLParser import HTMLParser

9from imp import load_source

7from imp import load_source

10from os.path import splitext, join

8from os.path import splitext, join

11from random import choice

9from random import choice

@@ -16,6 +14,19 @@ from searx.languages import language_codes

16from searx import settings

14from searx import settings

17from searx import logger

15from searx import logger

18

16

17+try:

18+ from cStringIO import StringIO

19+except:

20+ from io import StringIO

21+

22+try:

23+ from HTMLParser import HTMLParser

24+except:

25+ from html.parser import HTMLParser

26+

27+if sys.version_info[0] == 3:

28+ unichr = chr

29+ unicode = str

19

30

20logger = logger.getChild('utils')

31logger = logger.getChild('utils')

21

32

@@ -140,7 +151,7 @@ class UnicodeWriter:

140

151

141 def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):

152 def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):

142 # Redirect output to a queue

153 # Redirect output to a queue

143− self.queue = cStringIO.StringIO()

154+ self.queue = StringIO()

144 self.writer = csv.writer(self.queue, dialect=dialect, **kwds)

155 self.writer = csv.writer(self.queue, dialect=dialect, **kwds)

145 self.stream = f

156 self.stream = f

146 self.encoder = getincrementalencoder(encoding)()

157 self.encoder = getincrementalencoder(encoding)()

@@ -152,14 +163,13 @@ class UnicodeWriter:

152 unicode_row.append(col.encode('utf-8').strip())

163 unicode_row.append(col.encode('utf-8').strip())

153 else:

164 else:

154 unicode_row.append(col)

165 unicode_row.append(col)

155− self.writer.writerow(unicode_row)

166+ self.writer.writerow([x.decode('utf-8') if hasattr(x, 'decode') else x for x in unicode_row])

156 # Fetch UTF-8 output from the queue ...

167 # Fetch UTF-8 output from the queue ...

157− data = self.queue.getvalue()

168+ data = self.queue.getvalue().strip('\x00')

158− data = data.decode("utf-8")

159 # ... and reencode it into the target encoding

169 # ... and reencode it into the target encoding

160 data = self.encoder.encode(data)

170 data = self.encoder.encode(data)

161 # write to the target stream

171 # write to the target stream

162− self.stream.write(data)

172+ self.stream.write(data.decode('utf-8'))

163 # empty queue

173 # empty queue

164 self.queue.truncate(0)

174 self.queue.truncate(0)

165

175

@@ -231,7 +241,7 @@ def dict_subset(d, properties):

231

241

232def prettify_url(url, max_length=74):

242def prettify_url(url, max_length=74):

233 if len(url) > max_length:

243 if len(url) > max_length:

234− chunk_len = max_length / 2 + 1

244+ chunk_len = int(max_length / 2 + 1)

235 return u'{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])

245 return u'{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])

236 else:

246 else:

237 return url

247 return url

searx/webapp.py+23−13

@@ -22,11 +22,12 @@ if __name__ == '__main__':

22 from os.path import realpath, dirname

23 path.append(realpath(dirname(realpath(__file__)) + '/../'))

24

25−import cStringIO

26import hashlib

25import hashlib

27import hmac

26import hmac

28import json

27import json

29import os

28import os

29+import sys

30+

30import requests

31import requests

31

32

32from searx import logger

33from searx import logger

@@ -42,8 +43,6 @@ except:

42 exit(1)

43 exit(1)

43from cgi import escape

44from cgi import escape

44from datetime import datetime, timedelta

45from datetime import datetime, timedelta

45−from urllib import urlencode

46−from urlparse import urlparse, urljoin

47from werkzeug.contrib.fixers import ProxyFix

46from werkzeug.contrib.fixers import ProxyFix

48from flask import (

47from flask import (

49 Flask, request, render_template, url_for, Response, make_response,

48 Flask, request, render_template, url_for, Response, make_response,

@@ -52,7 +51,7 @@ from flask import (

52from flask_babel import Babel, gettext, format_date, format_decimal

51from flask_babel import Babel, gettext, format_date, format_decimal

53from flask.json import jsonify

52from flask.json import jsonify

54from searx import settings, searx_dir, searx_debug

53from searx import settings, searx_dir, searx_debug

55−from searx.exceptions import SearxException, SearxParameterException

54+from searx.exceptions import SearxParameterException

56from searx.engines import (

55from searx.engines import (

57 categories, engines, engine_shortcuts, get_engines_stats, initialize_engines

56 categories, engines, engine_shortcuts, get_engines_stats, initialize_engines

58)

57)

@@ -69,6 +68,7 @@ from searx.autocomplete import searx_bang, backends as autocomplete_backends

69from searx.plugins import plugins

68from searx.plugins import plugins

70from searx.preferences import Preferences, ValidationException

69from searx.preferences import Preferences, ValidationException

71from searx.answerers import answerers

70from searx.answerers import answerers

71+from searx.url_utils import urlencode, urlparse, urljoin

72

73# check if the pyopenssl package is installed.

74# It is needed for SSL connection without trouble, see #298

@@ -78,6 +78,15 @@ except ImportError:

78 logger.critical("The pyopenssl package has to be installed.\n"

79 "Some HTTPS connections will fail")

80

81+try:

82+ from cStringIO import StringIO

83+except:

84+ from io import StringIO

85+

86+

87+if sys.version_info[0] == 3:

88+ unicode = str

89+

81# serve pages with HTTP/1.1

90# serve pages with HTTP/1.1

82from werkzeug.serving import WSGIRequestHandler

91from werkzeug.serving import WSGIRequestHandler

83WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))

92WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))

@@ -357,6 +366,8 @@ def render(template_name, override_theme=None, **kwargs):

357

366

358 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

367 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

359

368

369+ kwargs['unicode'] = unicode

370+

360 kwargs['scripts'] = set()

371 kwargs['scripts'] = set()

361 for plugin in request.user_plugins:

372 for plugin in request.user_plugins:

362 for script in plugin.js_dependencies:

373 for script in plugin.js_dependencies:

@@ -375,7 +386,7 @@ def render(template_name, override_theme=None, **kwargs):

375def pre_request():

386def pre_request():

376 request.errors = []

387 request.errors = []

377

388

378− preferences = Preferences(themes, categories.keys(), engines, plugins)

389+ preferences = Preferences(themes, list(categories.keys()), engines, plugins)

379 request.preferences = preferences

390 request.preferences = preferences

380 try:

391 try:

381 preferences.parse_cookies(request.cookies)

392 preferences.parse_cookies(request.cookies)

@@ -479,10 +490,8 @@ def index():

479 for result in results:

490 for result in results:

480 if output_format == 'html':

491 if output_format == 'html':

481 if 'content' in result and result['content']:

492 if 'content' in result and result['content']:

482− result['content'] = highlight_content(escape(result['content'][:1024]),

493+ result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)

483− search_query.query.encode('utf-8'))

484− result['title'] = highlight_content(escape(result['title'] or u''),

494+ result['title'] = highlight_content(escape(result['title'] or u''), search_query.query)

485− search_query.query.encode('utf-8'))

486 else:

495 else:

487 if result.get('content'):

496 if result.get('content'):

488 result['content'] = html_to_text(result['content']).strip()

497 result['content'] = html_to_text(result['content']).strip()

@@ -510,7 +519,7 @@ def index():

510 result['publishedDate'] = format_date(result['publishedDate'])

519 result['publishedDate'] = format_date(result['publishedDate'])

511

520

512 if output_format == 'json':

521 if output_format == 'json':

513− return Response(json.dumps({'query': search_query.query,

522+ return Response(json.dumps({'query': search_query.query.decode('utf-8'),

514 'number_of_results': number_of_results,

523 'number_of_results': number_of_results,

515 'results': results,

524 'results': results,

516 'answers': list(result_container.answers),

525 'answers': list(result_container.answers),

@@ -519,7 +528,7 @@ def index():

519 'suggestions': list(result_container.suggestions)}),

528 'suggestions': list(result_container.suggestions)}),

520 mimetype='application/json')

529 mimetype='application/json')

521 elif output_format == 'csv':

530 elif output_format == 'csv':

522− csv = UnicodeWriter(cStringIO.StringIO())

531+ csv = UnicodeWriter(StringIO())

523 keys = ('title', 'url', 'content', 'host', 'engine', 'score')

532 keys = ('title', 'url', 'content', 'host', 'engine', 'score')

524 csv.writerow(keys)

533 csv.writerow(keys)

525 for row in results:

534 for row in results:

@@ -527,7 +536,7 @@ def index():

527 csv.writerow([row.get(key, '') for key in keys])

536 csv.writerow([row.get(key, '') for key in keys])

528 csv.stream.seek(0)

537 csv.stream.seek(0)

529 response = Response(csv.stream.read(), mimetype='application/csv')

538 response = Response(csv.stream.read(), mimetype='application/csv')

530− cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query.encode('utf-8'))

539+ cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)

531 response.headers.add('Content-Disposition', cont_disp)

540 response.headers.add('Content-Disposition', cont_disp)

532 return response

541 return response

533 elif output_format == 'rss':

542 elif output_format == 'rss':

@@ -578,7 +587,7 @@ def autocompleter():

578 disabled_engines = request.preferences.engines.get_disabled()

587 disabled_engines = request.preferences.engines.get_disabled()

579

588

580 # parse query

589 # parse query

581− raw_text_query = RawTextQuery(request.form.get('q', '').encode('utf-8'), disabled_engines)

590+ raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)

582 raw_text_query.parse_query()

591 raw_text_query.parse_query()

583

592

584 # check if search query is set

593 # check if search query is set

@@ -820,6 +829,7 @@ def page_not_found(e):

820

829

821

830

822def run():

831def run():

832+ logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])

823 app.run(

833 app.run(

824 debug=searx_debug,

834 debug=searx_debug,

825 use_debugger=searx_debug,

835 use_debugger=searx_debug,

tests/robot/__init__.py+75

@@ -0,0 +1,75 @@

1+# -*- coding: utf-8 -*-

2+

3+from time import sleep

4+

5+url = "http://localhost:11111/"

6+

7+

8+def test_index(browser):

9+ # Visit URL

10+ browser.visit(url)

11+ assert browser.is_text_present('about')

12+

13+

14+def test_404(browser):

15+ # Visit URL

16+ browser.visit(url + 'missing_link')

17+ assert browser.is_text_present('Page not found')

18+

19+

20+def test_about(browser):

21+ browser.visit(url)

22+ browser.click_link_by_text('about')

23+ assert browser.is_text_present('Why use searx?')

24+

25+

26+def test_preferences(browser):

27+ browser.visit(url)

28+ browser.click_link_by_text('preferences')

29+ assert browser.is_text_present('Preferences')

30+ assert browser.is_text_present('Cookies')

31+

32+ assert browser.is_element_present_by_xpath('//label[@for="checkbox_dummy"]')

33+

34+

35+def test_preferences_engine_select(browser):

36+ browser.visit(url)

37+ browser.click_link_by_text('preferences')

38+

39+ assert browser.is_element_present_by_xpath('//a[@href="#tab_engine"]')

40+ browser.find_by_xpath('//a[@href="#tab_engine"]').first.click()

41+

42+ assert not browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked

43+ browser.find_by_xpath('//label[@for="engine_general_dummy__general"]').first.check()

44+ browser.find_by_xpath('//input[@value="save"]').first.click()

45+

46+ # waiting for the redirect - without this the test is flaky..

47+ sleep(1)

48+

49+ browser.visit(url)

50+ browser.click_link_by_text('preferences')

51+ browser.find_by_xpath('//a[@href="#tab_engine"]').first.click()

52+

53+ assert browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked

54+

55+

56+def test_preferences_locale(browser):

57+ browser.visit(url)

58+ browser.click_link_by_text('preferences')

59+

60+ browser.select('locale', 'hu')

61+ browser.find_by_xpath('//input[@value="save"]').first.click()

62+

63+ # waiting for the redirect - without this the test is flaky..

64+ sleep(1)

65+

66+ browser.visit(url)

67+ browser.click_link_by_text('beállítások')

68+ browser.is_text_present('Beállítások')

69+

70+

71+def test_search(browser):

72+ browser.visit(url)

73+ browser.fill('q', 'test search query')

74+ browser.find_by_xpath('//button[@type="submit"]').first.click()

75+ assert browser.is_text_present('didn\'t find any results')

tests/robot/test_basic.robotdeleted−153

@@ -1,153 +0,0 @@

1−*** Settings ***

2−Library Selenium2Library timeout=10 implicit_wait=0.5

3−Test Setup Open Browser http://localhost:11111/

4−Test Teardown Close All Browsers

5−

6−

7−*** Keywords ***

8−Submit Preferences

9− Set Selenium Speed 2 seconds

10− Submit Form id=search_form

11− Location Should Be http://localhost:11111/

12− Set Selenium Speed 0 seconds

13−

14−

15−*** Test Cases ***

16−Front page

17− Page Should Contain about

18− Page Should Contain preferences

19−

20−404 page

21− Go To http://localhost:11111/no-such-page

22− Page Should Contain Page not found

23− Page Should Contain Go to search page

24−

25−About page

26− Click Element link=about

27− Page Should Contain Why use searx?

28− Page Should Contain Element link=search engines

29−

30−Preferences page

31− Click Element link=preferences

32− Page Should Contain Preferences

33− Page Should Contain Default categories

34− Page Should Contain Currently used search engines

35− Page Should Contain dummy dummy

36− Page Should Contain general dummy

37−

38−Switch category

39− Go To http://localhost:11111/preferences

40− Page Should Contain Checkbox category_general

41− Page Should Contain Checkbox category_dummy

42− Click Element xpath=//*[.="general"]

43− Click Element xpath=//*[.="dummy"]

44− Submit Preferences

45− Checkbox Should Not Be Selected category_general

46− Checkbox Should Be Selected category_dummy

47−

48−Change language

49− Page Should Contain about

50− Page Should Contain preferences

51− Go To http://localhost:11111/preferences

52− Select From List locale hu

53− Submit Preferences

54− Page Should Contain rólunk

55− Page Should Contain beállítások

56−

57−Change method

58− Page Should Contain about

59− Page Should Contain preferences

60− Go To http://localhost:11111/preferences

61− Select From List method GET

62− Submit Preferences

63− Go To http://localhost:11111/preferences

64− List Selection Should Be method GET

65− Select From List method POST

66− Submit Preferences

67− Go To http://localhost:11111/preferences

68− List Selection Should Be method POST

69−

70−Change theme

71− Page Should Contain about

72− Page Should Contain preferences

73− Go To http://localhost:11111/preferences

74− List Selection Should Be theme legacy

75− Select From List theme oscar

76− Submit Preferences

77− Go To http://localhost:11111/preferences

78− List Selection Should Be theme oscar

79−

80−Change safesearch

81− Page Should Contain about

82− Page Should Contain preferences

83− Go To http://localhost:11111/preferences

84− List Selection Should Be safesearch None

85− Select From List safesearch Strict

86− Submit Preferences

87− Go To http://localhost:11111/preferences

88− List Selection Should Be safesearch Strict

89−

90−Change image proxy

91− Page Should Contain about

92− Page Should Contain preferences

93− Go To http://localhost:11111/preferences

94− List Selection Should Be image_proxy Disabled

95− Select From List image_proxy Enabled

96− Submit Preferences

97− Go To http://localhost:11111/preferences

98− List Selection Should Be image_proxy Enabled

99−

100−Change search language

101− Page Should Contain about

102− Page Should Contain preferences

103− Go To http://localhost:11111/preferences

104− List Selection Should Be language Default language

105− Select From List language Türkçe - tr-TR

106− Submit Preferences

107− Go To http://localhost:11111/preferences

108− List Selection Should Be language Türkçe - tr-TR

109−

110−Change autocomplete

111− Page Should Contain about

112− Page Should Contain preferences

113− Go To http://localhost:11111/preferences

114− List Selection Should Be autocomplete -

115− Select From List autocomplete google

116− Submit Preferences

117− Go To http://localhost:11111/preferences

118− List Selection Should Be autocomplete google

119−

120−Change allowed/disabled engines

121− Page Should Contain about

122− Page Should Contain preferences

123− Go To http://localhost:11111/preferences

124− Page Should Contain Engine name

125− Element Should Contain xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy'] Block

126− Element Should Contain xpath=//label[@class="deny"][@for='engine_general_general_dummy'] Block

127− Click Element xpath=//label[@class="deny"][@for='engine_general_general_dummy']

128− Submit Preferences

129− Page Should Contain about

130− Page Should Contain preferences

131− Go To http://localhost:11111/preferences

132− Page Should Contain Engine name

133− Element Should Contain xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy'] Block

134− Element Should Contain xpath=//label[@class="deny"][@for='engine_general_general_dummy'] \

135−

136−Block a plugin

137− Page Should Contain about

138− Page Should Contain preferences

139− Go To http://localhost:11111/preferences

140− List Selection Should Be theme legacy

141− Select From List theme oscar

142− Submit Preferences

143− Go To http://localhost:11111/preferences

144− List Selection Should Be theme oscar

145− Page Should Contain Plugins

146− Click Link Plugins

147− Checkbox Should Not Be Selected id=plugin_HTTPS_rewrite

148− Click Element xpath=//label[@for='plugin_HTTPS_rewrite']

149− Submit Preferences

150− Go To http://localhost:11111/preferences

151− Page Should Contain Plugins

152− Click Link Plugins

153− Checkbox Should Be Selected id=plugin_HTTPS_rewrite

tests/unit/engines/test_archlinux.py+2−2

@@ -25,7 +25,7 @@ class TestArchLinuxEngine(SearxTestCase):

25 self.assertTrue(query in params['url'])

26 self.assertTrue('wiki.archlinux.org' in params['url'])

27

28− for lang, domain in domains.iteritems():

28+ for lang, domain in domains.items():

29 dic['language'] = lang

30 params = archlinux.request(query, dic)

31 self.assertTrue(domain in params['url'])

@@ -102,5 +102,5 @@ class TestArchLinuxEngine(SearxTestCase):

102 for exp in expected:

103 res = results[i]

104 i += 1

105− for key, value in exp.iteritems():

105+ for key, value in exp.items():

106 self.assertEqual(res[key], value)

tests/unit/engines/test_bing.py+3−3

@@ -7,18 +7,18 @@ from searx.testing import SearxTestCase

7class TestBingEngine(SearxTestCase):

8

9 def test_request(self):

10− query = 'test_query'

10+ query = u'test_query'

11 dicto = defaultdict(dict)

12 dicto['pageno'] = 0

13 dicto['language'] = 'fr_FR'

14− params = bing.request(query, dicto)

14+ params = bing.request(query.encode('utf-8'), dicto)

15 self.assertTrue('url' in params)

16 self.assertTrue(query in params['url'])

17 self.assertTrue('language%3AFR' in params['url'])

18 self.assertTrue('bing.com' in params['url'])

19

20 dicto['language'] = 'all'

21− params = bing.request(query, dicto)

21+ params = bing.request(query.encode('utf-8'), dicto)

22 self.assertTrue('language' in params['url'])

23

24 def test_response(self):

tests/unit/engines/test_bing_news.py+6−6

@@ -36,10 +36,10 @@ class TestBingNewsEngine(SearxTestCase):

36 self.assertRaises(AttributeError, bing_news.response, '')

37 self.assertRaises(AttributeError, bing_news.response, '[]')

38

39− response = mock.Mock(content='<html></html>')

39+ response = mock.Mock(text='<html></html>')

40 self.assertEqual(bing_news.response(response), [])

41

42− response = mock.Mock(content='<html></html>')

42+ response = mock.Mock(text='<html></html>')

43 self.assertEqual(bing_news.response(response), [])

44

45 html = """<?xml version="1.0" encoding="utf-8" ?>

@@ -74,7 +74,7 @@ class TestBingNewsEngine(SearxTestCase):

74 </item>

75 </channel>

76</rss>""" # noqa

77− response = mock.Mock(content=html)

77+ response = mock.Mock(text=html.encode('utf-8'))

78 results = bing_news.response(response)

79 self.assertEqual(type(results), list)

80 self.assertEqual(len(results), 2)

@@ -113,7 +113,7 @@ class TestBingNewsEngine(SearxTestCase):

113 </item>

114 </channel>

115</rss>""" # noqa

116− response = mock.Mock(content=html)

116+ response = mock.Mock(text=html.encode('utf-8'))

117 results = bing_news.response(response)

118 self.assertEqual(type(results), list)

119 self.assertEqual(len(results), 1)

@@ -136,11 +136,11 @@ class TestBingNewsEngine(SearxTestCase):

136 </channel>

137</rss>""" # noqa

138

139− response = mock.Mock(content=html)

139+ response = mock.Mock(text=html.encode('utf-8'))

140 results = bing_news.response(response)

141 self.assertEqual(type(results), list)

142 self.assertEqual(len(results), 0)

143

144 html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""

145− response = mock.Mock(content=html)

145+ response = mock.Mock(text=html.encode('utf-8'))

146 self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)

tests/unit/engines/test_btdigg.py+6−6

@@ -22,10 +22,10 @@ class TestBtdiggEngine(SearxTestCase):

22 self.assertRaises(AttributeError, btdigg.response, '')

23 self.assertRaises(AttributeError, btdigg.response, '[]')

24

25− response = mock.Mock(content='<html></html>')

25+ response = mock.Mock(text='<html></html>')

26 self.assertEqual(btdigg.response(response), [])

27

28− html = """

28+ html = u"""

29 <div id="search_res">

30 <table>

31 <tr>

@@ -82,7 +82,7 @@ class TestBtdiggEngine(SearxTestCase):

82 </table>

83 </div>

84 """

85− response = mock.Mock(content=html)

85+ response = mock.Mock(text=html.encode('utf-8'))

86 results = btdigg.response(response)

87 self.assertEqual(type(results), list)

88 self.assertEqual(len(results), 1)

@@ -101,12 +101,12 @@ class TestBtdiggEngine(SearxTestCase):

101 </table>

102 </div>

103 """

104− response = mock.Mock(content=html)

104+ response = mock.Mock(text=html.encode('utf-8'))

105 results = btdigg.response(response)

106 self.assertEqual(type(results), list)

107 self.assertEqual(len(results), 0)

108

109− html = """

109+ html = u"""

110 <div id="search_res">

111 <table>

112 <tr>

@@ -367,7 +367,7 @@ class TestBtdiggEngine(SearxTestCase):

367 </table>

368 </div>

369 """

370− response = mock.Mock(content=html)

370+ response = mock.Mock(text=html.encode('utf-8'))

371 results = btdigg.response(response)

372 self.assertEqual(type(results), list)

373 self.assertEqual(len(results), 5)

tests/unit/engines/test_currency_convert.py+2−2

@@ -8,13 +8,13 @@ from searx.testing import SearxTestCase

8class TestCurrencyConvertEngine(SearxTestCase):

9

10 def test_request(self):

11− query = 'test_query'

11+ query = b'test_query'

12 dicto = defaultdict(dict)

13 dicto['pageno'] = 1

14 params = currency_convert.request(query, dicto)

15 self.assertNotIn('url', params)

16

17− query = 'convert 10 Pound Sterlings to United States Dollars'

17+ query = b'convert 10 Pound Sterlings to United States Dollars'

18 params = currency_convert.request(query, dicto)

19 self.assertIn('url', params)

20 self.assertIn('finance.yahoo.com', params['url'])

tests/unit/engines/test_digbt.py+2−2

@@ -21,7 +21,7 @@ class TestDigBTEngine(SearxTestCase):

21 self.assertRaises(AttributeError, digbt.response, '')

22 self.assertRaises(AttributeError, digbt.response, '[]')

23

24− response = mock.Mock(content='<html></html>')

24+ response = mock.Mock(text='<html></html>')

25 self.assertEqual(digbt.response(response), [])

26

27 html = """

@@ -50,7 +50,7 @@ class TestDigBTEngine(SearxTestCase):

50 </td></tr>

51 </table>

52 """

53− response = mock.Mock(content=html)

53+ response = mock.Mock(text=html.encode('utf-8'))

54 results = digbt.response(response)

55 self.assertEqual(type(results), list)

56 self.assertEqual(len(results), 1)

tests/unit/engines/test_duckduckgo.py+1−2

@@ -90,8 +90,7 @@ class TestDuckduckgoEngine(SearxTestCase):

90 "wt-wt":"All Results","ar-es":"Argentina","au-en":"Australia","at-de":"Austria","be-fr":"Belgium (fr)"

91 }some more code..."""

92 response = mock.Mock(text=js)

93− languages = duckduckgo._fetch_supported_languages(response)

93+ languages = list(duckduckgo._fetch_supported_languages(response))

94− self.assertEqual(type(languages), list)

95 self.assertEqual(len(languages), 5)

94 self.assertEqual(len(languages), 5)

96 self.assertIn('wt-WT', languages)

95 self.assertIn('wt-WT', languages)

97 self.assertIn('es-AR', languages)

96 self.assertIn('es-AR', languages)

tests/unit/engines/test_frinkiac.py+3−2

@@ -1,7 +1,6 @@

1# -*- coding: utf-8 -*-

2from collections import defaultdict

3import mock

4−from json import dumps

5from searx.engines import frinkiac

4from searx.engines import frinkiac

6from searx.testing import SearxTestCase

5from searx.testing import SearxTestCase

7

6

@@ -44,6 +43,8 @@ class TestFrinkiacEngine(SearxTestCase):

44 self.assertEqual(type(results), list)

43 self.assertEqual(type(results), list)

45 self.assertEqual(len(results), 4)

44 self.assertEqual(len(results), 4)

46 self.assertEqual(results[0]['title'], u'S06E18')

45 self.assertEqual(results[0]['title'], u'S06E18')

47− self.assertEqual(results[0]['url'], 'https://frinkiac.com/?p=caption&e=S06E18&t=534616')

46+ self.assertIn('p=caption', results[0]['url'])

47+ self.assertIn('e=S06E18', results[0]['url'])

48+ self.assertIn('t=534616', results[0]['url'])

48 self.assertEqual(results[0]['thumbnail_src'], 'https://frinkiac.com/img/S06E18/534616/medium.jpg')

49 self.assertEqual(results[0]['thumbnail_src'], 'https://frinkiac.com/img/S06E18/534616/medium.jpg')

49 self.assertEqual(results[0]['img_src'], 'https://frinkiac.com/img/S06E18/534616.jpg')

50 self.assertEqual(results[0]['img_src'], 'https://frinkiac.com/img/S06E18/534616.jpg')

tests/unit/engines/test_gigablast.py+1

@@ -10,6 +10,7 @@ class TestGigablastEngine(SearxTestCase):

10 query = 'test_query'

11 dicto = defaultdict(dict)

12 dicto['pageno'] = 0

13+ dicto['safesearch'] = 0

13 dicto['language'] = 'all'

14 dicto['language'] = 'all'

14 params = gigablast.request(query, dicto)

15 params = gigablast.request(query, dicto)

15 self.assertTrue('url' in params)

16 self.assertTrue('url' in params)

tests/unit/engines/test_soundcloud.py+1−1

@@ -2,7 +2,7 @@ from collections import defaultdict

2import mock

3from searx.engines import soundcloud

4from searx.testing import SearxTestCase

5−from urllib import quote_plus

5+from searx.url_utils import quote_plus

6

7

8class TestSoundcloudEngine(SearxTestCase):

tests/unit/engines/test_startpage.py+3−3

@@ -31,7 +31,7 @@ class TestStartpageEngine(SearxTestCase):

31 self.assertRaises(AttributeError, startpage.response, '')

32 self.assertRaises(AttributeError, startpage.response, '[]')

33

34− response = mock.Mock(content='<html></html>')

34+ response = mock.Mock(text='<html></html>')

35 self.assertEqual(startpage.response(response), [])

36

37 html = """

@@ -62,7 +62,7 @@ class TestStartpageEngine(SearxTestCase):

62 </p>

63 </div>

64 """

65− response = mock.Mock(content=html)

65+ response = mock.Mock(text=html.encode('utf-8'))

66 results = startpage.response(response)

67 self.assertEqual(type(results), list)

68 self.assertEqual(len(results), 1)

@@ -133,7 +133,7 @@ class TestStartpageEngine(SearxTestCase):

133 </p>

134 </div>

135 """

136− response = mock.Mock(content=html)

136+ response = mock.Mock(text=html.encode('utf-8'))

137 results = startpage.response(response)

138 self.assertEqual(type(results), list)

139 self.assertEqual(len(results), 1)

tests/unit/engines/test_swisscows.py+4−4

@@ -33,13 +33,13 @@ class TestSwisscowsEngine(SearxTestCase):

33 self.assertRaises(AttributeError, swisscows.response, '')

34 self.assertRaises(AttributeError, swisscows.response, '[]')

35

36− response = mock.Mock(content='<html></html>')

36+ response = mock.Mock(text=b'<html></html>')

37 self.assertEqual(swisscows.response(response), [])

38

39− response = mock.Mock(content='<html></html>')

39+ response = mock.Mock(text=b'<html></html>')

40 self.assertEqual(swisscows.response(response), [])

41

42− html = u"""

42+ html = b"""

43 <script>

44 App.Dispatcher.dispatch("initialize", {

45 html5history: true,

@@ -111,7 +111,7 @@ class TestSwisscowsEngine(SearxTestCase):

111 });

112 </script>

113 """

114− response = mock.Mock(content=html)

114+ response = mock.Mock(text=html)

115 results = swisscows.response(response)

116 self.assertEqual(type(results), list)

117 self.assertEqual(len(results), 3)

tests/unit/engines/test_tokyotoshokan.py+1−1

@@ -91,7 +91,7 @@ class TestTokyotoshokanEngine(SearxTestCase):

91 self.assertEqual(r['title'], 'Koyomimonogatari')

92 self.assertEqual(r['magnetlink'], 'magnet:?xt=urn:btih:4c19eb46b5113685fbd2288ed2531b0b')

93 self.assertEqual(r['filesize'], int(1024 * 1024 * 10.5))

94− self.assertEqual(r['publishedDate'], datetime(2016, 03, 26, 16, 41))

94+ self.assertEqual(r['publishedDate'], datetime(2016, 3, 26, 16, 41))

95 self.assertEqual(r['content'], 'Comment: sample comment')

96 self.assertEqual(r['seed'], 53)

97 self.assertEqual(r['leech'], 18)

tests/unit/engines/test_wikidata.py+1−2

@@ -1,5 +1,4 @@

1# -*- coding: utf-8 -*-

2−from json import loads

3from lxml.html import fromstring

2from lxml.html import fromstring

4from collections import defaultdict

3from collections import defaultdict

5import mock

4import mock

@@ -31,7 +30,7 @@ class TestWikidataEngine(SearxTestCase):

31 self.assertRaises(AttributeError, wikidata.response, '')

30 self.assertRaises(AttributeError, wikidata.response, '')

32 self.assertRaises(AttributeError, wikidata.response, '[]')

31 self.assertRaises(AttributeError, wikidata.response, '[]')

33

32

34− response = mock.Mock(content='<html></html>', search_params={"language": "all"})

33+ response = mock.Mock(text='<html></html>', search_params={"language": "all"})

35 self.assertEqual(wikidata.response(response), [])

34 self.assertEqual(wikidata.response(response), [])

36

35

37 def test_getDetail(self):

36 def test_getDetail(self):

tests/unit/engines/test_wikipedia.py+9−9

@@ -13,15 +13,15 @@ class TestWikipediaEngine(SearxTestCase):

13 query = 'test_query'

14 dicto = defaultdict(dict)

15 dicto['language'] = 'fr-FR'

16− params = wikipedia.request(query, dicto)

16+ params = wikipedia.request(query.encode('utf-8'), dicto)

17 self.assertIn('url', params)

18 self.assertIn(query, params['url'])

19 self.assertIn('test_query', params['url'])

20 self.assertIn('Test_Query', params['url'])

21 self.assertIn('fr.wikipedia.org', params['url'])

22

23− query = 'Test_Query'

23+ query = u'Test_Query'

24− params = wikipedia.request(query, dicto)

24+ params = wikipedia.request(query.encode('utf-8'), dicto)

25 self.assertIn('Test_Query', params['url'])

26 self.assertNotIn('test_query', params['url'])

27

@@ -57,7 +57,7 @@ class TestWikipediaEngine(SearxTestCase):

57 }

58 }

59 }"""

60− response = mock.Mock(content=json, search_params=dicto)

60+ response = mock.Mock(text=json, search_params=dicto)

61 self.assertEqual(wikipedia.response(response), [])

62

63 # normal case

@@ -80,7 +80,7 @@ class TestWikipediaEngine(SearxTestCase):

80 }

81 }

82 }"""

83− response = mock.Mock(content=json, search_params=dicto)

83+ response = mock.Mock(text=json, search_params=dicto)

84 results = wikipedia.response(response)

85 self.assertEqual(type(results), list)

86 self.assertEqual(len(results), 2)

@@ -108,10 +108,10 @@ class TestWikipediaEngine(SearxTestCase):

108 }

109 }

110 }"""

111− response = mock.Mock(content=json, search_params=dicto)

111+ response = mock.Mock(text=json, search_params=dicto)

112 results = wikipedia.response(response)

113 self.assertEqual(type(results), list)

114− self.assertEqual(len(results), 0)

114+ self.assertEqual(len(results), 2)

115

116 # no image

117 json = """

@@ -130,7 +130,7 @@ class TestWikipediaEngine(SearxTestCase):

130 }

131 }

132 }"""

133− response = mock.Mock(content=json, search_params=dicto)

133+ response = mock.Mock(text=json, search_params=dicto)

134 results = wikipedia.response(response)

135 self.assertEqual(type(results), list)

136 self.assertEqual(len(results), 2)

@@ -158,7 +158,7 @@ class TestWikipediaEngine(SearxTestCase):

158 }

159 }

160 }"""

161− response = mock.Mock(content=json, search_params=dicto)

161+ response = mock.Mock(text=json, search_params=dicto)

162 results = wikipedia.response(response)

163 self.assertEqual(type(results), list)

164 self.assertEqual(len(results), 2)

tests/unit/engines/test_wolframalpha_api.py+5−5

@@ -35,11 +35,11 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

35 xml = '''<?xml version='1.0' encoding='UTF-8'?>

36 <queryresult success='false' error='false' />

37 '''

38− response = mock.Mock(content=xml)

38+ response = mock.Mock(text=xml.encode('utf-8'))

39 self.assertEqual(wolframalpha_api.response(response), [])

40

41 # test basic case

42− xml = """<?xml version='1.0' encoding='UTF-8'?>

42+ xml = b"""<?xml version='1.0' encoding='UTF-8'?>

43 <queryresult success='true'

44 error='false'

45 numpods='3'

@@ -83,7 +83,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

83 </pod>

84 </queryresult>

85 """

86− response = mock.Mock(content=xml, request=request)

86+ response = mock.Mock(text=xml, request=request)

87 results = wolframalpha_api.response(response)

88 self.assertEqual(type(results), list)

89 self.assertEqual(len(results), 2)

@@ -107,7 +107,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

107 self.assertIn('result_plaintext', results[1]['content'])

108

109 # test calc

110− xml = """<?xml version='1.0' encoding='UTF-8'?>

110+ xml = b"""<?xml version='1.0' encoding='UTF-8'?>

111 <queryresult success='true'

112 error='false'

113 numpods='2'

@@ -144,7 +144,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

144 </pod>

145 </queryresult>

146 """

147− response = mock.Mock(content=xml, request=request)

147+ response = mock.Mock(text=xml, request=request)

148 results = wolframalpha_api.response(response)

149 self.assertEqual(type(results), list)

150 self.assertEqual(len(results), 2)

tests/unit/test_plugins.py+8−8

@@ -48,11 +48,11 @@ class SelfIPTest(SearxTestCase):

48 # IP test

49 request = Mock(remote_addr='127.0.0.1')

50 request.headers.getlist.return_value = []

51− search = get_search_mock(query='ip', pageno=1)

51+ search = get_search_mock(query=b'ip', pageno=1)

52 store.call(store.plugins, 'post_search', request, search)

53 self.assertTrue('127.0.0.1' in search.result_container.answers)

54

55− search = get_search_mock(query='ip', pageno=2)

55+ search = get_search_mock(query=b'ip', pageno=2)

56 store.call(store.plugins, 'post_search', request, search)

57 self.assertFalse('127.0.0.1' in search.result_container.answers)

58

@@ -60,26 +60,26 @@ class SelfIPTest(SearxTestCase):

60 request = Mock(user_agent='Mock')

61 request.headers.getlist.return_value = []

62

63− search = get_search_mock(query='user-agent', pageno=1)

63+ search = get_search_mock(query=b'user-agent', pageno=1)

64 store.call(store.plugins, 'post_search', request, search)

65 self.assertTrue('Mock' in search.result_container.answers)

66

67− search = get_search_mock(query='user-agent', pageno=2)

67+ search = get_search_mock(query=b'user-agent', pageno=2)

68 store.call(store.plugins, 'post_search', request, search)

69 self.assertFalse('Mock' in search.result_container.answers)

70

71− search = get_search_mock(query='user-agent', pageno=1)

71+ search = get_search_mock(query=b'user-agent', pageno=1)

72 store.call(store.plugins, 'post_search', request, search)

73 self.assertTrue('Mock' in search.result_container.answers)

74

75− search = get_search_mock(query='user-agent', pageno=2)

75+ search = get_search_mock(query=b'user-agent', pageno=2)

76 store.call(store.plugins, 'post_search', request, search)

77 self.assertFalse('Mock' in search.result_container.answers)

78

79− search = get_search_mock(query='What is my User-Agent?', pageno=1)

79+ search = get_search_mock(query=b'What is my User-Agent?', pageno=1)

80 store.call(store.plugins, 'post_search', request, search)

81 self.assertTrue('Mock' in search.result_container.answers)

82

83− search = get_search_mock(query='What is my User-Agent?', pageno=2)

83+ search = get_search_mock(query=b'What is my User-Agent?', pageno=2)

84 store.call(store.plugins, 'post_search', request, search)

85 self.assertFalse('Mock' in search.result_container.answers)

tests/unit/test_utils.py+6−2

@@ -1,8 +1,12 @@

1# -*- coding: utf-8 -*-

2import mock

3+import sys

3from searx.testing import SearxTestCase

4from searx.testing import SearxTestCase

4from searx import utils

5from searx import utils

5

6

7+if sys.version_info[0] == 3:

8+ unicode = str

9+

6

10

7class TestUtils(SearxTestCase):

11class TestUtils(SearxTestCase):

8

12

@@ -30,9 +34,9 @@ class TestUtils(SearxTestCase):

30 self.assertEqual(utils.highlight_content(content, None), content)

34 self.assertEqual(utils.highlight_content(content, None), content)

31

35

32 content = 'a'

36 content = 'a'

33− query = 'test'

37+ query = b'test'

34 self.assertEqual(utils.highlight_content(content, query), content)

38 self.assertEqual(utils.highlight_content(content, query), content)

35− query = 'a test'

39+ query = b'a test'

36 self.assertEqual(utils.highlight_content(content, query), content)

40 self.assertEqual(utils.highlight_content(content, query), content)

37

41

38 def test_html_to_text(self):

42 def test_html_to_text(self):

tests/unit/test_webapp.py+22−24

@@ -2,10 +2,10 @@

2

3import json

4from mock import Mock

5−from urlparse import ParseResult

6from searx import webapp

5from searx import webapp

7from searx.testing import SearxTestCase

6from searx.testing import SearxTestCase

8from searx.search import Search

7from searx.search import Search

8+from searx.url_utils import ParseResult

9

10

11class ViewsTestCase(SearxTestCase):

@@ -57,37 +57,35 @@ class ViewsTestCase(SearxTestCase):

57 def test_index_empty(self):

58 result = self.app.post('/')

59 self.assertEqual(result.status_code, 200)

60− self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)

60+ self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)

61

62 def test_index_html(self):

63 result = self.app.post('/', data={'q': 'test'})

64 self.assertIn(

65− '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>', # noqa

65+ b'<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>', # noqa

66 result.data

67 )

68 self.assertIn(

69− '<p class="content">first <span class="highlight">test</span> content<br class="last"/></p>', # noqa

69+ b'<p class="content">first <span class="highlight">test</span> content<br class="last"/></p>', # noqa

70 result.data

71 )

72

73 def test_index_json(self):

74 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

75

76− result_dict = json.loads(result.data)

76+ result_dict = json.loads(result.data.decode('utf-8'))

77

78 self.assertEqual('test', result_dict['query'])

79− self.assertEqual(

80− result_dict['results'][0]['content'], 'first test content')

79+ self.assertEqual(result_dict['results'][0]['content'], 'first test content')

81− self.assertEqual(

82− result_dict['results'][0]['url'], 'http://first.test.xyz')

80+ self.assertEqual(result_dict['results'][0]['url'], 'http://first.test.xyz')

83

81

84 def test_index_csv(self):

82 def test_index_csv(self):

85 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

83 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

86

84

87 self.assertEqual(

85 self.assertEqual(

88− 'title,url,content,host,engine,score\r\n'

86+ b'title,url,content,host,engine,score\r\n'

89− 'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,\r\n' # noqa

87+ b'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,\r\n' # noqa

90− 'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,\r\n', # noqa

88+ b'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,\r\n', # noqa

91 result.data

89 result.data

92 )

90 )

93

91

@@ -95,65 +93,65 @@ class ViewsTestCase(SearxTestCase):

95 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

93 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

96

94

97 self.assertIn(

95 self.assertIn(

98− '<description>Search results for "test" - searx</description>',

96+ b'<description>Search results for "test" - searx</description>',

99 result.data

97 result.data

100 )

98 )

101

99

102 self.assertIn(

100 self.assertIn(

103− '<opensearch:totalResults>3</opensearch:totalResults>',

101+ b'<opensearch:totalResults>3</opensearch:totalResults>',

104 result.data

102 result.data

105 )

103 )

106

104

107 self.assertIn(

105 self.assertIn(

108− '<title>First Test</title>',

106+ b'<title>First Test</title>',

109 result.data

107 result.data

110 )

108 )

111

109

112 self.assertIn(

110 self.assertIn(

113− '<link>http://first.test.xyz</link>',

111+ b'<link>http://first.test.xyz</link>',

114 result.data

112 result.data

115 )

113 )

116

114

117 self.assertIn(

115 self.assertIn(

118− '<description>first test content</description>',

116+ b'<description>first test content</description>',

119 result.data

117 result.data

120 )

118 )

121

119

122 def test_about(self):

120 def test_about(self):

123 result = self.app.get('/about')

121 result = self.app.get('/about')

124 self.assertEqual(result.status_code, 200)

122 self.assertEqual(result.status_code, 200)

125− self.assertIn('<h1>About <a href="/">searx</a></h1>', result.data)

123+ self.assertIn(b'<h1>About <a href="/">searx</a></h1>', result.data)

126

124

127 def test_preferences(self):

125 def test_preferences(self):

128 result = self.app.get('/preferences')

126 result = self.app.get('/preferences')

129 self.assertEqual(result.status_code, 200)

127 self.assertEqual(result.status_code, 200)

130 self.assertIn(

128 self.assertIn(

131− '<form method="post" action="/preferences" id="search_form">',

129+ b'<form method="post" action="/preferences" id="search_form">',

132 result.data

130 result.data

133 )

131 )

134 self.assertIn(

132 self.assertIn(

135− '<legend>Default categories</legend>',

133+ b'<legend>Default categories</legend>',

136 result.data

134 result.data

137 )

135 )

138 self.assertIn(

136 self.assertIn(

139− '<legend>Interface language</legend>',

137+ b'<legend>Interface language</legend>',

140 result.data

138 result.data

141 )

139 )

142

140

143 def test_stats(self):

141 def test_stats(self):

144 result = self.app.get('/stats')

142 result = self.app.get('/stats')

145 self.assertEqual(result.status_code, 200)

143 self.assertEqual(result.status_code, 200)

146− self.assertIn('<h2>Engine stats</h2>', result.data)

144+ self.assertIn(b'<h2>Engine stats</h2>', result.data)

147

145

148 def test_robots_txt(self):

146 def test_robots_txt(self):

149 result = self.app.get('/robots.txt')

147 result = self.app.get('/robots.txt')

150 self.assertEqual(result.status_code, 200)

148 self.assertEqual(result.status_code, 200)

151− self.assertIn('Allow: /', result.data)

149+ self.assertIn(b'Allow: /', result.data)

152

150

153 def test_opensearch_xml(self):

151 def test_opensearch_xml(self):

154 result = self.app.get('/opensearch.xml')

152 result = self.app.get('/opensearch.xml')

155 self.assertEqual(result.status_code, 200)

153 self.assertEqual(result.status_code, 200)

156− self.assertIn('<Description>a privacy-respecting, hackable metasearch engine</Description>', result.data)

154+ self.assertIn(b'<Description>a privacy-respecting, hackable metasearch engine</Description>', result.data)

157

155

158 def test_favicon(self):

156 def test_favicon(self):

159 result = self.app.get('/favicon.ico')

157 result = self.app.get('/favicon.ico')