git.iko.soy/searx

on master help sign in

.travis.yml+3−2

@@ -9,6 +9,7 @@ addons:

99language: python

1010python:

1111 - "2.7"

12+ - "3.6"

1213before_install:

1314 - "export DISPLAY=:99.0"

1415 - "sh -e /etc/init.d/xvfb start"

@@ -24,9 +25,9 @@ script:

2425 - ./manage.sh styles

2526 - ./manage.sh grunt_build

2627 - ./manage.sh tests

27− - ./manage.sh py_test_coverage

2828after_success:

29− coveralls

29+ - ./manage.sh py_test_coverage

30+ - coveralls

3031notifications:

3132 irc:

3233 channels:

requirements-dev.txt+1−2

@@ -3,8 +3,7 @@ mock==2.0.0

33nose2[coverage-plugin]

44pep8==1.7.0

55plone.testing==5.0.0

6−robotframework-selenium2library==1.8.0

7−robotsuite==1.7.0

6+splinter==0.7.5

87transifex-client==0.12.2

98unittest2==1.1.0

109zope.testrunner==4.5.1

searx/answerers/__init__.py+8−4

@@ -1,8 +1,12 @@

11from os import listdir

22from os.path import realpath, dirname, join, isdir

3+from sys import version_info

34from searx.utils import load_module

45from collections import defaultdict

56

7+if version_info[0] == 3:

8+ unicode = str

9+

610

711answerers_dir = dirname(realpath(__file__))

812

@@ -10,7 +14,7 @@ answerers_dir = dirname(realpath(__file__))

1014def load_answerers():

1115 answerers = []

1216 for filename in listdir(answerers_dir):

13− if not isdir(join(answerers_dir, filename)):

17+ if not isdir(join(answerers_dir, filename)) or filename.startswith('_'):

1418 continue

1519 module = load_module('answerer.py', join(answerers_dir, filename))

1620 if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):

@@ -30,12 +34,12 @@ def get_answerers_by_keywords(answerers):

3034

3135def ask(query):

3236 results = []

33− query_parts = filter(None, query.query.split())

37+ query_parts = list(filter(None, query.query.split()))

3438

35− if query_parts[0] not in answerers_by_keywords:

39+ if query_parts[0].decode('utf-8') not in answerers_by_keywords:

3640 return results

3741

38− for answerer in answerers_by_keywords[query_parts[0]]:

42+ for answerer in answerers_by_keywords[query_parts[0].decode('utf-8')]:

3943 result = answerer(query)

4044 if result:

4145 results.append(result)

searx/answerers/random/answerer.py+9−4

@@ -1,5 +1,6 @@

11import random

22import string

3+import sys

34from flask_babel import gettext

45

56# required answerer attribute

@@ -8,7 +9,11 @@ keywords = ('random',)

89

910random_int_max = 2**31

1011

11−random_string_letters = string.lowercase + string.digits + string.uppercase

12+if sys.version_info[0] == 2:

13+ random_string_letters = string.lowercase + string.digits + string.uppercase

14+else:

15+ unicode = str

16+ random_string_letters = string.ascii_lowercase + string.digits + string.ascii_uppercase

1217

1318

1419def random_string():

@@ -24,9 +29,9 @@ def random_int():

2429 return unicode(random.randint(-random_int_max, random_int_max))

2530

2631

27−random_types = {u'string': random_string,

28− u'int': random_int,

29− u'float': random_float}

32+random_types = {b'string': random_string,

33+ b'int': random_int,

34+ b'float': random_float}

3035

3136

3237# required answerer function

searx/answerers/statistics/answerer.py+10−6

@@ -1,8 +1,12 @@

1+from sys import version_info

12from functools import reduce

23from operator import mul

34

45from flask_babel import gettext

56

7+if version_info[0] == 3:

8+ unicode = str

9+

610keywords = ('min',

711 'max',

812 'avg',

@@ -19,22 +23,22 @@ def answer(query):

1923 return []

2024

2125 try:

22− args = map(float, parts[1:])

26+ args = list(map(float, parts[1:]))

2327 except:

2428 return []

2529

2630 func = parts[0]

2731 answer = None

2832

29− if func == 'min':

33+ if func == b'min':

3034 answer = min(args)

31− elif func == 'max':

35+ elif func == b'max':

3236 answer = max(args)

33− elif func == 'avg':

37+ elif func == b'avg':

3438 answer = sum(args) / len(args)

35− elif func == 'sum':

39+ elif func == b'sum':

3640 answer = sum(args)

37− elif func == 'prod':

41+ elif func == b'prod':

3842 answer = reduce(mul, args, 1)

3943

4044 if answer is None:

searx/autocomplete.py+5−1

@@ -18,7 +18,6 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1818

1919from lxml import etree

2020from json import loads

21−from urllib import urlencode

2221from searx import settings

2322from searx.languages import language_codes

2423from searx.engines import (

@@ -26,6 +25,11 @@ from searx.engines import (

2625)

2726from searx.poolrequests import get as http_get

2827

28+try:

29+ from urllib import urlencode

30+except:

31+ from urllib.parse import urlencode

32+

2933

3034def get(*args, **kwargs):

3135 if 'timeout' not in kwargs:

searx/engines/1337x.py+1−2

@@ -1,8 +1,7 @@

1−from urllib import quote

21from lxml import html

32from searx.engines.xpath import extract_text

43from searx.utils import get_torrent_size

5−from urlparse import urljoin

4+from searx.url_utils import quote, urljoin

65

76url = 'https://1337x.to/'

87search_url = url + 'search/{search_term}/{pageno}/'

searx/engines/__init__.py+2−3

@@ -72,12 +72,11 @@ def load_engine(engine_data):

7272 if engine_data['categories'] == 'none':

7373 engine.categories = []

7474 else:

75− engine.categories = map(

76− str.strip, engine_data['categories'].split(','))

75+ engine.categories = list(map(str.strip, engine_data['categories'].split(',')))

7776 continue

7877 setattr(engine, param_name, engine_data[param_name])

7978

80− for arg_name, arg_value in engine_default_args.iteritems():

79+ for arg_name, arg_value in engine_default_args.items():

8180 if not hasattr(engine, arg_name):

8281 setattr(engine, arg_name, arg_value)

8382

searx/engines/archlinux.py+1−2

@@ -11,10 +11,9 @@

1111 @parse url, title

1212"""

1313

14−from urlparse import urljoin

15−from urllib import urlencode

1614from lxml import html

1715from searx.engines.xpath import extract_text

16+from searx.url_utils import urlencode, urljoin

1817

1918# engine dependent config

2019categories = ['it']

searx/engines/base.py+3−3

@@ -14,10 +14,10 @@

1414"""

1515

1616from lxml import etree

17−from urllib import urlencode

18−from searx.utils import searx_useragent

1917from datetime import datetime

2018import re

19+from searx.url_utils import urlencode

20+from searx.utils import searx_useragent

2121

2222

2323categories = ['science']

@@ -73,7 +73,7 @@ def request(query, params):

7373def response(resp):

7474 results = []

7575

76− search_results = etree.XML(resp.content)

76+ search_results = etree.XML(resp.text)

7777

7878 for entry in search_results.xpath('./result/doc'):

7979 content = "No description available"

searx/engines/bing.py+1−1

@@ -13,9 +13,9 @@

1313 @todo publishedDate

1414"""

1515

16−from urllib import urlencode

1716from lxml import html

1817from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

1919

2020# engine dependent config

2121categories = ['general']

searx/engines/bing_images.py+1−1

@@ -15,11 +15,11 @@

1515 limited response to 10 images

1616"""

1717

18−from urllib import urlencode

1918from lxml import html

2019from json import loads

2120import re

2221from searx.engines.bing import _fetch_supported_languages, supported_languages_url

22+from searx.url_utils import urlencode

2323

2424# engine dependent config

2525categories = ['images']

searx/engines/bing_news.py+2−3

@@ -11,13 +11,12 @@

1111 @parse url, title, content, publishedDate, thumbnail

1212"""

1313

14−from urllib import urlencode

15−from urlparse import urlparse, parse_qsl

1614from datetime import datetime

1715from dateutil import parser

1816from lxml import etree

1917from searx.utils import list_get

2018from searx.engines.bing import _fetch_supported_languages, supported_languages_url

19+from searx.url_utils import urlencode, urlparse, parse_qsl

2120

2221# engine dependent config

2322categories = ['news']

@@ -86,7 +85,7 @@ def request(query, params):

8685def response(resp):

8786 results = []

8887

89− rss = etree.fromstring(resp.content)

88+ rss = etree.fromstring(resp.text)

9089

9190 ns = rss.nsmap

9291

searx/engines/blekko_images.py+1−1

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['images']

searx/engines/btdigg.py+2−3

@@ -10,11 +10,10 @@

1010 @parse url, title, content, seed, leech, magnetlink

1111"""

1212

13−from urlparse import urljoin

14−from urllib import quote

1513from lxml import html

1614from operator import itemgetter

1715from searx.engines.xpath import extract_text

16+from searx.url_utils import quote, urljoin

1817from searx.utils import get_torrent_size

1918

2019# engine dependent config

@@ -38,7 +37,7 @@ def request(query, params):

3837def response(resp):

3938 results = []

4039

41− dom = html.fromstring(resp.content)

40+ dom = html.fromstring(resp.text)

4241

4342 search_res = dom.xpath('//div[@id="search_res"]/table/tr')

4443

searx/engines/currency_convert.py+9−5

@@ -1,21 +1,25 @@

1−from datetime import datetime

1+import json

22import re

33import os

4−import json

4+import sys

55import unicodedata

66

7+from datetime import datetime

8+

9+if sys.version_info[0] == 3:

10+ unicode = str

711

812categories = []

913url = 'https://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s={query}=X'

1014weight = 100

1115

12−parser_re = re.compile(u'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I) # noqa

16+parser_re = re.compile(b'.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

1317

1418db = 1

1519

1620

1721def normalize_name(name):

18− name = name.lower().replace('-', ' ').rstrip('s')

22+ name = name.decode('utf-8').lower().replace('-', ' ').rstrip('s')

1923 name = re.sub(' +', ' ', name)

2024 return unicodedata.normalize('NFKD', name).lower()

2125

@@ -35,7 +39,7 @@ def iso4217_to_name(iso4217, language):

3539

3640

3741def request(query, params):

38− m = parser_re.match(unicode(query, 'utf8'))

42+ m = parser_re.match(query)

3943 if not m:

4044 # wrong query

4145 return params

searx/engines/dailymotion.py+1−2

@@ -12,10 +12,9 @@

1212 @todo set content-parameter with correct data

1313"""

1414

15−from urllib import urlencode

1615from json import loads

1716from datetime import datetime

18−from requests import get

17+from searx.url_utils import urlencode

1918

2019# engine dependent config

2120categories = ['videos']

searx/engines/deezer.py+2−3

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['music']

@@ -30,8 +30,7 @@ embedded_url = '<iframe scrolling="no" frameborder="0" allowTransparency="true"

3030def request(query, params):

3131 offset = (params['pageno'] - 1) * 25

3232

33− params['url'] = search_url.format(query=urlencode({'q': query}),

34− offset=offset)

33+ params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)

3534

3635 return params

3736

searx/engines/deviantart.py+1−1

@@ -12,10 +12,10 @@

1212 @todo rewrite to api

1313"""

1414

15−from urllib import urlencode

1615from lxml import html

1716import re

1817from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

1919

2020# engine dependent config

2121categories = ['images']

searx/engines/dictzone.py+3−3

@@ -10,20 +10,20 @@

1010"""

1111

1212import re

13−from urlparse import urljoin

1413from lxml import html

1514from searx.utils import is_valid_lang

15+from searx.url_utils import urljoin

1616

1717categories = ['general']

1818url = u'http://dictzone.com/{from_lang}-{to_lang}-dictionary/{query}'

1919weight = 100

2020

21−parser_re = re.compile(u'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

21+parser_re = re.compile(b'.*?([a-z]+)-([a-z]+) ([^ ]+)$', re.I)

2222results_xpath = './/table[@id="r"]/tr'

2323

2424

2525def request(query, params):

26− m = parser_re.match(unicode(query, 'utf8'))

26+ m = parser_re.match(query)

2727 if not m:

2828 return params

2929

searx/engines/digbt.py+6−2

@@ -10,10 +10,14 @@

1010 @parse url, title, content, magnetlink

1111"""

1212

13−from urlparse import urljoin

13+from sys import version_info

1414from lxml import html

1515from searx.engines.xpath import extract_text

1616from searx.utils import get_torrent_size

17+from searx.url_utils import urljoin

18+

19+if version_info[0] == 3:

20+ unicode = str

1721

1822categories = ['videos', 'music', 'files']

1923paging = True

@@ -31,7 +35,7 @@ def request(query, params):

3135

3236

3337def response(resp):

34− dom = html.fromstring(resp.content)

38+ dom = html.fromstring(resp.text)

3539 search_res = dom.xpath('.//td[@class="x-item"]')

3640

3741 if not search_res:

searx/engines/digg.py+2−2

@@ -10,10 +10,10 @@

1010 @parse url, title, content, publishedDate, thumbnail

1111"""

1212

13−from urllib import quote_plus

13+from dateutil import parser

1414from json import loads

1515from lxml import html

16−from dateutil import parser

16+from searx.url_utils import quote_plus

1717

1818# engine dependent config

1919categories = ['news', 'social media']

searx/engines/doku.py+1−1

@@ -9,9 +9,9 @@

99# @stable yes

1010# @parse (general) url, title, content

1111

12−from urllib import urlencode

1312from lxml.html import fromstring

1413from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py+1−1

@@ -13,11 +13,11 @@

1313 @todo rewrite to api

1414"""

1515

16−from urllib import urlencode

1716from lxml.html import fromstring

1817from requests import get

1918from json import loads

2019from searx.engines.xpath import extract_text

20+from searx.url_utils import urlencode

2121

2222# engine dependent config

2323categories = ['general']

searx/engines/duckduckgo_definitions.py+3−3

@@ -1,10 +1,10 @@

11import json

2−from urllib import urlencode

3−from re import compile, sub

42from lxml import html

5−from searx.utils import html_to_text

3+from re import compile

64from searx.engines.xpath import extract_text

75from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url

6+from searx.url_utils import urlencode

7+from searx.utils import html_to_text

88

99url = 'https://api.duckduckgo.com/'\

1010 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

searx/engines/faroo.py+1−1

@@ -10,10 +10,10 @@

1010 @parse url, title, content, publishedDate, img_src

1111"""

1212

13−from urllib import urlencode

1413from json import loads

1514import datetime

1615from searx.utils import searx_useragent

16+from searx.url_utils import urlencode

1717

1818# engine dependent config

1919categories = ['general', 'news']

searx/engines/fdroid.py+3−4

@@ -9,9 +9,9 @@

99 @parse url, title, content

1010"""

1111

12−from urllib import urlencode

13−from searx.engines.xpath import extract_text

1412from lxml import html

13+from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['files']

@@ -24,8 +24,7 @@ search_url = base_url + 'repository/browse/?{query}'

2424

2525# do search-request

2626def request(query, params):

27− query = urlencode({'fdfilter': query,

28− 'fdpage': params['pageno']})

27+ query = urlencode({'fdfilter': query, 'fdpage': params['pageno']})

2928 params['url'] = search_url.format(query=query)

3029 return params

3130

searx/engines/filecrop.py+7−4

@@ -1,5 +1,9 @@

1−from urllib import urlencode

2−from HTMLParser import HTMLParser

1+from searx.url_utils import urlencode

2+

3+try:

4+ from HTMLParser import HTMLParser

5+except:

6+ from html.parser import HTMLParser

37

48url = 'http://www.filecrop.com/'

59search_url = url + '/search.php?{query}&size_i=0&size_f=100000000&engine_r=1&engine_d=1&engine_e=1&engine_4=1&engine_m=1&pos={index}' # noqa

@@ -73,8 +77,7 @@ class FilecropResultParser(HTMLParser):

7377

7478def request(query, params):

7579 index = 1 + (params['pageno'] - 1) * 30

76− params['url'] = search_url.format(query=urlencode({'w': query}),

77− index=index)

80+ params['url'] = search_url.format(query=urlencode({'w': query}), index=index)

7881 return params

7982

8083

searx/engines/flickr.py+1−1

@@ -13,8 +13,8 @@

1313 More info on api-key : https://www.flickr.com/services/apps/create/

1414"""

1515

16−from urllib import urlencode

1716from json import loads

17+from searx.url_utils import urlencode

1818

1919categories = ['images']

2020

searx/engines/flickr_noapi.py+1−1

@@ -12,11 +12,11 @@

1212 @parse url, title, thumbnail, img_src

1313"""

1414

15−from urllib import urlencode

1615from json import loads

1716from time import time

1817import re

1918from searx.engines import logger

19+from searx.url_utils import urlencode

2020

2121

2222logger = logger.getChild('flickr-noapi')

searx/engines/framalibre.py+1−3

@@ -10,12 +10,10 @@

1010 @parse url, title, content, thumbnail, img_src

1111"""

1212

13−from urlparse import urljoin

1413from cgi import escape

15−from urllib import urlencode

1614from lxml import html

1715from searx.engines.xpath import extract_text

18−from dateutil import parser

16+from searx.url_utils import urljoin, urlencode

1917

2018# engine dependent config

2119categories = ['it']

searx/engines/frinkiac.py+1−1

@@ -10,7 +10,7 @@ Frinkiac (Images)

1010"""

1111

1212from json import loads

13−from urllib import urlencode

13+from searx.url_utils import urlencode

1414

1515categories = ['images']

1616

searx/engines/gigablast.py+1−2

@@ -11,10 +11,9 @@

1111"""

1212

1313from json import loads

14−from random import randint

1514from time import time

16−from urllib import urlencode

1715from lxml.html import fromstring

16+from searx.url_utils import urlencode

1817

1918# engine dependent config

2019categories = ['general']

searx/engines/github.py+1−1

@@ -10,8 +10,8 @@

1010 @parse url, title, content

1111"""

1212

13−from urllib import urlencode

1413from json import loads

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['it']

searx/engines/google.py+2−3

@@ -9,11 +9,10 @@

99# @parse url, title, content, suggestion

1010

1111import re

12−from urllib import urlencode

13−from urlparse import urlparse, parse_qsl

1412from lxml import html, etree

1513from searx.engines.xpath import extract_text, extract_url

16−from searx.search import logger

14+from searx import logger

15+from searx.url_utils import urlencode, urlparse, parse_qsl

1716

1817logger = logger.getChild('google engine')

1918

searx/engines/google_images.py+1−1

@@ -11,9 +11,9 @@

1111"""

1212

1313from datetime import date, timedelta

14−from urllib import urlencode

1514from json import loads

1615from lxml import html

16+from searx.url_utils import urlencode

1717

1818

1919# engine dependent config

searx/engines/google_news.py+1−2

@@ -11,9 +11,8 @@

1111"""

1212

1313from lxml import html

14−from urllib import urlencode

15−from json import loads

1614from searx.engines.google import _fetch_supported_languages, supported_languages_url

15+from searx.url_utils import urlencode

1716

1817# search-url

1918categories = ['news']

searx/engines/ina.py+7−3

@@ -12,11 +12,15 @@

1212# @todo embedded (needs some md5 from video page)

1313

1414from json import loads

15−from urllib import urlencode

1615from lxml import html

17−from HTMLParser import HTMLParser

18−from searx.engines.xpath import extract_text

1916from dateutil import parser

17+from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode

19+

20+try:

21+ from HTMLParser import HTMLParser

22+except:

23+ from html.parser import HTMLParser

2024

2125# engine dependent config

2226categories = ['videos']

searx/engines/json_engine.py+8−3

@@ -1,11 +1,16 @@

1−from urllib import urlencode

2−from json import loads

31from collections import Iterable

2+from json import loads

3+from sys import version_info

4+from searx.url_utils import urlencode

5+

6+if version_info[0] == 3:

7+ unicode = str

48

59search_url = None

610url_query = None

711content_query = None

812title_query = None

13+paging = False

914suggestion_query = ''

1015results_query = ''

1116

@@ -20,7 +25,7 @@ first_page_num = 1

2025

2126def iterate(iterable):

2227 if type(iterable) == dict:

23− it = iterable.iteritems()

28+ it = iterable.items()

2429

2530 else:

2631 it = enumerate(iterable)

searx/engines/kickass.py+1−2

@@ -10,12 +10,11 @@

1010 @parse url, title, content, seed, leech, magnetlink

1111"""

1212

13−from urlparse import urljoin

14−from urllib import quote

1513from lxml import html

1614from operator import itemgetter

1715from searx.engines.xpath import extract_text

1816from searx.utils import get_torrent_size, convert_str_to_int

17+from searx.url_utils import quote, urljoin

1918

2019# engine dependent config

2120categories = ['videos', 'music', 'files']

searx/engines/mediawiki.py+1−1

@@ -14,7 +14,7 @@

1414

1515from json import loads

1616from string import Formatter

17−from urllib import urlencode, quote

17+from searx.url_utils import urlencode, quote

1818

1919# engine dependent config

2020categories = ['general']

searx/engines/mixcloud.py+1−1

@@ -11,8 +11,8 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode

1514from dateutil import parser

15+from searx.url_utils import urlencode

1616

1717# engine dependent config

1818categories = ['music']

searx/engines/nyaa.py+1−1

@@ -9,9 +9,9 @@

99 @parse url, title, content, seed, leech, torrentfile

1010"""

1111

12−from urllib import urlencode

1312from lxml import html

1413from searx.engines.xpath import extract_text

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['files', 'images', 'videos', 'music']

searx/engines/openstreetmap.py−4

@@ -11,7 +11,6 @@

1111"""

1212

1313from json import loads

14−from searx.utils import searx_useragent

1514

1615# engine dependent config

1716categories = ['map']

@@ -27,9 +26,6 @@ result_base_url = 'https://openstreetmap.org/{osm_type}/{osm_id}'

2726def request(query, params):

2827 params['url'] = base_url + search_string.format(query=query)

2928

30− # using searx User-Agent

31− params['headers']['User-Agent'] = searx_useragent()

32−

3329 return params

3430

3531

searx/engines/photon.py+1−1

@@ -10,9 +10,9 @@

1010 @parse url, title

1111"""

1212

13−from urllib import urlencode

1413from json import loads

1514from searx.utils import searx_useragent

15+from searx.url_utils import urlencode

1616

1717# engine dependent config

1818categories = ['map']

searx/engines/piratebay.py+1−2

@@ -8,11 +8,10 @@

88# @stable yes (HTML can change)

99# @parse url, title, content, seed, leech, magnetlink

1010

11−from urlparse import urljoin

12−from urllib import quote

1311from lxml import html

1412from operator import itemgetter

1513from searx.engines.xpath import extract_text

14+from searx.url_utils import quote, urljoin

1615

1716# engine dependent config

1817categories = ['videos', 'music', 'files']

searx/engines/qwant.py+1−2

@@ -12,9 +12,8 @@

1212

1313from datetime import datetime

1414from json import loads

15−from urllib import urlencode

16−

1715from searx.utils import html_to_text

16+from searx.url_utils import urlencode

1817

1918# engine dependent config

2019categories = None

searx/engines/reddit.py+2−4

@@ -11,9 +11,8 @@

1111"""

1212

1313import json

14−from urllib import urlencode

15−from urlparse import urlparse, urljoin

1614from datetime import datetime

15+from searx.url_utils import urlencode, urljoin, urlparse

1716

1817# engine dependent config

1918categories = ['general', 'images', 'news', 'social media']

@@ -26,8 +25,7 @@ search_url = base_url + 'search.json?{query}'

2625

2726# do search-request

2827def request(query, params):

29− query = urlencode({'q': query,

30− 'limit': page_size})

28+ query = urlencode({'q': query, 'limit': page_size})

3129 params['url'] = search_url.format(query=query)

3230

3331 return params

searx/engines/scanr_structures.py+1−3

@@ -10,9 +10,7 @@

1010 @parse url, title, content, img_src

1111"""

1212

13−from urllib import urlencode

1413from json import loads, dumps

15−from dateutil import parser

1614from searx.utils import html_to_text

1715

1816# engine dependent config

@@ -48,7 +46,7 @@ def response(resp):

4846 search_res = loads(resp.text)

4947

5048 # return empty array if there are no results

51− if search_res.get('total') < 1:

49+ if search_res.get('total', 0) < 1:

5250 return []

5351

5452 # parse results

searx/engines/searchcode_code.py+2−3

@@ -10,8 +10,8 @@

1010 @parse url, title, content

1111"""

1212

13−from urllib import urlencode

1413from json import loads

14+from searx.url_utils import urlencode

1515

1616

1717# engine dependent config

@@ -31,8 +31,7 @@ code_endings = {'cs': 'c#',

3131

3232# do search-request

3333def request(query, params):

34− params['url'] = search_url.format(query=urlencode({'q': query}),

35− pageno=params['pageno'] - 1)

34+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)

3635

3736 return params

3837

searx/engines/searchcode_doc.py+2−3

@@ -10,8 +10,8 @@

1010 @parse url, title, content

1111"""

1212

13−from urllib import urlencode

1413from json import loads

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['it']

@@ -24,8 +24,7 @@ search_url = url + 'api/search_IV/?{query}&p={pageno}'

2424

2525# do search-request

2626def request(query, params):

27− params['url'] = search_url.format(query=urlencode({'q': query}),

28− pageno=params['pageno'] - 1)

27+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'] - 1)

2928

3029 return params

3130

searx/engines/seedpeer.py+1−3

@@ -8,11 +8,9 @@

88# @stable yes (HTML can change)

99# @parse url, title, content, seed, leech, magnetlink

1010

11−from urlparse import urljoin

12−from urllib import quote

1311from lxml import html

1412from operator import itemgetter

15−from searx.engines.xpath import extract_text

13+from searx.url_utils import quote, urljoin

1614

1715

1816url = 'http://www.seedpeer.eu/'

searx/engines/soundcloud.py+12−7

@@ -11,13 +11,17 @@

1111"""

1212

1313import re

14−from StringIO import StringIO

1514from json import loads

16−from lxml import etree

17−from urllib import urlencode, quote_plus

15+from lxml import html

1816from dateutil import parser

1917from searx import logger

2018from searx.poolrequests import get as http_get

19+from searx.url_utils import quote_plus, urlencode

20+

21+try:

22+ from cStringIO import StringIO

23+except:

24+ from io import StringIO

2125

2226# engine dependent config

2327categories = ['music']

@@ -36,14 +40,15 @@ embedded_url = '<iframe width="100%" height="166" ' +\

3640 'scrolling="no" frameborder="no" ' +\

3741 'data-src="https://w.soundcloud.com/player/?url={uri}"></iframe>'

3842

43+cid_re = re.compile(r'client_id:"([^"]*)"', re.I | re.U)

44+

3945

4046def get_client_id():

4147 response = http_get("https://soundcloud.com")

42− rx_namespace = {"re": "http://exslt.org/regular-expressions"}

4348

4449 if response.ok:

45− tree = etree.parse(StringIO(response.content), etree.HTMLParser())

46− script_tags = tree.xpath("//script[re:match(@src, '(.*app.*js)')]", namespaces=rx_namespace)

50+ tree = html.fromstring(response.content)

51+ script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")

4752 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

4853

4954 # extracts valid app_js urls from soundcloud.com content

@@ -51,7 +56,7 @@ def get_client_id():

5156 # gets app_js and searches for the clientid

5257 response = http_get(app_js_url)

5358 if response.ok:

54− cids = re.search(r'client_id:"([^"]*)"', response.content, re.M | re.I)

59+ cids = cid_re.search(response.text)

5560 if cids is not None and len(cids.groups()):

5661 return cids.groups()[0]

5762 logger.warning("Unable to fetch guest client_id from SoundCloud, check parser!")

searx/engines/spotify.py+2−3

@@ -11,7 +11,7 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode

14+from searx.url_utils import urlencode

1515

1616# engine dependent config

1717categories = ['music']

@@ -29,8 +29,7 @@ embedded_url = '<iframe data-src="https://embed.spotify.com/?uri=spotify:track:{

2929def request(query, params):

3030 offset = (params['pageno'] - 1) * 20

3131

32− params['url'] = search_url.format(query=urlencode({'q': query}),

33− offset=offset)

32+ params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)

3433

3534 return params

3635

searx/engines/stackoverflow.py+2−4

@@ -10,10 +10,9 @@

1010 @parse url, title, content

1111"""

1212

13−from urlparse import urljoin

14−from urllib import urlencode

1513from lxml import html

1614from searx.engines.xpath import extract_text

15+from searx.url_utils import urlencode, urljoin

1716

1817# engine dependent config

1918categories = ['it']

@@ -31,8 +30,7 @@ content_xpath = './/div[@class="excerpt"]'

3130

3231# do search-request

3332def request(query, params):

34− params['url'] = search_url.format(query=urlencode({'q': query}),

35− pageno=params['pageno'])

33+ params['url'] = search_url.format(query=urlencode({'q': query}), pageno=params['pageno'])

3634

3735 return params

3836

searx/engines/startpage.py+1−1

@@ -56,7 +56,7 @@ def request(query, params):

5656def response(resp):

5757 results = []

5858

59− dom = html.fromstring(resp.content)

59+ dom = html.fromstring(resp.text)

6060

6161 # parse results

6262 for result in dom.xpath(results_xpath):

searx/engines/subtitleseeker.py+1−1

@@ -10,10 +10,10 @@

1010 @parse url, title, content

1111"""

1212

13−from urllib import quote_plus

1413from lxml import html

1514from searx.languages import language_codes

1615from searx.engines.xpath import extract_text

16+from searx.url_utils import quote_plus

1717

1818# engine dependent config

1919categories = ['videos']

searx/engines/swisscows.py+13−14

@@ -11,9 +11,9 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode, unquote

1514import re

1615from lxml.html import fromstring

16+from searx.url_utils import unquote, urlencode

1717

1818# engine dependent config

1919categories = ['general', 'images']

@@ -27,10 +27,10 @@ search_string = '?{query}&page={page}'

2727supported_languages_url = base_url

2828

2929# regex

30−regex_json = re.compile(r'initialData: {"Request":(.|\n)*},\s*environment')

31−regex_json_remove_start = re.compile(r'^initialData:\s*')

32−regex_json_remove_end = re.compile(r',\s*environment$')

33−regex_img_url_remove_start = re.compile(r'^https?://i\.swisscows\.ch/\?link=')

30+regex_json = re.compile(b'initialData: {"Request":(.|\n)*},\s*environment')

31+regex_json_remove_start = re.compile(b'^initialData:\s*')

32+regex_json_remove_end = re.compile(b',\s*environment$')

33+regex_img_url_remove_start = re.compile(b'^https?://i\.swisscows\.ch/\?link=')

3434

3535

3636# do search-request

@@ -45,10 +45,9 @@ def request(query, params):

4545 ui_language = params['language'].split('-')[0]

4646

4747 search_path = search_string.format(

48− query=urlencode({'query': query,

49− 'uiLanguage': ui_language,

50− 'region': region}),

51− page=params['pageno'])

48+ query=urlencode({'query': query, 'uiLanguage': ui_language, 'region': region}),

49+ page=params['pageno']

50+ )

5251

5352 # image search query is something like 'image?{query}&page={page}'

5453 if params['category'] == 'images':

@@ -63,14 +62,14 @@ def request(query, params):

6362def response(resp):

6463 results = []

6564

66− json_regex = regex_json.search(resp.content)

65+ json_regex = regex_json.search(resp.text)

6766

6867 # check if results are returned

6968 if not json_regex:

7069 return []

7170

72− json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))

73− json = loads(json_raw)

71+ json_raw = regex_json_remove_end.sub(b'', regex_json_remove_start.sub(b'', json_regex.group()))

72+ json = loads(json_raw.decode('utf-8'))

7473

7574 # parse results

7675 for result in json['Results'].get('items', []):

@@ -78,7 +77,7 @@ def response(resp):

7877

7978 # parse image results

8079 if result.get('ContentType', '').startswith('image'):

81− img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

80+ img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))

8281

8382 # append result

8483 results.append({'url': result['SourceUrl'],

@@ -100,7 +99,7 @@ def response(resp):

10099 # parse images

101100 for result in json.get('Images', []):

102101 # decode image url

103− img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))

102+ img_url = unquote(regex_img_url_remove_start.sub(b'', result['Url'].encode('utf-8')).decode('utf-8'))

104103

105104 # append result

106105 results.append({'url': result['SourceUrl'],

searx/engines/tokyotoshokan.py+5−6

@@ -11,11 +11,11 @@

1111"""

1212

1313import re

14−from urllib import urlencode

1514from lxml import html

1615from searx.engines.xpath import extract_text

1716from datetime import datetime

1817from searx.engines.nyaa import int_or_zero, get_filesize_mul

18+from searx.url_utils import urlencode

1919

2020# engine dependent config

2121categories = ['files', 'videos', 'music']

@@ -28,8 +28,7 @@ search_url = base_url + 'search.php?{query}'

2828

2929# do search-request

3030def request(query, params):

31− query = urlencode({'page': params['pageno'],

32− 'terms': query})

31+ query = urlencode({'page': params['pageno'], 'terms': query})

3332 params['url'] = search_url.format(query=query)

3433 return params

3534

@@ -50,7 +49,7 @@ def response(resp):

5049 size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)

5150

5251 # processing the results, two rows at a time

53− for i in xrange(0, len(rows), 2):

52+ for i in range(0, len(rows), 2):

5453 # parse the first row

5554 name_row = rows[i]

5655

@@ -79,14 +78,14 @@ def response(resp):

7978 groups = size_re.match(item).groups()

8079 multiplier = get_filesize_mul(groups[1])

8180 params['filesize'] = int(multiplier * float(groups[0]))

82− except Exception as e:

81+ except:

8382 pass

8483 elif item.startswith('Date:'):

8584 try:

8685 # Date: 2016-02-21 21:44 UTC

8786 date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')

8887 params['publishedDate'] = date

89− except Exception as e:

88+ except:

9089 pass

9190 elif item.startswith('Comment:'):

9291 params['content'] = item

searx/engines/torrentz.py+4−4

@@ -12,11 +12,11 @@

1212"""

1313

1414import re

15−from urllib import urlencode

1615from lxml import html

17−from searx.engines.xpath import extract_text

1816from datetime import datetime

1917from searx.engines.nyaa import int_or_zero, get_filesize_mul

18+from searx.engines.xpath import extract_text

19+from searx.url_utils import urlencode

2020

2121# engine dependent config

2222categories = ['files', 'videos', 'music']

@@ -70,7 +70,7 @@ def response(resp):

7070 size_str = result.xpath('./dd/span[@class="s"]/text()')[0]

7171 size, suffix = size_str.split()

7272 params['filesize'] = int(size) * get_filesize_mul(suffix)

73− except Exception as e:

73+ except:

7474 pass

7575

7676 # does our link contain a valid SHA1 sum?

@@ -84,7 +84,7 @@ def response(resp):

8484 # Fri, 25 Mar 2016 16:29:01

8585 date = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')

8686 params['publishedDate'] = date

87− except Exception as e:

87+ except:

8888 pass

8989

9090 results.append(params)

searx/engines/translated.py+4

@@ -9,8 +9,12 @@

99 @parse url, title, content

1010"""

1111import re

12+from sys import version_info

1213from searx.utils import is_valid_lang

1314

15+if version_info[0] == 3:

16+ unicode = str

17+

1418categories = ['general']

1519url = u'http://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

1620web_url = u'http://mymemory.translated.net/en/{from_lang}/{to_lang}/{query}'

searx/engines/twitter.py+1−2

@@ -12,11 +12,10 @@

1212 @todo publishedDate

1313"""

1414

15−from urlparse import urljoin

16−from urllib import urlencode

1715from lxml import html

1816from datetime import datetime

1917from searx.engines.xpath import extract_text

18+from searx.url_utils import urlencode, urljoin

2019

2120# engine dependent config

2221categories = ['social media']

searx/engines/vimeo.py+1−1

@@ -13,8 +13,8 @@

1313# @todo set content-parameter with correct data

1414

1515from json import loads

16−from urllib import urlencode

1716from dateutil import parser

17+from searx.url_utils import urlencode

1818

1919# engine dependent config

2020categories = ['videos']

searx/engines/wikidata.py+5−8

@@ -14,12 +14,11 @@

1414from searx import logger

1515from searx.poolrequests import get

1616from searx.engines.xpath import extract_text

17−from searx.utils import format_date_by_locale

1817from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

18+from searx.url_utils import urlencode

1919

2020from json import loads

2121from lxml.html import fromstring

22−from urllib import urlencode

2322

2423logger = logger.getChild('wikidata')

2524result_count = 1

@@ -62,14 +61,13 @@ def request(query, params):

6261 language = 'en'

6362

6463 params['url'] = url_search.format(

65− query=urlencode({'label': query,

66− 'language': language}))

64+ query=urlencode({'label': query, 'language': language}))

6765 return params

6866

6967

7068def response(resp):

7169 results = []

72− html = fromstring(resp.content)

70+ html = fromstring(resp.text)

7371 wikidata_ids = html.xpath(wikidata_ids_xpath)

7472

7573 language = resp.search_params['language'].split('-')[0]

@@ -78,10 +76,9 @@ def response(resp):

7876

7977 # TODO: make requests asynchronous to avoid timeout when result_count > 1

8078 for wikidata_id in wikidata_ids[:result_count]:

81− url = url_detail.format(query=urlencode({'page': wikidata_id,

82− 'uselang': language}))

79+ url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))

8380 htmlresponse = get(url)

84− jsonresponse = loads(htmlresponse.content)

81+ jsonresponse = loads(htmlresponse.text)

8582 results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])

8683

8784 return results

searx/engines/wikipedia.py+9−12

@@ -11,13 +11,12 @@

1111"""

1212

1313from json import loads

14−from urllib import urlencode, quote

1514from lxml.html import fromstring

16−

15+from searx.url_utils import quote, urlencode

1716

1817# search-url

19−base_url = 'https://{language}.wikipedia.org/'

20−search_postfix = 'w/api.php?'\

18+base_url = u'https://{language}.wikipedia.org/'

19+search_url = base_url + u'w/api.php?'\

2120 'action=query'\

2221 '&format=json'\

2322 '&{query}'\

@@ -37,16 +36,16 @@ def url_lang(lang):

3736 else:

3837 language = lang

3938

40− return base_url.format(language=language)

39+ return language

4140

4241

4342# do search-request

4443def request(query, params):

4544 if query.islower():

46− query += '|' + query.title()

45+ query = u'{0}|{1}'.format(query.decode('utf-8'), query.decode('utf-8').title()).encode('utf-8')

4746

48− params['url'] = url_lang(params['language']) \

49− + search_postfix.format(query=urlencode({'titles': query}))

47+ params['url'] = search_url.format(query=urlencode({'titles': query}),

48+ language=url_lang(params['language']))

5049

5150 return params

5251

@@ -78,7 +77,7 @@ def extract_first_paragraph(content, title, image):

7877def response(resp):

7978 results = []

8079

81− search_result = loads(resp.content)

80+ search_result = loads(resp.text)

8281

8382 # wikipedia article's unique id

8483 # first valid id is assumed to be the requested article

@@ -99,11 +98,9 @@ def response(resp):

9998 extract = page.get('extract')

10099

101100 summary = extract_first_paragraph(extract, title, image)

102− if not summary:

103− return []

104101

105102 # link to wikipedia article

106− wikipedia_link = url_lang(resp.search_params['language']) \

103+ wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \

107104 + 'wiki/' + quote(title.replace(' ', '_').encode('utf8'))

108105

109106 results.append({'url': wikipedia_link, 'title': title})

searx/engines/wolframalpha_api.py+6−7

@@ -8,8 +8,8 @@

88# @stable yes

99# @parse url, infobox

1010

11−from urllib import urlencode

1211from lxml import etree

12+from searx.url_utils import urlencode

1313

1414# search-url

1515search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'

@@ -37,8 +37,7 @@ image_pods = {'VisualRepresentation',

3737

3838# do search-request

3939def request(query, params):

40− params['url'] = search_url.format(query=urlencode({'input': query}),

41− api_key=api_key)

40+ params['url'] = search_url.format(query=urlencode({'input': query}), api_key=api_key)

4241 params['headers']['Referer'] = site_url.format(query=urlencode({'i': query}))

4342

4443 return params

@@ -56,7 +55,7 @@ def replace_pua_chars(text):

5655 u'\uf74e': 'i', # imaginary number

5756 u'\uf7d9': '='} # equals sign

5857

59− for k, v in pua_chars.iteritems():

58+ for k, v in pua_chars.items():

6059 text = text.replace(k, v)

6160

6261 return text

@@ -66,7 +65,7 @@ def replace_pua_chars(text):

6665def response(resp):

6766 results = []

6867

69− search_results = etree.XML(resp.content)

68+ search_results = etree.XML(resp.text)

7069

7170 # return empty array if there are no results

7271 if search_results.xpath(failure_xpath):

@@ -120,10 +119,10 @@ def response(resp):

120119 # append infobox

121120 results.append({'infobox': infobox_title,

122121 'attributes': result_chunks,

123− 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})

122+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})

124123

125124 # append link to site

126− results.append({'url': resp.request.headers['Referer'].decode('utf8'),

125+ results.append({'url': resp.request.headers['Referer'],

127126 'title': title,

128127 'content': result_content})

129128

searx/engines/wolframalpha_noapi.py+4−5

@@ -10,10 +10,9 @@

1010

1111from json import loads

1212from time import time

13−from urllib import urlencode

14−from lxml.etree import XML

1513

1614from searx.poolrequests import get as http_get

15+from searx.url_utils import urlencode

1716

1817# search-url

1918url = 'https://www.wolframalpha.com/'

@@ -62,7 +61,7 @@ obtain_token()

6261# do search-request

6362def request(query, params):

6463 # obtain token if last update was more than an hour

65− if time() - token['last_updated'] > 3600:

64+ if time() - (token['last_updated'] or 0) > 3600:

6665 obtain_token()

6766 params['url'] = search_url.format(query=urlencode({'input': query}), token=token['value'])

6867 params['headers']['Referer'] = referer_url.format(query=urlencode({'i': query}))

@@ -112,9 +111,9 @@ def response(resp):

112111

113112 results.append({'infobox': infobox_title,

114113 'attributes': result_chunks,

115− 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer'].decode('utf8')}]})

114+ 'urls': [{'title': 'Wolfram|Alpha', 'url': resp.request.headers['Referer']}]})

116115

117− results.append({'url': resp.request.headers['Referer'].decode('utf8'),

116+ results.append({'url': resp.request.headers['Referer'],

118117 'title': 'Wolfram|Alpha (' + infobox_title + ')',

119118 'content': result_content})

120119

searx/engines/www1x.py+2−4

@@ -10,11 +10,9 @@

1010 @parse url, title, thumbnail, img_src, content

1111"""

1212

13−from urllib import urlencode

14−from urlparse import urljoin

1513from lxml import html

16−import string

1714import re

15+from searx.url_utils import urlencode, urljoin

1816

1917# engine dependent config

2018categories = ['images']

@@ -55,7 +53,7 @@ def response(resp):

5553 cur_element += result_part

5654

5755 # fix xml-error

58− cur_element = string.replace(cur_element, '"></a>', '"/></a>')

56+ cur_element = cur_element.replace('"></a>', '"/></a>')

5957

6058 dom = html.fromstring(cur_element)

6159 link = dom.xpath('//a')[0]

searx/engines/www500px.py+1−2

@@ -13,8 +13,7 @@

1313"""

1414

1515from json import loads

16−from urllib import urlencode

17−from urlparse import urljoin

16+from searx.url_utils import urlencode, urljoin

1817

1918# engine dependent config

2019categories = ['images']

searx/engines/xpath.py+2−2

@@ -1,13 +1,13 @@

11from lxml import html

2−from urllib import urlencode, unquote

3−from urlparse import urlparse, urljoin

42from lxml.etree import _ElementStringResult, _ElementUnicodeResult

53from searx.utils import html_to_text

4+from searx.url_utils import unquote, urlencode, urljoin, urlparse

65

76search_url = None

87url_xpath = None

98content_xpath = None

109title_xpath = None

10+paging = False

1111suggestion_xpath = ''

1212results_xpath = ''

1313

searx/engines/yacy.py+1−1

@@ -13,8 +13,8 @@

1313# @todo parse video, audio and file results

1414

1515from json import loads

16−from urllib import urlencode

1716from dateutil import parser

17+from searx.url_utils import urlencode

1818

1919from searx.utils import html_to_text

2020

searx/engines/yahoo.py+1−2

@@ -11,10 +11,9 @@

1111 @parse url, title, content, suggestion

1212"""

1313

14−from urllib import urlencode

15−from urlparse import unquote

1614from lxml import html

1715from searx.engines.xpath import extract_text, extract_url

16+from searx.url_utils import unquote, urlencode

1817

1918# engine dependent config

2019categories = ['general']

searx/engines/yahoo_news.py+3−3

@@ -9,13 +9,13 @@

99# @stable no (HTML can change)

1010# @parse url, title, content, publishedDate

1111

12−from urllib import urlencode

12+import re

13+from datetime import datetime, timedelta

1314from lxml import html

1415from searx.engines.xpath import extract_text, extract_url

1516from searx.engines.yahoo import parse_url, _fetch_supported_languages, supported_languages_url

16−from datetime import datetime, timedelta

17−import re

1817from dateutil import parser

18+from searx.url_utils import urlencode

1919

2020# engine dependent config

2121categories = ['news']

searx/engines/yandex.py+2−2

@@ -9,9 +9,9 @@

99 @parse url, title, content

1010"""

1111

12−from urllib import urlencode

1312from lxml import html

14−from searx.search import logger

13+from searx import logger

14+from searx.url_utils import urlencode

1515

1616logger = logger.getChild('yandex engine')

1717

searx/engines/youtube_api.py+1−1

@@ -9,8 +9,8 @@

99# @parse url, title, content, publishedDate, thumbnail, embedded

1010

1111from json import loads

12−from urllib import urlencode

1312from dateutil import parser

13+from searx.url_utils import urlencode

1414

1515# engine dependent config

1616categories = ['videos', 'music']

searx/engines/youtube_noapi.py+1−1

@@ -8,10 +8,10 @@

88# @stable no

99# @parse url, title, content, publishedDate, thumbnail, embedded

1010

11−from urllib import quote_plus

1211from lxml import html

1312from searx.engines.xpath import extract_text

1413from searx.utils import list_get

14+from searx.url_utils import quote_plus

1515

1616# engine dependent config

1717categories = ['videos', 'music']

searx/plugins/__init__.py+4−1

@@ -14,9 +14,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1414

1515(C) 2015 by Adam Tauber, <asciimoo@gmail.com>

1616'''

17−from sys import exit

17+from sys import exit, version_info

1818from searx import logger

1919

20+if version_info[0] == 3:

21+ unicode = str

22+

2023logger = logger.getChild('plugins')

2124

2225from searx.plugins import (doai_rewrite,

searx/plugins/doai_rewrite.py+1−1

@@ -1,6 +1,6 @@

11from flask_babel import gettext

22import re

3−from urlparse import urlparse, parse_qsl

3+from searx.url_utils import urlparse, parse_qsl

44

55regex = re.compile(r'10\.\d{4,9}/[^\s]+')

66

searx/plugins/https_rewrite.py+4−1

@@ -16,14 +16,17 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1616'''

1717

1818import re

19−from urlparse import urlparse

19+import sys

2020from lxml import etree

2121from os import listdir, environ

2222from os.path import isfile, isdir, join

2323from searx.plugins import logger

2424from flask_babel import gettext

2525from searx import searx_dir

26+from searx.url_utils import urlparse

2627

28+if sys.version_info[0] == 3:

29+ unicode = str

2730

2831name = "HTTPS rewrite"

2932description = gettext('Rewrite HTTP links to HTTPS if possible')

searx/plugins/self_info.py+2−2

@@ -22,7 +22,7 @@ default_on = True

2222

2323

2424# Self User Agent regex

25−p = re.compile('.*user[ -]agent.*', re.IGNORECASE)

25+p = re.compile(b'.*user[ -]agent.*', re.IGNORECASE)

2626

2727

2828# attach callback to the post search hook

@@ -31,7 +31,7 @@ p = re.compile('.*user[ -]agent.*', re.IGNORECASE)

3131def post_search(request, search):

3232 if search.search_query.pageno > 1:

3333 return True

34− if search.search_query.query == 'ip':

34+ if search.search_query.query == b'ip':

3535 x_forwarded_for = request.headers.getlist("X-Forwarded-For")

3636 if x_forwarded_for:

3737 ip = x_forwarded_for[0]

searx/plugins/tracker_url_remover.py+1−1

@@ -17,7 +17,7 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1717

1818from flask_babel import gettext

1919import re

20−from urlparse import urlunparse

20+from searx.url_utils import urlunparse

2121

2222regexes = {re.compile(r'utm_[^&]+&?'),

2323 re.compile(r'(wkey|wemail)[^&]+&?'),

searx/preferences.py+9−9

@@ -23,7 +23,7 @@ class Setting(object):

2323 def __init__(self, default_value, **kwargs):

2424 super(Setting, self).__init__()

2525 self.value = default_value

26− for key, value in kwargs.iteritems():

26+ for key, value in kwargs.items():

2727 setattr(self, key, value)

2828

2929 self._post_init()

@@ -38,7 +38,7 @@ class Setting(object):

3838 return self.value

3939

4040 def save(self, name, resp):

41− resp.set_cookie(name, bytes(self.value), max_age=COOKIE_MAX_AGE)

41+ resp.set_cookie(name, self.value, max_age=COOKIE_MAX_AGE)

4242

4343

4444class StringSetting(Setting):

@@ -133,7 +133,7 @@ class MapSetting(Setting):

133133

134134 def save(self, name, resp):

135135 if hasattr(self, 'key'):

136− resp.set_cookie(name, bytes(self.key), max_age=COOKIE_MAX_AGE)

136+ resp.set_cookie(name, self.key, max_age=COOKIE_MAX_AGE)

137137

138138

139139class SwitchableSetting(Setting):

@@ -194,7 +194,7 @@ class EnginesSetting(SwitchableSetting):

194194 def _post_init(self):

195195 super(EnginesSetting, self)._post_init()

196196 transformed_choices = []

197− for engine_name, engine in self.choices.iteritems():

197+ for engine_name, engine in self.choices.items():

198198 for category in engine.categories:

199199 transformed_choice = dict()

200200 transformed_choice['default_on'] = not engine.disabled

@@ -241,9 +241,9 @@ class Preferences(object):

241241 'language': SearchLanguageSetting(settings['search']['language'],

242242 choices=LANGUAGE_CODES),

243243 'locale': EnumStringSetting(settings['ui']['default_locale'],

244− choices=settings['locales'].keys() + ['']),

244+ choices=list(settings['locales'].keys()) + ['']),

245245 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],

246− choices=autocomplete.backends.keys() + ['']),

246+ choices=list(autocomplete.backends.keys()) + ['']),

247247 'image_proxy': MapSetting(settings['server']['image_proxy'],

248248 map={'': settings['server']['image_proxy'],

249249 '0': False,

@@ -260,7 +260,7 @@ class Preferences(object):

260260 self.unknown_params = {}

261261

262262 def parse_cookies(self, input_data):

263− for user_setting_name, user_setting in input_data.iteritems():

263+ for user_setting_name, user_setting in input_data.items():

264264 if user_setting_name in self.key_value_settings:

265265 self.key_value_settings[user_setting_name].parse(user_setting)

266266 elif user_setting_name == 'disabled_engines':

@@ -274,7 +274,7 @@ class Preferences(object):

274274 disabled_engines = []

275275 enabled_categories = []

276276 disabled_plugins = []

277− for user_setting_name, user_setting in input_data.iteritems():

277+ for user_setting_name, user_setting in input_data.items():

278278 if user_setting_name in self.key_value_settings:

279279 self.key_value_settings[user_setting_name].parse(user_setting)

280280 elif user_setting_name.startswith('engine_'):

@@ -295,7 +295,7 @@ class Preferences(object):

295295 return self.key_value_settings[user_setting_name].get_value()

296296

297297 def save(self, resp):

298− for user_setting_name, user_setting in self.key_value_settings.iteritems():

298+ for user_setting_name, user_setting in self.key_value_settings.items():

299299 user_setting.save(user_setting_name, resp)

300300 self.engines.save(resp)

301301 self.plugins.save(resp)

searx/query.py+6−2

@@ -21,8 +21,12 @@ from searx.languages import language_codes

2121from searx.engines import (

2222 categories, engines, engine_shortcuts

2323)

24−import string

2524import re

25+import string

26+import sys

27+

28+if sys.version_info[0] == 3:

29+ unicode = str

2630

2731VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$')

2832

@@ -146,7 +150,7 @@ class SearchQuery(object):

146150 """container for all the search parameters (query, language, etc...)"""

147151

148152 def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):

149− self.query = query

153+ self.query = query.encode('utf-8')

150154 self.engines = engines

151155 self.categories = categories

152156 self.lang = lang

searx/results.py+5−1

@@ -1,9 +1,13 @@

11import re

2+import sys

23from collections import defaultdict

34from operator import itemgetter

45from threading import RLock

5−from urlparse import urlparse, unquote

66from searx.engines import engines

7+from searx.url_utils import urlparse, unquote

8+

9+if sys.version_info[0] == 3:

10+ basestring = str

711

812CONTENT_LEN_IGNORED_CHARS_REGEX = re.compile(r'[,;:!?\./\\\\ ()-_]', re.M | re.U)

913WHITESPACE_REGEX = re.compile('( |\t|\n)+', re.M | re.U)

searx/search.py+10−2

@@ -16,8 +16,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1616'''

1717

1818import gc

19+import sys

1920import threading

20−from thread import start_new_thread

2121from time import time

2222from uuid import uuid4

2323import requests.exceptions

@@ -33,6 +33,14 @@ from searx import logger

3333from searx.plugins import plugins

3434from searx.exceptions import SearxParameterException

3535

36+try:

37+ from thread import start_new_thread

38+except:

39+ from _thread import start_new_thread

40+

41+if sys.version_info[0] == 3:

42+ unicode = str

43+

3644logger = logger.getChild('search')

3745

3846number_of_searches = 0

@@ -387,7 +395,7 @@ class Search(object):

387395 request_params['time_range'] = search_query.time_range

388396

389397 # append request to list

390− requests.append((selected_engine['name'], search_query.query.encode('utf-8'), request_params))

398+ requests.append((selected_engine['name'], search_query.query, request_params))

391399

392400 # update timeout_limit

393401 timeout_limit = max(timeout_limit, engine.timeout)

searx/settings_robot.yml+1−1

@@ -17,7 +17,7 @@ server:

1717

1818ui:

1919 themes_path : ""

20− default_theme : legacy

20+ default_theme : oscar

2121 default_locale : ""

2222

2323outgoing:

searx/templates/courgette/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/legacy/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/oscar/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="text-center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/templates/pix-art/404.html+1−1

@@ -3,7 +3,7 @@

33<div class="center">

44 <h1>{{ _('Page not found') }}</h1>

55 {% autoescape false %}

6− <p>{{ _('Go to %(search_page)s.', search_page='<a href="{}">{}</a>'.decode('utf-8').format(url_for('index'), _('search page'))) }}</p>

6+ <p>{{ _('Go to %(search_page)s.', search_page=unicode('<a href="{}">{}</a>').format(url_for('index'), _('search page'))) }}</p>

77 {% endautoescape %}

88</div>

99{% endblock %}

searx/testing.py+26−16

@@ -1,13 +1,16 @@

11# -*- coding: utf-8 -*-

22"""Shared testing code."""

33

4−from plone.testing import Layer

5−from unittest2 import TestCase

6−from os.path import dirname, join, abspath

7−

84

95import os

106import subprocess

7+import traceback

8+

9+

10+from os.path import dirname, join, abspath

11+

12+from splinter import Browser

13+from unittest2 import TestCase

1114

1215

1316class SearxTestLayer:

@@ -32,7 +35,7 @@ class SearxTestLayer:

3235 testTearDown = classmethod(testTearDown)

3336

3437

35−class SearxRobotLayer(Layer):

38+class SearxRobotLayer():

3639 """Searx Robot Test Layer"""

3740

3841 def setUp(self):

@@ -62,7 +65,12 @@ class SearxRobotLayer(Layer):

6265 del os.environ['SEARX_SETTINGS_PATH']

6366

6467

65−SEARXROBOTLAYER = SearxRobotLayer()

68+# SEARXROBOTLAYER = SearxRobotLayer()

69+def run_robot_tests(tests):

70+ print('Running {0} tests'.format(len(tests)))

71+ for test in tests:

72+ with Browser() as browser:

73+ test(browser)

6674

6775

6876class SearxTestCase(TestCase):

@@ -72,17 +80,19 @@ class SearxTestCase(TestCase):

7280

7381

7482if __name__ == '__main__':

75− from tests.test_robot import test_suite

7683 import sys

77− from zope.testrunner.runner import Runner

84+ # test cases

85+ from tests import robot

7886

7987 base_dir = abspath(join(dirname(__file__), '../tests'))

8088 if sys.argv[1] == 'robot':

81− r = Runner(['--color',

82− '--auto-progress',

83− '--stop-on-error',

84− '--path',

85− base_dir],

86− found_suites=[test_suite()])

87− r.run()

88− sys.exit(int(r.failed))

89+ test_layer = SearxRobotLayer()

90+ errors = False

91+ try:

92+ test_layer.setUp()

93+ run_robot_tests([getattr(robot, x) for x in dir(robot) if x.startswith('test_')])

94+ except Exception:

95+ errors = True

96+ print('Error occured: {0}'.format(traceback.format_exc()))

97+ test_layer.tearDown()

98+ sys.exit(1 if errors else 0)

searx/url_utils.pynew file+28

@@ -0,0 +1,28 @@

1+from sys import version_info

2+

3+if version_info[0] == 2:

4+ from urllib import quote, quote_plus, unquote, urlencode

5+ from urlparse import parse_qsl, urljoin, urlparse, urlunparse, ParseResult

6+else:

7+ from urllib.parse import (

8+ parse_qsl,

9+ quote,

10+ quote_plus,

11+ unquote,

12+ urlencode,

13+ urljoin,

14+ urlparse,

15+ urlunparse,

16+ ParseResult

17+ )

18+

19+

20+__export__ = (parse_qsl,

21+ quote,

22+ quote_plus,

23+ unquote,

24+ urlencode,

25+ urljoin,

26+ urlparse,

27+ urlunparse,

28+ ParseResult)

searx/utils.py+18−8

@@ -1,11 +1,9 @@

1−import cStringIO

21import csv

32import os

43import re

54

65from babel.dates import format_date

76from codecs import getincrementalencoder

8−from HTMLParser import HTMLParser

97from imp import load_source

108from os.path import splitext, join

119from random import choice

@@ -16,6 +14,19 @@ from searx.languages import language_codes

1614from searx import settings

1715from searx import logger

1816

17+try:

18+ from cStringIO import StringIO

19+except:

20+ from io import StringIO

21+

22+try:

23+ from HTMLParser import HTMLParser

24+except:

25+ from html.parser import HTMLParser

26+

27+if sys.version_info[0] == 3:

28+ unichr = chr

29+ unicode = str

1930

2031logger = logger.getChild('utils')

2132

@@ -140,7 +151,7 @@ class UnicodeWriter:

140151

141152 def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):

142153 # Redirect output to a queue

143− self.queue = cStringIO.StringIO()

154+ self.queue = StringIO()

144155 self.writer = csv.writer(self.queue, dialect=dialect, **kwds)

145156 self.stream = f

146157 self.encoder = getincrementalencoder(encoding)()

@@ -152,14 +163,13 @@ class UnicodeWriter:

152163 unicode_row.append(col.encode('utf-8').strip())

153164 else:

154165 unicode_row.append(col)

155− self.writer.writerow(unicode_row)

166+ self.writer.writerow([x.decode('utf-8') if hasattr(x, 'decode') else x for x in unicode_row])

156167 # Fetch UTF-8 output from the queue ...

157− data = self.queue.getvalue()

158− data = data.decode("utf-8")

168+ data = self.queue.getvalue().strip('\x00')

159169 # ... and reencode it into the target encoding

160170 data = self.encoder.encode(data)

161171 # write to the target stream

162− self.stream.write(data)

172+ self.stream.write(data.decode('utf-8'))

163173 # empty queue

164174 self.queue.truncate(0)

165175

@@ -231,7 +241,7 @@ def dict_subset(d, properties):

231241

232242def prettify_url(url, max_length=74):

233243 if len(url) > max_length:

234− chunk_len = max_length / 2 + 1

244+ chunk_len = int(max_length / 2 + 1)

235245 return u'{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])

236246 else:

237247 return url

searx/webapp.py+23−13

@@ -22,11 +22,12 @@ if __name__ == '__main__':

2222 from os.path import realpath, dirname

2323 path.append(realpath(dirname(realpath(__file__)) + '/../'))

2424

25−import cStringIO

2625import hashlib

2726import hmac

2827import json

2928import os

29+import sys

30+

3031import requests

3132

3233from searx import logger

@@ -42,8 +43,6 @@ except:

4243 exit(1)

4344from cgi import escape

4445from datetime import datetime, timedelta

45−from urllib import urlencode

46−from urlparse import urlparse, urljoin

4746from werkzeug.contrib.fixers import ProxyFix

4847from flask import (

4948 Flask, request, render_template, url_for, Response, make_response,

@@ -52,7 +51,7 @@ from flask import (

5251from flask_babel import Babel, gettext, format_date, format_decimal

5352from flask.json import jsonify

5453from searx import settings, searx_dir, searx_debug

55−from searx.exceptions import SearxException, SearxParameterException

54+from searx.exceptions import SearxParameterException

5655from searx.engines import (

5756 categories, engines, engine_shortcuts, get_engines_stats, initialize_engines

5857)

@@ -69,6 +68,7 @@ from searx.autocomplete import searx_bang, backends as autocomplete_backends

6968from searx.plugins import plugins

7069from searx.preferences import Preferences, ValidationException

7170from searx.answerers import answerers

71+from searx.url_utils import urlencode, urlparse, urljoin

7272

7373# check if the pyopenssl package is installed.

7474# It is needed for SSL connection without trouble, see #298

@@ -78,6 +78,15 @@ except ImportError:

7878 logger.critical("The pyopenssl package has to be installed.\n"

7979 "Some HTTPS connections will fail")

8080

81+try:

82+ from cStringIO import StringIO

83+except:

84+ from io import StringIO

85+

86+

87+if sys.version_info[0] == 3:

88+ unicode = str

89+

8190# serve pages with HTTP/1.1

8291from werkzeug.serving import WSGIRequestHandler

8392WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0'))

@@ -357,6 +366,8 @@ def render(template_name, override_theme=None, **kwargs):

357366

358367 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

359368

369+ kwargs['unicode'] = unicode

370+

360371 kwargs['scripts'] = set()

361372 for plugin in request.user_plugins:

362373 for script in plugin.js_dependencies:

@@ -375,7 +386,7 @@ def render(template_name, override_theme=None, **kwargs):

375386def pre_request():

376387 request.errors = []

377388

378− preferences = Preferences(themes, categories.keys(), engines, plugins)

389+ preferences = Preferences(themes, list(categories.keys()), engines, plugins)

379390 request.preferences = preferences

380391 try:

381392 preferences.parse_cookies(request.cookies)

@@ -479,10 +490,8 @@ def index():

479490 for result in results:

480491 if output_format == 'html':

481492 if 'content' in result and result['content']:

482− result['content'] = highlight_content(escape(result['content'][:1024]),

483− search_query.query.encode('utf-8'))

484− result['title'] = highlight_content(escape(result['title'] or u''),

485− search_query.query.encode('utf-8'))

493+ result['content'] = highlight_content(escape(result['content'][:1024]), search_query.query)

494+ result['title'] = highlight_content(escape(result['title'] or u''), search_query.query)

486495 else:

487496 if result.get('content'):

488497 result['content'] = html_to_text(result['content']).strip()

@@ -510,7 +519,7 @@ def index():

510519 result['publishedDate'] = format_date(result['publishedDate'])

511520

512521 if output_format == 'json':

513− return Response(json.dumps({'query': search_query.query,

522+ return Response(json.dumps({'query': search_query.query.decode('utf-8'),

514523 'number_of_results': number_of_results,

515524 'results': results,

516525 'answers': list(result_container.answers),

@@ -519,7 +528,7 @@ def index():

519528 'suggestions': list(result_container.suggestions)}),

520529 mimetype='application/json')

521530 elif output_format == 'csv':

522− csv = UnicodeWriter(cStringIO.StringIO())

531+ csv = UnicodeWriter(StringIO())

523532 keys = ('title', 'url', 'content', 'host', 'engine', 'score')

524533 csv.writerow(keys)

525534 for row in results:

@@ -527,7 +536,7 @@ def index():

527536 csv.writerow([row.get(key, '') for key in keys])

528537 csv.stream.seek(0)

529538 response = Response(csv.stream.read(), mimetype='application/csv')

530− cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query.encode('utf-8'))

539+ cont_disp = 'attachment;Filename=searx_-_{0}.csv'.format(search_query.query)

531540 response.headers.add('Content-Disposition', cont_disp)

532541 return response

533542 elif output_format == 'rss':

@@ -578,7 +587,7 @@ def autocompleter():

578587 disabled_engines = request.preferences.engines.get_disabled()

579588

580589 # parse query

581− raw_text_query = RawTextQuery(request.form.get('q', '').encode('utf-8'), disabled_engines)

590+ raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)

582591 raw_text_query.parse_query()

583592

584593 # check if search query is set

@@ -820,6 +829,7 @@ def page_not_found(e):

820829

821830

822831def run():

832+ logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])

823833 app.run(

824834 debug=searx_debug,

825835 use_debugger=searx_debug,

tests/robot/__init__.py+75

@@ -0,0 +1,75 @@

1+# -*- coding: utf-8 -*-

2+

3+from time import sleep

4+

5+url = "http://localhost:11111/"

6+

7+

8+def test_index(browser):

9+ # Visit URL

10+ browser.visit(url)

11+ assert browser.is_text_present('about')

12+

13+

14+def test_404(browser):

15+ # Visit URL

16+ browser.visit(url + 'missing_link')

17+ assert browser.is_text_present('Page not found')

18+

19+

20+def test_about(browser):

21+ browser.visit(url)

22+ browser.click_link_by_text('about')

23+ assert browser.is_text_present('Why use searx?')

24+

25+

26+def test_preferences(browser):

27+ browser.visit(url)

28+ browser.click_link_by_text('preferences')

29+ assert browser.is_text_present('Preferences')

30+ assert browser.is_text_present('Cookies')

31+

32+ assert browser.is_element_present_by_xpath('//label[@for="checkbox_dummy"]')

33+

34+

35+def test_preferences_engine_select(browser):

36+ browser.visit(url)

37+ browser.click_link_by_text('preferences')

38+

39+ assert browser.is_element_present_by_xpath('//a[@href="#tab_engine"]')

40+ browser.find_by_xpath('//a[@href="#tab_engine"]').first.click()

41+

42+ assert not browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked

43+ browser.find_by_xpath('//label[@for="engine_general_dummy__general"]').first.check()

44+ browser.find_by_xpath('//input[@value="save"]').first.click()

45+

46+ # waiting for the redirect - without this the test is flaky..

47+ sleep(1)

48+

49+ browser.visit(url)

50+ browser.click_link_by_text('preferences')

51+ browser.find_by_xpath('//a[@href="#tab_engine"]').first.click()

52+

53+ assert browser.find_by_xpath('//input[@id="engine_general_dummy__general"]').first.checked

54+

55+

56+def test_preferences_locale(browser):

57+ browser.visit(url)

58+ browser.click_link_by_text('preferences')

59+

60+ browser.select('locale', 'hu')

61+ browser.find_by_xpath('//input[@value="save"]').first.click()

62+

63+ # waiting for the redirect - without this the test is flaky..

64+ sleep(1)

65+

66+ browser.visit(url)

67+ browser.click_link_by_text('beállítások')

68+ browser.is_text_present('Beállítások')

69+

70+

71+def test_search(browser):

72+ browser.visit(url)

73+ browser.fill('q', 'test search query')

74+ browser.find_by_xpath('//button[@type="submit"]').first.click()

75+ assert browser.is_text_present('didn\'t find any results')

tests/robot/test_basic.robotdeleted−153

@@ -1,153 +0,0 @@

1−*** Settings ***

2−Library Selenium2Library timeout=10 implicit_wait=0.5

3−Test Setup Open Browser http://localhost:11111/

4−Test Teardown Close All Browsers

5−

6−

7−*** Keywords ***

8−Submit Preferences

9− Set Selenium Speed 2 seconds

10− Submit Form id=search_form

11− Location Should Be http://localhost:11111/

12− Set Selenium Speed 0 seconds

13−

14−

15−*** Test Cases ***

16−Front page

17− Page Should Contain about

18− Page Should Contain preferences

19−

20−404 page

21− Go To http://localhost:11111/no-such-page

22− Page Should Contain Page not found

23− Page Should Contain Go to search page

24−

25−About page

26− Click Element link=about

27− Page Should Contain Why use searx?

28− Page Should Contain Element link=search engines

29−

30−Preferences page

31− Click Element link=preferences

32− Page Should Contain Preferences

33− Page Should Contain Default categories

34− Page Should Contain Currently used search engines

35− Page Should Contain dummy dummy

36− Page Should Contain general dummy

37−

38−Switch category

39− Go To http://localhost:11111/preferences

40− Page Should Contain Checkbox category_general

41− Page Should Contain Checkbox category_dummy

42− Click Element xpath=//*[.="general"]

43− Click Element xpath=//*[.="dummy"]

44− Submit Preferences

45− Checkbox Should Not Be Selected category_general

46− Checkbox Should Be Selected category_dummy

47−

48−Change language

49− Page Should Contain about

50− Page Should Contain preferences

51− Go To http://localhost:11111/preferences

52− Select From List locale hu

53− Submit Preferences

54− Page Should Contain rólunk

55− Page Should Contain beállítások

56−

57−Change method

58− Page Should Contain about

59− Page Should Contain preferences

60− Go To http://localhost:11111/preferences

61− Select From List method GET

62− Submit Preferences

63− Go To http://localhost:11111/preferences

64− List Selection Should Be method GET

65− Select From List method POST

66− Submit Preferences

67− Go To http://localhost:11111/preferences

68− List Selection Should Be method POST

69−

70−Change theme

71− Page Should Contain about

72− Page Should Contain preferences

73− Go To http://localhost:11111/preferences

74− List Selection Should Be theme legacy

75− Select From List theme oscar

76− Submit Preferences

77− Go To http://localhost:11111/preferences

78− List Selection Should Be theme oscar

79−

80−Change safesearch

81− Page Should Contain about

82− Page Should Contain preferences

83− Go To http://localhost:11111/preferences

84− List Selection Should Be safesearch None

85− Select From List safesearch Strict

86− Submit Preferences

87− Go To http://localhost:11111/preferences

88− List Selection Should Be safesearch Strict

89−

90−Change image proxy

91− Page Should Contain about

92− Page Should Contain preferences

93− Go To http://localhost:11111/preferences

94− List Selection Should Be image_proxy Disabled

95− Select From List image_proxy Enabled

96− Submit Preferences

97− Go To http://localhost:11111/preferences

98− List Selection Should Be image_proxy Enabled

99−

100−Change search language

101− Page Should Contain about

102− Page Should Contain preferences

103− Go To http://localhost:11111/preferences

104− List Selection Should Be language Default language

105− Select From List language Türkçe - tr-TR

106− Submit Preferences

107− Go To http://localhost:11111/preferences

108− List Selection Should Be language Türkçe - tr-TR

109−

110−Change autocomplete

111− Page Should Contain about

112− Page Should Contain preferences

113− Go To http://localhost:11111/preferences

114− List Selection Should Be autocomplete -

115− Select From List autocomplete google

116− Submit Preferences

117− Go To http://localhost:11111/preferences

118− List Selection Should Be autocomplete google

119−

120−Change allowed/disabled engines

121− Page Should Contain about

122− Page Should Contain preferences

123− Go To http://localhost:11111/preferences

124− Page Should Contain Engine name

125− Element Should Contain xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy'] Block

126− Element Should Contain xpath=//label[@class="deny"][@for='engine_general_general_dummy'] Block

127− Click Element xpath=//label[@class="deny"][@for='engine_general_general_dummy']

128− Submit Preferences

129− Page Should Contain about

130− Page Should Contain preferences

131− Go To http://localhost:11111/preferences

132− Page Should Contain Engine name

133− Element Should Contain xpath=//label[@class="deny"][@for='engine_dummy_dummy_dummy'] Block

134− Element Should Contain xpath=//label[@class="deny"][@for='engine_general_general_dummy'] \

135−

136−Block a plugin

137− Page Should Contain about

138− Page Should Contain preferences

139− Go To http://localhost:11111/preferences

140− List Selection Should Be theme legacy

141− Select From List theme oscar

142− Submit Preferences

143− Go To http://localhost:11111/preferences

144− List Selection Should Be theme oscar

145− Page Should Contain Plugins

146− Click Link Plugins

147− Checkbox Should Not Be Selected id=plugin_HTTPS_rewrite

148− Click Element xpath=//label[@for='plugin_HTTPS_rewrite']

149− Submit Preferences

150− Go To http://localhost:11111/preferences

151− Page Should Contain Plugins

152− Click Link Plugins

153− Checkbox Should Be Selected id=plugin_HTTPS_rewrite

tests/unit/engines/test_archlinux.py+2−2

@@ -25,7 +25,7 @@ class TestArchLinuxEngine(SearxTestCase):

2525 self.assertTrue(query in params['url'])

2626 self.assertTrue('wiki.archlinux.org' in params['url'])

2727

28− for lang, domain in domains.iteritems():

28+ for lang, domain in domains.items():

2929 dic['language'] = lang

3030 params = archlinux.request(query, dic)

3131 self.assertTrue(domain in params['url'])

@@ -102,5 +102,5 @@ class TestArchLinuxEngine(SearxTestCase):

102102 for exp in expected:

103103 res = results[i]

104104 i += 1

105− for key, value in exp.iteritems():

105+ for key, value in exp.items():

106106 self.assertEqual(res[key], value)

tests/unit/engines/test_bing.py+3−3

@@ -7,18 +7,18 @@ from searx.testing import SearxTestCase

77class TestBingEngine(SearxTestCase):

88

99 def test_request(self):

10− query = 'test_query'

10+ query = u'test_query'

1111 dicto = defaultdict(dict)

1212 dicto['pageno'] = 0

1313 dicto['language'] = 'fr_FR'

14− params = bing.request(query, dicto)

14+ params = bing.request(query.encode('utf-8'), dicto)

1515 self.assertTrue('url' in params)

1616 self.assertTrue(query in params['url'])

1717 self.assertTrue('language%3AFR' in params['url'])

1818 self.assertTrue('bing.com' in params['url'])

1919

2020 dicto['language'] = 'all'

21− params = bing.request(query, dicto)

21+ params = bing.request(query.encode('utf-8'), dicto)

2222 self.assertTrue('language' in params['url'])

2323

2424 def test_response(self):

tests/unit/engines/test_bing_news.py+6−6

@@ -36,10 +36,10 @@ class TestBingNewsEngine(SearxTestCase):

3636 self.assertRaises(AttributeError, bing_news.response, '')

3737 self.assertRaises(AttributeError, bing_news.response, '[]')

3838

39− response = mock.Mock(content='<html></html>')

39+ response = mock.Mock(text='<html></html>')

4040 self.assertEqual(bing_news.response(response), [])

4141

42− response = mock.Mock(content='<html></html>')

42+ response = mock.Mock(text='<html></html>')

4343 self.assertEqual(bing_news.response(response), [])

4444

4545 html = """<?xml version="1.0" encoding="utf-8" ?>

@@ -74,7 +74,7 @@ class TestBingNewsEngine(SearxTestCase):

7474 </item>

7575 </channel>

7676</rss>""" # noqa

77− response = mock.Mock(content=html)

77+ response = mock.Mock(text=html.encode('utf-8'))

7878 results = bing_news.response(response)

7979 self.assertEqual(type(results), list)

8080 self.assertEqual(len(results), 2)

@@ -113,7 +113,7 @@ class TestBingNewsEngine(SearxTestCase):

113113 </item>

114114 </channel>

115115</rss>""" # noqa

116− response = mock.Mock(content=html)

116+ response = mock.Mock(text=html.encode('utf-8'))

117117 results = bing_news.response(response)

118118 self.assertEqual(type(results), list)

119119 self.assertEqual(len(results), 1)

@@ -136,11 +136,11 @@ class TestBingNewsEngine(SearxTestCase):

136136 </channel>

137137</rss>""" # noqa

138138

139− response = mock.Mock(content=html)

139+ response = mock.Mock(text=html.encode('utf-8'))

140140 results = bing_news.response(response)

141141 self.assertEqual(type(results), list)

142142 self.assertEqual(len(results), 0)

143143

144144 html = """<?xml version="1.0" encoding="utf-8" ?>gabarge"""

145− response = mock.Mock(content=html)

145+ response = mock.Mock(text=html.encode('utf-8'))

146146 self.assertRaises(lxml.etree.XMLSyntaxError, bing_news.response, response)

tests/unit/engines/test_btdigg.py+6−6

@@ -22,10 +22,10 @@ class TestBtdiggEngine(SearxTestCase):

2222 self.assertRaises(AttributeError, btdigg.response, '')

2323 self.assertRaises(AttributeError, btdigg.response, '[]')

2424

25− response = mock.Mock(content='<html></html>')

25+ response = mock.Mock(text='<html></html>')

2626 self.assertEqual(btdigg.response(response), [])

2727

28− html = """

28+ html = u"""

2929 <div id="search_res">

3030 <table>

3131 <tr>

@@ -82,7 +82,7 @@ class TestBtdiggEngine(SearxTestCase):

8282 </table>

8383 </div>

8484 """

85− response = mock.Mock(content=html)

85+ response = mock.Mock(text=html.encode('utf-8'))

8686 results = btdigg.response(response)

8787 self.assertEqual(type(results), list)

8888 self.assertEqual(len(results), 1)

@@ -101,12 +101,12 @@ class TestBtdiggEngine(SearxTestCase):

101101 </table>

102102 </div>

103103 """

104− response = mock.Mock(content=html)

104+ response = mock.Mock(text=html.encode('utf-8'))

105105 results = btdigg.response(response)

106106 self.assertEqual(type(results), list)

107107 self.assertEqual(len(results), 0)

108108

109− html = """

109+ html = u"""

110110 <div id="search_res">

111111 <table>

112112 <tr>

@@ -367,7 +367,7 @@ class TestBtdiggEngine(SearxTestCase):

367367 </table>

368368 </div>

369369 """

370− response = mock.Mock(content=html)

370+ response = mock.Mock(text=html.encode('utf-8'))

371371 results = btdigg.response(response)

372372 self.assertEqual(type(results), list)

373373 self.assertEqual(len(results), 5)

tests/unit/engines/test_currency_convert.py+2−2

@@ -8,13 +8,13 @@ from searx.testing import SearxTestCase

88class TestCurrencyConvertEngine(SearxTestCase):

99

1010 def test_request(self):

11− query = 'test_query'

11+ query = b'test_query'

1212 dicto = defaultdict(dict)

1313 dicto['pageno'] = 1

1414 params = currency_convert.request(query, dicto)

1515 self.assertNotIn('url', params)

1616

17− query = 'convert 10 Pound Sterlings to United States Dollars'

17+ query = b'convert 10 Pound Sterlings to United States Dollars'

1818 params = currency_convert.request(query, dicto)

1919 self.assertIn('url', params)

2020 self.assertIn('finance.yahoo.com', params['url'])

tests/unit/engines/test_digbt.py+2−2

@@ -21,7 +21,7 @@ class TestDigBTEngine(SearxTestCase):

2121 self.assertRaises(AttributeError, digbt.response, '')

2222 self.assertRaises(AttributeError, digbt.response, '[]')

2323

24− response = mock.Mock(content='<html></html>')

24+ response = mock.Mock(text='<html></html>')

2525 self.assertEqual(digbt.response(response), [])

2626

2727 html = """

@@ -50,7 +50,7 @@ class TestDigBTEngine(SearxTestCase):

5050 </td></tr>

5151 </table>

5252 """

53− response = mock.Mock(content=html)

53+ response = mock.Mock(text=html.encode('utf-8'))

5454 results = digbt.response(response)

5555 self.assertEqual(type(results), list)

5656 self.assertEqual(len(results), 1)

tests/unit/engines/test_duckduckgo.py+1−2

@@ -90,8 +90,7 @@ class TestDuckduckgoEngine(SearxTestCase):

9090 "wt-wt":"All Results","ar-es":"Argentina","au-en":"Australia","at-de":"Austria","be-fr":"Belgium (fr)"

9191 }some more code..."""

9292 response = mock.Mock(text=js)

93− languages = duckduckgo._fetch_supported_languages(response)

94− self.assertEqual(type(languages), list)

93+ languages = list(duckduckgo._fetch_supported_languages(response))

9594 self.assertEqual(len(languages), 5)

9695 self.assertIn('wt-WT', languages)

9796 self.assertIn('es-AR', languages)

tests/unit/engines/test_frinkiac.py+3−2

@@ -1,7 +1,6 @@

11# -*- coding: utf-8 -*-

22from collections import defaultdict

33import mock

4−from json import dumps

54from searx.engines import frinkiac

65from searx.testing import SearxTestCase

76

@@ -44,6 +43,8 @@ class TestFrinkiacEngine(SearxTestCase):

4443 self.assertEqual(type(results), list)

4544 self.assertEqual(len(results), 4)

4645 self.assertEqual(results[0]['title'], u'S06E18')

47− self.assertEqual(results[0]['url'], 'https://frinkiac.com/?p=caption&e=S06E18&t=534616')

46+ self.assertIn('p=caption', results[0]['url'])

47+ self.assertIn('e=S06E18', results[0]['url'])

48+ self.assertIn('t=534616', results[0]['url'])

4849 self.assertEqual(results[0]['thumbnail_src'], 'https://frinkiac.com/img/S06E18/534616/medium.jpg')

4950 self.assertEqual(results[0]['img_src'], 'https://frinkiac.com/img/S06E18/534616.jpg')

tests/unit/engines/test_gigablast.py+1

@@ -10,6 +10,7 @@ class TestGigablastEngine(SearxTestCase):

1010 query = 'test_query'

1111 dicto = defaultdict(dict)

1212 dicto['pageno'] = 0

13+ dicto['safesearch'] = 0

1314 dicto['language'] = 'all'

1415 params = gigablast.request(query, dicto)

1516 self.assertTrue('url' in params)

tests/unit/engines/test_soundcloud.py+1−1

@@ -2,7 +2,7 @@ from collections import defaultdict

22import mock

33from searx.engines import soundcloud

44from searx.testing import SearxTestCase

5−from urllib import quote_plus

5+from searx.url_utils import quote_plus

66

77

88class TestSoundcloudEngine(SearxTestCase):

tests/unit/engines/test_startpage.py+3−3

@@ -31,7 +31,7 @@ class TestStartpageEngine(SearxTestCase):

3131 self.assertRaises(AttributeError, startpage.response, '')

3232 self.assertRaises(AttributeError, startpage.response, '[]')

3333

34− response = mock.Mock(content='<html></html>')

34+ response = mock.Mock(text='<html></html>')

3535 self.assertEqual(startpage.response(response), [])

3636

3737 html = """

@@ -62,7 +62,7 @@ class TestStartpageEngine(SearxTestCase):

6262 </p>

6363 </div>

6464 """

65− response = mock.Mock(content=html)

65+ response = mock.Mock(text=html.encode('utf-8'))

6666 results = startpage.response(response)

6767 self.assertEqual(type(results), list)

6868 self.assertEqual(len(results), 1)

@@ -133,7 +133,7 @@ class TestStartpageEngine(SearxTestCase):

133133 </p>

134134 </div>

135135 """

136− response = mock.Mock(content=html)

136+ response = mock.Mock(text=html.encode('utf-8'))

137137 results = startpage.response(response)

138138 self.assertEqual(type(results), list)

139139 self.assertEqual(len(results), 1)

tests/unit/engines/test_swisscows.py+4−4

@@ -33,13 +33,13 @@ class TestSwisscowsEngine(SearxTestCase):

3333 self.assertRaises(AttributeError, swisscows.response, '')

3434 self.assertRaises(AttributeError, swisscows.response, '[]')

3535

36− response = mock.Mock(content='<html></html>')

36+ response = mock.Mock(text=b'<html></html>')

3737 self.assertEqual(swisscows.response(response), [])

3838

39− response = mock.Mock(content='<html></html>')

39+ response = mock.Mock(text=b'<html></html>')

4040 self.assertEqual(swisscows.response(response), [])

4141

42− html = u"""

42+ html = b"""

4343 <script>

4444 App.Dispatcher.dispatch("initialize", {

4545 html5history: true,

@@ -111,7 +111,7 @@ class TestSwisscowsEngine(SearxTestCase):

111111 });

112112 </script>

113113 """

114− response = mock.Mock(content=html)

114+ response = mock.Mock(text=html)

115115 results = swisscows.response(response)

116116 self.assertEqual(type(results), list)

117117 self.assertEqual(len(results), 3)

tests/unit/engines/test_tokyotoshokan.py+1−1

@@ -91,7 +91,7 @@ class TestTokyotoshokanEngine(SearxTestCase):

9191 self.assertEqual(r['title'], 'Koyomimonogatari')

9292 self.assertEqual(r['magnetlink'], 'magnet:?xt=urn:btih:4c19eb46b5113685fbd2288ed2531b0b')

9393 self.assertEqual(r['filesize'], int(1024 * 1024 * 10.5))

94− self.assertEqual(r['publishedDate'], datetime(2016, 03, 26, 16, 41))

94+ self.assertEqual(r['publishedDate'], datetime(2016, 3, 26, 16, 41))

9595 self.assertEqual(r['content'], 'Comment: sample comment')

9696 self.assertEqual(r['seed'], 53)

9797 self.assertEqual(r['leech'], 18)

tests/unit/engines/test_wikidata.py+1−2

@@ -1,5 +1,4 @@

11# -*- coding: utf-8 -*-

2−from json import loads

32from lxml.html import fromstring

43from collections import defaultdict

54import mock

@@ -31,7 +30,7 @@ class TestWikidataEngine(SearxTestCase):

3130 self.assertRaises(AttributeError, wikidata.response, '')

3231 self.assertRaises(AttributeError, wikidata.response, '[]')

3332

34− response = mock.Mock(content='<html></html>', search_params={"language": "all"})

33+ response = mock.Mock(text='<html></html>', search_params={"language": "all"})

3534 self.assertEqual(wikidata.response(response), [])

3635

3736 def test_getDetail(self):

tests/unit/engines/test_wikipedia.py+9−9

@@ -13,15 +13,15 @@ class TestWikipediaEngine(SearxTestCase):

1313 query = 'test_query'

1414 dicto = defaultdict(dict)

1515 dicto['language'] = 'fr-FR'

16− params = wikipedia.request(query, dicto)

16+ params = wikipedia.request(query.encode('utf-8'), dicto)

1717 self.assertIn('url', params)

1818 self.assertIn(query, params['url'])

1919 self.assertIn('test_query', params['url'])

2020 self.assertIn('Test_Query', params['url'])

2121 self.assertIn('fr.wikipedia.org', params['url'])

2222

23− query = 'Test_Query'

24− params = wikipedia.request(query, dicto)

23+ query = u'Test_Query'

24+ params = wikipedia.request(query.encode('utf-8'), dicto)

2525 self.assertIn('Test_Query', params['url'])

2626 self.assertNotIn('test_query', params['url'])

2727

@@ -57,7 +57,7 @@ class TestWikipediaEngine(SearxTestCase):

5757 }

5858 }

5959 }"""

60− response = mock.Mock(content=json, search_params=dicto)

60+ response = mock.Mock(text=json, search_params=dicto)

6161 self.assertEqual(wikipedia.response(response), [])

6262

6363 # normal case

@@ -80,7 +80,7 @@ class TestWikipediaEngine(SearxTestCase):

8080 }

8181 }

8282 }"""

83− response = mock.Mock(content=json, search_params=dicto)

83+ response = mock.Mock(text=json, search_params=dicto)

8484 results = wikipedia.response(response)

8585 self.assertEqual(type(results), list)

8686 self.assertEqual(len(results), 2)

@@ -108,10 +108,10 @@ class TestWikipediaEngine(SearxTestCase):

108108 }

109109 }

110110 }"""

111− response = mock.Mock(content=json, search_params=dicto)

111+ response = mock.Mock(text=json, search_params=dicto)

112112 results = wikipedia.response(response)

113113 self.assertEqual(type(results), list)

114− self.assertEqual(len(results), 0)

114+ self.assertEqual(len(results), 2)

115115

116116 # no image

117117 json = """

@@ -130,7 +130,7 @@ class TestWikipediaEngine(SearxTestCase):

130130 }

131131 }

132132 }"""

133− response = mock.Mock(content=json, search_params=dicto)

133+ response = mock.Mock(text=json, search_params=dicto)

134134 results = wikipedia.response(response)

135135 self.assertEqual(type(results), list)

136136 self.assertEqual(len(results), 2)

@@ -158,7 +158,7 @@ class TestWikipediaEngine(SearxTestCase):

158158 }

159159 }

160160 }"""

161− response = mock.Mock(content=json, search_params=dicto)

161+ response = mock.Mock(text=json, search_params=dicto)

162162 results = wikipedia.response(response)

163163 self.assertEqual(type(results), list)

164164 self.assertEqual(len(results), 2)

tests/unit/engines/test_wolframalpha_api.py+5−5

@@ -35,11 +35,11 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

3535 xml = '''<?xml version='1.0' encoding='UTF-8'?>

3636 <queryresult success='false' error='false' />

3737 '''

38− response = mock.Mock(content=xml)

38+ response = mock.Mock(text=xml.encode('utf-8'))

3939 self.assertEqual(wolframalpha_api.response(response), [])

4040

4141 # test basic case

42− xml = """<?xml version='1.0' encoding='UTF-8'?>

42+ xml = b"""<?xml version='1.0' encoding='UTF-8'?>

4343 <queryresult success='true'

4444 error='false'

4545 numpods='3'

@@ -83,7 +83,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

8383 </pod>

8484 </queryresult>

8585 """

86− response = mock.Mock(content=xml, request=request)

86+ response = mock.Mock(text=xml, request=request)

8787 results = wolframalpha_api.response(response)

8888 self.assertEqual(type(results), list)

8989 self.assertEqual(len(results), 2)

@@ -107,7 +107,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

107107 self.assertIn('result_plaintext', results[1]['content'])

108108

109109 # test calc

110− xml = """<?xml version='1.0' encoding='UTF-8'?>

110+ xml = b"""<?xml version='1.0' encoding='UTF-8'?>

111111 <queryresult success='true'

112112 error='false'

113113 numpods='2'

@@ -144,7 +144,7 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

144144 </pod>

145145 </queryresult>

146146 """

147− response = mock.Mock(content=xml, request=request)

147+ response = mock.Mock(text=xml, request=request)

148148 results = wolframalpha_api.response(response)

149149 self.assertEqual(type(results), list)

150150 self.assertEqual(len(results), 2)

tests/unit/test_plugins.py+8−8

@@ -48,11 +48,11 @@ class SelfIPTest(SearxTestCase):

4848 # IP test

4949 request = Mock(remote_addr='127.0.0.1')

5050 request.headers.getlist.return_value = []

51− search = get_search_mock(query='ip', pageno=1)

51+ search = get_search_mock(query=b'ip', pageno=1)

5252 store.call(store.plugins, 'post_search', request, search)

5353 self.assertTrue('127.0.0.1' in search.result_container.answers)

5454

55− search = get_search_mock(query='ip', pageno=2)

55+ search = get_search_mock(query=b'ip', pageno=2)

5656 store.call(store.plugins, 'post_search', request, search)

5757 self.assertFalse('127.0.0.1' in search.result_container.answers)

5858

@@ -60,26 +60,26 @@ class SelfIPTest(SearxTestCase):

6060 request = Mock(user_agent='Mock')

6161 request.headers.getlist.return_value = []

6262

63− search = get_search_mock(query='user-agent', pageno=1)

63+ search = get_search_mock(query=b'user-agent', pageno=1)

6464 store.call(store.plugins, 'post_search', request, search)

6565 self.assertTrue('Mock' in search.result_container.answers)

6666

67− search = get_search_mock(query='user-agent', pageno=2)

67+ search = get_search_mock(query=b'user-agent', pageno=2)

6868 store.call(store.plugins, 'post_search', request, search)

6969 self.assertFalse('Mock' in search.result_container.answers)

7070

71− search = get_search_mock(query='user-agent', pageno=1)

71+ search = get_search_mock(query=b'user-agent', pageno=1)

7272 store.call(store.plugins, 'post_search', request, search)

7373 self.assertTrue('Mock' in search.result_container.answers)

7474

75− search = get_search_mock(query='user-agent', pageno=2)

75+ search = get_search_mock(query=b'user-agent', pageno=2)

7676 store.call(store.plugins, 'post_search', request, search)

7777 self.assertFalse('Mock' in search.result_container.answers)

7878

79− search = get_search_mock(query='What is my User-Agent?', pageno=1)

79+ search = get_search_mock(query=b'What is my User-Agent?', pageno=1)

8080 store.call(store.plugins, 'post_search', request, search)

8181 self.assertTrue('Mock' in search.result_container.answers)

8282

83− search = get_search_mock(query='What is my User-Agent?', pageno=2)

83+ search = get_search_mock(query=b'What is my User-Agent?', pageno=2)

8484 store.call(store.plugins, 'post_search', request, search)

8585 self.assertFalse('Mock' in search.result_container.answers)

tests/unit/test_utils.py+6−2

@@ -1,8 +1,12 @@

11# -*- coding: utf-8 -*-

22import mock

3+import sys

34from searx.testing import SearxTestCase

45from searx import utils

56

7+if sys.version_info[0] == 3:

8+ unicode = str

9+

610

711class TestUtils(SearxTestCase):

812

@@ -30,9 +34,9 @@ class TestUtils(SearxTestCase):

3034 self.assertEqual(utils.highlight_content(content, None), content)

3135

3236 content = 'a'

33− query = 'test'

37+ query = b'test'

3438 self.assertEqual(utils.highlight_content(content, query), content)

35− query = 'a test'

39+ query = b'a test'

3640 self.assertEqual(utils.highlight_content(content, query), content)

3741

3842 def test_html_to_text(self):

tests/unit/test_webapp.py+22−24

@@ -2,10 +2,10 @@

22

33import json

44from mock import Mock

5−from urlparse import ParseResult

65from searx import webapp

76from searx.testing import SearxTestCase

87from searx.search import Search

8+from searx.url_utils import ParseResult

99

1010

1111class ViewsTestCase(SearxTestCase):

@@ -57,37 +57,35 @@ class ViewsTestCase(SearxTestCase):

5757 def test_index_empty(self):

5858 result = self.app.post('/')

5959 self.assertEqual(result.status_code, 200)

60− self.assertIn('<div class="title"><h1>searx</h1></div>', result.data)

60+ self.assertIn(b'<div class="title"><h1>searx</h1></div>', result.data)

6161

6262 def test_index_html(self):

6363 result = self.app.post('/', data={'q': 'test'})

6464 self.assertIn(

65− '<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>', # noqa

65+ b'<h3 class="result_title"><img width="14" height="14" class="favicon" src="/static/themes/legacy/img/icons/icon_youtube.ico" alt="youtube" /><a href="http://second.test.xyz" rel="noreferrer">Second <span class="highlight">Test</span></a></h3>', # noqa

6666 result.data

6767 )

6868 self.assertIn(

69− '<p class="content">first <span class="highlight">test</span> content<br class="last"/></p>', # noqa

69+ b'<p class="content">first <span class="highlight">test</span> content<br class="last"/></p>', # noqa

7070 result.data

7171 )

7272

7373 def test_index_json(self):

7474 result = self.app.post('/', data={'q': 'test', 'format': 'json'})

7575

76− result_dict = json.loads(result.data)

76+ result_dict = json.loads(result.data.decode('utf-8'))

7777

7878 self.assertEqual('test', result_dict['query'])

79− self.assertEqual(

80− result_dict['results'][0]['content'], 'first test content')

81− self.assertEqual(

82− result_dict['results'][0]['url'], 'http://first.test.xyz')

79+ self.assertEqual(result_dict['results'][0]['content'], 'first test content')

80+ self.assertEqual(result_dict['results'][0]['url'], 'http://first.test.xyz')

8381

8482 def test_index_csv(self):

8583 result = self.app.post('/', data={'q': 'test', 'format': 'csv'})

8684

8785 self.assertEqual(

88− 'title,url,content,host,engine,score\r\n'

89− 'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,\r\n' # noqa

90− 'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,\r\n', # noqa

86+ b'title,url,content,host,engine,score\r\n'

87+ b'First Test,http://first.test.xyz,first test content,first.test.xyz,startpage,\r\n' # noqa

88+ b'Second Test,http://second.test.xyz,second test content,second.test.xyz,youtube,\r\n', # noqa

9189 result.data

9290 )

9391

@@ -95,65 +93,65 @@ class ViewsTestCase(SearxTestCase):

9593 result = self.app.post('/', data={'q': 'test', 'format': 'rss'})

9694

9795 self.assertIn(

98− '<description>Search results for "test" - searx</description>',

96+ b'<description>Search results for "test" - searx</description>',

9997 result.data

10098 )

10199

102100 self.assertIn(

103− '<opensearch:totalResults>3</opensearch:totalResults>',

101+ b'<opensearch:totalResults>3</opensearch:totalResults>',

104102 result.data

105103 )

106104

107105 self.assertIn(

108− '<title>First Test</title>',

106+ b'<title>First Test</title>',

109107 result.data

110108 )

111109

112110 self.assertIn(

113− '<link>http://first.test.xyz</link>',

111+ b'<link>http://first.test.xyz</link>',

114112 result.data

115113 )

116114

117115 self.assertIn(

118− '<description>first test content</description>',

116+ b'<description>first test content</description>',

119117 result.data

120118 )

121119

122120 def test_about(self):

123121 result = self.app.get('/about')

124122 self.assertEqual(result.status_code, 200)

125− self.assertIn('<h1>About <a href="/">searx</a></h1>', result.data)

123+ self.assertIn(b'<h1>About <a href="/">searx</a></h1>', result.data)

126124

127125 def test_preferences(self):

128126 result = self.app.get('/preferences')

129127 self.assertEqual(result.status_code, 200)

130128 self.assertIn(

131− '<form method="post" action="/preferences" id="search_form">',

129+ b'<form method="post" action="/preferences" id="search_form">',

132130 result.data

133131 )

134132 self.assertIn(

135− '<legend>Default categories</legend>',

133+ b'<legend>Default categories</legend>',

136134 result.data

137135 )

138136 self.assertIn(

139− '<legend>Interface language</legend>',

137+ b'<legend>Interface language</legend>',

140138 result.data

141139 )

142140

143141 def test_stats(self):

144142 result = self.app.get('/stats')

145143 self.assertEqual(result.status_code, 200)

146− self.assertIn('<h2>Engine stats</h2>', result.data)

144+ self.assertIn(b'<h2>Engine stats</h2>', result.data)

147145

148146 def test_robots_txt(self):

149147 result = self.app.get('/robots.txt')

150148 self.assertEqual(result.status_code, 200)

151− self.assertIn('Allow: /', result.data)

149+ self.assertIn(b'Allow: /', result.data)

152150

153151 def test_opensearch_xml(self):

154152 result = self.app.get('/opensearch.xml')

155153 self.assertEqual(result.status_code, 200)

156− self.assertIn('<Description>a privacy-respecting, hackable metasearch engine</Description>', result.data)

154+ self.assertIn(b'<Description>a privacy-respecting, hackable metasearch engine</Description>', result.data)

157155

158156 def test_favicon(self):

159157 result = self.app.get('/favicon.ico')