e48f07a · searx · git.iko.soy

searx/answerers/__init__.pynew file+46

@@ -0,0 +1,46 @@

1+from os import listdir

2+from os.path import realpath, dirname, join, isdir

3+from searx.utils import load_module

4+from collections import defaultdict

5+

6+

7+answerers_dir = dirname(realpath(__file__))

8+

9+

10+def load_answerers():

11+ answerers = []

12+ for filename in listdir(answerers_dir):

13+ if not isdir(join(answerers_dir, filename)):

14+ continue

15+ module = load_module('answerer.py', join(answerers_dir, filename))

16+ if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):

17+ exit(2)

18+ answerers.append(module)

19+ return answerers

20+

21+

22+def get_answerers_by_keywords(answerers):

23+ by_keyword = defaultdict(list)

24+ for answerer in answerers:

25+ for keyword in answerer.keywords:

26+ for keyword in answerer.keywords:

27+ by_keyword[keyword].append(answerer.answer)

28+ return by_keyword

29+

30+

31+def ask(query):

32+ results = []

33+ query_parts = filter(None, query.query.split())

34+

35+ if query_parts[0] not in answerers_by_keywords:

36+ return results

37+

38+ for answerer in answerers_by_keywords[query_parts[0]]:

39+ result = answerer(query)

40+ if result:

41+ results.append(result)

42+ return results

43+

44+

45+answerers = load_answerers()

46+answerers_by_keywords = get_answerers_by_keywords(answerers)

searx/answerers/random/answerer.pynew file+50

@@ -0,0 +1,50 @@

1+import random

2+import string

3+from flask_babel import gettext

4+

5+# required answerer attribute

6+# specifies which search query keywords triggers this answerer

7+keywords = ('random',)

8+

9+random_int_max = 2**31

10+

11+random_string_letters = string.lowercase + string.digits + string.uppercase

12+

13+

14+def random_string():

15+ return u''.join(random.choice(random_string_letters)

16+ for _ in range(random.randint(8, 32)))

17+

18+

19+def random_float():

20+ return unicode(random.random())

21+

22+

23+def random_int():

24+ return unicode(random.randint(-random_int_max, random_int_max))

25+

26+

27+random_types = {u'string': random_string,

28+ u'int': random_int,

29+ u'float': random_float}

30+

31+

32+# required answerer function

33+# can return a list of results (any result type) for a given query

34+def answer(query):

35+ parts = query.query.split()

36+ if len(parts) != 2:

37+ return []

38+

39+ if parts[1] not in random_types:

40+ return []

41+

42+ return [{'answer': random_types[parts[1]]()}]

43+

44+

45+# required answerer function

46+# returns information about the answerer

47+def self_info():

48+ return {'name': gettext('Random value generator'),

49+ 'description': gettext('Generate different random values'),

50+ 'examples': [u'random {}'.format(x) for x in random_types]}

searx/answerers/statistics/answerer.pynew file+51

@@ -0,0 +1,51 @@

1+from functools import reduce

2+from operator import mul

3+

4+from flask_babel import gettext

5+

6+keywords = ('min',

7+ 'max',

8+ 'avg',

9+ 'sum',

10+ 'prod')

11+

12+

13+# required answerer function

14+# can return a list of results (any result type) for a given query

15+def answer(query):

16+ parts = query.query.split()

17+

18+ if len(parts) < 2:

19+ return []

20+

21+ try:

22+ args = map(float, parts[1:])

23+ except:

24+ return []

25+

26+ func = parts[0]

27+ answer = None

28+

29+ if func == 'min':

30+ answer = min(args)

31+ elif func == 'max':

32+ answer = max(args)

33+ elif func == 'avg':

34+ answer = sum(args) / len(args)

35+ elif func == 'sum':

36+ answer = sum(args)

37+ elif func == 'prod':

38+ answer = reduce(mul, args, 1)

39+

40+ if answer is None:

41+ return []

42+

43+ return [{'answer': unicode(answer)}]

44+

45+

46+# required answerer function

47+# returns information about the answerer

48+def self_info():

49+ return {'name': gettext('Statistics functions'),

50+ 'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),

51+ 'examples': ['avg 123 548 2.04 24.2']}

searx/engines/__init__.py+4−13

@@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1717'''

1818

19−from os.path import realpath, dirname, splitext, join

19+from os.path import realpath, dirname

2020import sys

21−from imp import load_source

2221from flask_babel import gettext

2322from operator import itemgetter

2423from searx import settings

2524from searx import logger

25+from searx.utils import load_module

2626

2727

2828logger = logger.getChild('engines')

@@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__))

3232engines = {}

3333

3434categories = {'general': []}

35+_initialized = False

3536

3637engine_shortcuts = {}

3738engine_default_args = {'paging': False,

@@ -46,16 +47,6 @@ engine_default_args = {'paging': False,

4647 'time_range_support': False}

4748

4849

49−def load_module(filename):

50− modname = splitext(filename)[0]

51− if modname in sys.modules:

52− del sys.modules[modname]

53− filepath = join(engine_dir, filename)

54− module = load_source(modname, filepath)

55− module.name = modname

56− return module

57−

58−

5950def load_engine(engine_data):

6051

6152 if '_' in engine_data['name']:

@@ -65,7 +56,7 @@ def load_engine(engine_data):

6556 engine_module = engine_data['engine']

6657

6758 try:

68− engine = load_module(engine_module + '.py')

59+ engine = load_module(engine_module + '.py', engine_dir)

6960 except:

7061 logger.exception('Cannot load engine "{}"'.format(engine_module))

7162 return None

searx/engines/archlinux.py+1−2

@@ -12,7 +12,6 @@

1212"""

1313

1414from urlparse import urljoin

15−from cgi import escape

1615from urllib import urlencode

1716from lxml import html

1817from searx.engines.xpath import extract_text

@@ -135,7 +134,7 @@ def response(resp):

135134 for result in dom.xpath(xpath_results):

136135 link = result.xpath(xpath_link)[0]

137136 href = urljoin(base_url, link.attrib.get('href'))

138− title = escape(extract_text(link))

137+ title = extract_text(link)

139138

140139 results.append({'url': href,

141140 'title': title})

searx/engines/base.py+1−2

@@ -16,7 +16,6 @@

1616from lxml import etree

1717from urllib import urlencode

1818from searx.utils import searx_useragent

19−from cgi import escape

2019from datetime import datetime

2120import re

2221

@@ -94,7 +93,7 @@ def response(resp):

9493 url = item.text

9594

9695 elif item.attrib["name"] == "dcdescription":

97− content = escape(item.text[:300])

96+ content = item.text[:300]

9897 if len(item.text) > 300:

9998 content += "..."

10099

searx/engines/bing.py+6−11

@@ -14,7 +14,6 @@

1414"""

1515

1616from urllib import urlencode

17−from cgi import escape

1817from lxml import html

1918from searx.engines.xpath import extract_text

2019

@@ -32,18 +31,14 @@ search_string = 'search?{query}&first={offset}'

3231def request(query, params):

3332 offset = (params['pageno'] - 1) * 10 + 1

3433

35− if params['language'] == 'all':

36− language = 'en-US'

37− else:

38− language = params['language'].replace('_', '-')

34+ if params['language'] != 'all':

35+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),

36+ query.decode('utf-8')).encode('utf-8')

3937

4038 search_path = search_string.format(

41− query=urlencode({'q': query, 'setmkt': language}),

39+ query=urlencode({'q': query}),

4240 offset=offset)

4341

44− params['cookies']['SRCHHPGUSR'] = \

45− 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]

46−

4742 params['url'] = base_url + search_path

4843 return params

4944

@@ -65,7 +60,7 @@ def response(resp):

6560 link = result.xpath('.//h3/a')[0]

6661 url = link.attrib.get('href')

6762 title = extract_text(link)

68− content = escape(extract_text(result.xpath('.//p')))

63+ content = extract_text(result.xpath('.//p'))

6964

7065 # append result

7166 results.append({'url': url,

@@ -77,7 +72,7 @@ def response(resp):

7772 link = result.xpath('.//h2/a')[0]

7873 url = link.attrib.get('href')

7974 title = extract_text(link)

80− content = escape(extract_text(result.xpath('.//p')))

75+ content = extract_text(result.xpath('.//p'))

8176

8277 # append result

8378 results.append({'url': url,

searx/engines/btdigg.py+2−3

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import quote

1615from lxml import html

1716from operator import itemgetter

@@ -51,8 +50,8 @@ def response(resp):

5150 for result in search_res:

5251 link = result.xpath('.//td[@class="torrent_name"]//a')[0]

5352 href = urljoin(url, link.attrib.get('href'))

54− title = escape(extract_text(link))

55− content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))

53+ title = extract_text(link)

54+ content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])

5655 content = "<br />".join(content.split("\n"))

5756

5857 filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

searx/engines/dailymotion.py+1−2

@@ -14,7 +14,6 @@

1414

1515from urllib import urlencode

1616from json import loads

17−from cgi import escape

1817from datetime import datetime

1918

2019# engine dependent config

@@ -57,7 +56,7 @@ def response(resp):

5756 for res in search_res['list']:

5857 title = res['title']

5958 url = res['url']

60− content = escape(res['description'])

59+ content = res['description']

6160 thumbnail = res['thumbnail_360_url']

6261 publishedDate = datetime.fromtimestamp(res['created_time'], None)

6362 embedded = embedded_url.format(videoid=res['id'])

searx/engines/deezer.py+5−4

@@ -51,10 +51,11 @@ def response(resp):

5151 if url.startswith('http://'):

5252 url = 'https' + url[4:]

5353

54− content = result['artist']['name'] +\

55− " • " +\

56− result['album']['title'] +\

57− " • " + result['title']

54+ content = '{} - {} - {}'.format(

55+ result['artist']['name'],

56+ result['album']['title'],

57+ result['title'])

58+

5859 embedded = embedded_url.format(audioid=result['id'])

5960

6061 # append result

searx/engines/dictzone.py+2−3

@@ -12,7 +12,6 @@

1212import re

1313from urlparse import urljoin

1414from lxml import html

15−from cgi import escape

1615from searx.utils import is_valid_lang

1716

1817categories = ['general']

@@ -62,8 +61,8 @@ def response(resp):

6261

6362 results.append({

6463 'url': urljoin(resp.url, '?%d' % k),

65− 'title': escape(from_result.text_content()),

66− 'content': escape('; '.join(to_results))

64+ 'title': from_result.text_content(),

65+ 'content': '; '.join(to_results)

6766 })

6867

6968 return results

searx/engines/digg.py+1−2

@@ -13,7 +13,6 @@

1313from urllib import quote_plus

1414from json import loads

1515from lxml import html

16−from cgi import escape

1716from dateutil import parser

1817

1918# engine dependent config

@@ -56,7 +55,7 @@ def response(resp):

5655 url = result.attrib.get('data-contenturl')

5756 thumbnail = result.xpath('.//img')[0].attrib.get('src')

5857 title = ''.join(result.xpath(title_xpath))

59− content = escape(''.join(result.xpath(content_xpath)))

58+ content = ''.join(result.xpath(content_xpath))

6059 pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

6160 publishedDate = parser.parse(pubdate)

6261

searx/engines/fdroid.py+1−2

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from searx.engines.xpath import extract_text

1514from lxml import html

@@ -43,7 +42,7 @@ def response(resp):

4342 img_src = app.xpath('.//img/@src')[0]

4443

4544 content = extract_text(app.xpath('./p')[0])

46− content = escape(content.replace(title, '', 1).strip())

45+ content = content.replace(title, '', 1).strip()

4746

4847 results.append({'url': url,

4948 'title': title,

searx/engines/flickr.py+3−11

@@ -77,21 +77,13 @@ def response(resp):

7777

7878 url = build_flickr_url(photo['owner'], photo['id'])

7979

80− title = photo['title']

81−

82− content = '<span class="photo-author">' +\

83− photo['ownername'] +\

84− '</span><br />' +\

85− '<span class="description">' +\

86− photo['description']['_content'] +\

87− '</span>'

88−

8980 # append result

9081 results.append({'url': url,

91− 'title': title,

82+ 'title': photo['title'],

9283 'img_src': img_src,

9384 'thumbnail_src': thumbnail_src,

94− 'content': content,

85+ 'content': photo['description']['_content'],

86+ 'author': photo['ownername'],

9587 'template': 'images.html'})

9688

9789 # return results

searx/engines/flickr_noapi.py+3−4

@@ -102,16 +102,15 @@ def response(resp):

102102

103103 title = photo.get('title', '')

104104

105− content = '<span class="photo-author">' +\

106− photo['username'] +\

107− '</span><br />'

105+ author = photo['username']

108106

109107 # append result

110108 results.append({'url': url,

111109 'title': title,

112110 'img_src': img_src,

113111 'thumbnail_src': thumbnail_src,

114− 'content': content,

112+ 'content': '',

113+ 'author': author,

115114 'template': 'images.html'})

116115

117116 return results

searx/engines/gigablast.py+2−3

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from json import loads

1514from random import randint

1615from time import time

@@ -78,8 +77,8 @@ def response(resp):

7877 for result in response_json['results']:

7978 # append result

8079 results.append({'url': result['url'],

81− 'title': escape(result['title']),

82− 'content': escape(result['sum'])})

80+ 'title': result['title'],

81+ 'content': result['sum']})

8382

8483 # return results

8584 return results

searx/engines/github.py+1−2

@@ -12,7 +12,6 @@

1212

1313from urllib import urlencode

1414from json import loads

15−from cgi import escape

1615

1716# engine dependent config

1817categories = ['it']

@@ -48,7 +47,7 @@ def response(resp):

4847 url = res['html_url']

4948

5049 if res['description']:

51− content = escape(res['description'][:500])

50+ content = res['description'][:500]

5251 else:

5352 content = ''

5453

searx/engines/google.py+2−3

@@ -9,7 +9,6 @@

99# @parse url, title, content, suggestion

1010

1111import re

12−from cgi import escape

1312from urllib import urlencode

1413from urlparse import urlparse, parse_qsl

1514from lxml import html, etree

@@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):

155154def extract_text_from_dom(result, xpath):

156155 r = result.xpath(xpath)

157156 if len(r) > 0:

158− return escape(extract_text(r[0]))

157+ return extract_text(r[0])

159158 return None

160159

161160

@@ -264,7 +263,7 @@ def response(resp):

264263 # parse suggestion

265264 for suggestion in dom.xpath(suggestion_xpath):

266265 # append suggestion

267− results.append({'suggestion': escape(extract_text(suggestion))})

266+ results.append({'suggestion': extract_text(suggestion)})

268267

269268 # return results

270269 return results

searx/engines/kickass.py+1−2

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import quote

1615from lxml import html

1716from operator import itemgetter

@@ -57,7 +56,7 @@ def response(resp):

5756 link = result.xpath('.//a[@class="cellMainLink"]')[0]

5857 href = urljoin(url, link.attrib['href'])

5958 title = extract_text(link)

60− content = escape(extract_text(result.xpath(content_xpath)))

59+ content = extract_text(result.xpath(content_xpath))

6160 seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))

6261 leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))

6362 filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

searx/engines/nyaa.py+2−4

@@ -9,7 +9,6 @@

99 @parse url, title, content, seed, leech, torrentfile

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from lxml import html

1514from searx.engines.xpath import extract_text

@@ -78,7 +77,7 @@ def response(resp):

7877

7978 # torrent title

8079 page_a = result.xpath(xpath_title)[0]

81− title = escape(extract_text(page_a))

80+ title = extract_text(page_a)

8281

8382 # link to the page

8483 href = page_a.attrib.get('href')

@@ -90,7 +89,7 @@ def response(resp):

9089 try:

9190 file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')

9291 file_size = int(float(file_size) * get_filesize_mul(suffix))

93− except Exception as e:

92+ except:

9493 file_size = None

9594

9695 # seed count

@@ -105,7 +104,6 @@ def response(resp):

105104 # content string contains all information not included into template

106105 content = 'Category: "{category}". Downloaded {downloads} times.'

107106 content = content.format(category=category, downloads=downloads)

108− content = escape(content)

109107

110108 results.append({'url': href,

111109 'title': title,

searx/engines/openstreetmap.py+1−1

@@ -43,7 +43,7 @@ def response(resp):

4343 if 'display_name' not in r:

4444 continue

4545

46− title = r['display_name']

46+ title = r['display_name'] or u''

4747 osm_type = r.get('osm_type', r.get('type'))

4848 url = result_base_url.format(osm_type=osm_type,

4949 osm_id=r['osm_id'])

searx/engines/piratebay.py+1−2

@@ -9,7 +9,6 @@

99# @parse url, title, content, seed, leech, magnetlink

1010

1111from urlparse import urljoin

12−from cgi import escape

1312from urllib import quote

1413from lxml import html

1514from operator import itemgetter

@@ -62,7 +61,7 @@ def response(resp):

6261 link = result.xpath('.//div[@class="detName"]//a')[0]

6362 href = urljoin(url, link.attrib.get('href'))

6463 title = extract_text(link)

65− content = escape(extract_text(result.xpath(content_xpath)))

64+ content = extract_text(result.xpath(content_xpath))

6665 seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]

6766

6867 # convert seed to int if possible

searx/engines/reddit.py+1−2

@@ -11,7 +11,6 @@

1111"""

1212

1313import json

14−from cgi import escape

1514from urllib import urlencode

1615from urlparse import urlparse, urljoin

1716from datetime import datetime

@@ -68,7 +67,7 @@ def response(resp):

6867 img_results.append(params)

6968 else:

7069 created = datetime.fromtimestamp(data['created_utc'])

71− content = escape(data['selftext'])

70+ content = data['selftext']

7271 if len(content) > 500:

7372 content = content[:500] + '...'

7473 params['content'] = content

searx/engines/searchcode_doc.py+2−10

@@ -44,20 +44,12 @@ def response(resp):

4444 # parse results

4545 for result in search_results.get('results', []):

4646 href = result['url']

47− title = "[" + result['type'] + "] " +\

48− result['namespace'] +\

49− " " + result['name']

50− content = '<span class="highlight">[' +\

51− result['type'] + "] " +\

52− result['name'] + " " +\

53− result['synopsis'] +\

54− "</span><br />" +\

55− result['description']

47+ title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])

5648

5749 # append result

5850 results.append({'url': href,

5951 'title': title,

60− 'content': content})

52+ 'content': result['description']})

6153

6254 # return results

6355 return results

searx/engines/seedpeer.py−1

@@ -9,7 +9,6 @@

99# @parse url, title, content, seed, leech, magnetlink

1010

1111from urlparse import urljoin

12−from cgi import escape

1312from urllib import quote

1413from lxml import html

1514from operator import itemgetter

searx/engines/spotify.py+5−4

@@ -46,10 +46,11 @@ def response(resp):

4646 if result['type'] == 'track':

4747 title = result['name']

4848 url = result['external_urls']['spotify']

49− content = result['artists'][0]['name'] +\

50− " • " +\

51− result['album']['name'] +\

52− " • " + result['name']

49+ content = '{} - {} - {}'.format(

50+ result['artists'][0]['name'],

51+ result['album']['name'],

52+ result['name'])

53+

5354 embedded = embedded_url.format(audioid=result['id'])

5455

5556 # append result

searx/engines/stackoverflow.py+2−3

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import urlencode

1615from lxml import html

1716from searx.engines.xpath import extract_text

@@ -48,8 +47,8 @@ def response(resp):

4847 for result in dom.xpath(results_xpath):

4948 link = result.xpath(link_xpath)[0]

5049 href = urljoin(url, link.attrib.get('href'))

51− title = escape(extract_text(link))

52− content = escape(extract_text(result.xpath(content_xpath)))

50+ title = extract_text(link)

51+ content = extract_text(result.xpath(content_xpath))

5352

5453 # append result

5554 results.append({'url': href,

searx/engines/startpage.py+2−3

@@ -11,7 +11,6 @@

1111# @todo paging

1212

1313from lxml import html

14−from cgi import escape

1514from dateutil import parser

1615from datetime import datetime, timedelta

1716import re

@@ -79,10 +78,10 @@ def response(resp):

7978 if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):

8079 continue

8180

82− title = escape(extract_text(link))

81+ title = extract_text(link)

8382

8483 if result.xpath('./p[@class="desc clk"]'):

85− content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))

84+ content = extract_text(result.xpath('./p[@class="desc clk"]'))

8685 else:

8786 content = ''

8887

searx/engines/subtitleseeker.py+2−3

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from urllib import quote_plus

1514from lxml import html

1615from searx.languages import language_codes

@@ -59,7 +58,7 @@ def response(resp):

5958 elif search_lang:

6059 href = href + search_lang + '/'

6160

62− title = escape(extract_text(link))

61+ title = extract_text(link)

6362

6463 content = extract_text(result.xpath('.//div[contains(@class,"red")]'))

6564 content = content + " - "

@@ -75,7 +74,7 @@ def response(resp):

7574 # append result

7675 results.append({'url': href,

7776 'title': title,

78− 'content': escape(content)})

77+ 'content': content})

7978

8079 # return results

8180 return results

searx/engines/swisscows.py+4−5

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from json import loads

1514from urllib import urlencode, unquote

1615import re

@@ -78,7 +77,7 @@ def response(resp):

7877

7978 # append result

8079 results.append({'url': result['SourceUrl'],

81− 'title': escape(result['Title']),

80+ 'title': result['Title'],

8281 'content': '',

8382 'img_src': img_url,

8483 'template': 'images.html'})

@@ -90,8 +89,8 @@ def response(resp):

9089

9190 # append result

9291 results.append({'url': result_url,

93− 'title': escape(result_title),

94− 'content': escape(result_content)})

92+ 'title': result_title,

93+ 'content': result_content})

9594

9695 # parse images

9796 for result in json.get('Images', []):

@@ -100,7 +99,7 @@ def response(resp):

10099

101100 # append result

102101 results.append({'url': result['SourceUrl'],

103− 'title': escape(result['Title']),

102+ 'title': result['Title'],

104103 'content': '',

105104 'img_src': img_url,

106105 'template': 'images.html'})

searx/engines/tokyotoshokan.py−1

@@ -11,7 +11,6 @@

1111"""

1212

1313import re

14−from cgi import escape

1514from urllib import urlencode

1615from lxml import html

1716from searx.engines.xpath import extract_text

searx/engines/torrentz.py−1

@@ -12,7 +12,6 @@

1212"""

1313

1414import re

15−from cgi import escape

1615from urllib import urlencode

1716from lxml import html

1817from searx.engines.xpath import extract_text

searx/engines/translated.py+5−6

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111import re

12−from cgi import escape

1312from searx.utils import is_valid_lang

1413

1514categories = ['general']

@@ -52,14 +51,14 @@ def request(query, params):

5251def response(resp):

5352 results = []

5453 results.append({

55− 'url': escape(web_url.format(

54+ 'url': web_url.format(

5655 from_lang=resp.search_params['from_lang'][2],

5756 to_lang=resp.search_params['to_lang'][2],

58− query=resp.search_params['query'])),

59− 'title': escape('[{0}-{1}] {2}'.format(

57+ query=resp.search_params['query']),

58+ 'title': '[{0}-{1}] {2}'.format(

6059 resp.search_params['from_lang'][1],

6160 resp.search_params['to_lang'][1],

62− resp.search_params['query'])),

63− 'content': escape(resp.json()['responseData']['translatedText'])

61+ resp.search_params['query']),

62+ 'content': resp.json()['responseData']['translatedText']

6463 })

6564 return results

searx/engines/wolframalpha_noapi.py−1

@@ -8,7 +8,6 @@

88# @stable no

99# @parse url, infobox

1010

11−from cgi import escape

1211from json import loads

1312from time import time

1413from urllib import urlencode

searx/engines/yandex.py+2−3

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from lxml import html

1514from searx.search import logger

@@ -52,8 +51,8 @@ def response(resp):

5251 for result in dom.xpath(results_xpath):

5352 try:

5453 res = {'url': result.xpath(url_xpath)[0],

55− 'title': escape(''.join(result.xpath(title_xpath))),

56− 'content': escape(''.join(result.xpath(content_xpath)))}

54+ 'title': ''.join(result.xpath(title_xpath)),

55+ 'content': ''.join(result.xpath(content_xpath))}

5756 except:

5857 logger.exception('yandex parse crash')

5958 continue

searx/plugins/doai_rewrite.py+1−1

@@ -27,5 +27,5 @@ def on_result(request, search, result):

2727 if doi.endswith(suffix):

2828 doi = doi[:-len(suffix)]

2929 result['url'] = 'http://doai.io/' + doi

30− result['parsed_url'] = urlparse(ctx['result']['url'])

30+ result['parsed_url'] = urlparse(result['url'])

3131 return True

searx/preferences.py+17−14

@@ -49,28 +49,32 @@ class StringSetting(Setting):

4949class EnumStringSetting(Setting):

5050 """Setting of a value which can only come from the given choices"""

5151

52+ def _validate_selection(self, selection):

53+ if selection not in self.choices:

54+ raise ValidationException('Invalid value: "{0}"'.format(selection))

55+

5256 def _post_init(self):

5357 if not hasattr(self, 'choices'):

5458 raise MissingArgumentException('Missing argument: choices')

55−

56− if self.value != '' and self.value not in self.choices:

57− raise ValidationException('Invalid default value: {0}'.format(self.value))

59+ self._validate_selection(self.value)

5860

5961 def parse(self, data):

60− if data not in self.choices and data != self.value:

61− raise ValidationException('Invalid choice: {0}'.format(data))

62+ self._validate_selection(data)

6263 self.value = data

6364

6465

6566class MultipleChoiceSetting(EnumStringSetting):

6667 """Setting of values which can only come from the given choices"""

6768

69+ def _validate_selections(self, selections):

70+ for item in selections:

71+ if item not in self.choices:

72+ raise ValidationException('Invalid value: "{0}"'.format(selections))

73+

6874 def _post_init(self):

6975 if not hasattr(self, 'choices'):

7076 raise MissingArgumentException('Missing argument: choices')

71− for item in self.value:

72− if item not in self.choices:

73− raise ValidationException('Invalid default value: {0}'.format(self.value))

77+ self._validate_selections(self.value)

7478

7579 def parse(self, data):

7680 if data == '':

@@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):

7882 return

7983

8084 elements = data.split(',')

81− for item in elements:

82− if item not in self.choices:

83− raise ValidationException('Invalid choice: {0}'.format(item))

85+ self._validate_selections(elements)

8486 self.value = elements

8587

8688 def parse_form(self, data):

@@ -214,11 +216,12 @@ class Preferences(object):

214216 super(Preferences, self).__init__()

215217

216218 self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),

217− 'language': EnumStringSetting('all', choices=LANGUAGE_CODES),

219+ 'language': EnumStringSetting(settings['search']['language'],

220+ choices=LANGUAGE_CODES),

218221 'locale': EnumStringSetting(settings['ui']['default_locale'],

219− choices=settings['locales'].keys()),

222+ choices=settings['locales'].keys() + ['']),

220223 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],

221− choices=autocomplete.backends.keys()),

224+ choices=autocomplete.backends.keys() + ['']),

222225 'image_proxy': MapSetting(settings['server']['image_proxy'],

223226 map={'': settings['server']['image_proxy'],

224227 '0': False,

searx/results.py+5−4

@@ -146,16 +146,17 @@ class ResultContainer(object):

146146 self._number_of_results.append(result['number_of_results'])

147147 results.remove(result)

148148

149− with RLock():

150− engines[engine_name].stats['search_count'] += 1

151− engines[engine_name].stats['result_count'] += len(results)

149+ if engine_name in engines:

150+ with RLock():

151+ engines[engine_name].stats['search_count'] += 1

152+ engines[engine_name].stats['result_count'] += len(results)

152153

153154 if not results:

154155 return

155156

156157 self.results[engine_name].extend(results)

157158

158− if not self.paging and engines[engine_name].paging:

159+ if not self.paging and engine_name in engines and engines[engine_name].paging:

159160 self.paging = True

160161

161162 for i, result in enumerate(results):

searx/search.py+9

@@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib

2424from searx.engines import (

2525 categories, engines

2626)

27+from searx.answerers import ask

2728from searx.utils import gen_useragent

2829from searx.query import RawTextQuery, SearchQuery

2930from searx.results import ResultContainer

@@ -300,6 +301,14 @@ class Search(object):

300301 # start time

301302 start_time = time()

302303

304+ # answeres ?

305+ answerers_results = ask(self.search_query)

306+

307+ if answerers_results:

308+ for results in answerers_results:

309+ self.result_container.extend('answer', results)

310+ return self.result_container

311+

303312 # init vars

304313 requests = []

305314

searx/settings.yml+1

@@ -5,6 +5,7 @@ general:

55search:

66 safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict

77 autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default

8+ language : "all"

89

910server:

1011 port : 8888

searx/settings_robot.yml+1

@@ -5,6 +5,7 @@ general:

55search:

66 safe_search : 0

77 autocomplete : ""

8+ language: "all"

89

910server:

1011 port : 11111

searx/static/plugins/js/infinite_scroll.js+1−1

@@ -5,7 +5,7 @@ $(document).ready(function() {

55 var formData = $('#pagination form:last').serialize();

66 if (formData) {

77 $('#pagination').html('<div class="loading-spinner"></div>');

8− $.post('/', formData, function (data) {

8+ $.post('./', formData, function (data) {

99 var body = $(data);

1010 $('#pagination').remove();

1111 $('#main_results').append('<hr/>');

searx/templates/courgette/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

33 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

44 xmlns:atom="http://www.w3.org/2005/Atom">

55 <channel>

6− <title>Searx search: {{ q }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

6+ <title>Searx search: {{ q|e }}</title>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8+ <description>Search results for "{{ q|e }}" - searx</description>

99 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

1010 <opensearch:startIndex>1</opensearch:startIndex>

1111 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

1212 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

1414 {% for r in results %}

1515 <item>

1616 <title>{{ r.title }}</title>

searx/templates/courgette/results.html+5−5

@@ -1,6 +1,6 @@

11{% extends "courgette/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

44{% block content %}

55<div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div>

66<div class="small search center">

@@ -17,7 +17,7 @@

1717 {% for output_type in ('csv', 'json', 'rss') %}

1818 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

1919 <div class="left">

20− <input type="hidden" name="q" value="{{ q }}" />

20+ <input type="hidden" name="q" value="{{ q|e }}" />

2121 <input type="hidden" name="format" value="{{ output_type }}" />

2222 {% for category in selected_categories %}

2323 <input type="hidden" name="category_{{ category }}" value="1"/>

@@ -62,7 +62,7 @@

6262 {% if pageno > 1 %}

6363 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

6464 <div class="left">

65− <input type="hidden" name="q" value="{{ q }}" />

65+ <input type="hidden" name="q" value="{{ q|e }}" />

6666 {% for category in selected_categories %}

6767 <input type="hidden" name="category_{{ category }}" value="1"/>

6868 {% endfor %}

@@ -76,7 +76,7 @@

7676 {% for category in selected_categories %}

7777 <input type="hidden" name="category_{{ category }}" value="1"/>

7878 {% endfor %}

79− <input type="hidden" name="q" value="{{ q }}" />

79+ <input type="hidden" name="q" value="{{ q|e }}" />

8080 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

8181 <input type="submit" value="{{ _('next page') }} >>" />

8282 </div>

searx/templates/legacy/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

33 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

44 xmlns:atom="http://www.w3.org/2005/Atom">

55 <channel>

6− <title>Searx search: {{ q }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

6+ <title>Searx search: {{ q|e }}</title>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8+ <description>Search results for "{{ q|e }}" - searx</description>

99 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

1010 <opensearch:startIndex>1</opensearch:startIndex>

1111 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

1212 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

1414 {% for r in results %}

1515 <item>

1616 <title>{{ r.title }}</title>

searx/templates/legacy/results.html+5−5

@@ -1,6 +1,6 @@

11{% extends "legacy/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

44{% block content %}

55<div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>

66<div class="small search center">

@@ -18,7 +18,7 @@

1818 {% for output_type in ('csv', 'json', 'rss') %}

1919 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

2020 <div class="left">

21− <input type="hidden" name="q" value="{{ q }}" />

21+ <input type="hidden" name="q" value="{{ q|e }}" />

2222 <input type="hidden" name="format" value="{{ output_type }}" />

2323 {% for category in selected_categories %}

2424 <input type="hidden" name="category_{{ category }}" value="1"/>

@@ -73,7 +73,7 @@

7373 {% if pageno > 1 %}

7474 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

7575 <div class="{% if rtl %}right{% else %}left{% endif %}">

76− <input type="hidden" name="q" value="{{ q }}" />

76+ <input type="hidden" name="q" value="{{ q|e }}" />

7777 {% for category in selected_categories %}

7878 <input type="hidden" name="category_{{ category }}" value="1"/>

7979 {% endfor %}

@@ -87,7 +87,7 @@

8787 {% for category in selected_categories %}

8888 <input type="hidden" name="category_{{ category }}" value="1"/>

8989 {% endfor %}

90− <input type="hidden" name="q" value="{{ q }}" />

90+ <input type="hidden" name="q" value="{{ q|e }}" />

9191 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

9292 <input type="submit" value="{{ _('next page') }} >>" />

9393 </div>

searx/templates/oscar/base.html+15

@@ -1,3 +1,4 @@

1+{% from 'oscar/macros.html' import icon %}

12<!DOCTYPE html>

23<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>

34<head>

@@ -54,6 +55,20 @@

5455<body>

5556 {% include 'oscar/navbar.html' %}

5657 <div class="container">

58+ {% if errors %}

59+ <div class="alert alert-danger fade in" role="alert">

60+ <button class="close" data-dismiss="alert" type="button">

61+ <span aria-hidden="true">×</span>

62+ <span class="sr-only">{{ _('Close') }}</span>

63+ </button>

64+ <strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong>

65+ <ul>

66+ {% for message in errors %}

67+ <li>{{ message }}</li>

68+ {% endfor %}

69+ </ul>

70+ </div>

71+ {% endif %}

5772

5873 {% block site_alert_error %}

5974 {% endblock %}

searx/templates/oscar/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

33 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

44 xmlns:atom="http://www.w3.org/2005/Atom">

55 <channel>

6− <title>Searx search: {{ q }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

6+ <title>Searx search: {{ q|e }}</title>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8+ <description>Search results for "{{ q|e }}" - searx</description>

99 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

1010 <opensearch:startIndex>1</opensearch:startIndex>

1111 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

1212 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

1414 {% for r in results %}

1515 <item>

1616 <title>{{ r.title }}</title>

searx/templates/oscar/preferences.html+29

@@ -12,6 +12,7 @@

1212 <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>

1313 <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>

1414 <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>

15+ {% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %}

1516 <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>

1617 </ul>

1718

@@ -224,6 +225,34 @@

224225 </fieldset>

225226 </div>

226227

228+ {% if answerers %}

229+ <div class="tab-pane active_if_nojs" id="tab_answerers">

230+ <noscript>

231+ <h3>{{ _('Answerers') }}</h3>

232+ </noscript>

233+ <p class="text-muted" style="margin:20px 0;">

234+ {{ _('This is the list of searx\'s instant answering modules.') }}

235+ </p>

236+ <table class="table table-striped">

237+ <tr>

238+ <th class="text-muted">{{ _('Name') }}</th>

239+ <th class="text-muted">{{ _('Keywords') }}</th>

240+ <th class="text-muted">{{ _('Description') }}</th>

241+ <th class="text-muted">{{ _('Examples') }}</th>

242+ </tr>

243+

244+ {% for answerer in answerers %}

245+ <tr>

246+ <td class="text-muted">{{ answerer.info.name }}</td>

247+ <td class="text-muted">{{ answerer.keywords|join(', ') }}</td>

248+ <td class="text-muted">{{ answerer.info.description }}</td>

249+ <td class="text-muted">{{ answerer.info.examples|join(', ') }}</td>

250+ </tr>

251+ {% endfor %}

252+ </table>

253+ </div>

254+ {% endif %}

255+

227256 <div class="tab-pane active_if_nojs" id="tab_cookies">

228257 <noscript>

229258 <h3>{{ _('Cookies') }}</h3>

searx/templates/oscar/result_templates/images.html+6−1

@@ -13,7 +13,12 @@

1313 </div>

1414 <div class="modal-body">

1515 <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">

16− {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}

16+ {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}

17+ {% if result.content %}

18+ <p class="result-content">

19+ {{ result.content }}

20+ </p>

21+ {% endif %}

1722 </div>

1823 <div class="modal-footer">

1924 <div class="clearfix"></div>

searx/templates/oscar/results.html+7−7

@@ -1,6 +1,6 @@

11{% extends "oscar/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}

44{% block content %}

55 <div class="row">

66 <div class="col-sm-8" id="main_results">

@@ -37,9 +37,9 @@

3737 <div id="pagination">

3838 <div class="pull-left">

3939 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

40− <input type="hidden" name="q" value="{{ q }}" />

40+ <input type="hidden" name="q" value="{{ q|e }}" />

4141 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

42− <input type="hidden" name="q" value="{{ q }}" />

42+ <input type="hidden" name="q" value="{{ q|e }}" />

4343 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

4444 <input type="hidden" name="time_range" value="{{ time_range }}" />

4545 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>

@@ -59,7 +59,7 @@

5959 <div id="pagination">

6060 <div class="pull-left">

6161 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

62− <input type="hidden" name="q" value="{{ q }}" />

62+ <input type="hidden" name="q" value="{{ q|e }}" />

6363 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

6464 <input type="hidden" name="pageno" value="{{ pageno-1 }}" />

6565 <input type="hidden" name="time_range" value="{{ time_range }}" />

@@ -69,7 +69,7 @@

6969 <div class="pull-right">

7070 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

7171 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

72− <input type="hidden" name="q" value="{{ q }}" />

72+ <input type="hidden" name="q" value="{{ q|e }}" />

7373 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

7474 <input type="hidden" name="time_range" value="{{ time_range }}" />

7575 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>

@@ -130,7 +130,7 @@

130130 <div class="clearfix"></div>

131131 {% for output_type in ('csv', 'json', 'rss') %}

132132 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">

133− <input type="hidden" name="q" value="{{ q }}">

133+ <input type="hidden" name="q" value="{{ q|e }}">

134134 <input type="hidden" name="format" value="{{ output_type }}">

135135 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}

136136 <input type="hidden" name="pageno" value="{{ pageno }}">

searx/templates/pix-art/results.html+3−3

@@ -5,7 +5,7 @@

55 {% endfor %}

66{% else %}

77{% extends "pix-art/base.html" %}

8−{% block title %}{{ q }} - {% endblock %}

8+{% block title %}{{ q|e }} - {% endblock %}

99{% block meta %}{% endblock %}

1010{% block content %}

1111<div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div>

@@ -25,8 +25,8 @@

2525 </span>

2626 <div id="pagination">

2727 <br />

28− <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />

28+ <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />

2929 </div>

3030</div>

3131{% endblock %}

32−{% endif %}

32+{% endif %}

searx/utils.py+13

@@ -6,7 +6,10 @@ import re

66from babel.dates import format_date

77from codecs import getincrementalencoder

88from HTMLParser import HTMLParser

9+from imp import load_source

10+from os.path import splitext, join

911from random import choice

12+import sys

1013

1114from searx.version import VERSION_STRING

1215from searx.languages import language_codes

@@ -285,3 +288,13 @@ def is_valid_lang(lang):

285288 if l[1].lower() == lang.lower():

286289 return (True, l[0][:2], l[1].lower())

287290 return False

291+

292+

293+def load_module(filename, module_dir):

294+ modname = splitext(filename)[0]

295+ if modname in sys.modules:

296+ del sys.modules[modname]

297+ filepath = join(module_dir, filename)

298+ module = load_source(modname, filepath)

299+ module.name = modname

300+ return module

searx/webapp.py+20−11

@@ -40,7 +40,7 @@ except:

4040 logger.critical("cannot import dependency: pygments")

4141 from sys import exit

4242 exit(1)

43−

43+from cgi import escape

4444from datetime import datetime, timedelta

4545from urllib import urlencode

4646from urlparse import urlparse, urljoin

@@ -62,11 +62,12 @@ from searx.utils import (

6262)

6363from searx.version import VERSION_STRING

6464from searx.languages import language_codes

65−from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp

66−from searx.query import RawTextQuery, SearchQuery

65+from searx.search import SearchWithPlugins, get_search_query_from_webapp

66+from searx.query import RawTextQuery

6767from searx.autocomplete import searx_bang, backends as autocomplete_backends

6868from searx.plugins import plugins

6969from searx.preferences import Preferences, ValidationException

70+from searx.answerers import answerers

7071

7172# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.

7273# They are needed for SSL connection without trouble, see #298

@@ -344,6 +345,8 @@ def render(template_name, override_theme=None, **kwargs):

344345

345346 kwargs['cookies'] = request.cookies

346347

348+ kwargs['errors'] = request.errors

349+

347350 kwargs['instance_name'] = settings['general']['instance_name']

348351

349352 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

@@ -364,15 +367,16 @@ def render(template_name, override_theme=None, **kwargs):

364367

365368@app.before_request

366369def pre_request():

367− # merge GET, POST vars

370+ request.errors = []

371+

368372 preferences = Preferences(themes, categories.keys(), engines, plugins)

373+ request.preferences = preferences

369374 try:

370375 preferences.parse_cookies(request.cookies)

371376 except:

372− # TODO throw error message to the user

373− logger.warning('Invalid config')

374− request.preferences = preferences

377+ request.errors.append(gettext('Invalid settings, please edit your preferences'))

375378

379+ # merge GET, POST vars

376380 # request.form

377381 request.form = dict(request.form.items())

378382 for k, v in request.args.items():

@@ -397,7 +401,7 @@ def index():

397401 Supported outputs: html, json, csv, rss.

398402 """

399403

400− if not request.args and not request.form:

404+ if request.form.get('q') is None:

401405 return render(

402406 'index.html',

403407 )

@@ -411,6 +415,8 @@ def index():

411415 search = SearchWithPlugins(search_query, request)

412416 result_container = search.search()

413417 except:

418+ request.errors.append(gettext('search error'))

419+ logger.exception('search error')

414420 return render(

415421 'index.html',

416422 )

@@ -427,8 +433,10 @@ def index():

427433 for result in results:

428434 if output_format == 'html':

429435 if 'content' in result and result['content']:

430− result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8'))

431− result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8'))

436+ result['content'] = highlight_content(escape(result['content'][:1024]),

437+ search_query.query.encode('utf-8'))

438+ result['title'] = highlight_content(escape(result['title'] or u''),

439+ search_query.query.encode('utf-8'))

432440 else:

433441 if result.get('content'):

434442 result['content'] = html_to_text(result['content']).strip()

@@ -572,7 +580,7 @@ def preferences():

572580 try:

573581 request.preferences.parse_form(request.form)

574582 except ValidationException:

575− # TODO use flash feature of flask

583+ request.errors.append(gettext('Invalid settings, please edit your preferences'))

576584 return resp

577585 return request.preferences.save(resp)

578586

@@ -609,6 +617,7 @@ def preferences():

609617 language_codes=language_codes,

610618 engines_by_category=categories,

611619 stats=stats,

620+ answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],

612621 disabled_engines=disabled_engines,

613622 autocomplete_backends=autocomplete_backends,

614623 shortcuts={y: x for x, y in engine_shortcuts.items()},

tests/unit/engines/test_bing.py+2−4

@@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase):

1414 params = bing.request(query, dicto)

1515 self.assertTrue('url' in params)

1616 self.assertTrue(query in params['url'])

17+ self.assertTrue('language%3AFR' in params['url'])

1718 self.assertTrue('bing.com' in params['url'])

18− self.assertTrue('SRCHHPGUSR' in params['cookies'])

19− self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])

2019

2120 dicto['language'] = 'all'

2221 params = bing.request(query, dicto)

23− self.assertTrue('SRCHHPGUSR' in params['cookies'])

24− self.assertTrue('en' in params['cookies']['SRCHHPGUSR'])

22+ self.assertTrue('language' not in params['url'])

2523

2624 def test_response(self):

2725 self.assertRaises(AttributeError, bing.response, None)

tests/unit/engines/test_deezer.py+1−1

@@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase):

4242 self.assertEqual(len(results), 1)

4343 self.assertEqual(results[0]['title'], 'Title of track')

4444 self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')

45− self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')

45+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')

4646 self.assertTrue('100' in results[0]['embedded'])

4747

4848 json = r"""

tests/unit/engines/test_flickr.py+3−3

@@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase):

5252 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

5353 self.assertTrue('o.jpg' in results[0]['img_src'])

5454 self.assertTrue('n.jpg' in results[0]['thumbnail_src'])

55− self.assertTrue('Owner' in results[0]['content'])

55+ self.assertTrue('Owner' in results[0]['author'])

5656 self.assertTrue('Description' in results[0]['content'])

5757

5858 json = r"""

@@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase):

7676 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

7777 self.assertTrue('z.jpg' in results[0]['img_src'])

7878 self.assertTrue('z.jpg' in results[0]['thumbnail_src'])

79− self.assertTrue('Owner' in results[0]['content'])

79+ self.assertTrue('Owner' in results[0]['author'])

8080 self.assertTrue('Description' in results[0]['content'])

8181

8282 json = r"""

@@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase):

100100 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

101101 self.assertTrue('o.jpg' in results[0]['img_src'])

102102 self.assertTrue('o.jpg' in results[0]['thumbnail_src'])

103− self.assertTrue('Owner' in results[0]['content'])

103+ self.assertTrue('Owner' in results[0]['author'])

104104 self.assertTrue('Description' in results[0]['content'])

105105

106106 json = r"""

tests/unit/engines/test_flickr_noapi.py+3−3

@@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

145145 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

146146 self.assertIn('k.jpg', results[0]['img_src'])

147147 self.assertIn('n.jpg', results[0]['thumbnail_src'])

148− self.assertIn('Owner', results[0]['content'])

148+ self.assertIn('Owner', results[0]['author'])

149149

150150 # no n size, only the z size

151151 json = """

@@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

188188 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

189189 self.assertIn('z.jpg', results[0]['img_src'])

190190 self.assertIn('z.jpg', results[0]['thumbnail_src'])

191− self.assertIn('Owner', results[0]['content'])

191+ self.assertIn('Owner', results[0]['author'])

192192

193193 # no z or n size

194194 json = """

@@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

231231 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

232232 self.assertIn('o.jpg', results[0]['img_src'])

233233 self.assertIn('o.jpg', results[0]['thumbnail_src'])

234− self.assertIn('Owner', results[0]['content'])

234+ self.assertIn('Owner', results[0]['author'])

235235

236236 # no image test

237237 json = """

tests/unit/engines/test_kickass.py+2−2

@@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase):

9898 self.assertEqual(len(results), 1)

9999 self.assertEqual(results[0]['title'], 'This should be the title')

100100 self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')

101− self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

101+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

102102 self.assertEqual(results[0]['seed'], 10)

103103 self.assertEqual(results[0]['leech'], 1)

104104 self.assertEqual(results[0]['filesize'], 449)

@@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase):

381381 self.assertEqual(len(results), 5)

382382 self.assertEqual(results[0]['title'], 'This should be the title')

383383 self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')

384− self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

384+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

385385 self.assertEqual(results[0]['seed'], 10)

386386 self.assertEqual(results[0]['leech'], 1)

387387 self.assertEqual(results[0]['files'], 4)

tests/unit/engines/test_searchcode_doc.py−3

@@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase):

5656 self.assertEqual(len(results), 1)

5757 self.assertEqual(results[0]['title'], '[Type] Namespace test')

5858 self.assertEqual(results[0]['url'], 'http://url')

59− self.assertIn('Synopsis', results[0]['content'])

60− self.assertIn('Type', results[0]['content'])

61− self.assertIn('test', results[0]['content'])

6259 self.assertIn('Description', results[0]['content'])

6360

6461 json = r"""

tests/unit/engines/test_spotify.py+1−1

@@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase):

9090 self.assertEqual(len(results), 1)

9191 self.assertEqual(results[0]['title'], 'Title of track')

9292 self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa')

93− self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')

93+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')

9494 self.assertIn('1000', results[0]['embedded'])

9595

9696 json = """

tests/unit/test_answerers.pynew file+16

@@ -0,0 +1,16 @@

1+# -*- coding: utf-8 -*-

2+

3+from mock import Mock

4+

5+from searx.answerers import answerers

6+from searx.testing import SearxTestCase

7+

8+

9+class AnswererTest(SearxTestCase):

10+

11+ def test_unicode_input(self):

12+ query = Mock()

13+ unicode_payload = u'árvíztűrő tükörfúrógép'

14+ for answerer in answerers:

15+ query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)

16+ self.assertTrue(isinstance(answerer.answer(query), list))

tests/unit/test_webapp.py+2−1

@@ -5,6 +5,7 @@ from mock import Mock

55from urlparse import ParseResult

66from searx import webapp

77from searx.testing import SearxTestCase

8+from searx.search import Search

89

910

1011class ViewsTestCase(SearxTestCase):

@@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase):

4142 results_number=lambda: 3,

4243 results_length=lambda: len(self.test_results))

4344

44− webapp.Search.search = search_mock

45+ Search.search = search_mock

4546

4647 def get_current_theme_name_mock(override=None):

4748 return 'legacy'