e48f07a · searx · git.iko.soy

searx/answerers/__init__.pynew file+46

compare with master

unifiedsplit

@@ -0,0 +1,46 @@

1+from os import listdir

2+from os.path import realpath, dirname, join, isdir

3+from searx.utils import load_module

4+from collections import defaultdict

5+

6+

7+answerers_dir = dirname(realpath(__file__))

8+

9+

10+def load_answerers():

11+ answerers = []

12+ for filename in listdir(answerers_dir):

13+ if not isdir(join(answerers_dir, filename)):

14+ continue

15+ module = load_module('answerer.py', join(answerers_dir, filename))

16+ if not hasattr(module, 'keywords') or not isinstance(module.keywords, tuple) or not len(module.keywords):

17+ exit(2)

18+ answerers.append(module)

19+ return answerers

20+

21+

22+def get_answerers_by_keywords(answerers):

23+ by_keyword = defaultdict(list)

24+ for answerer in answerers:

25+ for keyword in answerer.keywords:

26+ for keyword in answerer.keywords:

27+ by_keyword[keyword].append(answerer.answer)

28+ return by_keyword

29+

30+

31+def ask(query):

32+ results = []

33+ query_parts = filter(None, query.query.split())

34+

35+ if query_parts[0] not in answerers_by_keywords:

36+ return results

37+

38+ for answerer in answerers_by_keywords[query_parts[0]]:

39+ result = answerer(query)

40+ if result:

41+ results.append(result)

42+ return results

43+

44+

45+answerers = load_answerers()

46+answerers_by_keywords = get_answerers_by_keywords(answerers)

searx/answerers/random/answerer.pynew file+50

@@ -0,0 +1,50 @@

1+import random

2+import string

3+from flask_babel import gettext

4+

5+# required answerer attribute

6+# specifies which search query keywords triggers this answerer

7+keywords = ('random',)

8+

9+random_int_max = 2**31

10+

11+random_string_letters = string.lowercase + string.digits + string.uppercase

12+

13+

14+def random_string():

15+ return u''.join(random.choice(random_string_letters)

16+ for _ in range(random.randint(8, 32)))

17+

18+

19+def random_float():

20+ return unicode(random.random())

21+

22+

23+def random_int():

24+ return unicode(random.randint(-random_int_max, random_int_max))

25+

26+

27+random_types = {u'string': random_string,

28+ u'int': random_int,

29+ u'float': random_float}

30+

31+

32+# required answerer function

33+# can return a list of results (any result type) for a given query

34+def answer(query):

35+ parts = query.query.split()

36+ if len(parts) != 2:

37+ return []

38+

39+ if parts[1] not in random_types:

40+ return []

41+

42+ return [{'answer': random_types[parts[1]]()}]

43+

44+

45+# required answerer function

46+# returns information about the answerer

47+def self_info():

48+ return {'name': gettext('Random value generator'),

49+ 'description': gettext('Generate different random values'),

50+ 'examples': [u'random {}'.format(x) for x in random_types]}

searx/answerers/statistics/answerer.pynew file+51

@@ -0,0 +1,51 @@

1+from functools import reduce

2+from operator import mul

3+

4+from flask_babel import gettext

5+

6+keywords = ('min',

7+ 'max',

8+ 'avg',

9+ 'sum',

10+ 'prod')

11+

12+

13+# required answerer function

14+# can return a list of results (any result type) for a given query

15+def answer(query):

16+ parts = query.query.split()

17+

18+ if len(parts) < 2:

19+ return []

20+

21+ try:

22+ args = map(float, parts[1:])

23+ except:

24+ return []

25+

26+ func = parts[0]

27+ answer = None

28+

29+ if func == 'min':

30+ answer = min(args)

31+ elif func == 'max':

32+ answer = max(args)

33+ elif func == 'avg':

34+ answer = sum(args) / len(args)

35+ elif func == 'sum':

36+ answer = sum(args)

37+ elif func == 'prod':

38+ answer = reduce(mul, args, 1)

39+

40+ if answer is None:

41+ return []

42+

43+ return [{'answer': unicode(answer)}]

44+

45+

46+# required answerer function

47+# returns information about the answerer

48+def self_info():

49+ return {'name': gettext('Statistics functions'),

50+ 'description': gettext('Compute {functions} of the arguments').format(functions='/'.join(keywords)),

51+ 'examples': ['avg 123 548 2.04 24.2']}

searx/engines/__init__.py+4−13

@@ -16,13 +16,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

17'''

18

19−from os.path import realpath, dirname, splitext, join

19+from os.path import realpath, dirname

20import sys

21−from imp import load_source

22from flask_babel import gettext

21from flask_babel import gettext

23from operator import itemgetter

22from operator import itemgetter

24from searx import settings

23from searx import settings

25from searx import logger

24from searx import logger

25+from searx.utils import load_module

26

27

28logger = logger.getChild('engines')

@@ -32,6 +32,7 @@ engine_dir = dirname(realpath(__file__))

32engines = {}

33

34categories = {'general': []}

35+_initialized = False

35

36

36engine_shortcuts = {}

37engine_shortcuts = {}

37engine_default_args = {'paging': False,

38engine_default_args = {'paging': False,

@@ -46,16 +47,6 @@ engine_default_args = {'paging': False,

46 'time_range_support': False}

47 'time_range_support': False}

47

48

49

49−def load_module(filename):

50− modname = splitext(filename)[0]

51− if modname in sys.modules:

52− del sys.modules[modname]

53− filepath = join(engine_dir, filename)

54− module = load_source(modname, filepath)

55− module.name = modname

56− return module

57−

58−

59def load_engine(engine_data):

50def load_engine(engine_data):

60

51

61 if '_' in engine_data['name']:

52 if '_' in engine_data['name']:

@@ -65,7 +56,7 @@ def load_engine(engine_data):

65 engine_module = engine_data['engine']

56 engine_module = engine_data['engine']

66

57

67 try:

58 try:

68− engine = load_module(engine_module + '.py')

59+ engine = load_module(engine_module + '.py', engine_dir)

69 except:

60 except:

70 logger.exception('Cannot load engine "{}"'.format(engine_module))

61 logger.exception('Cannot load engine "{}"'.format(engine_module))

71 return None

62 return None

searx/engines/archlinux.py+1−2

@@ -12,7 +12,6 @@

12"""

13

14from urlparse import urljoin

15−from cgi import escape

16from urllib import urlencode

15from urllib import urlencode

17from lxml import html

16from lxml import html

18from searx.engines.xpath import extract_text

17from searx.engines.xpath import extract_text

@@ -135,7 +134,7 @@ def response(resp):

135 for result in dom.xpath(xpath_results):

134 for result in dom.xpath(xpath_results):

136 link = result.xpath(xpath_link)[0]

135 link = result.xpath(xpath_link)[0]

137 href = urljoin(base_url, link.attrib.get('href'))

136 href = urljoin(base_url, link.attrib.get('href'))

138− title = escape(extract_text(link))

137+ title = extract_text(link)

139

138

140 results.append({'url': href,

139 results.append({'url': href,

141 'title': title})

140 'title': title})

searx/engines/base.py+1−2

@@ -16,7 +16,6 @@

16from lxml import etree

17from urllib import urlencode

18from searx.utils import searx_useragent

19−from cgi import escape

20from datetime import datetime

19from datetime import datetime

21import re

20import re

22

21

@@ -94,7 +93,7 @@ def response(resp):

94 url = item.text

93 url = item.text

95

94

96 elif item.attrib["name"] == "dcdescription":

95 elif item.attrib["name"] == "dcdescription":

97− content = escape(item.text[:300])

96+ content = item.text[:300]

98 if len(item.text) > 300:

97 if len(item.text) > 300:

99 content += "..."

98 content += "..."

100

99

searx/engines/bing.py+6−11

@@ -14,7 +14,6 @@

14"""

15

16from urllib import urlencode

17−from cgi import escape

18from lxml import html

17from lxml import html

19from searx.engines.xpath import extract_text

18from searx.engines.xpath import extract_text

20

19

@@ -32,18 +31,14 @@ search_string = 'search?{query}&first={offset}'

32def request(query, params):

31def request(query, params):

33 offset = (params['pageno'] - 1) * 10 + 1

32 offset = (params['pageno'] - 1) * 10 + 1

34

33

35− if params['language'] == 'all':

34+ if params['language'] != 'all':

36− language = 'en-US'

35+ query = u'language:{} {}'.format(params['language'].split('_')[0].upper(),

37− else:

36+ query.decode('utf-8')).encode('utf-8')

38− language = params['language'].replace('_', '-')

39

37

40 search_path = search_string.format(

38 search_path = search_string.format(

41− query=urlencode({'q': query, 'setmkt': language}),

39+ query=urlencode({'q': query}),

42 offset=offset)

40 offset=offset)

43

41

44− params['cookies']['SRCHHPGUSR'] = \

45− 'NEWWND=0&NRSLT=-1&SRCHLANG=' + language.split('-')[0]

46−

47 params['url'] = base_url + search_path

42 params['url'] = base_url + search_path

48 return params

43 return params

49

44

@@ -65,7 +60,7 @@ def response(resp):

65 link = result.xpath('.//h3/a')[0]

60 link = result.xpath('.//h3/a')[0]

66 url = link.attrib.get('href')

61 url = link.attrib.get('href')

67 title = extract_text(link)

62 title = extract_text(link)

68− content = escape(extract_text(result.xpath('.//p')))

63+ content = extract_text(result.xpath('.//p'))

69

64

70 # append result

65 # append result

71 results.append({'url': url,

66 results.append({'url': url,

@@ -77,7 +72,7 @@ def response(resp):

77 link = result.xpath('.//h2/a')[0]

72 link = result.xpath('.//h2/a')[0]

78 url = link.attrib.get('href')

73 url = link.attrib.get('href')

79 title = extract_text(link)

74 title = extract_text(link)

80− content = escape(extract_text(result.xpath('.//p')))

75+ content = extract_text(result.xpath('.//p'))

81

76

82 # append result

77 # append result

83 results.append({'url': url,

78 results.append({'url': url,

searx/engines/btdigg.py+2−3

@@ -11,7 +11,6 @@

11"""

12

13from urlparse import urljoin

14−from cgi import escape

15from urllib import quote

14from urllib import quote

16from lxml import html

15from lxml import html

17from operator import itemgetter

16from operator import itemgetter

@@ -51,8 +50,8 @@ def response(resp):

51 for result in search_res:

50 for result in search_res:

52 link = result.xpath('.//td[@class="torrent_name"]//a')[0]

51 link = result.xpath('.//td[@class="torrent_name"]//a')[0]

53 href = urljoin(url, link.attrib.get('href'))

52 href = urljoin(url, link.attrib.get('href'))

54− title = escape(extract_text(link))

53+ title = extract_text(link)

55− content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))

54+ content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])

56 content = "<br />".join(content.split("\n"))

55 content = "<br />".join(content.split("\n"))

57

56

58 filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

57 filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

searx/engines/dailymotion.py+1−2

@@ -14,7 +14,6 @@

14

15from urllib import urlencode

16from json import loads

17−from cgi import escape

18from datetime import datetime

17from datetime import datetime

19

18

20# engine dependent config

19# engine dependent config

@@ -57,7 +56,7 @@ def response(resp):

57 for res in search_res['list']:

56 for res in search_res['list']:

58 title = res['title']

57 title = res['title']

59 url = res['url']

58 url = res['url']

60− content = escape(res['description'])

59+ content = res['description']

61 thumbnail = res['thumbnail_360_url']

60 thumbnail = res['thumbnail_360_url']

62 publishedDate = datetime.fromtimestamp(res['created_time'], None)

61 publishedDate = datetime.fromtimestamp(res['created_time'], None)

63 embedded = embedded_url.format(videoid=res['id'])

62 embedded = embedded_url.format(videoid=res['id'])

searx/engines/deezer.py+5−4

@@ -51,10 +51,11 @@ def response(resp):

51 if url.startswith('http://'):

52 url = 'https' + url[4:]

53

54+ content = '{} - {} - {}'.format(

54− content = result['artist']['name'] +\

55+ result['artist']['name'],

55− " • " +\

56− result['album']['title'] +\

56+ result['album']['title'],

57− " • " + result['title']

57+ result['title'])

58+

58 embedded = embedded_url.format(audioid=result['id'])

59 embedded = embedded_url.format(audioid=result['id'])

59

60

60 # append result

61 # append result

searx/engines/dictzone.py+2−3

@@ -12,7 +12,6 @@

12import re

13from urlparse import urljoin

14from lxml import html

15−from cgi import escape

16from searx.utils import is_valid_lang

15from searx.utils import is_valid_lang

17

16

18categories = ['general']

17categories = ['general']

@@ -62,8 +61,8 @@ def response(resp):

62

61

63 results.append({

62 results.append({

64 'url': urljoin(resp.url, '?%d' % k),

63 'url': urljoin(resp.url, '?%d' % k),

65− 'title': escape(from_result.text_content()),

64+ 'title': from_result.text_content(),

66− 'content': escape('; '.join(to_results))

65+ 'content': '; '.join(to_results)

67 })

66 })

68

67

69 return results

68 return results

searx/engines/digg.py+1−2

@@ -13,7 +13,6 @@

13from urllib import quote_plus

14from json import loads

15from lxml import html

16−from cgi import escape

17from dateutil import parser

16from dateutil import parser

18

17

19# engine dependent config

18# engine dependent config

@@ -56,7 +55,7 @@ def response(resp):

56 url = result.attrib.get('data-contenturl')

55 url = result.attrib.get('data-contenturl')

57 thumbnail = result.xpath('.//img')[0].attrib.get('src')

56 thumbnail = result.xpath('.//img')[0].attrib.get('src')

58 title = ''.join(result.xpath(title_xpath))

57 title = ''.join(result.xpath(title_xpath))

59− content = escape(''.join(result.xpath(content_xpath)))

58+ content = ''.join(result.xpath(content_xpath))

60 pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

59 pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

61 publishedDate = parser.parse(pubdate)

60 publishedDate = parser.parse(pubdate)

62

61

searx/engines/fdroid.py+1−2

@@ -9,7 +9,6 @@

9 @parse url, title, content

10"""

11

12−from cgi import escape

13from urllib import urlencode

12from urllib import urlencode

14from searx.engines.xpath import extract_text

13from searx.engines.xpath import extract_text

15from lxml import html

14from lxml import html

@@ -43,7 +42,7 @@ def response(resp):

43 img_src = app.xpath('.//img/@src')[0]

42 img_src = app.xpath('.//img/@src')[0]

44

43

45 content = extract_text(app.xpath('./p')[0])

44 content = extract_text(app.xpath('./p')[0])

46− content = escape(content.replace(title, '', 1).strip())

45+ content = content.replace(title, '', 1).strip()

47

46

48 results.append({'url': url,

47 results.append({'url': url,

49 'title': title,

48 'title': title,

searx/engines/flickr.py+3−11

@@ -77,21 +77,13 @@ def response(resp):

77

78 url = build_flickr_url(photo['owner'], photo['id'])

79

80− title = photo['title']

81−

82− content = '<span class="photo-author">' +\

83− photo['ownername'] +\

84− '</span><br />' +\

85− '<span class="description">' +\

86− photo['description']['_content'] +\

87− '</span>'

88−

89 # append result

80 # append result

90 results.append({'url': url,

81 results.append({'url': url,

91− 'title': title,

82+ 'title': photo['title'],

92 'img_src': img_src,

83 'img_src': img_src,

93 'thumbnail_src': thumbnail_src,

84 'thumbnail_src': thumbnail_src,

94− 'content': content,

85+ 'content': photo['description']['_content'],

86+ 'author': photo['ownername'],

95 'template': 'images.html'})

87 'template': 'images.html'})

96

88

97 # return results

89 # return results

searx/engines/flickr_noapi.py+3−4

@@ -102,16 +102,15 @@ def response(resp):

102

103 title = photo.get('title', '')

104

105− content = '<span class="photo-author">' +\

106− photo['username'] +\

105+ author = photo['username']

107− '</span><br />'

108

106

109 # append result

107 # append result

110 results.append({'url': url,

108 results.append({'url': url,

111 'title': title,

109 'title': title,

112 'img_src': img_src,

110 'img_src': img_src,

113 'thumbnail_src': thumbnail_src,

111 'thumbnail_src': thumbnail_src,

114− 'content': content,

112+ 'content': '',

113+ 'author': author,

115 'template': 'images.html'})

114 'template': 'images.html'})

116

115

117 return results

116 return results

searx/engines/gigablast.py+2−3

@@ -10,7 +10,6 @@

10 @parse url, title, content

11"""

12

13−from cgi import escape

14from json import loads

13from json import loads

15from random import randint

14from random import randint

16from time import time

15from time import time

@@ -78,8 +77,8 @@ def response(resp):

78 for result in response_json['results']:

77 for result in response_json['results']:

79 # append result

78 # append result

80 results.append({'url': result['url'],

79 results.append({'url': result['url'],

81− 'title': escape(result['title']),

80+ 'title': result['title'],

82− 'content': escape(result['sum'])})

81+ 'content': result['sum']})

83

82

84 # return results

83 # return results

85 return results

84 return results

searx/engines/github.py+1−2

@@ -12,7 +12,6 @@

12

13from urllib import urlencode

14from json import loads

15−from cgi import escape

16

15

17# engine dependent config

16# engine dependent config

18categories = ['it']

17categories = ['it']

@@ -48,7 +47,7 @@ def response(resp):

48 url = res['html_url']

47 url = res['html_url']

49

48

50 if res['description']:

49 if res['description']:

51− content = escape(res['description'][:500])

50+ content = res['description'][:500]

52 else:

51 else:

53 content = ''

52 content = ''

54

53

searx/engines/google.py+2−3

@@ -9,7 +9,6 @@

9# @parse url, title, content, suggestion

10

11import re

12−from cgi import escape

13from urllib import urlencode

12from urllib import urlencode

14from urlparse import urlparse, parse_qsl

13from urlparse import urlparse, parse_qsl

15from lxml import html, etree

14from lxml import html, etree

@@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):

155def extract_text_from_dom(result, xpath):

154def extract_text_from_dom(result, xpath):

156 r = result.xpath(xpath)

155 r = result.xpath(xpath)

157 if len(r) > 0:

156 if len(r) > 0:

158− return escape(extract_text(r[0]))

157+ return extract_text(r[0])

159 return None

158 return None

160

159

161

160

@@ -264,7 +263,7 @@ def response(resp):

264 # parse suggestion

263 # parse suggestion

265 for suggestion in dom.xpath(suggestion_xpath):

264 for suggestion in dom.xpath(suggestion_xpath):

266 # append suggestion

265 # append suggestion

267− results.append({'suggestion': escape(extract_text(suggestion))})

266+ results.append({'suggestion': extract_text(suggestion)})

268

267

269 # return results

268 # return results

270 return results

269 return results

searx/engines/kickass.py+1−2

@@ -11,7 +11,6 @@

11"""

12

13from urlparse import urljoin

14−from cgi import escape

15from urllib import quote

14from urllib import quote

16from lxml import html

15from lxml import html

17from operator import itemgetter

16from operator import itemgetter

@@ -57,7 +56,7 @@ def response(resp):

57 link = result.xpath('.//a[@class="cellMainLink"]')[0]

56 link = result.xpath('.//a[@class="cellMainLink"]')[0]

58 href = urljoin(url, link.attrib['href'])

57 href = urljoin(url, link.attrib['href'])

59 title = extract_text(link)

58 title = extract_text(link)

60− content = escape(extract_text(result.xpath(content_xpath)))

59+ content = extract_text(result.xpath(content_xpath))

61 seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))

60 seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))

62 leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))

61 leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))

63 filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

62 filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

searx/engines/nyaa.py+2−4

@@ -9,7 +9,6 @@

9 @parse url, title, content, seed, leech, torrentfile

10"""

11

12−from cgi import escape

13from urllib import urlencode

12from urllib import urlencode

14from lxml import html

13from lxml import html

15from searx.engines.xpath import extract_text

14from searx.engines.xpath import extract_text

@@ -78,7 +77,7 @@ def response(resp):

78

77

79 # torrent title

78 # torrent title

80 page_a = result.xpath(xpath_title)[0]

79 page_a = result.xpath(xpath_title)[0]

81− title = escape(extract_text(page_a))

80+ title = extract_text(page_a)

82

81

83 # link to the page

82 # link to the page

84 href = page_a.attrib.get('href')

83 href = page_a.attrib.get('href')

@@ -90,7 +89,7 @@ def response(resp):

90 try:

89 try:

91 file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')

90 file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')

92 file_size = int(float(file_size) * get_filesize_mul(suffix))

91 file_size = int(float(file_size) * get_filesize_mul(suffix))

93− except Exception as e:

92+ except:

94 file_size = None

93 file_size = None

95

94

96 # seed count

95 # seed count

@@ -105,7 +104,6 @@ def response(resp):

105 # content string contains all information not included into template

104 # content string contains all information not included into template

106 content = 'Category: "{category}". Downloaded {downloads} times.'

105 content = 'Category: "{category}". Downloaded {downloads} times.'

107 content = content.format(category=category, downloads=downloads)

106 content = content.format(category=category, downloads=downloads)

108− content = escape(content)

109

107

110 results.append({'url': href,

108 results.append({'url': href,

111 'title': title,

109 'title': title,

searx/engines/openstreetmap.py+1−1

@@ -43,7 +43,7 @@ def response(resp):

43 if 'display_name' not in r:

44 continue

45

46− title = r['display_name']

46+ title = r['display_name'] or u''

47 osm_type = r.get('osm_type', r.get('type'))

48 url = result_base_url.format(osm_type=osm_type,

49 osm_id=r['osm_id'])

searx/engines/piratebay.py+1−2

@@ -9,7 +9,6 @@

9# @parse url, title, content, seed, leech, magnetlink

10

11from urlparse import urljoin

12−from cgi import escape

13from urllib import quote

12from urllib import quote

14from lxml import html

13from lxml import html

15from operator import itemgetter

14from operator import itemgetter

@@ -62,7 +61,7 @@ def response(resp):

62 link = result.xpath('.//div[@class="detName"]//a')[0]

61 link = result.xpath('.//div[@class="detName"]//a')[0]

63 href = urljoin(url, link.attrib.get('href'))

62 href = urljoin(url, link.attrib.get('href'))

64 title = extract_text(link)

63 title = extract_text(link)

65− content = escape(extract_text(result.xpath(content_xpath)))

64+ content = extract_text(result.xpath(content_xpath))

66 seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]

65 seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]

67

66

68 # convert seed to int if possible

67 # convert seed to int if possible

searx/engines/reddit.py+1−2

@@ -11,7 +11,6 @@

11"""

12

13import json

14−from cgi import escape

15from urllib import urlencode

14from urllib import urlencode

16from urlparse import urlparse, urljoin

15from urlparse import urlparse, urljoin

17from datetime import datetime

16from datetime import datetime

@@ -68,7 +67,7 @@ def response(resp):

68 img_results.append(params)

67 img_results.append(params)

69 else:

68 else:

70 created = datetime.fromtimestamp(data['created_utc'])

69 created = datetime.fromtimestamp(data['created_utc'])

71− content = escape(data['selftext'])

70+ content = data['selftext']

72 if len(content) > 500:

71 if len(content) > 500:

73 content = content[:500] + '...'

72 content = content[:500] + '...'

74 params['content'] = content

73 params['content'] = content

searx/engines/searchcode_doc.py+2−10

@@ -44,20 +44,12 @@ def response(resp):

44 # parse results

45 for result in search_results.get('results', []):

46 href = result['url']

47− title = "[" + result['type'] + "] " +\

47+ title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])

48− result['namespace'] +\

49− " " + result['name']

50− content = '<span class="highlight">[' +\

51− result['type'] + "] " +\

52− result['name'] + " " +\

53− result['synopsis'] +\

54− "</span><br />" +\

55− result['description']

56

48

57 # append result

49 # append result

58 results.append({'url': href,

50 results.append({'url': href,

59 'title': title,

51 'title': title,

60− 'content': content})

52+ 'content': result['description']})

61

53

62 # return results

54 # return results

63 return results

55 return results

searx/engines/seedpeer.py−1

@@ -9,7 +9,6 @@

9# @parse url, title, content, seed, leech, magnetlink

10

11from urlparse import urljoin

12−from cgi import escape

13from urllib import quote

12from urllib import quote

14from lxml import html

13from lxml import html

15from operator import itemgetter

14from operator import itemgetter

searx/engines/spotify.py+5−4

@@ -46,10 +46,11 @@ def response(resp):

46 if result['type'] == 'track':

47 title = result['name']

48 url = result['external_urls']['spotify']

49+ content = '{} - {} - {}'.format(

49− content = result['artists'][0]['name'] +\

50+ result['artists'][0]['name'],

50− " • " +\

51− result['album']['name'] +\

51+ result['album']['name'],

52− " • " + result['name']

52+ result['name'])

53+

53 embedded = embedded_url.format(audioid=result['id'])

54 embedded = embedded_url.format(audioid=result['id'])

54

55

55 # append result

56 # append result

searx/engines/stackoverflow.py+2−3

@@ -11,7 +11,6 @@

11"""

12

13from urlparse import urljoin

14−from cgi import escape

15from urllib import urlencode

14from urllib import urlencode

16from lxml import html

15from lxml import html

17from searx.engines.xpath import extract_text

16from searx.engines.xpath import extract_text

@@ -48,8 +47,8 @@ def response(resp):

48 for result in dom.xpath(results_xpath):

47 for result in dom.xpath(results_xpath):

49 link = result.xpath(link_xpath)[0]

48 link = result.xpath(link_xpath)[0]

50 href = urljoin(url, link.attrib.get('href'))

49 href = urljoin(url, link.attrib.get('href'))

51− title = escape(extract_text(link))

50+ title = extract_text(link)

52− content = escape(extract_text(result.xpath(content_xpath)))

51+ content = extract_text(result.xpath(content_xpath))

53

52

54 # append result

53 # append result

55 results.append({'url': href,

54 results.append({'url': href,

searx/engines/startpage.py+2−3

@@ -11,7 +11,6 @@

11# @todo paging

12

13from lxml import html

14−from cgi import escape

15from dateutil import parser

14from dateutil import parser

16from datetime import datetime, timedelta

15from datetime import datetime, timedelta

17import re

16import re

@@ -79,10 +78,10 @@ def response(resp):

79 if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):

78 if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):

80 continue

79 continue

81

80

82− title = escape(extract_text(link))

81+ title = extract_text(link)

83

82

84 if result.xpath('./p[@class="desc clk"]'):

83 if result.xpath('./p[@class="desc clk"]'):

85− content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))

84+ content = extract_text(result.xpath('./p[@class="desc clk"]'))

86 else:

85 else:

87 content = ''

86 content = ''

88

87

searx/engines/subtitleseeker.py+2−3

@@ -10,7 +10,6 @@

10 @parse url, title, content

11"""

12

13−from cgi import escape

14from urllib import quote_plus

13from urllib import quote_plus

15from lxml import html

14from lxml import html

16from searx.languages import language_codes

15from searx.languages import language_codes

@@ -59,7 +58,7 @@ def response(resp):

59 elif search_lang:

58 elif search_lang:

60 href = href + search_lang + '/'

59 href = href + search_lang + '/'

61

60

62− title = escape(extract_text(link))

61+ title = extract_text(link)

63

62

64 content = extract_text(result.xpath('.//div[contains(@class,"red")]'))

63 content = extract_text(result.xpath('.//div[contains(@class,"red")]'))

65 content = content + " - "

64 content = content + " - "

@@ -75,7 +74,7 @@ def response(resp):

75 # append result

74 # append result

76 results.append({'url': href,

75 results.append({'url': href,

77 'title': title,

76 'title': title,

78− 'content': escape(content)})

77+ 'content': content})

79

78

80 # return results

79 # return results

81 return results

80 return results

searx/engines/swisscows.py+4−5

@@ -10,7 +10,6 @@

10 @parse url, title, content

11"""

12

13−from cgi import escape

14from json import loads

13from json import loads

15from urllib import urlencode, unquote

14from urllib import urlencode, unquote

16import re

15import re

@@ -78,7 +77,7 @@ def response(resp):

78

77

79 # append result

78 # append result

80 results.append({'url': result['SourceUrl'],

79 results.append({'url': result['SourceUrl'],

81− 'title': escape(result['Title']),

80+ 'title': result['Title'],

82 'content': '',

81 'content': '',

83 'img_src': img_url,

82 'img_src': img_url,

84 'template': 'images.html'})

83 'template': 'images.html'})

@@ -90,8 +89,8 @@ def response(resp):

90

89

91 # append result

90 # append result

92 results.append({'url': result_url,

91 results.append({'url': result_url,

93− 'title': escape(result_title),

92+ 'title': result_title,

94− 'content': escape(result_content)})

93+ 'content': result_content})

95

94

96 # parse images

95 # parse images

97 for result in json.get('Images', []):

96 for result in json.get('Images', []):

@@ -100,7 +99,7 @@ def response(resp):

100

99

101 # append result

100 # append result

102 results.append({'url': result['SourceUrl'],

101 results.append({'url': result['SourceUrl'],

103− 'title': escape(result['Title']),

102+ 'title': result['Title'],

104 'content': '',

103 'content': '',

105 'img_src': img_url,

104 'img_src': img_url,

106 'template': 'images.html'})

105 'template': 'images.html'})

searx/engines/tokyotoshokan.py−1

@@ -11,7 +11,6 @@

11"""

12

13import re

14−from cgi import escape

15from urllib import urlencode

14from urllib import urlencode

16from lxml import html

15from lxml import html

17from searx.engines.xpath import extract_text

16from searx.engines.xpath import extract_text

searx/engines/torrentz.py−1

@@ -12,7 +12,6 @@

12"""

13

14import re

15−from cgi import escape

16from urllib import urlencode

15from urllib import urlencode

17from lxml import html

16from lxml import html

18from searx.engines.xpath import extract_text

17from searx.engines.xpath import extract_text

searx/engines/translated.py+5−6

@@ -9,7 +9,6 @@

9 @parse url, title, content

10"""

11import re

12−from cgi import escape

13from searx.utils import is_valid_lang

12from searx.utils import is_valid_lang

14

13

15categories = ['general']

14categories = ['general']

@@ -52,14 +51,14 @@ def request(query, params):

52def response(resp):

51def response(resp):

53 results = []

52 results = []

54 results.append({

53 results.append({

55− 'url': escape(web_url.format(

54+ 'url': web_url.format(

56 from_lang=resp.search_params['from_lang'][2],

55 from_lang=resp.search_params['from_lang'][2],

57 to_lang=resp.search_params['to_lang'][2],

56 to_lang=resp.search_params['to_lang'][2],

58− query=resp.search_params['query'])),

57+ query=resp.search_params['query']),

59− 'title': escape('[{0}-{1}] {2}'.format(

58+ 'title': '[{0}-{1}] {2}'.format(

60 resp.search_params['from_lang'][1],

59 resp.search_params['from_lang'][1],

61 resp.search_params['to_lang'][1],

60 resp.search_params['to_lang'][1],

62− resp.search_params['query'])),

61+ resp.search_params['query']),

63− 'content': escape(resp.json()['responseData']['translatedText'])

62+ 'content': resp.json()['responseData']['translatedText']

64 })

63 })

65 return results

64 return results

searx/engines/wolframalpha_noapi.py−1

@@ -8,7 +8,6 @@

8# @stable no

9# @parse url, infobox

10

11−from cgi import escape

12from json import loads

11from json import loads

13from time import time

12from time import time

14from urllib import urlencode

13from urllib import urlencode

searx/engines/yandex.py+2−3

@@ -9,7 +9,6 @@

9 @parse url, title, content

10"""

11

12−from cgi import escape

13from urllib import urlencode

12from urllib import urlencode

14from lxml import html

13from lxml import html

15from searx.search import logger

14from searx.search import logger

@@ -52,8 +51,8 @@ def response(resp):

52 for result in dom.xpath(results_xpath):

51 for result in dom.xpath(results_xpath):

53 try:

52 try:

54 res = {'url': result.xpath(url_xpath)[0],

53 res = {'url': result.xpath(url_xpath)[0],

55− 'title': escape(''.join(result.xpath(title_xpath))),

54+ 'title': ''.join(result.xpath(title_xpath)),

56− 'content': escape(''.join(result.xpath(content_xpath)))}

55+ 'content': ''.join(result.xpath(content_xpath))}

57 except:

56 except:

58 logger.exception('yandex parse crash')

57 logger.exception('yandex parse crash')

59 continue

58 continue

searx/plugins/doai_rewrite.py+1−1

@@ -27,5 +27,5 @@ def on_result(request, search, result):

27 if doi.endswith(suffix):

28 doi = doi[:-len(suffix)]

29 result['url'] = 'http://doai.io/' + doi

30− result['parsed_url'] = urlparse(ctx['result']['url'])

30+ result['parsed_url'] = urlparse(result['url'])

31 return True

searx/preferences.py+17−14

@@ -49,28 +49,32 @@ class StringSetting(Setting):

49class EnumStringSetting(Setting):

50 """Setting of a value which can only come from the given choices"""

51

52+ def _validate_selection(self, selection):

53+ if selection not in self.choices:

54+ raise ValidationException('Invalid value: "{0}"'.format(selection))

55+

52 def _post_init(self):

56 def _post_init(self):

53 if not hasattr(self, 'choices'):

57 if not hasattr(self, 'choices'):

54 raise MissingArgumentException('Missing argument: choices')

58 raise MissingArgumentException('Missing argument: choices')

55−

59+ self._validate_selection(self.value)

56− if self.value != '' and self.value not in self.choices:

57− raise ValidationException('Invalid default value: {0}'.format(self.value))

58

60

59 def parse(self, data):

61 def parse(self, data):

60− if data not in self.choices and data != self.value:

62+ self._validate_selection(data)

61− raise ValidationException('Invalid choice: {0}'.format(data))

62 self.value = data

63 self.value = data

63

64

65

65class MultipleChoiceSetting(EnumStringSetting):

66class MultipleChoiceSetting(EnumStringSetting):

66 """Setting of values which can only come from the given choices"""

67 """Setting of values which can only come from the given choices"""

67

68

69+ def _validate_selections(self, selections):

70+ for item in selections:

71+ if item not in self.choices:

72+ raise ValidationException('Invalid value: "{0}"'.format(selections))

73+

68 def _post_init(self):

74 def _post_init(self):

69 if not hasattr(self, 'choices'):

75 if not hasattr(self, 'choices'):

70 raise MissingArgumentException('Missing argument: choices')

76 raise MissingArgumentException('Missing argument: choices')

71− for item in self.value:

77+ self._validate_selections(self.value)

72− if item not in self.choices:

73− raise ValidationException('Invalid default value: {0}'.format(self.value))

74

78

75 def parse(self, data):

79 def parse(self, data):

76 if data == '':

80 if data == '':

@@ -78,9 +82,7 @@ class MultipleChoiceSetting(EnumStringSetting):

78 return

82 return

79

83

80 elements = data.split(',')

84 elements = data.split(',')

81− for item in elements:

85+ self._validate_selections(elements)

82− if item not in self.choices:

83− raise ValidationException('Invalid choice: {0}'.format(item))

84 self.value = elements

86 self.value = elements

85

87

86 def parse_form(self, data):

88 def parse_form(self, data):

@@ -214,11 +216,12 @@ class Preferences(object):

214 super(Preferences, self).__init__()

216 super(Preferences, self).__init__()

215

217

216 self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),

218 self.key_value_settings = {'categories': MultipleChoiceSetting(['general'], choices=categories),

219+ 'language': EnumStringSetting(settings['search']['language'],

217− 'language': EnumStringSetting('all', choices=LANGUAGE_CODES),

220+ choices=LANGUAGE_CODES),

218 'locale': EnumStringSetting(settings['ui']['default_locale'],

221 'locale': EnumStringSetting(settings['ui']['default_locale'],

219− choices=settings['locales'].keys()),

222+ choices=settings['locales'].keys() + ['']),

220 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],

223 'autocomplete': EnumStringSetting(settings['search']['autocomplete'],

221− choices=autocomplete.backends.keys()),

224+ choices=autocomplete.backends.keys() + ['']),

222 'image_proxy': MapSetting(settings['server']['image_proxy'],

225 'image_proxy': MapSetting(settings['server']['image_proxy'],

223 map={'': settings['server']['image_proxy'],

226 map={'': settings['server']['image_proxy'],

224 '0': False,

227 '0': False,

searx/results.py+5−4

@@ -146,16 +146,17 @@ class ResultContainer(object):

146 self._number_of_results.append(result['number_of_results'])

147 results.remove(result)

148

149+ if engine_name in engines:

149− with RLock():

150+ with RLock():

150− engines[engine_name].stats['search_count'] += 1

151+ engines[engine_name].stats['search_count'] += 1

151− engines[engine_name].stats['result_count'] += len(results)

152+ engines[engine_name].stats['result_count'] += len(results)

152

153

153 if not results:

154 if not results:

154 return

155 return

155

156

156 self.results[engine_name].extend(results)

157 self.results[engine_name].extend(results)

157

158

158− if not self.paging and engines[engine_name].paging:

159+ if not self.paging and engine_name in engines and engines[engine_name].paging:

159 self.paging = True

160 self.paging = True

160

161

161 for i, result in enumerate(results):

162 for i, result in enumerate(results):

searx/search.py+9

@@ -24,6 +24,7 @@ import searx.poolrequests as requests_lib

24from searx.engines import (

25 categories, engines

26)

27+from searx.answerers import ask

27from searx.utils import gen_useragent

28from searx.utils import gen_useragent

28from searx.query import RawTextQuery, SearchQuery

29from searx.query import RawTextQuery, SearchQuery

29from searx.results import ResultContainer

30from searx.results import ResultContainer

@@ -300,6 +301,14 @@ class Search(object):

300 # start time

301 # start time

301 start_time = time()

302 start_time = time()

302

303

304+ # answeres ?

305+ answerers_results = ask(self.search_query)

306+

307+ if answerers_results:

308+ for results in answerers_results:

309+ self.result_container.extend('answer', results)

310+ return self.result_container

311+

303 # init vars

312 # init vars

304 requests = []

313 requests = []

305

314

searx/settings.yml+1

@@ -5,6 +5,7 @@ general:

5search:

6 safe_search : 0 # Filter results. 0: None, 1: Moderate, 2: Strict

7 autocomplete : "" # Existing autocomplete backends: "dbpedia", "duckduckgo", "google", "startpage", "wikipedia" - leave blank to turn it off by default

8+ language : "all"

8

9

9server:

10server:

10 port : 8888

11 port : 8888

searx/settings_robot.yml+1

@@ -5,6 +5,7 @@ general:

5search:

6 safe_search : 0

7 autocomplete : ""

8+ language: "all"

8

9

9server:

10server:

10 port : 11111

11 port : 11111

searx/static/plugins/js/infinite_scroll.js+1−1

@@ -5,7 +5,7 @@ $(document).ready(function() {

5 var formData = $('#pagination form:last').serialize();

6 if (formData) {

7 $('#pagination').html('<div class="loading-spinner"></div>');

8− $.post('/', formData, function (data) {

8+ $.post('./', formData, function (data) {

9 var body = $(data);

10 $('#pagination').remove();

11 $('#main_results').append('<hr/>');

searx/templates/courgette/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

3 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

4 xmlns:atom="http://www.w3.org/2005/Atom">

5 <channel>

6− <title>Searx search: {{ q }}</title>

6+ <title>Searx search: {{ q|e }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

8+ <description>Search results for "{{ q|e }}" - searx</description>

9 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

10 <opensearch:startIndex>1</opensearch:startIndex>

11 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

12 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

14 {% for r in results %}

15 <item>

16 <title>{{ r.title }}</title>

searx/templates/courgette/results.html+5−5

@@ -1,6 +1,6 @@

1{% extends "courgette/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

4{% block content %}

5<div class="right"><a href="{{ url_for('preferences') }}" id="preferences"><span>{{ _('preferences') }}</span></a></div>

6<div class="small search center">

@@ -17,7 +17,7 @@

17 {% for output_type in ('csv', 'json', 'rss') %}

18 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

19 <div class="left">

20− <input type="hidden" name="q" value="{{ q }}" />

20+ <input type="hidden" name="q" value="{{ q|e }}" />

21 <input type="hidden" name="format" value="{{ output_type }}" />

22 {% for category in selected_categories %}

23 <input type="hidden" name="category_{{ category }}" value="1"/>

@@ -62,7 +62,7 @@

62 {% if pageno > 1 %}

63 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

64 <div class="left">

65− <input type="hidden" name="q" value="{{ q }}" />

65+ <input type="hidden" name="q" value="{{ q|e }}" />

66 {% for category in selected_categories %}

67 <input type="hidden" name="category_{{ category }}" value="1"/>

68 {% endfor %}

@@ -76,7 +76,7 @@

76 {% for category in selected_categories %}

77 <input type="hidden" name="category_{{ category }}" value="1"/>

78 {% endfor %}

79− <input type="hidden" name="q" value="{{ q }}" />

79+ <input type="hidden" name="q" value="{{ q|e }}" />

80 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

81 <input type="submit" value="{{ _('next page') }} >>" />

82 </div>

searx/templates/legacy/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

3 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

4 xmlns:atom="http://www.w3.org/2005/Atom">

5 <channel>

6− <title>Searx search: {{ q }}</title>

6+ <title>Searx search: {{ q|e }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

8+ <description>Search results for "{{ q|e }}" - searx</description>

9 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

10 <opensearch:startIndex>1</opensearch:startIndex>

11 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

12 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

14 {% for r in results %}

15 <item>

16 <title>{{ r.title }}</title>

searx/templates/legacy/results.html+5−5

@@ -1,6 +1,6 @@

1{% extends "legacy/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}">{% endblock %}

4{% block content %}

5<div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>

6<div class="small search center">

@@ -18,7 +18,7 @@

18 {% for output_type in ('csv', 'json', 'rss') %}

19 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

20 <div class="left">

21− <input type="hidden" name="q" value="{{ q }}" />

21+ <input type="hidden" name="q" value="{{ q|e }}" />

22 <input type="hidden" name="format" value="{{ output_type }}" />

23 {% for category in selected_categories %}

24 <input type="hidden" name="category_{{ category }}" value="1"/>

@@ -73,7 +73,7 @@

73 {% if pageno > 1 %}

74 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

75 <div class="{% if rtl %}right{% else %}left{% endif %}">

76− <input type="hidden" name="q" value="{{ q }}" />

76+ <input type="hidden" name="q" value="{{ q|e }}" />

77 {% for category in selected_categories %}

78 <input type="hidden" name="category_{{ category }}" value="1"/>

79 {% endfor %}

@@ -87,7 +87,7 @@

87 {% for category in selected_categories %}

88 <input type="hidden" name="category_{{ category }}" value="1"/>

89 {% endfor %}

90− <input type="hidden" name="q" value="{{ q }}" />

90+ <input type="hidden" name="q" value="{{ q|e }}" />

91 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

92 <input type="submit" value="{{ _('next page') }} >>" />

93 </div>

searx/templates/oscar/base.html+15

@@ -1,3 +1,4 @@

1+{% from 'oscar/macros.html' import icon %}

1<!DOCTYPE html>

2<!DOCTYPE html>

2<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>

3<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"{% if rtl %} dir="rtl"{% endif %}>

3<head>

4<head>

@@ -54,6 +55,20 @@

54<body>

55<body>

55 {% include 'oscar/navbar.html' %}

56 {% include 'oscar/navbar.html' %}

56 <div class="container">

57 <div class="container">

58+ {% if errors %}

59+ <div class="alert alert-danger fade in" role="alert">

60+ <button class="close" data-dismiss="alert" type="button">

61+ <span aria-hidden="true">×</span>

62+ <span class="sr-only">{{ _('Close') }}</span>

63+ </button>

64+ <strong class="lead">{{ icon('info-sign') }} {{ _('Error!') }}</strong>

65+ <ul>

66+ {% for message in errors %}

67+ <li>{{ message }}</li>

68+ {% endfor %}

69+ </ul>

70+ </div>

71+ {% endif %}

57

72

58 {% block site_alert_error %}

73 {% block site_alert_error %}

59 {% endblock %}

74 {% endblock %}

searx/templates/oscar/opensearch_response_rss.xml+4−4

@@ -3,14 +3,14 @@

3 xmlns:opensearch="http://a9.com/-/spec/opensearch/1.1/"

4 xmlns:atom="http://www.w3.org/2005/Atom">

5 <channel>

6− <title>Searx search: {{ q }}</title>

6+ <title>Searx search: {{ q|e }}</title>

7− <link>{{ base_url }}?q={{ q }}</link>

7+ <link>{{ base_url }}?q={{ q|e }}</link>

8− <description>Search results for "{{ q }}" - searx</description>

8+ <description>Search results for "{{ q|e }}" - searx</description>

9 <opensearch:totalResults>{{ number_of_results }}</opensearch:totalResults>

10 <opensearch:startIndex>1</opensearch:startIndex>

11 <opensearch:itemsPerPage>{{ number_of_results }}</opensearch:itemsPerPage>

12 <atom:link rel="search" type="application/opensearchdescription+xml" href="{{ base_url }}opensearch.xml"/>

13− <opensearch:Query role="request" searchTerms="{{ q }}" startPage="1" />

13+ <opensearch:Query role="request" searchTerms="{{ q|e }}" startPage="1" />

14 {% for r in results %}

15 <item>

16 <title>{{ r.title }}</title>

searx/templates/oscar/preferences.html+29

@@ -12,6 +12,7 @@

12 <li class="active"><a href="#tab_general" role="tab" data-toggle="tab">{{ _('General') }}</a></li>

13 <li><a href="#tab_engine" role="tab" data-toggle="tab">{{ _('Engines') }}</a></li>

14 <li><a href="#tab_plugins" role="tab" data-toggle="tab">{{ _('Plugins') }}</a></li>

15+ {% if answerers %}<li><a href="#tab_answerers" role="tab" data-toggle="tab">{{ _('Answerers') }}</a></li>{% endif %}

15 <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>

16 <li><a href="#tab_cookies" role="tab" data-toggle="tab">{{ _('Cookies') }}</a></li>

16 </ul>

17 </ul>

17

18

@@ -224,6 +225,34 @@

224 </fieldset>

225 </fieldset>

225 </div>

226 </div>

226

227

228+ {% if answerers %}

229+ <div class="tab-pane active_if_nojs" id="tab_answerers">

230+ <noscript>

231+ <h3>{{ _('Answerers') }}</h3>

232+ </noscript>

233+ <p class="text-muted" style="margin:20px 0;">

234+ {{ _('This is the list of searx\'s instant answering modules.') }}

235+ </p>

236+ <table class="table table-striped">

237+ <tr>

238+ <th class="text-muted">{{ _('Name') }}</th>

239+ <th class="text-muted">{{ _('Keywords') }}</th>

240+ <th class="text-muted">{{ _('Description') }}</th>

241+ <th class="text-muted">{{ _('Examples') }}</th>

242+ </tr>

243+

244+ {% for answerer in answerers %}

245+ <tr>

246+ <td class="text-muted">{{ answerer.info.name }}</td>

247+ <td class="text-muted">{{ answerer.keywords|join(', ') }}</td>

248+ <td class="text-muted">{{ answerer.info.description }}</td>

249+ <td class="text-muted">{{ answerer.info.examples|join(', ') }}</td>

250+ </tr>

251+ {% endfor %}

252+ </table>

253+ </div>

254+ {% endif %}

255+

227 <div class="tab-pane active_if_nojs" id="tab_cookies">

256 <div class="tab-pane active_if_nojs" id="tab_cookies">

228 <noscript>

257 <noscript>

229 <h3>{{ _('Cookies') }}</h3>

258 <h3>{{ _('Cookies') }}</h3>

searx/templates/oscar/result_templates/images.html+6−1

@@ -13,7 +13,12 @@

13 </div>

14 <div class="modal-body">

15 <img class="img-responsive center-block" src="{% if result.thumbnail_src %}{{ image_proxify(result.thumbnail_src) }}{% else %}{{ image_proxify(result.img_src) }}{% endif %}" alt="{{ result.title|striptags }}">

16+ {% if result.author %}<span class="photo-author">{{ result.author }}</span><br />{% endif %}

17+ {% if result.content %}

16− {% if result.content %}<p class="result-content">{{ result.content|safe }}</p>{% endif %}

18+ <p class="result-content">

19+ {{ result.content }}

20+ </p>

21+ {% endif %}

17 </div>

22 </div>

18 <div class="modal-footer">

23 <div class="modal-footer">

19 <div class="clearfix"></div>

24 <div class="clearfix"></div>

searx/templates/oscar/results.html+7−7

@@ -1,6 +1,6 @@

1{% extends "oscar/base.html" %}

2−{% block title %}{{ q }} - {% endblock %}

2+{% block title %}{{ q|e }} - {% endblock %}

3−{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}

3+{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&format=rss&{% for category in selected_categories %}category_{{ category }}=1&{% endfor %}pageno={{ pageno }}&time_range={{ time_range }}">{% endblock %}

4{% block content %}

5 <div class="row">

6 <div class="col-sm-8" id="main_results">

@@ -37,9 +37,9 @@

37 <div id="pagination">

38 <div class="pull-left">

39 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

40− <input type="hidden" name="q" value="{{ q }}" />

40+ <input type="hidden" name="q" value="{{ q|e }}" />

41 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

42− <input type="hidden" name="q" value="{{ q }}" />

42+ <input type="hidden" name="q" value="{{ q|e }}" />

43 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

44 <input type="hidden" name="time_range" value="{{ time_range }}" />

45 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>

@@ -59,7 +59,7 @@

59 <div id="pagination">

60 <div class="pull-left">

61 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

62− <input type="hidden" name="q" value="{{ q }}" />

62+ <input type="hidden" name="q" value="{{ q|e }}" />

63 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

64 <input type="hidden" name="pageno" value="{{ pageno-1 }}" />

65 <input type="hidden" name="time_range" value="{{ time_range }}" />

@@ -69,7 +69,7 @@

69 <div class="pull-right">

70 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="pull-left">

71 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1"/>{% endfor %}

72− <input type="hidden" name="q" value="{{ q }}" />

72+ <input type="hidden" name="q" value="{{ q|e }}" />

73 <input type="hidden" name="pageno" value="{{ pageno+1 }}" />

74 <input type="hidden" name="time_range" value="{{ time_range }}" />

75 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>

@@ -130,7 +130,7 @@

130 <div class="clearfix"></div>

131 {% for output_type in ('csv', 'json', 'rss') %}

132 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}" class="form-inline pull-{% if rtl %}right{% else %}left{% endif %} result_download">

133− <input type="hidden" name="q" value="{{ q }}">

133+ <input type="hidden" name="q" value="{{ q|e }}">

134 <input type="hidden" name="format" value="{{ output_type }}">

135 {% for category in selected_categories %}<input type="hidden" name="category_{{ category }}" value="1">{% endfor %}

136 <input type="hidden" name="pageno" value="{{ pageno }}">

searx/templates/pix-art/results.html+3−3

@@ -5,7 +5,7 @@

5 {% endfor %}

6{% else %}

7{% extends "pix-art/base.html" %}

8−{% block title %}{{ q }} - {% endblock %}

8+{% block title %}{{ q|e }} - {% endblock %}

9{% block meta %}{% endblock %}

10{% block content %}

11<div id="logo"><a href="./"><img src="{{ url_for('static', filename='img/searx-pixel-small.png') }}" alt="searx Logo"/></a></div>

@@ -25,8 +25,8 @@

25 </span>

26 <div id="pagination">

27 <br />

28− <input type="button" onclick="load_more('{{ q }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />

28+ <input type="button" onclick="load_more('{{ q|e }}', {{ pageno+1 }})" id="load_more" value="{{ _('Load more...') }}" />

29 </div>

30</div>

31{% endblock %}

32−{% endif %}

32+{% endif %}

searx/utils.py+13

@@ -6,7 +6,10 @@ import re

6from babel.dates import format_date

7from codecs import getincrementalencoder

8from HTMLParser import HTMLParser

9+from imp import load_source

10+from os.path import splitext, join

9from random import choice

11from random import choice

12+import sys

10

13

11from searx.version import VERSION_STRING

14from searx.version import VERSION_STRING

12from searx.languages import language_codes

15from searx.languages import language_codes

@@ -285,3 +288,13 @@ def is_valid_lang(lang):

285 if l[1].lower() == lang.lower():

288 if l[1].lower() == lang.lower():

286 return (True, l[0][:2], l[1].lower())

289 return (True, l[0][:2], l[1].lower())

287 return False

290 return False

291+

292+

293+def load_module(filename, module_dir):

294+ modname = splitext(filename)[0]

295+ if modname in sys.modules:

296+ del sys.modules[modname]

297+ filepath = join(module_dir, filename)

298+ module = load_source(modname, filepath)

299+ module.name = modname

300+ return module

searx/webapp.py+20−11

@@ -40,7 +40,7 @@ except:

40 logger.critical("cannot import dependency: pygments")

41 from sys import exit

42 exit(1)

43−

43+from cgi import escape

44from datetime import datetime, timedelta

45from urllib import urlencode

46from urlparse import urlparse, urljoin

@@ -62,11 +62,12 @@ from searx.utils import (

62)

63from searx.version import VERSION_STRING

64from searx.languages import language_codes

65−from searx.search import Search, SearchWithPlugins, get_search_query_from_webapp

65+from searx.search import SearchWithPlugins, get_search_query_from_webapp

66−from searx.query import RawTextQuery, SearchQuery

66+from searx.query import RawTextQuery

67from searx.autocomplete import searx_bang, backends as autocomplete_backends

68from searx.plugins import plugins

69from searx.preferences import Preferences, ValidationException

70+from searx.answerers import answerers

70

71

71# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.

72# check if the pyopenssl, ndg-httpsclient, pyasn1 packages are installed.

72# They are needed for SSL connection without trouble, see #298

73# They are needed for SSL connection without trouble, see #298

@@ -344,6 +345,8 @@ def render(template_name, override_theme=None, **kwargs):

344

345

345 kwargs['cookies'] = request.cookies

346 kwargs['cookies'] = request.cookies

346

347

348+ kwargs['errors'] = request.errors

349+

347 kwargs['instance_name'] = settings['general']['instance_name']

350 kwargs['instance_name'] = settings['general']['instance_name']

348

351

349 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

352 kwargs['results_on_new_tab'] = request.preferences.get_value('results_on_new_tab')

@@ -364,15 +367,16 @@ def render(template_name, override_theme=None, **kwargs):

364

367

365@app.before_request

368@app.before_request

366def pre_request():

369def pre_request():

367− # merge GET, POST vars

370+ request.errors = []

371+

368 preferences = Preferences(themes, categories.keys(), engines, plugins)

372 preferences = Preferences(themes, categories.keys(), engines, plugins)

373+ request.preferences = preferences

369 try:

374 try:

370 preferences.parse_cookies(request.cookies)

375 preferences.parse_cookies(request.cookies)

371 except:

376 except:

372− # TODO throw error message to the user

377+ request.errors.append(gettext('Invalid settings, please edit your preferences'))

373− logger.warning('Invalid config')

374− request.preferences = preferences

375

378

379+ # merge GET, POST vars

376 # request.form

380 # request.form

377 request.form = dict(request.form.items())

381 request.form = dict(request.form.items())

378 for k, v in request.args.items():

382 for k, v in request.args.items():

@@ -397,7 +401,7 @@ def index():

397 Supported outputs: html, json, csv, rss.

401 Supported outputs: html, json, csv, rss.

398 """

402 """

399

403

400− if not request.args and not request.form:

404+ if request.form.get('q') is None:

401 return render(

405 return render(

402 'index.html',

406 'index.html',

403 )

407 )

@@ -411,6 +415,8 @@ def index():

411 search = SearchWithPlugins(search_query, request)

415 search = SearchWithPlugins(search_query, request)

412 result_container = search.search()

416 result_container = search.search()

413 except:

417 except:

418+ request.errors.append(gettext('search error'))

419+ logger.exception('search error')

414 return render(

420 return render(

415 'index.html',

421 'index.html',

416 )

422 )

@@ -427,8 +433,10 @@ def index():

427 for result in results:

433 for result in results:

428 if output_format == 'html':

434 if output_format == 'html':

429 if 'content' in result and result['content']:

435 if 'content' in result and result['content']:

430− result['content'] = highlight_content(result['content'][:1024], search_query.query.encode('utf-8'))

436+ result['content'] = highlight_content(escape(result['content'][:1024]),

437+ search_query.query.encode('utf-8'))

431− result['title'] = highlight_content(result['title'], search_query.query.encode('utf-8'))

438+ result['title'] = highlight_content(escape(result['title'] or u''),

439+ search_query.query.encode('utf-8'))

432 else:

440 else:

433 if result.get('content'):

441 if result.get('content'):

434 result['content'] = html_to_text(result['content']).strip()

442 result['content'] = html_to_text(result['content']).strip()

@@ -572,7 +580,7 @@ def preferences():

572 try:

580 try:

573 request.preferences.parse_form(request.form)

581 request.preferences.parse_form(request.form)

574 except ValidationException:

582 except ValidationException:

575− # TODO use flash feature of flask

583+ request.errors.append(gettext('Invalid settings, please edit your preferences'))

576 return resp

584 return resp

577 return request.preferences.save(resp)

585 return request.preferences.save(resp)

578

586

@@ -609,6 +617,7 @@ def preferences():

609 language_codes=language_codes,

617 language_codes=language_codes,

610 engines_by_category=categories,

618 engines_by_category=categories,

611 stats=stats,

619 stats=stats,

620+ answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],

612 disabled_engines=disabled_engines,

621 disabled_engines=disabled_engines,

613 autocomplete_backends=autocomplete_backends,

622 autocomplete_backends=autocomplete_backends,

614 shortcuts={y: x for x, y in engine_shortcuts.items()},

623 shortcuts={y: x for x, y in engine_shortcuts.items()},

tests/unit/engines/test_bing.py+2−4

@@ -14,14 +14,12 @@ class TestBingEngine(SearxTestCase):

14 params = bing.request(query, dicto)

15 self.assertTrue('url' in params)

16 self.assertTrue(query in params['url'])

17+ self.assertTrue('language%3AFR' in params['url'])

17 self.assertTrue('bing.com' in params['url'])

18 self.assertTrue('bing.com' in params['url'])

18− self.assertTrue('SRCHHPGUSR' in params['cookies'])

19− self.assertTrue('fr' in params['cookies']['SRCHHPGUSR'])

20

19

21 dicto['language'] = 'all'

20 dicto['language'] = 'all'

22 params = bing.request(query, dicto)

21 params = bing.request(query, dicto)

23− self.assertTrue('SRCHHPGUSR' in params['cookies'])

22+ self.assertTrue('language' not in params['url'])

24− self.assertTrue('en' in params['cookies']['SRCHHPGUSR'])

25

23

26 def test_response(self):

24 def test_response(self):

27 self.assertRaises(AttributeError, bing.response, None)

25 self.assertRaises(AttributeError, bing.response, None)

tests/unit/engines/test_deezer.py+1−1

@@ -42,7 +42,7 @@ class TestDeezerEngine(SearxTestCase):

42 self.assertEqual(len(results), 1)

43 self.assertEqual(results[0]['title'], 'Title of track')

44 self.assertEqual(results[0]['url'], 'https://www.deezer.com/track/1094042')

45− self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')

45+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')

46 self.assertTrue('100' in results[0]['embedded'])

47

48 json = r"""

tests/unit/engines/test_flickr.py+3−3

@@ -52,7 +52,7 @@ class TestFlickrEngine(SearxTestCase):

52 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

53 self.assertTrue('o.jpg' in results[0]['img_src'])

54 self.assertTrue('n.jpg' in results[0]['thumbnail_src'])

55− self.assertTrue('Owner' in results[0]['content'])

55+ self.assertTrue('Owner' in results[0]['author'])

56 self.assertTrue('Description' in results[0]['content'])

57

58 json = r"""

@@ -76,7 +76,7 @@ class TestFlickrEngine(SearxTestCase):

76 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

77 self.assertTrue('z.jpg' in results[0]['img_src'])

78 self.assertTrue('z.jpg' in results[0]['thumbnail_src'])

79− self.assertTrue('Owner' in results[0]['content'])

79+ self.assertTrue('Owner' in results[0]['author'])

80 self.assertTrue('Description' in results[0]['content'])

81

82 json = r"""

@@ -100,7 +100,7 @@ class TestFlickrEngine(SearxTestCase):

100 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/66847915@N08/15751017054')

101 self.assertTrue('o.jpg' in results[0]['img_src'])

102 self.assertTrue('o.jpg' in results[0]['thumbnail_src'])

103− self.assertTrue('Owner' in results[0]['content'])

103+ self.assertTrue('Owner' in results[0]['author'])

104 self.assertTrue('Description' in results[0]['content'])

105

106 json = r"""

tests/unit/engines/test_flickr_noapi.py+3−3

@@ -145,7 +145,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

145 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

146 self.assertIn('k.jpg', results[0]['img_src'])

147 self.assertIn('n.jpg', results[0]['thumbnail_src'])

148− self.assertIn('Owner', results[0]['content'])

148+ self.assertIn('Owner', results[0]['author'])

149

150 # no n size, only the z size

151 json = """

@@ -188,7 +188,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

188 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

189 self.assertIn('z.jpg', results[0]['img_src'])

190 self.assertIn('z.jpg', results[0]['thumbnail_src'])

191− self.assertIn('Owner', results[0]['content'])

191+ self.assertIn('Owner', results[0]['author'])

192

193 # no z or n size

194 json = """

@@ -231,7 +231,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

231 self.assertEqual(results[0]['url'], 'https://www.flickr.com/photos/59729010@N00/14001294434')

232 self.assertIn('o.jpg', results[0]['img_src'])

233 self.assertIn('o.jpg', results[0]['thumbnail_src'])

234− self.assertIn('Owner', results[0]['content'])

234+ self.assertIn('Owner', results[0]['author'])

235

236 # no image test

237 json = """

tests/unit/engines/test_kickass.py+2−2

@@ -98,7 +98,7 @@ class TestKickassEngine(SearxTestCase):

98 self.assertEqual(len(results), 1)

99 self.assertEqual(results[0]['title'], 'This should be the title')

100 self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')

101− self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

101+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

102 self.assertEqual(results[0]['seed'], 10)

103 self.assertEqual(results[0]['leech'], 1)

104 self.assertEqual(results[0]['filesize'], 449)

@@ -381,7 +381,7 @@ class TestKickassEngine(SearxTestCase):

381 self.assertEqual(len(results), 5)

382 self.assertEqual(results[0]['title'], 'This should be the title')

383 self.assertEqual(results[0]['url'], 'https://kickass.cd/url.html')

384− self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

384+ self.assertEqual(results[0]['content'], 'Posted by riri in Other > Unsorted')

385 self.assertEqual(results[0]['seed'], 10)

386 self.assertEqual(results[0]['leech'], 1)

387 self.assertEqual(results[0]['files'], 4)

tests/unit/engines/test_searchcode_doc.py−3

@@ -56,9 +56,6 @@ class TestSearchcodeDocEngine(SearxTestCase):

56 self.assertEqual(len(results), 1)

57 self.assertEqual(results[0]['title'], '[Type] Namespace test')

58 self.assertEqual(results[0]['url'], 'http://url')

59− self.assertIn('Synopsis', results[0]['content'])

60− self.assertIn('Type', results[0]['content'])

61− self.assertIn('test', results[0]['content'])

62 self.assertIn('Description', results[0]['content'])

59 self.assertIn('Description', results[0]['content'])

63

60

64 json = r"""

61 json = r"""

tests/unit/engines/test_spotify.py+1−1

@@ -90,7 +90,7 @@ class TestSpotifyEngine(SearxTestCase):

90 self.assertEqual(len(results), 1)

91 self.assertEqual(results[0]['title'], 'Title of track')

92 self.assertEqual(results[0]['url'], 'https://open.spotify.com/track/2GzvFiedqW8hgqUpWcASZa')

93− self.assertEqual(results[0]['content'], 'Artist Name • Album Title • Title of track')

93+ self.assertEqual(results[0]['content'], 'Artist Name - Album Title - Title of track')

94 self.assertIn('1000', results[0]['embedded'])

95

96 json = """

tests/unit/test_answerers.pynew file+16

@@ -0,0 +1,16 @@

1+# -*- coding: utf-8 -*-

2+

3+from mock import Mock

4+

5+from searx.answerers import answerers

6+from searx.testing import SearxTestCase

7+

8+

9+class AnswererTest(SearxTestCase):

10+

11+ def test_unicode_input(self):

12+ query = Mock()

13+ unicode_payload = u'árvíztűrő tükörfúrógép'

14+ for answerer in answerers:

15+ query.query = u'{} {}'.format(answerer.keywords[0], unicode_payload)

16+ self.assertTrue(isinstance(answerer.answer(query), list))

tests/unit/test_webapp.py+2−1

@@ -5,6 +5,7 @@ from mock import Mock

5from urlparse import ParseResult

6from searx import webapp

7from searx.testing import SearxTestCase

8+from searx.search import Search

8

9

10

10class ViewsTestCase(SearxTestCase):

11class ViewsTestCase(SearxTestCase):

@@ -41,7 +42,7 @@ class ViewsTestCase(SearxTestCase):

41 results_number=lambda: 3,

42 results_number=lambda: 3,

42 results_length=lambda: len(self.test_results))

43 results_length=lambda: len(self.test_results))

43

44

44− webapp.Search.search = search_mock

45+ Search.search = search_mock

45

46

46 def get_current_theme_name_mock(override=None):

47 def get_current_theme_name_mock(override=None):

47 return 'legacy'

48 return 'legacy'