85b3723 · searx · git.iko.soy

searx/engines/bing.py+10−10

@@ -18,7 +18,7 @@ from lxml import html

1818from searx import logger, utils

1919from searx.engines.xpath import extract_text

2020from searx.url_utils import urlencode

21−from searx.utils import match_language, gen_useragent

21+from searx.utils import match_language, gen_useragent, eval_xpath

2222

2323logger = logger.getChild('bing engine')

2424

@@ -65,11 +65,11 @@ def response(resp):

6565

6666 dom = html.fromstring(resp.text)

6767 # parse results

68− for result in dom.xpath('//div[@class="sa_cc"]'):

69− link = result.xpath('.//h3/a')[0]

68+ for result in eval_xpath(dom, '//div[@class="sa_cc"]'):

69+ link = eval_xpath(result, './/h3/a')[0]

7070 url = link.attrib.get('href')

7171 title = extract_text(link)

72− content = extract_text(result.xpath('.//p'))

72+ content = extract_text(eval_xpath(result, './/p'))

7373

7474 # append result

7575 results.append({'url': url,

@@ -77,11 +77,11 @@ def response(resp):

7777 'content': content})

7878

7979 # parse results again if nothing is found yet

80− for result in dom.xpath('//li[@class="b_algo"]'):

81− link = result.xpath('.//h2/a')[0]

80+ for result in eval_xpath(dom, '//li[@class="b_algo"]'):

81+ link = eval_xpath(result, './/h2/a')[0]

8282 url = link.attrib.get('href')

8383 title = extract_text(link)

84− content = extract_text(result.xpath('.//p'))

84+ content = extract_text(eval_xpath(result, './/p'))

8585

8686 # append result

8787 results.append({'url': url,

@@ -89,7 +89,7 @@ def response(resp):

8989 'content': content})

9090

9191 try:

92− result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))

92+ result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))

9393 result_len_container = utils.to_string(result_len_container)

9494 if "-" in result_len_container:

9595 # Remove the part "from-to" for paginated request ...

@@ -113,9 +113,9 @@ def response(resp):

113113def _fetch_supported_languages(resp):

114114 supported_languages = []

115115 dom = html.fromstring(resp.text)

116− options = dom.xpath('//div[@id="limit-languages"]//input')

116+ options = eval_xpath(dom, '//div[@id="limit-languages"]//input')

117117 for option in options:

118− code = option.xpath('./@id')[0].replace('_', '-')

118+ code = eval_xpath(option, './@id')[0].replace('_', '-')

119119 if code == 'nb':

120120 code = 'no'

121121 supported_languages.append(code)

searx/engines/dictzone.py+4−4

@@ -11,7 +11,7 @@

1111

1212import re

1313from lxml import html

14−from searx.utils import is_valid_lang

14+from searx.utils import is_valid_lang, eval_xpath

1515from searx.url_utils import urljoin

1616

1717categories = ['general']

@@ -47,14 +47,14 @@ def response(resp):

4747

4848 dom = html.fromstring(resp.text)

4949

50− for k, result in enumerate(dom.xpath(results_xpath)[1:]):

50+ for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):

5151 try:

52− from_result, to_results_raw = result.xpath('./td')

52+ from_result, to_results_raw = eval_xpath(result, './td')

5353 except:

5454 continue

5555

5656 to_results = []

57− for to_result in to_results_raw.xpath('./p/a'):

57+ for to_result in eval_xpath(to_results_raw, './p/a'):

5858 t = to_result.text_content()

5959 if t.strip():

6060 to_results.append(to_result.text_content())

searx/engines/doku.py+8−7

@@ -11,6 +11,7 @@

1111

1212from lxml.html import fromstring

1313from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

1415from searx.url_utils import urlencode

1516

1617# engine dependent config

@@ -45,16 +46,16 @@ def response(resp):

4546

4647 # parse results

4748 # Quickhits

48− for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):

49+ for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):

4950 try:

50− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

51+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

5152 except:

5253 continue

5354

5455 if not res_url:

5556 continue

5657

57− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

58+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

5859

5960 # append result

6061 results.append({'title': title,

@@ -62,13 +63,13 @@ def response(resp):

6263 'url': base_url + res_url})

6364

6465 # Search results

65− for r in doc.xpath('//dl[@class="search_results"]/*'):

66+ for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):

6667 try:

6768 if r.tag == "dt":

68− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

69− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

69+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

70+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

7071 elif r.tag == "dd":

71− content = extract_text(r.xpath('.'))

72+ content = extract_text(eval_xpath(r, '.'))

7273

7374 # append result

7475 results.append({'title': title,

searx/engines/duckduckgo.py+5−5

@@ -18,7 +18,7 @@ from json import loads

1818from searx.engines.xpath import extract_text

1919from searx.poolrequests import get

2020from searx.url_utils import urlencode

21−from searx.utils import match_language

21+from searx.utils import match_language, eval_xpath

2222

2323# engine dependent config

2424categories = ['general']

@@ -106,19 +106,19 @@ def response(resp):

106106 doc = fromstring(resp.text)

107107

108108 # parse results

109− for i, r in enumerate(doc.xpath(result_xpath)):

109+ for i, r in enumerate(eval_xpath(doc, result_xpath)):

110110 if i >= 30:

111111 break

112112 try:

113− res_url = r.xpath(url_xpath)[-1]

113+ res_url = eval_xpath(r, url_xpath)[-1]

114114 except:

115115 continue

116116

117117 if not res_url:

118118 continue

119119

120− title = extract_text(r.xpath(title_xpath))

121− content = extract_text(r.xpath(content_xpath))

120+ title = extract_text(eval_xpath(r, title_xpath))

121+ content = extract_text(eval_xpath(r, content_xpath))

122122

123123 # append result

124124 results.append({'title': title,

searx/engines/duden.py+8−7

@@ -11,6 +11,7 @@

1111from lxml import html, etree

1212import re

1313from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

1415from searx.url_utils import quote, urljoin

1516from searx import logger

1617

@@ -52,9 +53,9 @@ def response(resp):

5253 dom = html.fromstring(resp.text)

5354

5455 try:

55− number_of_results_string = re.sub('[^0-9]', '', dom.xpath(

56− '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]

57− )

56+ number_of_results_string =\

57+ re.sub('[^0-9]', '',

58+ eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])

5859

5960 results.append({'number_of_results': int(number_of_results_string)})

6061

@@ -62,12 +63,12 @@ def response(resp):

6263 logger.debug("Couldn't read number of results.")

6364 pass

6465

65− for result in dom.xpath('//section[not(contains(@class, "essay"))]'):

66+ for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):

6667 try:

67− url = result.xpath('.//h2/a')[0].get('href')

68+ url = eval_xpath(result, './/h2/a')[0].get('href')

6869 url = urljoin(base_url, url)

69− title = result.xpath('string(.//h2/a)').strip()

70− content = extract_text(result.xpath('.//p'))

70+ title = eval_xpath(result, 'string(.//h2/a)').strip()

71+ content = extract_text(eval_xpath(result, './/p'))

7172 # append result

7273 results.append({'url': url,

7374 'title': title,

searx/engines/gigablast.py+3−2

@@ -15,6 +15,7 @@ from json import loads

1515from time import time

1616from lxml.html import fromstring

1717from searx.url_utils import urlencode

18+from searx.utils import eval_xpath

1819

1920# engine dependent config

2021categories = ['general']

@@ -99,9 +100,9 @@ def response(resp):

99100def _fetch_supported_languages(resp):

100101 supported_languages = []

101102 dom = fromstring(resp.text)

102− links = dom.xpath('//span[@id="menu2"]/a')

103+ links = eval_xpath(dom, '//span[@id="menu2"]/a')

103104 for link in links:

104− href = link.xpath('./@href')[0].split('lang%3A')

105+ href = eval_xpath(link, './@href')[0].split('lang%3A')

105106 if len(href) == 2:

106107 code = href[1].split('_')

107108 if len(code) == 2:

searx/engines/google.py+16−16

@@ -14,7 +14,7 @@ from lxml import html, etree

1414from searx.engines.xpath import extract_text, extract_url

1515from searx import logger

1616from searx.url_utils import urlencode, urlparse, parse_qsl

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

1818

1919logger = logger.getChild('google engine')

2020

@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):

156156

157157# returns extract_text on the first result selected by the xpath or None

158158def extract_text_from_dom(result, xpath):

159− r = result.xpath(xpath)

159+ r = eval_xpath(result, xpath)

160160 if len(r) > 0:

161161 return extract_text(r[0])

162162 return None

@@ -226,21 +226,21 @@ def response(resp):

226226 # convert the text to dom

227227 dom = html.fromstring(resp.text)

228228

229− instant_answer = dom.xpath('//div[@id="_vBb"]//text()')

229+ instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')

230230 if instant_answer:

231231 results.append({'answer': u' '.join(instant_answer)})

232232 try:

233− results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]

233+ results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]

234234 .split()[1].replace(',', ''))

235235 results.append({'number_of_results': results_num})

236236 except:

237237 pass

238238

239239 # parse results

240− for result in dom.xpath(results_xpath):

240+ for result in eval_xpath(dom, results_xpath):

241241 try:

242− title = extract_text(result.xpath(title_xpath)[0])

243− url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)

242+ title = extract_text(eval_xpath(result, title_xpath)[0])

243+ url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)

244244 parsed_url = urlparse(url, google_hostname)

245245

246246 # map result

@@ -249,7 +249,7 @@ def response(resp):

249249 continue

250250 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):

251251 # print "yooooo"*30

252− # x = result.xpath(map_near)

252+ # x = eval_xpath(result, map_near)

253253 # if len(x) > 0:

254254 # # map : near the location

255255 # results = results + parse_map_near(parsed_url, x, google_hostname)

@@ -286,11 +286,11 @@ def response(resp):

286286 continue

287287

288288 # parse suggestion

289− for suggestion in dom.xpath(suggestion_xpath):

289+ for suggestion in eval_xpath(dom, suggestion_xpath):

290290 # append suggestion

291291 results.append({'suggestion': extract_text(suggestion)})

292292

293− for correction in dom.xpath(spelling_suggestion_xpath):

293+ for correction in eval_xpath(dom, spelling_suggestion_xpath):

294294 results.append({'correction': extract_text(correction)})

295295

296296 # return results

@@ -299,9 +299,9 @@ def response(resp):

299299

300300def parse_images(result, google_hostname):

301301 results = []

302− for image in result.xpath(images_xpath):

303− url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)

304− img_src = extract_text(image.xpath(image_img_src_xpath)[0])

302+ for image in eval_xpath(result, images_xpath):

303+ url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)

304+ img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])

305305

306306 # append result

307307 results.append({'url': url,

@@ -388,10 +388,10 @@ def attributes_to_html(attributes):

388388def _fetch_supported_languages(resp):

389389 supported_languages = {}

390390 dom = html.fromstring(resp.text)

391− options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')

391+ options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

392392 for option in options:

393− code = option.xpath('./@value')[0].split('_')[-1]

394− name = option.xpath('./@data-name')[0].title()

393+ code = eval_xpath(option, './@value')[0].split('_')[-1]

394+ name = eval_xpath(option, './@data-name')[0].title()

395395 supported_languages[code] = {"name": name}

396396

397397 return supported_languages

searx/engines/startpage.py+5−4

@@ -16,6 +16,7 @@ from datetime import datetime, timedelta

1616import re

1717from searx.engines.xpath import extract_text

1818from searx.languages import language_codes

19+from searx.utils import eval_xpath

1920

2021# engine dependent config

2122categories = ['general']

@@ -70,8 +71,8 @@ def response(resp):

7071 dom = html.fromstring(resp.text)

7172

7273 # parse results

73− for result in dom.xpath(results_xpath):

74− links = result.xpath(link_xpath)

74+ for result in eval_xpath(dom, results_xpath):

75+ links = eval_xpath(result, link_xpath)

7576 if not links:

7677 continue

7778 link = links[0]

@@ -87,8 +88,8 @@ def response(resp):

8788

8889 title = extract_text(link)

8990

90− if result.xpath(content_xpath):

91− content = extract_text(result.xpath(content_xpath))

91+ if eval_xpath(result, content_xpath):

92+ content = extract_text(eval_xpath(result, content_xpath))

9293 else:

9394 content = ''

9495

searx/engines/wikidata.py+1−17

@@ -16,7 +16,7 @@ from searx.poolrequests import get

1616from searx.engines.xpath import extract_text

1717from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

1818from searx.url_utils import urlencode

19−from searx.utils import match_language

19+from searx.utils import match_language, eval_xpath

2020

2121from json import loads

2222from lxml.html import fromstring

@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator

5757calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

5858media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'

5959

60−# xpath_cache

61−xpath_cache = {}

62−

63−

64−def get_xpath(xpath_str):

65− result = xpath_cache.get(xpath_str, None)

66− if not result:

67− result = etree.XPath(xpath_str)

68− xpath_cache[xpath_str] = result

69− return result

70−

71−

72−def eval_xpath(element, xpath_str):

73− xpath = get_xpath(xpath_str)

74− return xpath(element)

75−

7660

7761def get_id_cache(result):

7862 id_cache = {}

searx/engines/xpath.py+10−10

@@ -1,6 +1,6 @@

11from lxml import html

22from lxml.etree import _ElementStringResult, _ElementUnicodeResult

3−from searx.utils import html_to_text

3+from searx.utils import html_to_text, eval_xpath

44from searx.url_utils import unquote, urlencode, urljoin, urlparse

55

66search_url = None

@@ -104,15 +104,15 @@ def response(resp):

104104 results = []

105105 dom = html.fromstring(resp.text)

106106 if results_xpath:

107− for result in dom.xpath(results_xpath):

108− url = extract_url(result.xpath(url_xpath), search_url)

109− title = extract_text(result.xpath(title_xpath))

110− content = extract_text(result.xpath(content_xpath))

107+ for result in eval_xpath(dom, results_xpath):

108+ url = extract_url(eval_xpath(result, url_xpath), search_url)

109+ title = extract_text(eval_xpath(result, title_xpath))

110+ content = extract_text(eval_xpath(result, content_xpath))

111111 tmp_result = {'url': url, 'title': title, 'content': content}

112112

113113 # add thumbnail if available

114114 if thumbnail_xpath:

115− thumbnail_xpath_result = result.xpath(thumbnail_xpath)

115+ thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)

116116 if len(thumbnail_xpath_result) > 0:

117117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

118118

@@ -120,14 +120,14 @@ def response(resp):

120120 else:

121121 for url, title, content in zip(

122122 (extract_url(x, search_url) for

123− x in dom.xpath(url_xpath)),

124− map(extract_text, dom.xpath(title_xpath)),

125− map(extract_text, dom.xpath(content_xpath))

123+ x in eval_xpath(dom, url_xpath)),

124+ map(extract_text, eval_xpath(dom, title_xpath)),

125+ map(extract_text, eval_xpath(dom, content_xpath))

126126 ):

127127 results.append({'url': url, 'title': title, 'content': content})

128128

129129 if not suggestion_xpath:

130130 return results

131− for suggestion in dom.xpath(suggestion_xpath):

131+ for suggestion in eval_xpath(dom, suggestion_xpath):

132132 results.append({'suggestion': extract_text(suggestion)})

133133 return results

searx/engines/yahoo.py+9−9

@@ -14,7 +14,7 @@

1414from lxml import html

1515from searx.engines.xpath import extract_text, extract_url

1616from searx.url_utils import unquote, urlencode

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

1818

1919# engine dependent config

2020categories = ['general']

@@ -109,21 +109,21 @@ def response(resp):

109109 dom = html.fromstring(resp.text)

110110

111111 try:

112− results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]

112+ results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]

113113 .split()[0].replace(',', ''))

114114 results.append({'number_of_results': results_num})

115115 except:

116116 pass

117117

118118 # parse results

119− for result in dom.xpath(results_xpath):

119+ for result in eval_xpath(dom, results_xpath):

120120 try:

121− url = parse_url(extract_url(result.xpath(url_xpath), search_url))

122− title = extract_text(result.xpath(title_xpath)[0])

121+ url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))

122+ title = extract_text(eval_xpath(result, title_xpath)[0])

123123 except:

124124 continue

125125

126− content = extract_text(result.xpath(content_xpath)[0])

126+ content = extract_text(eval_xpath(result, content_xpath)[0])

127127

128128 # append result

129129 results.append({'url': url,

@@ -131,7 +131,7 @@ def response(resp):

131131 'content': content})

132132

133133 # if no suggestion found, return results

134− suggestions = dom.xpath(suggestion_xpath)

134+ suggestions = eval_xpath(dom, suggestion_xpath)

135135 if not suggestions:

136136 return results

137137

@@ -148,9 +148,9 @@ def response(resp):

148148def _fetch_supported_languages(resp):

149149 supported_languages = []

150150 dom = html.fromstring(resp.text)

151− options = dom.xpath('//div[@id="yschlang"]/span/label/input')

151+ options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')

152152 for option in options:

153− code_parts = option.xpath('./@value')[0][5:].split('_')

153+ code_parts = eval_xpath(option, './@value')[0][5:].split('_')

154154 if len(code_parts) == 2:

155155 code = code_parts[0] + '-' + code_parts[1].upper()

156156 else:

searx/results.py+5−3

@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):

6767

6868 for url2 in infobox2.get('urls', []):

6969 unique_url = True

70− for url1 in infobox1.get('urls', []):

71− if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):

70+ parsed_url2 = urlparse(url2.get('url', ''))

71+ for url1 in urls1:

72+ if compare_urls(urlparse(url1.get('url', '')), parsed_url2):

7273 unique_url = False

7374 break

7475 if unique_url:

@@ -188,8 +189,9 @@ class ResultContainer(object):

188189 add_infobox = True

189190 infobox_id = infobox.get('id', None)

190191 if infobox_id is not None:

192+ parsed_url_infobox_id = urlparse(infobox_id)

191193 for existingIndex in self.infoboxes:

192− if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):

194+ if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):

193195 merge_two_infoboxes(existingIndex, infobox)

194196 add_infobox = False

195197

searx/utils.py+15

@@ -13,6 +13,7 @@ from numbers import Number

1313from os.path import splitext, join

1414from io import open

1515from random import choice

16+from lxml.etree import XPath

1617import sys

1718import json

1819

@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

5152useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

5253 + "/data/useragents.json", 'r', encoding='utf-8').read())

5354

55+xpath_cache = dict()

5456lang_to_lc_cache = dict()

5557

5658

@@ -450,3 +452,16 @@ def get_engine_from_settings(name):

450452 return engine

451453

452454 return {}

455+

456+

457+def get_xpath(xpath_str):

458+ result = xpath_cache.get(xpath_str, None)

459+ if result is None:

460+ result = XPath(xpath_str)

461+ xpath_cache[xpath_str] = result

462+ return result

463+

464+

465+def eval_xpath(element, xpath_str):

466+ xpath = get_xpath(xpath_str)

467+ return xpath(element)

searx/webapp.py+7−9

@@ -154,20 +154,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None

154154

155155@babel.localeselector

156156def get_locale():

157− locale = request.accept_languages.best_match(settings['locales'].keys())

158−

159− if request.preferences.get_value('locale') != '':

160− locale = request.preferences.get_value('locale')

157+ if 'locale' in request.form\

158+ and request.form['locale'] in settings['locales']:

159+ return request.form['locale']

161160

162161 if 'locale' in request.args\

163162 and request.args['locale'] in settings['locales']:

164− locale = request.args['locale']

163+ return request.args['locale']

165164

166− if 'locale' in request.form\

167− and request.form['locale'] in settings['locales']:

168− locale = request.form['locale']

165+ if request.preferences.get_value('locale') != '':

166+ return request.preferences.get_value('locale')

169167

170− return locale

168+ return request.accept_languages.best_match(settings['locales'].keys())

171169

172170

173171# code-highlighter