on master help sign in
[mod] speed optimization
85b3723Dalf
searx/engines/bing.py+10−10
@@ -18,7 +18,7 @@ from lxml import html
18from searx import logger, utils
18from searx import logger, utils
19from searx.engines.xpath import extract_text
19from searx.engines.xpath import extract_text
20from searx.url_utils import urlencode
20from searx.url_utils import urlencode
21−from searx.utils import match_language, gen_useragent
21+from searx.utils import match_language, gen_useragent, eval_xpath
22
22
23logger = logger.getChild('bing engine')
23logger = logger.getChild('bing engine')
24
24
@@ -65,11 +65,11 @@ def response(resp):
65
65
66 dom = html.fromstring(resp.text)
66 dom = html.fromstring(resp.text)
67 # parse results
67 # parse results
68− for result in dom.xpath('//div[@class="sa_cc"]'):
68+ for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
69− link = result.xpath('.//h3/a')[0]
69+ link = eval_xpath(result, './/h3/a')[0]
70 url = link.attrib.get('href')
70 url = link.attrib.get('href')
71 title = extract_text(link)
71 title = extract_text(link)
72− content = extract_text(result.xpath('.//p'))
72+ content = extract_text(eval_xpath(result, './/p'))
73
73
74 # append result
74 # append result
75 results.append({'url': url,
75 results.append({'url': url,
@@ -77,11 +77,11 @@ def response(resp):
77 'content': content})
77 'content': content})
78
78
79 # parse results again if nothing is found yet
79 # parse results again if nothing is found yet
80− for result in dom.xpath('//li[@class="b_algo"]'):
80+ for result in eval_xpath(dom, '//li[@class="b_algo"]'):
81− link = result.xpath('.//h2/a')[0]
81+ link = eval_xpath(result, './/h2/a')[0]
82 url = link.attrib.get('href')
82 url = link.attrib.get('href')
83 title = extract_text(link)
83 title = extract_text(link)
84− content = extract_text(result.xpath('.//p'))
84+ content = extract_text(eval_xpath(result, './/p'))
85
85
86 # append result
86 # append result
87 results.append({'url': url,
87 results.append({'url': url,
@@ -89,7 +89,7 @@ def response(resp):
89 'content': content})
89 'content': content})
90
90
91 try:
91 try:
92− result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
92+ result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
93 result_len_container = utils.to_string(result_len_container)
93 result_len_container = utils.to_string(result_len_container)
94 if "-" in result_len_container:
94 if "-" in result_len_container:
95 # Remove the part "from-to" for paginated request ...
95 # Remove the part "from-to" for paginated request ...
@@ -113,9 +113,9 @@ def response(resp):
113def _fetch_supported_languages(resp):
113def _fetch_supported_languages(resp):
114 supported_languages = []
114 supported_languages = []
115 dom = html.fromstring(resp.text)
115 dom = html.fromstring(resp.text)
116− options = dom.xpath('//div[@id="limit-languages"]//input')
116+ options = eval_xpath(dom, '//div[@id="limit-languages"]//input')
117 for option in options:
117 for option in options:
118− code = option.xpath('./@id')[0].replace('_', '-')
118+ code = eval_xpath(option, './@id')[0].replace('_', '-')
119 if code == 'nb':
119 if code == 'nb':
120 code = 'no'
120 code = 'no'
121 supported_languages.append(code)
121 supported_languages.append(code)
searx/engines/dictzone.py+4−4
@@ -11,7 +11,7 @@
11
11
12import re
12import re
13from lxml import html
13from lxml import html
14−from searx.utils import is_valid_lang
14+from searx.utils import is_valid_lang, eval_xpath
15from searx.url_utils import urljoin
15from searx.url_utils import urljoin
16
16
17categories = ['general']
17categories = ['general']
@@ -47,14 +47,14 @@ def response(resp):
47
47
48 dom = html.fromstring(resp.text)
48 dom = html.fromstring(resp.text)
49
49
50− for k, result in enumerate(dom.xpath(results_xpath)[1:]):
50+ for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):
51 try:
51 try:
52− from_result, to_results_raw = result.xpath('./td')
52+ from_result, to_results_raw = eval_xpath(result, './td')
53 except:
53 except:
54 continue
54 continue
55
55
56 to_results = []
56 to_results = []
57− for to_result in to_results_raw.xpath('./p/a'):
57+ for to_result in eval_xpath(to_results_raw, './p/a'):
58 t = to_result.text_content()
58 t = to_result.text_content()
59 if t.strip():
59 if t.strip():
60 to_results.append(to_result.text_content())
60 to_results.append(to_result.text_content())
searx/engines/doku.py+8−7
@@ -11,6 +11,7 @@
11
11
12from lxml.html import fromstring
12from lxml.html import fromstring
13from searx.engines.xpath import extract_text
13from searx.engines.xpath import extract_text
14+from searx.utils import eval_xpath
14from searx.url_utils import urlencode
15from searx.url_utils import urlencode
15
16
16# engine dependent config
17# engine dependent config
@@ -45,16 +46,16 @@ def response(resp):
45
46
46 # parse results
47 # parse results
47 # Quickhits
48 # Quickhits
48− for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):
49+ for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):
49 try:
50 try:
50− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
51+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
51 except:
52 except:
52 continue
53 continue
53
54
54 if not res_url:
55 if not res_url:
55 continue
56 continue
56
57
57− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
58+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
58
59
59 # append result
60 # append result
60 results.append({'title': title,
61 results.append({'title': title,
@@ -62,13 +63,13 @@ def response(resp):
62 'url': base_url + res_url})
63 'url': base_url + res_url})
63
64
64 # Search results
65 # Search results
65− for r in doc.xpath('//dl[@class="search_results"]/*'):
66+ for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):
66 try:
67 try:
67 if r.tag == "dt":
68 if r.tag == "dt":
68− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
69+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
69− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
70+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
70 elif r.tag == "dd":
71 elif r.tag == "dd":
71− content = extract_text(r.xpath('.'))
72+ content = extract_text(eval_xpath(r, '.'))
72
73
73 # append result
74 # append result
74 results.append({'title': title,
75 results.append({'title': title,
searx/engines/duckduckgo.py+5−5
@@ -18,7 +18,7 @@ from json import loads
18from searx.engines.xpath import extract_text
18from searx.engines.xpath import extract_text
19from searx.poolrequests import get
19from searx.poolrequests import get
20from searx.url_utils import urlencode
20from searx.url_utils import urlencode
21−from searx.utils import match_language
21+from searx.utils import match_language, eval_xpath
22
22
23# engine dependent config
23# engine dependent config
24categories = ['general']
24categories = ['general']
@@ -106,19 +106,19 @@ def response(resp):
106 doc = fromstring(resp.text)
106 doc = fromstring(resp.text)
107
107
108 # parse results
108 # parse results
109− for i, r in enumerate(doc.xpath(result_xpath)):
109+ for i, r in enumerate(eval_xpath(doc, result_xpath)):
110 if i >= 30:
110 if i >= 30:
111 break
111 break
112 try:
112 try:
113− res_url = r.xpath(url_xpath)[-1]
113+ res_url = eval_xpath(r, url_xpath)[-1]
114 except:
114 except:
115 continue
115 continue
116
116
117 if not res_url:
117 if not res_url:
118 continue
118 continue
119
119
120− title = extract_text(r.xpath(title_xpath))
120+ title = extract_text(eval_xpath(r, title_xpath))
121− content = extract_text(r.xpath(content_xpath))
121+ content = extract_text(eval_xpath(r, content_xpath))
122
122
123 # append result
123 # append result
124 results.append({'title': title,
124 results.append({'title': title,
searx/engines/duden.py+8−7
@@ -11,6 +11,7 @@
11from lxml import html, etree
11from lxml import html, etree
12import re
12import re
13from searx.engines.xpath import extract_text
13from searx.engines.xpath import extract_text
14+from searx.utils import eval_xpath
14from searx.url_utils import quote, urljoin
15from searx.url_utils import quote, urljoin
15from searx import logger
16from searx import logger
16
17
@@ -52,9 +53,9 @@ def response(resp):
52 dom = html.fromstring(resp.text)
53 dom = html.fromstring(resp.text)
53
54
54 try:
55 try:
55− number_of_results_string = re.sub('[^0-9]', '', dom.xpath(
56+ number_of_results_string =\
57+ re.sub('[^0-9]', '',
56− '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]
58+ eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])
57− )
58
59
59 results.append({'number_of_results': int(number_of_results_string)})
60 results.append({'number_of_results': int(number_of_results_string)})
60
61
@@ -62,12 +63,12 @@ def response(resp):
62 logger.debug("Couldn't read number of results.")
63 logger.debug("Couldn't read number of results.")
63 pass
64 pass
64
65
65− for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
66+ for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):
66 try:
67 try:
67− url = result.xpath('.//h2/a')[0].get('href')
68+ url = eval_xpath(result, './/h2/a')[0].get('href')
68 url = urljoin(base_url, url)
69 url = urljoin(base_url, url)
69− title = result.xpath('string(.//h2/a)').strip()
70+ title = eval_xpath(result, 'string(.//h2/a)').strip()
70− content = extract_text(result.xpath('.//p'))
71+ content = extract_text(eval_xpath(result, './/p'))
71 # append result
72 # append result
72 results.append({'url': url,
73 results.append({'url': url,
73 'title': title,
74 'title': title,
searx/engines/gigablast.py+3−2
@@ -15,6 +15,7 @@ from json import loads
15from time import time
15from time import time
16from lxml.html import fromstring
16from lxml.html import fromstring
17from searx.url_utils import urlencode
17from searx.url_utils import urlencode
18+from searx.utils import eval_xpath
18
19
19# engine dependent config
20# engine dependent config
20categories = ['general']
21categories = ['general']
@@ -99,9 +100,9 @@ def response(resp):
99def _fetch_supported_languages(resp):
100def _fetch_supported_languages(resp):
100 supported_languages = []
101 supported_languages = []
101 dom = fromstring(resp.text)
102 dom = fromstring(resp.text)
102− links = dom.xpath('//span[@id="menu2"]/a')
103+ links = eval_xpath(dom, '//span[@id="menu2"]/a')
103 for link in links:
104 for link in links:
104− href = link.xpath('./@href')[0].split('lang%3A')
105+ href = eval_xpath(link, './@href')[0].split('lang%3A')
105 if len(href) == 2:
106 if len(href) == 2:
106 code = href[1].split('_')
107 code = href[1].split('_')
107 if len(code) == 2:
108 if len(code) == 2:
searx/engines/google.py+16−16
@@ -14,7 +14,7 @@ from lxml import html, etree
14from searx.engines.xpath import extract_text, extract_url
14from searx.engines.xpath import extract_text, extract_url
15from searx import logger
15from searx import logger
16from searx.url_utils import urlencode, urlparse, parse_qsl
16from searx.url_utils import urlencode, urlparse, parse_qsl
17−from searx.utils import match_language
17+from searx.utils import match_language, eval_xpath
18
18
19logger = logger.getChild('google engine')
19logger = logger.getChild('google engine')
20
20
@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):
156
156
157# returns extract_text on the first result selected by the xpath or None
157# returns extract_text on the first result selected by the xpath or None
158def extract_text_from_dom(result, xpath):
158def extract_text_from_dom(result, xpath):
159− r = result.xpath(xpath)
159+ r = eval_xpath(result, xpath)
160 if len(r) > 0:
160 if len(r) > 0:
161 return extract_text(r[0])
161 return extract_text(r[0])
162 return None
162 return None
@@ -226,21 +226,21 @@ def response(resp):
226 # convert the text to dom
226 # convert the text to dom
227 dom = html.fromstring(resp.text)
227 dom = html.fromstring(resp.text)
228
228
229− instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
229+ instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')
230 if instant_answer:
230 if instant_answer:
231 results.append({'answer': u' '.join(instant_answer)})
231 results.append({'answer': u' '.join(instant_answer)})
232 try:
232 try:
233− results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
233+ results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]
234 .split()[1].replace(',', ''))
234 .split()[1].replace(',', ''))
235 results.append({'number_of_results': results_num})
235 results.append({'number_of_results': results_num})
236 except:
236 except:
237 pass
237 pass
238
238
239 # parse results
239 # parse results
240− for result in dom.xpath(results_xpath):
240+ for result in eval_xpath(dom, results_xpath):
241 try:
241 try:
242− title = extract_text(result.xpath(title_xpath)[0])
242+ title = extract_text(eval_xpath(result, title_xpath)[0])
243− url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
243+ url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)
244 parsed_url = urlparse(url, google_hostname)
244 parsed_url = urlparse(url, google_hostname)
245
245
246 # map result
246 # map result
@@ -249,7 +249,7 @@ def response(resp):
249 continue
249 continue
250 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
250 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
251 # print "yooooo"*30
251 # print "yooooo"*30
252− # x = result.xpath(map_near)
252+ # x = eval_xpath(result, map_near)
253 # if len(x) > 0:
253 # if len(x) > 0:
254 # # map : near the location
254 # # map : near the location
255 # results = results + parse_map_near(parsed_url, x, google_hostname)
255 # results = results + parse_map_near(parsed_url, x, google_hostname)
@@ -286,11 +286,11 @@ def response(resp):
286 continue
286 continue
287
287
288 # parse suggestion
288 # parse suggestion
289− for suggestion in dom.xpath(suggestion_xpath):
289+ for suggestion in eval_xpath(dom, suggestion_xpath):
290 # append suggestion
290 # append suggestion
291 results.append({'suggestion': extract_text(suggestion)})
291 results.append({'suggestion': extract_text(suggestion)})
292
292
293− for correction in dom.xpath(spelling_suggestion_xpath):
293+ for correction in eval_xpath(dom, spelling_suggestion_xpath):
294 results.append({'correction': extract_text(correction)})
294 results.append({'correction': extract_text(correction)})
295
295
296 # return results
296 # return results
@@ -299,9 +299,9 @@ def response(resp):
299
299
300def parse_images(result, google_hostname):
300def parse_images(result, google_hostname):
301 results = []
301 results = []
302− for image in result.xpath(images_xpath):
302+ for image in eval_xpath(result, images_xpath):
303− url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
303+ url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)
304− img_src = extract_text(image.xpath(image_img_src_xpath)[0])
304+ img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])
305
305
306 # append result
306 # append result
307 results.append({'url': url,
307 results.append({'url': url,
@@ -388,10 +388,10 @@ def attributes_to_html(attributes):
388def _fetch_supported_languages(resp):
388def _fetch_supported_languages(resp):
389 supported_languages = {}
389 supported_languages = {}
390 dom = html.fromstring(resp.text)
390 dom = html.fromstring(resp.text)
391− options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
391+ options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')
392 for option in options:
392 for option in options:
393− code = option.xpath('./@value')[0].split('_')[-1]
393+ code = eval_xpath(option, './@value')[0].split('_')[-1]
394− name = option.xpath('./@data-name')[0].title()
394+ name = eval_xpath(option, './@data-name')[0].title()
395 supported_languages[code] = {"name": name}
395 supported_languages[code] = {"name": name}
396
396
397 return supported_languages
397 return supported_languages
searx/engines/startpage.py+5−4
@@ -16,6 +16,7 @@ from datetime import datetime, timedelta
16import re
16import re
17from searx.engines.xpath import extract_text
17from searx.engines.xpath import extract_text
18from searx.languages import language_codes
18from searx.languages import language_codes
19+from searx.utils import eval_xpath
19
20
20# engine dependent config
21# engine dependent config
21categories = ['general']
22categories = ['general']
@@ -70,8 +71,8 @@ def response(resp):
70 dom = html.fromstring(resp.text)
71 dom = html.fromstring(resp.text)
71
72
72 # parse results
73 # parse results
73− for result in dom.xpath(results_xpath):
74+ for result in eval_xpath(dom, results_xpath):
74− links = result.xpath(link_xpath)
75+ links = eval_xpath(result, link_xpath)
75 if not links:
76 if not links:
76 continue
77 continue
77 link = links[0]
78 link = links[0]
@@ -87,8 +88,8 @@ def response(resp):
87
88
88 title = extract_text(link)
89 title = extract_text(link)
89
90
90− if result.xpath(content_xpath):
91+ if eval_xpath(result, content_xpath):
91− content = extract_text(result.xpath(content_xpath))
92+ content = extract_text(eval_xpath(result, content_xpath))
92 else:
93 else:
93 content = ''
94 content = ''
94
95
searx/engines/wikidata.py+1−17
@@ -16,7 +16,7 @@ from searx.poolrequests import get
16from searx.engines.xpath import extract_text
16from searx.engines.xpath import extract_text
17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
18from searx.url_utils import urlencode
18from searx.url_utils import urlencode
19−from searx.utils import match_language
19+from searx.utils import match_language, eval_xpath
20
20
21from json import loads
21from json import loads
22from lxml.html import fromstring
22from lxml.html import fromstring
@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
59
59
60−# xpath_cache
61−xpath_cache = {}
62−
63−
64−def get_xpath(xpath_str):
65− result = xpath_cache.get(xpath_str, None)
66− if not result:
67− result = etree.XPath(xpath_str)
68− xpath_cache[xpath_str] = result
69− return result
70−
71−
72−def eval_xpath(element, xpath_str):
73− xpath = get_xpath(xpath_str)
74− return xpath(element)
75−
76
60
77def get_id_cache(result):
61def get_id_cache(result):
78 id_cache = {}
62 id_cache = {}
searx/engines/xpath.py+10−10
@@ -1,6 +1,6 @@
1from lxml import html
1from lxml import html
2from lxml.etree import _ElementStringResult, _ElementUnicodeResult
2from lxml.etree import _ElementStringResult, _ElementUnicodeResult
3−from searx.utils import html_to_text
3+from searx.utils import html_to_text, eval_xpath
4from searx.url_utils import unquote, urlencode, urljoin, urlparse
4from searx.url_utils import unquote, urlencode, urljoin, urlparse
5
5
6search_url = None
6search_url = None
@@ -104,15 +104,15 @@ def response(resp):
104 results = []
104 results = []
105 dom = html.fromstring(resp.text)
105 dom = html.fromstring(resp.text)
106 if results_xpath:
106 if results_xpath:
107− for result in dom.xpath(results_xpath):
107+ for result in eval_xpath(dom, results_xpath):
108− url = extract_url(result.xpath(url_xpath), search_url)
108+ url = extract_url(eval_xpath(result, url_xpath), search_url)
109− title = extract_text(result.xpath(title_xpath))
109+ title = extract_text(eval_xpath(result, title_xpath))
110− content = extract_text(result.xpath(content_xpath))
110+ content = extract_text(eval_xpath(result, content_xpath))
111 tmp_result = {'url': url, 'title': title, 'content': content}
111 tmp_result = {'url': url, 'title': title, 'content': content}
112
112
113 # add thumbnail if available
113 # add thumbnail if available
114 if thumbnail_xpath:
114 if thumbnail_xpath:
115− thumbnail_xpath_result = result.xpath(thumbnail_xpath)
115+ thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)
116 if len(thumbnail_xpath_result) > 0:
116 if len(thumbnail_xpath_result) > 0:
117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
118
118
@@ -120,14 +120,14 @@ def response(resp):
120 else:
120 else:
121 for url, title, content in zip(
121 for url, title, content in zip(
122 (extract_url(x, search_url) for
122 (extract_url(x, search_url) for
123− x in dom.xpath(url_xpath)),
123+ x in eval_xpath(dom, url_xpath)),
124− map(extract_text, dom.xpath(title_xpath)),
124+ map(extract_text, eval_xpath(dom, title_xpath)),
125− map(extract_text, dom.xpath(content_xpath))
125+ map(extract_text, eval_xpath(dom, content_xpath))
126 ):
126 ):
127 results.append({'url': url, 'title': title, 'content': content})
127 results.append({'url': url, 'title': title, 'content': content})
128
128
129 if not suggestion_xpath:
129 if not suggestion_xpath:
130 return results
130 return results
131− for suggestion in dom.xpath(suggestion_xpath):
131+ for suggestion in eval_xpath(dom, suggestion_xpath):
132 results.append({'suggestion': extract_text(suggestion)})
132 results.append({'suggestion': extract_text(suggestion)})
133 return results
133 return results
searx/engines/yahoo.py+9−9
@@ -14,7 +14,7 @@
14from lxml import html
14from lxml import html
15from searx.engines.xpath import extract_text, extract_url
15from searx.engines.xpath import extract_text, extract_url
16from searx.url_utils import unquote, urlencode
16from searx.url_utils import unquote, urlencode
17−from searx.utils import match_language
17+from searx.utils import match_language, eval_xpath
18
18
19# engine dependent config
19# engine dependent config
20categories = ['general']
20categories = ['general']
@@ -109,21 +109,21 @@ def response(resp):
109 dom = html.fromstring(resp.text)
109 dom = html.fromstring(resp.text)
110
110
111 try:
111 try:
112− results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]
112+ results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]
113 .split()[0].replace(',', ''))
113 .split()[0].replace(',', ''))
114 results.append({'number_of_results': results_num})
114 results.append({'number_of_results': results_num})
115 except:
115 except:
116 pass
116 pass
117
117
118 # parse results
118 # parse results
119− for result in dom.xpath(results_xpath):
119+ for result in eval_xpath(dom, results_xpath):
120 try:
120 try:
121− url = parse_url(extract_url(result.xpath(url_xpath), search_url))
121+ url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))
122− title = extract_text(result.xpath(title_xpath)[0])
122+ title = extract_text(eval_xpath(result, title_xpath)[0])
123 except:
123 except:
124 continue
124 continue
125
125
126− content = extract_text(result.xpath(content_xpath)[0])
126+ content = extract_text(eval_xpath(result, content_xpath)[0])
127
127
128 # append result
128 # append result
129 results.append({'url': url,
129 results.append({'url': url,
@@ -131,7 +131,7 @@ def response(resp):
131 'content': content})
131 'content': content})
132
132
133 # if no suggestion found, return results
133 # if no suggestion found, return results
134− suggestions = dom.xpath(suggestion_xpath)
134+ suggestions = eval_xpath(dom, suggestion_xpath)
135 if not suggestions:
135 if not suggestions:
136 return results
136 return results
137
137
@@ -148,9 +148,9 @@ def response(resp):
148def _fetch_supported_languages(resp):
148def _fetch_supported_languages(resp):
149 supported_languages = []
149 supported_languages = []
150 dom = html.fromstring(resp.text)
150 dom = html.fromstring(resp.text)
151− options = dom.xpath('//div[@id="yschlang"]/span/label/input')
151+ options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')
152 for option in options:
152 for option in options:
153− code_parts = option.xpath('./@value')[0][5:].split('_')
153+ code_parts = eval_xpath(option, './@value')[0][5:].split('_')
154 if len(code_parts) == 2:
154 if len(code_parts) == 2:
155 code = code_parts[0] + '-' + code_parts[1].upper()
155 code = code_parts[0] + '-' + code_parts[1].upper()
156 else:
156 else:
searx/results.py+5−3
@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):
67
67
68 for url2 in infobox2.get('urls', []):
68 for url2 in infobox2.get('urls', []):
69 unique_url = True
69 unique_url = True
70+ parsed_url2 = urlparse(url2.get('url', ''))
70− for url1 in infobox1.get('urls', []):
71+ for url1 in urls1:
71− if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):
72+ if compare_urls(urlparse(url1.get('url', '')), parsed_url2):
72 unique_url = False
73 unique_url = False
73 break
74 break
74 if unique_url:
75 if unique_url:
@@ -188,8 +189,9 @@ class ResultContainer(object):
188 add_infobox = True
189 add_infobox = True
189 infobox_id = infobox.get('id', None)
190 infobox_id = infobox.get('id', None)
190 if infobox_id is not None:
191 if infobox_id is not None:
192+ parsed_url_infobox_id = urlparse(infobox_id)
191 for existingIndex in self.infoboxes:
193 for existingIndex in self.infoboxes:
192− if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):
194+ if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
193 merge_two_infoboxes(existingIndex, infobox)
195 merge_two_infoboxes(existingIndex, infobox)
194 add_infobox = False
196 add_infobox = False
195
197
searx/utils.py+15
@@ -13,6 +13,7 @@ from numbers import Number
13from os.path import splitext, join
13from os.path import splitext, join
14from io import open
14from io import open
15from random import choice
15from random import choice
16+from lxml.etree import XPath
16import sys
17import sys
17import json
18import json
18
19
@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
51useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
52useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
52 + "/data/useragents.json", 'r', encoding='utf-8').read())
53 + "/data/useragents.json", 'r', encoding='utf-8').read())
53
54
55+xpath_cache = dict()
54lang_to_lc_cache = dict()
56lang_to_lc_cache = dict()
55
57
56
58
@@ -450,3 +452,16 @@ def get_engine_from_settings(name):
450 return engine
452 return engine
451
453
452 return {}
454 return {}
455+
456+
457+def get_xpath(xpath_str):
458+ result = xpath_cache.get(xpath_str, None)
459+ if result is None:
460+ result = XPath(xpath_str)
461+ xpath_cache[xpath_str] = result
462+ return result
463+
464+
465+def eval_xpath(element, xpath_str):
466+ xpath = get_xpath(xpath_str)
467+ return xpath(element)
searx/webapp.py+7−9
@@ -154,20 +154,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None
154
154
155@babel.localeselector
155@babel.localeselector
156def get_locale():
156def get_locale():
157− locale = request.accept_languages.best_match(settings['locales'].keys())
157+ if 'locale' in request.form\
158−
158+ and request.form['locale'] in settings['locales']:
159− if request.preferences.get_value('locale') != '':
160− locale = request.preferences.get_value('locale')
159+ return request.form['locale']
161
160
162 if 'locale' in request.args\
161 if 'locale' in request.args\
163 and request.args['locale'] in settings['locales']:
162 and request.args['locale'] in settings['locales']:
164− locale = request.args['locale']
163+ return request.args['locale']
165
164
166− if 'locale' in request.form\
165+ if request.preferences.get_value('locale') != '':
167− and request.form['locale'] in settings['locales']:
168− locale = request.form['locale']
166+ return request.preferences.get_value('locale')
169
167
170− return locale
168+ return request.accept_languages.best_match(settings['locales'].keys())
171
169
172
170
173# code-highlighter
171# code-highlighter