on master help sign in
Merge pull request #1744 from dalf/optimizations
731e342Adam Tauber
searx/engines/bing.py+10−10
@@ -18,7 +18,7 @@ from lxml import html
18from searx import logger, utils
18from searx import logger, utils
19from searx.engines.xpath import extract_text
19from searx.engines.xpath import extract_text
20from searx.url_utils import urlencode
20from searx.url_utils import urlencode
21−from searx.utils import match_language, gen_useragent
21+from searx.utils import match_language, gen_useragent, eval_xpath
22
22
23logger = logger.getChild('bing engine')
23logger = logger.getChild('bing engine')
24
24
@@ -65,11 +65,11 @@ def response(resp):
65
65
66 dom = html.fromstring(resp.text)
66 dom = html.fromstring(resp.text)
67 # parse results
67 # parse results
68− for result in dom.xpath('//div[@class="sa_cc"]'):
68+ for result in eval_xpath(dom, '//div[@class="sa_cc"]'):
69− link = result.xpath('.//h3/a')[0]
69+ link = eval_xpath(result, './/h3/a')[0]
70 url = link.attrib.get('href')
70 url = link.attrib.get('href')
71 title = extract_text(link)
71 title = extract_text(link)
72− content = extract_text(result.xpath('.//p'))
72+ content = extract_text(eval_xpath(result, './/p'))
73
73
74 # append result
74 # append result
75 results.append({'url': url,
75 results.append({'url': url,
@@ -77,11 +77,11 @@ def response(resp):
77 'content': content})
77 'content': content})
78
78
79 # parse results again if nothing is found yet
79 # parse results again if nothing is found yet
80− for result in dom.xpath('//li[@class="b_algo"]'):
80+ for result in eval_xpath(dom, '//li[@class="b_algo"]'):
81− link = result.xpath('.//h2/a')[0]
81+ link = eval_xpath(result, './/h2/a')[0]
82 url = link.attrib.get('href')
82 url = link.attrib.get('href')
83 title = extract_text(link)
83 title = extract_text(link)
84− content = extract_text(result.xpath('.//p'))
84+ content = extract_text(eval_xpath(result, './/p'))
85
85
86 # append result
86 # append result
87 results.append({'url': url,
87 results.append({'url': url,
@@ -89,7 +89,7 @@ def response(resp):
89 'content': content})
89 'content': content})
90
90
91 try:
91 try:
92− result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))
92+ result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))
93 result_len_container = utils.to_string(result_len_container)
93 result_len_container = utils.to_string(result_len_container)
94 if "-" in result_len_container:
94 if "-" in result_len_container:
95 # Remove the part "from-to" for paginated request ...
95 # Remove the part "from-to" for paginated request ...
@@ -113,9 +113,9 @@ def response(resp):
113def _fetch_supported_languages(resp):
113def _fetch_supported_languages(resp):
114 supported_languages = []
114 supported_languages = []
115 dom = html.fromstring(resp.text)
115 dom = html.fromstring(resp.text)
116− options = dom.xpath('//div[@id="limit-languages"]//input')
116+ options = eval_xpath(dom, '//div[@id="limit-languages"]//input')
117 for option in options:
117 for option in options:
118− code = option.xpath('./@id')[0].replace('_', '-')
118+ code = eval_xpath(option, './@id')[0].replace('_', '-')
119 if code == 'nb':
119 if code == 'nb':
120 code = 'no'
120 code = 'no'
121 supported_languages.append(code)
121 supported_languages.append(code)
searx/engines/dictzone.py+4−4
@@ -11,7 +11,7 @@
11
11
12import re
12import re
13from lxml import html
13from lxml import html
14−from searx.utils import is_valid_lang
14+from searx.utils import is_valid_lang, eval_xpath
15from searx.url_utils import urljoin
15from searx.url_utils import urljoin
16
16
17categories = ['general']
17categories = ['general']
@@ -47,14 +47,14 @@ def response(resp):
47
47
48 dom = html.fromstring(resp.text)
48 dom = html.fromstring(resp.text)
49
49
50− for k, result in enumerate(dom.xpath(results_xpath)[1:]):
50+ for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):
51 try:
51 try:
52− from_result, to_results_raw = result.xpath('./td')
52+ from_result, to_results_raw = eval_xpath(result, './td')
53 except:
53 except:
54 continue
54 continue
55
55
56 to_results = []
56 to_results = []
57− for to_result in to_results_raw.xpath('./p/a'):
57+ for to_result in eval_xpath(to_results_raw, './p/a'):
58 t = to_result.text_content()
58 t = to_result.text_content()
59 if t.strip():
59 if t.strip():
60 to_results.append(to_result.text_content())
60 to_results.append(to_result.text_content())
searx/engines/doku.py+8−7
@@ -11,6 +11,7 @@
11
11
12from lxml.html import fromstring
12from lxml.html import fromstring
13from searx.engines.xpath import extract_text
13from searx.engines.xpath import extract_text
14+from searx.utils import eval_xpath
14from searx.url_utils import urlencode
15from searx.url_utils import urlencode
15
16
16# engine dependent config
17# engine dependent config
@@ -45,16 +46,16 @@ def response(resp):
45
46
46 # parse results
47 # parse results
47 # Quickhits
48 # Quickhits
48− for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):
49+ for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):
49 try:
50 try:
50− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
51+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
51 except:
52 except:
52 continue
53 continue
53
54
54 if not res_url:
55 if not res_url:
55 continue
56 continue
56
57
57− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
58+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
58
59
59 # append result
60 # append result
60 results.append({'title': title,
61 results.append({'title': title,
@@ -62,13 +63,13 @@ def response(resp):
62 'url': base_url + res_url})
63 'url': base_url + res_url})
63
64
64 # Search results
65 # Search results
65− for r in doc.xpath('//dl[@class="search_results"]/*'):
66+ for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):
66 try:
67 try:
67 if r.tag == "dt":
68 if r.tag == "dt":
68− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]
69+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]
69− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))
70+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))
70 elif r.tag == "dd":
71 elif r.tag == "dd":
71− content = extract_text(r.xpath('.'))
72+ content = extract_text(eval_xpath(r, '.'))
72
73
73 # append result
74 # append result
74 results.append({'title': title,
75 results.append({'title': title,
searx/engines/duckduckgo.py+5−5
@@ -18,7 +18,7 @@ from json import loads
18from searx.engines.xpath import extract_text
18from searx.engines.xpath import extract_text
19from searx.poolrequests import get
19from searx.poolrequests import get
20from searx.url_utils import urlencode
20from searx.url_utils import urlencode
21−from searx.utils import match_language
21+from searx.utils import match_language, eval_xpath
22
22
23# engine dependent config
23# engine dependent config
24categories = ['general']
24categories = ['general']
@@ -106,19 +106,19 @@ def response(resp):
106 doc = fromstring(resp.text)
106 doc = fromstring(resp.text)
107
107
108 # parse results
108 # parse results
109− for i, r in enumerate(doc.xpath(result_xpath)):
109+ for i, r in enumerate(eval_xpath(doc, result_xpath)):
110 if i >= 30:
110 if i >= 30:
111 break
111 break
112 try:
112 try:
113− res_url = r.xpath(url_xpath)[-1]
113+ res_url = eval_xpath(r, url_xpath)[-1]
114 except:
114 except:
115 continue
115 continue
116
116
117 if not res_url:
117 if not res_url:
118 continue
118 continue
119
119
120− title = extract_text(r.xpath(title_xpath))
120+ title = extract_text(eval_xpath(r, title_xpath))
121− content = extract_text(r.xpath(content_xpath))
121+ content = extract_text(eval_xpath(r, content_xpath))
122
122
123 # append result
123 # append result
124 results.append({'title': title,
124 results.append({'title': title,
searx/engines/duden.py+8−7
@@ -11,6 +11,7 @@
11from lxml import html, etree
11from lxml import html, etree
12import re
12import re
13from searx.engines.xpath import extract_text
13from searx.engines.xpath import extract_text
14+from searx.utils import eval_xpath
14from searx.url_utils import quote, urljoin
15from searx.url_utils import quote, urljoin
15from searx import logger
16from searx import logger
16
17
@@ -52,9 +53,9 @@ def response(resp):
52 dom = html.fromstring(resp.text)
53 dom = html.fromstring(resp.text)
53
54
54 try:
55 try:
55− number_of_results_string = re.sub('[^0-9]', '', dom.xpath(
56+ number_of_results_string =\
57+ re.sub('[^0-9]', '',
56− '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]
58+ eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])
57− )
58
59
59 results.append({'number_of_results': int(number_of_results_string)})
60 results.append({'number_of_results': int(number_of_results_string)})
60
61
@@ -62,12 +63,12 @@ def response(resp):
62 logger.debug("Couldn't read number of results.")
63 logger.debug("Couldn't read number of results.")
63 pass
64 pass
64
65
65− for result in dom.xpath('//section[not(contains(@class, "essay"))]'):
66+ for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):
66 try:
67 try:
67− url = result.xpath('.//h2/a')[0].get('href')
68+ url = eval_xpath(result, './/h2/a')[0].get('href')
68 url = urljoin(base_url, url)
69 url = urljoin(base_url, url)
69− title = result.xpath('string(.//h2/a)').strip()
70+ title = eval_xpath(result, 'string(.//h2/a)').strip()
70− content = extract_text(result.xpath('.//p'))
71+ content = extract_text(eval_xpath(result, './/p'))
71 # append result
72 # append result
72 results.append({'url': url,
73 results.append({'url': url,
73 'title': title,
74 'title': title,
searx/engines/gigablast.py+3−2
@@ -15,6 +15,7 @@ from json import loads
15from time import time
15from time import time
16from lxml.html import fromstring
16from lxml.html import fromstring
17from searx.url_utils import urlencode
17from searx.url_utils import urlencode
18+from searx.utils import eval_xpath
18
19
19# engine dependent config
20# engine dependent config
20categories = ['general']
21categories = ['general']
@@ -99,9 +100,9 @@ def response(resp):
99def _fetch_supported_languages(resp):
100def _fetch_supported_languages(resp):
100 supported_languages = []
101 supported_languages = []
101 dom = fromstring(resp.text)
102 dom = fromstring(resp.text)
102− links = dom.xpath('//span[@id="menu2"]/a')
103+ links = eval_xpath(dom, '//span[@id="menu2"]/a')
103 for link in links:
104 for link in links:
104− href = link.xpath('./@href')[0].split('lang%3A')
105+ href = eval_xpath(link, './@href')[0].split('lang%3A')
105 if len(href) == 2:
106 if len(href) == 2:
106 code = href[1].split('_')
107 code = href[1].split('_')
107 if len(code) == 2:
108 if len(code) == 2:
searx/engines/google.py+16−16
@@ -14,7 +14,7 @@ from lxml import html, etree
14from searx.engines.xpath import extract_text, extract_url
14from searx.engines.xpath import extract_text, extract_url
15from searx import logger
15from searx import logger
16from searx.url_utils import urlencode, urlparse, parse_qsl
16from searx.url_utils import urlencode, urlparse, parse_qsl
17−from searx.utils import match_language
17+from searx.utils import match_language, eval_xpath
18
18
19logger = logger.getChild('google engine')
19logger = logger.getChild('google engine')
20
20
@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):
156
156
157# returns extract_text on the first result selected by the xpath or None
157# returns extract_text on the first result selected by the xpath or None
158def extract_text_from_dom(result, xpath):
158def extract_text_from_dom(result, xpath):
159− r = result.xpath(xpath)
159+ r = eval_xpath(result, xpath)
160 if len(r) > 0:
160 if len(r) > 0:
161 return extract_text(r[0])
161 return extract_text(r[0])
162 return None
162 return None
@@ -227,21 +227,21 @@ def response(resp):
227 # convert the text to dom
227 # convert the text to dom
228 dom = html.fromstring(resp.text)
228 dom = html.fromstring(resp.text)
229
229
230− instant_answer = dom.xpath('//div[@id="_vBb"]//text()')
230+ instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')
231 if instant_answer:
231 if instant_answer:
232 results.append({'answer': u' '.join(instant_answer)})
232 results.append({'answer': u' '.join(instant_answer)})
233 try:
233 try:
234− results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]
234+ results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]
235 .split()[1].replace(',', ''))
235 .split()[1].replace(',', ''))
236 results.append({'number_of_results': results_num})
236 results.append({'number_of_results': results_num})
237 except:
237 except:
238 pass
238 pass
239
239
240 # parse results
240 # parse results
241− for result in dom.xpath(results_xpath):
241+ for result in eval_xpath(dom, results_xpath):
242 try:
242 try:
243− title = extract_text(result.xpath(title_xpath)[0])
243+ title = extract_text(eval_xpath(result, title_xpath)[0])
244− url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)
244+ url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)
245 parsed_url = urlparse(url, google_hostname)
245 parsed_url = urlparse(url, google_hostname)
246
246
247 # map result
247 # map result
@@ -250,7 +250,7 @@ def response(resp):
250 continue
250 continue
251 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
251 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):
252 # print "yooooo"*30
252 # print "yooooo"*30
253− # x = result.xpath(map_near)
253+ # x = eval_xpath(result, map_near)
254 # if len(x) > 0:
254 # if len(x) > 0:
255 # # map : near the location
255 # # map : near the location
256 # results = results + parse_map_near(parsed_url, x, google_hostname)
256 # results = results + parse_map_near(parsed_url, x, google_hostname)
@@ -287,11 +287,11 @@ def response(resp):
287 continue
287 continue
288
288
289 # parse suggestion
289 # parse suggestion
290− for suggestion in dom.xpath(suggestion_xpath):
290+ for suggestion in eval_xpath(dom, suggestion_xpath):
291 # append suggestion
291 # append suggestion
292 results.append({'suggestion': extract_text(suggestion)})
292 results.append({'suggestion': extract_text(suggestion)})
293
293
294− for correction in dom.xpath(spelling_suggestion_xpath):
294+ for correction in eval_xpath(dom, spelling_suggestion_xpath):
295 results.append({'correction': extract_text(correction)})
295 results.append({'correction': extract_text(correction)})
296
296
297 # return results
297 # return results
@@ -300,9 +300,9 @@ def response(resp):
300
300
301def parse_images(result, google_hostname):
301def parse_images(result, google_hostname):
302 results = []
302 results = []
303− for image in result.xpath(images_xpath):
303+ for image in eval_xpath(result, images_xpath):
304− url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)
304+ url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)
305− img_src = extract_text(image.xpath(image_img_src_xpath)[0])
305+ img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])
306
306
307 # append result
307 # append result
308 results.append({'url': url,
308 results.append({'url': url,
@@ -389,10 +389,10 @@ def attributes_to_html(attributes):
389def _fetch_supported_languages(resp):
389def _fetch_supported_languages(resp):
390 supported_languages = {}
390 supported_languages = {}
391 dom = html.fromstring(resp.text)
391 dom = html.fromstring(resp.text)
392− options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')
392+ options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')
393 for option in options:
393 for option in options:
394− code = option.xpath('./@value')[0].split('_')[-1]
394+ code = eval_xpath(option, './@value')[0].split('_')[-1]
395− name = option.xpath('./@data-name')[0].title()
395+ name = eval_xpath(option, './@data-name')[0].title()
396 supported_languages[code] = {"name": name}
396 supported_languages[code] = {"name": name}
397
397
398 return supported_languages
398 return supported_languages
searx/engines/startpage.py+5−4
@@ -16,6 +16,7 @@ from datetime import datetime, timedelta
16import re
16import re
17from searx.engines.xpath import extract_text
17from searx.engines.xpath import extract_text
18from searx.languages import language_codes
18from searx.languages import language_codes
19+from searx.utils import eval_xpath
19
20
20# engine dependent config
21# engine dependent config
21categories = ['general']
22categories = ['general']
@@ -70,8 +71,8 @@ def response(resp):
70 dom = html.fromstring(resp.text)
71 dom = html.fromstring(resp.text)
71
72
72 # parse results
73 # parse results
73− for result in dom.xpath(results_xpath):
74+ for result in eval_xpath(dom, results_xpath):
74− links = result.xpath(link_xpath)
75+ links = eval_xpath(result, link_xpath)
75 if not links:
76 if not links:
76 continue
77 continue
77 link = links[0]
78 link = links[0]
@@ -87,8 +88,8 @@ def response(resp):
87
88
88 title = extract_text(link)
89 title = extract_text(link)
89
90
90− if result.xpath(content_xpath):
91+ if eval_xpath(result, content_xpath):
91− content = extract_text(result.xpath(content_xpath))
92+ content = extract_text(eval_xpath(result, content_xpath))
92 else:
93 else:
93 content = ''
94 content = ''
94
95
searx/engines/wikidata.py+1−17
@@ -16,7 +16,7 @@ from searx.poolrequests import get
16from searx.engines.xpath import extract_text
16from searx.engines.xpath import extract_text
17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url
18from searx.url_utils import urlencode
18from searx.url_utils import urlencode
19−from searx.utils import match_language
19+from searx.utils import match_language, eval_xpath
20
20
21from json import loads
21from json import loads
22from lxml.html import fromstring
22from lxml.html import fromstring
@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
59
59
60−# xpath_cache
61−xpath_cache = {}
62−
63−
64−def get_xpath(xpath_str):
65− result = xpath_cache.get(xpath_str, None)
66− if not result:
67− result = etree.XPath(xpath_str)
68− xpath_cache[xpath_str] = result
69− return result
70−
71−
72−def eval_xpath(element, xpath_str):
73− xpath = get_xpath(xpath_str)
74− return xpath(element)
75−
76
60
77def get_id_cache(result):
61def get_id_cache(result):
78 id_cache = {}
62 id_cache = {}
searx/engines/xpath.py+10−10
@@ -1,6 +1,6 @@
1from lxml import html
1from lxml import html
2from lxml.etree import _ElementStringResult, _ElementUnicodeResult
2from lxml.etree import _ElementStringResult, _ElementUnicodeResult
3−from searx.utils import html_to_text
3+from searx.utils import html_to_text, eval_xpath
4from searx.url_utils import unquote, urlencode, urljoin, urlparse
4from searx.url_utils import unquote, urlencode, urljoin, urlparse
5
5
6search_url = None
6search_url = None
@@ -104,15 +104,15 @@ def response(resp):
104 results = []
104 results = []
105 dom = html.fromstring(resp.text)
105 dom = html.fromstring(resp.text)
106 if results_xpath:
106 if results_xpath:
107− for result in dom.xpath(results_xpath):
107+ for result in eval_xpath(dom, results_xpath):
108− url = extract_url(result.xpath(url_xpath), search_url)
108+ url = extract_url(eval_xpath(result, url_xpath), search_url)
109− title = extract_text(result.xpath(title_xpath))
109+ title = extract_text(eval_xpath(result, title_xpath))
110− content = extract_text(result.xpath(content_xpath))
110+ content = extract_text(eval_xpath(result, content_xpath))
111 tmp_result = {'url': url, 'title': title, 'content': content}
111 tmp_result = {'url': url, 'title': title, 'content': content}
112
112
113 # add thumbnail if available
113 # add thumbnail if available
114 if thumbnail_xpath:
114 if thumbnail_xpath:
115− thumbnail_xpath_result = result.xpath(thumbnail_xpath)
115+ thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)
116 if len(thumbnail_xpath_result) > 0:
116 if len(thumbnail_xpath_result) > 0:
117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)
118
118
@@ -120,14 +120,14 @@ def response(resp):
120 else:
120 else:
121 for url, title, content in zip(
121 for url, title, content in zip(
122 (extract_url(x, search_url) for
122 (extract_url(x, search_url) for
123− x in dom.xpath(url_xpath)),
123+ x in eval_xpath(dom, url_xpath)),
124− map(extract_text, dom.xpath(title_xpath)),
124+ map(extract_text, eval_xpath(dom, title_xpath)),
125− map(extract_text, dom.xpath(content_xpath))
125+ map(extract_text, eval_xpath(dom, content_xpath))
126 ):
126 ):
127 results.append({'url': url, 'title': title, 'content': content})
127 results.append({'url': url, 'title': title, 'content': content})
128
128
129 if not suggestion_xpath:
129 if not suggestion_xpath:
130 return results
130 return results
131− for suggestion in dom.xpath(suggestion_xpath):
131+ for suggestion in eval_xpath(dom, suggestion_xpath):
132 results.append({'suggestion': extract_text(suggestion)})
132 results.append({'suggestion': extract_text(suggestion)})
133 return results
133 return results
searx/engines/yahoo.py+9−9
@@ -14,7 +14,7 @@
14from lxml import html
14from lxml import html
15from searx.engines.xpath import extract_text, extract_url
15from searx.engines.xpath import extract_text, extract_url
16from searx.url_utils import unquote, urlencode
16from searx.url_utils import unquote, urlencode
17−from searx.utils import match_language
17+from searx.utils import match_language, eval_xpath
18
18
19# engine dependent config
19# engine dependent config
20categories = ['general']
20categories = ['general']
@@ -109,21 +109,21 @@ def response(resp):
109 dom = html.fromstring(resp.text)
109 dom = html.fromstring(resp.text)
110
110
111 try:
111 try:
112− results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]
112+ results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]
113 .split()[0].replace(',', ''))
113 .split()[0].replace(',', ''))
114 results.append({'number_of_results': results_num})
114 results.append({'number_of_results': results_num})
115 except:
115 except:
116 pass
116 pass
117
117
118 # parse results
118 # parse results
119− for result in dom.xpath(results_xpath):
119+ for result in eval_xpath(dom, results_xpath):
120 try:
120 try:
121− url = parse_url(extract_url(result.xpath(url_xpath), search_url))
121+ url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))
122− title = extract_text(result.xpath(title_xpath)[0])
122+ title = extract_text(eval_xpath(result, title_xpath)[0])
123 except:
123 except:
124 continue
124 continue
125
125
126− content = extract_text(result.xpath(content_xpath)[0])
126+ content = extract_text(eval_xpath(result, content_xpath)[0])
127
127
128 # append result
128 # append result
129 results.append({'url': url,
129 results.append({'url': url,
@@ -131,7 +131,7 @@ def response(resp):
131 'content': content})
131 'content': content})
132
132
133 # if no suggestion found, return results
133 # if no suggestion found, return results
134− suggestions = dom.xpath(suggestion_xpath)
134+ suggestions = eval_xpath(dom, suggestion_xpath)
135 if not suggestions:
135 if not suggestions:
136 return results
136 return results
137
137
@@ -148,9 +148,9 @@ def response(resp):
148def _fetch_supported_languages(resp):
148def _fetch_supported_languages(resp):
149 supported_languages = []
149 supported_languages = []
150 dom = html.fromstring(resp.text)
150 dom = html.fromstring(resp.text)
151− options = dom.xpath('//div[@id="yschlang"]/span/label/input')
151+ options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')
152 for option in options:
152 for option in options:
153− code_parts = option.xpath('./@value')[0][5:].split('_')
153+ code_parts = eval_xpath(option, './@value')[0][5:].split('_')
154 if len(code_parts) == 2:
154 if len(code_parts) == 2:
155 code = code_parts[0] + '-' + code_parts[1].upper()
155 code = code_parts[0] + '-' + code_parts[1].upper()
156 else:
156 else:
searx/results.py+5−3
@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):
67
67
68 for url2 in infobox2.get('urls', []):
68 for url2 in infobox2.get('urls', []):
69 unique_url = True
69 unique_url = True
70+ parsed_url2 = urlparse(url2.get('url', ''))
70− for url1 in infobox1.get('urls', []):
71+ for url1 in urls1:
71− if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):
72+ if compare_urls(urlparse(url1.get('url', '')), parsed_url2):
72 unique_url = False
73 unique_url = False
73 break
74 break
74 if unique_url:
75 if unique_url:
@@ -188,8 +189,9 @@ class ResultContainer(object):
188 add_infobox = True
189 add_infobox = True
189 infobox_id = infobox.get('id', None)
190 infobox_id = infobox.get('id', None)
190 if infobox_id is not None:
191 if infobox_id is not None:
192+ parsed_url_infobox_id = urlparse(infobox_id)
191 for existingIndex in self.infoboxes:
193 for existingIndex in self.infoboxes:
192− if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):
194+ if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):
193 merge_two_infoboxes(existingIndex, infobox)
195 merge_two_infoboxes(existingIndex, infobox)
194 add_infobox = False
196 add_infobox = False
195
197
searx/utils.py+15
@@ -13,6 +13,7 @@ from numbers import Number
13from os.path import splitext, join
13from os.path import splitext, join
14from io import open
14from io import open
15from random import choice
15from random import choice
16+from lxml.etree import XPath
16import sys
17import sys
17import json
18import json
18
19
@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
51useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
52useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
52 + "/data/useragents.json", 'r', encoding='utf-8').read())
53 + "/data/useragents.json", 'r', encoding='utf-8').read())
53
54
55+xpath_cache = dict()
54lang_to_lc_cache = dict()
56lang_to_lc_cache = dict()
55
57
56
58
@@ -450,3 +452,16 @@ def get_engine_from_settings(name):
450 return engine
452 return engine
451
453
452 return {}
454 return {}
455+
456+
457+def get_xpath(xpath_str):
458+ result = xpath_cache.get(xpath_str, None)
459+ if result is None:
460+ result = XPath(xpath_str)
461+ xpath_cache[xpath_str] = result
462+ return result
463+
464+
465+def eval_xpath(element, xpath_str):
466+ xpath = get_xpath(xpath_str)
467+ return xpath(element)
searx/webapp.py+7−9
@@ -157,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None
157
157
158@babel.localeselector
158@babel.localeselector
159def get_locale():
159def get_locale():
160− locale = request.accept_languages.best_match(settings['locales'].keys())
160+ if 'locale' in request.form\
161−
161+ and request.form['locale'] in settings['locales']:
162− if request.preferences.get_value('locale') != '':
163− locale = request.preferences.get_value('locale')
162+ return request.form['locale']
164
163
165 if 'locale' in request.args\
164 if 'locale' in request.args\
166 and request.args['locale'] in settings['locales']:
165 and request.args['locale'] in settings['locales']:
167− locale = request.args['locale']
166+ return request.args['locale']
168
167
169− if 'locale' in request.form\
168+ if request.preferences.get_value('locale') != '':
170− and request.form['locale'] in settings['locales']:
171− locale = request.form['locale']
169+ return request.preferences.get_value('locale')
172
170
173− return locale
171+ return request.accept_languages.best_match(settings['locales'].keys())
174
172
175
173
176# code-highlighter
174# code-highlighter