1b90e14 · searx · git.iko.soy

Dockerfile+15−15

@@ -1,10 +1,17 @@

11FROM alpine:3.10

2+ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

3+EXPOSE 8080

4+VOLUME /etc/searx

5+VOLUME /var/log/uwsgi

26

3−ARG VERSION_GITCOMMIT=unknow

4−ARG SEARX_GIT_VERSION=unknow

7+ARG VERSION_GITCOMMIT=unknown

8+ARG SEARX_GIT_VERSION=unknown

59

6−ARG SEARX_GID=1000

7−ARG SEARX_UID=1000

10+ARG SEARX_GID=977

11+ARG SEARX_UID=977

12+

13+RUN addgroup -g ${SEARX_GID} searx && \

14+ adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx

815

916ARG TIMESTAMP_SETTINGS=0

1017ARG TIMESTAMP_UWSGI=0

@@ -16,19 +23,14 @@ ENV INSTANCE_NAME=searx \

1623 BASE_URL= \

1724 MORTY_KEY= \

1825 MORTY_URL=

19−EXPOSE 8080

20−VOLUME /etc/searx

21−VOLUME /var/log/uwsgi

2226

2327WORKDIR /usr/local/searx

2428

25−RUN addgroup -g ${SEARX_GID} searx && \

26− adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx

2729

2830COPY requirements.txt ./requirements.txt

2931

30−RUN apk -U upgrade \

31− && apk add -t build-dependencies \

32+RUN apk upgrade --no-cache \

33+ && apk add --no-cache -t build-dependencies \

3234 build-base \

3335 py3-setuptools \

3436 python3-dev \

@@ -38,7 +40,7 @@ RUN apk -U upgrade \

3840 openssl-dev \

3941 tar \

4042 git \

41− && apk add \

43+ && apk add --no-cache \

4244 ca-certificates \

4345 su-exec \

4446 python3 \

@@ -50,8 +52,7 @@ RUN apk -U upgrade \

5052 uwsgi-python3 \

5153 && pip3 install --upgrade pip \

5254 && pip3 install --no-cache -r requirements.txt \

53− && apk del build-dependencies \

54− && rm -f /var/cache/apk/*

55+ && apk del build-dependencies

5556

5657COPY --chown=searx:searx . .

5758

@@ -62,7 +63,6 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

6263 echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \

6364 fi

6465

65−ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

6666

6767# Keep this argument at the end since it change each time

6868ARG LABEL_DATE=

searx/engines/bing.py+10−10

@@ -18,7 +18,7 @@ from lxml import html

1818from searx import logger, utils

1919from searx.engines.xpath import extract_text

2020from searx.url_utils import urlencode

21−from searx.utils import match_language, gen_useragent

21+from searx.utils import match_language, gen_useragent, eval_xpath

2222

2323logger = logger.getChild('bing engine')

2424

@@ -65,11 +65,11 @@ def response(resp):

6565

6666 dom = html.fromstring(resp.text)

6767 # parse results

68− for result in dom.xpath('//div[@class="sa_cc"]'):

69− link = result.xpath('.//h3/a')[0]

68+ for result in eval_xpath(dom, '//div[@class="sa_cc"]'):

69+ link = eval_xpath(result, './/h3/a')[0]

7070 url = link.attrib.get('href')

7171 title = extract_text(link)

72− content = extract_text(result.xpath('.//p'))

72+ content = extract_text(eval_xpath(result, './/p'))

7373

7474 # append result

7575 results.append({'url': url,

@@ -77,11 +77,11 @@ def response(resp):

7777 'content': content})

7878

7979 # parse results again if nothing is found yet

80− for result in dom.xpath('//li[@class="b_algo"]'):

81− link = result.xpath('.//h2/a')[0]

80+ for result in eval_xpath(dom, '//li[@class="b_algo"]'):

81+ link = eval_xpath(result, './/h2/a')[0]

8282 url = link.attrib.get('href')

8383 title = extract_text(link)

84− content = extract_text(result.xpath('.//p'))

84+ content = extract_text(eval_xpath(result, './/p'))

8585

8686 # append result

8787 results.append({'url': url,

@@ -89,7 +89,7 @@ def response(resp):

8989 'content': content})

9090

9191 try:

92− result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))

92+ result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))

9393 result_len_container = utils.to_string(result_len_container)

9494 if "-" in result_len_container:

9595 # Remove the part "from-to" for paginated request ...

@@ -113,9 +113,9 @@ def response(resp):

113113def _fetch_supported_languages(resp):

114114 supported_languages = []

115115 dom = html.fromstring(resp.text)

116− options = dom.xpath('//div[@id="limit-languages"]//input')

116+ options = eval_xpath(dom, '//div[@id="limit-languages"]//input')

117117 for option in options:

118− code = option.xpath('./@id')[0].replace('_', '-')

118+ code = eval_xpath(option, './@id')[0].replace('_', '-')

119119 if code == 'nb':

120120 code = 'no'

121121 supported_languages.append(code)

searx/engines/dictzone.py+4−4

@@ -11,7 +11,7 @@

1111

1212import re

1313from lxml import html

14−from searx.utils import is_valid_lang

14+from searx.utils import is_valid_lang, eval_xpath

1515from searx.url_utils import urljoin

1616

1717categories = ['general']

@@ -47,14 +47,14 @@ def response(resp):

4747

4848 dom = html.fromstring(resp.text)

4949

50− for k, result in enumerate(dom.xpath(results_xpath)[1:]):

50+ for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):

5151 try:

52− from_result, to_results_raw = result.xpath('./td')

52+ from_result, to_results_raw = eval_xpath(result, './td')

5353 except:

5454 continue

5555

5656 to_results = []

57− for to_result in to_results_raw.xpath('./p/a'):

57+ for to_result in eval_xpath(to_results_raw, './p/a'):

5858 t = to_result.text_content()

5959 if t.strip():

6060 to_results.append(to_result.text_content())

searx/engines/doku.py+8−7

@@ -11,6 +11,7 @@

1111

1212from lxml.html import fromstring

1313from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

1415from searx.url_utils import urlencode

1516

1617# engine dependent config

@@ -45,16 +46,16 @@ def response(resp):

4546

4647 # parse results

4748 # Quickhits

48− for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):

49+ for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):

4950 try:

50− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

51+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

5152 except:

5253 continue

5354

5455 if not res_url:

5556 continue

5657

57− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

58+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

5859

5960 # append result

6061 results.append({'title': title,

@@ -62,13 +63,13 @@ def response(resp):

6263 'url': base_url + res_url})

6364

6465 # Search results

65− for r in doc.xpath('//dl[@class="search_results"]/*'):

66+ for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):

6667 try:

6768 if r.tag == "dt":

68− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

69− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

69+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

70+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

7071 elif r.tag == "dd":

71− content = extract_text(r.xpath('.'))

72+ content = extract_text(eval_xpath(r, '.'))

7273

7374 # append result

7475 results.append({'title': title,

searx/engines/duckduckgo.py+5−5

@@ -18,7 +18,7 @@ from json import loads

1818from searx.engines.xpath import extract_text

1919from searx.poolrequests import get

2020from searx.url_utils import urlencode

21−from searx.utils import match_language

21+from searx.utils import match_language, eval_xpath

2222

2323# engine dependent config

2424categories = ['general']

@@ -106,19 +106,19 @@ def response(resp):

106106 doc = fromstring(resp.text)

107107

108108 # parse results

109− for i, r in enumerate(doc.xpath(result_xpath)):

109+ for i, r in enumerate(eval_xpath(doc, result_xpath)):

110110 if i >= 30:

111111 break

112112 try:

113− res_url = r.xpath(url_xpath)[-1]

113+ res_url = eval_xpath(r, url_xpath)[-1]

114114 except:

115115 continue

116116

117117 if not res_url:

118118 continue

119119

120− title = extract_text(r.xpath(title_xpath))

121− content = extract_text(r.xpath(content_xpath))

120+ title = extract_text(eval_xpath(r, title_xpath))

121+ content = extract_text(eval_xpath(r, content_xpath))

122122

123123 # append result

124124 results.append({'title': title,

searx/engines/duckduckgo_definitions.py+16−3

@@ -1,3 +1,14 @@

1+"""

2+DuckDuckGo (definitions)

3+

4+- `Instant Answer API`_

5+- `DuckDuckGo query`_

6+

7+.. _Instant Answer API: https://duckduckgo.com/api

8+.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1

9+

10+"""

11+

112import json

213from lxml import html

314from re import compile

@@ -25,7 +36,8 @@ def result_to_text(url, text, htmlResult):

2536def request(query, params):

2637 params['url'] = url.format(query=urlencode({'q': query}))

2738 language = match_language(params['language'], supported_languages, language_aliases)

28− params['headers']['Accept-Language'] = language.split('-')[0]

39+ language = language.split('-')[0]

40+ params['headers']['Accept-Language'] = language

2941 return params

3042

3143

@@ -43,8 +55,9 @@ def response(resp):

4355

4456 # add answer if there is one

4557 answer = search_res.get('Answer', '')

46− if answer != '':

47− results.append({'answer': html_to_text(answer)})

58+ if answer:

59+ if search_res.get('AnswerType', '') not in ['calc']:

60+ results.append({'answer': html_to_text(answer)})

4861

4962 # add infobox

5063 if 'Definition' in search_res:

searx/engines/duden.py+8−7

@@ -11,6 +11,7 @@

1111from lxml import html, etree

1212import re

1313from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

1415from searx.url_utils import quote, urljoin

1516from searx import logger

1617

@@ -52,9 +53,9 @@ def response(resp):

5253 dom = html.fromstring(resp.text)

5354

5455 try:

55− number_of_results_string = re.sub('[^0-9]', '', dom.xpath(

56− '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]

57− )

56+ number_of_results_string =\

57+ re.sub('[^0-9]', '',

58+ eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])

5859

5960 results.append({'number_of_results': int(number_of_results_string)})

6061

@@ -62,12 +63,12 @@ def response(resp):

6263 logger.debug("Couldn't read number of results.")

6364 pass

6465

65− for result in dom.xpath('//section[not(contains(@class, "essay"))]'):

66+ for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):

6667 try:

67− url = result.xpath('.//h2/a')[0].get('href')

68+ url = eval_xpath(result, './/h2/a')[0].get('href')

6869 url = urljoin(base_url, url)

69− title = result.xpath('string(.//h2/a)').strip()

70− content = extract_text(result.xpath('.//p'))

70+ title = eval_xpath(result, 'string(.//h2/a)').strip()

71+ content = extract_text(eval_xpath(result, './/p'))

7172 # append result

7273 results.append({'url': url,

7374 'title': title,

searx/engines/gigablast.py+3−2

@@ -15,6 +15,7 @@ from json import loads

1515from time import time

1616from lxml.html import fromstring

1717from searx.url_utils import urlencode

18+from searx.utils import eval_xpath

1819

1920# engine dependent config

2021categories = ['general']

@@ -99,9 +100,9 @@ def response(resp):

99100def _fetch_supported_languages(resp):

100101 supported_languages = []

101102 dom = fromstring(resp.text)

102− links = dom.xpath('//span[@id="menu2"]/a')

103+ links = eval_xpath(dom, '//span[@id="menu2"]/a')

103104 for link in links:

104− href = link.xpath('./@href')[0].split('lang%3A')

105+ href = eval_xpath(link, './@href')[0].split('lang%3A')

105106 if len(href) == 2:

106107 code = href[1].split('_')

107108 if len(code) == 2:

searx/engines/google.py+16−16

@@ -14,7 +14,7 @@ from lxml import html, etree

1414from searx.engines.xpath import extract_text, extract_url

1515from searx import logger

1616from searx.url_utils import urlencode, urlparse, parse_qsl

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

1818

1919logger = logger.getChild('google engine')

2020

@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):

156156

157157# returns extract_text on the first result selected by the xpath or None

158158def extract_text_from_dom(result, xpath):

159− r = result.xpath(xpath)

159+ r = eval_xpath(result, xpath)

160160 if len(r) > 0:

161161 return extract_text(r[0])

162162 return None

@@ -227,21 +227,21 @@ def response(resp):

227227 # convert the text to dom

228228 dom = html.fromstring(resp.text)

229229

230− instant_answer = dom.xpath('//div[@id="_vBb"]//text()')

230+ instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')

231231 if instant_answer:

232232 results.append({'answer': u' '.join(instant_answer)})

233233 try:

234− results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]

234+ results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]

235235 .split()[1].replace(',', ''))

236236 results.append({'number_of_results': results_num})

237237 except:

238238 pass

239239

240240 # parse results

241− for result in dom.xpath(results_xpath):

241+ for result in eval_xpath(dom, results_xpath):

242242 try:

243− title = extract_text(result.xpath(title_xpath)[0])

244− url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)

243+ title = extract_text(eval_xpath(result, title_xpath)[0])

244+ url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)

245245 parsed_url = urlparse(url, google_hostname)

246246

247247 # map result

@@ -250,7 +250,7 @@ def response(resp):

250250 continue

251251 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):

252252 # print "yooooo"*30

253− # x = result.xpath(map_near)

253+ # x = eval_xpath(result, map_near)

254254 # if len(x) > 0:

255255 # # map : near the location

256256 # results = results + parse_map_near(parsed_url, x, google_hostname)

@@ -287,11 +287,11 @@ def response(resp):

287287 continue

288288

289289 # parse suggestion

290− for suggestion in dom.xpath(suggestion_xpath):

290+ for suggestion in eval_xpath(dom, suggestion_xpath):

291291 # append suggestion

292292 results.append({'suggestion': extract_text(suggestion)})

293293

294− for correction in dom.xpath(spelling_suggestion_xpath):

294+ for correction in eval_xpath(dom, spelling_suggestion_xpath):

295295 results.append({'correction': extract_text(correction)})

296296

297297 # return results

@@ -300,9 +300,9 @@ def response(resp):

300300

301301def parse_images(result, google_hostname):

302302 results = []

303− for image in result.xpath(images_xpath):

304− url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)

305− img_src = extract_text(image.xpath(image_img_src_xpath)[0])

303+ for image in eval_xpath(result, images_xpath):

304+ url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)

305+ img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])

306306

307307 # append result

308308 results.append({'url': url,

@@ -389,10 +389,10 @@ def attributes_to_html(attributes):

389389def _fetch_supported_languages(resp):

390390 supported_languages = {}

391391 dom = html.fromstring(resp.text)

392− options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')

392+ options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

393393 for option in options:

394− code = option.xpath('./@value')[0].split('_')[-1]

395− name = option.xpath('./@data-name')[0].title()

394+ code = eval_xpath(option, './@value')[0].split('_')[-1]

395+ name = eval_xpath(option, './@data-name')[0].title()

396396 supported_languages[code] = {"name": name}

397397

398398 return supported_languages

searx/engines/seedpeer.pynew file+78

@@ -0,0 +1,78 @@

1+# Seedpeer (Videos, Music, Files)

2+#

3+# @website https://seedpeer.me

4+# @provide-api no (nothing found)

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable yes (HTML can change)

9+# @parse url, title, content, seed, leech, magnetlink

10+

11+from lxml import html

12+from json import loads

13+from operator import itemgetter

14+from searx.url_utils import quote, urljoin

15+from searx.engines.xpath import extract_text

16+

17+

18+url = 'https://seedpeer.me/'

19+search_url = url + 'search/{search_term}?page={page_no}'

20+torrent_file_url = url + 'torrent/{torrent_hash}'

21+

22+# specific xpath variables

23+script_xpath = '//script[@type="text/javascript"][not(@src)]'

24+torrent_xpath = '(//table)[2]/tbody/tr'

25+link_xpath = '(./td)[1]/a/@href'

26+age_xpath = '(./td)[2]'

27+size_xpath = '(./td)[3]'

28+

29+

30+# do search-request

31+def request(query, params):

32+ params['url'] = search_url.format(search_term=quote(query),

33+ page_no=params['pageno'])

34+ return params

35+

36+

37+# get response from search-request

38+def response(resp):

39+ results = []

40+ dom = html.fromstring(resp.text)

41+ result_rows = dom.xpath(torrent_xpath)

42+

43+ try:

44+ script_element = dom.xpath(script_xpath)[0]

45+ json_string = script_element.text[script_element.text.find('{'):]

46+ torrents_json = loads(json_string)

47+ except:

48+ return []

49+

50+ # parse results

51+ for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']):

52+ title = torrent_json['name']

53+ seed = int(torrent_json['seeds'])

54+ leech = int(torrent_json['peers'])

55+ size = int(torrent_json['size'])

56+ torrent_hash = torrent_json['hash']

57+

58+ torrentfile = torrent_file_url.format(torrent_hash=torrent_hash)

59+ magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash)

60+

61+ age = extract_text(torrent_row.xpath(age_xpath))

62+ link = torrent_row.xpath(link_xpath)[0]

63+

64+ href = urljoin(url, link)

65+

66+ # append result

67+ results.append({'url': href,

68+ 'title': title,

69+ 'content': age,

70+ 'seed': seed,

71+ 'leech': leech,

72+ 'filesize': size,

73+ 'torrentfile': torrentfile,

74+ 'magnetlink': magnetlink,

75+ 'template': 'torrent.html'})

76+

77+ # return results sorted by seeder

78+ return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py+3−1

@@ -51,7 +51,9 @@ def get_client_id():

5151

5252 if response.ok:

5353 tree = html.fromstring(response.content)

54− script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")

54+ # script_tags has been moved from /assets/app/ to /assets/ path. I

55+ # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js

56+ script_tags = tree.xpath("//script[contains(@src, '/assets/')]")

5557 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

5658

5759 # extracts valid app_js urls from soundcloud.com content

searx/engines/startpage.py+5−4

@@ -16,6 +16,7 @@ from datetime import datetime, timedelta

1616import re

1717from searx.engines.xpath import extract_text

1818from searx.languages import language_codes

19+from searx.utils import eval_xpath

1920

2021# engine dependent config

2122categories = ['general']

@@ -70,8 +71,8 @@ def response(resp):

7071 dom = html.fromstring(resp.text)

7172

7273 # parse results

73− for result in dom.xpath(results_xpath):

74− links = result.xpath(link_xpath)

74+ for result in eval_xpath(dom, results_xpath):

75+ links = eval_xpath(result, link_xpath)

7576 if not links:

7677 continue

7778 link = links[0]

@@ -87,8 +88,8 @@ def response(resp):

8788

8889 title = extract_text(link)

8990

90− if result.xpath(content_xpath):

91− content = extract_text(result.xpath(content_xpath))

91+ if eval_xpath(result, content_xpath):

92+ content = extract_text(eval_xpath(result, content_xpath))

9293 else:

9394 content = ''

9495

searx/engines/wikidata.py+1−17

@@ -16,7 +16,7 @@ from searx.poolrequests import get

1616from searx.engines.xpath import extract_text

1717from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

1818from searx.url_utils import urlencode

19−from searx.utils import match_language

19+from searx.utils import match_language, eval_xpath

2020

2121from json import loads

2222from lxml.html import fromstring

@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator

5757calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

5858media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'

5959

60−# xpath_cache

61−xpath_cache = {}

62−

63−

64−def get_xpath(xpath_str):

65− result = xpath_cache.get(xpath_str, None)

66− if not result:

67− result = etree.XPath(xpath_str)

68− xpath_cache[xpath_str] = result

69− return result

70−

71−

72−def eval_xpath(element, xpath_str):

73− xpath = get_xpath(xpath_str)

74− return xpath(element)

75−

7660

7761def get_id_cache(result):

7862 id_cache = {}

searx/engines/xpath.py+10−10

@@ -1,6 +1,6 @@

11from lxml import html

22from lxml.etree import _ElementStringResult, _ElementUnicodeResult

3−from searx.utils import html_to_text

3+from searx.utils import html_to_text, eval_xpath

44from searx.url_utils import unquote, urlencode, urljoin, urlparse

55

66search_url = None

@@ -104,15 +104,15 @@ def response(resp):

104104 results = []

105105 dom = html.fromstring(resp.text)

106106 if results_xpath:

107− for result in dom.xpath(results_xpath):

108− url = extract_url(result.xpath(url_xpath), search_url)

109− title = extract_text(result.xpath(title_xpath))

110− content = extract_text(result.xpath(content_xpath))

107+ for result in eval_xpath(dom, results_xpath):

108+ url = extract_url(eval_xpath(result, url_xpath), search_url)

109+ title = extract_text(eval_xpath(result, title_xpath))

110+ content = extract_text(eval_xpath(result, content_xpath))

111111 tmp_result = {'url': url, 'title': title, 'content': content}

112112

113113 # add thumbnail if available

114114 if thumbnail_xpath:

115− thumbnail_xpath_result = result.xpath(thumbnail_xpath)

115+ thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)

116116 if len(thumbnail_xpath_result) > 0:

117117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

118118

@@ -120,14 +120,14 @@ def response(resp):

120120 else:

121121 for url, title, content in zip(

122122 (extract_url(x, search_url) for

123− x in dom.xpath(url_xpath)),

124− map(extract_text, dom.xpath(title_xpath)),

125− map(extract_text, dom.xpath(content_xpath))

123+ x in eval_xpath(dom, url_xpath)),

124+ map(extract_text, eval_xpath(dom, title_xpath)),

125+ map(extract_text, eval_xpath(dom, content_xpath))

126126 ):

127127 results.append({'url': url, 'title': title, 'content': content})

128128

129129 if not suggestion_xpath:

130130 return results

131− for suggestion in dom.xpath(suggestion_xpath):

131+ for suggestion in eval_xpath(dom, suggestion_xpath):

132132 results.append({'suggestion': extract_text(suggestion)})

133133 return results

searx/engines/yahoo.py+9−9

@@ -14,7 +14,7 @@

1414from lxml import html

1515from searx.engines.xpath import extract_text, extract_url

1616from searx.url_utils import unquote, urlencode

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

1818

1919# engine dependent config

2020categories = ['general']

@@ -109,21 +109,21 @@ def response(resp):

109109 dom = html.fromstring(resp.text)

110110

111111 try:

112− results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]

112+ results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]

113113 .split()[0].replace(',', ''))

114114 results.append({'number_of_results': results_num})

115115 except:

116116 pass

117117

118118 # parse results

119− for result in dom.xpath(results_xpath):

119+ for result in eval_xpath(dom, results_xpath):

120120 try:

121− url = parse_url(extract_url(result.xpath(url_xpath), search_url))

122− title = extract_text(result.xpath(title_xpath)[0])

121+ url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))

122+ title = extract_text(eval_xpath(result, title_xpath)[0])

123123 except:

124124 continue

125125

126− content = extract_text(result.xpath(content_xpath)[0])

126+ content = extract_text(eval_xpath(result, content_xpath)[0])

127127

128128 # append result

129129 results.append({'url': url,

@@ -131,7 +131,7 @@ def response(resp):

131131 'content': content})

132132

133133 # if no suggestion found, return results

134− suggestions = dom.xpath(suggestion_xpath)

134+ suggestions = eval_xpath(dom, suggestion_xpath)

135135 if not suggestions:

136136 return results

137137

@@ -148,9 +148,9 @@ def response(resp):

148148def _fetch_supported_languages(resp):

149149 supported_languages = []

150150 dom = html.fromstring(resp.text)

151− options = dom.xpath('//div[@id="yschlang"]/span/label/input')

151+ options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')

152152 for option in options:

153− code_parts = option.xpath('./@value')[0][5:].split('_')

153+ code_parts = eval_xpath(option, './@value')[0][5:].split('_')

154154 if len(code_parts) == 2:

155155 code = code_parts[0] + '-' + code_parts[1].upper()

156156 else:

searx/results.py+5−3

@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):

6767

6868 for url2 in infobox2.get('urls', []):

6969 unique_url = True

70− for url1 in infobox1.get('urls', []):

71− if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):

70+ parsed_url2 = urlparse(url2.get('url', ''))

71+ for url1 in urls1:

72+ if compare_urls(urlparse(url1.get('url', '')), parsed_url2):

7273 unique_url = False

7374 break

7475 if unique_url:

@@ -188,8 +189,9 @@ class ResultContainer(object):

188189 add_infobox = True

189190 infobox_id = infobox.get('id', None)

190191 if infobox_id is not None:

192+ parsed_url_infobox_id = urlparse(infobox_id)

191193 for existingIndex in self.infoboxes:

192− if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):

194+ if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):

193195 merge_two_infoboxes(existingIndex, infobox)

194196 add_infobox = False

195197

searx/settings.yml+5

@@ -748,6 +748,11 @@ engines:

748748 page_size : 10

749749 disabled : True

750750

751+ - name : seedpeer

752+ shortcut : speu

753+ engine : seedpeer

754+ categories: files, music, videos

755+

751756# - name : yacy

752757# engine : yacy

753758# shortcut : ya

searx/templates/courgette/result_templates/torrent.html+1−1

@@ -4,7 +4,7 @@

44 {% endif %}

55 <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>

66 {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %}

7− {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}

7+ {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}

88 <span>

99 {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %}

1010 {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %}

searx/templates/legacy/result_templates/torrent.html+1−1

@@ -8,6 +8,6 @@

88 <p>

99 {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %}

1010 {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} -

11− {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %}

11+ {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %}

1212 </p>

1313</div>

searx/templates/oscar/result_templates/torrent.html+1−1

@@ -3,7 +3,7 @@

33{{ result_header(result, favicons) }}

44{{ result_sub_header(result) }}

55

6−{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}

6+{% if result.seed is defined %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}

77{% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }}

88 <span class="badge">

99 {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/templates/simple/result_templates/torrent.html+1−1

@@ -6,7 +6,7 @@

66{% if result.magnetlink %}<p class="altlink"> • {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</p>{% endif %}

77{% if result.torrentfile %}<p class="altlink"> • {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</p>{% endif %}

88

9−{% if result.seed %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}

9+{% if result.seed is defined %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}

1010

1111{%- if result.filesize %}<p class="stat">{{ icon('floppy-disk') }} {{ _('Filesize') }}<span class="badge">

1212 {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/utils.py+15

@@ -13,6 +13,7 @@ from numbers import Number

1313from os.path import splitext, join

1414from io import open

1515from random import choice

16+from lxml.etree import XPath

1617import sys

1718import json

1819

@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

5152useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

5253 + "/data/useragents.json", 'r', encoding='utf-8').read())

5354

55+xpath_cache = dict()

5456lang_to_lc_cache = dict()

5557

5658

@@ -450,3 +452,16 @@ def get_engine_from_settings(name):

450452 return engine

451453

452454 return {}

455+

456+

457+def get_xpath(xpath_str):

458+ result = xpath_cache.get(xpath_str, None)

459+ if result is None:

460+ result = XPath(xpath_str)

461+ xpath_cache[xpath_str] = result

462+ return result

463+

464+

465+def eval_xpath(element, xpath_str):

466+ xpath = get_xpath(xpath_str)

467+ return xpath(element)

searx/webapp.py+7−9

@@ -157,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None

157157

158158@babel.localeselector

159159def get_locale():

160− locale = request.accept_languages.best_match(settings['locales'].keys())

161−

162− if request.preferences.get_value('locale') != '':

163− locale = request.preferences.get_value('locale')

160+ if 'locale' in request.form\

161+ and request.form['locale'] in settings['locales']:

162+ return request.form['locale']

164163

165164 if 'locale' in request.args\

166165 and request.args['locale'] in settings['locales']:

167− locale = request.args['locale']

166+ return request.args['locale']

168167

169− if 'locale' in request.form\

170− and request.form['locale'] in settings['locales']:

171− locale = request.form['locale']

168+ if request.preferences.get_value('locale') != '':

169+ return request.preferences.get_value('locale')

172170

173− return locale

171+ return request.accept_languages.best_match(settings['locales'].keys())

174172

175173

176174# code-highlighter

tests/unit/engines/test_seedpeer.pynew file+66

@@ -0,0 +1,66 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import seedpeer

5+from searx.testing import SearxTestCase

6+

7+

8+class TestBtdiggEngine(SearxTestCase):

9+

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ params = seedpeer.request(query, dicto)

15+ self.assertIn('url', params)

16+ self.assertIn(query, params['url'])

17+ self.assertIn('seedpeer', params['url'])

18+

19+ def test_response(self):

20+ self.assertRaises(AttributeError, seedpeer.response, None)

21+ self.assertRaises(AttributeError, seedpeer.response, [])

22+ self.assertRaises(AttributeError, seedpeer.response, '')

23+ self.assertRaises(AttributeError, seedpeer.response, '[]')

24+

25+ response = mock.Mock(text='<html></html>')

26+ self.assertEqual(seedpeer.response(response), [])

27+

28+ html = u"""

29+ <html>

30+ <head>

31+ <script></script>

32+ <script type="text/javascript" src="not_here.js"></script>

33+ <script type="text/javascript">

34+ window.initialData=

35+ {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}}

36+ </script>

37+ </head>

38+ <body>

39+ <table></table>

40+ <table>

41+ <thead><tr></tr></thead>

42+ <tbody>

43+ <tr>

44+ <td><a href="link">Title</a></td>

45+ <td>1 year</td>

46+ <td>1 KB</td>

47+ <td>10</td>

48+ <td>20</td>

49+ <td></td>

50+ </tr>

51+ </tbody>

52+ </table>

53+ </body>

54+ </html>

55+ """

56+ response = mock.Mock(text=html)

57+ results = seedpeer.response(response)

58+ self.assertEqual(type(results), list)

59+ self.assertEqual(len(results), 1)

60+ self.assertEqual(results[0]['title'], 'Title')

61+ self.assertEqual(results[0]['url'], 'https://seedpeer.me/link')

62+ self.assertEqual(results[0]['seed'], 10)

63+ self.assertEqual(results[0]['leech'], 20)

64+ self.assertEqual(results[0]['filesize'], 1024)

65+ self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123')

66+ self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')