1b90e14 · searx · git.iko.soy

Dockerfile+15−15

compare with master

unifiedsplit

@@ -1,10 +1,17 @@

1FROM alpine:3.10

2+ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

3+EXPOSE 8080

4+VOLUME /etc/searx

5+VOLUME /var/log/uwsgi

2

6

3−ARG VERSION_GITCOMMIT=unknow

7+ARG VERSION_GITCOMMIT=unknown

4−ARG SEARX_GIT_VERSION=unknow

8+ARG SEARX_GIT_VERSION=unknown

5

9

6−ARG SEARX_GID=1000

10+ARG SEARX_GID=977

7−ARG SEARX_UID=1000

11+ARG SEARX_UID=977

12+

13+RUN addgroup -g ${SEARX_GID} searx && \

14+ adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx

8

15

9ARG TIMESTAMP_SETTINGS=0

16ARG TIMESTAMP_SETTINGS=0

10ARG TIMESTAMP_UWSGI=0

17ARG TIMESTAMP_UWSGI=0

@@ -16,19 +23,14 @@ ENV INSTANCE_NAME=searx \

16 BASE_URL= \

23 BASE_URL= \

17 MORTY_KEY= \

24 MORTY_KEY= \

18 MORTY_URL=

25 MORTY_URL=

19−EXPOSE 8080

20−VOLUME /etc/searx

21−VOLUME /var/log/uwsgi

22

26

23WORKDIR /usr/local/searx

27WORKDIR /usr/local/searx

24

28

25−RUN addgroup -g ${SEARX_GID} searx && \

26− adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx

27

29

28COPY requirements.txt ./requirements.txt

30COPY requirements.txt ./requirements.txt

29

31

30−RUN apk -U upgrade \

32+RUN apk upgrade --no-cache \

31− && apk add -t build-dependencies \

33+ && apk add --no-cache -t build-dependencies \

32 build-base \

34 build-base \

33 py3-setuptools \

35 py3-setuptools \

34 python3-dev \

36 python3-dev \

@@ -38,7 +40,7 @@ RUN apk -U upgrade \

38 openssl-dev \

40 openssl-dev \

39 tar \

41 tar \

40 git \

42 git \

41− && apk add \

43+ && apk add --no-cache \

42 ca-certificates \

44 ca-certificates \

43 su-exec \

45 su-exec \

44 python3 \

46 python3 \

@@ -50,8 +52,7 @@ RUN apk -U upgrade \

50 uwsgi-python3 \

52 uwsgi-python3 \

51 && pip3 install --upgrade pip \

53 && pip3 install --upgrade pip \

52 && pip3 install --no-cache -r requirements.txt \

54 && pip3 install --no-cache -r requirements.txt \

53− && apk del build-dependencies \

55+ && apk del build-dependencies

54− && rm -f /var/cache/apk/*

55

56

56COPY --chown=searx:searx . .

57COPY --chown=searx:searx . .

57

58

@@ -62,7 +63,6 @@ RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \

62 echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \

63 echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \

63 fi

64 fi

64

65

65−ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"]

66

67# Keep this argument at the end since it change each time

68ARG LABEL_DATE=

searx/engines/bing.py+10−10

@@ -18,7 +18,7 @@ from lxml import html

18from searx import logger, utils

19from searx.engines.xpath import extract_text

20from searx.url_utils import urlencode

21−from searx.utils import match_language, gen_useragent

21+from searx.utils import match_language, gen_useragent, eval_xpath

22

23logger = logger.getChild('bing engine')

24

@@ -65,11 +65,11 @@ def response(resp):

65

66 dom = html.fromstring(resp.text)

67 # parse results

68− for result in dom.xpath('//div[@class="sa_cc"]'):

68+ for result in eval_xpath(dom, '//div[@class="sa_cc"]'):

69− link = result.xpath('.//h3/a')[0]

69+ link = eval_xpath(result, './/h3/a')[0]

70 url = link.attrib.get('href')

71 title = extract_text(link)

72− content = extract_text(result.xpath('.//p'))

72+ content = extract_text(eval_xpath(result, './/p'))

73

74 # append result

75 results.append({'url': url,

@@ -77,11 +77,11 @@ def response(resp):

77 'content': content})

78

79 # parse results again if nothing is found yet

80− for result in dom.xpath('//li[@class="b_algo"]'):

80+ for result in eval_xpath(dom, '//li[@class="b_algo"]'):

81− link = result.xpath('.//h2/a')[0]

81+ link = eval_xpath(result, './/h2/a')[0]

82 url = link.attrib.get('href')

83 title = extract_text(link)

84− content = extract_text(result.xpath('.//p'))

84+ content = extract_text(eval_xpath(result, './/p'))

85

86 # append result

87 results.append({'url': url,

@@ -89,7 +89,7 @@ def response(resp):

89 'content': content})

90

91 try:

92− result_len_container = "".join(dom.xpath('//span[@class="sb_count"]/text()'))

92+ result_len_container = "".join(eval_xpath(dom, '//span[@class="sb_count"]/text()'))

93 result_len_container = utils.to_string(result_len_container)

94 if "-" in result_len_container:

95 # Remove the part "from-to" for paginated request ...

@@ -113,9 +113,9 @@ def response(resp):

113def _fetch_supported_languages(resp):

114 supported_languages = []

115 dom = html.fromstring(resp.text)

116− options = dom.xpath('//div[@id="limit-languages"]//input')

116+ options = eval_xpath(dom, '//div[@id="limit-languages"]//input')

117 for option in options:

118− code = option.xpath('./@id')[0].replace('_', '-')

118+ code = eval_xpath(option, './@id')[0].replace('_', '-')

119 if code == 'nb':

120 code = 'no'

121 supported_languages.append(code)

searx/engines/dictzone.py+4−4

@@ -11,7 +11,7 @@

11

12import re

13from lxml import html

14−from searx.utils import is_valid_lang

14+from searx.utils import is_valid_lang, eval_xpath

15from searx.url_utils import urljoin

16

17categories = ['general']

@@ -47,14 +47,14 @@ def response(resp):

47

48 dom = html.fromstring(resp.text)

49

50− for k, result in enumerate(dom.xpath(results_xpath)[1:]):

50+ for k, result in enumerate(eval_xpath(dom, results_xpath)[1:]):

51 try:

52− from_result, to_results_raw = result.xpath('./td')

52+ from_result, to_results_raw = eval_xpath(result, './td')

53 except:

54 continue

55

56 to_results = []

57− for to_result in to_results_raw.xpath('./p/a'):

57+ for to_result in eval_xpath(to_results_raw, './p/a'):

58 t = to_result.text_content()

59 if t.strip():

60 to_results.append(to_result.text_content())

searx/engines/doku.py+8−7

@@ -11,6 +11,7 @@

11

12from lxml.html import fromstring

13from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

14from searx.url_utils import urlencode

15from searx.url_utils import urlencode

15

16

16# engine dependent config

17# engine dependent config

@@ -45,16 +46,16 @@ def response(resp):

45

46

46 # parse results

47 # parse results

47 # Quickhits

48 # Quickhits

48− for r in doc.xpath('//div[@class="search_quickresult"]/ul/li'):

49+ for r in eval_xpath(doc, '//div[@class="search_quickresult"]/ul/li'):

49 try:

50 try:

50− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

51+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

51 except:

52 except:

52 continue

53 continue

53

54

54 if not res_url:

55 if not res_url:

55 continue

56 continue

56

57

57− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

58+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

58

59

59 # append result

60 # append result

60 results.append({'title': title,

61 results.append({'title': title,

@@ -62,13 +63,13 @@ def response(resp):

62 'url': base_url + res_url})

63 'url': base_url + res_url})

63

64

64 # Search results

65 # Search results

65− for r in doc.xpath('//dl[@class="search_results"]/*'):

66+ for r in eval_xpath(doc, '//dl[@class="search_results"]/*'):

66 try:

67 try:

67 if r.tag == "dt":

68 if r.tag == "dt":

68− res_url = r.xpath('.//a[@class="wikilink1"]/@href')[-1]

69+ res_url = eval_xpath(r, './/a[@class="wikilink1"]/@href')[-1]

69− title = extract_text(r.xpath('.//a[@class="wikilink1"]/@title'))

70+ title = extract_text(eval_xpath(r, './/a[@class="wikilink1"]/@title'))

70 elif r.tag == "dd":

71 elif r.tag == "dd":

71− content = extract_text(r.xpath('.'))

72+ content = extract_text(eval_xpath(r, '.'))

72

73

73 # append result

74 # append result

74 results.append({'title': title,

75 results.append({'title': title,

searx/engines/duckduckgo.py+5−5

@@ -18,7 +18,7 @@ from json import loads

18from searx.engines.xpath import extract_text

19from searx.poolrequests import get

20from searx.url_utils import urlencode

21−from searx.utils import match_language

21+from searx.utils import match_language, eval_xpath

22

23# engine dependent config

24categories = ['general']

@@ -106,19 +106,19 @@ def response(resp):

106 doc = fromstring(resp.text)

107

108 # parse results

109− for i, r in enumerate(doc.xpath(result_xpath)):

109+ for i, r in enumerate(eval_xpath(doc, result_xpath)):

110 if i >= 30:

111 break

112 try:

113− res_url = r.xpath(url_xpath)[-1]

113+ res_url = eval_xpath(r, url_xpath)[-1]

114 except:

115 continue

116

117 if not res_url:

118 continue

119

120− title = extract_text(r.xpath(title_xpath))

120+ title = extract_text(eval_xpath(r, title_xpath))

121− content = extract_text(r.xpath(content_xpath))

121+ content = extract_text(eval_xpath(r, content_xpath))

122

123 # append result

124 results.append({'title': title,

searx/engines/duckduckgo_definitions.py+16−3

@@ -1,3 +1,14 @@

1+"""

2+DuckDuckGo (definitions)

3+

4+- `Instant Answer API`_

5+- `DuckDuckGo query`_

6+

7+.. _Instant Answer API: https://duckduckgo.com/api

8+.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1

9+

10+"""

11+

1import json

12import json

2from lxml import html

13from lxml import html

3from re import compile

14from re import compile

@@ -25,7 +36,8 @@ def result_to_text(url, text, htmlResult):

25def request(query, params):

36def request(query, params):

26 params['url'] = url.format(query=urlencode({'q': query}))

37 params['url'] = url.format(query=urlencode({'q': query}))

27 language = match_language(params['language'], supported_languages, language_aliases)

38 language = match_language(params['language'], supported_languages, language_aliases)

28− params['headers']['Accept-Language'] = language.split('-')[0]

39+ language = language.split('-')[0]

40+ params['headers']['Accept-Language'] = language

29 return params

41 return params

30

42

31

43

@@ -43,8 +55,9 @@ def response(resp):

43

55

44 # add answer if there is one

56 # add answer if there is one

45 answer = search_res.get('Answer', '')

57 answer = search_res.get('Answer', '')

46− if answer != '':

58+ if answer:

59+ if search_res.get('AnswerType', '') not in ['calc']:

47− results.append({'answer': html_to_text(answer)})

60+ results.append({'answer': html_to_text(answer)})

48

61

49 # add infobox

62 # add infobox

50 if 'Definition' in search_res:

63 if 'Definition' in search_res:

searx/engines/duden.py+8−7

@@ -11,6 +11,7 @@

11from lxml import html, etree

12import re

13from searx.engines.xpath import extract_text

14+from searx.utils import eval_xpath

14from searx.url_utils import quote, urljoin

15from searx.url_utils import quote, urljoin

15from searx import logger

16from searx import logger

16

17

@@ -52,9 +53,9 @@ def response(resp):

52 dom = html.fromstring(resp.text)

53 dom = html.fromstring(resp.text)

53

54

54 try:

55 try:

55− number_of_results_string = re.sub('[^0-9]', '', dom.xpath(

56+ number_of_results_string =\

57+ re.sub('[^0-9]', '',

56− '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0]

58+ eval_xpath(dom, '//a[@class="active" and contains(@href,"/suchen/dudenonline")]/span/text()')[0])

57− )

58

59

59 results.append({'number_of_results': int(number_of_results_string)})

60 results.append({'number_of_results': int(number_of_results_string)})

60

61

@@ -62,12 +63,12 @@ def response(resp):

62 logger.debug("Couldn't read number of results.")

63 logger.debug("Couldn't read number of results.")

63 pass

64 pass

64

65

65− for result in dom.xpath('//section[not(contains(@class, "essay"))]'):

66+ for result in eval_xpath(dom, '//section[not(contains(@class, "essay"))]'):

66 try:

67 try:

67− url = result.xpath('.//h2/a')[0].get('href')

68+ url = eval_xpath(result, './/h2/a')[0].get('href')

68 url = urljoin(base_url, url)

69 url = urljoin(base_url, url)

69− title = result.xpath('string(.//h2/a)').strip()

70+ title = eval_xpath(result, 'string(.//h2/a)').strip()

70− content = extract_text(result.xpath('.//p'))

71+ content = extract_text(eval_xpath(result, './/p'))

71 # append result

72 # append result

72 results.append({'url': url,

73 results.append({'url': url,

73 'title': title,

74 'title': title,

searx/engines/gigablast.py+3−2

@@ -15,6 +15,7 @@ from json import loads

15from time import time

16from lxml.html import fromstring

17from searx.url_utils import urlencode

18+from searx.utils import eval_xpath

18

19

19# engine dependent config

20# engine dependent config

20categories = ['general']

21categories = ['general']

@@ -99,9 +100,9 @@ def response(resp):

99def _fetch_supported_languages(resp):

100def _fetch_supported_languages(resp):

100 supported_languages = []

101 supported_languages = []

101 dom = fromstring(resp.text)

102 dom = fromstring(resp.text)

102− links = dom.xpath('//span[@id="menu2"]/a')

103+ links = eval_xpath(dom, '//span[@id="menu2"]/a')

103 for link in links:

104 for link in links:

104− href = link.xpath('./@href')[0].split('lang%3A')

105+ href = eval_xpath(link, './@href')[0].split('lang%3A')

105 if len(href) == 2:

106 if len(href) == 2:

106 code = href[1].split('_')

107 code = href[1].split('_')

107 if len(code) == 2:

108 if len(code) == 2:

searx/engines/google.py+16−16

@@ -14,7 +14,7 @@ from lxml import html, etree

14from searx.engines.xpath import extract_text, extract_url

15from searx import logger

16from searx.url_utils import urlencode, urlparse, parse_qsl

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

18

19logger = logger.getChild('google engine')

20

@@ -156,7 +156,7 @@ def parse_url(url_string, google_hostname):

156

157# returns extract_text on the first result selected by the xpath or None

158def extract_text_from_dom(result, xpath):

159− r = result.xpath(xpath)

159+ r = eval_xpath(result, xpath)

160 if len(r) > 0:

161 return extract_text(r[0])

162 return None

@@ -227,21 +227,21 @@ def response(resp):

227 # convert the text to dom

228 dom = html.fromstring(resp.text)

229

230− instant_answer = dom.xpath('//div[@id="_vBb"]//text()')

230+ instant_answer = eval_xpath(dom, '//div[@id="_vBb"]//text()')

231 if instant_answer:

232 results.append({'answer': u' '.join(instant_answer)})

233 try:

234− results_num = int(dom.xpath('//div[@id="resultStats"]//text()')[0]

234+ results_num = int(eval_xpath(dom, '//div[@id="resultStats"]//text()')[0]

235 .split()[1].replace(',', ''))

236 results.append({'number_of_results': results_num})

237 except:

238 pass

239

240 # parse results

241− for result in dom.xpath(results_xpath):

241+ for result in eval_xpath(dom, results_xpath):

242 try:

243− title = extract_text(result.xpath(title_xpath)[0])

243+ title = extract_text(eval_xpath(result, title_xpath)[0])

244− url = parse_url(extract_url(result.xpath(url_xpath), google_url), google_hostname)

244+ url = parse_url(extract_url(eval_xpath(result, url_xpath), google_url), google_hostname)

245 parsed_url = urlparse(url, google_hostname)

246

247 # map result

@@ -250,7 +250,7 @@ def response(resp):

250 continue

251 # if parsed_url.path.startswith(maps_path) or parsed_url.netloc.startswith(map_hostname_start):

252 # print "yooooo"*30

253− # x = result.xpath(map_near)

253+ # x = eval_xpath(result, map_near)

254 # if len(x) > 0:

255 # # map : near the location

256 # results = results + parse_map_near(parsed_url, x, google_hostname)

@@ -287,11 +287,11 @@ def response(resp):

287 continue

288

289 # parse suggestion

290− for suggestion in dom.xpath(suggestion_xpath):

290+ for suggestion in eval_xpath(dom, suggestion_xpath):

291 # append suggestion

292 results.append({'suggestion': extract_text(suggestion)})

293

294− for correction in dom.xpath(spelling_suggestion_xpath):

294+ for correction in eval_xpath(dom, spelling_suggestion_xpath):

295 results.append({'correction': extract_text(correction)})

296

297 # return results

@@ -300,9 +300,9 @@ def response(resp):

300

301def parse_images(result, google_hostname):

302 results = []

303− for image in result.xpath(images_xpath):

303+ for image in eval_xpath(result, images_xpath):

304− url = parse_url(extract_text(image.xpath(image_url_xpath)[0]), google_hostname)

304+ url = parse_url(extract_text(eval_xpath(image, image_url_xpath)[0]), google_hostname)

305− img_src = extract_text(image.xpath(image_img_src_xpath)[0])

305+ img_src = extract_text(eval_xpath(image, image_img_src_xpath)[0])

306

307 # append result

308 results.append({'url': url,

@@ -389,10 +389,10 @@ def attributes_to_html(attributes):

389def _fetch_supported_languages(resp):

390 supported_languages = {}

391 dom = html.fromstring(resp.text)

392− options = dom.xpath('//*[@id="langSec"]//input[@name="lr"]')

392+ options = eval_xpath(dom, '//*[@id="langSec"]//input[@name="lr"]')

393 for option in options:

394− code = option.xpath('./@value')[0].split('_')[-1]

394+ code = eval_xpath(option, './@value')[0].split('_')[-1]

395− name = option.xpath('./@data-name')[0].title()

395+ name = eval_xpath(option, './@data-name')[0].title()

396 supported_languages[code] = {"name": name}

397

398 return supported_languages

searx/engines/seedpeer.pynew file+78

@@ -0,0 +1,78 @@

1+# Seedpeer (Videos, Music, Files)

2+#

3+# @website https://seedpeer.me

4+# @provide-api no (nothing found)

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable yes (HTML can change)

9+# @parse url, title, content, seed, leech, magnetlink

10+

11+from lxml import html

12+from json import loads

13+from operator import itemgetter

14+from searx.url_utils import quote, urljoin

15+from searx.engines.xpath import extract_text

16+

17+

18+url = 'https://seedpeer.me/'

19+search_url = url + 'search/{search_term}?page={page_no}'

20+torrent_file_url = url + 'torrent/{torrent_hash}'

21+

22+# specific xpath variables

23+script_xpath = '//script[@type="text/javascript"][not(@src)]'

24+torrent_xpath = '(//table)[2]/tbody/tr'

25+link_xpath = '(./td)[1]/a/@href'

26+age_xpath = '(./td)[2]'

27+size_xpath = '(./td)[3]'

28+

29+

30+# do search-request

31+def request(query, params):

32+ params['url'] = search_url.format(search_term=quote(query),

33+ page_no=params['pageno'])

34+ return params

35+

36+

37+# get response from search-request

38+def response(resp):

39+ results = []

40+ dom = html.fromstring(resp.text)

41+ result_rows = dom.xpath(torrent_xpath)

42+

43+ try:

44+ script_element = dom.xpath(script_xpath)[0]

45+ json_string = script_element.text[script_element.text.find('{'):]

46+ torrents_json = loads(json_string)

47+ except:

48+ return []

49+

50+ # parse results

51+ for torrent_row, torrent_json in zip(result_rows, torrents_json['data']['list']):

52+ title = torrent_json['name']

53+ seed = int(torrent_json['seeds'])

54+ leech = int(torrent_json['peers'])

55+ size = int(torrent_json['size'])

56+ torrent_hash = torrent_json['hash']

57+

58+ torrentfile = torrent_file_url.format(torrent_hash=torrent_hash)

59+ magnetlink = 'magnet:?xt=urn:btih:{}'.format(torrent_hash)

60+

61+ age = extract_text(torrent_row.xpath(age_xpath))

62+ link = torrent_row.xpath(link_xpath)[0]

63+

64+ href = urljoin(url, link)

65+

66+ # append result

67+ results.append({'url': href,

68+ 'title': title,

69+ 'content': age,

70+ 'seed': seed,

71+ 'leech': leech,

72+ 'filesize': size,

73+ 'torrentfile': torrentfile,

74+ 'magnetlink': magnetlink,

75+ 'template': 'torrent.html'})

76+

77+ # return results sorted by seeder

78+ return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py+3−1

@@ -51,7 +51,9 @@ def get_client_id():

51

52 if response.ok:

53 tree = html.fromstring(response.content)

54+ # script_tags has been moved from /assets/app/ to /assets/ path. I

55+ # found client_id in https://a-v2.sndcdn.com/assets/49-a0c01933-3.js

54− script_tags = tree.xpath("//script[contains(@src, '/assets/app')]")

56+ script_tags = tree.xpath("//script[contains(@src, '/assets/')]")

55 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

57 app_js_urls = [script_tag.get('src') for script_tag in script_tags if script_tag is not None]

56

58

57 # extracts valid app_js urls from soundcloud.com content

59 # extracts valid app_js urls from soundcloud.com content

searx/engines/startpage.py+5−4

@@ -16,6 +16,7 @@ from datetime import datetime, timedelta

16import re

17from searx.engines.xpath import extract_text

18from searx.languages import language_codes

19+from searx.utils import eval_xpath

19

20

20# engine dependent config

21# engine dependent config

21categories = ['general']

22categories = ['general']

@@ -70,8 +71,8 @@ def response(resp):

70 dom = html.fromstring(resp.text)

71 dom = html.fromstring(resp.text)

71

72

72 # parse results

73 # parse results

73− for result in dom.xpath(results_xpath):

74+ for result in eval_xpath(dom, results_xpath):

74− links = result.xpath(link_xpath)

75+ links = eval_xpath(result, link_xpath)

75 if not links:

76 if not links:

76 continue

77 continue

77 link = links[0]

78 link = links[0]

@@ -87,8 +88,8 @@ def response(resp):

87

88

88 title = extract_text(link)

89 title = extract_text(link)

89

90

90− if result.xpath(content_xpath):

91+ if eval_xpath(result, content_xpath):

91− content = extract_text(result.xpath(content_xpath))

92+ content = extract_text(eval_xpath(result, content_xpath))

92 else:

93 else:

93 content = ''

94 content = ''

94

95

searx/engines/wikidata.py+1−17

@@ -16,7 +16,7 @@ from searx.poolrequests import get

16from searx.engines.xpath import extract_text

17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

18from searx.url_utils import urlencode

19−from searx.utils import match_language

19+from searx.utils import match_language, eval_xpath

20

21from json import loads

22from lxml.html import fromstring

@@ -57,22 +57,6 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator

57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'

59

60−# xpath_cache

61−xpath_cache = {}

62−

63−

64−def get_xpath(xpath_str):

65− result = xpath_cache.get(xpath_str, None)

66− if not result:

67− result = etree.XPath(xpath_str)

68− xpath_cache[xpath_str] = result

69− return result

70−

71−

72−def eval_xpath(element, xpath_str):

73− xpath = get_xpath(xpath_str)

74− return xpath(element)

75−

76

60

77def get_id_cache(result):

61def get_id_cache(result):

78 id_cache = {}

62 id_cache = {}

searx/engines/xpath.py+10−10

@@ -1,6 +1,6 @@

1from lxml import html

2from lxml.etree import _ElementStringResult, _ElementUnicodeResult

3−from searx.utils import html_to_text

3+from searx.utils import html_to_text, eval_xpath

4from searx.url_utils import unquote, urlencode, urljoin, urlparse

5

6search_url = None

@@ -104,15 +104,15 @@ def response(resp):

104 results = []

105 dom = html.fromstring(resp.text)

106 if results_xpath:

107− for result in dom.xpath(results_xpath):

107+ for result in eval_xpath(dom, results_xpath):

108− url = extract_url(result.xpath(url_xpath), search_url)

108+ url = extract_url(eval_xpath(result, url_xpath), search_url)

109− title = extract_text(result.xpath(title_xpath))

109+ title = extract_text(eval_xpath(result, title_xpath))

110− content = extract_text(result.xpath(content_xpath))

110+ content = extract_text(eval_xpath(result, content_xpath))

111 tmp_result = {'url': url, 'title': title, 'content': content}

112

113 # add thumbnail if available

114 if thumbnail_xpath:

115− thumbnail_xpath_result = result.xpath(thumbnail_xpath)

115+ thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)

116 if len(thumbnail_xpath_result) > 0:

117 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

118

@@ -120,14 +120,14 @@ def response(resp):

120 else:

121 for url, title, content in zip(

122 (extract_url(x, search_url) for

123− x in dom.xpath(url_xpath)),

123+ x in eval_xpath(dom, url_xpath)),

124− map(extract_text, dom.xpath(title_xpath)),

124+ map(extract_text, eval_xpath(dom, title_xpath)),

125− map(extract_text, dom.xpath(content_xpath))

125+ map(extract_text, eval_xpath(dom, content_xpath))

126 ):

127 results.append({'url': url, 'title': title, 'content': content})

128

129 if not suggestion_xpath:

130 return results

131− for suggestion in dom.xpath(suggestion_xpath):

131+ for suggestion in eval_xpath(dom, suggestion_xpath):

132 results.append({'suggestion': extract_text(suggestion)})

133 return results

searx/engines/yahoo.py+9−9

@@ -14,7 +14,7 @@

14from lxml import html

15from searx.engines.xpath import extract_text, extract_url

16from searx.url_utils import unquote, urlencode

17−from searx.utils import match_language

17+from searx.utils import match_language, eval_xpath

18

19# engine dependent config

20categories = ['general']

@@ -109,21 +109,21 @@ def response(resp):

109 dom = html.fromstring(resp.text)

110

111 try:

112− results_num = int(dom.xpath('//div[@class="compPagination"]/span[last()]/text()')[0]

112+ results_num = int(eval_xpath(dom, '//div[@class="compPagination"]/span[last()]/text()')[0]

113 .split()[0].replace(',', ''))

114 results.append({'number_of_results': results_num})

115 except:

116 pass

117

118 # parse results

119− for result in dom.xpath(results_xpath):

119+ for result in eval_xpath(dom, results_xpath):

120 try:

121− url = parse_url(extract_url(result.xpath(url_xpath), search_url))

121+ url = parse_url(extract_url(eval_xpath(result, url_xpath), search_url))

122− title = extract_text(result.xpath(title_xpath)[0])

122+ title = extract_text(eval_xpath(result, title_xpath)[0])

123 except:

124 continue

125

126− content = extract_text(result.xpath(content_xpath)[0])

126+ content = extract_text(eval_xpath(result, content_xpath)[0])

127

128 # append result

129 results.append({'url': url,

@@ -131,7 +131,7 @@ def response(resp):

131 'content': content})

132

133 # if no suggestion found, return results

134− suggestions = dom.xpath(suggestion_xpath)

134+ suggestions = eval_xpath(dom, suggestion_xpath)

135 if not suggestions:

136 return results

137

@@ -148,9 +148,9 @@ def response(resp):

148def _fetch_supported_languages(resp):

149 supported_languages = []

150 dom = html.fromstring(resp.text)

151− options = dom.xpath('//div[@id="yschlang"]/span/label/input')

151+ options = eval_xpath(dom, '//div[@id="yschlang"]/span/label/input')

152 for option in options:

153− code_parts = option.xpath('./@value')[0][5:].split('_')

153+ code_parts = eval_xpath(option, './@value')[0][5:].split('_')

154 if len(code_parts) == 2:

155 code = code_parts[0] + '-' + code_parts[1].upper()

156 else:

searx/results.py+5−3

@@ -67,8 +67,9 @@ def merge_two_infoboxes(infobox1, infobox2):

67

68 for url2 in infobox2.get('urls', []):

69 unique_url = True

70+ parsed_url2 = urlparse(url2.get('url', ''))

70− for url1 in infobox1.get('urls', []):

71+ for url1 in urls1:

71− if compare_urls(urlparse(url1.get('url', '')), urlparse(url2.get('url', ''))):

72+ if compare_urls(urlparse(url1.get('url', '')), parsed_url2):

72 unique_url = False

73 unique_url = False

73 break

74 break

74 if unique_url:

75 if unique_url:

@@ -188,8 +189,9 @@ class ResultContainer(object):

188 add_infobox = True

189 add_infobox = True

189 infobox_id = infobox.get('id', None)

190 infobox_id = infobox.get('id', None)

190 if infobox_id is not None:

191 if infobox_id is not None:

192+ parsed_url_infobox_id = urlparse(infobox_id)

191 for existingIndex in self.infoboxes:

193 for existingIndex in self.infoboxes:

192− if compare_urls(urlparse(existingIndex.get('id', '')), urlparse(infobox_id)):

194+ if compare_urls(urlparse(existingIndex.get('id', '')), parsed_url_infobox_id):

193 merge_two_infoboxes(existingIndex, infobox)

195 merge_two_infoboxes(existingIndex, infobox)

194 add_infobox = False

196 add_infobox = False

195

197

searx/settings.yml+5

@@ -748,6 +748,11 @@ engines:

748 page_size : 10

749 disabled : True

750

751+ - name : seedpeer

752+ shortcut : speu

753+ engine : seedpeer

754+ categories: files, music, videos

755+

751# - name : yacy

756# - name : yacy

752# engine : yacy

757# engine : yacy

753# shortcut : ya

758# shortcut : ya

searx/templates/courgette/result_templates/torrent.html+1−1

@@ -4,7 +4,7 @@

4 {% endif %}

5 <h3 class="result_title"><a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>

6 {% if result.content %}<span class="content">{{ result.content|safe }}</span><br />{% endif %}

7− {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}

7+ {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span><br />{% endif %}

8 <span>

9 {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %}

10 {% if result.torrentfile %}<a href="{{ result.torrentfile }}" class="torrentfile" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('torrent file') }}</a>{% endif %}

searx/templates/legacy/result_templates/torrent.html+1−1

@@ -8,6 +8,6 @@

8 <p>

9 {% if result.magnetlink %}<a href="{{ result.magnetlink }}" class="magnetlink">{{ _('magnet link') }}</a>{% endif %}

10 {% if result.torrentfile %}<a href="{{ result.torrentfile }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %} class="torrentfile">{{ _('torrent file') }}</a>{% endif %} -

11− {% if result.seed %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %}

11+ {% if result.seed is defined %}<span class="stats">{{ _('Seeder') }} : {{ result.seed }}, {{ _('Leecher') }} : {{ result.leech }}</span>{% endif %}

12 </p>

13</div>

searx/templates/oscar/result_templates/torrent.html+1−1

@@ -3,7 +3,7 @@

3{{ result_header(result, favicons) }}

4{{ result_sub_header(result) }}

5

6−{% if result.seed %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}

6+{% if result.seed is defined %}<p class="result-content">{{ icon('transfer') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span>{% endif %}

7{% if result.filesize %}<br />{{ icon('floppy-disk') }} {{ _('Filesize') }}

8 <span class="badge">

9 {% if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/templates/simple/result_templates/torrent.html+1−1

@@ -6,7 +6,7 @@

6{% if result.magnetlink %}<p class="altlink"> • {{ result_link(result.magnetlink, icon('magnet') + _('magnet link'), "magnetlink") }}</p>{% endif %}

7{% if result.torrentfile %}<p class="altlink"> • {{ result_link(result.torrentfile, icon('download-alt') + _('torrent file'), "torrentfile") }}</p>{% endif %}

8

9−{% if result.seed %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}

9+{% if result.seed is defined %}<p class="stat"> • {{ icon('arrow-swap') }} {{ _('Seeder') }} <span class="badge">{{ result.seed }}</span> • {{ _('Leecher') }} <span class="badge">{{ result.leech }}</span></p>{% endif %}

10

11{%- if result.filesize %}<p class="stat">{{ icon('floppy-disk') }} {{ _('Filesize') }}<span class="badge">

12 {%- if result.filesize < 1024 %}{{ result.filesize }} {{ _('Bytes') }}

searx/utils.py+15

@@ -13,6 +13,7 @@ from numbers import Number

13from os.path import splitext, join

14from io import open

15from random import choice

16+from lxml.etree import XPath

16import sys

17import sys

17import json

18import json

18

19

@@ -51,6 +52,7 @@ ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

51useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

52useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

52 + "/data/useragents.json", 'r', encoding='utf-8').read())

53 + "/data/useragents.json", 'r', encoding='utf-8').read())

53

54

55+xpath_cache = dict()

54lang_to_lc_cache = dict()

56lang_to_lc_cache = dict()

55

57

56

58

@@ -450,3 +452,16 @@ def get_engine_from_settings(name):

450 return engine

452 return engine

451

453

452 return {}

454 return {}

455+

456+

457+def get_xpath(xpath_str):

458+ result = xpath_cache.get(xpath_str, None)

459+ if result is None:

460+ result = XPath(xpath_str)

461+ xpath_cache[xpath_str] = result

462+ return result

463+

464+

465+def eval_xpath(element, xpath_str):

466+ xpath = get_xpath(xpath_str)

467+ return xpath(element)

searx/webapp.py+7−9

@@ -157,20 +157,18 @@ outgoing_proxies = settings['outgoing'].get('proxies') or None

157

158@babel.localeselector

159def get_locale():

160− locale = request.accept_languages.best_match(settings['locales'].keys())

160+ if 'locale' in request.form\

161−

161+ and request.form['locale'] in settings['locales']:

162− if request.preferences.get_value('locale') != '':

163− locale = request.preferences.get_value('locale')

162+ return request.form['locale']

164

163

165 if 'locale' in request.args\

164 if 'locale' in request.args\

166 and request.args['locale'] in settings['locales']:

165 and request.args['locale'] in settings['locales']:

167− locale = request.args['locale']

166+ return request.args['locale']

168

167

169− if 'locale' in request.form\

168+ if request.preferences.get_value('locale') != '':

170− and request.form['locale'] in settings['locales']:

171− locale = request.form['locale']

169+ return request.preferences.get_value('locale')

172

170

173− return locale

171+ return request.accept_languages.best_match(settings['locales'].keys())

174

172

175

173

176# code-highlighter

174# code-highlighter

tests/unit/engines/test_seedpeer.pynew file+66

@@ -0,0 +1,66 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import seedpeer

5+from searx.testing import SearxTestCase

6+

7+

8+class TestBtdiggEngine(SearxTestCase):

9+

10+ def test_request(self):

11+ query = 'test_query'

12+ dicto = defaultdict(dict)

13+ dicto['pageno'] = 1

14+ params = seedpeer.request(query, dicto)

15+ self.assertIn('url', params)

16+ self.assertIn(query, params['url'])

17+ self.assertIn('seedpeer', params['url'])

18+

19+ def test_response(self):

20+ self.assertRaises(AttributeError, seedpeer.response, None)

21+ self.assertRaises(AttributeError, seedpeer.response, [])

22+ self.assertRaises(AttributeError, seedpeer.response, '')

23+ self.assertRaises(AttributeError, seedpeer.response, '[]')

24+

25+ response = mock.Mock(text='<html></html>')

26+ self.assertEqual(seedpeer.response(response), [])

27+

28+ html = u"""

29+ <html>

30+ <head>

31+ <script></script>

32+ <script type="text/javascript" src="not_here.js"></script>

33+ <script type="text/javascript">

34+ window.initialData=

35+ {"data": {"list": [{"name": "Title", "seeds": "10", "peers": "20", "size": "1024", "hash": "abc123"}]}}

36+ </script>

37+ </head>

38+ <body>

39+ <table></table>

40+ <table>

41+ <thead><tr></tr></thead>

42+ <tbody>

43+ <tr>

44+ <td><a href="link">Title</a></td>

45+ <td>1 year</td>

46+ <td>1 KB</td>

47+ <td>10</td>

48+ <td>20</td>

49+ <td></td>

50+ </tr>

51+ </tbody>

52+ </table>

53+ </body>

54+ </html>

55+ """

56+ response = mock.Mock(text=html)

57+ results = seedpeer.response(response)

58+ self.assertEqual(type(results), list)

59+ self.assertEqual(len(results), 1)

60+ self.assertEqual(results[0]['title'], 'Title')

61+ self.assertEqual(results[0]['url'], 'https://seedpeer.me/link')

62+ self.assertEqual(results[0]['seed'], 10)

63+ self.assertEqual(results[0]['leech'], 20)

64+ self.assertEqual(results[0]['filesize'], 1024)

65+ self.assertEqual(results[0]['torrentfile'], 'https://seedpeer.me/torrent/abc123')

66+ self.assertEqual(results[0]['magnetlink'], 'magnet:?xt=urn:btih:abc123')