c3daa08 · searx · git.iko.soy

.gitignore+1

compare with master

unifiedsplit

@@ -15,6 +15,7 @@ setup.cfg

15*.pyc

16*/*.pyc

17*~

18+*.swp

18

19

19/node_modules

20/node_modules

20

21

searx/engines/__init__.py+13−1

@@ -142,6 +142,17 @@ def load_engine(engine_data):

142 engine.stats['page_load_time'] = 0

143 engine.stats['page_load_count'] = 0

144

145+ # tor related settings

146+ if settings['outgoing'].get('using_tor_proxy'):

147+ # use onion url if using tor.

148+ if hasattr(engine, 'onion_url'):

149+ engine.search_url = engine.onion_url + getattr(engine, 'search_path', '')

150+ elif 'onions' in engine.categories:

151+ # exclude onion engines if not using tor.

152+ return None

153+

154+ engine.timeout += settings['outgoing'].get('extra_proxy_timeout', 0)

155+

145 for category_name in engine.categories:

156 for category_name in engine.categories:

146 categories.setdefault(category_name, []).append(engine)

157 categories.setdefault(category_name, []).append(engine)

147

158

@@ -252,8 +263,9 @@ def get_engines_stats(preferences):

252

263

253

264

254def load_engines(engine_list):

265def load_engines(engine_list):

255− global engines

266+ global engines, engine_shortcuts

256 engines.clear()

267 engines.clear()

268+ engine_shortcuts.clear()

257 for engine_data in engine_list:

269 for engine_data in engine_list:

258 engine = load_engine(engine_data)

270 engine = load_engine(engine_data)

259 if engine is not None:

271 if engine is not None:

searx/engines/ahmia.pynew file+82

@@ -0,0 +1,82 @@

1+"""

2+ Ahmia (Onions)

3+

4+ @website http://msydqstlz2kzerdg.onion

5+ @provides-api no

6+

7+ @using-api no

8+ @results HTML

9+ @stable no

10+ @parse url, title, content

11+"""

12+

13+from urllib.parse import urlencode, urlparse, parse_qs

14+from lxml.html import fromstring

15+from searx.engines.xpath import extract_url, extract_text

16+

17+# engine config

18+categories = ['onions']

19+paging = True

20+page_size = 10

21+

22+# search url

23+search_url = 'http://msydqstlz2kzerdg.onion/search/?{query}'

24+time_range_support = True

25+time_range_dict = {'day': 1,

26+ 'week': 7,

27+ 'month': 30}

28+

29+# xpaths

30+results_xpath = '//li[@class="result"]'

31+url_xpath = './h4/a/@href'

32+title_xpath = './h4/a[1]'

33+content_xpath = './/p[1]'

34+correction_xpath = '//*[@id="didYouMean"]//a'

35+number_of_results_xpath = '//*[@id="totalResults"]'

36+

37+

38+def request(query, params):

39+ params['url'] = search_url.format(query=urlencode({'q': query}))

40+

41+ if params['time_range'] in time_range_dict:

42+ params['url'] += '&' + urlencode({'d': time_range_dict[params['time_range']]})

43+

44+ return params

45+

46+

47+def response(resp):

48+ results = []

49+ dom = fromstring(resp.text)

50+

51+ # trim results so there's not way too many at once

52+ first_result_index = page_size * (resp.search_params.get('pageno', 1) - 1)

53+ all_results = dom.xpath(results_xpath)

54+ trimmed_results = all_results[first_result_index:first_result_index + page_size]

55+

56+ # get results

57+ for result in trimmed_results:

58+ # remove ahmia url and extract the actual url for the result

59+ raw_url = extract_url(result.xpath(url_xpath), search_url)

60+ cleaned_url = parse_qs(urlparse(raw_url).query).get('redirect_url', [''])[0]

61+

62+ title = extract_text(result.xpath(title_xpath))

63+ content = extract_text(result.xpath(content_xpath))

64+

65+ results.append({'url': cleaned_url,

66+ 'title': title,

67+ 'content': content,

68+ 'is_onion': True})

69+

70+ # get spelling corrections

71+ for correction in dom.xpath(correction_xpath):

72+ results.append({'correction': extract_text(correction)})

73+

74+ # get number of results

75+ number_of_results = dom.xpath(number_of_results_xpath)

76+ if number_of_results:

77+ try:

78+ results.append({'number_of_results': int(extract_text(number_of_results))})

79+ except:

80+ pass

81+

82+ return results

searx/engines/not_evil.pynew file+64

@@ -0,0 +1,64 @@

1+"""

2+ not Evil (Onions)

3+

4+ @website http://hss3uro2hsxfogfq.onion

5+ @provide-api yes (http://hss3uro2hsxfogfq.onion/api.htm)

6+

7+ @using-api no

8+ @results HTML

9+ @stable no

10+ @parse url, title, content

11+"""

12+

13+from urllib.parse import urlencode

14+from lxml import html

15+from searx.engines.xpath import extract_text

16+

17+# engine dependent config

18+categories = ['onions']

19+paging = True

20+page_size = 20

21+

22+# search-url

23+base_url = 'http://hss3uro2hsxfogfq.onion/'

24+search_url = 'index.php?{query}&hostLimit=20&start={pageno}&numRows={page_size}'

25+

26+# specific xpath variables

27+results_xpath = '//*[@id="content"]/div/p'

28+url_xpath = './span[1]'

29+title_xpath = './a[1]'

30+content_xpath = './text()'

31+

32+

33+# do search-request

34+def request(query, params):

35+ offset = (params['pageno'] - 1) * page_size

36+

37+ params['url'] = base_url + search_url.format(pageno=offset,

38+ query=urlencode({'q': query}),

39+ page_size=page_size)

40+

41+ return params

42+

43+

44+# get response from search-request

45+def response(resp):

46+ results = []

47+

48+ # needed because otherwise requests guesses wrong encoding

49+ resp.encoding = 'utf8'

50+ dom = html.fromstring(resp.text)

51+

52+ # parse results

53+ for result in dom.xpath(results_xpath):

54+ url = extract_text(result.xpath(url_xpath)[0])

55+ title = extract_text(result.xpath(title_xpath)[0])

56+ content = extract_text(result.xpath(content_xpath))

57+

58+ # append result

59+ results.append({'url': url,

60+ 'title': title,

61+ 'content': content,

62+ 'is_onion': True})

63+

64+ return results

searx/engines/xpath.py+29−7

@@ -10,6 +10,8 @@ thumbnail_xpath = False

10paging = False

11suggestion_xpath = ''

12results_xpath = ''

13+cached_xpath = ''

14+cached_url = ''

13

15

14# parameters for engines with paging support

16# parameters for engines with paging support

15#

17#

@@ -36,6 +38,8 @@ def request(query, params):

36def response(resp):

38def response(resp):

37 results = []

39 results = []

38 dom = html.fromstring(resp.text)

40 dom = html.fromstring(resp.text)

41+ is_onion = True if 'onions' in categories else False

42+

39 if results_xpath:

43 if results_xpath:

40 for result in eval_xpath(dom, results_xpath):

44 for result in eval_xpath(dom, results_xpath):

41 url = extract_url(eval_xpath(result, url_xpath), search_url)

45 url = extract_url(eval_xpath(result, url_xpath), search_url)

@@ -49,15 +53,33 @@ def response(resp):

49 if len(thumbnail_xpath_result) > 0:

53 if len(thumbnail_xpath_result) > 0:

50 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

54 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

51

55

56+ # add alternative cached url if available

57+ if cached_xpath:

58+ tmp_result['cached_url'] = cached_url + extract_text(result.xpath(cached_xpath))

59+

60+ if is_onion:

61+ tmp_result['is_onion'] = True

62+

52 results.append(tmp_result)

63 results.append(tmp_result)

53 else:

64 else:

65+ if cached_xpath:

54− for url, title, content in zip(

66+ for url, title, content, cached in zip(

55− (extract_url(x, search_url) for

67+ (extract_url(x, search_url) for

56− x in eval_xpath(dom, url_xpath)),

68+ x in dom.xpath(url_xpath)),

69+ map(extract_text, dom.xpath(title_xpath)),

70+ map(extract_text, dom.xpath(content_xpath)),

57− map(extract_text, eval_xpath(dom, title_xpath)),

71+ map(extract_text, dom.xpath(cached_xpath))

72+ ):

73+ results.append({'url': url, 'title': title, 'content': content,

74+ 'cached_url': cached_url + cached, 'is_onion': is_onion})

75+ else:

76+ for url, title, content in zip(

77+ (extract_url(x, search_url) for

78+ x in dom.xpath(url_xpath)),

58− map(extract_text, eval_xpath(dom, content_xpath))

79+ map(extract_text, dom.xpath(title_xpath)),

80+ map(extract_text, dom.xpath(content_xpath))

59− ):

81+ ):

60− results.append({'url': url, 'title': title, 'content': content})

82+ results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion})

61

83

62 if not suggestion_xpath:

84 if not suggestion_xpath:

63 return results

85 return results

searx/settings.yml+28−3

@@ -60,8 +60,10 @@ outgoing: # communication with search engines

60# see http://docs.python-requests.org/en/latest/user/advanced/#proxies

61# SOCKS proxies are also supported: see http://requests.readthedocs.io/en/master/user/advanced/#socks

62# proxies :

63−# http : http://127.0.0.1:8080

63+# http : socks5h://127.0.0.1:9050

64−# https: http://127.0.0.1:8080

64+# https: socks5h://127.0.0.1:9050

65+# using_tor_proxy : True

66+# extra_proxy_timeout : 10.0 # Extra seconds to add in order to account for the time taken by the proxy

65# uncomment below section only if you have more than one network interface

67# uncomment below section only if you have more than one network interface

66# which can be the source of outgoing search requests

68# which can be the source of outgoing search requests

67# source_ips:

69# source_ips:

@@ -89,6 +91,12 @@ engines:

89 shortcut: apkm

91 shortcut: apkm

90 disabled: True

92 disabled: True

91

93

94+# Requires Tor

95+ - name : ahmia

96+ engine : ahmia

97+ categories : onions

98+ shortcut : ah

99+

92 - name : arch linux wiki

100 - name : arch linux wiki

93 engine : archlinux

101 engine : archlinux

94 shortcut : al

102 shortcut : al

@@ -185,7 +193,7 @@ engines:

185 - name : deviantart

193 - name : deviantart

186 engine : deviantart

194 engine : deviantart

187 shortcut : da

195 shortcut : da

188− timeout: 3.0

196+ timeout : 3.0

189

197

190 - name : ddg definitions

198 - name : ddg definitions

191 engine : duckduckgo_definitions

199 engine : duckduckgo_definitions

@@ -514,6 +522,11 @@ engines:

514 timeout: 5.0

522 timeout: 5.0

515 shortcut : npm

523 shortcut : npm

516

524

525+# Requires Tor

526+ - name : not evil

527+ engine : not_evil

528+ shortcut : ne

529+

517 - name : nyaa

530 - name : nyaa

518 engine : nyaa

531 engine : nyaa

519 shortcut : nt

532 shortcut : nt

@@ -698,6 +711,18 @@ engines:

698 url: https://torrentz2.eu/

711 url: https://torrentz2.eu/

699 timeout : 3.0

712 timeout : 3.0

700

713

714+# Requires Tor

715+ - name : torch

716+ engine : xpath

717+ paging : True

718+ search_url : http://xmh57jrknzkhv6y3ls3ubitzfqnkrwxhopf5aygthi7d6rplyvk3noyd.onion/cgi-bin/omega/omega?P={query}&DEFAULTOP=and

719+ results_xpath : //table//tr

720+ url_xpath : ./td[2]/a

721+ title_xpath : ./td[2]/b

722+ content_xpath : ./td[2]/small

723+ categories : onions

724+ shortcut : tch

725+

701 - name : twitter

726 - name : twitter

702 engine : twitter

727 engine : twitter

703 shortcut : tw

728 shortcut : tw

searx/templates/legacy/result_templates/default.html+6−1

@@ -1,6 +1,11 @@

1<div class="result {{ result.class }}{% for e in result.engines %} {{ e }}{% endfor %}">

2 <h3 class="result_title">{% if "icon_"~result.engine~".ico" in favicons %}<img width="14" height="14" class="favicon" src="{{ url_for('static', filename='img/icons/icon_'+result.engine+'.ico') }}" alt="{{result.engine}}" />{% endif %}<a href="{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ result.title|safe }}</a></h3>

3+ <p class="url">{{ result.pretty_url }}&lrm;

4+ {% if result.cached_url %}

5+ <a class="cache_link" href="{{ result.cached_url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('cached') }}</a>

6+ {% elif not result.is_onion %}

3− <p class="url">{{ result.pretty_url }}&lrm; <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('cached') }}</a>

7+ <a class="cache_link" href="https://web.archive.org/web/{{ result.url }}" {% if results_on_new_tab %}target="_blank" rel="noopener noreferrer"{% else %}rel="noreferrer"{% endif %}>{{ _('cached') }}</a>

8+ {% endif %}

4 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}</p>

9 {% if result.publishedDate %}<span class="published_date">{{ result.publishedDate }}</span>{% endif %}</p>

5 <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

10 <p class="content">{% if result.img_src %}<img src="{{ image_proxify(result.img_src) }}" class="image" />{% endif %}{% if result.content %}{{ result.content|safe }}<br class="last"/>{% endif %}</p>

6</div>

11</div>

searx/templates/oscar/macros.html+10−2

@@ -32,7 +32,11 @@

32 <span class="label label-default">{{ engine }}</span>

33 {%- endfor -%}

34 {%- if result.url -%}

35+ {% if result.cached_url %}

36+ <small>{{ result_link(result.cached_url, icon('link') + _('cached'), "text-info", id) }}</small>

37+ {% elif not result.is_onion %}

35− <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info", id) }}</small>

38+ <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info", id) }}</small>

39+ {% endif %}

36 {%- endif -%}

40 {%- endif -%}

37 {%- if proxify -%}

41 {%- if proxify -%}

38 <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info", id) }}</small>

42 <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info", id) }}</small>

@@ -50,7 +54,11 @@

50 <span class="label label-default">{{ engine }}</span>

54 <span class="label label-default">{{ engine }}</span>

51 {%- endfor %}

55 {%- endfor %}

52 {%- if result.url -%}

56 {%- if result.url -%}

57+ {% if result.cached_url %}

58+ <small>{{ result_link(result.cached_url, icon('link') + _('cached'), "text-info", id) }}</small>

59+ {% elif not result.is_onion %}

53− <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info", id) }}</small>

60+ <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info", id) }}</small>

61+ {% endif %}

54 {%- endif -%}

62 {%- endif -%}

55 {% if proxify -%}

63 {% if proxify -%}

56 <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info", id) }}</small>

64 <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info", id) }}</small>

searx/webapp.py+1

@@ -146,6 +146,7 @@ _category_names = (gettext('files'),

146 gettext('it'),

147 gettext('news'),

148 gettext('map'),

149+ gettext('onions'),

149 gettext('science'))

150 gettext('science'))

150

151

151outgoing_proxies = settings['outgoing'].get('proxies') or None

152outgoing_proxies = settings['outgoing'].get('proxies') or None

tests/unit/engines/test_xpath.pynew file+121

@@ -0,0 +1,121 @@

1+# -*- coding: utf-8 -*-

2+from collections import defaultdict

3+import mock

4+from searx.engines import xpath

5+from searx.testing import SearxTestCase

6+

7+

8+class TestXpathEngine(SearxTestCase):

9+

10+ def test_request(self):

11+ xpath.search_url = 'https://url.com/{query}'

12+ xpath.categories = []

13+ xpath.paging = False

14+ query = 'test_query'

15+ dicto = defaultdict(dict)

16+ params = xpath.request(query, dicto)

17+ self.assertIn('url', params)

18+ self.assertEquals('https://url.com/test_query', params['url'])

19+

20+ xpath.search_url = 'https://url.com/q={query}&p={pageno}'

21+ xpath.paging = True

22+ query = 'test_query'

23+ dicto = defaultdict(dict)

24+ dicto['pageno'] = 1

25+ params = xpath.request(query, dicto)

26+ self.assertIn('url', params)

27+ self.assertEquals('https://url.com/q=test_query&p=1', params['url'])

28+

29+ def test_response(self):

30+ # without results_xpath

31+ xpath.url_xpath = '//div[@class="search_result"]//a[@class="result"]/@href'

32+ xpath.title_xpath = '//div[@class="search_result"]//a[@class="result"]'

33+ xpath.content_xpath = '//div[@class="search_result"]//p[@class="content"]'

34+

35+ self.assertRaises(AttributeError, xpath.response, None)

36+ self.assertRaises(AttributeError, xpath.response, [])

37+ self.assertRaises(AttributeError, xpath.response, '')

38+ self.assertRaises(AttributeError, xpath.response, '[]')

39+

40+ response = mock.Mock(text='<html></html>')

41+ self.assertEqual(xpath.response(response), [])

42+

43+ html = u"""

44+ <div>

45+ <div class="search_result">

46+ <a class="result" href="https://result1.com">Result 1</a>

47+ <p class="content">Content 1</p>

48+ <a class="cached" href="https://cachedresult1.com">Cache</a>

49+ </div>

50+ <div class="search_result">

51+ <a class="result" href="https://result2.com">Result 2</a>

52+ <p class="content">Content 2</p>

53+ <a class="cached" href="https://cachedresult2.com">Cache</a>

54+ </div>

55+ </div>

56+ """

57+ response = mock.Mock(text=html)

58+ results = xpath.response(response)

59+ self.assertEqual(type(results), list)

60+ self.assertEqual(len(results), 2)

61+ self.assertEqual(results[0]['title'], 'Result 1')

62+ self.assertEqual(results[0]['url'], 'https://result1.com/')

63+ self.assertEqual(results[0]['content'], 'Content 1')

64+ self.assertEqual(results[1]['title'], 'Result 2')

65+ self.assertEqual(results[1]['url'], 'https://result2.com/')

66+ self.assertEqual(results[1]['content'], 'Content 2')

67+

68+ # with cached urls, without results_xpath

69+ xpath.cached_xpath = '//div[@class="search_result"]//a[@class="cached"]/@href'

70+ results = xpath.response(response)

71+ self.assertEqual(type(results), list)

72+ self.assertEqual(len(results), 2)

73+ self.assertEqual(results[0]['cached_url'], 'https://cachedresult1.com')

74+ self.assertEqual(results[1]['cached_url'], 'https://cachedresult2.com')

75+ self.assertFalse(results[0].get('is_onion', False))

76+

77+ # results are onion urls (no results_xpath)

78+ xpath.categories = ['onions']

79+ results = xpath.response(response)

80+ self.assertTrue(results[0]['is_onion'])

81+

82+ # with results_xpath

83+ xpath.results_xpath = '//div[@class="search_result"]'

84+ xpath.url_xpath = './/a[@class="result"]/@href'

85+ xpath.title_xpath = './/a[@class="result"]'

86+ xpath.content_xpath = './/p[@class="content"]'

87+ xpath.cached_xpath = None

88+ xpath.categories = []

89+

90+ self.assertRaises(AttributeError, xpath.response, None)

91+ self.assertRaises(AttributeError, xpath.response, [])

92+ self.assertRaises(AttributeError, xpath.response, '')

93+ self.assertRaises(AttributeError, xpath.response, '[]')

94+

95+ response = mock.Mock(text='<html></html>')

96+ self.assertEqual(xpath.response(response), [])

97+

98+ response = mock.Mock(text=html)

99+ results = xpath.response(response)

100+ self.assertEqual(type(results), list)

101+ self.assertEqual(len(results), 2)

102+ self.assertEqual(results[0]['title'], 'Result 1')

103+ self.assertEqual(results[0]['url'], 'https://result1.com/')

104+ self.assertEqual(results[0]['content'], 'Content 1')

105+ self.assertEqual(results[1]['title'], 'Result 2')

106+ self.assertEqual(results[1]['url'], 'https://result2.com/')

107+ self.assertEqual(results[1]['content'], 'Content 2')

108+

109+ # with cached urls, with results_xpath

110+ xpath.cached_xpath = './/a[@class="cached"]/@href'

111+ results = xpath.response(response)

112+ self.assertEqual(type(results), list)

113+ self.assertEqual(len(results), 2)

114+ self.assertEqual(results[0]['cached_url'], 'https://cachedresult1.com')

115+ self.assertEqual(results[1]['cached_url'], 'https://cachedresult2.com')

116+ self.assertFalse(results[0].get('is_onion', False))

117+

118+ # results are onion urls (with results_xpath)

119+ xpath.categories = ['onions']

120+ results = xpath.response(response)

121+ self.assertTrue(results[0]['is_onion'])

tests/unit/test_engines_init.pynew file+44

@@ -0,0 +1,44 @@

1+from searx.testing import SearxTestCase

2+from searx import settings, engines

3+

4+

5+class TestEnginesInit(SearxTestCase):

6+

7+ @classmethod

8+ def tearDownClass(cls):

9+ settings['outgoing']['using_tor_proxy'] = False

10+ settings['outgoing']['extra_proxy_timeout'] = 0

11+

12+ def test_initialize_engines_default(self):

13+ engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1'},

14+ {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2'}]

15+

16+ engines.initialize_engines(engine_list)

17+ self.assertEqual(len(engines.engines), 2)

18+ self.assertIn('engine1', engines.engines)

19+ self.assertIn('engine2', engines.engines)

20+

21+ def test_initialize_engines_exclude_onions(self):

22+ settings['outgoing']['using_tor_proxy'] = False

23+ engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general'},

24+ {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]

25+

26+ engines.initialize_engines(engine_list)

27+ self.assertEqual(len(engines.engines), 1)

28+ self.assertIn('engine1', engines.engines)

29+ self.assertNotIn('onions', engines.categories)

30+

31+ def test_initialize_engines_include_onions(self):

32+ settings['outgoing']['using_tor_proxy'] = True

33+ settings['outgoing']['extra_proxy_timeout'] = 100.0

34+ engine_list = [{'engine': 'dummy', 'name': 'engine1', 'shortcut': 'e1', 'categories': 'general',

35+ 'timeout': 20.0, 'onion_url': 'http://engine1.onion'},

36+ {'engine': 'dummy', 'name': 'engine2', 'shortcut': 'e2', 'categories': 'onions'}]

37+

38+ engines.initialize_engines(engine_list)

39+ self.assertEqual(len(engines.engines), 2)

40+ self.assertIn('engine1', engines.engines)

41+ self.assertIn('engine2', engines.engines)

42+ self.assertIn('onions', engines.categories)

43+ self.assertIn('http://engine1.onion', engines.engines['engine1'].search_url)

44+ self.assertEqual(engines.engines['engine1'].timeout, 120.0)