on master help sign in
Update onion engines to v3 (#2904)

downgrade httpx: PR https://github.com/encode/httpx/pull/1522 made some breaking changes in AsyncHTTPTransport that affect our code in https://github.com/searx/searx/blob/master/searx/network/client.py remove not_evil which has been down for a while now: https://old.reddit.com/r/onions/search/?q=not+evil&restrict_sr=on&t=year

commita5839a66d6453e50c23e0738c87c561cf4791fc4 authorMarc Abonce Seguin <marc-abonce@mailbox.org> dateMon 2 Aug 2021, 18:03 (5 years ago) parent05b9ced
4 files changed+3−77
requirements.txt+1−1 searx/engines/ahmia.py+2−2 searx/engines/not_evil.py−67 searx/settings.yml−7
requirements.txt+1−1
@@ -7,7 +7,7 @@ lxml==4.6.3
77pygments==2.8.0
88python-dateutil==2.8.1
99pyyaml==5.4.1
10−httpx[http2]==0.18.2
10+httpx[http2]==0.17.1
1111Brotli==1.0.9
1212uvloop==0.15.2; python_version >= '3.7'
1313uvloop==0.14.0; python_version < '3.7'
searx/engines/ahmia.py+2−2
@@ -9,7 +9,7 @@ from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval
99
1010# about
1111about = {
12− "website": 'http://msydqstlz2kzerdg.onion',
12+ "website": 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion',
1313 "wikidata_id": 'Q18693938',
1414 "official_api_documentation": None,
1515 "use_official_api": False,
@@ -23,7 +23,7 @@ paging = True
2323page_size = 10
2424
2525# search url
26−search_url = 'http://msydqstlz2kzerdg.onion/search/?{query}'
26+search_url = 'http://juhanurmihxlp77nkq76byazcldy2hlmovfu2epvl5ankdibsot4csyd.onion/search/?{query}'
2727time_range_support = True
2828time_range_dict = {'day': 1,
2929 'week': 7,
searx/engines/not_evil.pydeleted−67
@@ -1,67 +0,0 @@
1−# SPDX-License-Identifier: AGPL-3.0-or-later
2−"""
3− not Evil (Onions)
4−"""
5−
6−from urllib.parse import urlencode
7−from lxml import html
8−from searx.engines.xpath import extract_text
9−
10−# about
11−about = {
12− "website": 'http://hss3uro2hsxfogfq.onion',
13− "wikidata_id": None,
14− "official_api_documentation": 'http://hss3uro2hsxfogfq.onion/api.htm',
15− "use_official_api": False,
16− "require_api_key": False,
17− "results": 'HTML',
18−}
19−
20−# engine dependent config
21−categories = ['onions']
22−paging = True
23−page_size = 20
24−
25−# search-url
26−base_url = 'http://hss3uro2hsxfogfq.onion/'
27−search_url = 'index.php?{query}&hostLimit=20&start={pageno}&numRows={page_size}'
28−
29−# specific xpath variables
30−results_xpath = '//*[@id="content"]/div/p'
31−url_xpath = './span[1]'
32−title_xpath = './a[1]'
33−content_xpath = './text()'
34−
35−
36−# do search-request
37−def request(query, params):
38− offset = (params['pageno'] - 1) * page_size
39−
40− params['url'] = base_url + search_url.format(pageno=offset,
41− query=urlencode({'q': query}),
42− page_size=page_size)
43−
44− return params
45−
46−
47−# get response from search-request
48−def response(resp):
49− results = []
50−
51− # needed because otherwise requests guesses wrong encoding
52− resp.encoding = 'utf8'
53− dom = html.fromstring(resp.text)
54−
55− # parse results
56− for result in dom.xpath(results_xpath):
57− url = extract_text(result.xpath(url_xpath)[0])
58− title = extract_text(result.xpath(title_xpath)[0])
59− content = extract_text(result.xpath(content_xpath))
60−
61− # append result
62− results.append({'url': url,
63− 'title': title,
64− 'content': content,
65− 'is_onion': True})
66−
67− return results
searx/settings.yml−7
@@ -793,13 +793,6 @@ engines:
793793 require_api_key: false
794794 results: JSON
795795
796−# Requires Tor
797− - name : not evil
798− engine : not_evil
799− categories : onions
800− enable_http : True
801− shortcut : ne
802−
803796 - name : nyaa
804797 engine : nyaa
805798 shortcut : nt