4a27dab · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge branch 'master' into conditional-sigusr1

4a27dabDr. Rolf Jansen6 years ago

Makefile+1

compare with master

unifiedsplit

@@ -192,6 +192,7 @@ PYLINT_FILES=\

192 searx/engines/google_images.py \

193 searx/engines/mediathekviewweb.py \

194 searx/engines/solidtorrents.py \

195+ searx/engines/solr.py \

195 searx/engines/google_scholar.py \

196 searx/engines/google_scholar.py \

196 searx/engines/yahoo_news.py \

197 searx/engines/yahoo_news.py \

197 searx/engines/apkmirror.py \

198 searx/engines/apkmirror.py \

searx/engines/acgsou.pydeleted−74

@@ -1,74 +0,0 @@

1−# SPDX-License-Identifier: AGPL-3.0-or-later

2−"""

3− Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)

4−"""

5−

6−from urllib.parse import urlencode

7−from lxml import html

8−from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex

9−

10−# about

11−about = {

12− "website": 'https://www.acgsou.com/',

13− "wikidata_id": None,

14− "official_api_documentation": None,

15− "use_official_api": False,

16− "require_api_key": False,

17− "results": 'HTML',

18−}

19−

20−# engine dependent config

21−categories = ['files', 'images', 'videos', 'music']

22−paging = True

23−

24−# search-url

25−base_url = 'https://www.acgsou.com/'

26−search_url = base_url + 'search.php?{query}&page={offset}'

27−# xpath queries

28−xpath_results = '//table[contains(@class, "list_style table_fixed")]//tr[not(th)]'

29−xpath_category = './/td[2]/a[1]'

30−xpath_title = './/td[3]/a[last()]'

31−xpath_torrent_links = './/td[3]/a'

32−xpath_filesize = './/td[4]/text()'

33−

34−

35−def request(query, params):

36− query = urlencode({'keyword': query})

37− params['url'] = search_url.format(query=query, offset=params['pageno'])

38− return params

39−

40−

41−def response(resp):

42− results = []

43− dom = html.fromstring(resp.text)

44− for result in eval_xpath_list(dom, xpath_results):

45− # defaults

46− filesize = 0

47− magnet_link = "magnet:?xt=urn:btih:{}&tr=https://tracker.acgsou.com:2710/announce"

48−

49− category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[]))

50− page_a = eval_xpath_getindex(result, xpath_title, 0)

51− title = extract_text(page_a)

52− href = base_url + page_a.attrib.get('href')

53−

54− magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])

55−

56− filesize_info = eval_xpath_getindex(result, xpath_filesize, 0, default=None)

57− if filesize_info:

58− try:

59− filesize = filesize_info[:-2]

60− filesize_multiplier = filesize_info[-2:]

61− filesize = get_torrent_size(filesize, filesize_multiplier)

62− except:

63− pass

64− # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime

65− content = 'Category: "{category}".'

66− content = content.format(category=category)

67−

68− results.append({'url': href,

69− 'title': title,

70− 'content': content,

71− 'filesize': filesize,

72− 'magnetlink': magnet_link,

73− 'template': 'torrent.html'})

74− return results

searx/engines/microsoft_academic.py+25−32

@@ -3,10 +3,7 @@

3 Microsoft Academic (Science)

4"""

6−from datetime import datetime

7−from json import loads

6+from json import dumps, loads

8−from uuid import uuid4

9−from urllib.parse import urlencode

10from searx.utils import html_to_text

7from searx.utils import html_to_text

12# about

9# about

@@ -21,26 +18,25 @@ about = {

22categories = ['images']

19categories = ['images']

23paging = True

20paging = True

24−result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}'

21+search_url = 'https://academic.microsoft.com/api/search'

22+_paper_url = 'https://academic.microsoft.com/paper/{id}/reference'

27def request(query, params):

25def request(query, params):

28− correlation_id = uuid4()

26+ params['url'] = search_url

29− msacademic = uuid4()

30− time_now = datetime.now()

31−

32− params['url'] = result_url.format(query=urlencode({'correlationId': correlation_id}))

33− params['cookies']['msacademic'] = str(msacademic)

34− params['cookies']['ai_user'] = 'vhd0H|{now}'.format(now=str(time_now))

35 params['method'] = 'POST'

27 params['method'] = 'POST'

36− params['data'] = {

28+ params['headers']['content-type'] = 'application/json; charset=utf-8'

37− 'Query': '@{query}@'.format(query=query),

29+ params['data'] = dumps({

38− 'Limit': 10,

30+ 'query': query,

31+ 'queryExpression': '',

32+ 'filters': [],

33+ 'orderBy': 0,

39− 'Offset': params['pageno'] - 1,

34+ 'skip': (params['pageno'] - 1) * 10,

40− 'Filters': '',

35+ 'sortAscending': True,

41− 'OrderBy': '',

36+ 'take': 10,

42− 'SortAscending': False,

37+ 'includeCitationContexts': False,

38+ 'profileId': '',

43− }

39+ })

45 return params

41 return params

@@ -51,10 +47,13 @@ def response(resp):

51 if not response_data:

47 if not response_data:

52 return results

48 return results

54− for result in response_data['results']:

50+ for result in response_data['pr']:

55− url = _get_url(result)

51+ if 'dn' not in result['paper']:

52+ continue

53+

56− title = result['e']['dn']

54+ title = result['paper']['dn']

57− content = _get_content(result)

55+ content = _get_content(result['paper'])

56+ url = _paper_url.format(id=result['paper']['id'])

58 results.append({

57 results.append({

59 'url': url,

58 'url': url,

60 'title': html_to_text(title),

59 'title': html_to_text(title),

@@ -64,15 +63,9 @@ def response(resp):

64 return results

63 return results

67−def _get_url(result):

68− if 's' in result['e']:

69− return result['e']['s'][0]['u']

70− return 'https://academic.microsoft.com/#/detail/{pid}'.format(pid=result['id'])

71−

72−

73def _get_content(result):

66def _get_content(result):

74− if 'd' in result['e']:

67+ if 'd' in result:

75− content = result['e']['d']

68+ content = result['d']

76 if len(content) > 300:

69 if len(content) > 300:

77 return content[:300] + '...'

70 return content[:300] + '...'

78 return content

71 return content

searx/engines/solr.pynew file+74

@@ -0,0 +1,74 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Solr

4+"""

6+# pylint: disable=global-statement, missing-function-docstring

8+from json import loads

9+from urllib.parse import urlencode

10+from searx.exceptions import SearxEngineAPIException

11+

12+

13+base_url = 'http://localhost:8983'

14+collection = ''

15+rows = 10

16+sort = '' # sorting: asc or desc

17+field_list = 'name' # list of field names to display on the UI

18+default_fields = '' # default field to query

19+query_fields = '' # query fields

20+_search_url = ''

21+paging = True

22+

23+

24+def init(_):

25+ if collection == '':

26+ raise ValueError('collection cannot be empty')

27+

28+ global _search_url

29+ _search_url = base_url + '/solr/' + collection + '/select?{params}'

30+

31+

32+def request(query, params):

33+ query_params = {'q': query, 'rows': rows}

34+ if field_list != '':

35+ query_params['fl'] = field_list

36+ if query_fields != '':

37+ query_params['qf'] = query_fields

38+ if default_fields != '':

39+ query_params['df'] = default_fields

40+ if sort != '':

41+ query_params['sort'] = sort

42+

43+ if 'pageno' in params:

44+ query_params['start'] = rows * (params['pageno'] - 1)

45+

46+ params['url'] = _search_url.format(params=urlencode(query_params))

47+

48+ return params

49+

50+

51+def response(resp):

52+ resp_json = __get_response(resp)

53+

54+ results = []

55+ for result in resp_json['response']['docs']:

56+ r = {key: str(value) for key, value in result.items()}

57+ if len(r) == 0:

58+ continue

59+ r['template'] = 'key-value.html'

60+ results.append(r)

61+

62+ return results

63+

64+

65+def __get_response(resp):

66+ try:

67+ resp_json = loads(resp.text)

68+ except Exception as e:

69+ raise SearxEngineAPIException("failed to parse response") from e

70+

71+ if 'error' in resp_json:

72+ raise SearxEngineAPIException(resp_json['error']['msg'])

73+

74+ return resp_json

searx/engines/yahoo_news.py−7

@@ -22,13 +22,6 @@ from searx.utils import (

23from searx.engines.yahoo import parse_url

25−# pylint: disable=unused-import

26−from searx.engines.yahoo import (

27− _fetch_supported_languages,

28− supported_languages_url,

29−)

30−# pylint: enable=unused-import

31−

32logger = logger.getChild('yahoo_news engine')

25logger = logger.getChild('yahoo_news engine')

34# about

27# about

searx/engines/youtube_noapi.py+72−7

@@ -4,7 +4,7 @@

4"""

6from functools import reduce

7−from json import loads

7+from json import loads, dumps

8from urllib.parse import quote_plus

10# about

@@ -20,12 +20,15 @@ about = {

20# engine dependent config

21categories = ['videos', 'music']

22paging = True

23+language_support = False

23time_range_support = True

24time_range_support = True

25# search-url

26# search-url

26base_url = 'https://www.youtube.com/results'

27base_url = 'https://www.youtube.com/results'

27search_url = base_url + '?search_query={query}&page={page}'

28search_url = base_url + '?search_query={query}&page={page}'

28time_range_url = '&sp=EgII{time_range}%253D%253D'

29time_range_url = '&sp=EgII{time_range}%253D%253D'

30+# the key seems to be constant

31+next_page_url = 'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'

29time_range_dict = {'day': 'Ag',

32time_range_dict = {'day': 'Ag',

30 'week': 'Aw',

33 'week': 'Aw',

31 'month': 'BA',

34 'month': 'BA',

@@ -40,21 +43,73 @@ base_youtube_url = 'https://www.youtube.com/watch?v='

41# do search-request

44# do search-request

42def request(query, params):

45def request(query, params):

46+ if not params['engine_data'].get('next_page_token'):

43− params['url'] = search_url.format(query=quote_plus(query),

47+ params['url'] = search_url.format(query=quote_plus(query), page=params['pageno'])

44− page=params['pageno'])

45− if params['time_range'] in time_range_dict:

48+ if params['time_range'] in time_range_dict:

46− params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']])

49+ params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']])

50+ else:

51+ print(params['engine_data']['next_page_token'])

52+ params['url'] = next_page_url

53+ params['method'] = 'POST'

54+ params['data'] = dumps({

55+ 'context': {"client": {"clientName": "WEB", "clientVersion": "2.20210310.12.01"}},

56+ 'continuation': params['engine_data']['next_page_token'],

57+ })

58+ params['headers']['Content-Type'] = 'application/json'

48 return params

60 return params

51# get response from search-request

63# get response from search-request

52def response(resp):

64def response(resp):

65+ if resp.search_params.get('engine_data'):

66+ return parse_next_page_response(resp.text)

67+ return parse_first_page_response(resp.text)

68+

69+

70+def parse_next_page_response(response_text):

53 results = []

71 results = []

72+ result_json = loads(response_text)

73+ with open("/tmp/x", "w") as f:

74+ f.write(response_text)

75+ for section in (result_json['onResponseReceivedCommands'][0]

76+ .get('appendContinuationItemsAction')['continuationItems'][0]

77+ .get('itemSectionRenderer')['contents']):

78+ if 'videoRenderer' not in section:

79+ continue

80+ section = section['videoRenderer']

81+ content = "-"

82+ if 'descriptionSnippet' in section:

83+ content = ' '.join(x['text'] for x in section['descriptionSnippet']['runs'])

84+ results.append({

85+ 'url': base_youtube_url + section['videoId'],

86+ 'title': ' '.join(x['text'] for x in section['title']['runs']),

87+ 'content': content,

88+ 'author': section['ownerText']['runs'][0]['text'],

89+ 'length': section['lengthText']['simpleText'],

90+ 'template': 'videos.html',

91+ 'embedded': embedded_url.format(videoid=section['videoId']),

92+ 'thumbnail': section['thumbnail']['thumbnails'][-1]['url'],

93+ })

94+ try:

95+ token = result_json['onResponseReceivedCommands'][0]\

96+ .get('appendContinuationItemsAction')['continuationItems'][1]\

97+ .get('continuationItemRenderer')['continuationEndpoint']\

98+ .get('continuationCommand')['token']

99+ results.append({

100+ "engine_data": token,

101+ "key": "next_page_token",

102+ })

103+ except:

104+ pass

105+

106+ return results

107

55− results_data = resp.text[resp.text.find('ytInitialData'):]

56− results_data = results_data[results_data.find('{'):results_data.find(';</script>')]

108

109+def parse_first_page_response(response_text):

110+ results = []

111+ results_data = response_text[response_text.find('ytInitialData'):]

112+ results_data = results_data[results_data.find('{'):results_data.find(';</script>')]

58 results_json = loads(results_data) if results_data else {}

113 results_json = loads(results_data) if results_data else {}

59 sections = results_json.get('contents', {})\

114 sections = results_json.get('contents', {})\

60 .get('twoColumnSearchResultsRenderer', {})\

115 .get('twoColumnSearchResultsRenderer', {})\

@@ -63,6 +118,16 @@ def response(resp):

63 .get('contents', [])

118 .get('contents', [])

119

65 for section in sections:

120 for section in sections:

121+ if "continuationItemRenderer" in section:

122+ next_page_token = section["continuationItemRenderer"]\

123+ .get("continuationEndpoint", {})\

124+ .get("continuationCommand", {})\

125+ .get("token", "")

126+ if next_page_token:

127+ results.append({

128+ "engine_data": next_page_token,

129+ "key": "next_page_token",

130+ })

66 for video_container in section.get('itemSectionRenderer', {}).get('contents', []):

131 for video_container in section.get('itemSectionRenderer', {}).get('contents', []):

67 video = video_container.get('videoRenderer', {})

132 video = video_container.get('videoRenderer', {})

68 videoid = video.get('videoId')

133 videoid = video.get('videoId')

searx/settings.yml+17−6

@@ -82,6 +82,8 @@ outgoing: # communication with search engines

82# https:

83# - http://proxy1:8080

84# - http://proxy2:8080

85+# using_tor_proxy : True

86+# extra_proxy_timeout : 10.0 # Extra seconds to add in order to account for the time taken by the proxy

85# uncomment below section only if you have more than one network interface

87# uncomment below section only if you have more than one network interface

86# which can be the source of outgoing search requests

88# which can be the source of outgoing search requests

87# source_ips:

89# source_ips:

@@ -159,6 +161,7 @@ engines:

159 - name : ahmia

161 - name : ahmia

160 engine : ahmia

162 engine : ahmia

161 categories : onions

163 categories : onions

164+ enable_http : True

162 shortcut : ah

165 shortcut : ah

163

166

164 - name : arch linux wiki

167 - name : arch linux wiki

@@ -730,6 +733,8 @@ engines:

730# Requires Tor

733# Requires Tor

731 - name : not evil

734 - name : not evil

732 engine : not_evil

735 engine : not_evil

736+ categories : onions

737+ enable_http : True

733 shortcut : ne

738 shortcut : ne

734

739

735 - name : nyaa

740 - name : nyaa

@@ -737,12 +742,6 @@ engines:

737 shortcut : nt

742 shortcut : nt

738 disabled : True

743 disabled : True

739

744

740− - name : acgsou

741− engine : acgsou

742− shortcut : acg

743− disabled : True

744− timeout: 5.0

745−

746 - name : openairedatasets

745 - name : openairedatasets

747 engine : json_engine

746 engine : json_engine

748 paging : True

747 paging : True

@@ -943,6 +942,17 @@ engines:

943# api_client_id : *******

942# api_client_id : *******

944# api_client_secret : *******

943# api_client_secret : *******

945

944

945+# - name : solr

946+# engine : solr

947+# shortcut : slr

948+# base_url : http://localhost:8983

949+# collection : collection_name

950+# sort : '' # sorting: asc or desc

951+# field_list : '' # comma separated list of field names to display on the UI

952+# default_fields : '' # default field to query

953+# query_fields : '' # query fields

954+# enable_http : True

955+

946 - name : startpage

956 - name : startpage

947 engine : startpage

957 engine : startpage

948 shortcut : sp

958 shortcut : sp

@@ -979,6 +989,7 @@ engines:

979 title_xpath : ./td[2]/b

989 title_xpath : ./td[2]/b

980 content_xpath : ./td[2]/small

990 content_xpath : ./td[2]/small

981 categories : onions

991 categories : onions

992+ enable_http : True

982 shortcut : tch

993 shortcut : tch

983

994

984# maybe in a fun category

995# maybe in a fun category