44f4a9d · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[enh] add ability to send engine data to subsequent requests

44f4a9dAdam Tauber6 years ago

searx/engines/__init__.py+1

compare with master

unifiedsplit

@@ -327,6 +327,7 @@ def _set_https_support_for_engine(engine):

327 'is_test': True,

328 'category': 'files',

329 'raise_for_status': True,

330+ 'engine_data': {},

330 })

331 })

331

332

332 if 'url' not in params:

333 if 'url' not in params:

searx/results.py+5−1

@@ -1,4 +1,5 @@

1import re

2+from collections import defaultdict

2from operator import itemgetter

3from operator import itemgetter

3from threading import RLock

4from threading import RLock

4from urllib.parse import urlparse, unquote

5from urllib.parse import urlparse, unquote

@@ -144,7 +145,7 @@ class ResultContainer:

144 """docstring for ResultContainer"""

145 """docstring for ResultContainer"""

145

146

146 __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\

147 __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\

147− '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url'

148+ '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data'

148

149

149 def __init__(self):

150 def __init__(self):

150 super().__init__()

151 super().__init__()

@@ -154,6 +155,7 @@ class ResultContainer:

154 self.answers = {}

155 self.answers = {}

155 self.corrections = set()

156 self.corrections = set()

156 self._number_of_results = []

157 self._number_of_results = []

158+ self.engine_data = defaultdict(dict)

157 self._ordered = False

159 self._ordered = False

158 self.paging = False

160 self.paging = False

159 self.unresponsive_engines = set()

161 self.unresponsive_engines = set()

@@ -175,6 +177,8 @@ class ResultContainer:

175 self._merge_infobox(result)

177 self._merge_infobox(result)

176 elif 'number_of_results' in result:

178 elif 'number_of_results' in result:

177 self._number_of_results.append(result['number_of_results'])

179 self._number_of_results.append(result['number_of_results'])

180+ elif 'engine_data' in result:

181+ self.engine_data[engine_name][result['key']] = result['engine_data']

178 else:

182 else:

179 # standard result (url, title, content)

183 # standard result (url, title, content)

180 if 'url' in result and not isinstance(result['url'], str):

184 if 'url' in result and not isinstance(result['url'], str):

searx/search/__init__.py+2

@@ -111,6 +111,8 @@ class Search:

111 if request_params is None:

112 continue

113

114+ request_params['engine_data'] = self.search_query.engine_data.get(engineref.name, {})

115+

114 with threading.RLock():

116 with threading.RLock():

115 processor.engine.stats['sent_search_count'] += 1

117 processor.engine.stats['sent_search_count'] += 1

116

118

searx/search/models.py+6−2

@@ -25,7 +25,7 @@ class SearchQuery:

25 """container for all the search parameters (query, language, etc...)"""

27 __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\

28− 'timeout_limit', 'external_bang'

28+ 'timeout_limit', 'external_bang', 'engine_data'

30 def __init__(self,

31 query: str,

@@ -35,7 +35,8 @@ class SearchQuery:

35 pageno: int=1,

36 time_range: typing.Optional[str]=None,

37 timeout_limit: typing.Optional[float]=None,

38− external_bang: typing.Optional[str]=None):

38+ external_bang: typing.Optional[str]=None,

39+ engine_data: typing.Optional[dict]=None):

39 self.query = query

40 self.query = query

40 self.engineref_list = engineref_list

41 self.engineref_list = engineref_list

41 self.lang = lang

42 self.lang = lang

@@ -44,6 +45,9 @@ class SearchQuery:

44 self.time_range = time_range

45 self.time_range = time_range

45 self.timeout_limit = timeout_limit

46 self.timeout_limit = timeout_limit

46 self.external_bang = external_bang

47 self.external_bang = external_bang

48+ self.engine_data = engine_data

49+ if engine_data is None:

50+ self.engine_data = {}

48 @property

52 @property

49 def categories(self):

53 def categories(self):

searx/templates/oscar/results.html+11

@@ -7,6 +7,13 @@

7 <input type="hidden" name="language" value="{{ current_language }}" />{{- "" -}}

8 {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" />{% endif -%}

9{%- endmacro %}

10+{% macro engine_data_form(engine_data) -%}

11+ {% for engine_name, kv_data in engine_data.items() %}

12+ {% for k, v in kv_data.items() %}

13+ <input type="hidden" name="engine_data-{{ engine_name }}-{{ k|e }}" value="{{ v|e }}" />

14+ {% endfor %}

15+ {% endfor %}

16+{%- endmacro %}

10{%- macro search_url() %}{{ url_for('search', _external=True) }}?q={{ q|urlencode }}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if time_range %}&time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&language={{ current_language }}{% endif %}{% endmacro -%}

17{%- macro search_url() %}{{ url_for('search', _external=True) }}?q={{ q|urlencode }}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if time_range %}&time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&language={{ current_language }}{% endif %}{% endmacro -%}

12{% block title %}{{ q|e }} - {% endblock %}

19{% block title %}{{ q|e }} - {% endblock %}

@@ -142,12 +149,14 @@

142 <div class="pull-left">{{- "" -}}

149 <div class="pull-left">{{- "" -}}

143 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

150 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

144 {{- search_form_attrs(pageno+1) -}}

151 {{- search_form_attrs(pageno+1) -}}

152+ {{- engine_data_form(engine_data) -}}

145 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>{{- "" -}}

153 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>{{- "" -}}

146 </form>{{- "" -}}

154 </form>{{- "" -}}

147 </div>

155 </div>

148 <div class="pull-right">{{- "" -}}

156 <div class="pull-right">{{- "" -}}

149 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

157 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

150 {{- search_form_attrs(pageno-1) -}}

158 {{- search_form_attrs(pageno-1) -}}

159+ {{- engine_data_form(engine_data) -}}

151 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>{{- "" -}}

160 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>{{- "" -}}

152 </form>{{- "" -}}

161 </form>{{- "" -}}

153 </div>

162 </div>

@@ -158,12 +167,14 @@

158 <div class="pull-left">{{- "" -}}

167 <div class="pull-left">{{- "" -}}

159 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

168 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

160 {{- search_form_attrs(pageno-1) -}}

169 {{- search_form_attrs(pageno-1) -}}

170+ {{- engine_data_form(engine_data) -}}

161 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>{{- "" -}}

171 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>{{- "" -}}

162 </form>{{- "" -}}

172 </form>{{- "" -}}

163 </div>

173 </div>

164 <div class="pull-right">{{- "" -}}

174 <div class="pull-right">{{- "" -}}

165 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

175 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

166 {{- search_form_attrs(pageno+1) -}}

176 {{- search_form_attrs(pageno+1) -}}

177+ {{- engine_data_form(engine_data) -}}

167 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>{{- "" -}}

178 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>{{- "" -}}

168 </form>{{- "" -}}

179 </form>{{- "" -}}

169 </div>

180 </div>

searx/templates/simple/results.html+9

@@ -1,5 +1,12 @@

1{% extends "simple/base.html" %}

2{% from 'simple/macros.html' import icon, icon_small %}

3+{% macro engine_data_form(engine_data) -%}

4+ {% for engine_name, kv_data in engine_data.items() %}

5+ {% for k, v in kv_data.items() %}

6+ <input type="hidden" name="engine_data-{{ engine_name }}-{{ k|e }}" value="{{ v|e }}" />

7+ {% endfor %}

8+ {% endfor %}

9+{%- endmacro %}

3{% block title %}{% if method == 'GET' %}{{- q|e -}} -{% endif %}{% endblock %}

10{% block title %}{% if method == 'GET' %}{{- q|e -}} -{% endif %}{% endblock %}

4{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('search', _external=True) }}?q={{ q|urlencode }}&categories={{ selected_categories|join(",") | replace(' ','+') }}&pageno={{ pageno }}&time_range={{ time_range }}&language={{ current_language }}&safesearch={{ safesearch }}&format=rss">{% endblock %}

11{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('search', _external=True) }}?q={{ q|urlencode }}&categories={{ selected_categories|join(",") | replace(' ','+') }}&pageno={{ pageno }}&time_range={{ time_range }}&language={{ current_language }}&safesearch={{ safesearch }}&format=rss">{% endblock %}

5{% block content %}

12{% block content %}

@@ -136,6 +143,7 @@

136 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

143 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

137 <div class="{% if rtl %}right{% else %}left{% endif %}">

144 <div class="{% if rtl %}right{% else %}left{% endif %}">

138 <input type="hidden" name="q" value="{{ q|e }}" >

145 <input type="hidden" name="q" value="{{ q|e }}" >

146+ {{- engine_data_form(engine_data) -}}

139 {% for category in selected_categories %}

147 {% for category in selected_categories %}

140 <input type="hidden" name="category_{{ category }}" value="1" >

148 <input type="hidden" name="category_{{ category }}" value="1" >

141 {% endfor %}

149 {% endfor %}

@@ -152,6 +160,7 @@

152 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

160 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

153 <div class="{% if rtl %}left{% else %}right{% endif %}">

161 <div class="{% if rtl %}left{% else %}right{% endif %}">

154 <input type="hidden" name="q" value="{{ q|e }}" >

162 <input type="hidden" name="q" value="{{ q|e }}" >

163+ {{- engine_data_form(engine_data) -}}

155 {% for category in selected_categories %}

164 {% for category in selected_categories %}

156 <input type="hidden" name="category_{{ category }}" value="1" >

165 <input type="hidden" name="category_{{ category }}" value="1" >

157 {% endfor %}

166 {% endfor %}

searx/webadapter.py+13−1

@@ -1,3 +1,4 @@

1+from collections import defaultdict

1from typing import Dict, List, Optional, Tuple

2from typing import Dict, List, Optional, Tuple

2from searx.exceptions import SearxParameterException

3from searx.exceptions import SearxParameterException

3from searx.webutils import VALID_LANGUAGE_CODE

4from searx.webutils import VALID_LANGUAGE_CODE

@@ -196,6 +197,15 @@ def parse_generic(preferences: Preferences, form: Dict[str, str], disabled_engin

196 return query_engineref_list

197 return query_engineref_list

197

198

199

200+def parse_engine_data(form):

201+ engine_data = defaultdict(dict)

202+ for k, v in form.items():

203+ if k.startswith("engine_data"):

204+ _, engine, key = k.split('-')

205+ engine_data[engine][key] = v

206+ return engine_data

207+

208+

199def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\

209def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\

200 -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]:

210 -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]:

201 # no text for the query ?

211 # no text for the query ?

@@ -217,6 +227,7 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])

217 query_time_range = parse_time_range(form)

227 query_time_range = parse_time_range(form)

218 query_timeout = parse_timeout(form, raw_text_query)

228 query_timeout = parse_timeout(form, raw_text_query)

219 external_bang = raw_text_query.external_bang

229 external_bang = raw_text_query.external_bang

230+ engine_data = parse_engine_data(form)

220

231

221 if not is_locked('categories') and raw_text_query.enginerefs and raw_text_query.specific:

232 if not is_locked('categories') and raw_text_query.enginerefs and raw_text_query.specific:

222 # if engines are calculated from query,

233 # if engines are calculated from query,

@@ -232,7 +243,8 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])

232 validate_engineref_list(query_engineref_list, preferences)

243 validate_engineref_list(query_engineref_list, preferences)

233

244

234 return (SearchQuery(query, query_engineref_list, query_lang, query_safesearch, query_pageno,

245 return (SearchQuery(query, query_engineref_list, query_lang, query_safesearch, query_pageno,

235− query_time_range, query_timeout, external_bang=external_bang),

246+ query_time_range, query_timeout, external_bang=external_bang,

247+ engine_data=engine_data),

236 raw_text_query,

248 raw_text_query,

237 query_engineref_list_unknown,

249 query_engineref_list_unknown,

238 query_engineref_list_notoken)

250 query_engineref_list_notoken)

searx/webapp.py+1

@@ -730,6 +730,7 @@ def search():

730 answers=result_container.answers,

731 corrections=correction_urls,

732 infoboxes=result_container.infoboxes,

733+ engine_data=result_container.engine_data,

733 paging=result_container.paging,

734 paging=result_container.paging,

734 unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),

735 unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),

735 current_language=match_language(search_query.lang,

736 current_language=match_language(search_query.lang,

tests/unit/test_webapp.py+2−1

@@ -57,7 +57,8 @@ class ViewsTestCase(SearxTestCase):

57 results_number=lambda: 3,

58 results_length=lambda: len(test_results),

59 get_timings=lambda: timings,

60− redirect_url=None)

60+ redirect_url=None,

61+ engine_data={})

62 self.setattr4test(Search, 'search', search_mock)

63 self.setattr4test(Search, 'search', search_mock)