0d8b369 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2615 from searx/engine-data

Add ability to send engine data to subsequent requests

commit0d8b369b5b300e8a575d6715fc75067d09db63a5 authorNoémi Ványi <kvch@users.noreply.github.com> dateSat 6 Mar 2021, 12:17 (6 years ago) parent87f4cc4 44f4a9d

9 files changed+50−5

compare with master

unifiedsplit

searx/engines/__init__.py+1 searx/results.py+5−1 searx/search/__init__.py+2 searx/search/models.py+6−2 searx/templates/oscar/results.html+11 searx/templates/simple/results.html+9 searx/webadapter.py+13−1 searx/webapp.py+1 tests/unit/test_webapp.py+2−1

searx/engines/__init__.py+1

@@ -327,6 +327,7 @@ def _set_https_support_for_engine(engine):

327327 'is_test': True,

328328 'category': 'files',

329329 'raise_for_status': True,

330+ 'engine_data': {},

330331 })

331332

332333 if 'url' not in params:

searx/results.py+5−1

@@ -1,4 +1,5 @@

11import re

2+from collections import defaultdict

23from operator import itemgetter

34from threading import RLock

45from urllib.parse import urlparse, unquote

@@ -144,7 +145,7 @@ class ResultContainer:

144145 """docstring for ResultContainer"""

145146

146147 __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\

147− '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url'

148+ '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url', 'engine_data'

148149

149150 def __init__(self):

150151 super().__init__()

@@ -154,6 +155,7 @@ class ResultContainer:

154155 self.answers = {}

155156 self.corrections = set()

156157 self._number_of_results = []

158+ self.engine_data = defaultdict(dict)

157159 self._ordered = False

158160 self.paging = False

159161 self.unresponsive_engines = set()

@@ -175,6 +177,8 @@ class ResultContainer:

175177 self._merge_infobox(result)

176178 elif 'number_of_results' in result:

177179 self._number_of_results.append(result['number_of_results'])

180+ elif 'engine_data' in result:

181+ self.engine_data[engine_name][result['key']] = result['engine_data']

178182 else:

179183 # standard result (url, title, content)

180184 if 'url' in result and not isinstance(result['url'], str):

searx/search/__init__.py+2

@@ -111,6 +111,8 @@ class Search:

111111 if request_params is None:

112112 continue

113113

114+ request_params['engine_data'] = self.search_query.engine_data.get(engineref.name, {})

115+

114116 with threading.RLock():

115117 processor.engine.stats['sent_search_count'] += 1

116118

searx/search/models.py+6−2

@@ -25,7 +25,7 @@ class SearchQuery:

2525 """container for all the search parameters (query, language, etc...)"""

2626

2727 __slots__ = 'query', 'engineref_list', 'lang', 'safesearch', 'pageno', 'time_range',\

28− 'timeout_limit', 'external_bang'

28+ 'timeout_limit', 'external_bang', 'engine_data'

2929

3030 def __init__(self,

3131 query: str,

@@ -35,7 +35,8 @@ class SearchQuery:

3535 pageno: int=1,

3636 time_range: typing.Optional[str]=None,

3737 timeout_limit: typing.Optional[float]=None,

38− external_bang: typing.Optional[str]=None):

38+ external_bang: typing.Optional[str]=None,

39+ engine_data: typing.Optional[dict]=None):

3940 self.query = query

4041 self.engineref_list = engineref_list

4142 self.lang = lang

@@ -44,6 +45,9 @@ class SearchQuery:

4445 self.time_range = time_range

4546 self.timeout_limit = timeout_limit

4647 self.external_bang = external_bang

48+ self.engine_data = engine_data

49+ if engine_data is None:

50+ self.engine_data = {}

4751

4852 @property

4953 def categories(self):

searx/templates/oscar/results.html+11

@@ -7,6 +7,13 @@

77 <input type="hidden" name="language" value="{{ current_language }}" />{{- "" -}}

88 {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" />{% endif -%}

99{%- endmacro %}

10+{% macro engine_data_form(engine_data) -%}

11+ {% for engine_name, kv_data in engine_data.items() %}

12+ {% for k, v in kv_data.items() %}

13+ <input type="hidden" name="engine_data-{{ engine_name }}-{{ k|e }}" value="{{ v|e }}" />

14+ {% endfor %}

15+ {% endfor %}

16+{%- endmacro %}

1017{%- macro search_url() %}{{ url_for('search', _external=True) }}?q={{ q|urlencode }}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if time_range %}&time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&language={{ current_language }}{% endif %}{% endmacro -%}

1118

1219{% block title %}{{ q|e }} - {% endblock %}

@@ -142,12 +149,14 @@

142149 <div class="pull-left">{{- "" -}}

143150 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

144151 {{- search_form_attrs(pageno+1) -}}

152+ {{- engine_data_form(engine_data) -}}

145153 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-backward"></span> {{ _('next page') }}</button>{{- "" -}}

146154 </form>{{- "" -}}

147155 </div>

148156 <div class="pull-right">{{- "" -}}

149157 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

150158 {{- search_form_attrs(pageno-1) -}}

159+ {{- engine_data_form(engine_data) -}}

151160 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-forward"></span> {{ _('previous page') }}</button>{{- "" -}}

152161 </form>{{- "" -}}

153162 </div>

@@ -158,12 +167,14 @@

158167 <div class="pull-left">{{- "" -}}

159168 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

160169 {{- search_form_attrs(pageno-1) -}}

170+ {{- engine_data_form(engine_data) -}}

161171 <button type="submit" class="btn btn-default" {% if pageno == 1 %}disabled{% endif %}><span class="glyphicon glyphicon-backward"></span> {{ _('previous page') }}</button>{{- "" -}}

162172 </form>{{- "" -}}

163173 </div>

164174 <div class="pull-right">{{- "" -}}

165175 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}" class="pull-left">

166176 {{- search_form_attrs(pageno+1) -}}

177+ {{- engine_data_form(engine_data) -}}

167178 <button type="submit" class="btn btn-default"><span class="glyphicon glyphicon-forward"></span> {{ _('next page') }}</button>{{- "" -}}

168179 </form>{{- "" -}}

169180 </div>

searx/templates/simple/results.html+9

@@ -1,5 +1,12 @@

11{% extends "simple/base.html" %}

22{% from 'simple/macros.html' import icon, icon_small %}

3+{% macro engine_data_form(engine_data) -%}

4+ {% for engine_name, kv_data in engine_data.items() %}

5+ {% for k, v in kv_data.items() %}

6+ <input type="hidden" name="engine_data-{{ engine_name }}-{{ k|e }}" value="{{ v|e }}" />

7+ {% endfor %}

8+ {% endfor %}

9+{%- endmacro %}

310{% block title %}{% if method == 'GET' %}{{- q|e -}} -{% endif %}{% endblock %}

411{% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q|e }}" href="{{ url_for('search', _external=True) }}?q={{ q|urlencode }}&categories={{ selected_categories|join(",") | replace(' ','+') }}&pageno={{ pageno }}&time_range={{ time_range }}&language={{ current_language }}&safesearch={{ safesearch }}&format=rss">{% endblock %}

512{% block content %}

@@ -136,6 +143,7 @@

136143 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

137144 <div class="{% if rtl %}right{% else %}left{% endif %}">

138145 <input type="hidden" name="q" value="{{ q|e }}" >

146+ {{- engine_data_form(engine_data) -}}

139147 {% for category in selected_categories %}

140148 <input type="hidden" name="category_{{ category }}" value="1" >

141149 {% endfor %}

@@ -152,6 +160,7 @@

152160 <form method="{{ method or 'POST' }}" action="{{ url_for('search') }}">

153161 <div class="{% if rtl %}left{% else %}right{% endif %}">

154162 <input type="hidden" name="q" value="{{ q|e }}" >

163+ {{- engine_data_form(engine_data) -}}

155164 {% for category in selected_categories %}

156165 <input type="hidden" name="category_{{ category }}" value="1" >

157166 {% endfor %}

searx/webadapter.py+13−1

@@ -1,3 +1,4 @@

1+from collections import defaultdict

12from typing import Dict, List, Optional, Tuple

23from searx.exceptions import SearxParameterException

34from searx.webutils import VALID_LANGUAGE_CODE

@@ -196,6 +197,15 @@ def parse_generic(preferences: Preferences, form: Dict[str, str], disabled_engin

196197 return query_engineref_list

197198

198199

200+def parse_engine_data(form):

201+ engine_data = defaultdict(dict)

202+ for k, v in form.items():

203+ if k.startswith("engine_data"):

204+ _, engine, key = k.split('-')

205+ engine_data[engine][key] = v

206+ return engine_data

207+

208+

199209def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])\

200210 -> Tuple[SearchQuery, RawTextQuery, List[EngineRef], List[EngineRef]]:

201211 # no text for the query ?

@@ -217,6 +227,7 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])

217227 query_time_range = parse_time_range(form)

218228 query_timeout = parse_timeout(form, raw_text_query)

219229 external_bang = raw_text_query.external_bang

230+ engine_data = parse_engine_data(form)

220231

221232 if not is_locked('categories') and raw_text_query.enginerefs and raw_text_query.specific:

222233 # if engines are calculated from query,

@@ -232,7 +243,8 @@ def get_search_query_from_webapp(preferences: Preferences, form: Dict[str, str])

232243 validate_engineref_list(query_engineref_list, preferences)

233244

234245 return (SearchQuery(query, query_engineref_list, query_lang, query_safesearch, query_pageno,

235− query_time_range, query_timeout, external_bang=external_bang),

246+ query_time_range, query_timeout, external_bang=external_bang,

247+ engine_data=engine_data),

236248 raw_text_query,

237249 query_engineref_list_unknown,

238250 query_engineref_list_notoken)

searx/webapp.py+1

@@ -730,6 +730,7 @@ def search():

730730 answers=result_container.answers,

731731 corrections=correction_urls,

732732 infoboxes=result_container.infoboxes,

733+ engine_data=result_container.engine_data,

733734 paging=result_container.paging,

734735 unresponsive_engines=__get_translated_errors(result_container.unresponsive_engines),

735736 current_language=match_language(search_query.lang,

tests/unit/test_webapp.py+2−1

@@ -57,7 +57,8 @@ class ViewsTestCase(SearxTestCase):

5757 results_number=lambda: 3,

5858 results_length=lambda: len(test_results),

5959 get_timings=lambda: timings,

60− redirect_url=None)

60+ redirect_url=None,

61+ engine_data={})

6162

6263 self.setattr4test(Search, 'search', search_mock)

6364