72029d2 · searx · git.iko.soy

git.iko.soy/searx

[enh] Add timeout limit per request (#1640)

The new url parameter "timeout_limit" set timeout limit defined in second. Example "timeout_limit=1.5" means the timeout limit is 1.5 seconds. In addition, the query can start with <[number] to set the timeout limit. For number between 0 and 99, the unit is the second : Example: "<30 searx" means the timeout limit is 3 seconds For number above 100, the unit is the millisecond: Example: "<850 searx" means the timeout is 850 milliseconds. In addition, there is a new optional setting: outgoing.max_request_timeout. If not set, the user timeout can't go above searx configuration (as before: the max timeout of selected engine for a query). If the value is set, the user can set a timeout between 0 and max_request_timeout using <[number] or timeout_limit query parameter. Related to #1077 Updated version of PR #1413 from @isj-privacore

commit72029d27ded8d93ab891c616d6bffbe8d3a67dd2 authorAlexandre Flament <alex@al-f.net> dateFri 2 Aug 2019, 13:50 (7 years ago) parent2179079

10 files changed+181−16

compare with master

unifiedsplit

searx/query.py+18−1 searx/search.py+48−7 searx/settings.yml+2−1 searx/templates/oscar/results.html+1 searx/templates/simple/infobox.html+5 searx/templates/simple/results.html+9−3 searx/templates/simple/search.html+1 searx/webapp.py+2−1 tests/unit/test_query.py+42 tests/unit/test_search.py+53−3

searx/query.py+18−1

@@ -43,6 +43,7 @@ class RawTextQuery(object):

4343 self.query_parts = []

4444 self.engines = []

4545 self.languages = []

46+ self.timeout_limit = None

4647 self.specific = False

4748

4849 # parse query, if tags are set, which

@@ -69,6 +70,21 @@ class RawTextQuery(object):

6970 self.query_parts.append(query_part)

7071 continue

7172

73+ # this force the timeout

74+ if query_part[0] == '<':

75+ try:

76+ raw_timeout_limit = int(query_part[1:])

77+ if raw_timeout_limit < 100:

78+ # below 100, the unit is the second ( <3 = 3 seconds timeout )

79+ self.timeout_limit = float(raw_timeout_limit)

80+ else:

81+ # 100 or above, the unit is the millisecond ( <850 = 850 milliseconds timeout )

82+ self.timeout_limit = raw_timeout_limit / 1000.0

83+ parse_next = True

84+ except ValueError:

85+ # error not reported to the user

86+ pass

87+

7288 # this force a language

7389 if query_part[0] == ':':

7490 lang = query_part[1:].lower().replace('_', '-')

@@ -161,7 +177,7 @@ class RawTextQuery(object):

161177class SearchQuery(object):

162178 """container for all the search parameters (query, language, etc...)"""

163179

164− def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range):

180+ def __init__(self, query, engines, categories, lang, safesearch, pageno, time_range, timeout_limit=None):

165181 self.query = query.encode('utf-8')

166182 self.engines = engines

167183 self.categories = categories

@@ -169,6 +185,7 @@ class SearchQuery(object):

169185 self.safesearch = safesearch

170186 self.pageno = pageno

171187 self.time_range = time_range

188+ self.timeout_limit = timeout_limit

172189

173190 def __str__(self):

174191 return str(self.query) + ";" + str(self.engines)

searx/search.py+48−7

@@ -45,6 +45,16 @@ if sys.version_info[0] == 3:

4545logger = logger.getChild('search')

4646

4747number_of_searches = 0

48+max_request_timeout = settings.get('outgoing', {}).get('max_request_timeout' or None)

49+if max_request_timeout is None:

50+ logger.info('max_request_timeout={0}'.format(max_request_timeout))

51+else:

52+ if isinstance(max_request_timeout, float):

53+ logger.info('max_request_timeout={0} second(s)'.format(max_request_timeout))

54+ else:

55+ logger.critical('outgoing.max_request_timeout if defined has to be float')

56+ from sys import exit

57+ exit(1)

4858

4959

5060def send_http_request(engine, request_params):

@@ -265,6 +275,15 @@ def get_search_query_from_webapp(preferences, form):

265275 # query_engines

266276 query_engines = raw_text_query.engines

267277

278+ # timeout_limit

279+ query_timeout = raw_text_query.timeout_limit

280+ if query_timeout is None and 'timeout_limit' in form:

281+ raw_time_limit = form.get('timeout_limit')

282+ try:

283+ query_timeout = float(raw_time_limit)

284+ except ValueError:

285+ raise SearxParameterException('timeout_limit', raw_time_limit)

286+

268287 # query_categories

269288 query_categories = []

270289

@@ -338,7 +357,8 @@ def get_search_query_from_webapp(preferences, form):

338357 query_engines = deduplicate_query_engines(query_engines)

339358

340359 return (SearchQuery(query, query_engines, query_categories,

341− query_lang, query_safesearch, query_pageno, query_time_range),

360+ query_lang, query_safesearch, query_pageno,

361+ query_time_range, query_timeout),

342362 raw_text_query)

343363

344364

@@ -351,6 +371,7 @@ class Search(object):

351371 super(Search, self).__init__()

352372 self.search_query = search_query

353373 self.result_container = ResultContainer()

374+ self.actual_timeout = None

354375

355376 # do search-request

356377 def search(self):

@@ -380,7 +401,7 @@ class Search(object):

380401 search_query = self.search_query

381402

382403 # max of all selected engine timeout

383− timeout_limit = 0

404+ default_timeout = 0

384405

385406 # start search-reqest for all selected engines

386407 for selected_engine in search_query.engines:

@@ -420,12 +441,32 @@ class Search(object):

420441 # append request to list

421442 requests.append((selected_engine['name'], search_query.query, request_params))

422443

423− # update timeout_limit

424− timeout_limit = max(timeout_limit, engine.timeout)

425−

444+ # update default_timeout

445+ default_timeout = max(default_timeout, engine.timeout)

446+

447+ # adjust timeout

448+ self.actual_timeout = default_timeout

449+ query_timeout = self.search_query.timeout_limit

450+

451+ if max_request_timeout is None and query_timeout is None:

452+ # No max, no user query: default_timeout

453+ pass

454+ elif max_request_timeout is None and query_timeout is not None:

455+ # No max, but user query: From user query except if above default

456+ self.actual_timeout = min(default_timeout, query_timeout)

457+ elif max_request_timeout is not None and query_timeout is None:

458+ # Max, no user query: Default except if above max

459+ self.actual_timeout = min(default_timeout, max_request_timeout)

460+ elif max_request_timeout is not None and query_timeout is not None:

461+ # Max & user query: From user query except if above max

462+ self.actual_timeout = min(query_timeout, max_request_timeout)

463+

464+ logger.debug("actual_timeout={0} (default_timeout={1}, ?timeout_limit={2}, max_request_timeout={3})"

465+ .format(self.actual_timeout, default_timeout, query_timeout, max_request_timeout))

466+

467+ # send all search-request

426468 if requests:

427− # send all search-request

428− search_multiple_requests(requests, self.result_container, start_time, timeout_limit)

469+ search_multiple_requests(requests, self.result_container, start_time, self.actual_timeout)

429470 start_new_thread(gc.collect, tuple())

430471

431472 # return results, suggestions, answers and infoboxes

searx/settings.yml+2−1

@@ -34,7 +34,8 @@ ui:

3434# key : !!binary "your_morty_proxy_key"

3535

3636outgoing: # communication with search engines

37− request_timeout : 2.0 # seconds

37+ request_timeout : 2.0 # default timeout in seconds, can be override by engine

38+ # max_request_timeout: 10.0 # the maximum timeout in seconds

3839 useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator

3940 pool_connections : 100 # Number of different hosts

4041 pool_maxsize : 10 # Number of simultaneous requests by host

searx/templates/oscar/results.html+1

@@ -5,6 +5,7 @@

55 <input type="hidden" name="pageno" value="{{ pageno }}" />

66 <input type="hidden" name="time_range" value="{{ time_range }}" />

77 <input type="hidden" name="language" value="{{ current_language }}" />

8+ <input type="hidden" name="timeout_limit" value="{{ timeout_limit }}" />

89{%- endmacro %}

910{%- macro search_url() %}{{ base_url }}?q={{ q|urlencode }}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if time_range %}&time_range={{ time_range }}{% endif %}{% if current_language != 'all' %}&language={{ current_language }}{% endif %}{% endmacro -%}

1011

searx/templates/simple/infobox.html+5

@@ -36,6 +36,11 @@

3636 {% for suggestion in topic.suggestions %}

3737 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

3838 <input type="hidden" name="q" value="{{ suggestion }}">

39+ <input type="hidden" name="time_range" value="{{ time_range }}">

40+ <input type="hidden" name="language" value="{{ current_language }}">

41+ <input type="hidden" name="safesearch" value="{{ safesearch }}">

42+ <input type="hidden" name="theme" value="{{ theme }}">

43+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

3944 <input type="submit" value="{{ suggestion }}" />

4045 </form>

4146 {% endfor %}

searx/templates/simple/results.html+9−3

@@ -51,9 +51,11 @@

5151 {% for suggestion in suggestions %}

5252 <form method="{{ method or 'POST' }}" action="{{ url_for('index') }}">

5353 <input type="hidden" name="q" value="{{ suggestion.url }}">

54+ <input type="hidden" name="time_range" value="{{ time_range }}">

5455 <input type="hidden" name="language" value="{{ current_language }}">

5556 <input type="hidden" name="safesearch" value="{{ safesearch }}">

5657 <input type="hidden" name="theme" value="{{ theme }}">

58+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

5759 <input type="submit" class="suggestion" value="• {{ suggestion.title }}">

5860 </form>

5961 {% endfor %}

@@ -63,7 +65,7 @@

6365

6466 <div id="search_url">

6567 <h4 class="title">{{ _('Search URL') }} :</h4>

66− <div class="selectable_url"><pre>{{ base_url }}?q={{ q|urlencode }}&language={{ current_language }}&time_range={{ time_range }}&safesearch={{ safesearch }}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}</pre></div>

68+ <div class="selectable_url"><pre>{{ base_url }}?q={{ q|urlencode }}&language={{ current_language }}&time_range={{ time_range }}&safesearch={{ safesearch }}{% if pageno > 1 %}&pageno={{ pageno }}{% endif %}{% if selected_categories %}&categories={{ selected_categories|join(",") | replace(' ','+') }}{% endif %}{% if timeout_limit %}&timeout_limit={{ timeout_limit|urlencode }}{% endif %}</pre></div>

6769 </div>

6870 <div id="apis">

6971 <h4 class="title">{{ _('Download results') }}</h4>

@@ -79,6 +81,7 @@

7981 <input type="hidden" name="language" value="{{ current_language }}">

8082 <input type="hidden" name="safesearch" value="{{ safesearch }}">

8183 <input type="hidden" name="format" value="{{ output_type }}">

84+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

8285 <input type="submit" value="{{ output_type }}">

8386 </form>

8487 </div>

@@ -97,6 +100,7 @@

97100 <input type="hidden" name="language" value="{{ current_language }}">

98101 <input type="hidden" name="safesearch" value="{{ safesearch }}">

99102 <input type="hidden" name="theme" value="{{ theme }}">

103+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit }}" >{% endif %}

100104 <input type="submit" value="{{ correction }}">

101105 </form>

102106 </div>

@@ -134,7 +138,8 @@

134138 <input type="hidden" name="language" value="{{ current_language }}" >

135139 <input type="hidden" name="safesearch" value="{{ safesearch }}" >

136140 <input type="hidden" name="theme" value="{{ theme }}" >

137− <button type="submit">{{ icon_small('chevron-left') }} {{ _('previous page') }}</button>

141+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

142+ <button type="submit">{{ icon_small('chevron-left') }} {{ _('previous page') }}</button>

138143 </div>

139144 </form>

140145 {% endif %}

@@ -149,7 +154,8 @@

149154 <input type="hidden" name="language" value="{{ current_language }}" >

150155 <input type="hidden" name="safesearch" value="{{ safesearch }}" >

151156 <input type="hidden" name="theme" value="{{ theme }}" >

152− <button type="submit">{{ _('next page') }} {{ icon_small('chevron-right') }}</button>

157+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

158+ <button type="submit">{{ _('next page') }} {{ icon_small('chevron-right') }}</button>

153159 </div>

154160 </form>

155161 </nav>

searx/templates/simple/search.html+1

@@ -14,4 +14,5 @@

1414 {% include 'simple/categories.html' %}

1515 <input type="hidden" name="safesearch" value="{{ safesearch }}" >

1616 <input type="hidden" name="theme" value="{{ theme }}" >

17+ {% if timeout_limit %}<input type="hidden" name="timeout_limit" value="{{ timeout_limit|e }}" >{% endif %}

1718</form>

searx/webapp.py+2−1

@@ -628,7 +628,8 @@ def index():

628628 fallback=settings['search']['language']),

629629 base_url=get_base_url(),

630630 theme=get_current_theme_name(),

631− favicons=global_favicons[themes.index(get_current_theme_name())]

631+ favicons=global_favicons[themes.index(get_current_theme_name())],

632+ timeout_limit=request.form.get('timeout_limit', None)

632633 )

633634

634635

tests/unit/test_query.py+42

@@ -62,3 +62,45 @@ class TestQuery(SearxTestCase):

6262 self.assertEquals(len(query.query_parts), 1)

6363 self.assertEquals(len(query.languages), 0)

6464 self.assertFalse(query.specific)

65+

66+ def test_timeout_below100(self):

67+ query_text = '<3 the query'

68+ query = RawTextQuery(query_text, [])

69+ query.parse_query()

70+

71+ self.assertEquals(query.getFullQuery(), query_text)

72+ self.assertEquals(len(query.query_parts), 3)

73+ self.assertEquals(query.timeout_limit, 3)

74+ self.assertFalse(query.specific)

75+

76+ def test_timeout_above100(self):

77+ query_text = '<350 the query'

78+ query = RawTextQuery(query_text, [])

79+ query.parse_query()

80+

81+ self.assertEquals(query.getFullQuery(), query_text)

82+ self.assertEquals(len(query.query_parts), 3)

83+ self.assertEquals(query.timeout_limit, 0.35)

84+ self.assertFalse(query.specific)

85+

86+ def test_timeout_above1000(self):

87+ query_text = '<3500 the query'

88+ query = RawTextQuery(query_text, [])

89+ query.parse_query()

90+

91+ self.assertEquals(query.getFullQuery(), query_text)

92+ self.assertEquals(len(query.query_parts), 3)

93+ self.assertEquals(query.timeout_limit, 3.5)

94+ self.assertFalse(query.specific)

95+

96+ def test_timeout_invalid(self):

97+ # invalid number: it is not bang but it is part of the query

98+ query_text = '<xxx the query'

99+ query = RawTextQuery(query_text, [])

100+ query.parse_query()

101+

102+ self.assertEquals(query.getFullQuery(), query_text)

103+ self.assertEquals(len(query.query_parts), 1)

104+ self.assertEquals(query.query_parts[0], query_text)

105+ self.assertEquals(query.timeout_limit, None)

106+ self.assertFalse(query.specific)

tests/unit/test_search.py+53−3

@@ -2,9 +2,59 @@

33from searx.testing import SearxTestCase

5+import searx.preferences

6+import searx.search

7+import searx.engines

6−# TODO

710class SearchTestCase(SearxTestCase):

811

9− def test_(self):

10− pass

12+ @classmethod

13+ def setUpClass(cls):

14+ searx.engines.initialize_engines([{

15+ 'name': 'general dummy',

16+ 'engine': 'dummy',

17+ 'categories': 'general',

18+ 'shortcut': 'gd',

19+ 'timeout': 3.0

20+ }])

21+

22+ def test_timeout_simple(self):

23+ searx.search.max_request_timeout = None

24+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

25+ ['general'], 'en-US', 0, 1, None, None)

26+ search = searx.search.Search(search_query)

27+ search.search()

28+ self.assertEquals(search.actual_timeout, 3.0)

29+

30+ def test_timeout_query_above_default_nomax(self):

31+ searx.search.max_request_timeout = None

32+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

33+ ['general'], 'en-US', 0, 1, None, 5.0)

34+ search = searx.search.Search(search_query)

35+ search.search()

36+ self.assertEquals(search.actual_timeout, 3.0)

37+

38+ def test_timeout_query_below_default_nomax(self):

39+ searx.search.max_request_timeout = None

40+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

41+ ['general'], 'en-US', 0, 1, None, 1.0)

42+ search = searx.search.Search(search_query)

43+ search.search()

44+ self.assertEquals(search.actual_timeout, 1.0)

45+

46+ def test_timeout_query_below_max(self):

47+ searx.search.max_request_timeout = 10.0

48+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

49+ ['general'], 'en-US', 0, 1, None, 5.0)

50+ search = searx.search.Search(search_query)

51+ search.search()

52+ self.assertEquals(search.actual_timeout, 5.0)

53+

54+ def test_timeout_query_above_max(self):

55+ searx.search.max_request_timeout = 10.0

56+ search_query = searx.query.SearchQuery('test', [{'category': 'general', 'name': 'general dummy'}],

57+ ['general'], 'en-US', 0, 1, None, 15.0)

58+ search = searx.search.Search(search_query)

59+ search.search()

60+ self.assertEquals(search.actual_timeout, 10.0)