a1a492b · searx · git.iko.soy

docs/dev/engine_overview.rst+2

compare with master

unifiedsplit

@@ -58,6 +58,8 @@ argument type information

58name string name of search-engine

59engine string name of searx-engine

60 (filename without ``.py``)

61+enable_http bool enable HTTP

62+ (by default only HTTPS is enabled).

61shortcut string shortcut of search-engine

63shortcut string shortcut of search-engine

62timeout string specific timeout for search-engine

64timeout string specific timeout for search-engine

63display_error_messages boolean display error messages on the web UI

65display_error_messages boolean display error messages on the web UI

searx/engines/__init__.py+1−32

@@ -50,6 +50,7 @@ engine_default_args = {'paging': False,

50 'timeout': settings['outgoing']['request_timeout'],

51 'shortcut': '-',

52 'disabled': False,

53+ 'enable_http': False,

53 'suspend_end_time': 0,

54 'suspend_end_time': 0,

54 'continuous_errors': 0,

55 'continuous_errors': 0,

55 'time_range_support': False,

56 'time_range_support': False,

@@ -305,35 +306,3 @@ def initialize_engines(engine_list):

305 if init_fn:

306 if init_fn:

306 logger.debug('%s engine: Starting background initialization', engine_name)

307 logger.debug('%s engine: Starting background initialization', engine_name)

307 threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()

308 threading.Thread(target=engine_init, args=(engine_name, init_fn)).start()

308−

309− _set_https_support_for_engine(engine)

310−

311−

312−def _set_https_support_for_engine(engine):

313− # check HTTPS support if it is not disabled

314− if engine.engine_type != 'offline' and not hasattr(engine, 'https_support'):

315− params = engine.request('http_test', {

316− 'method': 'GET',

317− 'headers': {},

318− 'data': {},

319− 'url': '',

320− 'cookies': {},

321− 'verify': True,

322− 'auth': None,

323− 'pageno': 1,

324− 'time_range': None,

325− 'language': '',

326− 'safesearch': False,

327− 'is_test': True,

328− 'category': 'files',

329− 'raise_for_status': True,

330− 'engine_data': {},

331− })

332−

333− if 'url' not in params:

334− return

335−

336− parsed_url = urlparse(params['url'])

337− https_support = parsed_url.scheme == 'https'

338−

339− setattr(engine, 'https_support', https_support)

searx/poolrequests.py+14−2

@@ -91,9 +91,10 @@ class SessionSinglePool(requests.Session):

91 self.adapters.clear()

92

93 https_adapter = threadLocal.__dict__.setdefault('https_adapter', next(https_adapters))

94− http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))

95 self.mount('https://', https_adapter)

94 self.mount('https://', https_adapter)

95+ if get_enable_http_protocol():

96+ http_adapter = threadLocal.__dict__.setdefault('http_adapter', next(http_adapters))

96− self.mount('http://', http_adapter)

97+ self.mount('http://', http_adapter)

97

98

98 def close(self):

99 def close(self):

99 """Call super, but clear adapters since there are managed globaly"""

100 """Call super, but clear adapters since there are managed globaly"""

@@ -106,6 +107,17 @@ def set_timeout_for_thread(timeout, start_time=None):

106 threadLocal.start_time = start_time

107 threadLocal.start_time = start_time

107

108

109

110+def set_enable_http_protocol(enable_http):

111+ threadLocal.enable_http = enable_http

112+

113+

114+def get_enable_http_protocol():

115+ try:

116+ return threadLocal.enable_http

117+ except AttributeError:

118+ return False

119+

120+

109def reset_time_for_thread():

121def reset_time_for_thread():

110 threadLocal.total_time = 0

122 threadLocal.total_time = 0

111

123

searx/search/processors/online.py+2

@@ -131,6 +131,8 @@ class OnlineProcessor(EngineProcessor):

131 poolrequests.set_timeout_for_thread(timeout_limit, start_time=start_time)

132 # reset the HTTP total time

133 poolrequests.reset_time_for_thread()

134+ # enable HTTP only if explicitly enabled

135+ poolrequests.set_enable_http_protocol(self.engine.enable_http)

134

136

135 # suppose everything will be alright

137 # suppose everything will be alright

136 requests_exception = False

138 requests_exception = False

searx/settings.yml+1

@@ -656,6 +656,7 @@ engines:

656

657 - name : library genesis

658 engine : xpath

659+ enable_http: True

659 search_url : http://libgen.rs/search.php?req={query}

660 search_url : http://libgen.rs/search.php?req={query}

660 url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href

661 url_xpath : //a[contains(@href,"bookfi.net/md5")]/@href

661 title_xpath : //a[contains(@href,"book/")]/text()[1]

662 title_xpath : //a[contains(@href,"book/")]/text()[1]

searx/templates/oscar/preferences.html+1−1

@@ -230,7 +230,7 @@

230 <td class="onoff-checkbox">

231 {{ checkbox_toggle('engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_'), (search_engine.name, categ) in disabled_engines) }}

232 </td>

233− <th scope="row">{% if not search_engine.https_support %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>

233+ <th scope="row">{% if search_engine.enable_http %}{{ icon('exclamation-sign', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</td></th>

234 <td class="name">{{ shortcuts[search_engine.name] }}

235 <td>{{ support_toggle(stats[search_engine.name].supports_selected_language) }}</td>

236 <td>{{ support_toggle(search_engine.safesearch==True) }}</td>

searx/templates/simple/preferences.html+1−1

@@ -121,7 +121,7 @@

121 {% set engine_id = 'engine_' + search_engine.name|replace(' ', '_') + '__' + categ|replace(' ', '_') %}

122 <tr>

123 <td class="engine_checkbox">{{ checkbox_onoff(engine_id, (search_engine.name, categ) in disabled_engines) }}</td>

124− <th class="name">{% if not search_engine.https_support %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>

124+ <th class="name">{% if search_engine.enable_http %}{{ icon('warning', 'No HTTPS') }}{% endif %} {{ search_engine.name }}</th>

125 <td class="shortcut">{{ shortcuts[search_engine.name] }}</td>

126 <td>{{ checkbox(engine_id + '_supported_languages', current_language == 'all' or current_language in search_engine.supported_languages or current_language.split('-')[0] in search_engine.supported_languages, true, true) }}</td>

127 <td>{{ checkbox(engine_id + '_safesearch', search_engine.safesearch==True, true, true) }}</td>