@@ -16,13 +16,8 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
1616'''
1717
1818import threading
19−import re
2019import searx.poolrequests as requests_lib
21−from itertools import izip_longest, chain
22−from operator import itemgetter
23−from Queue import Queue
2420from time import time
25−from urlparse import urlparse, unquote
2621from searx import settings
2722from searx.engines import (
2823 categories, engines
@@ -30,6 +25,7 @@ from searx.engines import (
3025from searx.languages import language_codes
3126from searx.utils import gen_useragent, get_blocked_engines
3227from searx.query import Query
28+from searx.results import ResultContainer
3329from searx import logger
3430
3531logger = logger.getChild('search')
@@ -42,7 +38,8 @@ def search_request_wrapper(fn, url, engine_name, **kwargs):
4238 return fn(url, **kwargs)
4339 except:
4440 # increase errors stats
45− engines[engine_name].stats['errors'] += 1
41+ with threading.RLock():
42+ engines[engine_name].stats['errors'] += 1
4643
4744 # print engine name and specific error message
4845 logger.exception('engine crash: {0}'.format(engine_name))
@@ -84,7 +81,7 @@ def default_request_params():
8481
8582
8683# create a callback wrapper for the search engine results
87−def make_callback(engine_name, results_queue, callback, params):
84+def make_callback(engine_name, callback, params, result_container):
8885
8986 # creating a callback wrapper for the search engine results
9087 def process_callback(response, **kwargs):
@@ -96,12 +93,17 @@ def make_callback(engine_name, results_queue, callback, params):
9693
9794 response.search_params = params
9895
99− timeout_overhead = 0.2 # seconds
10096 search_duration = time() - params['started']
97+ # update stats with current page-load-time
98+ with threading.RLock():
99+ engines[engine_name].stats['page_load_time'] += search_duration
100+
101+ timeout_overhead = 0.2 # seconds
101102 timeout_limit = engines[engine_name].timeout + timeout_overhead
103+
102104 if search_duration > timeout_limit:
103− engines[engine_name].stats['page_load_time'] += timeout_limit
104− engines[engine_name].stats['errors'] += 1
105+ with threading.RLock():
106+ engines[engine_name].stats['errors'] += 1
105107 return
106108
107109 # callback
@@ -111,212 +113,11 @@ def make_callback(engine_name, results_queue, callback, params):
111113 for result in search_results:
112114 result['engine'] = engine_name
113115
114− results_queue.put_nowait((engine_name, search_results))
115−
116− # update stats with current page-load-time
117− engines[engine_name].stats['page_load_time'] += search_duration
116+ result_container.extend(engine_name, search_results)
118117
119118 return process_callback
120119
121120
122−# return the meaningful length of the content for a result
123−def content_result_len(content):
124− if isinstance(content, basestring):
125− content = re.sub('[,;:!?\./\\\\ ()-_]', '', content)
126− return len(content)
127− else:
128− return 0
129−
130−
131−# score results and remove duplications
132−def score_results(results):
133− # calculate scoring parameters
134− flat_res = filter(
135− None, chain.from_iterable(izip_longest(*results.values())))
136− flat_len = len(flat_res)
137− engines_len = len(results)
138−
139− results = []
140−
141− # pass 1: deduplication + scoring
142− for i, res in enumerate(flat_res):
143−
144− res['parsed_url'] = urlparse(res['url'])
145−
146− # if the result has no scheme, use http as default
147− if not res['parsed_url'].scheme:
148− res['parsed_url'] = res['parsed_url']._replace(scheme="http")
149−
150− res['host'] = res['parsed_url'].netloc
151−
152− if res['host'].startswith('www.'):
153− res['host'] = res['host'].replace('www.', '', 1)
154−
155− res['engines'] = [res['engine']]
156−
157− weight = 1.0
158−
159− # strip multiple spaces and cariage returns from content
160− if res.get('content'):
161− res['content'] = re.sub(' +', ' ',
162− res['content'].strip().replace('\n', ''))
163−
164− # get weight of this engine if possible
165− if hasattr(engines[res['engine']], 'weight'):
166− weight = float(engines[res['engine']].weight)
167−
168− # calculate score for that engine
169− score = int((flat_len - i) / engines_len) * weight + 1
170−
171− # check for duplicates
172− duplicated = False
173− for new_res in results:
174− # remove / from the end of the url if required
175− p1 = res['parsed_url'].path[:-1]\
176− if res['parsed_url'].path.endswith('/')\
177− else res['parsed_url'].path
178− p2 = new_res['parsed_url'].path[:-1]\
179− if new_res['parsed_url'].path.endswith('/')\
180− else new_res['parsed_url'].path
181−
182− # check if that result is a duplicate
183− if res['host'] == new_res['host'] and\
184− unquote(p1) == unquote(p2) and\
185− res['parsed_url'].query == new_res['parsed_url'].query and\
186− res.get('template') == new_res.get('template'):
187− duplicated = new_res
188− break
189−
190− # merge duplicates together
191− if duplicated:
192− # using content with more text
193− if content_result_len(res.get('content', '')) >\
194− content_result_len(duplicated.get('content', '')):
195− duplicated['content'] = res['content']
196−
197− # increase result-score
198− duplicated['score'] += score
199−
200− # add engine to list of result-engines
201− duplicated['engines'].append(res['engine'])
202−
203− # using https if possible
204− if duplicated['parsed_url'].scheme == 'https':
205− continue
206− elif res['parsed_url'].scheme == 'https':
207− duplicated['url'] = res['parsed_url'].geturl()
208− duplicated['parsed_url'] = res['parsed_url']
209−
210− # if there is no duplicate found, append result
211− else:
212− res['score'] = score
213−
214− results.append(res)
215−
216− results = sorted(results, key=itemgetter('score'), reverse=True)
217−
218− # pass 2 : group results by category and template
219− gresults = []
220− categoryPositions = {}
221−
222− for i, res in enumerate(results):
223− # FIXME : handle more than one category per engine
224− category = engines[res['engine']].categories[0] + ':' + ''\
225− if 'template' not in res\
226− else res['template']
227−
228− current = None if category not in categoryPositions\
229− else categoryPositions[category]
230−
231− # group with previous results using the same category
232− # if the group can accept more result and is not too far
233− # from the current position
234− if current is not None and (current['count'] > 0)\
235− and (len(gresults) - current['index'] < 20):
236− # group with the previous results using
237− # the same category with this one
238− index = current['index']
239− gresults.insert(index, res)
240−
241− # update every index after the current one
242− # (including the current one)
243− for k in categoryPositions:
244− v = categoryPositions[k]['index']
245− if v >= index:
246− categoryPositions[k]['index'] = v + 1
247−
248− # update this category
249− current['count'] -= 1
250−
251− else:
252− # same category
253− gresults.append(res)
254−
255− # update categoryIndex
256− categoryPositions[category] = {'index': len(gresults), 'count': 8}
257−
258− # return gresults
259− return gresults
260−
261−
262−def merge_two_infoboxes(infobox1, infobox2):
263− if 'urls' in infobox2:
264− urls1 = infobox1.get('urls', None)
265− if urls1 is None:
266− urls1 = []
267− infobox1.set('urls', urls1)
268−
269− urlSet = set()
270− for url in infobox1.get('urls', []):
271− urlSet.add(url.get('url', None))
272−
273− for url in infobox2.get('urls', []):
274− if url.get('url', None) not in urlSet:
275− urls1.append(url)
276−
277− if 'attributes' in infobox2:
278− attributes1 = infobox1.get('attributes', None)
279− if attributes1 is None:
280− attributes1 = []
281− infobox1.set('attributes', attributes1)
282−
283− attributeSet = set()
284− for attribute in infobox1.get('attributes', []):
285− if attribute.get('label', None) not in attributeSet:
286− attributeSet.add(attribute.get('label', None))
287−
288− for attribute in infobox2.get('attributes', []):
289− attributes1.append(attribute)
290−
291− if 'content' in infobox2:
292− content1 = infobox1.get('content', None)
293− content2 = infobox2.get('content', '')
294− if content1 is not None:
295− if content_result_len(content2) > content_result_len(content1):
296− infobox1['content'] = content2
297− else:
298− infobox1.set('content', content2)
299−
300−
301−def merge_infoboxes(infoboxes):
302− results = []
303− infoboxes_id = {}
304− for infobox in infoboxes:
305− add_infobox = True
306− infobox_id = infobox.get('id', None)
307− if infobox_id is not None:
308− existingIndex = infoboxes_id.get(infobox_id, None)
309− if existingIndex is not None:
310− merge_two_infoboxes(results[existingIndex], infobox)
311− add_infobox = False
312−
313− if add_infobox:
314− results.append(infobox)
315− infoboxes_id[infobox_id] = len(results) - 1
316−
317− return results
318−
319−
320121class Search(object):
321122
322123 """Search information container"""
@@ -334,10 +135,7 @@ class Search(object):
334135 # set blocked engines
335136 self.blocked_engines = get_blocked_engines(engines, request.cookies)
336137
337− self.results = []
338− self.suggestions = set()
339− self.answers = set()
340− self.infoboxes = []
138+ self.result_container = ResultContainer()
341139 self.request_data = {}
342140
343141 # set specific language if set
@@ -449,8 +247,6 @@ class Search(object):
449247
450248 # init vars
451249 requests = []
452− results_queue = Queue()
453− results = {}
454250
455251 # increase number of searches
456252 number_of_searches += 1
@@ -504,9 +300,9 @@ class Search(object):
504300 # create a callback wrapper for the search engine results
505301 callback = make_callback(
506302 selected_engine['name'],
507− results_queue,
508303 engine.response,
509− request_params)
304+ request_params,
305+ self.result_container)
510306
511307 # create dictionary which contain all
512308 # informations about the request
@@ -539,42 +335,5 @@ class Search(object):
539335 # send all search-request
540336 threaded_requests(requests)
541337
542− while not results_queue.empty():
543− engine_name, engine_results = results_queue.get_nowait()
544−
545− # TODO type checks
546− [self.suggestions.add(x['suggestion'])
547− for x in list(engine_results)
548− if 'suggestion' in x
549− and engine_results.remove(x) is None]
550−
551− [self.answers.add(x['answer'])
552− for x in list(engine_results)
553− if 'answer' in x
554− and engine_results.remove(x) is None]
555−
556− self.infoboxes.extend(x for x in list(engine_results)
557− if 'infobox' in x
558− and engine_results.remove(x) is None)
559−
560− results[engine_name] = engine_results
561−
562− # update engine-specific stats
563− for engine_name, engine_results in results.items():
564− engines[engine_name].stats['search_count'] += 1
565− engines[engine_name].stats['result_count'] += len(engine_results)
566−
567− # score results and remove duplications
568− self.results = score_results(results)
569−
570− # merge infoboxes according to their ids
571− self.infoboxes = merge_infoboxes(self.infoboxes)
572−
573− # update engine stats, using calculated score
574− for result in self.results:
575− for res_engine in result['engines']:
576− engines[result['engine']]\
577− .stats['score_count'] += result['score']
578−
579338 # return results, suggestions, answers and infoboxes
580339 return self