@@ -1,3 +1,20 @@
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
15+(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
1import grequests
18import grequests
2from itertools import izip_longest, chain
19from itertools import izip_longest, chain
3from datetime import datetime
20from datetime import datetime
@@ -9,45 +26,65 @@ from searx.engines import (
9from searx.languages import language_codes
26from searx.languages import language_codes
10from searx.utils import gen_useragent
27from searx.utils import gen_useragent
12number_of_searches = 0
30number_of_searches = 0
33+# get default reqest parameter
15def default_request_params():
34def default_request_params():
17 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
36 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
39+# create a callback wrapper for the search engine results
20def make_callback(engine_name, results, suggestions, callback, params):
40def make_callback(engine_name, results, suggestions, callback, params):
21 # creating a callback wrapper for the search engine results
42 # creating a callback wrapper for the search engine results
22 def process_callback(response, **kwargs):
43 def process_callback(response, **kwargs):
23 cb_res = []
44 cb_res = []
24 response.search_params = params
45 response.search_params = params
47+ # update stats with current page-load-time
25 engines[engine_name].stats['page_load_time'] += \
48 engines[engine_name].stats['page_load_time'] += \
26 (datetime.now() - params['started']).total_seconds()
49 (datetime.now() - params['started']).total_seconds()
28 search_results = callback(response)
52 search_results = callback(response)
29 except Exception, e:
53 except Exception, e:
54+ # increase errors stats
30 engines[engine_name].stats['errors'] += 1
55 engines[engine_name].stats['errors'] += 1
31 results[engine_name] = cb_res
56 results[engine_name] = cb_res
58+ # print engine name and specific error message
32 print '[E] Error with engine "{0}":\n\t{1}'.format(
59 print '[E] Error with engine "{0}":\n\t{1}'.format(
33 engine_name, str(e))
60 engine_name, str(e))
35 for result in search_results:
63 for result in search_results:
36 result['engine'] = engine_name
64 result['engine'] = engine_name
66+ # if it is a suggestion, add it to list of suggestions
37 if 'suggestion' in result:
67 if 'suggestion' in result:
38 # TODO type checks
68 # TODO type checks
39 suggestions.add(result['suggestion'])
69 suggestions.add(result['suggestion'])
41 cb_res.append(result)
73 cb_res.append(result)
42 results[engine_name] = cb_res
75 results[engine_name] = cb_res
43 return process_callback
77 return process_callback
80+# score results and remove duplications
46def score_results(results):
81def score_results(results):
82+ # calculate scoring parameters
47 flat_res = filter(
83 flat_res = filter(
48 None, chain.from_iterable(izip_longest(*results.values())))
84 None, chain.from_iterable(izip_longest(*results.values())))
49 flat_len = len(flat_res)
85 flat_len = len(flat_res)
50 engines_len = len(results)
86 engines_len = len(results)
51 results = []
88 results = []
53 # pass 1: deduplication + scoring
90 # pass 1: deduplication + scoring
@@ -63,34 +100,53 @@ def score_results(results):
63 res['engines'] = [res['engine']]
100 res['engines'] = [res['engine']]
64 weight = 1.0
101 weight = 1.0
103+ # get weight of this engine if possible
66 if hasattr(engines[res['engine']], 'weight'):
104 if hasattr(engines[res['engine']], 'weight'):
67 weight = float(engines[res['engine']].weight)
105 weight = float(engines[res['engine']].weight)
107+ # calculate score for that engine
69 score = int((flat_len - i) / engines_len) * weight + 1
108 score = int((flat_len - i) / engines_len) * weight + 1
70 duplicated = False
110 duplicated = False
112+ # check for duplicates
72 for new_res in results:
113 for new_res in results:
114+ # remove / from the end of the url if required
73 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
115 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
74 p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
116 p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
118+ # check if that result is a duplicate
75 if res['host'] == new_res['host'] and\
119 if res['host'] == new_res['host'] and\
76 unquote(p1) == unquote(p2) and\
120 unquote(p1) == unquote(p2) and\
77 res['parsed_url'].query == new_res['parsed_url'].query and\
121 res['parsed_url'].query == new_res['parsed_url'].query and\
78 res.get('template') == new_res.get('template'):
122 res.get('template') == new_res.get('template'):
79 duplicated = new_res
123 duplicated = new_res
126+ # merge duplicates together
81 if duplicated:
127 if duplicated:
128+ # using content with more text
82 if res.get('content') > duplicated.get('content'):
129 if res.get('content') > duplicated.get('content'):
83 duplicated['content'] = res['content']
130 duplicated['content'] = res['content']
132+ # increase result-score
84 duplicated['score'] += score
133 duplicated['score'] += score
135+ # add engine to list of result-engines
85 duplicated['engines'].append(res['engine'])
136 duplicated['engines'].append(res['engine'])
138+ # using https if possible
86 if duplicated['parsed_url'].scheme == 'https':
139 if duplicated['parsed_url'].scheme == 'https':
88 elif res['parsed_url'].scheme == 'https':
141 elif res['parsed_url'].scheme == 'https':
89 duplicated['url'] = res['parsed_url'].geturl()
142 duplicated['url'] = res['parsed_url'].geturl()
90 duplicated['parsed_url'] = res['parsed_url']
143 duplicated['parsed_url'] = res['parsed_url']
145+ # if there is no duplicate found, append result
92 res['score'] = score
147 res['score'] = score
93 results.append(res)
148 results.append(res)
94 results = sorted(results, key=itemgetter('score'), reverse=True)
150 results = sorted(results, key=itemgetter('score'), reverse=True)
96 # pass 2 : group results by category and template
152 # pass 2 : group results by category and template
@@ -99,7 +155,7 @@ def score_results(results):
100 for i, res in enumerate(results):
156 for i, res in enumerate(results):
101 # FIXME : handle more than one category per engine
157 # FIXME : handle more than one category per engine
102− category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']
158+ category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']
104 current = None if category not in categoryPositions else categoryPositions[category]
160 current = None if category not in categoryPositions else categoryPositions[category]
@@ -134,6 +190,7 @@ class Search(object):
134 """Search information container"""
190 """Search information container"""
136 def __init__(self, request):
192 def __init__(self, request):
137 super(Search, self).__init__()
194 super(Search, self).__init__()
138 self.query = None
195 self.query = None
139 self.engines = []
196 self.engines = []
@@ -141,18 +198,23 @@ class Search(object):
141 self.paging = False
198 self.paging = False
142 self.pageno = 1
199 self.pageno = 1
143 self.lang = 'all'
200 self.lang = 'all'
202+ # set blocked engines
144 if request.cookies.get('blocked_engines'):
203 if request.cookies.get('blocked_engines'):
145 self.blocked_engines = request.cookies['blocked_engines'].split(',') # noqa
204 self.blocked_engines = request.cookies['blocked_engines'].split(',') # noqa
147 self.blocked_engines = []
206 self.blocked_engines = []
148 self.results = []
208 self.results = []
149 self.suggestions = []
209 self.suggestions = []
150 self.request_data = {}
210 self.request_data = {}
212+ # set specific language if set
152 if request.cookies.get('language')\
213 if request.cookies.get('language')\
153 and request.cookies['language'] in (x[0] for x in language_codes):
214 and request.cookies['language'] in (x[0] for x in language_codes):
154 self.lang = request.cookies['language']
215 self.lang = request.cookies['language']
217+ # set request method
156 if request.method == 'POST':
218 if request.method == 'POST':
157 self.request_data = request.form
219 self.request_data = request.form
@@ -162,51 +224,72 @@ class Search(object):
162 if not self.request_data.get('q'):
224 if not self.request_data.get('q'):
163 raise Exception('noquery')
225 raise Exception('noquery')
165 self.query = self.request_data['q']
228 self.query = self.request_data['q']
167 pageno_param = self.request_data.get('pageno', '1')
231 pageno_param = self.request_data.get('pageno', '1')
168 if not pageno_param.isdigit() or int(pageno_param) < 1:
232 if not pageno_param.isdigit() or int(pageno_param) < 1:
169 raise Exception('wrong pagenumber')
233 raise Exception('wrong pagenumber')
171 self.pageno = int(pageno_param)
235 self.pageno = int(pageno_param)
237+ # parse query, if tags are set, which change the serch engine or search-language
173 self.parse_query()
238 self.parse_query()
175 self.categories = []
240 self.categories = []
242+ # if engines are calculated from query, set categories by using that informations
177 if self.engines:
243 if self.engines:
178 self.categories = list(set(engine['category']
244 self.categories = list(set(engine['category']
179 for engine in self.engines))
245 for engine in self.engines))
247+ # otherwise, using defined categories to calculate which engines should be used
249+ # set used categories
181 for pd_name, pd in self.request_data.items():
250 for pd_name, pd in self.request_data.items():
182 if pd_name.startswith('category_'):
251 if pd_name.startswith('category_'):
183 category = pd_name[9:]
252 category = pd_name[9:]
253+ # if category is not found in list, skip
184 if not category in categories:
254 if not category in categories:
257+ # add category to list
186 self.categories.append(category)
258 self.categories.append(category)
260+ # if no category is specified for this search, using user-defined default-configuration which (is stored in cookie)
187 if not self.categories:
261 if not self.categories:
188 cookie_categories = request.cookies.get('categories', '')
262 cookie_categories = request.cookies.get('categories', '')
189 cookie_categories = cookie_categories.split(',')
263 cookie_categories = cookie_categories.split(',')
190 for ccateg in cookie_categories:
264 for ccateg in cookie_categories:
191 if ccateg in categories:
265 if ccateg in categories:
192 self.categories.append(ccateg)
266 self.categories.append(ccateg)
268+ # if still no category is specified, using general as default-category
193 if not self.categories:
269 if not self.categories:
194 self.categories = ['general']
270 self.categories = ['general']
272+ # using all engines for that search, which are declared under the specific categories
196 for categ in self.categories:
273 for categ in self.categories:
197 self.engines.extend({'category': categ,
274 self.engines.extend({'category': categ,
198 'name': x.name}
275 'name': x.name}
199 for x in categories[categ]
276 for x in categories[categ]
200 if not x.name in self.blocked_engines)
277 if not x.name in self.blocked_engines)
279+ # parse query, if tags are set, which change the serch engine or search-language
202 def parse_query(self):
280 def parse_query(self):
203 query_parts = self.query.split()
281 query_parts = self.query.split()
204 modified = False
282 modified = False
284+ # check if language-prefix is set
205 if query_parts[0].startswith(':'):
285 if query_parts[0].startswith(':'):
206 lang = query_parts[0][1:].lower()
286 lang = query_parts[0][1:].lower()
288+ # check if any language-code is equal with declared language-codes
208 for lc in language_codes:
289 for lc in language_codes:
209 lang_id, lang_name, country = map(str.lower, lc)
290 lang_id, lang_name, country = map(str.lower, lc)
292+ # if correct language-code is found, set it as new search-language
210 if lang == lang_id\
293 if lang == lang_id\
211 or lang_id.startswith(lang)\
294 or lang_id.startswith(lang)\
212 or lang == lang_name\
295 or lang == lang_name\
@@ -215,56 +298,78 @@ class Search(object):
215 modified = True
298 modified = True
301+ # check if category/engine prefix is set
218 elif query_parts[0].startswith('!'):
302 elif query_parts[0].startswith('!'):
219 prefix = query_parts[0][1:].replace('_', ' ')
303 prefix = query_parts[0][1:].replace('_', ' ')
305+ # check if prefix is equal with engine shortcut
221 if prefix in engine_shortcuts\
306 if prefix in engine_shortcuts\
222 and not engine_shortcuts[prefix] in self.blocked_engines:
307 and not engine_shortcuts[prefix] in self.blocked_engines:
223 modified = True
308 modified = True
224 self.engines.append({'category': 'none',
309 self.engines.append({'category': 'none',
225 'name': engine_shortcuts[prefix]})
310 'name': engine_shortcuts[prefix]})
312+ # check if prefix is equal with engine name
226 elif prefix in engines\
313 elif prefix in engines\
227 and not prefix in self.blocked_engines:
314 and not prefix in self.blocked_engines:
228 modified = True
315 modified = True
229 self.engines.append({'category': 'none',
316 self.engines.append({'category': 'none',
230 'name': prefix})
317 'name': prefix})
319+ # check if prefix is equal with categorie name
231 elif prefix in categories:
320 elif prefix in categories:
232 modified = True
321 modified = True
322+ # using all engines for that search, which are declared under that categorie name
233 self.engines.extend({'category': prefix,
323 self.engines.extend({'category': prefix,
234 'name': engine.name}
324 'name': engine.name}
235 for engine in categories[prefix]
325 for engine in categories[prefix]
236 if not engine in self.blocked_engines)
326 if not engine in self.blocked_engines)
328+ # if language, category or engine were specificed in this query, search for more tags which does the same
237 if modified:
329 if modified:
238 self.query = self.query.replace(query_parts[0], '', 1).strip()
330 self.query = self.query.replace(query_parts[0], '', 1).strip()
239 self.parse_query()
331 self.parse_query()
241 def search(self, request):
334 def search(self, request):
242 global number_of_searches
335 global number_of_searches
243 requests = []
338 requests = []
244 results = {}
339 results = {}
245 suggestions = set()
340 suggestions = set()
342+ # increase number of searches
246 number_of_searches += 1
343 number_of_searches += 1
345+ # set default useragent
247 #user_agent = request.headers.get('User-Agent', '')
346 #user_agent = request.headers.get('User-Agent', '')
248 user_agent = gen_useragent()
347 user_agent = gen_useragent()
349+ # start search-reqest for all selected engines
250 for selected_engine in self.engines:
350 for selected_engine in self.engines:
251 if selected_engine['name'] not in engines:
351 if selected_engine['name'] not in engines:
254 engine = engines[selected_engine['name']]
354 engine = engines[selected_engine['name']]
356+ # if paging is not supported, skip
256 if self.pageno > 1 and not engine.paging:
357 if self.pageno > 1 and not engine.paging:
360+ # if search-language is set and engine does not provide language-support, skip
259 if self.lang != 'all' and not engine.language_support:
361 if self.lang != 'all' and not engine.language_support:
364+ # set default request parameters
262 request_params = default_request_params()
365 request_params = default_request_params()
263 request_params['headers']['User-Agent'] = user_agent
366 request_params['headers']['User-Agent'] = user_agent
264 request_params['category'] = selected_engine['category']
367 request_params['category'] = selected_engine['category']
265 request_params['started'] = datetime.now()
368 request_params['started'] = datetime.now()
266 request_params['pageno'] = self.pageno
369 request_params['pageno'] = self.pageno
267 request_params['language'] = self.lang
370 request_params['language'] = self.lang
372+ # update request parameters dependent on search-engine (contained in engines folder)
268 request_params = engine.request(self.query.encode('utf-8'),
373 request_params = engine.request(self.query.encode('utf-8'),
269 request_params)
374 request_params)
@@ -272,6 +377,7 @@ class Search(object):
272 # TODO add support of offline engines
377 # TODO add support of offline engines
380+ # create a callback wrapper for the search engine results
275 callback = make_callback(
381 callback = make_callback(
276 selected_engine['name'],
382 selected_engine['name'],
@@ -280,6 +386,7 @@ class Search(object):
280 request_params
386 request_params
389+ # create dictionary which contain all informations about the request
283 request_args = dict(
390 request_args = dict(
284 headers=request_params['headers'],
391 headers=request_params['headers'],
285 hooks=dict(response=callback),
392 hooks=dict(response=callback),
@@ -287,6 +394,7 @@ class Search(object):
287 timeout=engine.timeout
394 timeout=engine.timeout
397+ # specific type of request (GET or POST)
290 if request_params['method'] == 'GET':
398 if request_params['method'] == 'GET':
291 req = grequests.get
399 req = grequests.get
@@ -297,17 +405,25 @@ class Search(object):
297 if not request_params['url']:
405 if not request_params['url']:
408+ # append request to list
300 requests.append(req(request_params['url'], **request_args))
409 requests.append(req(request_params['url'], **request_args))
411+ # send all search-request
301 grequests.map(requests)
412 grequests.map(requests)
414+ # update engine-specific stats
302 for engine_name, engine_results in results.items():
415 for engine_name, engine_results in results.items():
303 engines[engine_name].stats['search_count'] += 1
416 engines[engine_name].stats['search_count'] += 1
304 engines[engine_name].stats['result_count'] += len(engine_results)
417 engines[engine_name].stats['result_count'] += len(engine_results)
419+ # score results and remove duplications
306 results = score_results(results)
420 results = score_results(results)
422+ # update engine stats, using calculated score
308 for result in results:
423 for result in results:
309 for res_engine in result['engines']:
424 for res_engine in result['engines']:
310 engines[result['engine']]\
425 engines[result['engine']]\
311 .stats['score_count'] += result['score']
426 .stats['score_count'] += result['score']
428+ # return results and suggestions
313 return results, suggestions
429 return results, suggestions