@@ -1,3 +1,20 @@
1+'''
2+searx is free software: you can redistribute it and/or modify
3+it under the terms of the GNU Affero General Public License as published by
4+the Free Software Foundation, either version 3 of the License, or
5+(at your option) any later version.
6+
7+searx is distributed in the hope that it will be useful,
8+but WITHOUT ANY WARRANTY; without even the implied warranty of
9+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10+GNU Affero General Public License for more details.
11+
12+You should have received a copy of the GNU Affero General Public License
13+along with searx. If not, see < http://www.gnu.org/licenses/ >.
14+
15+(C) 2013- by Adam Tauber, <asciimoo@gmail.com>
16+'''
17+
118import grequests
219from itertools import izip_longest, chain
320from datetime import datetime
@@ -9,45 +26,65 @@ from searx.engines import (
926from searx.languages import language_codes
1027from searx.utils import gen_useragent
1128
29+
1230number_of_searches = 0
1331
1432
33+# get default reqest parameter
1534def default_request_params():
1635 return {
1736 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}}
1837
1938
39+# create a callback wrapper for the search engine results
2040def make_callback(engine_name, results, suggestions, callback, params):
41+
2142 # creating a callback wrapper for the search engine results
2243 def process_callback(response, **kwargs):
2344 cb_res = []
2445 response.search_params = params
46+
47+ # update stats with current page-load-time
2548 engines[engine_name].stats['page_load_time'] += \
2649 (datetime.now() - params['started']).total_seconds()
50+
2751 try:
2852 search_results = callback(response)
2953 except Exception, e:
54+ # increase errors stats
3055 engines[engine_name].stats['errors'] += 1
3156 results[engine_name] = cb_res
57+
58+ # print engine name and specific error message
3259 print '[E] Error with engine "{0}":\n\t{1}'.format(
3360 engine_name, str(e))
3461 return
62+
3563 for result in search_results:
3664 result['engine'] = engine_name
65+
66+ # if it is a suggestion, add it to list of suggestions
3767 if 'suggestion' in result:
3868 # TODO type checks
3969 suggestions.add(result['suggestion'])
4070 continue
71+
72+ # append result
4173 cb_res.append(result)
74+
4275 results[engine_name] = cb_res
76+
4377 return process_callback
4478
4579
80+# score results and remove duplications
4681def score_results(results):
82+ # calculate scoring parameters
4783 flat_res = filter(
4884 None, chain.from_iterable(izip_longest(*results.values())))
4985 flat_len = len(flat_res)
5086 engines_len = len(results)
87+
5188 results = []
5289
5390 # pass 1: deduplication + scoring
@@ -63,34 +100,53 @@ def score_results(results):
63100 res['engines'] = [res['engine']]
64101 weight = 1.0
65102
103+ # get weight of this engine if possible
66104 if hasattr(engines[res['engine']], 'weight'):
67105 weight = float(engines[res['engine']].weight)
68106
107+ # calculate score for that engine
69108 score = int((flat_len - i) / engines_len) * weight + 1
109+
70110 duplicated = False
71111
112+ # check for duplicates
72113 for new_res in results:
114+ # remove / from the end of the url if required
73115 p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa
74116 p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa
117+
118+ # check if that result is a duplicate
75119 if res['host'] == new_res['host'] and\
76120 unquote(p1) == unquote(p2) and\
77121 res['parsed_url'].query == new_res['parsed_url'].query and\
78122 res.get('template') == new_res.get('template'):
79123 duplicated = new_res
80124 break
125+
126+ # merge duplicates together
81127 if duplicated:
128+ # using content with more text
82129 if res.get('content') > duplicated.get('content'):
83130 duplicated['content'] = res['content']
131+
132+ # increase result-score
84133 duplicated['score'] += score
134+
135+ # add engine to list of result-engines
85136 duplicated['engines'].append(res['engine'])
137+
138+ # using https if possible
86139 if duplicated['parsed_url'].scheme == 'https':
87140 continue
88141 elif res['parsed_url'].scheme == 'https':
89142 duplicated['url'] = res['parsed_url'].geturl()
90143 duplicated['parsed_url'] = res['parsed_url']
144+
145+ # if there is no duplicate found, append result
91146 else:
92147 res['score'] = score
93148 results.append(res)
149+
94150 results = sorted(results, key=itemgetter('score'), reverse=True)
95151
96152 # pass 2 : group results by category and template
@@ -99,7 +155,7 @@ def score_results(results):
99155
100156 for i, res in enumerate(results):
101157 # FIXME : handle more than one category per engine
102− category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']
158+ category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']
103159
104160 current = None if category not in categoryPositions else categoryPositions[category]
105161
@@ -134,6 +190,7 @@ class Search(object):
134190 """Search information container"""
135191
136192 def __init__(self, request):
193+ # init vars
137194 super(Search, self).__init__()
138195 self.query = None
139196 self.engines = []
@@ -141,18 +198,23 @@ class Search(object):
141198 self.paging = False
142199 self.pageno = 1
143200 self.lang = 'all'
201+
202+ # set blocked engines
144203 if request.cookies.get('blocked_engines'):
145204 self.blocked_engines = request.cookies['blocked_engines'].split(',') # noqa
146205 else:
147206 self.blocked_engines = []
207+
148208 self.results = []
149209 self.suggestions = []
150210 self.request_data = {}
151211
212+ # set specific language if set
152213 if request.cookies.get('language')\
153214 and request.cookies['language'] in (x[0] for x in language_codes):
154215 self.lang = request.cookies['language']
155216
217+ # set request method
156218 if request.method == 'POST':
157219 self.request_data = request.form
158220 else:
@@ -162,51 +224,72 @@ class Search(object):
162224 if not self.request_data.get('q'):
163225 raise Exception('noquery')
164226
227+ # set query
165228 self.query = self.request_data['q']
166229
230+ # set pagenumber
167231 pageno_param = self.request_data.get('pageno', '1')
168232 if not pageno_param.isdigit() or int(pageno_param) < 1:
169233 raise Exception('wrong pagenumber')
170234
171235 self.pageno = int(pageno_param)
172236
237+ # parse query, if tags are set, which change the serch engine or search-language
173238 self.parse_query()
174239
175240 self.categories = []
176241
242+ # if engines are calculated from query, set categories by using that informations
177243 if self.engines:
178244 self.categories = list(set(engine['category']
179245 for engine in self.engines))
246+
247+ # otherwise, using defined categories to calculate which engines should be used
180248 else:
249+ # set used categories
181250 for pd_name, pd in self.request_data.items():
182251 if pd_name.startswith('category_'):
183252 category = pd_name[9:]
253+ # if category is not found in list, skip
184254 if not category in categories:
185255 continue
256+
257+ # add category to list
186258 self.categories.append(category)
259+
260+ # if no category is specified for this search, using user-defined default-configuration which (is stored in cookie)
187261 if not self.categories:
188262 cookie_categories = request.cookies.get('categories', '')
189263 cookie_categories = cookie_categories.split(',')
190264 for ccateg in cookie_categories:
191265 if ccateg in categories:
192266 self.categories.append(ccateg)
267+
268+ # if still no category is specified, using general as default-category
193269 if not self.categories:
194270 self.categories = ['general']
195271
272+ # using all engines for that search, which are declared under the specific categories
196273 for categ in self.categories:
197274 self.engines.extend({'category': categ,
198275 'name': x.name}
199276 for x in categories[categ]
200277 if not x.name in self.blocked_engines)
201278
279+ # parse query, if tags are set, which change the serch engine or search-language
202280 def parse_query(self):
203281 query_parts = self.query.split()
204282 modified = False
283+
284+ # check if language-prefix is set
205285 if query_parts[0].startswith(':'):
206286 lang = query_parts[0][1:].lower()
207287
288+ # check if any language-code is equal with declared language-codes
208289 for lc in language_codes:
209290 lang_id, lang_name, country = map(str.lower, lc)
291+
292+ # if correct language-code is found, set it as new search-language
210293 if lang == lang_id\
211294 or lang_id.startswith(lang)\
212295 or lang == lang_name\
@@ -215,56 +298,78 @@ class Search(object):
215298 modified = True
216299 break
217300
301+ # check if category/engine prefix is set
218302 elif query_parts[0].startswith('!'):
219303 prefix = query_parts[0][1:].replace('_', ' ')
220304
305+ # check if prefix is equal with engine shortcut
221306 if prefix in engine_shortcuts\
222307 and not engine_shortcuts[prefix] in self.blocked_engines:
223308 modified = True
224309 self.engines.append({'category': 'none',
225310 'name': engine_shortcuts[prefix]})
311+
312+ # check if prefix is equal with engine name
226313 elif prefix in engines\
227314 and not prefix in self.blocked_engines:
228315 modified = True
229316 self.engines.append({'category': 'none',
230317 'name': prefix})
318+
319+ # check if prefix is equal with categorie name
231320 elif prefix in categories:
232321 modified = True
322+ # using all engines for that search, which are declared under that categorie name
233323 self.engines.extend({'category': prefix,
234324 'name': engine.name}
235325 for engine in categories[prefix]
236326 if not engine in self.blocked_engines)
327+
328+ # if language, category or engine were specificed in this query, search for more tags which does the same
237329 if modified:
238330 self.query = self.query.replace(query_parts[0], '', 1).strip()
239331 self.parse_query()
240332
333+ # do search-request
241334 def search(self, request):
242335 global number_of_searches
336+
337+ # init vars
243338 requests = []
244339 results = {}
245340 suggestions = set()
341+
342+ # increase number of searches
246343 number_of_searches += 1
344+
345+ # set default useragent
247346 #user_agent = request.headers.get('User-Agent', '')
248347 user_agent = gen_useragent()
249348
349+ # start search-reqest for all selected engines
250350 for selected_engine in self.engines:
251351 if selected_engine['name'] not in engines:
252352 continue
253353
254354 engine = engines[selected_engine['name']]
255355
356+ # if paging is not supported, skip
256357 if self.pageno > 1 and not engine.paging:
257358 continue
258359
360+ # if search-language is set and engine does not provide language-support, skip
259361 if self.lang != 'all' and not engine.language_support:
260362 continue
261363
364+ # set default request parameters
262365 request_params = default_request_params()
263366 request_params['headers']['User-Agent'] = user_agent
264367 request_params['category'] = selected_engine['category']
265368 request_params['started'] = datetime.now()
266369 request_params['pageno'] = self.pageno
267370 request_params['language'] = self.lang
371+
372+ # update request parameters dependent on search-engine (contained in engines folder)
268373 request_params = engine.request(self.query.encode('utf-8'),
269374 request_params)
270375
@@ -272,6 +377,7 @@ class Search(object):
272377 # TODO add support of offline engines
273378 pass
274379
380+ # create a callback wrapper for the search engine results
275381 callback = make_callback(
276382 selected_engine['name'],
277383 results,
@@ -280,6 +386,7 @@ class Search(object):
280386 request_params
281387 )
282388
389+ # create dictionary which contain all informations about the request
283390 request_args = dict(
284391 headers=request_params['headers'],
285392 hooks=dict(response=callback),
@@ -287,6 +394,7 @@ class Search(object):
287394 timeout=engine.timeout
288395 )
289396
397+ # specific type of request (GET or POST)
290398 if request_params['method'] == 'GET':
291399 req = grequests.get
292400 else:
@@ -297,17 +405,25 @@ class Search(object):
297405 if not request_params['url']:
298406 continue
299407
408+ # append request to list
300409 requests.append(req(request_params['url'], **request_args))
410+
411+ # send all search-request
301412 grequests.map(requests)
413+
414+ # update engine-specific stats
302415 for engine_name, engine_results in results.items():
303416 engines[engine_name].stats['search_count'] += 1
304417 engines[engine_name].stats['result_count'] += len(engine_results)
305418
419+ # score results and remove duplications
306420 results = score_results(results)
307421
422+ # update engine stats, using calculated score
308423 for result in results:
309424 for res_engine in result['engines']:
310425 engines[result['engine']]\
311426 .stats['score_count'] += result['score']
312427
428+ # return results and suggestions
313429 return results, suggestions