5740cfb · searx · git.iko.soy

git.iko.soy/searx

[fix] pep8 part II.

commit5740cfbf1cb468af74d0e7e1c9358ce702eb4f59 authorAdam Tauber <asciimoo@gmail.com> dateSun 19 Oct 2014, 12:41 (12 years ago) parentb0fd71b

6 files changed+121−80

compare with master

unifiedsplit

searx/__init__.py+2−1 searx/engines/__init__.py+6−5 searx/query.py+23−20 searx/search.py+67−37 searx/utils.py+5−4 searx/webapp.py+18−13

searx/__init__.py+2−1

@@ -28,7 +28,8 @@ except:

2828searx_dir = abspath(dirname(__file__))

2929engine_dir = dirname(realpath(__file__))

3030

31−# if possible set path to settings using the enviroment variable SEARX_SETTINGS_PATH

31+# if possible set path to settings using the

32+# enviroment variable SEARX_SETTINGS_PATH

3233if 'SEARX_SETTINGS_PATH' in environ:

3334 settings_path = environ['SEARX_SETTINGS_PATH']

3435# otherwise using default path

searx/engines/__init__.py+6−5

@@ -41,7 +41,7 @@ def load_module(filename):

4141 module.name = modname

4242 return module

4343

44−if not 'engines' in settings or not settings['engines']:

44+if 'engines' not in settings or not settings['engines']:

4545 print '[E] Error no engines found. Edit your settings.yml'

4646 exit(2)

4747

@@ -68,15 +68,15 @@ for engine_data in settings['engines']:

6868 engine.categories = ['general']

6969

7070 if not hasattr(engine, 'language_support'):

71− #engine.language_support = False

71+ # engine.language_support = False

7272 engine.language_support = True

7373

7474 if not hasattr(engine, 'timeout'):

75− #engine.language_support = False

75+ # engine.language_support = False

7676 engine.timeout = settings['server']['request_timeout']

7777

7878 if not hasattr(engine, 'shortcut'):

79− #engine.shortcut = '''

79+ # engine.shortcut = '''

8080 engine.shortcut = ''

8181

8282 # checking required variables

@@ -161,7 +161,8 @@ def get_engines_stats():

161161

162162 for engine in scores_per_result:

163163 if max_score_per_result:

164− engine['percentage'] = int(engine['avg'] / max_score_per_result * 100)

164+ engine['percentage'] = int(engine['avg']

165+ / max_score_per_result * 100)

165166 else:

166167 engine['percentage'] = 0

167168

searx/query.py+23−20

@@ -31,30 +31,31 @@ class Query(object):

3131 def __init__(self, query, blocked_engines):

3232 self.query = query

3333 self.blocked_engines = []

34−

34+

3535 if blocked_engines:

3636 self.blocked_engines = blocked_engines

37−

37+

3838 self.query_parts = []

3939 self.engines = []

4040 self.languages = []

41−

42− # parse query, if tags are set, which change the serch engine or search-language

41+

42+ # parse query, if tags are set, which

43+ # change the serch engine or search-language

4344 def parse_query(self):

4445 self.query_parts = []

45−

46+

4647 # split query, including whitespaces

4748 raw_query_parts = re.split(r'(\s+)', self.query)

48−

49+

4950 parse_next = True

50−

51+

5152 for query_part in raw_query_parts:

5253 if not parse_next:

5354 self.query_parts[-1] += query_part

5455 continue

55−

56+

5657 parse_next = False

57−

58+

5859 # part does only contain spaces, skip

5960 if query_part.isspace()\

6061 or query_part == '':

@@ -62,15 +63,17 @@ class Query(object):

6263 self.query_parts.append(query_part)

6364 continue

6465

65− # this force a language

66+ # this force a language

6667 if query_part[0] == ':':

6768 lang = query_part[1:].lower()

6869

69− # check if any language-code is equal with declared language-codes

70+ # check if any language-code is equal with

71+ # declared language-codes

7072 for lc in language_codes:

7173 lang_id, lang_name, country = map(str.lower, lc)

7274

73− # if correct language-code is found, set it as new search-language

75+ # if correct language-code is found

76+ # set it as new search-language

7477 if lang == lang_id\

7578 or lang_id.startswith(lang)\

7679 or lang == lang_name\

@@ -89,23 +92,24 @@ class Query(object):

8992 parse_next = True

9093 self.engines.append({'category': 'none',

9194 'name': engine_shortcuts[prefix]})

92−

95+

9396 # check if prefix is equal with engine name

9497 elif prefix in engines\

95− and not prefix in self.blocked_engines:

98+ and prefix not in self.blocked_engines:

9699 parse_next = True

97100 self.engines.append({'category': 'none',

98101 'name': prefix})

99102

100103 # check if prefix is equal with categorie name

101104 elif prefix in categories:

102− # using all engines for that search, which are declared under that categorie name

105+ # using all engines for that search, which

106+ # are declared under that categorie name

103107 parse_next = True

104108 self.engines.extend({'category': prefix,

105109 'name': engine.name}

106110 for engine in categories[prefix]

107− if not engine in self.blocked_engines)

108−

111+ if engine not in self.blocked_engines)

112+

109113 # append query part to query_part list

110114 self.query_parts.append(query_part)

111115

@@ -114,14 +118,13 @@ class Query(object):

114118 self.query_parts[-1] = search_query

115119 else:

116120 self.query_parts.append(search_query)

117−

121+

118122 def getSearchQuery(self):

119123 if len(self.query_parts):

120124 return self.query_parts[-1]

121125 else:

122126 return ''

123−

127+

124128 def getFullQuery(self):

125129 # get full querry including whitespaces

126130 return string.join(self.query_parts, '')

127−

searx/search.py+67−37

@@ -22,7 +22,7 @@ from datetime import datetime

2222from operator import itemgetter

2323from urlparse import urlparse, unquote

2424from searx.engines import (

25− categories, engines, engine_shortcuts

25+ categories, engines

2626)

2727from searx.languages import language_codes

2828from searx.utils import gen_useragent

@@ -39,7 +39,13 @@ def default_request_params():

3939

4040

4141# create a callback wrapper for the search engine results

42−def make_callback(engine_name, results, suggestions, answers, infoboxes, callback, params):

42+def make_callback(engine_name,

43+ results,

44+ suggestions,

45+ answers,

46+ infoboxes,

47+ callback,

48+ params):

4349

4450 # creating a callback wrapper for the search engine results

4551 def process_callback(response, **kwargs):

@@ -95,7 +101,7 @@ def make_callback(engine_name, results, suggestions, answers, infoboxes, callbac

95101def content_result_len(content):

96102 if isinstance(content, basestring):

97103 content = re.sub('[,;:!?\./\\\\ ()-_]', '', content)

98− return len(content)

104+ return len(content)

99105 else:

100106 return 0

101107

@@ -126,7 +132,8 @@ def score_results(results):

126132

127133 # strip multiple spaces and cariage returns from content

128134 if 'content' in res:

129− res['content'] = re.sub(' +', ' ', res['content'].strip().replace('\n', ''))

135+ res['content'] = re.sub(' +', ' ',

136+ res['content'].strip().replace('\n', ''))

130137

131138 # get weight of this engine if possible

132139 if hasattr(engines[res['engine']], 'weight'):

@@ -139,8 +146,12 @@ def score_results(results):

139146 duplicated = False

140147 for new_res in results:

141148 # remove / from the end of the url if required

142− p1 = res['parsed_url'].path[:-1] if res['parsed_url'].path.endswith('/') else res['parsed_url'].path # noqa

143− p2 = new_res['parsed_url'].path[:-1] if new_res['parsed_url'].path.endswith('/') else new_res['parsed_url'].path # noqa

149+ p1 = res['parsed_url'].path[:-1]\

150+ if res['parsed_url'].path.endswith('/')\

151+ else res['parsed_url'].path

152+ p2 = new_res['parsed_url'].path[:-1]\

153+ if new_res['parsed_url'].path.endswith('/')\

154+ else new_res['parsed_url'].path

144155

145156 # check if that result is a duplicate

146157 if res['host'] == new_res['host'] and\

@@ -153,7 +164,8 @@ def score_results(results):

153164 # merge duplicates together

154165 if duplicated:

155166 # using content with more text

156− if content_result_len(res.get('content', '')) > content_result_len(duplicated.get('content', '')):

167+ if content_result_len(res.get('content', '')) >\

168+ content_result_len(duplicated.get('content', '')):

157169 duplicated['content'] = res['content']

158170

159171 # increase result-score

@@ -182,17 +194,25 @@ def score_results(results):

182194

183195 for i, res in enumerate(results):

184196 # FIXME : handle more than one category per engine

185− category = engines[res['engine']].categories[0] + ':' + '' if 'template' not in res else res['template']

186−

187− current = None if category not in categoryPositions else categoryPositions[category]

188−

189− # group with previous results using the same category if the group can accept more result and is not too far from the current position

190− if current != None and (current['count'] > 0) and (len(gresults) - current['index'] < 20):

191− # group with the previous results using the same category with this one

197+ category = engines[res['engine']].categories[0] + ':' + ''\

198+ if 'template' not in res\

199+ else res['template']

200+

201+ current = None if category not in categoryPositions\

202+ else categoryPositions[category]

203+

204+ # group with previous results using the same category

205+ # if the group can accept more result and is not too far

206+ # from the current position

207+ if current is not None and (current['count'] > 0)\

208+ and (len(gresults) - current['index'] < 20):

209+ # group with the previous results using

210+ # the same category with this one

192211 index = current['index']

193212 gresults.insert(index, res)

194213

195− # update every index after the current one (including the current one)

214+ # update every index after the current one

215+ # (including the current one)

196216 for k in categoryPositions:

197217 v = categoryPositions[k]['index']

198218 if v >= index:

@@ -206,7 +226,7 @@ def score_results(results):

206226 gresults.append(res)

207227

208228 # update categoryIndex

209− categoryPositions[category] = { 'index' : len(gresults), 'count' : 8 }

229+ categoryPositions[category] = {'index': len(gresults), 'count': 8}

210230

211231 # return gresults

212232 return gresults

@@ -215,21 +235,21 @@ def score_results(results):

215235def merge_two_infoboxes(infobox1, infobox2):

216236 if 'urls' in infobox2:

217237 urls1 = infobox1.get('urls', None)

218− if urls1 == None:

238+ if urls1 is None:

219239 urls1 = []

220240 infobox1.set('urls', urls1)

221241

222242 urlSet = set()

223243 for url in infobox1.get('urls', []):

224244 urlSet.add(url.get('url', None))

225−

245+

226246 for url in infobox2.get('urls', []):

227247 if url.get('url', None) not in urlSet:

228248 urls1.append(url)

229249

230250 if 'attributes' in infobox2:

231251 attributes1 = infobox1.get('attributes', None)

232− if attributes1 == None:

252+ if attributes1 is None:

233253 attributes1 = []

234254 infobox1.set('attributes', attributes1)

235255

@@ -237,14 +257,14 @@ def merge_two_infoboxes(infobox1, infobox2):

237257 for attribute in infobox1.get('attributes', []):

238258 if attribute.get('label', None) not in attributeSet:

239259 attributeSet.add(attribute.get('label', None))

240−

260+

241261 for attribute in infobox2.get('attributes', []):

242262 attributes1.append(attribute)

243263

244264 if 'content' in infobox2:

245265 content1 = infobox1.get('content', None)

246266 content2 = infobox2.get('content', '')

247− if content1 != None:

267+ if content1 is not None:

248268 if content_result_len(content2) > content_result_len(content1):

249269 infobox1['content'] = content2

250270 else:

@@ -257,12 +277,12 @@ def merge_infoboxes(infoboxes):

257277 for infobox in infoboxes:

258278 add_infobox = True

259279 infobox_id = infobox.get('id', None)

260− if infobox_id != None:

280+ if infobox_id is not None:

261281 existingIndex = infoboxes_id.get(infobox_id, None)

262− if existingIndex != None:

282+ if existingIndex is not None:

263283 merge_two_infoboxes(results[existingIndex], infobox)

264− add_infobox=False

265−

284+ add_infobox = False

285+

266286 if add_infobox:

267287 results.append(infobox)

268288 infoboxes_id[infobox_id] = len(results)-1

@@ -318,7 +338,8 @@ class Search(object):

318338

319339 self.pageno = int(pageno_param)

320340

321− # parse query, if tags are set, which change the serch engine or search-language

341+ # parse query, if tags are set, which change

342+ # the serch engine or search-language

322343 query_obj = Query(self.request_data['q'], self.blocked_engines)

323344 query_obj.parse_query()

324345

@@ -334,25 +355,29 @@ class Search(object):

334355

335356 self.categories = []

336357

337− # if engines are calculated from query, set categories by using that informations

358+ # if engines are calculated from query,

359+ # set categories by using that informations

338360 if self.engines:

339361 self.categories = list(set(engine['category']

340362 for engine in self.engines))

341363

342− # otherwise, using defined categories to calculate which engines should be used

364+ # otherwise, using defined categories to

365+ # calculate which engines should be used

343366 else:

344367 # set used categories

345368 for pd_name, pd in self.request_data.items():

346369 if pd_name.startswith('category_'):

347370 category = pd_name[9:]

348371 # if category is not found in list, skip

349− if not category in categories:

372+ if category not in categories:

350373 continue

351374

352375 # add category to list

353376 self.categories.append(category)

354377

355− # if no category is specified for this search, using user-defined default-configuration which (is stored in cookie)

378+ # if no category is specified for this search,

379+ # using user-defined default-configuration which

380+ # (is stored in cookie)

356381 if not self.categories:

357382 cookie_categories = request.cookies.get('categories', '')

358383 cookie_categories = cookie_categories.split(',')

@@ -360,16 +385,18 @@ class Search(object):

360385 if ccateg in categories:

361386 self.categories.append(ccateg)

362387

363− # if still no category is specified, using general as default-category

388+ # if still no category is specified, using general

389+ # as default-category

364390 if not self.categories:

365391 self.categories = ['general']

366392

367− # using all engines for that search, which are declared under the specific categories

393+ # using all engines for that search, which are

394+ # declared under the specific categories

368395 for categ in self.categories:

369396 self.engines.extend({'category': categ,

370397 'name': x.name}

371398 for x in categories[categ]

372− if not x.name in self.blocked_engines)

399+ if x.name not in self.blocked_engines)

373400

374401 # do search-request

375402 def search(self, request):

@@ -386,7 +413,7 @@ class Search(object):

386413 number_of_searches += 1

387414

388415 # set default useragent

389− #user_agent = request.headers.get('User-Agent', '')

416+ # user_agent = request.headers.get('User-Agent', '')

390417 user_agent = gen_useragent()

391418

392419 # start search-reqest for all selected engines

@@ -400,7 +427,8 @@ class Search(object):

400427 if self.pageno > 1 and not engine.paging:

401428 continue

402429

403− # if search-language is set and engine does not provide language-support, skip

430+ # if search-language is set and engine does not

431+ # provide language-support, skip

404432 if self.lang != 'all' and not engine.language_support:

405433 continue

406434

@@ -412,7 +440,8 @@ class Search(object):

412440 request_params['pageno'] = self.pageno

413441 request_params['language'] = self.lang

414442

415− # update request parameters dependent on search-engine (contained in engines folder)

443+ # update request parameters dependent on

444+ # search-engine (contained in engines folder)

416445 request_params = engine.request(self.query.encode('utf-8'),

417446 request_params)

418447

@@ -431,7 +460,8 @@ class Search(object):

431460 request_params

432461 )

433462

434− # create dictionary which contain all informations about the request

463+ # create dictionary which contain all

464+ # informations about the request

435465 request_args = dict(

436466 headers=request_params['headers'],

437467 hooks=dict(response=callback),

searx/utils.py+5−4

@@ -1,4 +1,4 @@

1−#import htmlentitydefs

1+# import htmlentitydefs

22from codecs import getincrementalencoder

33from HTMLParser import HTMLParser

44from random import choice

@@ -22,7 +22,8 @@ def gen_useragent():

2222

2323def searx_useragent():

2424 return 'searx'

25−

25+

26+

2627def highlight_content(content, query):

2728

2829 if not content:

@@ -67,8 +68,8 @@ class HTMLTextExtractor(HTMLParser):

6768 self.result.append(unichr(codepoint))

6869

6970 def handle_entityref(self, name):

70− #codepoint = htmlentitydefs.name2codepoint[name]

71− #self.result.append(unichr(codepoint))

71+ # codepoint = htmlentitydefs.name2codepoint[name]

72+ # self.result.append(unichr(codepoint))

7273 self.result.append(name)

7374

7475 def get_text(self):

searx/webapp.py+18−13

@@ -71,7 +71,7 @@ app.secret_key = settings['server']['secret_key']

7171

7272babel = Babel(app)

7373

74−#TODO configurable via settings.yml

74+# TODO configurable via settings.yml

7575favicons = ['wikipedia', 'youtube', 'vimeo', 'soundcloud',

7676 'twitter', 'stackoverflow', 'github']

7777

@@ -146,14 +146,14 @@ def render(template_name, override_theme=None, **kwargs):

146146

147147 nonblocked_categories = set(chain.from_iterable(nonblocked_categories))

148148

149− if not 'categories' in kwargs:

149+ if 'categories' not in kwargs:

150150 kwargs['categories'] = ['general']

151151 kwargs['categories'].extend(x for x in

152152 sorted(categories.keys())

153153 if x != 'general'

154154 and x in nonblocked_categories)

155155

156− if not 'selected_categories' in kwargs:

156+ if 'selected_categories' not in kwargs:

157157 kwargs['selected_categories'] = []

158158 for arg in request.args:

159159 if arg.startswith('category_'):

@@ -168,7 +168,7 @@ def render(template_name, override_theme=None, **kwargs):

168168 if not kwargs['selected_categories']:

169169 kwargs['selected_categories'] = ['general']

170170

171− if not 'autocomplete' in kwargs:

171+ if 'autocomplete' not in kwargs:

172172 kwargs['autocomplete'] = autocomplete

173173

174174 kwargs['method'] = request.cookies.get('method', 'POST')

@@ -202,14 +202,15 @@ def index():

202202 'index.html',

203203 )

204204

205− search.results, search.suggestions, search.answers, search.infoboxes = search.search(request)

205+ search.results, search.suggestions,\

206+ search.answers, search.infoboxes = search.search(request)

206207

207208 for result in search.results:

208209

209210 if not search.paging and engines[result['engine']].paging:

210211 search.paging = True

211212

212− # check if HTTPS rewrite is required

213+ # check if HTTPS rewrite is required

213214 if settings['server']['https_rewrite']\

214215 and result['parsed_url'].scheme == 'http':

215216

@@ -236,7 +237,7 @@ def index():

236237 try:

237238 # TODO, precompile rule

238239 p = re.compile(rule[0])

239−

240+

240241 # rewrite url if possible

241242 new_result_url = p.sub(rule[1], result['url'])

242243 except:

@@ -250,17 +251,21 @@ def index():

250251 continue

251252

252253 # get domainname from result

253− # TODO, does only work correct with TLD's like asdf.com, not for asdf.com.de

254+ # TODO, does only work correct with TLD's like

255+ # asdf.com, not for asdf.com.de

254256 # TODO, using publicsuffix instead of this rewrite rule

255− old_result_domainname = '.'.join(result['parsed_url'].hostname.split('.')[-2:])

256− new_result_domainname = '.'.join(new_parsed_url.hostname.split('.')[-2:])

257+ old_result_domainname = '.'.join(

258+ result['parsed_url'].hostname.split('.')[-2:])

259+ new_result_domainname = '.'.join(

260+ new_parsed_url.hostname.split('.')[-2:])

257261

258− # check if rewritten hostname is the same, to protect against wrong or malicious rewrite rules

262+ # check if rewritten hostname is the same,

263+ # to protect against wrong or malicious rewrite rules

259264 if old_result_domainname == new_result_domainname:

260265 # set new url

261266 result['url'] = new_result_url

262267

263− # target has matched, do not search over the other rules

268+ # target has matched, do not search over the other rules

264269 break

265270

266271 if search.request_data.get('format', 'html') == 'html':

@@ -429,7 +434,7 @@ def preferences():

429434 for pd_name, pd in request.form.items():

430435 if pd_name.startswith('category_'):

431436 category = pd_name[9:]

432− if not category in categories:

437+ if category not in categories:

433438 continue

434439 selected_categories.append(category)

435440 elif pd_name == 'locale' and pd in settings['locales']: