d0f9778 · searx · git.iko.soy

3from collections import defaultdict

4from operator import itemgetter

5from threading import RLock

6+from searx import logger

6from searx.engines import engines

7from searx.engines import engines

7from searx.url_utils import urlparse, unquote

8from searx.url_utils import urlparse, unquote

8

9

127class ResultContainer(object):

128class ResultContainer(object):

128 """docstring for ResultContainer"""

129 """docstring for ResultContainer"""

129

130

131+ __slots__ = '_merged_results', 'infoboxes', 'suggestions', 'answers', 'corrections', '_number_of_results',\

132+ '_ordered', 'paging', 'unresponsive_engines', 'timings', 'redirect_url'

133+

130 def __init__(self):

134 def __init__(self):

131 super(ResultContainer, self).__init__()

135 super(ResultContainer, self).__init__()

132− self.results = defaultdict(list)

133 self._merged_results = []

136 self._merged_results = []

134 self.infoboxes = []

137 self.infoboxes = []

135 self.suggestions = set()

138 self.suggestions = set()

143 self.redirect_url = None

146 self.redirect_url = None

144

147

145 def extend(self, engine_name, results):

148 def extend(self, engine_name, results):

149+ standard_result_count = 0

146 for result in list(results):

150 for result in list(results):

147 result['engine'] = engine_name

151 result['engine'] = engine_name

148 if 'suggestion' in result:

152 if 'suggestion' in result:

149 self.suggestions.add(result['suggestion'])

153 self.suggestions.add(result['suggestion'])

150− results.remove(result)

151 elif 'answer' in result:

154 elif 'answer' in result:

152 self.answers[result['answer']] = result

155 self.answers[result['answer']] = result

153− results.remove(result)

154 elif 'correction' in result:

156 elif 'correction' in result:

155 self.corrections.add(result['correction'])

157 self.corrections.add(result['correction'])

156− results.remove(result)

157 elif 'infobox' in result:

158 elif 'infobox' in result:

158 self._merge_infobox(result)

159 self._merge_infobox(result)

159− results.remove(result)

160 elif 'number_of_results' in result:

161 self._number_of_results.append(result['number_of_results'])

162− results.remove(result)

162+ else:

163+ # standard result (url, title, content)

164+ if 'url' in result and not isinstance(result['url'], basestring):

165+ logger.debug('result: invalid URL: %s', str(result))

166+ elif 'title' in result and not isinstance(result['title'], basestring):

167+ logger.debug('result: invalid title: %s', str(result))

168+ elif 'content' in result and not isinstance(result['content'], basestring):

169+ logger.debug('result: invalid content: %s', str(result))

170+ else:

171+ self._merge_result(result, standard_result_count + 1)

172+ standard_result_count += 1

163

173

164 if engine_name in engines:

174 if engine_name in engines:

165 with RLock():

175 with RLock():

166 engines[engine_name].stats['search_count'] += 1

176 engines[engine_name].stats['search_count'] += 1

167− engines[engine_name].stats['result_count'] += len(results)

177+ engines[engine_name].stats['result_count'] += standard_result_count

168−

169− if not results:

170− return

171

178

172− self.results[engine_name].extend(results)

173−

174− if not self.paging and engine_name in engines and engines[engine_name].paging:

179+ if not self.paging and standard_result_count > 0 and engine_name in engines\

180+ and engines[engine_name].paging:

175 self.paging = True

181 self.paging = True

176

182

177− for i, result in enumerate(results):

178− if 'url' in result and not isinstance(result['url'], basestring):

179− continue

180− try:

181− result['url'] = result['url'].decode('utf-8')

182− except:

183− pass

184− if 'title' in result and not isinstance(result['title'], basestring):

185− continue

186− if 'content' in result and not isinstance(result['content'], basestring):

187− continue

188− position = i + 1

189− self._merge_result(result, position)

190−

191 def _merge_infobox(self, infobox):

183 def _merge_infobox(self, infobox):

192 add_infobox = True

184 add_infobox = True

193 infobox_id = infobox.get('id', None)

185 infobox_id = infobox.get('id', None)