git.iko.soy/searx
on master help sign in
[fix] highlighting only html
commit7b4ec5c5e9a89fc1bc3b3fc8dfad26450530a2da authorasciimoo <asciimoo@gmail.com> dateFri 10 Jan 2014, 23:38 (13 years ago) parent04c4083
3 files changed+35−28
compare with master
unifiedsplit
searx/engines/__init__.py−28 searx/utils.py+26 searx/webapp.py+9
searx/engines/__init__.py−28
@@ -25,7 +25,6 @@ from urlparse import urlparse
2525from searx import settings
2626import ConfigParser
2727import sys
28−import re
2928from datetime import datetime
3029
3130engine_dir = dirname(realpath(__file__))
@@ -106,31 +105,6 @@ def make_callback(engine_name, results, suggestions, callback, params):
106105 results[engine_name] = cb_res
107106 return process_callback
108107
109−def highlight_content(content, query):
110−
111− if not content:
112− return None
113− # ignoring html contents
114− # TODO better html content detection
115− if content.find('<') != -1:
116− return content
117−
118− query = query.decode('utf-8')
119− if content.lower().find(query.lower()) > -1:
120− query_regex = u'({0})'.format(re.escape(query))
121− content = re.sub(query_regex, '<b>\\1</b>', content, flags=re.I | re.U)
122− else:
123− regex_parts = []
124− for chunk in query.split():
125− if len(chunk) == 1:
126− regex_parts.append(u'\W+{0}\W+'.format(re.escape(chunk)))
127− else:
128− regex_parts.append(u'{0}'.format(re.escape(chunk)))
129− query_regex = u'({0})'.format('|'.join(regex_parts))
130− content = re.sub(query_regex, '<b>\\1</b>', content, flags=re.I | re.U)
131−
132− return content
133−
134108def score_results(results):
135109 flat_res = filter(None, chain.from_iterable(izip_longest(*results.values())))
136110 flat_len = len(flat_res)
@@ -218,8 +192,6 @@ def search(query, request, selected_engines):
218192 results = score_results(results)
219193
220194 for result in results:
221− if 'content' in result:
222− result['content'] = highlight_content(result['content'], query)
223195 for res_engine in result['engines']:
224196 engines[result['engine']].stats['score_count'] += result['score']
225197
searx/utils.py+26
@@ -3,6 +3,32 @@ from HTMLParser import HTMLParser
33import csv
44import codecs
55import cStringIO
6+import re
7+
8+def highlight_content(content, query):
9+
10+ if not content:
11+ return None
12+ # ignoring html contents
13+ # TODO better html content detection
14+ if content.find('<') != -1:
15+ return content
16+
17+ query = query.decode('utf-8')
18+ if content.lower().find(query.lower()) > -1:
19+ query_regex = u'({0})'.format(re.escape(query))
20+ content = re.sub(query_regex, '<b>\\1</b>', content, flags=re.I | re.U)
21+ else:
22+ regex_parts = []
23+ for chunk in query.split():
24+ if len(chunk) == 1:
25+ regex_parts.append(u'\W+{0}\W+'.format(re.escape(chunk)))
26+ else:
27+ regex_parts.append(u'{0}'.format(re.escape(chunk)))
28+ query_regex = u'({0})'.format('|'.join(regex_parts))
29+ content = re.sub(query_regex, '<b>\\1</b>', content, flags=re.I | re.U)
30+
31+ return content
632
733class HTMLTextExtractor(HTMLParser):
834 def __init__(self):
searx/webapp.py+9
@@ -29,6 +29,7 @@ import json
2929import cStringIO
3030from searx.utils import UnicodeWriter
3131from flask import send_from_directory
32+from searx.utils import highlight_content, html_to_text
3233
3334
3435
@@ -104,6 +105,14 @@ def index():
104105 results, suggestions = search(query, request, selected_engines)
105106
106107 for result in results:
108+ if request_data.get('format', 'html') == 'html':
109+ if 'content' in result:
110+ result['content'] = highlight_content(result['content'], query)
111+ result['title'] = highlight_content(result['title'], query)
112+ else:
113+ if 'content' in result:
114+ result['content'] = html_to_text(result['content']).strip()
115+ result['title'] = html_to_text(result['title']).strip()
107116 if len(result['url']) > 74:
108117 result['pretty_url'] = result['url'][:35] + '[..]' + result['url'][-35:]
109118 else: