17bf00e · searx · git.iko.soy

git.iko.soy/searx

[enh] removing result html tags

commit17bf00ee42583910e45794e1438a2bab459225ad authorasciimoo <asciimoo@gmail.com> dateSat 9 Nov 2013, 18:39 (13 years ago) parent14a53e3

4 files changed+7−6

compare with master

unifiedsplit

searx/engines/duckduckgo.py+2−1 searx/engines/startpage.py+2−2 searx/engines/twitter.py+2−1 searx/engines/xpath.py+1−2

searx/engines/duckduckgo.py+2−1

@@ -1,5 +1,6 @@

11from json import loads

22from urllib import urlencode

3+from searx.utils import html_to_text

45url = 'https://duckduckgo.com/'

56search_url = url + 'd.js?{query}&l=us-en&p=1&s=0'

@@ -16,7 +17,7 @@ def response(resp):

1617 if not r.get('t'):

1718 continue

1819 results.append({'title': r['t']

19− ,'content': r['a']

20+ ,'content': html_to_text(r['a'])

2021 ,'url': r['u']

2122 })

2223 return results

searx/engines/startpage.py+2−2

@@ -1,4 +1,4 @@

1−from urllib import quote

1+from urllib import urlencode

22from lxml import html

33from urlparse import urlparse

44from cgi import escape

@@ -8,7 +8,7 @@ search_url = base_url+'do/search'

99def request(query, params):

1010 global search_url

11− query = quote(query.replace(' ', '+'), safe='+')

11+ query = urlencode({'q': query})[2:]

1212 params['url'] = search_url

1313 params['method'] = 'POST'

1414 params['data'] = {'query': query}

searx/engines/twitter.py+2−1

@@ -1,6 +1,7 @@

11from urlparse import urljoin

22from urllib import urlencode

33from lxml import html

4+from cgi import escape

56categories = ['social media']

@@ -21,6 +22,6 @@ def response(resp):

2122 link = tweet.xpath('.//small[@class="time"]//a')[0]

2223 url = urljoin(base_url, link.attrib.get('href'))

2324 title = ''.join(tweet.xpath('.//span[@class="username js-action-profile-name"]//text()'))

24− content = ''.join(map(html.tostring, tweet.xpath('.//p[@class="js-tweet-text tweet-text"]//*')))

25+ content = escape(''.join(tweet.xpath('.//p[@class="js-tweet-text tweet-text"]//text()')))

2526 results.append({'url': url, 'title': title, 'content': content})

2627 return results

searx/engines/xpath.py+1−2

@@ -46,12 +46,11 @@ def request(query, params):

4646def response(resp):

4747 results = []

4848 dom = html.fromstring(resp.text)

49− query = resp.search_params['query']

5049 if results_xpath:

5150 for result in dom.xpath(results_xpath):

5251 url = extract_url(result.xpath(url_xpath))

5352 title = ' '.join(result.xpath(title_xpath))

54− content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))

53+ content = escape(' '.join(result.xpath(content_xpath)))

5554 results.append({'url': url, 'title': title, 'content': content})

5655 else:

5756 for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):