17bf00e · searx · git.iko.soy

searx/engines/duckduckgo.py+2−1

compare with master

unifiedsplit

@@ -1,5 +1,6 @@

1from json import loads

2from urllib import urlencode

3+from searx.utils import html_to_text

3

4

4url = 'https://duckduckgo.com/'

5url = 'https://duckduckgo.com/'

5search_url = url + 'd.js?{query}&l=us-en&p=1&s=0'

6search_url = url + 'd.js?{query}&l=us-en&p=1&s=0'

@@ -16,7 +17,7 @@ def response(resp):

16 if not r.get('t'):

17 if not r.get('t'):

17 continue

18 continue

18 results.append({'title': r['t']

19 results.append({'title': r['t']

19− ,'content': r['a']

20+ ,'content': html_to_text(r['a'])

20 ,'url': r['u']

21 ,'url': r['u']

21 })

22 })

22 return results

23 return results

searx/engines/startpage.py+2−2

@@ -1,4 +1,4 @@

1−from urllib import quote

1+from urllib import urlencode

2from lxml import html

3from urlparse import urlparse

4from cgi import escape

@@ -8,7 +8,7 @@ search_url = base_url+'do/search'

8

9def request(query, params):

10 global search_url

11− query = quote(query.replace(' ', '+'), safe='+')

11+ query = urlencode({'q': query})[2:]

12 params['url'] = search_url

13 params['method'] = 'POST'

14 params['data'] = {'query': query}

searx/engines/twitter.py+2−1

@@ -1,6 +1,7 @@

1from urlparse import urljoin

2from urllib import urlencode

3from lxml import html

4+from cgi import escape

4

5

5categories = ['social media']

6categories = ['social media']

6

7

@@ -21,6 +22,6 @@ def response(resp):

21 link = tweet.xpath('.//small[@class="time"]//a')[0]

22 link = tweet.xpath('.//small[@class="time"]//a')[0]

22 url = urljoin(base_url, link.attrib.get('href'))

23 url = urljoin(base_url, link.attrib.get('href'))

23 title = ''.join(tweet.xpath('.//span[@class="username js-action-profile-name"]//text()'))

24 title = ''.join(tweet.xpath('.//span[@class="username js-action-profile-name"]//text()'))

24− content = ''.join(map(html.tostring, tweet.xpath('.//p[@class="js-tweet-text tweet-text"]//*')))

25+ content = escape(''.join(tweet.xpath('.//p[@class="js-tweet-text tweet-text"]//text()')))

25 results.append({'url': url, 'title': title, 'content': content})

26 results.append({'url': url, 'title': title, 'content': content})

26 return results

27 return results

searx/engines/xpath.py+1−2

@@ -46,12 +46,11 @@ def request(query, params):

46def response(resp):

47 results = []

48 dom = html.fromstring(resp.text)

49− query = resp.search_params['query']

50 if results_xpath:

49 if results_xpath:

51 for result in dom.xpath(results_xpath):

50 for result in dom.xpath(results_xpath):

52 url = extract_url(result.xpath(url_xpath))

51 url = extract_url(result.xpath(url_xpath))

53 title = ' '.join(result.xpath(title_xpath))

52 title = ' '.join(result.xpath(title_xpath))

54− content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))

53+ content = escape(' '.join(result.xpath(content_xpath)))

55 results.append({'url': url, 'title': title, 'content': content})

54 results.append({'url': url, 'title': title, 'content': content})

56 else:

55 else:

57 for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):

56 for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):