16bdc0b · searx · git.iko.soy

git.iko.soy/searx

[mod] do not escape html content in engines

commit16bdc0baf4f2b56af000337c4a2fa1e689f1220c authorAdam Tauber <asciimoo@gmail.com> dateFri 9 Dec 2016, 11:44 (10 years ago) parent28f12ef

30 files changed+56−97

compare with master

unifiedsplit

searx/engines/archlinux.py+1−2

@@ -12,7 +12,6 @@

1212"""

1313

1414from urlparse import urljoin

15−from cgi import escape

1615from urllib import urlencode

1716from lxml import html

1817from searx.engines.xpath import extract_text

@@ -135,7 +134,7 @@ def response(resp):

135134 for result in dom.xpath(xpath_results):

136135 link = result.xpath(xpath_link)[0]

137136 href = urljoin(base_url, link.attrib.get('href'))

138− title = escape(extract_text(link))

137+ title = extract_text(link)

139138

140139 results.append({'url': href,

141140 'title': title})

searx/engines/base.py+1−2

@@ -16,7 +16,6 @@

1616from lxml import etree

1717from urllib import urlencode

1818from searx.utils import searx_useragent

19−from cgi import escape

2019from datetime import datetime

2120import re

2221

@@ -94,7 +93,7 @@ def response(resp):

9493 url = item.text

9594

9695 elif item.attrib["name"] == "dcdescription":

97− content = escape(item.text[:300])

96+ content = item.text[:300]

9897 if len(item.text) > 300:

9998 content += "..."

10099

searx/engines/bing.py+2−3

@@ -14,7 +14,6 @@

1414"""

1515

1616from urllib import urlencode

17−from cgi import escape

1817from lxml import html

1918from searx.engines.xpath import extract_text

2019

@@ -61,7 +60,7 @@ def response(resp):

6160 link = result.xpath('.//h3/a')[0]

6261 url = link.attrib.get('href')

6362 title = extract_text(link)

64− content = escape(extract_text(result.xpath('.//p')))

63+ content = extract_text(result.xpath('.//p'))

6564

6665 # append result

6766 results.append({'url': url,

@@ -73,7 +72,7 @@ def response(resp):

7372 link = result.xpath('.//h2/a')[0]

7473 url = link.attrib.get('href')

7574 title = extract_text(link)

76− content = escape(extract_text(result.xpath('.//p')))

75+ content = extract_text(result.xpath('.//p'))

7776

7877 # append result

7978 results.append({'url': url,

searx/engines/btdigg.py+2−3

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import quote

1615from lxml import html

1716from operator import itemgetter

@@ -51,8 +50,8 @@ def response(resp):

5150 for result in search_res:

5251 link = result.xpath('.//td[@class="torrent_name"]//a')[0]

5352 href = urljoin(url, link.attrib.get('href'))

54− title = escape(extract_text(link))

55− content = escape(extract_text(result.xpath('.//pre[@class="snippet"]')[0]))

53+ title = extract_text(link)

54+ content = extract_text(result.xpath('.//pre[@class="snippet"]')[0])

5655 content = " ".join(content.split("\n"))

5756

5857 filesize = result.xpath('.//span[@class="attr_val"]/text()')[0].split()[0]

searx/engines/dailymotion.py+1−2

@@ -14,7 +14,6 @@

1414

1515from urllib import urlencode

1616from json import loads

17−from cgi import escape

1817from datetime import datetime

1918

2019# engine dependent config

@@ -57,7 +56,7 @@ def response(resp):

5756 for res in search_res['list']:

5857 title = res['title']

5958 url = res['url']

60− content = escape(res['description'])

59+ content = res['description']

6160 thumbnail = res['thumbnail_360_url']

6261 publishedDate = datetime.fromtimestamp(res['created_time'], None)

6362 embedded = embedded_url.format(videoid=res['id'])

searx/engines/deezer.py+5−4

@@ -51,10 +51,11 @@ def response(resp):

5151 if url.startswith('http://'):

5252 url = 'https' + url[4:]

5353

54− content = result['artist']['name'] +\

55− " • " +\

56− result['album']['title'] +\

57− " • " + result['title']

54+ content = '{} - {} - {}'.format(

55+ result['artist']['name'],

56+ result['album']['title'],

57+ result['title'])

58+

5859 embedded = embedded_url.format(audioid=result['id'])

5960

6061 # append result

searx/engines/dictzone.py+2−3

@@ -12,7 +12,6 @@

1212import re

1313from urlparse import urljoin

1414from lxml import html

15−from cgi import escape

1615from searx.utils import is_valid_lang

1716

1817categories = ['general']

@@ -62,8 +61,8 @@ def response(resp):

6261

6362 results.append({

6463 'url': urljoin(resp.url, '?%d' % k),

65− 'title': escape(from_result.text_content()),

66− 'content': escape('; '.join(to_results))

64+ 'title': from_result.text_content(),

65+ 'content': '; '.join(to_results)

6766 })

6867

6968 return results

searx/engines/digg.py+1−2

@@ -13,7 +13,6 @@

1313from urllib import quote_plus

1414from json import loads

1515from lxml import html

16−from cgi import escape

1716from dateutil import parser

1817

1918# engine dependent config

@@ -56,7 +55,7 @@ def response(resp):

5655 url = result.attrib.get('data-contenturl')

5756 thumbnail = result.xpath('.//img')[0].attrib.get('src')

5857 title = ''.join(result.xpath(title_xpath))

59− content = escape(''.join(result.xpath(content_xpath)))

58+ content = ''.join(result.xpath(content_xpath))

6059 pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

6160 publishedDate = parser.parse(pubdate)

6261

searx/engines/fdroid.py+1−2

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from searx.engines.xpath import extract_text

1514from lxml import html

@@ -43,7 +42,7 @@ def response(resp):

4342 img_src = app.xpath('.//img/@src')[0]

4443

4544 content = extract_text(app.xpath('./p')[0])

46− content = escape(content.replace(title, '', 1).strip())

45+ content = content.replace(title, '', 1).strip()

4746

4847 results.append({'url': url,

4948 'title': title,

searx/engines/flickr.py+3−11

@@ -77,21 +77,13 @@ def response(resp):

7777

7878 url = build_flickr_url(photo['owner'], photo['id'])

7979

80− title = photo['title']

81−

82− content = '' +\

83− photo['ownername'] +\

84− ' ' +\

85− '' +\

86− photo['description']['_content'] +\

87− ''

88−

8980 # append result

9081 results.append({'url': url,

91− 'title': title,

82+ 'title': photo['title'],

9283 'img_src': img_src,

9384 'thumbnail_src': thumbnail_src,

94− 'content': content,

85+ 'content': content = photo['description']['_content'],

86+ 'author': photo['ownername'],

9587 'template': 'images.html'})

9688

9789 # return results

searx/engines/flickr_noapi.py+3−4

@@ -102,16 +102,15 @@ def response(resp):

102102

103103 title = photo.get('title', '')

104104

105− content = '' +\

106− photo['username'] +\

107− ' '

105+ author = photo['username']

108106

109107 # append result

110108 results.append({'url': url,

111109 'title': title,

112110 'img_src': img_src,

113111 'thumbnail_src': thumbnail_src,

114− 'content': content,

112+ 'content': '',

113+ 'author': author,

115114 'template': 'images.html'})

116115

117116 return results

searx/engines/gigablast.py+2−3

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from json import loads

1514from random import randint

1615from time import time

@@ -78,8 +77,8 @@ def response(resp):

7877 for result in response_json['results']:

7978 # append result

8079 results.append({'url': result['url'],

81− 'title': escape(result['title']),

82− 'content': escape(result['sum'])})

80+ 'title': result['title'],

81+ 'content': result['sum']})

8382

8483 # return results

8584 return results

searx/engines/github.py+1−2

@@ -12,7 +12,6 @@

1212

1313from urllib import urlencode

1414from json import loads

15−from cgi import escape

1615

1716# engine dependent config

1817categories = ['it']

@@ -48,7 +47,7 @@ def response(resp):

4847 url = res['html_url']

4948

5049 if res['description']:

51− content = escape(res['description'][:500])

50+ content = res['description'][:500]

5251 else:

5352 content = ''

5453

searx/engines/google.py+2−3

@@ -9,7 +9,6 @@

99# @parse url, title, content, suggestion

1010

1111import re

12−from cgi import escape

1312from urllib import urlencode

1413from urlparse import urlparse, parse_qsl

1514from lxml import html, etree

@@ -155,7 +154,7 @@ def parse_url(url_string, google_hostname):

155154def extract_text_from_dom(result, xpath):

156155 r = result.xpath(xpath)

157156 if len(r) > 0:

158− return escape(extract_text(r[0]))

157+ return extract_text(r[0])

159158 return None

160159

161160

@@ -264,7 +263,7 @@ def response(resp):

264263 # parse suggestion

265264 for suggestion in dom.xpath(suggestion_xpath):

266265 # append suggestion

267− results.append({'suggestion': escape(extract_text(suggestion))})

266+ results.append({'suggestion': extract_text(suggestion)})

268267

269268 # return results

270269 return results

searx/engines/kickass.py+1−2

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import quote

1615from lxml import html

1716from operator import itemgetter

@@ -57,7 +56,7 @@ def response(resp):

5756 link = result.xpath('.//a[@class="cellMainLink"]')[0]

5857 href = urljoin(url, link.attrib['href'])

5958 title = extract_text(link)

60− content = escape(extract_text(result.xpath(content_xpath)))

59+ content = extract_text(result.xpath(content_xpath))

6160 seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))

6261 leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))

6362 filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))

searx/engines/nyaa.py+2−4

@@ -9,7 +9,6 @@

99 @parse url, title, content, seed, leech, torrentfile

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from lxml import html

1514from searx.engines.xpath import extract_text

@@ -78,7 +77,7 @@ def response(resp):

7877

7978 # torrent title

8079 page_a = result.xpath(xpath_title)[0]

81− title = escape(extract_text(page_a))

80+ title = extract_text(page_a)

8281

8382 # link to the page

8483 href = page_a.attrib.get('href')

@@ -90,7 +89,7 @@ def response(resp):

9089 try:

9190 file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')

9291 file_size = int(float(file_size) * get_filesize_mul(suffix))

93− except Exception as e:

92+ except:

9493 file_size = None

9594

9695 # seed count

@@ -105,7 +104,6 @@ def response(resp):

105104 # content string contains all information not included into template

106105 content = 'Category: "{category}". Downloaded {downloads} times.'

107106 content = content.format(category=category, downloads=downloads)

108− content = escape(content)

109107

110108 results.append({'url': href,

111109 'title': title,

searx/engines/piratebay.py+1−2

@@ -9,7 +9,6 @@

99# @parse url, title, content, seed, leech, magnetlink

1010

1111from urlparse import urljoin

12−from cgi import escape

1312from urllib import quote

1413from lxml import html

1514from operator import itemgetter

@@ -62,7 +61,7 @@ def response(resp):

6261 link = result.xpath('.//div[@class="detName"]//a')[0]

6362 href = urljoin(url, link.attrib.get('href'))

6463 title = extract_text(link)

65− content = escape(extract_text(result.xpath(content_xpath)))

64+ content = extract_text(result.xpath(content_xpath))

6665 seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]

6766

6867 # convert seed to int if possible

searx/engines/reddit.py+1−2

@@ -11,7 +11,6 @@

1111"""

1212

1313import json

14−from cgi import escape

1514from urllib import urlencode

1615from urlparse import urlparse, urljoin

1716from datetime import datetime

@@ -68,7 +67,7 @@ def response(resp):

6867 img_results.append(params)

6968 else:

7069 created = datetime.fromtimestamp(data['created_utc'])

71− content = escape(data['selftext'])

70+ content = data['selftext']

7271 if len(content) > 500:

7372 content = content[:500] + '...'

7473 params['content'] = content

searx/engines/searchcode_doc.py+2−10

@@ -44,20 +44,12 @@ def response(resp):

4444 # parse results

4545 for result in search_results.get('results', []):

4646 href = result['url']

47− title = "[" + result['type'] + "] " +\

48− result['namespace'] +\

49− " " + result['name']

50− content = '[' +\

51− result['type'] + "] " +\

52− result['name'] + " " +\

53− result['synopsis'] +\

54− " " +\

55− result['description']

47+ title = "[{}] {} {}".format(result['type'], result['namespace'], result['name'])

5648

5749 # append result

5850 results.append({'url': href,

5951 'title': title,

60− 'content': content})

52+ 'content': result['description']})

6153

6254 # return results

6355 return results

searx/engines/seedpeer.py−1

@@ -9,7 +9,6 @@

99# @parse url, title, content, seed, leech, magnetlink

1010

1111from urlparse import urljoin

12−from cgi import escape

1312from urllib import quote

1413from lxml import html

1514from operator import itemgetter

searx/engines/spotify.py+5−4

@@ -46,10 +46,11 @@ def response(resp):

4646 if result['type'] == 'track':

4747 title = result['name']

4848 url = result['external_urls']['spotify']

49− content = result['artists'][0]['name'] +\

50− " • " +\

51− result['album']['name'] +\

52− " • " + result['name']

49+ content = '{} - {} - {}'.format(

50+ result['artists'][0]['name'],

51+ result['album']['name'],

52+ result['name'])

53+

5354 embedded = embedded_url.format(audioid=result['id'])

5455

5556 # append result

searx/engines/stackoverflow.py+2−3

@@ -11,7 +11,6 @@

1111"""

1212

1313from urlparse import urljoin

14−from cgi import escape

1514from urllib import urlencode

1615from lxml import html

1716from searx.engines.xpath import extract_text

@@ -48,8 +47,8 @@ def response(resp):

4847 for result in dom.xpath(results_xpath):

4948 link = result.xpath(link_xpath)[0]

5049 href = urljoin(url, link.attrib.get('href'))

51− title = escape(extract_text(link))

52− content = escape(extract_text(result.xpath(content_xpath)))

50+ title = extract_text(link)

51+ content = extract_text(result.xpath(content_xpath))

5352

5453 # append result

5554 results.append({'url': href,

searx/engines/startpage.py+2−3

@@ -11,7 +11,6 @@

1111# @todo paging

1212

1313from lxml import html

14−from cgi import escape

1514from dateutil import parser

1615from datetime import datetime, timedelta

1716import re

@@ -79,10 +78,10 @@ def response(resp):

7978 if re.match(r"^http(s|)://(www\.)?ixquick\.com/do/search\?.*$", url):

8079 continue

8180

82− title = escape(extract_text(link))

81+ title = extract_text(link)

8382

8483 if result.xpath('./p[@class="desc clk"]'):

85− content = escape(extract_text(result.xpath('./p[@class="desc clk"]')))

84+ content = extract_text(result.xpath('./p[@class="desc clk"]'))

8685 else:

8786 content = ''

8887

searx/engines/subtitleseeker.py+2−3

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from urllib import quote_plus

1514from lxml import html

1615from searx.languages import language_codes

@@ -59,7 +58,7 @@ def response(resp):

5958 elif search_lang:

6059 href = href + search_lang + '/'

6160

62− title = escape(extract_text(link))

61+ title = extract_text(link)

6362

6463 content = extract_text(result.xpath('.//div[contains(@class,"red")]'))

6564 content = content + " - "

@@ -75,7 +74,7 @@ def response(resp):

7574 # append result

7675 results.append({'url': href,

7776 'title': title,

78− 'content': escape(content)})

77+ 'content': content})

7978

8079 # return results

8180 return results

searx/engines/swisscows.py+4−5

@@ -10,7 +10,6 @@

1010 @parse url, title, content

1111"""

1212

13−from cgi import escape

1413from json import loads

1514from urllib import urlencode, unquote

1615import re

@@ -78,7 +77,7 @@ def response(resp):

7877

7978 # append result

8079 results.append({'url': result['SourceUrl'],

81− 'title': escape(result['Title']),

80+ 'title': result['Title'],

8281 'content': '',

8382 'img_src': img_url,

8483 'template': 'images.html'})

@@ -90,8 +89,8 @@ def response(resp):

9089

9190 # append result

9291 results.append({'url': result_url,

93− 'title': escape(result_title),

94− 'content': escape(result_content)})

92+ 'title': result_title,

93+ 'content': result_content})

9594

9695 # parse images

9796 for result in json.get('Images', []):

@@ -100,7 +99,7 @@ def response(resp):

10099

101100 # append result

102101 results.append({'url': result['SourceUrl'],

103− 'title': escape(result['Title']),

102+ 'title': result['Title'],

104103 'content': '',

105104 'img_src': img_url,

106105 'template': 'images.html'})

searx/engines/tokyotoshokan.py−1

@@ -11,7 +11,6 @@

1111"""

1212

1313import re

14−from cgi import escape

1514from urllib import urlencode

1615from lxml import html

1716from searx.engines.xpath import extract_text

searx/engines/torrentz.py−1

@@ -12,7 +12,6 @@

1212"""

1313

1414import re

15−from cgi import escape

1615from urllib import urlencode

1716from lxml import html

1817from searx.engines.xpath import extract_text

searx/engines/translated.py+5−6

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111import re

12−from cgi import escape

1312from searx.utils import is_valid_lang

1413

1514categories = ['general']

@@ -52,14 +51,14 @@ def request(query, params):

5251def response(resp):

5352 results = []

5453 results.append({

55− 'url': escape(web_url.format(

54+ 'url': web_url.format(

5655 from_lang=resp.search_params['from_lang'][2],

5756 to_lang=resp.search_params['to_lang'][2],

58− query=resp.search_params['query'])),

59− 'title': escape('[{0}-{1}] {2}'.format(

57+ query=resp.search_params['query']),

58+ 'title': '[{0}-{1}] {2}'.format(

6059 resp.search_params['from_lang'][1],

6160 resp.search_params['to_lang'][1],

62− resp.search_params['query'])),

63− 'content': escape(resp.json()['responseData']['translatedText'])

61+ resp.search_params['query']),

62+ 'content': resp.json()['responseData']['translatedText']

6463 })

6564 return results

searx/engines/wolframalpha_noapi.py−1

@@ -8,7 +8,6 @@

88# @stable no

99# @parse url, infobox

1010

11−from cgi import escape

1211from json import loads

1312from time import time

1413from urllib import urlencode

searx/engines/yandex.py+2−3

@@ -9,7 +9,6 @@

99 @parse url, title, content

1010"""

1111

12−from cgi import escape

1312from urllib import urlencode

1413from lxml import html

1514from searx.search import logger

@@ -52,8 +51,8 @@ def response(resp):

5251 for result in dom.xpath(results_xpath):

5352 try:

5453 res = {'url': result.xpath(url_xpath)[0],

55− 'title': escape(''.join(result.xpath(title_xpath))),

56− 'content': escape(''.join(result.xpath(content_xpath)))}

54+ 'title': ''.join(result.xpath(title_xpath)),

55+ 'content': ''.join(result.xpath(content_xpath))}

5756 except:

5857 logger.exception('yandex parse crash')

5958 continue