1467a2e · searx · git.iko.soy

searx/engines/google_news.py+9

compare with master

unifiedsplit

@@ -2,6 +2,7 @@

2

3from urllib import urlencode

4from json import loads

5+from datetime import datetime, timedelta

5

6

6categories = ['news']

7categories = ['news']

7

8

@@ -31,7 +32,15 @@ def response(resp):

31 return []

32 return []

32

33

33 for result in search_res['responseData']['results']:

34 for result in search_res['responseData']['results']:

35+# S.149 (159), library.pdf

36+# datetime.strptime("Mon, 10 Mar 2014 16:26:15 -0700", "%a, %d %b %Y %H:%M:%S %z")

37+# publishedDate = parse(result['publishedDate'])

38+ publishedDate = datetime.strptime(str.join(' ',result['publishedDate'].split(None)[0:5]), "%a, %d %b %Y %H:%M:%S")

39+ #utc_offset = timedelta(result['publishedDate'].split(None)[5]) # local = utc + offset

40+ #publishedDate = publishedDate + utc_offset

41+

34 results.append({'url': result['unescapedUrl'],

42 results.append({'url': result['unescapedUrl'],

35 'title': result['titleNoFormatting'],

43 'title': result['titleNoFormatting'],

44+ 'publishedDate': publishedDate,

36 'content': result['content']})

45 'content': result['content']})

37 return results

46 return results

searx/engines/yahoo_news.py+19−1

@@ -4,6 +4,8 @@ from urllib import urlencode

4from lxml import html

5from searx.engines.xpath import extract_text, extract_url

6from searx.engines.yahoo import parse_url

7+from datetime import datetime, timedelta

8+import re

7

9

8categories = ['news']

10categories = ['news']

9search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'

11search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'

@@ -11,6 +13,7 @@ results_xpath = '//div[@class="res"]'

11url_xpath = './/h3/a/@href'

13url_xpath = './/h3/a/@href'

12title_xpath = './/h3/a'

14title_xpath = './/h3/a'

13content_xpath = './/div[@class="abstr"]'

15content_xpath = './/div[@class="abstr"]'

16+publishedDate_xpath = './/span[@class="timestamp"]'

14suggestion_xpath = '//div[@id="satat"]//a'

17suggestion_xpath = '//div[@id="satat"]//a'

15

18

16paging = True

19paging = True

@@ -37,7 +40,22 @@ def response(resp):

37 url = parse_url(extract_url(result.xpath(url_xpath), search_url))

40 url = parse_url(extract_url(result.xpath(url_xpath), search_url))

38 title = extract_text(result.xpath(title_xpath)[0])

41 title = extract_text(result.xpath(title_xpath)[0])

39 content = extract_text(result.xpath(content_xpath)[0])

42 content = extract_text(result.xpath(content_xpath)[0])

43+ publishedDate = extract_text(result.xpath(publishedDate_xpath)[0])

44+

45+ if re.match("^[0-9]+ minute(s|) ago$", publishedDate):

46+ publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group()))

47+ else:

48+ if re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate):

49+ timeNumbers = re.findall(r'\d+', publishedDate)

50+ publishedDate = datetime.now() - timedelta(hours=int(timeNumbers[0])) - timedelta(minutes=int(timeNumbers[1]))

51+ else:

52+ # TODO year in string possible?

53+ publishedDate = datetime.strptime(publishedDate,"%b %d %H:%M%p")

54+

55+ if publishedDate.year == 1900:

56+ publishedDate = publishedDate.replace(year=datetime.now().year)

57+

40− results.append({'url': url, 'title': title, 'content': content})

58+ results.append({'url': url, 'title': title, 'content': content,'publishedDate':publishedDate})

41

59

42 if not suggestion_xpath:

60 if not suggestion_xpath:

43 return results

61 return results

searx/static/css/style.css+1

@@ -35,6 +35,7 @@ a{text-decoration:none;color:#1a11be}a:visited{color:#8e44ad}

35.result h3{font-size:1em;word-wrap:break-word;margin:5px 0 1px 0;padding:0}

36.result .content{font-size:.8em;margin:0;padding:0;max-width:54em;word-wrap:break-word;line-height:1.24}

37.result .url{font-size:.8em;margin:3px 0 0 0;padding:0;max-width:54em;word-wrap:break-word;color:#c0392b}

38+.result .published_date{font-size:.8em;color:#888;margin:5px 20px}

38.engines{color:#888}

39.engines{color:#888}

39.small_font{font-size:.8em}

40.small_font{font-size:.8em}

40.small p{margin:2px 0}

41.small p{margin:2px 0}

searx/static/less/definitions.less+3

@@ -64,6 +64,9 @@

64// Url to result

65@color-result-url-font: #C0392B;

66

67+// Publish Date

68+@color-result-publishdate-font: #888;

69+

67// Images

70// Images

68@color-result-image-span-background-hover: rgba(0, 0, 0, 0.6);

71@color-result-image-span-background-hover: rgba(0, 0, 0, 0.6);

69@color-result-image-span-font: #FFF;

72@color-result-image-span-font: #FFF;

searx/static/less/style.less+6

@@ -248,6 +248,12 @@ a {

248 word-wrap:break-word;

249 color: @color-result-url-font;

250 }

251+

252+ .published_date {

253+ font-size: 0.8em;

254+ color: @color-result-publishdate-font;

255+ margin: 5px 20px;

256+ }

251}

257}

252

258

253.engines {

259.engines {

searx/templates/result_templates/default.html+1

@@ -6,6 +6,7 @@

6

7 <div>

8 <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

9+ {% if result.publishedDate %}<p class="published_date">{{ result.publishedDate }}</p>{% endif %}

9 <p class="content">{% if result.content %}{{ result.content|safe }}<br />{% endif %}</p>

10 <p class="content">{% if result.content %}{{ result.content|safe }}<br />{% endif %}</p>

10 <p class="url">{{ result.pretty_url }}</p>

11 <p class="url">{{ result.pretty_url }}</p>

11 </div>

12 </div>

searx/translations/en/LC_MESSAGES/messages.po+8

@@ -37,6 +37,14 @@ msgstr ""

37msgid "Errors"

38msgstr ""

39

40+#: searx/webapp.py:167

41+msgid "{minutes} minute(s) ago"

42+msgstr ""

43+

44+#: searx/webapp.py:169

45+msgid "{hours} hour(s), {minutes} minute(s) ago"

46+msgstr ""

47+

40#: searx/templates/index.html:7

48#: searx/templates/index.html:7

41msgid "about"

49msgid "about"

42msgstr ""

50msgstr ""

searx/webapp.py+15−1

@@ -26,12 +26,13 @@ import json

26import cStringIO

27import os

28

29+from datetime import datetime, timedelta

29from itertools import chain

30from itertools import chain

30from flask import (

31from flask import (

31 Flask, request, render_template, url_for, Response, make_response,

32 Flask, request, render_template, url_for, Response, make_response,

32 redirect, send_from_directory

33 redirect, send_from_directory

33)

34)

34−from flask.ext.babel import Babel

35+from flask.ext.babel import Babel, gettext, ngettext, format_date

35from searx import settings, searx_dir

36from searx import settings, searx_dir

36from searx.engines import (

37from searx.engines import (

37 search as do_search, categories, engines, get_engines_stats,

38 search as do_search, categories, engines, get_engines_stats,

@@ -156,6 +157,19 @@ def index():

156 if engine in favicons:

157 if engine in favicons:

157 result['favicon'] = engine

158 result['favicon'] = engine

158

159

160+ # TODO, check if timezone is calculated right

161+ if 'publishedDate' in result:

162+ if result['publishedDate'] >= datetime.now() - timedelta(days=1):

163+ timedifference = datetime.now() - result['publishedDate']

164+ minutes = int((timedifference.seconds/60)%60)

165+ hours = int(timedifference.seconds/60/60)

166+ if hours == 0:

167+ result['publishedDate'] = gettext(u'{minutes} minute(s) ago').format(minutes=minutes)

168+ else:

169+ result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes)

170+ else:

171+ result['publishedDate'] = format_date(result['publishedDate'])

172+

159 if search.request_data.get('format') == 'json':

173 if search.request_data.get('format') == 'json':

160 return Response(json.dumps({'query': search.query,

174 return Response(json.dumps({'query': search.query,

161 'results': search.results}),

175 'results': search.results}),