018a144 · searx · git.iko.soy

requirements.txt+1

compare with master

unifiedsplit

@@ -3,3 +3,4 @@ flask-babel

3grequests

4lxml

5pyyaml

6+python-dateutil

searx/engines/google_news.py+4−10

@@ -2,6 +2,7 @@

2

3from urllib import urlencode

4from json import loads

5+from dateutil import parser

5from datetime import datetime

6from datetime import datetime

6

7

7categories = ['news']

8categories = ['news']

@@ -32,16 +33,9 @@ def response(resp):

32 return []

33 return []

33

34

34 for result in search_res['responseData']['results']:

35 for result in search_res['responseData']['results']:

35−# S.149 (159), library.pdf

36+

36−# datetime.strptime("Mon, 10 Mar 2014 16:26:15 -0700",

37+# Mon, 10 Mar 2014 16:26:15 -0700

37−# "%a, %d %b %Y %H:%M:%S %z")

38−# publishedDate = parse(result['publishedDate'])

38+ publishedDate = parser.parse(result['publishedDate'])

39− publishedDate = datetime.strptime(

40− str.join(' ', result['publishedDate'].split(None)[0:5]),

41− "%a, %d %b %Y %H:%M:%S")

42− #utc_offset = timedelta(result['publishedDate'].split(None)[5])

43− # local = utc + offset

44− #publishedDate = publishedDate + utc_offset

45

39

46 results.append({'url': result['unescapedUrl'],

40 results.append({'url': result['unescapedUrl'],

47 'title': result['titleNoFormatting'],

41 'title': result['titleNoFormatting'],

searx/engines/vimeo.py+6

@@ -2,6 +2,8 @@ from urllib import urlencode

2from HTMLParser import HTMLParser

3from lxml import html

4from xpath import extract_text

5+from datetime import datetime

6+from dateutil import parser

5

7

6base_url = 'http://vimeo.com'

8base_url = 'http://vimeo.com'

7search_url = base_url + '/search?{query}'

9search_url = base_url + '/search?{query}'

@@ -10,6 +12,7 @@ content_xpath = None

10title_xpath = None

12title_xpath = None

11results_xpath = ''

13results_xpath = ''

12content_tpl = '<a href="{0}"> <img src="{2}"/> </a>'

14content_tpl = '<a href="{0}"> <img src="{2}"/> </a>'

15+publishedDate_xpath = './/p[@class="meta"]//attribute::datetime'

13

16

14# the cookie set by vimeo contains all the following values,

17# the cookie set by vimeo contains all the following values,

15# but only __utma seems to be requiered

18# but only __utma seems to be requiered

@@ -40,9 +43,12 @@ def response(resp):

40 url = base_url + result.xpath(url_xpath)[0]

43 url = base_url + result.xpath(url_xpath)[0]

41 title = p.unescape(extract_text(result.xpath(title_xpath)))

44 title = p.unescape(extract_text(result.xpath(title_xpath)))

42 thumbnail = extract_text(result.xpath(content_xpath)[0])

45 thumbnail = extract_text(result.xpath(content_xpath)[0])

46+ publishedDate = parser.parse(extract_text(result.xpath(publishedDate_xpath)[0]))

47+

43 results.append({'url': url,

48 results.append({'url': url,

44 'title': title,

49 'title': title,

45 'content': content_tpl.format(url, title, thumbnail),

50 'content': content_tpl.format(url, title, thumbnail),

46 'template': 'videos.html',

51 'template': 'videos.html',

52+ 'publishedDate': publishedDate,

47 'thumbnail': thumbnail})

53 'thumbnail': thumbnail})

48 return results

54 return results

searx/engines/yahoo_news.py+2−3

@@ -6,6 +6,7 @@ from searx.engines.xpath import extract_text, extract_url

6from searx.engines.yahoo import parse_url

7from datetime import datetime, timedelta

8import re

9+from dateutil import parser

9

10

10categories = ['news']

11categories = ['news']

11search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'

12search_url = 'http://news.search.yahoo.com/search?{query}&b={offset}'

@@ -52,9 +53,7 @@ def response(resp):

52 - timedelta(hours=int(timeNumbers[0]))\

53 - timedelta(hours=int(timeNumbers[0]))\

53 - timedelta(minutes=int(timeNumbers[1]))

54 - timedelta(minutes=int(timeNumbers[1]))

54 else:

55 else:

55− # TODO year in string possible?

56− publishedDate = datetime.strptime(publishedDate,

56+ publishedDate = parser.parse(publishedDate)

57− "%b %d %H:%M%p")

58

57

59 if publishedDate.year == 1900:

58 if publishedDate.year == 1900:

60 publishedDate = publishedDate.replace(year=datetime.now().year)

59 publishedDate = publishedDate.replace(year=datetime.now().year)

searx/engines/youtube.py+7

@@ -1,5 +1,7 @@

1from json import loads

2from urllib import urlencode

3+from dateutil import parser

4+from datetime import datetime

3

5

4categories = ['videos']

6categories = ['videos']

5

7

@@ -35,6 +37,10 @@ def response(resp):

35 content = ''

37 content = ''

36 thumbnail = ''

38 thumbnail = ''

37

39

40+#"2013-12-31T15:22:51.000Z"

41+ pubdate = result['published']['$t']

42+ publishedDate = parser.parse(pubdate)

43+

38 if result['media$group']['media$thumbnail']:

44 if result['media$group']['media$thumbnail']:

39 thumbnail = result['media$group']['media$thumbnail'][0]['url']

45 thumbnail = result['media$group']['media$thumbnail'][0]['url']

40 content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail) # noqa

46 content += '<a href="{0}" title="{0}" ><img src="{1}" /></a>'.format(url, thumbnail) # noqa

@@ -48,6 +54,7 @@ def response(resp):

48 'title': title,

54 'title': title,

49 'content': content,

55 'content': content,

50 'template': 'videos.html',

56 'template': 'videos.html',

57+ 'publishedDate': publishedDate,

51 'thumbnail': thumbnail})

58 'thumbnail': thumbnail})

52

59

53 return results

60 return results

searx/templates/opensearch_response_rss.xml+1

@@ -16,6 +16,7 @@

16 <title>{{ r.title }}</title>

17 <link>{{ r.url }}</link>

18 <description>{{ r.content }}</description>

19+ {% if r.pubdate %}<pubDate>{{ r.pubdate }}</pubDate>{% endif %}

19 </item>

20 </item>

20 {% endfor %}

21 {% endfor %}

21 </channel>

22 </channel>

searx/templates/result_templates/videos.html+1

@@ -5,6 +5,7 @@

5

6 <p>

7 <h3 class="result_title"><a href="{{ result.url }}">{{ result.title|safe }}</a></h3>

8+ {% if result.publishedDate %}<p class="published_date">{{ result.publishedDate }}</p>{% endif %}

8 <a href="{{ result.url }}"><img width="400px" src="{{ result.thumbnail }}" title={{ result.title }} alt=" {{ result.title }}"/></a>

9 <a href="{{ result.url }}"><img width="400px" src="{{ result.thumbnail }}" title={{ result.title }} alt=" {{ result.title }}"/></a>

9 <p class="url">{{ result.url }}</p>

10 <p class="url">{{ result.url }}</p>

10 </p>

11 </p>

searx/webapp.py+3−2

@@ -159,8 +159,8 @@ def index():

159

160 # TODO, check if timezone is calculated right

161 if 'publishedDate' in result:

162− if result['publishedDate'] >= datetime.now() - timedelta(days=1):

162+ if result['publishedDate'].replace(tzinfo=None) >= datetime.now() - timedelta(days=1):

163− timedifference = datetime.now() - result['publishedDate']

163+ timedifference = datetime.now() - result['publishedDate'].replace(tzinfo=None)

164 minutes = int((timedifference.seconds / 60) % 60)

165 hours = int(timedifference.seconds / 60 / 60)

166 if hours == 0:

@@ -168,6 +168,7 @@ def index():

168 else:

169 result['publishedDate'] = gettext(u'{hours} hour(s), {minutes} minute(s) ago').format(hours=hours, minutes=minutes) # noqa

170 else:

171+ result['pubdate'] = result['publishedDate'].strftime('%a, %d %b %Y %H:%M:%S %z')

171 result['publishedDate'] = format_date(result['publishedDate'])

172 result['publishedDate'] = format_date(result['publishedDate'])

172

173

173 if search.request_data.get('format') == 'json':

174 if search.request_data.get('format') == 'json':

setup.py+1

@@ -35,6 +35,7 @@ setup(

35 'lxml',

36 'pyyaml',

37 'setuptools',

38+ 'python-dateutil',

38 ],

39 ],

39 extras_require={

40 extras_require={

40 'test': [

41 'test': [