e23c8f9 · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #746 from kvch/moar-time-range-support

Support time range search in more engines

commite23c8f954b6ae0f1100167d39b73e2037384c154 authorAdam Tauber <asciimoo@gmail.com> dateTue 1 Nov 2016, 20:01 (10 years ago) parent6243639 1490d6b

9 files changed+53−8

compare with master

unifiedsplit

searx/engines/bing_images.py+7 searx/engines/bing_news.py+19−5 searx/engines/flickr_noapi.py+14−3 searx/engines/youtube_noapi.py+7 searx/templates/oscar/preferences.html+2 tests/unit/engines/test_bing_images.py+1 tests/unit/engines/test_bing_news.py+1 tests/unit/engines/test_flickr_noapi.py+1 tests/unit/engines/test_youtube_noapi.py+1

searx/engines/bing_images.py+7

@@ -24,11 +24,16 @@ import re

2424categories = ['images']

2525paging = True

2626safesearch = True

27+time_range_support = True

2728

2829# search-url

2930base_url = 'https://www.bing.com/'

3031search_string = 'images/search?{query}&count=10&first={offset}'

32+time_range_string = '&qft=+filterui:age-lt{interval}'

3133thumb_url = "https://www.bing.com/th?id={ihk}"

34+time_range_dict = {'day': '1440',

35+ 'week': '10080',

36+ 'month': '43200'}

3237

3338# safesearch definitions

3439safesearch_types = {2: 'STRICT',

@@ -58,6 +63,8 @@ def request(query, params):

5863 '&ADLT=' + safesearch_types.get(params['safesearch'], 'DEMOTE')

5964

6065 params['url'] = base_url + search_path

66+ if params['time_range'] in time_range_dict:

67+ params['url'] += time_range_string.format(interval=time_range_dict[params['time_range']])

6168

6269 return params

6370

searx/engines/bing_news.py+19−5

@@ -22,10 +22,15 @@ from searx.utils import list_get

2222categories = ['news']

2323paging = True

2424language_support = True

25+time_range_support = True

2526

2627# search-url

2728base_url = 'https://www.bing.com/'

2829search_string = 'news/search?{query}&first={offset}&format=RSS'

30+search_string_with_time = 'news/search?{query}&first={offset}&qft=interval%3d"{interval}"&format=RSS'

31+time_range_dict = {'day': '7',

32+ 'week': '8',

33+ 'month': '9'}

2934

3035

3136# remove click

@@ -46,6 +51,19 @@ def image_url_cleanup(url_string):

4651 return url_string

4752

4853

54+def _get_url(query, language, offset, time_range):

55+ if time_range in time_range_dict:

56+ search_path = search_string_with_time.format(

57+ query=urlencode({'q': query, 'setmkt': language}),

58+ offset=offset,

59+ interval=time_range_dict[time_range])

60+ else:

61+ search_path = search_string.format(

62+ query=urlencode({'q': query, 'setmkt': language}),

63+ offset=offset)

64+ return base_url + search_path

65+

66+

4967# do search-request

5068def request(query, params):

5169 offset = (params['pageno'] - 1) * 10 + 1

@@ -55,11 +73,7 @@ def request(query, params):

5573 else:

5674 language = params['language'].replace('_', '-')

5775

58− search_path = search_string.format(

59− query=urlencode({'q': query, 'setmkt': language}),

60− offset=offset)

61−

62− params['url'] = base_url + search_path

76+ params['url'] = _get_url(query, language, offset, params['time_range'])

6377

6478 return params

6579

searx/engines/flickr_noapi.py+14−3

@@ -14,6 +14,7 @@

1414

1515from urllib import urlencode

1616from json import loads

17+from time import time

1718import re

1819from searx.engines import logger

1920

@@ -24,21 +25,31 @@ categories = ['images']

2425

2526url = 'https://www.flickr.com/'

2627search_url = url + 'search?{query}&page={page}'

28+time_range_url = '&min_upload_date={start}&max_upload_date={end}'

2729photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'

2830regex = re.compile(r"\"search-photos-lite-models\",\"photos\":(.*}),\"totalItems\":", re.DOTALL)

2931image_sizes = ('o', 'k', 'h', 'b', 'c', 'z', 'n', 'm', 't', 'q', 's')

3032

3133paging = True

34+time_range_support = True

35+time_range_dict = {'day': 60 * 60 * 24,

36+ 'week': 60 * 60 * 24 * 7,

37+ 'month': 60 * 60 * 24 * 7 * 4}

3238

3339

3440def build_flickr_url(user_id, photo_id):

3541 return photo_url.format(userid=user_id, photoid=photo_id)

3642

3743

38−def request(query, params):

39− params['url'] = search_url.format(query=urlencode({'text': query}),

40− page=params['pageno'])

44+def _get_time_range_url(time_range):

45+ if time_range in time_range_dict:

46+ return time_range_url.format(start=time(), end=str(int(time()) - time_range_dict[time_range]))

47+ return ''

48+

4149

50+def request(query, params):

51+ params['url'] = (search_url.format(query=urlencode({'text': query}), page=params['pageno'])

52+ + _get_time_range_url(params['time_range']))

4253 return params

4354

4455

searx/engines/youtube_noapi.py+7

@@ -17,10 +17,15 @@ from searx.utils import list_get

1717categories = ['videos', 'music']

1818paging = True

1919language_support = False

20+time_range_support = True

2021

2122# search-url

2223base_url = 'https://www.youtube.com/results'

2324search_url = base_url + '?search_query={query}&page={page}'

25+time_range_url = '&sp=EgII{time_range}%253D%253D'

26+time_range_dict = {'day': 'Ag',

27+ 'week': 'Aw',

28+ 'month': 'BA'}

2429

2530embedded_url = '<iframe width="540" height="304" ' +\

2631 'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\

@@ -47,6 +52,8 @@ def extract_text_from_dom(result, xpath):

4752def request(query, params):

4853 params['url'] = search_url.format(query=quote_plus(query),

4954 page=params['pageno'])

55+ if params['time_range'] in time_range_dict:

56+ params['url'] += time_range_url.format(time_range=time_range_dict[params['time_range']])

5057

5158 return params

5259

searx/templates/oscar/preferences.html+2

@@ -158,6 +158,7 @@

158158 <th>{{ _("Engine name") }}</th>

159159 <th>{{ _("Shortcut") }}</th>

160160 <th>{{ _("SafeSearch") }}</th>

161+ <th>{{ _("Time range") }}</th>

161162 <th>{{ _("Avg. time") }}</th>

162163 <th>{{ _("Max time") }}</th>

163164 {% else %}

@@ -179,6 +180,7 @@

179180 <th>{{ search_engine.name }}</th>

180181 <td>{{ shortcuts[search_engine.name] }}</td>

181182 <td><input type="checkbox" {{ "checked" if search_engine.safesearch==True else ""}} readonly="readonly" disabled="disabled"></td>

183+ <td><input type="checkbox" {{ "checked" if search_engine.time_range_support==True else ""}} readonly="readonly" disabled="disabled"></td>

182184 <td class="{{ 'danger' if stats[search_engine.name]['warn_time'] else '' }}">{{ 'N/A' if stats[search_engine.name].time==None else stats[search_engine.name].time }}</td>

183185 <td class="{{ 'danger' if stats[search_engine.name]['warn_timeout'] else '' }}">{{ search_engine.timeout }}</td>

184186 {% else %}

tests/unit/engines/test_bing_images.py+1

@@ -13,6 +13,7 @@ class TestBingImagesEngine(SearxTestCase):

1313 dicto['pageno'] = 1

1414 dicto['language'] = 'fr_FR'

1515 dicto['safesearch'] = 1

16+ dicto['time_range'] = ''

1617 params = bing_images.request(query, dicto)

1718 self.assertTrue('url' in params)

1819 self.assertTrue(query in params['url'])

tests/unit/engines/test_bing_news.py+1

@@ -12,6 +12,7 @@ class TestBingNewsEngine(SearxTestCase):

1212 dicto = defaultdict(dict)

1313 dicto['pageno'] = 1

1414 dicto['language'] = 'fr_FR'

15+ dicto['time_range'] = ''

1516 params = bing_news.request(query, dicto)

1617 self.assertIn('url', params)

1718 self.assertIn(query, params['url'])

tests/unit/engines/test_flickr_noapi.py+1

@@ -15,6 +15,7 @@ class TestFlickrNoapiEngine(SearxTestCase):

1515 query = 'test_query'

1616 dicto = defaultdict(dict)

1717 dicto['pageno'] = 1

18+ dicto['time_range'] = ''

1819 params = flickr_noapi.request(query, dicto)

1920 self.assertIn('url', params)

2021 self.assertIn(query, params['url'])

tests/unit/engines/test_youtube_noapi.py+1

@@ -11,6 +11,7 @@ class TestYoutubeNoAPIEngine(SearxTestCase):

1111 query = 'test_query'

1212 dicto = defaultdict(dict)

1313 dicto['pageno'] = 0

14+ dicto['time_range'] = ''

1415 params = youtube_noapi.request(query, dicto)

1516 self.assertIn('url', params)

1617 self.assertIn(query, params['url'])