git.iko.soy/searx
on master help sign in
Merge pull request #783 from kvch/time-range-search-year

add year support to engines which support it

commite58949b76fac7aa93341523ff0e2f35e0a03e057 authorAdam Tauber <asciimoo@gmail.com> dateMon 12 Dec 2016, 10:06 (10 years ago) parent6b6007fb034356
16 files changed+82−7
compare with master
unifiedsplit
searx/engines/bing_images.py+2−1 searx/engines/bing_news.py+3 searx/engines/deviantart.py+3 searx/engines/duckduckgo.py+3 searx/engines/flickr_noapi.py+2−1 searx/engines/google.py+2−1 searx/engines/google_images.py+9−1 searx/engines/google_news.py+2−1 searx/engines/yahoo.py+3 searx/engines/youtube_noapi.py+2−1 searx/templates/oscar/time-range.html+3 tests/unit/engines/test_bing_news.py+7 tests/unit/engines/test_deviantart.py+8−1 tests/unit/engines/test_duckduckgo.py+7 tests/unit/engines/test_yahoo.py+7 tests/unit/engines/test_youtube_noapi.py+19
searx/engines/bing_images.py+2−1
@@ -33,7 +33,8 @@ time_range_string = '&qft=+filterui:age-lt{interval}'
3333thumb_url = "https://www.bing.com/th?id={ihk}"
3434time_range_dict = {'day': '1440',
3535 'week': '10080',
36− 'month': '43200'}
36+ 'month': '43200',
37+ 'year': '525600'}
3738
3839# safesearch definitions
3940safesearch_types = {2: 'STRICT',
searx/engines/bing_news.py+3
@@ -66,6 +66,9 @@ def _get_url(query, language, offset, time_range):
6666
6767# do search-request
6868def request(query, params):
69+ if params['time_range'] and params['time_range'] not in time_range_dict:
70+ return params
71+
6972 offset = (params['pageno'] - 1) * 10 + 1
7073
7174 if params['language'] == 'all':
searx/engines/deviantart.py+3
@@ -34,6 +34,9 @@ time_range_dict = {'day': 11,
3434
3535# do search-request
3636def request(query, params):
37+ if params['time_range'] and params['time_range'] not in time_range_dict:
38+ return params
39+
3740 offset = (params['pageno'] - 1) * 24
3841
3942 params['url'] = search_url.format(offset=offset,
searx/engines/duckduckgo.py+3
@@ -41,6 +41,9 @@ content_xpath = './/a[@class="result__snippet"]'
4141
4242# do search-request
4343def request(query, params):
44+ if params['time_range'] and params['time_range'] not in time_range_dict:
45+ return params
46+
4447 offset = (params['pageno'] - 1) * 30
4548
4649 if params['language'] == 'all':
searx/engines/flickr_noapi.py+2−1
@@ -34,7 +34,8 @@ paging = True
3434time_range_support = True
3535time_range_dict = {'day': 60 * 60 * 24,
3636 'week': 60 * 60 * 24 * 7,
37− 'month': 60 * 60 * 24 * 7 * 4}
37+ 'month': 60 * 60 * 24 * 7 * 4,
38+ 'year': 60 * 60 * 24 * 7 * 52}
3839
3940
4041def build_flickr_url(user_id, photo_id):
searx/engines/google.py+2−1
@@ -95,7 +95,8 @@ search_url = ('https://{hostname}' +
9595time_range_search = "&tbs=qdr:{range}"
9696time_range_dict = {'day': 'd',
9797 'week': 'w',
98− 'month': 'm'}
98+ 'month': 'm',
99+ 'year': 'y'}
99100
100101# other URLs
101102map_hostname_start = 'maps.google.'
searx/engines/google_images.py+9−1
@@ -10,10 +10,12 @@
1010 @parse url, title, img_src
1111"""
1212
13+from datetime import date, timedelta
1314from urllib import urlencode
1415from json import loads
1516from lxml import html
1617
18+
1719# engine dependent config
1820categories = ['images']
1921paging = True
@@ -29,6 +31,7 @@ search_url = 'https://www.google.com/search'\
2931 '&yv=2'\
3032 '&{search_options}'
3133time_range_attr = "qdr:{range}"
34+time_range_custom_attr = "cdr:1,cd_min:{start},cd_max{end}"
3235time_range_dict = {'day': 'd',
3336 'week': 'w',
3437 'month': 'm'}
@@ -36,7 +39,6 @@ time_range_dict = {'day': 'd',
3639
3740# do search-request
3841def request(query, params):
39−
4042 search_options = {
4143 'ijn': params['pageno'] - 1,
4244 'start': (params['pageno'] - 1) * number_of_results
@@ -44,6 +46,12 @@ def request(query, params):
4446
4547 if params['time_range'] in time_range_dict:
4648 search_options['tbs'] = time_range_attr.format(range=time_range_dict[params['time_range']])
49+ elif params['time_range'] == 'year':
50+ now = date.today()
51+ then = now - timedelta(days=365)
52+ start = then.strftime('%m/%d/%Y')
53+ end = now.strftime('%m/%d/%Y')
54+ search_options['tbs'] = time_range_custom_attr.format(start=start, end=end)
4755
4856 if safesearch and params['safesearch']:
4957 search_options['safe'] = 'on'
searx/engines/google_news.py+2−1
@@ -29,7 +29,8 @@ search_url = 'https://www.google.com/search'\
2929time_range_attr = "qdr:{range}"
3030time_range_dict = {'day': 'd',
3131 'week': 'w',
32− 'month': 'm'}
32+ 'month': 'm',
33+ 'year': 'y'}
3334
3435
3536# do search-request
searx/engines/yahoo.py+3
@@ -77,6 +77,9 @@ def _get_language(params):
7777
7878# do search-request
7979def request(query, params):
80+ if params['time_range'] and params['time_range'] not in time_range_dict:
81+ return params
82+
8083 offset = (params['pageno'] - 1) * 10 + 1
8184 language = _get_language(params)
8285
searx/engines/youtube_noapi.py+2−1
@@ -25,7 +25,8 @@ search_url = base_url + '?search_query={query}&page={page}'
2525time_range_url = '&sp=EgII{time_range}%253D%253D'
2626time_range_dict = {'day': 'Ag',
2727 'week': 'Aw',
28− 'month': 'BA'}
28+ 'month': 'BA',
29+ 'year': 'BQ'}
2930
3031embedded_url = '<iframe width="540" height="304" ' +\
3132 'data-src="//www.youtube-nocookie.com/embed/{videoid}" ' +\
searx/templates/oscar/time-range.html+3
@@ -11,4 +11,7 @@
1111 <option id="time-range-month" value="month" {{ "selected" if time_range=="month" else ""}}>
1212 {{ _('Last month') }}
1313 </option>
14+ <option id="time-range-year" value="year" {{ "selected" if time_range=="year" else ""}}>
15+ {{ _('Last year') }}
16+ </option>
1417</select>
tests/unit/engines/test_bing_news.py+7
@@ -23,6 +23,13 @@ class TestBingNewsEngine(SearxTestCase):
2323 params = bing_news.request(query, dicto)
2424 self.assertIn('en', params['url'])
2525
26+ def test_no_url_in_request_year_time_range(self):
27+ dicto = defaultdict(dict)
28+ query = 'test_query'
29+ dicto['time_range'] = 'year'
30+ params = bing_news.request(query, dicto)
31+ self.assertEqual({}, params['url'])
32+
2633 def test_response(self):
2734 self.assertRaises(AttributeError, bing_news.response, None)
2835 self.assertRaises(AttributeError, bing_news.response, [])
tests/unit/engines/test_deviantart.py+8−1
@@ -7,8 +7,8 @@ from searx.testing import SearxTestCase
77class TestDeviantartEngine(SearxTestCase):
88
99 def test_request(self):
10− query = 'test_query'
1110 dicto = defaultdict(dict)
11+ query = 'test_query'
1212 dicto['pageno'] = 0
1313 dicto['time_range'] = ''
1414 params = deviantart.request(query, dicto)
@@ -16,6 +16,13 @@ class TestDeviantartEngine(SearxTestCase):
1616 self.assertTrue(query in params['url'])
1717 self.assertTrue('deviantart.com' in params['url'])
1818
19+ def test_no_url_in_request_year_time_range(self):
20+ dicto = defaultdict(dict)
21+ query = 'test_query'
22+ dicto['time_range'] = 'year'
23+ params = deviantart.request(query, dicto)
24+ self.assertEqual({}, params['url'])
25+
1926 def test_response(self):
2027 self.assertRaises(AttributeError, deviantart.response, None)
2128 self.assertRaises(AttributeError, deviantart.response, [])
tests/unit/engines/test_duckduckgo.py+7
@@ -19,6 +19,13 @@ class TestDuckduckgoEngine(SearxTestCase):
1919 self.assertIn('duckduckgo.com', params['url'])
2020 self.assertIn('ch-de', params['url'])
2121
22+ def test_no_url_in_request_year_time_range(self):
23+ dicto = defaultdict(dict)
24+ query = 'test_query'
25+ dicto['time_range'] = 'year'
26+ params = duckduckgo.request(query, dicto)
27+ self.assertEqual({}, params['url'])
28+
2229 def test_response(self):
2330 self.assertRaises(AttributeError, duckduckgo.response, None)
2431 self.assertRaises(AttributeError, duckduckgo.response, [])
tests/unit/engines/test_yahoo.py+7
@@ -46,6 +46,13 @@ class TestYahooEngine(SearxTestCase):
4646 self.assertIn('en', params['cookies']['sB'])
4747 self.assertIn('en', params['url'])
4848
49+ def test_no_url_in_request_year_time_range(self):
50+ dicto = defaultdict(dict)
51+ query = 'test_query'
52+ dicto['time_range'] = 'year'
53+ params = yahoo.request(query, dicto)
54+ self.assertEqual({}, params['url'])
55+
4956 def test_response(self):
5057 self.assertRaises(AttributeError, yahoo.response, None)
5158 self.assertRaises(AttributeError, yahoo.response, [])
tests/unit/engines/test_youtube_noapi.py+19
@@ -17,6 +17,25 @@ class TestYoutubeNoAPIEngine(SearxTestCase):
1717 self.assertIn(query, params['url'])
1818 self.assertIn('youtube.com', params['url'])
1919
20+ def test_time_range_search(self):
21+ dicto = defaultdict(dict)
22+ query = 'test_query'
23+ dicto['time_range'] = 'year'
24+ params = youtube_noapi.request(query, dicto)
25+ self.assertIn('&sp=EgIIBQ%253D%253D', params['url'])
26+
27+ dicto['time_range'] = 'month'
28+ params = youtube_noapi.request(query, dicto)
29+ self.assertIn('&sp=EgIIBA%253D%253D', params['url'])
30+
31+ dicto['time_range'] = 'week'
32+ params = youtube_noapi.request(query, dicto)
33+ self.assertIn('&sp=EgIIAw%253D%253D', params['url'])
34+
35+ dicto['time_range'] = 'day'
36+ params = youtube_noapi.request(query, dicto)
37+ self.assertIn('&sp=EgIIAg%253D%253D', params['url'])
38+
2039 def test_response(self):
2140 self.assertRaises(AttributeError, youtube_noapi.response, None)
2241 self.assertRaises(AttributeError, youtube_noapi.response, [])