93c0c49 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

add time range search with yahoo

93c0c49Noemi Vanyi10 years ago

searx/engines/__init__.py+2−1

compare with master

unifiedsplit

@@ -42,7 +42,8 @@ engine_default_args = {'paging': False,

42 'shortcut': '-',

43 'disabled': False,

44 'suspend_end_time': 0,

45− 'continuous_errors': 0}

45+ 'continuous_errors': 0,

46+ 'time_range_support': False}

48def load_module(filename):

49def load_module(filename):

searx/engines/yahoo.py+25−8

@@ -20,10 +20,12 @@ from searx.engines.xpath import extract_text, extract_url

20categories = ['general']

21paging = True

22language_support = True

23+time_range_support = True

24# search-url

25# search-url

25base_url = 'https://search.yahoo.com/'

26base_url = 'https://search.yahoo.com/'

26search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'

27search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'

28+search_url_with_time = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}&age={age}&btf={btf}&fr2=time'

28# specific xpath variables

30# specific xpath variables

29results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"

31results_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' Sr ')]"

@@ -32,6 +34,9 @@ title_xpath = './/h3/a'

32content_xpath = './/div[@class="compText aAbs"]'

34content_xpath = './/div[@class="compText aAbs"]'

33suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"

35suggestion_xpath = "//div[contains(concat(' ', normalize-space(@class), ' '), ' AlsoTry ')]//a"

37+time_range_dict = {'day': ['1d', 'd'],

38+ 'week': ['1w', 'w'],

39+ 'month': ['1m', 'm']}

36# remove yahoo-specific tracking-url

41# remove yahoo-specific tracking-url

37def parse_url(url_string):

42def parse_url(url_string):

@@ -51,18 +56,30 @@ def parse_url(url_string):

51 return unquote(url_string[start:end])

56 return unquote(url_string[start:end])

59+def _get_url(query, offset, language, time_range):

60+ if time_range:

61+ return base_url + search_url_with_time.format(offset=offset,

62+ query=urlencode({'p': query}),

63+ lang=language,

64+ age=time_range_dict[time_range][0],

65+ btf=time_range_dict[time_range][1])

66+ return base_url + search_url.format(offset=offset,

67+ query=urlencode({'p': query}),

68+ lang=language)

69+

70+

71+def _get_language(params):

72+ if params['language'] == 'all':

73+ return 'en'

74+ return params['language'].split('_')[0]

75+

76+

54# do search-request

77# do search-request

55def request(query, params):

78def request(query, params):

56 offset = (params['pageno'] - 1) * 10 + 1

79 offset = (params['pageno'] - 1) * 10 + 1

80+ language = _get_language(params)

58− if params['language'] == 'all':

82+ params['url'] = _get_url(query, offset, language, params['time_range'])

59− language = 'en'

60− else:

61− language = params['language'].split('_')[0]

62−

63− params['url'] = base_url + search_url.format(offset=offset,

64− query=urlencode({'p': query}),

65− lang=language)

67 # TODO required?

84 # TODO required?

68 params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\

85 params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\

searx/search.py+7−2

@@ -138,6 +138,7 @@ class Search(object):

138 self.paging = False

139 self.pageno = 1

140 self.lang = 'all'

141+ self.time_range = None

141

142

142 # set blocked engines

143 # set blocked engines

143 self.disabled_engines = request.preferences.engines.get_disabled()

144 self.disabled_engines = request.preferences.engines.get_disabled()

@@ -178,9 +179,9 @@ class Search(object):

178 if len(query_obj.languages):

179 if len(query_obj.languages):

179 self.lang = query_obj.languages[-1]

180 self.lang = query_obj.languages[-1]

180

181

181− self.engines = query_obj.engines

182+ self.time_range = self.request_data.get('time_range')

182

183

183− self.categories = []

184+ self.engines = query_obj.engines

184

185

185 # if engines are calculated from query,

186 # if engines are calculated from query,

186 # set categories by using that informations

187 # set categories by using that informations

@@ -279,6 +280,9 @@ class Search(object):

279 if self.lang != 'all' and not engine.language_support:

280 if self.lang != 'all' and not engine.language_support:

280 continue

281 continue

281

282

283+ if self.time_range and not engine.time_range_support:

284+ continue

285+

282 # set default request parameters

286 # set default request parameters

283 request_params = default_request_params()

287 request_params = default_request_params()

284 request_params['headers']['User-Agent'] = user_agent

288 request_params['headers']['User-Agent'] = user_agent

@@ -293,6 +297,7 @@ class Search(object):

293

297

294 # 0 = None, 1 = Moderate, 2 = Strict

298 # 0 = None, 1 = Moderate, 2 = Strict

295 request_params['safesearch'] = request.preferences.get_value('safesearch')

299 request_params['safesearch'] = request.preferences.get_value('safesearch')

300+ request_params['time_range'] = self.time_range

296

301

297 # update request parameters dependent on

302 # update request parameters dependent on

298 # search-engine (contained in engines folder)

303 # search-engine (contained in engines folder)

searx/webapp.py+1

@@ -459,6 +459,7 @@ def index():

459 paging=search.paging,

460 number_of_results=format_decimal(number_of_results),

461 pageno=search.pageno,

462+ time_range=search.time_range,

462 base_url=get_base_url(),

463 base_url=get_base_url(),

463 suggestions=search.result_container.suggestions,

464 suggestions=search.result_container.suggestions,

464 answers=search.result_container.answers,

465 answers=search.result_container.answers,