on master help sign in
[fix] yahoo_news engine
commit44ed4424f60533b73f452252f95f1bc00ec48bc8 authorAlexandre Flament <alex@al-f.net> dateSat 2 May 2015, 21:08 (11 years ago) parentbbd83f5
2 files changed+89−89
searx/engines/yahoo_news.py+18−7 searx/tests/engines/test_yahoo_news.py+71−82
searx/engines/yahoo_news.py+18−7
@@ -23,15 +23,15 @@ paging = True
2323language_support = True
2424
2525# search-url
26−search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}' # noqa
26+search_url = 'https://news.search.yahoo.com/search?{query}&b={offset}&{lang}=uh3_news_web_gs_1&pz=10&xargs=0&vl=lang_{lang}' # noqa
2727
2828# specific xpath variables
29−results_xpath = '//div[@class="res"]'
29+results_xpath = '//ol[contains(@class,"searchCenterMiddle")]//li'
3030url_xpath = './/h3/a/@href'
3131title_xpath = './/h3/a'
32−content_xpath = './/div[@class="abstr"]'
33−publishedDate_xpath = './/span[@class="timestamp"]'
34−suggestion_xpath = '//div[@id="satat"]//a'
32+content_xpath = './/div[@class="compText"]'
33+publishedDate_xpath = './/span[contains(@class,"tri")]'
34+suggestion_xpath = '//div[contains(@class,"VerALSOTRY")]//a'
3535
3636
3737# do search-request
@@ -48,11 +48,18 @@ def request(query, params):
4848 lang=language)
4949
5050 # TODO required?
51− params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\
51+ params['cookies']['sB'] = '"v=1&vm=p&fl=1&vl=lang_{lang}&sh=1&pn=10&rw=new'\
5252 .format(lang=language)
5353 return params
5454
5555
56+def sanitize_url(url):
57+ if ".yahoo.com/" in url:
58+ return re.sub(u"\;\_ylt\=.+$", "", url)
59+ else:
60+ return url
61+
62+
5663# get response from search-request
5764def response(resp):
5865 results = []
@@ -61,13 +68,17 @@ def response(resp):
6168
6269 # parse results
6370 for result in dom.xpath(results_xpath):
64− url = parse_url(extract_url(result.xpath(url_xpath), search_url))
71+ urls = result.xpath(url_xpath)
72+ if len(urls) != 1:
73+ continue
74+ url = sanitize_url(parse_url(extract_url(urls, search_url)))
6575 title = extract_text(result.xpath(title_xpath)[0])
6676 content = extract_text(result.xpath(content_xpath)[0])
6777
6878 # parse publishedDate
6979 publishedDate = extract_text(result.xpath(publishedDate_xpath)[0])
7080
81+ # still useful ?
7182 if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
7283 publishedDate = datetime.now() - timedelta(minutes=int(re.match(r'\d+', publishedDate).group())) # noqa
7384 else:
searx/tests/engines/test_yahoo_news.py+71−82
@@ -39,19 +39,24 @@ class TestYahooNewsEngine(SearxTestCase):
3939 self.assertEqual(yahoo_news.response(response), [])
4040
4141 html = """
42− <div class="res">
43− <div>
44− <h3>
45− <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
46− This is
47− the <b>title</b>...
48− </a>
49− </h3>
50− </div>
51− <span class="url">Business via Yahoo! Finance</span> &nbsp; <span class="timestamp">Feb 03 09:45am</span>
52− <div class="abstr">
53− This is the content
54− </div>
42+ <ol class=" reg searchCenterMiddle">
43+ <li class="first">
44+ <div class="compTitle">
45+ <h3>
46+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
47+ This is
48+ the <b>title</b>...
49+ </a>
50+ </h3>
51+ </div>
52+ <div>
53+ <span class="cite">Business via Yahoo!</span>
54+ <span class="tri fc-2nd ml-10">May 01 10:00 AM</span>
55+ </div>
56+ <div class="compText">
57+ This is the content
58+ </div>
59+ </li>
5560 </div>
5661 """
5762 response = mock.Mock(text=html)
@@ -63,48 +68,59 @@ class TestYahooNewsEngine(SearxTestCase):
6368 self.assertEqual(results[0]['content'], 'This is the content')
6469
6570 html = """
66− <div class="res">
67− <div>
68− <h3>
69− <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
70− This is
71− the <b>title</b>...
72− </a>
73− </h3>
74− </div>
75− <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">2 hours, 22 minutes ago</span>
76− <div class="abstr">
77− This is the content
78− </div>
79− </div>
80− <div class="res">
81− <div>
82− <h3>
83− <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
84− This is
85− the <b>title</b>...
86− </a>
87− </h3>
88− </div>
89− <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">22 minutes ago</span>
90− <div class="abstr">
91− This is the content
92− </div>
93− </div>
94− <div class="res">
95− <div>
96− <h3>
97− <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
98− This is
99− the <b>title</b>...
100− </a>
101− </h3>
102− </div>
103− <span class="url">Business via Yahoo!</span> &nbsp; <span class="timestamp">Feb 03 09:45am 1900</span>
104− <div class="abstr">
105− This is the content
106− </div>
107− </div>
71+ <ol class=" reg searchCenterMiddle">
72+ <li class="first">
73+ <div class="compTitle">
74+ <h3>
75+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
76+ This is
77+ the <b>title</b>...
78+ </a>
79+ </h3>
80+ </div>
81+ <div>
82+ <span class="cite">Business via Yahoo!</span>
83+ <span class="tri fc-2nd ml-10">2 hours, 22 minutes ago</span>
84+ </div>
85+ <div class="compText">
86+ This is the content
87+ </div>
88+ </li>
89+ <li>
90+ <div class="compTitle">
91+ <h3>
92+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
93+ This is
94+ the <b>title</b>...
95+ </a>
96+ </h3>
97+ </div>
98+ <div>
99+ <span class="cite">Business via Yahoo!</span>
100+ <span class="tri fc-2nd ml-10">22 minutes ago</span>
101+ </div>
102+ <div class="compText">
103+ This is the content
104+ </div>
105+ </li>
106+ <li>
107+ <div class="compTitle">
108+ <h3>
109+ <a class="yschttl spt" href="http://this.is.the.url" target="_blank">
110+ This is
111+ the <b>title</b>...
112+ </a>
113+ </h3>
114+ </div>
115+ <div>
116+ <span class="cite">Business via Yahoo!</span>
117+ <span class="tri fc-2nd ml-10">Feb 03 09:45AM 1900</span>
118+ </div>
119+ <div class="compText">
120+ This is the content
121+ </div>
122+ </li>
123+ </ol>
108124 """
109125 response = mock.Mock(text=html)
110126 results = yahoo_news.response(response)
@@ -114,30 +130,3 @@ class TestYahooNewsEngine(SearxTestCase):
114130 self.assertEqual(results[0]['url'], 'http://this.is.the.url/')
115131 self.assertEqual(results[0]['content'], 'This is the content')
116132 self.assertEqual(results[2]['publishedDate'].year, datetime.now().year)
117−
118− html = """
119− <li class="b_algo" u="0|5109|4755453613245655|UAGjXgIrPH5yh-o5oNHRx_3Zta87f_QO">
120− <div Class="sa_mc">
121− <div class="sb_tlst">
122− <h2>
123− <a href="http://this.should.be.the.link/" h="ID=SERP,5124.1">
124− <strong>This</strong> should be the title</a>
125− </h2>
126− </div>
127− <div class="sb_meta">
128− <cite>
129− <strong>this</strong>.meta.com</cite>
130− <span class="c_tlbxTrg">
131− <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5125.1">
132− </span>
133− </span>
134− </div>
135− <p>
136− <strong>This</strong> should be the content.</p>
137− </div>
138− </li>
139− """
140− response = mock.Mock(text=html)
141− results = yahoo_news.response(response)
142− self.assertEqual(type(results), list)
143− self.assertEqual(len(results), 0)