@@ -19,14 +19,13 @@ def response(resp):
19 global base_url
19 global base_url
20 results = []
20 results = []
21 dom = html.fromstring(resp.content)
21 dom = html.fromstring(resp.content)
22+ # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]
23+ # not ads : div[@class="result"] are the direct childs of div[@id="results"]
22− for result in dom.xpath('//div[@class="result"]'):
24+ for result in dom.xpath('//div[@id="results"]/div[@class="result"]'):
23 link = result.xpath('.//h3/a')[0]
25 link = result.xpath('.//h3/a')[0]
24 url = link.attrib.get('href')
26 url = link.attrib.get('href')
25 parsed_url = urlparse(url)
27 parsed_url = urlparse(url)
26− # TODO better google link detection
27− if parsed_url.netloc.find('www.google.com') >= 0:
29− title = ' '.join(link.xpath('.//text()'))
28+ title = link.text_content()
30− content = escape(' '.join(result.xpath('.//p[@class="desc"]//text()')))
29+ content = result.xpath('./p[@class="desc"]')[0].text_content()
31 results.append({'url': url, 'title': title, 'content': content})
30 results.append({'url': url, 'title': title, 'content': content})
32 return results
31 return results