@@ -5,10 +5,10 @@ from cgi import escape
5from lxml.etree import _ElementStringResult
5from lxml.etree import _ElementStringResult
7search_url = None
7search_url = None
9url_xpath = None
8url_xpath = None
10content_xpath = None
9content_xpath = None
11title_xpath = None
10title_xpath = None
13def extract_url(xpath_results):
13def extract_url(xpath_results):
@@ -26,7 +26,7 @@ def extract_url(xpath_results):
27 url = xpath_results[0].attrib.get('href')
27 url = xpath_results[0].attrib.get('href')
29− raise Exception('Cannot handle xpath url resultset')
29+ url = xpath_results.attrib.get('href')
30 if not url.startswith('http://') or not url.startswith('https://'):
30 if not url.startswith('http://') or not url.startswith('https://'):
31 url = 'http://'+url
31 url = 'http://'+url
32 parsed_url = urlparse(url)
32 parsed_url = urlparse(url)
@@ -45,10 +45,15 @@ def response(resp):
45 results = []
45 results = []
46 dom = html.fromstring(resp.text)
46 dom = html.fromstring(resp.text)
47 query = resp.search_params['query']
47 query = resp.search_params['query']
48− for result in dom.xpath(results_xpath):
49+ for result in dom.xpath(results_xpath):
49− url = extract_url(result.xpath(url_xpath))
50+ url = extract_url(result.xpath(url_xpath))
50− title = ' '.join(result.xpath(title_xpath))
51+ title = ' '.join(result.xpath(title_xpath))
51− content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))
52+ content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))
53+ results.append({'url': url, 'title': title, 'content': content})
55+ for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):
52− results.append({'url': url, 'title': title, 'content': content})
56+ results.append({'url': url, 'title': title, 'content': content})
54 return results
59 return results