@@ -5,10 +5,10 @@ from cgi import escape
55from lxml.etree import _ElementStringResult
66
77search_url = None
8−results_xpath = None
98url_xpath = None
109content_xpath = None
1110title_xpath = None
11+results_xpath = ''
1212
1313def extract_url(xpath_results):
1414 url = ''
@@ -26,7 +26,7 @@ def extract_url(xpath_results):
2626 else:
2727 url = xpath_results[0].attrib.get('href')
2828 else:
29− raise Exception('Cannot handle xpath url resultset')
29+ url = xpath_results.attrib.get('href')
3030 if not url.startswith('http://') or not url.startswith('https://'):
3131 url = 'http://'+url
3232 parsed_url = urlparse(url)
@@ -45,10 +45,15 @@ def response(resp):
4545 results = []
4646 dom = html.fromstring(resp.text)
4747 query = resp.search_params['query']
48− for result in dom.xpath(results_xpath):
49− url = extract_url(result.xpath(url_xpath))
50− title = ' '.join(result.xpath(title_xpath))
51− content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))
52− results.append({'url': url, 'title': title, 'content': content})
48+ if results_xpath:
49+ for result in dom.xpath(results_xpath):
50+ url = extract_url(result.xpath(url_xpath))
51+ title = ' '.join(result.xpath(title_xpath))
52+ content = escape(' '.join(result.xpath(content_xpath))).replace(query, '<b>{0}</b>'.format(query))
53+ results.append({'url': url, 'title': title, 'content': content})
54+ else:
55+ for content, url, title in zip(dom.xpath(content_xpath), map(extract_url, dom.xpath(url_xpath)), dom.xpath(title_xpath)):
56+ results.append({'url': url, 'title': title, 'content': content})
57+
5358
5459 return results