@@ -31,8 +31,6 @@ if xpath_results is a string element, then it's already done
3131def extract_text(xpath_results):
3232 if type(xpath_results) == list:
3333 # it's list of result : concat everything using recursive call
34− if not xpath_results:
35− raise Exception('Empty url resultset')
3634 result = ''
3735 for e in xpath_results:
3836 result = result + extract_text(e)
@@ -48,6 +46,8 @@ def extract_text(xpath_results):
4846
4947
5048def extract_url(xpath_results, search_url):
49+ if xpath_results == []:
50+ raise Exception('Empty url resultset')
5151 url = extract_text(xpath_results)
5252
5353 if url.startswith('//'):
@@ -103,8 +103,8 @@ def response(resp):
103103 if results_xpath:
104104 for result in dom.xpath(results_xpath):
105105 url = extract_url(result.xpath(url_xpath), search_url)
106− title = extract_text(result.xpath(title_xpath)[0])
107− content = extract_text(result.xpath(content_xpath)[0])
106+ title = extract_text(result.xpath(title_xpath))
107+ content = extract_text(result.xpath(content_xpath))
108108 results.append({'url': url, 'title': title, 'content': content})
109109 else:
110110 for url, title, content in zip(