@@ -116,12 +116,12 @@ g_section_with_header = './g-section-with-header'
116# the title is a h3 tag relative to the result group
116# the title is a h3 tag relative to the result group
117title_xpath = './/h3[1]'
117title_xpath = './/h3[1]'
119−# in the result group there is <div class="r" ../> it's first child is a <a
119+# in the result group there is <div class="yuRUbf" ../> it's first child is a <a
120−# href=...> (on some results, the <a> is the first "descendant", not ""child")
120+# href=...>
121−href_xpath = './/div[@class="r"]//a/@href'
121+href_xpath = './/div[@class="yuRUbf"]//a/@href'
123−# in the result group there is <div class="s" ../> containing he *content*
123+# in the result group there is <div class="IsZvec" ../> containing he *content*
124−content_xpath = './/div[@class="s"]'
124+content_xpath = './/div[@class="IsZvec"]'
126# Suggestions are links placed in a *card-section*, we extract only the text
126# Suggestions are links placed in a *card-section*, we extract only the text
127# from the links not the links itself.
127# from the links not the links itself.
@@ -249,7 +249,12 @@ def response(resp):
252− title = extract_text(eval_xpath(result, title_xpath)[0])
252+ title_tag = eval_xpath(result, title_xpath)
254+ # this not one of the common google results *section*
255+ logger.debug('ingoring <div class="g" ../> section: missing title')
257+ title = extract_text(title_tag[0])
253 url = eval_xpath(result, href_xpath)[0]
258 url = eval_xpath(result, href_xpath)[0]
254 content = extract_text_from_dom(result, content_xpath)
259 content = extract_text_from_dom(result, content_xpath)
255 results.append({
260 results.append({