9ee8e55 · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[fix] bing images

9ee8e55marc9 years ago

searx/engines/bing_images.py+11−10

compare with master

unifiedsplit

@@ -31,7 +31,6 @@ time_range_support = True

31base_url = 'https://www.bing.com/'

32search_string = 'images/search?{query}&count=10&first={offset}'

33time_range_string = '&qft=+filterui:age-lt{interval}'

34−thumb_url = "https://www.bing.com/th?id={ihk}"

35time_range_dict = {'day': '1440',

34time_range_dict = {'day': '1440',

36 'week': '10080',

35 'week': '10080',

37 'month': '43200',

36 'month': '43200',

@@ -78,30 +77,32 @@ def response(resp):

78 dom = html.fromstring(resp.text)

77 dom = html.fromstring(resp.text)

80 # parse results

79 # parse results

81− for result in dom.xpath('//div[@class="dg_u"]/div'):

80+ for result in dom.xpath('//div[@id="mmComponent_images_1"]/ul/li/div/div[@class="imgpt"]'):

82 link = result.xpath('./a')[0]

81 link = result.xpath('./a')[0]

83+ # TODO find actual title

84+ title = link.xpath('.//img/@alt')[0]

85+

84 # parse json-data (it is required to add a space, to make it parsable)

86 # parse json-data (it is required to add a space, to make it parsable)

85 json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m')))

87 json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('m')))

87− title = link.attrib.get('t1')

89+ url = json_data.get('purl')

88− ihk = link.attrib.get('ihk')

90+ img_src = json_data.get('murl')

90− # url = 'http://' + link.attrib.get('t3')

92+ thumb_json_data = loads(_quote_keys_regex.sub(r'\1"\2": \3', link.attrib.get('mad')))

91− url = json_data.get('surl')

93+ thumbnail = thumb_json_data.get('turl')

92− img_src = json_data.get('imgurl')

94 # append result

95 # append result

95 results.append({'template': 'images.html',

96 results.append({'template': 'images.html',

96 'url': url,

97 'url': url,

97 'title': title,

98 'title': title,

98 'content': '',

99 'content': '',

99− 'thumbnail_src': thumb_url.format(ihk=ihk),

100+ 'thumbnail_src': thumbnail,

100 'img_src': img_src})

101 'img_src': img_src})

101

102

102 # TODO stop parsing if 10 images are found

103 # TODO stop parsing if 10 images are found

103− if len(results) >= 10:

104+ # if len(results) >= 10:

104− break

105+ # break

105

106

106 # return results

107 # return results

107 return results

108 return results

tests/unit/engines/test_bing_images.py+37−76

@@ -39,85 +39,46 @@ class TestBingImagesEngine(SearxTestCase):

39 self.assertEqual(bing_images.response(response), [])

41 html = """

42− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>

42+ <div id="mmComponent_images_1">

43− <a href="/images/search?q=south&view=detailv2&&id=7E92863981CCFB89FBDD55205C742DFDA3290CF6&selectedIndex=9&ccid=vzvIfv5u&simid=608055786735667000&thid=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" ihk="OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0" t1="South Carolina" t2="747 x 589 · 29 kB · gif" t3="www.digital-topo-maps.com/county-map/south-carolina.shtml" hh="236" hw="300" m='{ns:"images",k:"5117",mid:"7E92863981CCFB89FBDD55205C742DFDA3290CF6",md5:"bf3bc87efe6e0e476be8cc34bf6cd80e",surl:"http://www.digital-topo-maps.com/county-map/south-carolina.shtml",imgurl:"http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif",tid:"OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0",ow:"480",docid:"608055786735667000",oh:"378",tft:"45"}' mid="7E92863981CCFB89FBDD55205C742DFDA3290CF6" h="ID=images,5117.1">

43+ <ul>

44− <img class="img_hid" src2="https://tse4.mm.bing.net/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0&w=210&h=154&c=7&rs=1&qlt=90&o=4&pid=1.1" style="width:210px;height:154px;" width="210" height="154">

44+ <li>

45− </a>

45+ <div>

46−

46+ <div class="imgpt">

47− </div></div>

47+ <a m='{"purl":"page_url","murl":"img_url"}' mad='{"turl":"thumb_url"}'>

48− """ # noqa

48+ <img src="" alt="alt text" />

49− html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')

49+ </a>

50− response = mock.Mock(text=html)

50+ </div>

51− results = bing_images.response(response)

51+ <div></div>

52− self.assertEqual(type(results), list)

52+ </div>

53− self.assertEqual(len(results), 1)

53+ <div>

54− self.assertEqual(results[0]['title'], 'South Carolina')

54+ <div class="imgpt">

55− self.assertEqual(results[0]['url'],

55+ <a m='{"purl":"page_url2","murl":"img_url2"}' mad='{"turl":"thumb_url2"}'>

56− 'http://www.digital-topo-maps.com/county-map/south-carolina.shtml')

56+ <img src="" alt="alt text 2" />

57− self.assertEqual(results[0]['content'], '')

57+ </a>

58− self.assertEqual(results[0]['thumbnail_src'],

58+ </div>

59− 'https://www.bing.com/th?id=OIP.Mbf3bc87efe6e0e476be8cc34bf6cd80eH0')

59+ </div>

60− self.assertEqual(results[0]['img_src'],

60+ </li>

61− 'http://www.digital-topo-maps.com/county-map/south-carolina-county-map.gif')

61+ </ul>

62−

62+ <ul>

63− html = """

63+ <li>

64− <a href="#" ihk="HN.608003696942779811"

64+ <div>

65− m="{ns:"images",k:"5045",

65+ <div class="imgpt">

66− mid:"59EB92C317974F34517A1CCAEBEF76A578E08DEE",

66+ <a m='{"purl":"page_url3","murl":"img_url3"}' mad='{"turl":"thumb_url3"}'>

67− surl:"http://www.page.url/",

67+ <img src="" alt="alt text 3" />

68− imgurl:"http://test.url/Test%20Query.jpg",oh:"238",

68+ </a>

69− tft:"0",oi:"http://www.image.url/Images/Test%20Query.jpg"}"

69+ </div>

70− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"

70+ </div>

71− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">

71+ </li>

72− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&o=4&pid=1.7"

72+ </ul>

73− style="height:144px;" width="178" height="144"/>

74− </a>

75− """

76− response = mock.Mock(text=html)

77− results = bing_images.response(response)

78− self.assertEqual(type(results), list)

79− self.assertEqual(len(results), 0)

80−

81− html = """

82− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>

83− <a href="#" ihk="HN.608003696942779811"

84− m="{ns:"images",k:"5045",

85−mid:"659EB92C317974F34517A1CCAEBEF76A578E08DEE",

86−surl:"http://www.page.url/",imgurl:"http://test.url/Test%20Query.jpg",

87−oh:"238",tft:"0",oi:"http://www.image.url/Images/Test%20Query.jpg"}"

88− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"

89− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">

90− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&o=4&pid=1.7"

91− style="height:144px;" width="178" height="144"/>

92− </a>

93− </div></div>

94− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>

95− <a href="#" ihk="HN.608003696942779811"

96− m="{ns:"images",k:"5045",

97−mid:"659EB92C317974F34517A1CCAEBEF76A578E08DEE",

98−surl:"http://www.page.url/",imgurl:"http://test.url/Test%20Query.jpg",

99−oh:"238",tft:"0",oi:"http://www.image.url/Images/Test%20Query.jpg"}"

100− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"

101− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">

102− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&o=4&pid=1.7"

103− style="height:144px;" width="178" height="144"/>

104− </a>

105− </div></div>

106− <div class="dg_u" style="width:178px;height:144px;left:17px;top:0px"><div>

107− <a href="#" ihk="HN.608003696942779811"

108− m="{ns:"images",k:"5045",

109−mid:"659EB92C317974F34517A1CCAEBEF76A578E08DEE",

110−surl:"http://www.page.url/",imgurl:"http://test.url/Test%20Query.jpg",

111−oh:"238",tft:"0",oi:"http://www.image.url/Images/Test%20Query.jpg"}"

112− mid="59EB92C317974F34517A1CCAEBEF76A578E08DEE" onclick="return false;"

113− t1="Test Query" t2="650 x 517 · 31 kB · jpeg" t3="www.short.url" h="ID=images,5045.1">

114− <img src="https://tse4.mm.bing.net/th?id=HN.608003696942779811&o=4&pid=1.7"

115− style="height:144px;" width="178" height="144"/>

116− </a>

117− </div></div>

73+ </div>

118 """

74 """

119 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')

75 html = html.replace('\r\n', '').replace('\n', '').replace('\r', '')

120 response = mock.Mock(text=html)

76 response = mock.Mock(text=html)

121 results = bing_images.response(response)

77 results = bing_images.response(response)

122 self.assertEqual(type(results), list)

78 self.assertEqual(type(results), list)

123 self.assertEqual(len(results), 3)

79 self.assertEqual(len(results), 3)

80+ self.assertEqual(results[0]['title'], 'alt text')

81+ self.assertEqual(results[0]['url'], 'page_url')

82+ self.assertEqual(results[0]['content'], '')

83+ self.assertEqual(results[0]['thumbnail_src'], 'thumb_url')

84+ self.assertEqual(results[0]['img_src'], 'img_url')