f5128c7 · searx · git.iko.soy

git.iko.soy/searx

[mod] add/modify image fetching for bing_news, qwant and twitter engines

commitf5128c7cb96d7564cc7bebeae13a319557b84aaa authorAlexandre Flament <alex@al-f.net> dateSun 12 Feb 2017, 14:58 (10 years ago) parent4cffd78

4 files changed+27−10

compare with master

unifiedsplit

searx/engines/bing_news.py+2−3 searx/engines/qwant.py+16−3 searx/engines/twitter.py+6−1 tests/unit/engines/test_bing_news.py+3−3

searx/engines/bing_news.py+2−3

@@ -112,12 +112,11 @@ def response(resp):

112112

113113 # append result

114114 if thumbnail is not None:

115− results.append({'template': 'videos.html',

116− 'url': url,

115+ results.append({'url': url,

117116 'title': title,

118117 'publishedDate': publishedDate,

119118 'content': content,

120− 'thumbnail': thumbnail})

119+ 'img_src': thumbnail})

121120 else:

122121 results.append({'url': url,

123122 'title': title,

searx/engines/qwant.py+16−3

@@ -96,14 +96,27 @@ def response(resp):

9696 'thumbnail_src': thumbnail_src,

9797 'img_src': img_src})

9898

99− elif (category_to_keyword.get(categories[0], '') == 'news' or

100− category_to_keyword.get(categories[0], '') == 'social'):

99+ elif category_to_keyword.get(categories[0], '') == 'social':

101100 published_date = datetime.fromtimestamp(result['date'], None)

101+ img_src = result.get('img', None)

102+ results.append({'url': res_url,

103+ 'title': title,

104+ 'publishedDate': published_date,

105+ 'content': content,

106+ 'img_src': img_src})

102107

108+ elif category_to_keyword.get(categories[0], '') == 'news':

109+ published_date = datetime.fromtimestamp(result['date'], None)

110+ media = result.get('media', [])

111+ if len(media) > 0:

112+ img_src = media[0].get('pict', {}).get('url', None)

113+ else:

114+ img_src = None

103115 results.append({'url': res_url,

104116 'title': title,

105117 'publishedDate': published_date,

106− 'content': content})

118+ 'content': content,

119+ 'img_src': img_src})

107120

108121 return results

109122

searx/engines/twitter.py+6−1

@@ -27,6 +27,7 @@ search_url = base_url + 'search?'

2727

2828# specific xpath variables

2929results_xpath = '//li[@data-item-type="tweet"]'

30+avatar_xpath = './/img[contains(@class, "avatar")]/@src'

3031link_xpath = './/small[@class="time"]//a'

3132title_xpath = './/span[contains(@class, "username")]'

3233content_xpath = './/p[contains(@class, "tweet-text")]'

@@ -57,6 +58,8 @@ def response(resp):

5758 try:

5859 link = tweet.xpath(link_xpath)[0]

5960 content = extract_text(tweet.xpath(content_xpath)[0])

61+ img_src = tweet.xpath(avatar_xpath)[0]

62+ img_src = img_src.replace('_bigger', '_normal')

6063 except Exception:

6164 continue

6265

@@ -71,12 +74,14 @@ def response(resp):

7174 results.append({'url': url,

7275 'title': title,

7376 'content': content,

77+ 'img_src': img_src,

7478 'publishedDate': publishedDate})

7579 else:

7680 # append result

7781 results.append({'url': url,

7882 'title': title,

79− 'content': content})

83+ 'content': content,

84+ 'img_src': img_src})

8085

8186 # return results

8287 return results

tests/unit/engines/test_bing_news.py+3−3

@@ -81,11 +81,11 @@ class TestBingNewsEngine(SearxTestCase):

8181 self.assertEqual(results[0]['title'], 'Title')

8282 self.assertEqual(results[0]['url'], 'http://url.of.article/')

8383 self.assertEqual(results[0]['content'], 'Article Content')

84− self.assertEqual(results[0]['thumbnail'], 'https://www.bing.com/th?id=ON.13371337133713371337133713371337')

84+ self.assertEqual(results[0]['img_src'], 'https://www.bing.com/th?id=ON.13371337133713371337133713371337')

8585 self.assertEqual(results[1]['title'], 'Another Title')

8686 self.assertEqual(results[1]['url'], 'http://another.url.of.article/')

8787 self.assertEqual(results[1]['content'], 'Another Article Content')

88− self.assertNotIn('thumbnail', results[1])

88+ self.assertNotIn('img_src', results[1])

8989

9090 html = """<?xml version="1.0" encoding="utf-8" ?>

9191<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS">

@@ -120,7 +120,7 @@ class TestBingNewsEngine(SearxTestCase):

120120 self.assertEqual(results[0]['title'], 'Title')

121121 self.assertEqual(results[0]['url'], 'http://another.url.of.article/')

122122 self.assertEqual(results[0]['content'], 'Article Content')

123− self.assertEqual(results[0]['thumbnail'], 'http://another.bing.com/image')

123+ self.assertEqual(results[0]['img_src'], 'http://another.bing.com/image')

124124

125125 html = """<?xml version="1.0" encoding="utf-8" ?>

126126<rss version="2.0" xmlns:News="https://www.bing.com:443/news/search?q=python&setmkt=en-US&first=1&format=RSS">