git.iko.soy/searx
on master help sign in
Merge pull request #305 from dalf/https_again

[enh] reduce the number of http outgoing connections.

commit59a65d0bdc98a76f03235055f214747f699923a9 authorAdam Tauber <asciimoo@gmail.com> dateSat 2 May 2015, 13:36 (11 years ago) parent01de1c659ee040
9 files changed+22−7
compare with master
unifiedsplit
searx/autocomplete.py+2−2 searx/engines/dailymotion.py+3 searx/engines/deviantart.py+7−1 searx/engines/digg.py+3 searx/engines/gigablast.py+1−1 searx/engines/google_images.py+3 searx/engines/www1x.py+1−1 searx/tests/engines/test_deviantart.py+1−1 searx/tests/engines/test_google_images.py+1−1
searx/autocomplete.py+2−2
@@ -111,7 +111,7 @@ def searx_bang(full_query):
111111
112112
113113def dbpedia(query):
114− # dbpedia autocompleter
114+ # dbpedia autocompleter, no HTTPS
115115 autocomplete_url = 'http://lookup.dbpedia.org/api/search.asmx/KeywordSearch?' # noqa
116116
117117 response = get(autocomplete_url
@@ -139,7 +139,7 @@ def duckduckgo(query):
139139
140140def google(query):
141141 # google autocompleter
142− autocomplete_url = 'http://suggestqueries.google.com/complete/search?client=toolbar&' # noqa
142+ autocomplete_url = 'https://suggestqueries.google.com/complete/search?client=toolbar&' # noqa
143143
144144 response = get(autocomplete_url
145145 + urlencode(dict(q=query)))
searx/engines/dailymotion.py+3
@@ -60,6 +60,9 @@ def response(resp):
6060 publishedDate = datetime.fromtimestamp(res['created_time'], None)
6161 embedded = embedded_url.format(videoid=res['id'])
6262
63+ # http to https
64+ thumbnail = thumbnail.replace("http://", "https://")
65+
6366 results.append({'template': 'videos.html',
6467 'url': url,
6568 'title': title,
searx/engines/deviantart.py+7−1
@@ -22,7 +22,7 @@ paging = True
2222
2323# search-url
2424base_url = 'https://www.deviantart.com/'
25−search_url = base_url+'search?offset={offset}&{query}'
25+search_url = base_url+'browse/all/?offset={offset}&{query}'
2626
2727
2828# do search-request
@@ -56,6 +56,12 @@ def response(resp):
5656 thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
5757 img_src = regex.sub('/', thumbnail_src)
5858
59+ # http to https, remove domain sharding
60+ thumbnail_src = re.sub(r"https?://(th|fc)\d+.", "https://th01.", thumbnail_src)
61+ thumbnail_src = re.sub(r"http://", "https://", thumbnail_src)
62+
63+ url = re.sub(r"http://(.*)\.deviantart\.com/", "https://\\1.deviantart.com/", url)
64+
5965 # append result
6066 results.append({'url': url,
6167 'title': title,
searx/engines/digg.py+3
@@ -58,6 +58,9 @@ def response(resp):
5858 pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')
5959 publishedDate = parser.parse(pubdate)
6060
61+ # http to https
62+ thumbnail = thumbnail.replace("http://static.digg.com", "https://static.digg.com")
63+
6164 # append result
6265 results.append({'url': url,
6366 'title': title,
searx/engines/gigablast.py+1−1
@@ -17,7 +17,7 @@ categories = ['general']
1717paging = True
1818number_of_results = 5
1919
20−# search-url
20+# search-url, invalid HTTPS certificate
2121base_url = 'http://gigablast.com/'
2222search_string = 'search?{query}&n={number_of_results}&s={offset}&xml=1&qh=0'
2323
searx/engines/google_images.py+3
@@ -56,6 +56,9 @@ def response(resp):
5656 continue
5757 thumbnail_src = result['tbUrl']
5858
59+ # http to https
60+ thumbnail_src = thumbnail_src.replace("http://", "https://")
61+
5962 # append result
6063 results.append({'url': href,
6164 'title': title,
searx/engines/www1x.py+1−1
@@ -19,7 +19,7 @@ import re
1919categories = ['images']
2020paging = False
2121
22−# search-url
22+# search-url, no HTTPS (there is a valid certificate for https://api2.1x.com/ )
2323base_url = 'http://1x.com'
2424search_url = base_url+'/backend/search.php?{query}'
2525
searx/tests/engines/test_deviantart.py+1−1
@@ -75,7 +75,7 @@ class TestDeviantartEngine(SearxTestCase):
7575 self.assertEqual(results[0]['title'], 'Title of image')
7676 self.assertEqual(results[0]['url'], 'http://url.of.result/2nd.part.of.url')
7777 self.assertNotIn('content', results[0])
78− self.assertEqual(results[0]['thumbnail_src'], 'http://url.of.thumbnail')
78+ self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail')
7979
8080 html = """
8181 <span class="tt-fh-tc" style="width: 202px;">
searx/tests/engines/test_google_images.py+1−1
@@ -65,7 +65,7 @@ class TestGoogleImagesEngine(SearxTestCase):
6565 self.assertEqual(len(results), 1)
6666 self.assertEqual(results[0]['title'], 'This is the title')
6767 self.assertEqual(results[0]['url'], 'http://this.is.the.url')
68− self.assertEqual(results[0]['thumbnail_src'], 'http://thumbnail.url')
68+ self.assertEqual(results[0]['thumbnail_src'], 'https://thumbnail.url')
6969 self.assertEqual(results[0]['img_src'], 'http://image.url.jpg')
7070 self.assertEqual(results[0]['content'], '<b>test</b>')
7171