git.iko.soy/searx
on master help sign in
Merge pull request #348 from pointhi/swisscows

implement swisscows engine

commit9c7578bab620ffe32afc1344817ce367cf096f00 authorAdam Tauber <asciimoo@gmail.com> dateMon 1 Jun 2015, 07:49 (11 years ago) parentaac8d3a2d81079
4 files changed+237
compare with master
unifiedsplit
searx/engines/swisscows.py+108 searx/settings.yml+4 searx/tests/engines/test_swisscows.py+124 searx/tests/test_engines.py+1
searx/engines/swisscows.pynew file+108
@@ -0,0 +1,108 @@
1+"""
2+ Swisscows (Web, Images)
3+
4+ @website https://swisscows.ch
5+ @provide-api no
6+
7+ @using-api no
8+ @results HTML (using search portal)
9+ @stable no (HTML can change)
10+ @parse url, title, content
11+"""
12+
13+from json import loads
14+from urllib import urlencode, unquote
15+import re
16+
17+# engine dependent config
18+categories = ['general', 'images']
19+paging = True
20+language_support = True
21+
22+# search-url
23+base_url = 'https://swisscows.ch/'
24+search_string = '?{query}&page={page}'
25+
26+# regex
27+regex_json = re.compile('initialData: {"Request":(.|\n)*},\s*environment')
28+regex_json_remove_start = re.compile('^initialData:\s*')
29+regex_json_remove_end = re.compile(',\s*environment$')
30+regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=')
31+
32+
33+# do search-request
34+def request(query, params):
35+ if params['language'] == 'all':
36+ ui_language = 'browser'
37+ region = 'browser'
38+ else:
39+ region = params['language'].replace('_', '-')
40+ ui_language = params['language'].split('_')[0]
41+
42+ search_path = search_string.format(
43+ query=urlencode({'query': query,
44+ 'uiLanguage': ui_language,
45+ 'region': region}),
46+ page=params['pageno'])
47+
48+ # image search query is something like 'image?{query}&page={page}'
49+ if params['category'] == 'images':
50+ search_path = 'image' + search_path
51+
52+ params['url'] = base_url + search_path
53+
54+ return params
55+
56+
57+# get response from search-request
58+def response(resp):
59+ results = []
60+
61+ json_regex = regex_json.search(resp.content)
62+
63+ # check if results are returned
64+ if not json_regex:
65+ return []
66+
67+ json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
68+ json = loads(json_raw)
69+
70+ # parse results
71+ for result in json['Results'].get('items', []):
72+ result_title = result['Title'].replace(u'\uE000', '').replace(u'\uE001', '')
73+
74+ # parse image results
75+ if result.get('ContentType', '').startswith('image'):
76+ img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
77+
78+ # append result
79+ results.append({'url': result['SourceUrl'],
80+ 'title': result['Title'],
81+ 'content': '',
82+ 'img_src': img_url,
83+ 'template': 'images.html'})
84+
85+ # parse general results
86+ else:
87+ result_url = result['Url'].replace(u'\uE000', '').replace(u'\uE001', '')
88+ result_content = result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')
89+
90+ # append result
91+ results.append({'url': result_url,
92+ 'title': result_title,
93+ 'content': result_content})
94+
95+ # parse images
96+ for result in json.get('Images', []):
97+ # decode image url
98+ img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
99+
100+ # append result
101+ results.append({'url': result['SourceUrl'],
102+ 'title': result['Title'],
103+ 'content': '',
104+ 'img_src': img_url,
105+ 'template': 'images.html'})
106+
107+ # return results
108+ return results
searx/settings.yml+4
@@ -213,6 +213,10 @@ engines:
213213 timeout : 6.0
214214 disabled : True
215215
216+ - name : swisscows
217+ engine : swisscows
218+ shortcut : sw
219+
216220 - name : twitter
217221 engine : twitter
218222 shortcut : tw
searx/tests/engines/test_swisscows.pynew file+124
@@ -0,0 +1,124 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import swisscows
4+from searx.testing import SearxTestCase
5+
6+
7+class TestSwisscowsEngine(SearxTestCase):
8+
9+ def test_request(self):
10+ query = 'test_query'
11+ dicto = defaultdict(dict)
12+ dicto['pageno'] = 1
13+ dicto['language'] = 'de_DE'
14+ params = swisscows.request(query, dicto)
15+ self.assertTrue('url' in params)
16+ self.assertTrue(query in params['url'])
17+ self.assertTrue('swisscows.ch' in params['url'])
18+ self.assertTrue('uiLanguage=de' in params['url'])
19+ self.assertTrue('region=de-DE' in params['url'])
20+
21+ dicto['language'] = 'all'
22+ params = swisscows.request(query, dicto)
23+ self.assertTrue('uiLanguage=browser' in params['url'])
24+ self.assertTrue('region=browser' in params['url'])
25+
26+ def test_response(self):
27+ self.assertRaises(AttributeError, swisscows.response, None)
28+ self.assertRaises(AttributeError, swisscows.response, [])
29+ self.assertRaises(AttributeError, swisscows.response, '')
30+ self.assertRaises(AttributeError, swisscows.response, '[]')
31+
32+ response = mock.Mock(content='<html></html>')
33+ self.assertEqual(swisscows.response(response), [])
34+
35+ response = mock.Mock(content='<html></html>')
36+ self.assertEqual(swisscows.response(response), [])
37+
38+ html = u"""
39+ <script>
40+ App.Dispatcher.dispatch("initialize", {
41+ html5history: true,
42+ initialData: {"Request":
43+ {"Page":1,
44+ "ItemsCount":1,
45+ "Query":"This should ",
46+ "NormalizedQuery":"This should ",
47+ "Region":"de-AT",
48+ "UILanguage":"de"},
49+ "Results":{"items":[
50+ {"Title":"\uE000This should\uE001 be the title",
51+ "Description":"\uE000This should\uE001 be the content.",
52+ "Url":"http://this.should.be.the.link/",
53+ "DisplayUrl":"www.\uE000this.should.be.the\uE001.link",
54+ "Id":"782ef287-e439-451c-b380-6ebc14ba033d"},
55+ {"Title":"Datei:This should1.svg",
56+ "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
57+ "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should1.svg",
58+ "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should1.svg",
59+ "Width":950,
60+ "Height":534,
61+ "FileSize":92100,
62+ "ContentType":"image/jpeg",
63+ "Thumbnail":{
64+ "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should1.png",
65+ "ContentType":"image/jpeg",
66+ "Width":300,
67+ "Height":168,
68+ "FileSize":9134},
69+ "Id":"6a97a542-8f65-425f-b7f6-1178c3aba7be"
70+ }
71+ ],"TotalCount":55300,
72+ "Query":"This should "
73+ },
74+ "Images":[{"Title":"Datei:This should.svg",
75+ "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
76+ "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should.svg",
77+ "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should.svg",
78+ "Width":1280,
79+ "Height":677,
80+ "FileSize":50053,
81+ "ContentType":"image/png",
82+ "Thumbnail":{"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
83+ "ContentType":"image/png",
84+ "Width":300,
85+ "Height":158,
86+ "FileSize":8023},
87+ "Id":"ae230fd8-a06a-47d6-99d5-e74766d8143a"}]},
88+ environment: "production"
89+ }).then(function (options) {
90+ $('#Search_Form').on('submit', function (e) {
91+ if (!Modernizr.history) return;
92+ e.preventDefault();
93+
94+ var $form = $(this),
95+ $query = $('#Query'),
96+ query = $.trim($query.val()),
97+ path = App.Router.makePath($form.attr('action'), null, $form.serializeObject())
98+
99+ if (query.length) {
100+ options.html5history ?
101+ ReactRouter.HistoryLocation.push(path) :
102+ ReactRouter.RefreshLocation.push(path);
103+ }
104+ else $('#Query').trigger('blur');
105+ });
106+
107+ });
108+ </script>
109+ """
110+ response = mock.Mock(content=html)
111+ results = swisscows.response(response)
112+ self.assertEqual(type(results), list)
113+ self.assertEqual(len(results), 3)
114+ self.assertEqual(results[0]['title'], 'This should be the title')
115+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
116+ self.assertEqual(results[0]['content'], 'This should be the content.')
117+ self.assertEqual(results[1]['title'], 'Datei:This should1.svg')
118+ self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should1.svg')
119+ self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should1.png')
120+ self.assertEqual(results[1]['template'], 'images.html')
121+ self.assertEqual(results[2]['title'], 'Datei:This should.svg')
122+ self.assertEqual(results[2]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg')
123+ self.assertEqual(results[2]['img_src'], 'http://ts2.mm.This/should.png')
124+ self.assertEqual(results[2]['template'], 'images.html')
searx/tests/test_engines.py+1
@@ -32,6 +32,7 @@ from searx.tests.engines.test_spotify import * # noqa
3232from searx.tests.engines.test_stackoverflow import * # noqa
3333from searx.tests.engines.test_startpage import * # noqa
3434from searx.tests.engines.test_subtitleseeker import * # noqa
35+from searx.tests.engines.test_swisscows import * # noqa
3536from searx.tests.engines.test_twitter import * # noqa
3637from searx.tests.engines.test_vimeo import * # noqa
3738from searx.tests.engines.test_www1x import * # noqa