git.iko.soy/searx
on master help sign in
[enh] implement swisscows engine, FIX #347

TODO: implement image engine

commitd923b2a420b77a3a29375516ed9929beeda0924a authorThomas Pointhuber <thomas.pointhuber@gmx.at> dateMon 1 Jun 2015, 09:15 (11 years ago) parentaac8d3a
4 files changed+195
compare with master
unifiedsplit
searx/engines/swisscows.py+86 searx/settings.yml+4 searx/tests/engines/test_swisscows.py+104 searx/tests/test_engines.py+1
searx/engines/swisscows.pynew file+86
@@ -0,0 +1,86 @@
1+"""
2+ Swisscows (Web)
3+
4+ @website https://swisscows.ch
5+ @provide-api no
6+
7+ @using-api no
8+ @results HTML (using search portal)
9+ @stable no (HTML can change)
10+ @parse url, title, content
11+"""
12+
13+from json import loads
14+from urllib import urlencode, unquote
15+import re
16+
17+# engine dependent config
18+categories = ['general']
19+paging = True
20+language_support = True
21+
22+# search-url
23+base_url = 'https://swisscows.ch/'
24+search_string = '?{query}&page={page}'
25+
26+# regex
27+regex_json = re.compile('initialData: {"Request":(.|\n)*}\]},\s*environment')
28+regex_json_remove_start = re.compile('^initialData:\s*')
29+regex_json_remove_end = re.compile(',\s*environment$')
30+regex_img_url_remove_start = re.compile('^https?://i\.swisscows\.ch/\?link=')
31+
32+
33+# do search-request
34+def request(query, params):
35+ if params['language'] == 'all':
36+ ui_language = 'browser'
37+ region = 'browser'
38+ else:
39+ region = params['language'].replace('_', '-')
40+ ui_language = params['language'].split('_')[0]
41+
42+ search_path = search_string.format(
43+ query=urlencode({'query': query,
44+ 'uiLanguage': ui_language,
45+ 'region': region}),
46+ page=params['pageno'])
47+
48+ params['url'] = base_url + search_path
49+
50+ return params
51+
52+
53+# get response from search-request
54+def response(resp):
55+ results = []
56+
57+ json_regex = regex_json.search(resp.content)
58+
59+ # check if results are returned
60+ if not json_regex:
61+ return []
62+
63+ json_raw = regex_json_remove_end.sub('', regex_json_remove_start.sub('', json_regex.group()))
64+ json = loads(json_raw)
65+
66+ # parse normal results
67+ for result in json['Results'].get('items', []):
68+ # append result
69+ results.append({'url': result['Url'].replace(u'\uE000', '').replace(u'\uE001', ''),
70+ 'title': result['Title'].replace(u'\uE000', '').replace(u'\uE001', ''),
71+ 'content': result['Description'].replace(u'\uE000', '').replace(u'\uE001', '')})
72+
73+ # parse images
74+ for result in json.get('Images', []):
75+ # decode image url
76+ img_url = unquote(regex_img_url_remove_start.sub('', result['Url']))
77+
78+ # append result
79+ results.append({'url': result['SourceUrl'],
80+ 'title': result['Title'],
81+ 'content': '',
82+ 'img_src': img_url,
83+ 'template': 'images.html'})
84+
85+ # return results
86+ return results
searx/settings.yml+4
@@ -213,6 +213,10 @@ engines:
213213 timeout : 6.0
214214 disabled : True
215215
216+ - name : swisscows
217+ engine : swisscows
218+ shortcut : sw
219+
216220 - name : twitter
217221 engine : twitter
218222 shortcut : tw
searx/tests/engines/test_swisscows.pynew file+104
@@ -0,0 +1,104 @@
1+from collections import defaultdict
2+import mock
3+from searx.engines import swisscows
4+from searx.testing import SearxTestCase
5+
6+
7+class TestSwisscowsEngine(SearxTestCase):
8+
9+ def test_request(self):
10+ query = 'test_query'
11+ dicto = defaultdict(dict)
12+ dicto['pageno'] = 1
13+ dicto['language'] = 'de_DE'
14+ params = swisscows.request(query, dicto)
15+ self.assertTrue('url' in params)
16+ self.assertTrue(query in params['url'])
17+ self.assertTrue('swisscows.ch' in params['url'])
18+ self.assertTrue('uiLanguage=de' in params['url'])
19+ self.assertTrue('region=de-DE' in params['url'])
20+
21+ dicto['language'] = 'all'
22+ params = swisscows.request(query, dicto)
23+ self.assertTrue('uiLanguage=browser' in params['url'])
24+ self.assertTrue('region=browser' in params['url'])
25+
26+ def test_response(self):
27+ self.assertRaises(AttributeError, swisscows.response, None)
28+ self.assertRaises(AttributeError, swisscows.response, [])
29+ self.assertRaises(AttributeError, swisscows.response, '')
30+ self.assertRaises(AttributeError, swisscows.response, '[]')
31+
32+ response = mock.Mock(content='<html></html>')
33+ self.assertEqual(swisscows.response(response), [])
34+
35+ response = mock.Mock(content='<html></html>')
36+ self.assertEqual(swisscows.response(response), [])
37+
38+ html = u"""
39+ <script>
40+ App.Dispatcher.dispatch("initialize", {
41+ html5history: true,
42+ initialData: {"Request":
43+ {"Page":1,
44+ "ItemsCount":1,
45+ "Query":"This should ",
46+ "NormalizedQuery":"This should ",
47+ "Region":"de-AT",
48+ "UILanguage":"de"},
49+ "Results":{"items":[
50+ {"Title":"\uE000This should\uE001 be the title",
51+ "Description":"\uE000This should\uE001 be the content.",
52+ "Url":"http://this.should.be.the.link/",
53+ "DisplayUrl":"www.\uE000this.should.be.the\uE001.link",
54+ "Id":"782ef287-e439-451c-b380-6ebc14ba033d"}
55+ ],"TotalCount":55300,
56+ "Query":"This should "
57+ },
58+ "Images":[{"Title":"Datei:This should.svg",
59+ "Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
60+ "SourceUrl":"http://de.wikipedia.org/wiki/Datei:This should.svg",
61+ "DisplayUrl":"de.wikipedia.org/wiki/Datei:This should.svg",
62+ "Width":1280,
63+ "Height":677,
64+ "FileSize":50053,
65+ "ContentType":"image/png",
66+ "Thumbnail":{"Url":"https://i.swisscows.ch/?link=http%3a%2f%2fts2.mm.This/should.png",
67+ "ContentType":"image/png",
68+ "Width":300,
69+ "Height":158,
70+ "FileSize":8023},
71+ "Id":"ae230fd8-a06a-47d6-99d5-e74766d8143a"}]},
72+ environment: "production"
73+ }).then(function (options) {
74+ $('#Search_Form').on('submit', function (e) {
75+ if (!Modernizr.history) return;
76+ e.preventDefault();
77+
78+ var $form = $(this),
79+ $query = $('#Query'),
80+ query = $.trim($query.val()),
81+ path = App.Router.makePath($form.attr('action'), null, $form.serializeObject())
82+
83+ if (query.length) {
84+ options.html5history ?
85+ ReactRouter.HistoryLocation.push(path) :
86+ ReactRouter.RefreshLocation.push(path);
87+ }
88+ else $('#Query').trigger('blur');
89+ });
90+
91+ });
92+ </script>
93+ """
94+ response = mock.Mock(content=html)
95+ results = swisscows.response(response)
96+ self.assertEqual(type(results), list)
97+ self.assertEqual(len(results), 2)
98+ self.assertEqual(results[0]['title'], 'This should be the title')
99+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
100+ self.assertEqual(results[0]['content'], 'This should be the content.')
101+ self.assertEqual(results[1]['title'], 'Datei:This should.svg')
102+ self.assertEqual(results[1]['url'], 'http://de.wikipedia.org/wiki/Datei:This should.svg')
103+ self.assertEqual(results[1]['img_src'], 'http://ts2.mm.This/should.png')
104+ self.assertEqual(results[1]['template'], 'images.html')
searx/tests/test_engines.py+1
@@ -32,6 +32,7 @@ from searx.tests.engines.test_spotify import * # noqa
3232from searx.tests.engines.test_stackoverflow import * # noqa
3333from searx.tests.engines.test_startpage import * # noqa
3434from searx.tests.engines.test_subtitleseeker import * # noqa
35+from searx.tests.engines.test_swisscows import * # noqa
3536from searx.tests.engines.test_twitter import * # noqa
3637from searx.tests.engines.test_vimeo import * # noqa
3738from searx.tests.engines.test_www1x import * # noqa