on master help sign in
Startpage's unit test
commitf1c10f4fe45f34c12994b9bbc4aca133202fd7ca authorCqoicebordel <Cqoicebordel@users.noreply.github.com> dateFri 6 Feb 2015, 17:31 (12 years ago) parent3a4d604
3 files changed+146−8
searx/engines/startpage.py+5−8 searx/tests/engines/test_startpage.py+140 searx/tests/test_engines.py+1
searx/engines/startpage.py+5−8
@@ -13,6 +13,7 @@
1313from lxml import html
1414from cgi import escape
1515import re
16+from searx.engines.xpath import extract_text
1617
1718# engine dependent config
1819categories = ['general']
@@ -45,8 +46,7 @@ def request(query, params):
4546
4647 # set language if specified
4748 if params['language'] != 'all':
48− params['data']['with_language'] = ('lang_' +
49− params['language'].split('_')[0])
49+ params['data']['with_language'] = ('lang_' + params['language'].split('_')[0])
5050
5151 return params
5252
@@ -64,18 +64,15 @@ def response(resp):
6464 continue
6565 link = links[0]
6666 url = link.attrib.get('href')
67− try:
68− title = escape(link.text_content())
69− except UnicodeDecodeError:
70− continue
7167
7268 # block google-ad url's
7369 if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):
7470 continue
7571
72+ title = escape(extract_text(link))
73+
7674 if result.xpath('./p[@class="desc"]'):
77− content = escape(result.xpath('./p[@class="desc"]')[0]
78− .text_content())
75+ content = escape(extract_text(result.xpath('./p[@class="desc"]')))
7976 else:
8077 content = ''
8178
searx/tests/engines/test_startpage.pynew file+140
@@ -0,0 +1,140 @@
1+# -*- coding: utf-8 -*-
2+from collections import defaultdict
3+import mock
4+from searx.engines import startpage
5+from searx.testing import SearxTestCase
6+
7+
8+class TestStartpageEngine(SearxTestCase):
9+
10+ def test_request(self):
11+ query = 'test_query'
12+ dicto = defaultdict(dict)
13+ dicto['pageno'] = 1
14+ dicto['language'] = 'fr_FR'
15+ params = startpage.request(query, dicto)
16+ self.assertIn('url', params)
17+ self.assertIn('startpage.com', params['url'])
18+ self.assertIn('data', params)
19+ self.assertIn('query', params['data'])
20+ self.assertIn(query, params['data']['query'])
21+ self.assertIn('with_language', params['data'])
22+ self.assertIn('lang_fr', params['data']['with_language'])
23+
24+ dicto['language'] = 'all'
25+ params = startpage.request(query, dicto)
26+ self.assertNotIn('with_language', params['data'])
27+
28+ def test_response(self):
29+ self.assertRaises(AttributeError, startpage.response, None)
30+ self.assertRaises(AttributeError, startpage.response, [])
31+ self.assertRaises(AttributeError, startpage.response, '')
32+ self.assertRaises(AttributeError, startpage.response, '[]')
33+
34+ response = mock.Mock(content='<html></html>')
35+ self.assertEqual(startpage.response(response), [])
36+
37+ html = """
38+ <div class='result' style=' *width : auto; *margin-right : 10%;'>
39+ <h3>
40+ <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >
41+ This should be the title
42+ </a>
43+ <span id='title_stars_2' name='title_stars_2'> </span>
44+ </h3>
45+ <p class='desc'>
46+ This should be the content.
47+ </p>
48+ <p>
49+ <span class='url'>www.speed<b>test</b>.net/fr/
50+ </span>
51+ -
52+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="
53+ class='proxy'>
54+ Navigation avec Ixquick Proxy
55+ </A>
56+ -
57+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=
58+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=
59+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>
60+ Mis en surbrillance
61+ </A>
62+ </p>
63+ </div>
64+ """
65+ response = mock.Mock(content=html)
66+ results = startpage.response(response)
67+ self.assertEqual(type(results), list)
68+ self.assertEqual(len(results), 1)
69+ self.assertEqual(results[0]['title'], 'This should be the title')
70+ self.assertEqual(results[0]['url'], 'http://this.should.be.the.link/')
71+ self.assertEqual(results[0]['content'], 'This should be the content.')
72+
73+ html = """
74+ <div class='result' style=' *width : auto; *margin-right : 10%;'>
75+ <h3>
76+ <a href='http://www.google.com/aclk?sa=l&ai=C' id='title_2' name='title_2' >
77+ This should be the title
78+ </a>
79+ <span id='title_stars_2' name='title_stars_2'> </span>
80+ </h3>
81+ <p class='desc'>
82+ This should be the content.
83+ </p>
84+ <p>
85+ <span class='url'>www.speed<b>test</b>.net/fr/
86+ </span>
87+ -
88+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="
89+ class='proxy'>
90+ Navigation avec Ixquick Proxy
91+ </A>
92+ -
93+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=
94+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=
95+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>
96+ Mis en surbrillance
97+ </A>
98+ </p>
99+ </div>
100+ <div class='result' style=' *width : auto; *margin-right : 10%;'>
101+ <h3>
102+ <span id='title_stars_2' name='title_stars_2'> </span>
103+ </h3>
104+ <p class='desc'>
105+ This should be the content.
106+ </p>
107+ <p>
108+ <span class='url'>www.speed<b>test</b>.net/fr/
109+ </span>
110+ </p>
111+ </div>
112+ <div class='result' style=' *width : auto; *margin-right : 10%;'>
113+ <h3>
114+ <a href='http://this.should.be.the.link/' id='title_2' name='title_2' >
115+ This should be the title
116+ </a>
117+ <span id='title_stars_2' name='title_stars_2'> </span>
118+ </h3>
119+ <p>
120+ <span class='url'>www.speed<b>test</b>.net/fr/
121+ </span>
122+ -
123+ <A class="proxy" id="proxy_link" HREF="https://ixquick-proxy.com/do/spg/proxy?ep=&edata=&ek=&ekdata="
124+ class='proxy'>
125+ Navigation avec Ixquick Proxy
126+ </A>
127+ -
128+ <A HREF="https://ixquick-proxy.com/do/spg/highlight.pl?l=francais&c=hf&cat=web&q=test&rl=NONE&rid=
129+ &hlq=https://startpage.com/do/search&mtabp=-1&mtcmd=process_search&mtlanguage=francais&mtengine0=
130+ &mtcat=web&u=http:%2F%2Fwww.speedtest.net%2Ffr%2F" class='proxy'>
131+ Mis en surbrillance
132+ </A>
133+ </p>
134+ </div>
135+ """
136+ response = mock.Mock(content=html)
137+ results = startpage.response(response)
138+ self.assertEqual(type(results), list)
139+ self.assertEqual(len(results), 1)
140+ self.assertEqual(results[0]['content'], '')
searx/tests/test_engines.py+1
@@ -23,6 +23,7 @@ from searx.tests.engines.test_searchcode_code import * # noqa
2323from searx.tests.engines.test_searchcode_doc import * # noqa
2424from searx.tests.engines.test_soundcloud import * # noqa
2525from searx.tests.engines.test_stackoverflow import * # noqa
26+from searx.tests.engines.test_startpage import * # noqa
2627from searx.tests.engines.test_subtitleseeker import * # noqa
2728from searx.tests.engines.test_twitter import * # noqa
2829from searx.tests.engines.test_vimeo import * # noqa