8f3b33d · searx · git.iko.soy

git.iko.soy/searx

[fix] remove unnecesary async calls in wolframalpha_noapi

setting async to false in the request did the job, lol.

commit8f3b33de23090e6d11a5a29a239719bb6915d2ec authora01200356 <a01200356@itesm.mx> dateSun 28 Feb 2016, 02:05 (11 years ago) parent4cea71e

3 files changed+46−114

compare with master

unifiedsplit

searx/engines/wolframalpha_noapi.py+2−54 tests/unit/engines/test_wolframalpha_api.py+16−16 tests/unit/engines/test_wolframalpha_noapi.py+28−44

searx/engines/wolframalpha_noapi.py+2−54

@@ -20,7 +20,7 @@ from searx.poolrequests import get as http_get

2020url = 'https://www.wolframalpha.com/'

2121

2222search_url = url + 'input/json.jsp'\

23− '?async=true'\

23+ '?async=false'\

2424 '&banners=raw'\

2525 '&debuggingdata=false'\

2626 '&format=image,plaintext,imagemap,minput,moutput'\

@@ -38,14 +38,6 @@ referer_url = url + 'input/?{query}'

3838token = {'value': '',

3939 'last_updated': None}

4040

41−# xpath variables

42−success_xpath = '/pod[attribute::error="false"]'

43−plaintext_xpath = './plaintext'

44−title_xpath = './@title'

45−image_xpath = './img'

46−img_src_xpath = './img/@src'

47−img_alt_xpath = './img/@alt'

48−

4941# pods to display as image in infobox

5042# this pods do return a plaintext, but they look better and are more useful as images

5143image_pods = {'VisualRepresentation',

@@ -79,42 +71,6 @@ def request(query, params):

7971 return params

8072

8173

82−# get additional pod

83−# NOTE: this makes an additional requests to server, so the response will take longer and might reach timeout

84−def get_async_pod(url):

85− try:

86− resp = http_get(url, timeout=2.0)

87− except:

88− return None

89−

90− if resp:

91− return parse_async_pod(resp)

92−

93−

94−def parse_async_pod(resp):

95− pod = {'subpods': []}

96−

97− resp_pod = XML(resp.content)

98−

99− if resp_pod.xpath(success_xpath):

100− for subpod in resp_pod:

101− new_subpod = {'title': subpod.xpath(title_xpath)[0]}

102−

103− plaintext = subpod.xpath(plaintext_xpath)[0].text

104− if plaintext:

105− new_subpod['plaintext'] = plaintext

106− else:

107− new_subpod['plaintext'] = ''

108−

109− if subpod.xpath(image_xpath):

110− new_subpod['img'] = {'src': subpod.xpath(img_src_xpath)[0],

111− 'alt': subpod.xpath(img_alt_xpath)[0]}

112−

113− pod['subpods'].append(new_subpod)

114−

115− return pod

116−

117−

11874# get response from search-request

11975def response(resp):

12076 results = []

@@ -132,15 +88,7 @@ def response(resp):

13288 pod_title = pod.get('title', '')

13389

13490 if 'subpods' not in pod:

135− # comment this section if your requests always reach timeout

136− if pod['async']:

137− result = get_async_pod(pod['async'])

138− if result:

139− pod = result

140− else:

141− continue

142− else:

143− continue

91+ continue

14492

14593 if pod_id == 'Input' or not infobox_title:

14694 infobox_title = pod['subpods'][0]['plaintext']

tests/unit/engines/test_wolframalpha_api.py+16−16

@@ -87,16 +87,16 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

8787 results = wolframalpha_api.response(response)

8888 self.assertEqual(type(results), list)

8989 self.assertEqual(len(results), 2)

90− self.assertIn('input_plaintext', results[0]['infobox'])

90+ self.assertEqual('input_plaintext', results[0]['infobox'])

9191

9292 self.assertEqual(len(results[0]['attributes']), 3)

93− self.assertIn('Input', results[0]['attributes'][0]['label'])

94− self.assertIn('input_plaintext', results[0]['attributes'][0]['value'])

95− self.assertIn('Result', results[0]['attributes'][1]['label'])

96− self.assertIn('result_plaintext', results[0]['attributes'][1]['value'])

97− self.assertIn('Manipulatives illustration', results[0]['attributes'][2]['label'])

98− self.assertIn('illustration_img_src.gif', results[0]['attributes'][2]['image']['src'])

99− self.assertIn('illustration_img_alt', results[0]['attributes'][2]['image']['alt'])

93+ self.assertEqual('Input', results[0]['attributes'][0]['label'])

94+ self.assertEqual('input_plaintext', results[0]['attributes'][0]['value'])

95+ self.assertEqual('Result', results[0]['attributes'][1]['label'])

96+ self.assertEqual('result_plaintext', results[0]['attributes'][1]['value'])

97+ self.assertEqual('Manipulatives illustration', results[0]['attributes'][2]['label'])

98+ self.assertEqual('illustration_img_src.gif', results[0]['attributes'][2]['image']['src'])

99+ self.assertEqual('illustration_img_alt', results[0]['attributes'][2]['image']['alt'])

100100

101101 self.assertEqual(len(results[0]['urls']), 1)

102102

@@ -129,8 +129,8 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

129129 <plaintext>integral_plaintext</plaintext>

130130 </subpod>

131131 </pod>

132− <pod title='Plot'

133− scanner='Plotter'

132+ <pod title='Plot of the integral'

133+ scanner='Integral'

134134 id='Plot'

135135 error='false'

136136 numsubpods='1'>

@@ -147,14 +147,14 @@ class TestWolframAlphaAPIEngine(SearxTestCase):

147147 results = wolframalpha_api.response(response)

148148 self.assertEqual(type(results), list)

149149 self.assertEqual(len(results), 2)

150− self.assertIn('integral_plaintext', results[0]['infobox'])

150+ self.assertEqual('integral_plaintext', results[0]['infobox'])

151151

152152 self.assertEqual(len(results[0]['attributes']), 2)

153− self.assertIn('Indefinite integral', results[0]['attributes'][0]['label'])

154− self.assertIn('integral_plaintext', results[0]['attributes'][0]['value'])

155− self.assertIn('Plot', results[0]['attributes'][1]['label'])

156− self.assertIn('plot.gif', results[0]['attributes'][1]['image']['src'])

157− self.assertIn('plot_alt', results[0]['attributes'][1]['image']['alt'])

153+ self.assertEqual('Indefinite integral', results[0]['attributes'][0]['label'])

154+ self.assertEqual('integral_plaintext', results[0]['attributes'][0]['value'])

155+ self.assertEqual('Plot of the integral', results[0]['attributes'][1]['label'])

156+ self.assertEqual('plot.gif', results[0]['attributes'][1]['image']['src'])

157+ self.assertEqual('plot_alt', results[0]['attributes'][1]['image']['alt'])

158158

159159 self.assertEqual(len(results[0]['urls']), 1)

160160

tests/unit/engines/test_wolframalpha_noapi.py+28−44

@@ -124,16 +124,16 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):

124124 results = wolframalpha_noapi.response(response)

125125 self.assertEqual(type(results), list)

126126 self.assertEqual(len(results), 2)

127− self.assertIn('input_plaintext', results[0]['infobox'])

127+ self.assertEqual('input_plaintext', results[0]['infobox'])

128128

129129 self.assertEqual(len(results[0]['attributes']), 3)

130− self.assertIn('Input', results[0]['attributes'][0]['label'])

131− self.assertIn('input_plaintext', results[0]['attributes'][0]['value'])

132− self.assertIn('Result', results[0]['attributes'][1]['label'])

133− self.assertIn('result_plaintext', results[0]['attributes'][1]['value'])

134− self.assertIn('Manipulatives illustration', results[0]['attributes'][2]['label'])

135− self.assertIn('illustration_img_src.gif', results[0]['attributes'][2]['image']['src'])

136− self.assertIn('illustration_img_alt', results[0]['attributes'][2]['image']['alt'])

130+ self.assertEqual('Input', results[0]['attributes'][0]['label'])

131+ self.assertEqual('input_plaintext', results[0]['attributes'][0]['value'])

132+ self.assertEqual('Result', results[0]['attributes'][1]['label'])

133+ self.assertEqual('result_plaintext', results[0]['attributes'][1]['value'])

134+ self.assertEqual('Manipulatives illustration', results[0]['attributes'][2]['label'])

135+ self.assertEqual('illustration_img_src.gif', results[0]['attributes'][2]['image']['src'])

136+ self.assertEqual('illustration_img_alt', results[0]['attributes'][2]['image']['alt'])

137137

138138 self.assertEqual(len(results[0]['urls']), 1)

139139

@@ -184,8 +184,19 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):

184184 ],

185185 "id" : "Plot",

186186 "error" : false,

187− "numsubpods" : 0,

188− "async" : "invalid_async_url"

187+ "numsubpods" : 1,

188+ "subpods" : [

189+ {

190+ "title" : "",

191+ "img" : {

192+ "src" : "plot.gif",

193+ "alt" : "plot_alt",

194+ "title" : "plot_title"

195+ },

196+ "plaintext" : "",

197+ "minput" : "plot_minput"

198+ }

199+ ]

189200 }

190201 ]

191202 }}

@@ -194,11 +205,14 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):

194205 results = wolframalpha_noapi.response(response)

195206 self.assertEqual(type(results), list)

196207 self.assertEqual(len(results), 2)

197− self.assertIn('integral_plaintext', results[0]['infobox'])

208+ self.assertEqual('integral_plaintext', results[0]['infobox'])

198209

199− self.assertEqual(len(results[0]['attributes']), 1)

200− self.assertIn('Indefinite integral', results[0]['attributes'][0]['label'])

201− self.assertIn('integral_plaintext', results[0]['attributes'][0]['value'])

210+ self.assertEqual(len(results[0]['attributes']), 2)

211+ self.assertEqual('Indefinite integral', results[0]['attributes'][0]['label'])

212+ self.assertEqual('integral_plaintext', results[0]['attributes'][0]['value'])

213+ self.assertEqual('Plot of the integral', results[0]['attributes'][1]['label'])

214+ self.assertEqual('plot.gif', results[0]['attributes'][1]['image']['src'])

215+ self.assertEqual('plot_alt', results[0]['attributes'][1]['image']['alt'])

202216

203217 self.assertEqual(len(results[0]['urls']), 1)

204218

@@ -206,33 +220,3 @@ class TestWolframAlphaNoAPIEngine(SearxTestCase):

206220 self.assertEqual('Wolfram|Alpha', results[0]['urls'][0]['title'])

207221 self.assertEqual(referer_url, results[1]['url'])

208222 self.assertEqual('Wolfram|Alpha', results[1]['title'])

209−

210− def test_parse_async_pod(self):

211− self.assertRaises(AttributeError, wolframalpha_noapi.parse_async_pod, None)

212− self.assertRaises(AttributeError, wolframalpha_noapi.parse_async_pod, [])

213− self.assertRaises(AttributeError, wolframalpha_noapi.parse_async_pod, '')

214− self.assertRaises(AttributeError, wolframalpha_noapi.parse_async_pod, '[]')

215−

216− # test plot

217− xml = '''<?xml version='1.0' encoding='UTF-8'?>

218− <pod title='Plot'

219− scanner='Plot'

220− id='Plot'

221− error='false'

222− numsubpods='1'>

223− <subpod title=''>

224− <img src='plot_img_src.gif'

225− alt='plot_img_alt'

226− title='plot_img_title' />

227− <plaintext>plot_plaintext</plaintext>

228− <minput>plot_minput</minput>

229− </subpod>

230− </pod>

231− '''

232− response = mock.Mock(content=xml)

233− pod = wolframalpha_noapi.parse_async_pod(response)

234− self.assertEqual(len(pod['subpods']), 1)

235− self.assertEqual('', pod['subpods'][0]['title'])

236− self.assertEqual('plot_plaintext', pod['subpods'][0]['plaintext'])

237− self.assertEqual('plot_img_src.gif', pod['subpods'][0]['img']['src'])

238− self.assertEqual('plot_img_alt', pod['subpods'][0]['img']['alt'])