git.iko.soy/searx
on master help sign in
[fix] deviantart engine xpaths
commitba590de7f1506b8b8e281403dc47b6dcdbe3ead1 authorNoemi Vanyi <sitbackandwait@gmail.com> dateTue 19 Jul 2016, 09:37 (10 years ago) parentaa09f96
2 files changed+12−44
compare with master
unifiedsplit
searx/engines/deviantart.py+4−5 tests/unit/engines/test_deviantart.py+8−39
searx/engines/deviantart.py+4−5
@@ -50,11 +50,10 @@ def response(resp):
5050 regex = re.compile(r'\/200H\/')
5151
5252 # parse results
53− for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):
54− link = result.xpath('.//a[contains(@class, "thumb")]')[0]
55− url = urljoin(base_url, link.attrib.get('href'))
56− title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]')
57− title = extract_text(title_links[0])
53+ for result in dom.xpath('.//span[@class="thumb wide"]'):
54+ link = result.xpath('.//a[@class="torpedo-thumb-link"]')[0]
55+ url = link.attrib.get('href')
56+ title = extract_text(result.xpath('.//span[@class="title"]'))
5857 thumbnail_src = link.xpath('.//img')[0].attrib.get('src')
5958 img_src = regex.sub('/', thumbnail_src)
6059
tests/unit/engines/test_deviantart.py+8−39
@@ -28,44 +28,13 @@ class TestDeviantartEngine(SearxTestCase):
2828 self.assertEqual(deviantart.response(response), [])
2929
3030 html = """
31− <div class="tt-a tt-fh tt-boxed" collect_rid="1:149167425"
32− usericon="http://a.deviantart.net/avatars/t/e/test-0.gif" userid="233301"
33− username="test-0" symbol="~" category="digitalart/animation">
34− <span class="tt-w" style="width: auto; max-width: 277px;">
35− <span class="tt-fh-tc" style="width: 202px;">
36− <span class="tt-bb" style="width: 202px;">
37− </span>
38− <span class="shadow">
39− <a class="thumb" href="http://url.of.result/2nd.part.of.url"
40− title="Behoimi BE Animation Test by test-0, Jan 4,
41− 2010 in Digital Art &gt; Animation"> <i></i>
42− <img width="200" height="200" alt="Test"
43− src="http://url.of.thumbnail" data-src="http://th08.deviantart.net/test.jpg">
44− </a>
45− </span>
46− <!-- ^TTT -->
47− </span>
48− <span class="details">
49− <a href="http://test-0.deviantart.com/art/Test" class="t"
50− title="Behoimi BE Animation Test by test-0, Jan 4, 2010">
51− <span class="tt-fh-oe">Title of image</span> </a>
52− <small>
53− <span class="category">
54− <span class="age">
55− 5 years ago
56− </span>
57− in <a title="Behoimi BE Animation Test by test-0, Jan 4, 2010"
58− href="http://www.deviantart.com/browse/all/digitalart/animation/">Animation</a>
59− </span>
60− <div class="commentcount">
61− <a href="http://test-0.deviantart.com/art/Test#comments">
62− <span class="iconcommentsstats"></span>9 Comments</a>
63− </div>
64− <a class="mlt-link" href="http://www.deviantart.com/morelikethis/149167425">
65− <span class="mlt-icon"></span> <span class="mlt-text">More Like This</span> </a>
66− </span>
67− </small> <!-- TTT$ -->
68− </span>
31+ <div id="page-1-results" class="page-results results-page-thumb torpedo-container">
32+ <span class="thumb wide" href="http://amai911.deviantart.com/art/Horse-195212845"
33+ data-super-full-width="900" data-super-full-height="600">
34+ <a class="torpedo-thumb-link" href="https://url.of.image">
35+ <img data-sigil="torpedo-img" src="https://url.of.thumbnail" />
36+ </a>
37+ <span class="info"><span class="title-wrap"><span class="title">Title of image</span></span>
6938 </div>
7039 """
7140 response = mock.Mock(text=html)
@@ -73,7 +42,7 @@ class TestDeviantartEngine(SearxTestCase):
7342 self.assertEqual(type(results), list)
7443 self.assertEqual(len(results), 1)
7544 self.assertEqual(results[0]['title'], 'Title of image')
76− self.assertEqual(results[0]['url'], 'http://url.of.result/2nd.part.of.url')
45+ self.assertEqual(results[0]['url'], 'https://url.of.image')
7746 self.assertNotIn('content', results[0])
7847 self.assertEqual(results[0]['thumbnail_src'], 'https://url.of.thumbnail')
7948