c3232b0 · searx · git.iko.soy

git.iko.soy/searx

Merge branch 'master' into nyaa

commitc3232b0e1a82315e5936fb3c0731548efd66a0b3 authormisnyo <misnyo@users.noreply.github.com> dateMon 4 Sep 2017, 17:48 (9 years ago) parent01330f7 6888010

5 files changed+77−84

compare with master

unifiedsplit

searx/engines/generalfile.py−62 searx/engines/gigablast.py+6−2 searx/engines/google_news.py+2−2 searx/settings.yml+13−12 tests/unit/engines/test_google_news.py+56−6

searx/engines/generalfile.pydeleted−62

@@ -1,62 +0,0 @@

1−"""

2− General Files (Files)

3−

4− @website http://www.general-files.org

5− @provide-api no (nothing found)

6−

7− @using-api no (because nothing found)

8− @results HTML (using search portal)

9− @stable no (HTML can change)

10− @parse url, title, content

11−

12− @todo detect torrents?

13−"""

14−

15−from lxml import html

16−

17−# engine dependent config

18−categories = ['files']

19−paging = True

20−

21−# search-url

22−base_url = 'http://www.general-file.com'

23−search_url = base_url + '/files-{letter}/{query}/{pageno}'

24−

25−# specific xpath variables

26−result_xpath = '//table[@class="block-file"]'

27−title_xpath = './/h2/a//text()'

28−url_xpath = './/h2/a/@href'

29−content_xpath = './/p//text()'

30−

31−

32−# do search-request

33−def request(query, params):

34−

35− params['url'] = search_url.format(query=query,

36− letter=query[0],

37− pageno=params['pageno'])

38−

39− return params

40−

41−

42−# get response from search-request

43−def response(resp):

44− results = []

45−

46− dom = html.fromstring(resp.text)

47−

48− # parse results

49− for result in dom.xpath(result_xpath):

50− url = result.xpath(url_xpath)[0]

51−

52− # skip fast download links

53− if not url.startswith('/'):

54− continue

55−

56− # append result

57− results.append({'url': base_url + url,

58− 'title': ''.join(result.xpath(title_xpath)),

59− 'content': ''.join(result.xpath(content_xpath))})

60−

61− # return results

62− return results

searx/engines/gigablast.py+6−2

@@ -10,6 +10,7 @@

1010 @parse url, title, content

1111"""

1212

13+import random

1314from json import loads

1415from time import time

1516from lxml.html import fromstring

@@ -32,7 +33,8 @@ search_string = 'search?{query}'\

3233 '&qh=0'\

3334 '&qlang={lang}'\

3435 '&ff={safesearch}'\

35− '&rxikd={rxikd}' # random number - 9 digits

36+ '&rxieu={rxieu}'\

37+ '&rand={rxikd}' # current unix timestamp

3638

3739# specific xpath variables

3840results_xpath = '//response//result'

@@ -59,10 +61,12 @@ def request(query, params):

5961 else:

6062 safesearch = 0

6163

64+ # rxieu is some kind of hash from the search query, but accepts random atm

6265 search_path = search_string.format(query=urlencode({'q': query}),

6366 offset=offset,

6467 number_of_results=number_of_results,

65− rxikd=str(time())[:9],

68+ rxikd=int(time() * 1000),

69+ rxieu=random.randint(1000000000, 9999999999),

6670 lang=language,

6771 safesearch=safesearch)

6872

searx/engines/google_news.py+2−2

@@ -67,8 +67,8 @@ def response(resp):

6767 for result in dom.xpath('//div[@class="g"]|//div[@class="g _cy"]'):

6868 try:

6969 r = {

70− 'url': result.xpath('.//div[@class="_cnc"]//a/@href')[0],

71− 'title': ''.join(result.xpath('.//div[@class="_cnc"]//h3//text()')),

70+ 'url': result.xpath('.//a[@class="l _PMs"]')[0].attrib.get("href"),

71+ 'title': ''.join(result.xpath('.//a[@class="l _PMs"]//text()')),

7272 'content': ''.join(result.xpath('.//div[@class="st"]//text()')),

7373 }

7474 except:

searx/settings.yml+13−12

@@ -242,15 +242,16 @@ engines:

242242 disabled: True

243243

244244 - name : gitlab

245− engine : xpath

245+ engine : json_engine

246246 paging : True

247− search_url : https://gitlab.com/search?page={pageno}&search={query}

248− url_xpath : //li[@class="project-row"]//a[@class="project"]/@href

249− title_xpath : //li[@class="project-row"]//span[contains(@class, "project-full-name")]

250− content_xpath : //li[@class="project-row"]//div[@class="description"]/p

247+ search_url : https://gitlab.com/api/v4/projects?search={query}&page={pageno}

248+ url_query : web_url

249+ title_query : name_with_namespace

250+ content_query : description

251+ page_size : 20

251252 categories : it

252253 shortcut : gl

253− timeout : 5.0

254+ timeout : 10.0

254255 disabled : True

255256

256257 - name : github

@@ -321,9 +322,9 @@ engines:

321322 engine : xpath

322323 paging : True

323324 search_url : https://geektimes.ru/search/page{pageno}/?q={query}

324− url_xpath : //div[@class="search_results"]//a[@class="post__title_link"]/@href

325− title_xpath : //div[@class="search_results"]//a[@class="post__title_link"]

326− content_xpath : //div[@class="search_results"]//div[contains(@class, "content")]

325+ url_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]/@href

326+ title_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]

327+ content_xpath : //article[contains(@class, "post")]//div[contains(@class, "post__text")]

327328 categories : it

328329 timeout : 4.0

329330 disabled : True

@@ -333,9 +334,9 @@ engines:

333334 engine : xpath

334335 paging : True

335336 search_url : https://habrahabr.ru/search/page{pageno}/?q={query}

336− url_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")]/@href

337− title_xpath : //div[@class="search_results"]//a[contains(@class, "post__title_link")]

338− content_xpath : //div[@class="search_results"]//div[contains(@class, "content")]

337+ url_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]/@href

338+ title_xpath : //article[contains(@class, "post")]//a[@class="post__title_link"]

339+ content_xpath : //article[contains(@class, "post")]//div[contains(@class, "post__text")]

339340 categories : it

340341 timeout : 4.0

341342 disabled : True

tests/unit/engines/test_google_news.py+56−6

@@ -37,14 +37,64 @@ class TestGoogleNewsEngine(SearxTestCase):

3737 self.assertEqual(google_news.response(response), [])

3838

3939 html = u"""

40−<div class="g">

41−<div class="ts _V6c _Zmc _XO _knc _d7c"><a class="top _vQb _mnc" href="http://this.is.the.url" onmousedown="return rwt(this,'','','','5','AFQjCNGixEtJGC3qTB9pYFLXlRj8XXwdiA','','0ahUKEwiG7O_M5-rQAhWDtRoKHd0RD5QQvIgBCCwwBA','','',event)"><img class="th _lub" id="news-thumbnail-image-52779299683347" src="data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBwgHBgkIBwgKCgkLDRYPDQwMDRsUFRAWIB0iIiAdHx8kKDQsJCYxJx8fLT0tMTU3Ojo6Iys/RD84QzQ5OjcBCgoKDQwNGg8PGjclHyU3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3N//AABEIAGQAZAMBIgACEQEDEQH/xAAbAAACAwEBAQAAAAAAAAAAAAAFBgADBAIBB//EADsQAAIBAwIEAwUGBAUFAAAAAAECAwAEEQUhBhIxURNBYRQiMnGBFSORobHBB1Ji4UKS0fDxFiQzNYL/xAAYAQEBAQEBAAAAAAAAAAAAAAACAwEEAP/EACERAAMBAAICAgMBAAAAAAAAAAABAhEhMQMSE2EEQVEi/9oADAMBAAIRAxEAPwDSkKr8JFWIzwnMTlSepFdiBgMiuxEdid6B7QbrEr8qE4L4wGNCIT7u/WifE0eIosHBz0oWnugCjQ4BfEipJHbpK/JG0gDP/KO9XT6Lw7BGDHrofIBIJQ4NXalAs8IDJnFLepS21pmOOFXl679FoY66ZTcNwWxtGzaaqwOf8JFcXupSSr4cupSOp6gnGaDQahJESQwU/wBIxXbX0l0jl+VgduYjJrfj/ofc9MluueV1z3xV2jlX1AlDkY61TY3MAk5LuJSnm4G4+nnTFb2ltEweFRvuCO1ebw8uTYNnQ9iDTtDKJYY8DflFJgXLp8xT5FbJHBGx/lFFM9a4MrQFjnFSilvPGiYaItv1zUrTnwHljgLViqwUDyNFEskKlmIDCs08JMWHfl32INZ8hnIscWII4oWPmdqCg9KLccsUtLZM596g6HmVT6Um9WlvGeXk/gWruCObGFz3pb0/R59Y1D2WDYk5lmfpGP3PpW7iK4aIxY3CqWA7t5UxcG3SQWxsZI2VI0MxM6hedj1Oeo9Kc5M6PPagQ3DdjpcpWeHx2U7s+9LuvPCsr+yoETIwAOgpq1DV59RmuJIpbS2hhOG5zvmlLVXE4OCpcnBKdDRW7ptZgMEuCCPrTJwteGYS2z78vvpnt5j/AH3pWIxRbhZ+XV4h5Orj8s/tTtcE5fI6xKWmjVepYfrX0trQrbxLICDyDavm0EvgXEU2M8jhsfI033fHiyxxSfZkwj+EtjP6VzpPR0E8ICQR0qUuT8Y2DSExxtj69alVwOINy3Esye7zKfnXdpZzzMjSfAO1KGm8UX99qNvbm1SNZZApYnoPOimr6rf2crBLmQoTsFIwo8htUHDRk+N0Zv4nQi2itMHALUAiP3SH0oxJrUtyqJcKk4GDiZeb9a0LqkOADY2nz8FaSeLCkxgsWUVrecWQRXwDQW8DzMpGckdNvPr0orxZfpa2if8AYywPIfDkY4YDYHbHbcVgucf9YWd3CqR+MxUgDC5C7bDy2FecRzXQsxJd3lmJN8wCPCvgkZBBOc1Vr2kccaA7PkuBOm6qzD3+XZh1/UflWG+isoGAgMjPzb8x2Jz2r37Sm8AouynqKFyOQxkk8unqa9KYKawolI529CRRLhr/ANxb46AOT/lNBgSTTPwTL7PqMlwpHOkXKuR3IzTvonPY2WqCa6hiPR3VT9TTfx3pzaRb2IskUJICpBPUgUvDXphjLR5HQ8oq6bii9uOUXExmC/CJBzY/GoTWFXOiKV1BZpvEjJYyEkgbb1Kdvt19j9zk9fuk/wBKlL5foz0Zk0vTWa0N/hyVYqiKcZ26/jSvdX81pqET+0PJCX8Nw3l6H5b19H0pccPWpxu2WO/9R3pQ4s0yzi1K4gcFGuW8WN+b3STny8t6xr9lU+MRe2AQVPukZGKq9qTxzCGJcLzbHIqm2uILSCO2uiyvF7pdh1HUNjtQO0uootbLSmQ25kIfwyOYrnyztmvT/oL4C91M3NHIh96GRZB9D+/T61xxNf6XqbBrC2LSbkhVyfypk1Cew1SP7F0exNjbW8LXeovKwMx5VJCs3/0vToT6EUhcHTqOIEfConIyLy+QNX9MWElYOmkYHDKVx5EYrFIrSNk7imriCFrm7kt+TM8R5ie696y6Do41G5naUlLa2TnkxjJ7L9TR3BOdeCzgbg82fntRvR1mtbpGYe5IvUdiP7isrWDSTuIwFIyewA+tbraC7js4J7WVpYz92VUZMTdeUr28wenX1rXyg+rTC7TkHrXonPeiGk6TFeWeNQuWsLot7kkihoXHrj3l+e4oZJbSxzTxY5mgcrJyHmAx57eXrXOVc1Pawt8dq8rPv5VKww+jWUgbhGymTO8QIPod6y8cadHfaQsgws8GWUjrjO4/erXxa8M2cETZCW6bEdNq513UlsbSO6mh9oRXXmh5+UPkk4Jx54NdUrQ08Qj2cQ1ezMDkpexA7k/EBQ+C3ubOVXliICts3l1rXqEr3+qPeWASK7n55vZ7bHhxDGeQf1coJP4dc13Y8SMiPDeRBiRjp19DUaiofBs3NLkKahcwcNaLJaQMtzquspz3jhuYRRE5EYPc9W/4oDDYGyvLO4upLdB4YzFFJmRQq5HMvcj9qtvI7SPXpGguxqFvFysZSvKHON167gdM1a/EUb6fLYz2FvI01yJZrphmVlyCVHYfLue9dTWyRisotfVpLm8D272suB0mURtjtkV3p2qwWF5P7VbmETAK6ZyrD0odrtq+nTiy1PT47Oc4mWWPBflYbA4OMenWsdnd3FusqxOkyMpVkYdR9a5nLXBer2tQyxRWcjznT5CpfDBMbtjoPl6VlN+sCLNbwMZjIysoAGPQj8OtBre5dJeaPKN5ods0aGoW2o2rQ3i/eHYSpsw+dTc4dng/L9J9WjSsK3Og3V/JeQWvhL4cUcr5Zj5j0JGw9aG6Jpst3NDFLcmzNzA8ts2ATIyZ265Xz/vtWbVbOzgQCW4eJ4oT4I8MkSHyUYHmepJAHY0PWGSbT11CFFSGKQwM6EhmYjOTv2OM7darCyeDl/J8r8nk1sIwXDSRK7jDHrnY586lNuh6Pouu6Xb3ZvE06ZV8KaFCFUuvVgD0zsa8oNE9CmpA/YtoRkhrdPptWPidZ5dGuIraETSye6V64jQB3IH82QD8gcb1gvuIDJp9vaW8JVkRYyZN8kDyArBbXx1XXLO2vb2SKI8yyGBSFRSu+MZJzhc/6V1QsYLeo1R3Oh3dnby6RYvDcWcQE1wUVRM3KASQCd/n39aUkhN7rltFJ1muI0OP6mA/ejeqX1jYaxLpOmzmfT4IQgmOMu5PMzbAdwPpS+l37Pq8NzGvM0Myui9yrAgfpXrfIZ6CkmjXWpa9qcPD9s3hQvIyoGGBGrcvmfP96OcI8YaTpelw2mp6MJkR2cThVkEj9dww28umdsVm0HSotb1O6tbK5v8ATp1R2dmcOo94Aqccp8/yoHxBbrpt39lLMswtCweRVwGdt2/ABR9KbWAX2aNX1231DiCXVru3M5kkLmGR/cIxhV+QAH4UHvbqC5lMsFvFa7Y5Yc4P4k0X1vS7XS9E0znwb+6BmkXkX3E8t8ZzuOpxsawXEa6Td3MKrHLJ4JhdiPgYgc2PUbrn51Nz/WPfoxpOcY8QEjbB3q2KUswzjbzWt+t3Ftcw2Hh28cKwRLCzqP8AydMtjA32rnUtG9jiS4srhJ7OT4JUYZ+TDqDQpCVFpmW/g9llYArvGxHwmrNX0v2HQdKkjcxx3HOZk5jhnGMk9+woRFKUO/Uedbprye8tGiZs28Eikcx+FmBG3ocfkKKWcG1zyPv8PLi2Xh3wpbOSQxTuodFHvA4PbuTUpG07XtW0qA29lMYoy3MVx5kD07AVKxzyZphmvZnWViR/KMeQrvQ4lutQm8bJ8G0lmUZ/xKuRn61KldDACoHYzu7MSx6sepo9pFtE/vMMloyT9cj8sV7UoPsS6G7+HX3Wm6jdLvM0ioWPYLn9SaQFka91NHuTzme4Bkz58zb/AK17Uq1dImg5rcrXPGcUU2GSIxqq+QGA36ml+bMkaXEjM0kw8R89yd6lSpiCfCcEV7fXdpcxq8TWUjjujAggr2NXcJ3k08F9pcpDWslpJNyEfA6gEFe29SpQZqAgbmAY9TvXAY5YZOCRUqV79ifRthucRhXgglxsGkTmIHbNSpUrAn//2Q==" alt="A(z) south témájának képe a következőből: CBC.ca" data-deferred="1" onload="google.aft&&google.aft(this)"></a><div class="_cnc"><h3 class="r _U6c"><a class="l _HId" href="http://this.is.the.url" onmousedown="return rwt(this,'','','','5','AFQjCNGixEtJGC3qTB9pYFLXlRj8XXwdiA','','0ahUKEwiG7O_M5-rQAhWDtRoKHd0RD5QQqQIILSgAMAQ','','',event)">Meet Thuli Madonsela — <em>South</em> Africa's conscience</a></h3><div class="slp"><span class="_tQb _IId">CBC.ca</span><span class="_v5">-</span><span class="f nsa _uQb">9 órával ezelőtt</span></div><div class="st"><em>South</em> African Public Protector</div></div><div class="_Xmc card-section"><a class="_sQb" href="http://www.news24.com/Columnists/Mpumelelo_Mkhabela/who-really-governs-south-africa-20161209" onmousedown="return rwt(this,'','','','5','AFQjCNHhc2MnYSZ5T4COqInzvgoju5k5bA','','0ahUKEwiG7O_M5-rQAhWDtRoKHd0RD5QQuogBCC4oATAE','','',event)">Who really governs <em>South</em> Africa?</a><br><span class="_Wmc _GId">Vélemény</span><span class="_v5">-</span><span class="_tQb _IId">News24</span><span class="_v5">-</span><span class="f nsa _uQb">2016. dec. 8.</span></div><div class="_Vmc"></div></div>

40+<h2 class="hd">Search Results</h2>

41+<div data-async-context="query:searx" id="ires">

42+ <div eid="oC2oWcGXCafR6ASkwoCwDA" id="rso">

43+ <div class="_NId">

44+

45+ <div class="g _cy">

46+ <div class="ts _JGs _JHs _tJs _KGs _jHs">

47+ <div class="_hJs">

48+ <h3 class="r _gJs">

49+ <a class="l _PMs" href="https://example.com/" onmousedown="return rwt(this,'','','','11','AFQjCNEyehpzD5cJK1KUfXBx9RmsbqqG9g','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAggiKAAwAA','','',event)">Example title</a>

50+ </h3>

51+ <div class="slp">

52+ <span class="_OHs _PHs">

53+ Mac & i</span>

54+ <span class="_QGs">

55+ -</span>

56+ <span class="f nsa _QHs">

57+ Mar 21, 2016</span>

58+ </div>

59+ <div class="st">Example description</div>

60+ </div>

61+ </div>

62+ </div>

63+ <div class="g _cy">

64+ <div class="ts _JGs _JHs _oGs _KGs _jHs">

65+ <a class="top _xGs _SHs" href="https://example2.com/" onmousedown="return rwt(this,'','','','12','AFQjCNHObfH7sYmLWI1SC-YhWXKZFRzRjw','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChC8iAEIJDAB','','',event)">

66+ <img class="th _RGs" src="https://example2.com/image.jpg" alt="Story image for searx from Golem.de" onload="typeof google==='object'&&google.aft&&google.aft(this)">

67+ </a>

68+ <div class="_hJs">

69+ <h3 class="r _gJs">

70+ <a class="l _PMs" href="https://example2.com/" onmousedown="return rwt(this,'','','','12','AFQjCNHObfH7sYmLWI1SC-YhWXKZFRzRjw','','0ahUKEwjB58OR54HWAhWnKJoKHSQhAMY4ChCpAgglKAAwAQ','','',event)">Example title 2</a>

71+ </h3>

72+ <div class="slp">

73+ <span class="_OHs _PHs">

74+ Golem.de</span>

75+ <span class="_QGs">

76+ -</span>

77+ <span class="f nsa _QHs">

78+ Oct 4, 2016</span>

79+ </div>

80+ <div class="st">Example description 2</div>

81+ </div>

82+ </div>

83+ </div>

84+ </div>

85+ </div>

4286</div>

87+

88+

4389 """ # noqa

4490 response = mock.Mock(text=html)

4591 results = google_news.response(response)

4692 self.assertEqual(type(results), list)

47− self.assertEqual(len(results), 1)

48− self.assertEqual(results[0]['title'], u'Meet Thuli Madonsela \u2014 South Africa\'s conscience')

49− self.assertEqual(results[0]['url'], 'http://this.is.the.url')

50− self.assertEqual(results[0]['content'], 'South African Public Protector')

93+ self.assertEqual(len(results), 2)

94+ self.assertEqual(results[0]['title'], u'Example title')

95+ self.assertEqual(results[0]['url'], 'https://example.com/')

96+ self.assertEqual(results[0]['content'], 'Example description')

97+ self.assertEqual(results[1]['title'], u'Example title 2')

98+ self.assertEqual(results[1]['url'], 'https://example2.com/')

99+ self.assertEqual(results[1]['content'], 'Example description 2')

100+ self.assertEqual(results[1]['img_src'], 'https://example2.com/image.jpg')