f825752 · searx · git.iko.soy

searx/engines/bing.py+1

@@ -23,6 +23,7 @@ language_support = True

2323base_url = 'https://www.bing.com/'

2424search_string = 'search?{query}&first={offset}'

2525

26+

2627# do search-request

2728def request(query, params):

2829 offset = (params['pageno'] - 1) * 10 + 1

searx/engines/bing_images.py+1

@@ -24,6 +24,7 @@ paging = True

2424base_url = 'https://www.bing.com/'

2525search_string = 'images/search?{query}&count=10&first={offset}'

2626

27+

2728# do search-request

2829def request(query, params):

2930 offset = (params['pageno'] - 1) * 10 + 1

searx/engines/bing_news.py+1

@@ -24,6 +24,7 @@ language_support = True

2424base_url = 'https://www.bing.com/'

2525search_string = 'news/search?{query}&first={offset}'

2626

27+

2728# do search-request

2829def request(query, params):

2930 offset = (params['pageno'] - 1) * 10 + 1

searx/engines/deviantart.py+29−3

@@ -1,35 +1,61 @@

1+## Deviantart (Images)

2+#

3+# @website https://www.deviantart.com/

4+# @provide-api yes (https://www.deviantart.com/developers/) (RSS)

5+#

6+# @using-api no (TODO, rewrite to api)

7+# @results HTML

8+# @stable no (HTML can change)

9+# @parse url, title, thumbnail, img_src

10+#

11+# @todo rewrite to api

12+

113from urllib import urlencode

214from urlparse import urljoin

315from lxml import html

416

17+# engine dependent config

518categories = ['images']

19+paging = True

620

21+# search-url

722base_url = 'https://www.deviantart.com/'

823search_url = base_url+'search?offset={offset}&{query}'

924

10−paging = True

11−

1225

26+# do search-request

1327def request(query, params):

1428 offset = (params['pageno'] - 1) * 24

29+

1530 params['url'] = search_url.format(offset=offset,

1631 query=urlencode({'q': query}))

32+

1733 return params

1834

1935

36+# get response from search-request

2037def response(resp):

2138 results = []

39+

40+ # return empty array if a redirection code is returned

2241 if resp.status_code == 302:

23− return results

42+ return []

43+

2444 dom = html.fromstring(resp.text)

45+

46+ # parse results

2547 for result in dom.xpath('//div[contains(@class, "tt-a tt-fh")]'):

2648 link = result.xpath('.//a[contains(@class, "thumb")]')[0]

2749 url = urljoin(base_url, link.attrib.get('href'))

2850 title_links = result.xpath('.//span[@class="details"]//a[contains(@class, "t")]') # noqa

2951 title = ''.join(title_links[0].xpath('.//text()'))

3052 img_src = link.xpath('.//img')[0].attrib['src']

53+

54+ # append result

3155 results.append({'url': url,

3256 'title': title,

3357 'img_src': img_src,

3458 'template': 'images.html'})

59+

60+ # return results

3561 return results

searx/engines/duckduckgo.py+38−34

@@ -1,65 +1,69 @@

1+## DuckDuckGo (Web)

2+#

3+# @website https://duckduckgo.com/

4+# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable no (HTML can change)

9+# @parse url, title, content

10+#

11+# @todo rewrite to api

12+# @todo language support (the current used site does not support language-change)

13+

114from urllib import urlencode

215from lxml.html import fromstring

316from searx.utils import html_to_text

417

5−url = 'https://duckduckgo.com/html?{query}&s={offset}'

18+# engine dependent config

19+categories = ['general']

20+paging = True

621locale = 'us-en'

722

23+# search-url

24+url = 'https://duckduckgo.com/html?{query}&s={offset}'

25+

26+# specific xpath variables

27+result_xpath = '//div[@class="results_links results_links_deep web-result"]' # noqa

28+url_xpath = './/a[@class="large"]/@href'

29+title_xpath = './/a[@class="large"]//text()'

30+content_xpath = './/div[@class="snippet"]//text()'

831

32+

33+# do search-request

934def request(query, params):

1035 offset = (params['pageno'] - 1) * 30

11− q = urlencode({'q': query,

12− 'l': locale})

13− params['url'] = url.format(query=q, offset=offset)

36+

37+ params['url'] = url.format(

38+ query=urlencode({'q': query, 'l': locale}),

39+ offset=offset)

40+

1441 return params

1542

1643

44+# get response from search-request

1745def response(resp):

18− result_xpath = '//div[@class="results_links results_links_deep web-result"]' # noqa

19− url_xpath = './/a[@class="large"]/@href'

20− title_xpath = './/a[@class="large"]//text()'

21− content_xpath = './/div[@class="snippet"]//text()'

2246 results = []

2347

2448 doc = fromstring(resp.text)

2549

50+ # parse results

2651 for r in doc.xpath(result_xpath):

2752 try:

2853 res_url = r.xpath(url_xpath)[-1]

2954 except:

3055 continue

56+

3157 if not res_url:

3258 continue

59+

3360 title = html_to_text(''.join(r.xpath(title_xpath)))

3461 content = html_to_text(''.join(r.xpath(content_xpath)))

62+

63+ # append result

3564 results.append({'title': title,

3665 'content': content,

3766 'url': res_url})

3867

68+ # return results

3969 return results

40−

41−

42−#from json import loads

43−#search_url = url + 'd.js?{query}&p=1&s={offset}'

44−#

45−#paging = True

46−#

47−#

48−#def request(query, params):

49−# offset = (params['pageno'] - 1) * 30

50−# q = urlencode({'q': query,

51−# 'l': locale})

52−# params['url'] = search_url.format(query=q, offset=offset)

53−# return params

54−#

55−#

56−#def response(resp):

57−# results = []

58−# search_res = loads(resp.text[resp.text.find('[{'):-2])[:-1]

59−# for r in search_res:

60−# if not r.get('t'):

61−# continue

62−# results.append({'title': r['t'],

63−# 'content': html_to_text(r['a']),

64−# 'url': r['u']})

65−# return results

searx/engines/dummy.py+8

@@ -1,6 +1,14 @@

1+## Dummy

2+#

3+# @results empty array

4+# @stable yes

5+

6+

7+# do search-request

18def request(query, params):

29 return params

310

411

12+# get response from search-request

513def response(resp):

614 return []

searx/engines/generalfile.py+28−3

@@ -1,35 +1,60 @@

1+## General Files (Files)

2+#

3+# @website http://www.general-files.org

4+# @provide-api no (nothing found)

5+#

6+# @using-api no (because nothing found)

7+# @results HTML (using search portal)

8+# @stable no (HTML can change)

9+# @parse url, title, content

10+#

11+# @todo detect torrents?

12+

113from lxml import html

214

15+# engine dependent config

16+categories = ['files']

17+paging = True

318

19+# search-url

420base_url = 'http://www.general-file.com'

521search_url = base_url + '/files-{letter}/{query}/{pageno}'

622

23+# specific xpath variables

724result_xpath = '//table[@class="block-file"]'

825title_xpath = './/h2/a//text()'

926url_xpath = './/h2/a/@href'

1027content_xpath = './/p//text()'

1128

12−paging = True

13−

1429

30+# do search-request

1531def request(query, params):

32+

1633 params['url'] = search_url.format(query=query,

1734 letter=query[0],

1835 pageno=params['pageno'])

36+

1937 return params

2038

2139

40+# get response from search-request

2241def response(resp):

23−

2442 results = []

43+

2544 dom = html.fromstring(resp.text)

45+

46+ # parse results

2647 for result in dom.xpath(result_xpath):

2748 url = result.xpath(url_xpath)[0]

49+

2850 # skip fast download links

2951 if not url.startswith('/'):

3052 continue

53+

54+ # append result

3155 results.append({'url': base_url + url,

3256 'title': ''.join(result.xpath(title_xpath)),

3357 'content': ''.join(result.xpath(content_xpath))})

3458

59+ # return results

3560 return results

searx/engines/github.py+30−2

@@ -1,31 +1,59 @@

1+## Github (It)

2+#

3+# @website https://github.com/

4+# @provide-api yes (https://developer.github.com/v3/)

5+#

6+# @using-api yes

7+# @results JSON

8+# @stable yes (using api)

9+# @parse url, title, content

10+

111from urllib import urlencode

212from json import loads

313from cgi import escape

414

15+# engine dependent config

516categories = ['it']

617

18+# search-url

719search_url = 'https://api.github.com/search/repositories?sort=stars&order=desc&{query}' # noqa

820

921accept_header = 'application/vnd.github.preview.text-match+json'

1022

1123

24+# do search-request

1225def request(query, params):

1326 params['url'] = search_url.format(query=urlencode({'q': query}))

27+

1428 params['headers']['Accept'] = accept_header

29+

1530 return params

1631

1732

33+# get response from search-request

1834def response(resp):

1935 results = []

36+

2037 search_res = loads(resp.text)

38+

39+ # check if items are recieved

2140 if not 'items' in search_res:

22− return results

41+ return []

42+

43+ # parse results

2344 for res in search_res['items']:

2445 title = res['name']

2546 url = res['html_url']

47+

2648 if res['description']:

2749 content = escape(res['description'][:500])

2850 else:

2951 content = ''

30− results.append({'url': url, 'title': title, 'content': content})

52+

53+ # append result

54+ results.append({'url': url,

55+ 'title': title,

56+ 'content': content})

57+

58+ # return results

3159 return results

searx/engines/piratebay.py+34−7

@@ -1,39 +1,61 @@

1+## Piratebay (Videos, Music, Files)

2+#

3+# @website https://thepiratebay.se

4+# @provide-api no (nothing found)

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable yes (HTML can change)

9+# @parse url, title, content, seed, leech, magnetlink

10+

111from urlparse import urljoin

212from cgi import escape

313from urllib import quote

414from lxml import html

515from operator import itemgetter

616

7−categories = ['videos', 'music']

17+# engine dependent config

18+categories = ['videos', 'music', 'files']

19+paging = True

820

21+# search-url

922url = 'https://thepiratebay.se/'

1023search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'

11−search_types = {'videos': '200',

24+

25+# piratebay specific type-definitions

26+search_types = {'files': '0',

1227 'music': '100',

13− 'files': '0'}

28+ 'videos': '200'}

1429

30+# specific xpath variables

1531magnet_xpath = './/a[@title="Download this torrent using magnet"]'

1632content_xpath = './/font[@class="detDesc"]//text()'

1733

18−paging = True

19−

2034

35+# do search-request

2136def request(query, params):

22− search_type = search_types.get(params['category'], '200')

37+ search_type = search_types.get(params['category'], '0')

38+

2339 params['url'] = search_url.format(search_term=quote(query),

2440 search_type=search_type,

2541 pageno=params['pageno'] - 1)

42+

2643 return params

2744

2845

46+# get response from search-request

2947def response(resp):

3048 results = []

49+

3150 dom = html.fromstring(resp.text)

51+

3252 search_res = dom.xpath('//table[@id="searchResult"]//tr')

3353

54+ # return empty array if nothing is found

3455 if not search_res:

35− return results

56+ return []

3657

58+ # parse results

3759 for result in search_res[1:]:

3860 link = result.xpath('.//div[@class="detName"]//a')[0]

3961 href = urljoin(url, link.attrib.get('href'))

@@ -41,17 +63,21 @@ def response(resp):

4163 content = escape(' '.join(result.xpath(content_xpath)))

4264 seed, leech = result.xpath('.//td[@align="right"]/text()')[:2]

4365

66+ # convert seed to int if possible

4467 if seed.isdigit():

4568 seed = int(seed)

4669 else:

4770 seed = 0

4871

72+ # convert leech to int if possible

4973 if leech.isdigit():

5074 leech = int(leech)

5175 else:

5276 leech = 0

5377

5478 magnetlink = result.xpath(magnet_xpath)[0]

79+

80+ # append result

5581 results.append({'url': href,

5682 'title': title,

5783 'content': content,

@@ -60,4 +86,5 @@ def response(resp):

6086 'magnetlink': magnetlink.attrib['href'],

6187 'template': 'torrent.html'})

6288

89+ # return results sorted by seeder

6390 return sorted(results, key=itemgetter('seed'), reverse=True)

searx/engines/soundcloud.py+29−4

@@ -1,30 +1,55 @@

1+## Soundcloud (Music)

2+#

3+# @website https://soundcloud.com

4+# @provide-api yes (https://developers.soundcloud.com/)

5+#

6+# @using-api yes

7+# @results JSON

8+# @stable yes

9+# @parse url, title, content

10+

111from json import loads

212from urllib import urlencode

313

14+# engine dependent config

415categories = ['music']

16+paging = True

517

18+# api-key

619guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'

7−url = 'https://api.soundcloud.com/'

8−search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id='+guest_client_id # noqa

920

10−paging = True

21+# search-url

22+url = 'https://api.soundcloud.com/'

23+search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id={client_id}'

1124

1225

26+# do search-request

1327def request(query, params):

1428 offset = (params['pageno'] - 1) * 20

29+

1530 params['url'] = search_url.format(query=urlencode({'q': query}),

16− offset=offset)

31+ offset=offset,

32+ client_id=guest_client_id)

33+

1734 return params

1835

1936

37+# get response from search-request

2038def response(resp):

2139 results = []

40+

2241 search_res = loads(resp.text)

42+

43+ # parse results

2344 for result in search_res.get('collection', []):

2445 if result['kind'] in ('track', 'playlist'):

2546 title = result['title']

2647 content = result['description']

48+

49+ # append result

2750 results.append({'url': result['permalink_url'],

2851 'title': title,

2952 'content': content})

53+

54+ # return results

3055 return results

searx/engines/stackoverflow.py+35−7

@@ -1,30 +1,58 @@

1+## Stackoverflow (It)

2+#

3+# @website https://stackoverflow.com/

4+# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)

5+#

6+# @using-api no

7+# @results HTML

8+# @stable no (HTML can change)

9+# @parse url, title, content

10+

111from urlparse import urljoin

212from cgi import escape

313from urllib import urlencode

414from lxml import html

515

16+# engine dependent config

617categories = ['it']

18+paging = True

719

20+# search-url

821url = 'http://stackoverflow.com/'

922search_url = url+'search?{query}&page={pageno}'

10−result_xpath = './/div[@class="excerpt"]//text()'

1123

12−paging = True

24+# specific xpath variables

25+results_xpath = '//div[contains(@class,"question-summary")]'

26+link_xpath = './/div[@class="result-link"]//a|.//div[@class="summary"]//h3//a'

27+title_xpath = './/text()'

28+content_xpath = './/div[@class="excerpt"]//text()'

1329

1430

31+# do search-request

1532def request(query, params):

1633 params['url'] = search_url.format(query=urlencode({'q': query}),

1734 pageno=params['pageno'])

35+

1836 return params

1937

2038

39+# get response from search-request

2140def response(resp):

2241 results = []

42+

2343 dom = html.fromstring(resp.text)

24− for result in dom.xpath('//div[@class="question-summary search-result"]'):

25− link = result.xpath('.//div[@class="result-link"]//a')[0]

44+

45+ # parse results

46+ for result in dom.xpath(results_xpath):

47+ link = result.xpath(link_xpath)[0]

2648 href = urljoin(url, link.attrib.get('href'))

27− title = escape(' '.join(link.xpath('.//text()')))

28− content = escape(' '.join(result.xpath(result_xpath)))

29− results.append({'url': href, 'title': title, 'content': content})

49+ title = escape(' '.join(link.xpath(title_xpath)))

50+ content = escape(' '.join(result.xpath(content_xpath)))

51+

52+ # append result

53+ results.append({'url': href,

54+ 'title': title,

55+ 'content': content})

56+

57+ # return results

3058 return results

searx/engines/startpage.py+53−21

@@ -1,47 +1,79 @@

1+## Startpage (Web)

2+#

3+# @website https://startpage.com

4+# @provide-api no (nothing found)

5+#

6+# @using-api no

7+# @results HTML

8+# @stable no (HTML can change)

9+# @parse url, title, content

10+#

11+# @todo paging

12+

113from urllib import urlencode

214from lxml import html

315from cgi import escape

16+import re

17+

18+# engine dependent config

19+categories = ['general']

20+# there is a mechanism to block "bot" search (probably the parameter qid), require storing of qid's between mulitble search-calls

21+#paging = False

22+language_support = True

423

5−base_url = None

6−search_url = None

24+# search-url

25+base_url = 'https://startpage.com/'

26+search_url = base_url + 'do/search'

727

8−# TODO paging

9−paging = False

10−# TODO complete list of country mapping

11−country_map = {'en_US': 'eng',

12− 'en_UK': 'uk',

13− 'nl_NL': 'ned'}

28+# specific xpath variables

29+# ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]

30+# not ads: div[@class="result"] are the direct childs of div[@id="results"]

31+results_xpath = '//div[@class="result"]'

32+link_xpath = './/h3/a'

1433

1534

35+# do search-request

1636def request(query, params):

37+ offset = (params['pageno'] - 1) * 10

1738 query = urlencode({'q': query})[2:]

39+

1840 params['url'] = search_url

1941 params['method'] = 'POST'

2042 params['data'] = {'query': query,

21− 'startat': (params['pageno'] - 1) * 10} # offset

22− country = country_map.get(params['language'], 'eng')

23− params['cookies']['preferences'] = \

24− 'lang_homepageEEEs/air/{country}/N1NsslEEE1N1Nfont_sizeEEEmediumN1Nrecent_results_filterEEE1N1Nlanguage_uiEEEenglishN1Ndisable_open_in_new_windowEEE0N1Ncolor_schemeEEEnewN1Nnum_of_resultsEEE10N1N'.format(country=country) # noqa

43+ 'startat': offset}

44+

45+ # set language if specified

46+ if params['language'] != 'all':

47+ params['data']['with_language'] = 'lang_' + params['language'].split('_')[0]

48+

2549 return params

2650

2751

52+# get response from search-request

2853def response(resp):

2954 results = []

55+

3056 dom = html.fromstring(resp.content)

31− # ads xpath //div[@id="results"]/div[@id="sponsored"]//div[@class="result"]

32− # not ads: div[@class="result"] are the direct childs of div[@id="results"]

33− for result in dom.xpath('//div[@class="result"]'):

34− link = result.xpath('.//h3/a')[0]

57+

58+ # parse results

59+ for result in dom.xpath(results_xpath):

60+ link = result.xpath(link_xpath)[0]

3561 url = link.attrib.get('href')

36− if url.startswith('http://www.google.')\

37− or url.startswith('https://www.google.'):

38− continue

3962 title = escape(link.text_content())

4063

41− content = ''

64+ # block google-ad url's

65+ if re.match("^http(s|)://www.google.[a-z]+/aclk.*$", url):

66+ continue

67+

4268 if result.xpath('./p[@class="desc"]'):

4369 content = escape(result.xpath('./p[@class="desc"]')[0].text_content())

70+ else:

71+ content = ''

4472

45− results.append({'url': url, 'title': title, 'content': content})

73+ # append result

74+ results.append({'url': url,

75+ 'title': title,

76+ 'content': content})

4677

78+ # return results

4779 return results

searx/engines/twitter.py+35−2

@@ -1,30 +1,63 @@

1+## Twitter (Social media)

2+#

3+# @website https://www.bing.com/news

4+# @provide-api yes (https://dev.twitter.com/docs/using-search)

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable no (HTML can change)

9+# @parse url, title, content

10+#

11+# @todo publishedDate

12+

113from urlparse import urljoin

214from urllib import urlencode

315from lxml import html

416from cgi import escape

517

18+# engine dependent config

619categories = ['social media']

20+language_support = True

721

22+# search-url

823base_url = 'https://twitter.com/'

924search_url = base_url+'search?'

25+

26+# specific xpath variables

27+results_xpath = '//li[@data-item-type="tweet"]'

28+link_xpath = './/small[@class="time"]//a'

1029title_xpath = './/span[@class="username js-action-profile-name"]//text()'

1130content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'

1231

1332

33+# do search-request

1434def request(query, params):

1535 params['url'] = search_url + urlencode({'q': query})

36+

37+ # set language if specified

38+ if params['language'] != 'all':

39+ params['cookies']['lang'] = params['language'].split('_')[0]

40+

1641 return params

1742

1843

44+# get response from search-request

1945def response(resp):

2046 results = []

47+

2148 dom = html.fromstring(resp.text)

22− for tweet in dom.xpath('//li[@data-item-type="tweet"]'):

23− link = tweet.xpath('.//small[@class="time"]//a')[0]

49+

50+ # parse results

51+ for tweet in dom.xpath(results_xpath):

52+ link = tweet.xpath(link_xpath)[0]

2453 url = urljoin(base_url, link.attrib.get('href'))

2554 title = ''.join(tweet.xpath(title_xpath))

2655 content = escape(''.join(tweet.xpath(content_xpath)))

56+

57+ # append result

2758 results.append({'url': url,

2859 'title': title,

2960 'content': content})

61+

62+ # return results

3063 return results

searx/engines/wikipedia.py+47−10

@@ -1,30 +1,67 @@

1+## Wikipedia (Web)

2+#

3+# @website http://www.wikipedia.org

4+# @provide-api yes (http://www.mediawiki.org/wiki/API:Search)

5+#

6+# @using-api yes

7+# @results JSON

8+# @stable yes

9+# @parse url, title

10+#

11+# @todo content

12+

113from json import loads

214from urllib import urlencode, quote

315

4−url = 'https://{language}.wikipedia.org/'

5−

6−search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}' # noqa

7−

8−number_of_results = 10

9−

16+# engine dependent config

17+categories = ['general']

1018language_support = True

19+paging = True

20+number_of_results = 1

21+

22+# search-url

23+url = 'https://{language}.wikipedia.org/'

24+search_url = url + 'w/api.php?action=query&list=search&{query}&srprop=timestamp&format=json&sroffset={offset}&srlimit={limit}' # noqa

1125

1226

27+# do search-request

1328def request(query, params):

14− offset = (params['pageno'] - 1) * 10

29+ offset = (params['pageno'] - 1) * number_of_results

30+

1531 if params['language'] == 'all':

1632 language = 'en'

1733 else:

1834 language = params['language'].split('_')[0]

35+

36+ # write search-language back to params, required in response

1937 params['language'] = language

38+

2039 params['url'] = search_url.format(query=urlencode({'srsearch': query}),

2140 offset=offset,

41+ limit=number_of_results,

2242 language=language)

43+

2344 return params

2445

2546

47+# get response from search-request

2648def response(resp):

49+ results = []

50+

2751 search_results = loads(resp.text)

28− res = search_results.get('query', {}).get('search', [])

29− return [{'url': url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8')), # noqa

30− 'title': result['title']} for result in res[:int(number_of_results)]]

52+

53+ # return empty array if there are no results

54+ if not search_results.get('query', {}).get('search'):

55+ return []

56+

57+ # parse results

58+ for result in search_results['query']['search']:

59+ res_url = url.format(language=resp.search_params['language']) + 'wiki/' + quote(result['title'].replace(' ', '_').encode('utf-8'))

60+

61+ # append result

62+ results.append({'url': res_url,

63+ 'title': result['title'],

64+ 'content': ''})

65+

66+ # return results

67+ return results

searx/engines/youtube.py+36−7

@@ -1,42 +1,69 @@

1+## Youtube (Videos)

2+#

3+# @website https://www.youtube.com/

4+# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)

5+#

6+# @using-api yes

7+# @results JSON

8+# @stable yes

9+# @parse url, title, content, publishedDate, thumbnail

10+

111from json import loads

212from urllib import urlencode

313from dateutil import parser

414

15+# engine dependent config

516categories = ['videos']

6−

7−search_url = ('https://gdata.youtube.com/feeds/api/videos'

8− '?alt=json&{query}&start-index={index}&max-results=25') # noqa

9−

1017paging = True

18+language_support = True

19+

20+# search-url

21+base_url = 'https://gdata.youtube.com/feeds/api/videos'

22+search_url = base_url + '?alt=json&{query}&start-index={index}&max-results=5' # noqa

1123

1224

25+# do search-request

1326def request(query, params):

14− index = (params['pageno'] - 1) * 25 + 1

27+ index = (params['pageno'] - 1) * 5 + 1

28+

1529 params['url'] = search_url.format(query=urlencode({'q': query}),

1630 index=index)

31+

32+ # add language tag if specified

33+ if params['language'] != 'all':

34+ params['url'] += '&lr=' + params['language'].split('_')[0]

35+

1736 return params

1837

1938

39+# get response from search-request

2040def response(resp):

2141 results = []

42+

2243 search_results = loads(resp.text)

44+

45+ # return empty array if there are no results

2346 if not 'feed' in search_results:

24− return results

47+ return []

48+

2549 feed = search_results['feed']

2650

51+ # parse results

2752 for result in feed['entry']:

2853 url = [x['href'] for x in result['link'] if x['type'] == 'text/html']

54+

2955 if not url:

3056 return

57+

3158 # remove tracking

3259 url = url[0].replace('feature=youtube_gdata', '')

3360 if url.endswith('&'):

3461 url = url[:-1]

62+

3563 title = result['title']['$t']

3664 content = ''

3765 thumbnail = ''

3866

39−#"2013-12-31T15:22:51.000Z"

4067 pubdate = result['published']['$t']

4168 publishedDate = parser.parse(pubdate)

4269

@@ -49,6 +76,7 @@ def response(resp):

4976 else:

5077 content = result['content']['$t']

5178

79+ # append result

5280 results.append({'url': url,

5381 'title': title,

5482 'content': content,

@@ -56,4 +84,5 @@ def response(resp):

5684 'publishedDate': publishedDate,

5785 'thumbnail': thumbnail})

5886

87+ # return results

5988 return results

searx/settings.yml+1−15

@@ -11,9 +11,8 @@ server:

1111engines:

1212 - name : wikipedia

1313 engine : wikipedia

14− number_of_results : 1

15− paging : False

1614 shortcut : wp

15+# number_of_results : 1 # default is 1

1716

1817 - name : bing

1918 engine : bing

@@ -37,7 +36,6 @@ engines:

3736

3837 - name : deviantart

3938 engine : deviantart

40− categories : images

4139 shortcut : da

4240 timeout: 3.0

4341

@@ -47,7 +45,6 @@ engines:

4745

4846 - name : duckduckgo

4947 engine : duckduckgo

50− locale : en-us

5148 shortcut : ddg

5249

5350# down - website is under criminal investigation by the UK

@@ -64,12 +61,10 @@ engines:

6461

6562 - name : general-file

6663 engine : generalfile

67− categories : files

6864 shortcut : gf

6965

7066 - name : github

7167 engine : github

72− categories : it

7368 shortcut : gh

7469

7570 - name : google

@@ -86,23 +81,18 @@ engines:

8681

8782 - name : piratebay

8883 engine : piratebay

89− categories : videos, music, files

9084 shortcut : tpb

9185

9286 - name : soundcloud

9387 engine : soundcloud

94− categories : music

9588 shortcut : sc

9689

9790 - name : stackoverflow

9891 engine : stackoverflow

99− categories : it

10092 shortcut : st

10193

10294 - name : startpage

10395 engine : startpage

104− base_url : 'https://startpage.com/'

105− search_url : 'https://startpage.com/do/search'

10696 shortcut : sp

10797

10898# +30% page load time

@@ -113,7 +103,6 @@ engines:

113103

114104 - name : twitter

115105 engine : twitter

116− categories : social media

117106 shortcut : tw

118107

119108# maybe in a fun category

@@ -142,13 +131,10 @@ engines:

142131

143132 - name : youtube

144133 engine : youtube

145− categories : videos

146134 shortcut : yt

147135

148136 - name : dailymotion

149137 engine : dailymotion

150− locale : en_US

151− categories : videos

152138 shortcut : dm

153139

154140 - name : vimeo