git.iko.soy/searx

on master help sign in

searx/engines/1337x.py+14

@@ -1,7 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ 1337x

4+"""

5+

16from urllib.parse import quote, urljoin

27from lxml import html

38from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex

49

10+# about

11+about = {

12+ "website": 'https://1337x.to/',

13+ "wikidata_id": 'Q28134166',

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

519

620url = 'https://1337x.to/'

721search_url = url + 'search/{search_term}/{pageno}/'

searx/engines/acgsou.py+11−7

@@ -1,18 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)

3−

4− @website https://www.acgsou.com/

5− @provide-api no

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, content, seed, leech, torrentfile

104"""

115

126from urllib.parse import urlencode

137from lxml import html

148from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex

159

10+# about

11+about = {

12+ "website": 'https://www.acgsou.com/',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1620# engine dependent config

1721categories = ['files', 'images', 'videos', 'music']

1822paging = True

searx/engines/ahmia.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Ahmia (Onions)

3−

4− @website http://msydqstlz2kzerdg.onion

5− @provides-api no

6−

7− @using-api no

8− @results HTML

9− @stable no

10− @parse url, title, content

114"""

125

136from urllib.parse import urlencode, urlparse, parse_qs

147from lxml.html import fromstring

158from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath

169

10+# about

11+about = {

12+ "website": 'http://msydqstlz2kzerdg.onion',

13+ "wikidata_id": 'Q18693938',

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1720# engine config

1821categories = ['onions']

1922paging = True

searx/engines/apkmirror.py+10−7

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 APK Mirror

3−

4− @website https://www.apkmirror.com

5−

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, thumbnail_src

104"""

115

126from urllib.parse import urlencode

137from lxml import html

148from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

159

10+# about

11+about = {

12+ "website": 'https://www.apkmirror.com',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

1619

1720# engine dependent config

1821categories = ['it']

searx/engines/archlinux.py+12−8

@@ -1,20 +1,24 @@

1−# -*- coding: utf-8 -*-

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 Arch Linux Wiki

54

6− @website https://wiki.archlinux.org

7− @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it

8− @using-api no

9− @results HTML

10− @stable no (HTML can change)

11− @parse url, title

5+ API: Mediawiki provides API, but Arch Wiki blocks access to it

126"""

137

148from urllib.parse import urlencode, urljoin

159from lxml import html

1610from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

1711

12+# about

13+about = {

14+ "website": 'https://wiki.archlinux.org/',

15+ "wikidata_id": 'Q101445877',

16+ "official_api_documentation": None,

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'HTML',

20+}

21+

1822# engine dependent config

1923categories = ['it']

2024language_support = True

searx/engines/arxiv.py+10−9

@@ -1,20 +1,21 @@

1−#!/usr/bin/env python

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 ArXiV (Scientific preprints)

5− @website https://arxiv.org

6− @provide-api yes (export.arxiv.org/api/query)

7− @using-api yes

8− @results XML-RSS

9− @stable yes

10− @parse url, title, publishedDate, content

11− More info on api: https://arxiv.org/help/api/user-manual

124"""

135

146from lxml import html

157from datetime import datetime

168from searx.utils import eval_xpath_list, eval_xpath_getindex

179

10+# about

11+about = {

12+ "website": 'https://arxiv.org',

13+ "wikidata_id": 'Q118398',

14+ "official_api_documentation": 'https://arxiv.org/help/api',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'XML-RSS',

18+}

1819

1920categories = ['science']

2021paging = True

searx/engines/base.py+10−11

@@ -1,16 +1,6 @@

1−#!/usr/bin/env python

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 BASE (Scholar publications)

5−

6− @website https://base-search.net

7− @provide-api yes with authorization (https://api.base-search.net/)

8−

9− @using-api yes

10− @results XML

11− @stable ?

12− @parse url, title, publishedDate, content

13− More info on api: http://base-search.net/about/download/base_interface.pdf

144"""

155

166from urllib.parse import urlencode

@@ -19,6 +9,15 @@ from datetime import datetime

199import re

2010from searx.utils import searx_useragent

2111

12+# about

13+about = {

14+ "website": 'https://base-search.net',

15+ "wikidata_id": 'Q448335',

16+ "official_api_documentation": 'https://api.base-search.net/',

17+ "use_official_api": True,

18+ "require_api_key": False,

19+ "results": 'XML',

20+}

2221

2322categories = ['science']

2423

searx/engines/bing.py+11−11

@@ -1,16 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Bing (Web)

3−

4− @website https://www.bing.com

5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

6− max. 5000 query/month

7−

8− @using-api no (because of query limit)

9− @results HTML (using search portal)

10− @stable no (HTML can change)

11− @parse url, title, content

12−

13− @todo publishedDate

144"""

155

166import re

@@ -21,6 +11,16 @@ from searx.utils import eval_xpath, extract_text, match_language

2111

2212logger = logger.getChild('bing engine')

2313

14+# about

15+about = {

16+ "website": 'https://www.bing.com',

17+ "wikidata_id": 'Q182496',

18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-web-search-api',

19+ "use_official_api": False,

20+ "require_api_key": False,

21+ "results": 'HTML',

22+}

23+

2424# engine dependent config

2525categories = ['general']

2626paging = True

searx/engines/bing_images.py+11−10

@@ -1,15 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Bing (Images)

3−

4− @website https://www.bing.com/images

5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

6− max. 5000 query/month

7−

8− @using-api no (because of query limit)

9− @results HTML (using search portal)

10− @stable no (HTML can change)

11− @parse url, title, img_src

12−

134"""

145

156from urllib.parse import urlencode

@@ -20,6 +11,16 @@ from searx.utils import match_language

2011from searx.engines.bing import language_aliases

2112from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2213

14+# about

15+about = {

16+ "website": 'https://www.bing.com/images',

17+ "wikidata_id": 'Q182496',

18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-image-search-api',

19+ "use_official_api": False,

20+ "require_api_key": False,

21+ "results": 'HTML',

22+}

23+

2324# engine dependent config

2425categories = ['images']

2526paging = True

searx/engines/bing_news.py+11−9

@@ -1,14 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Bing (News)

3−

4− @website https://www.bing.com/news

5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

6− max. 5000 query/month

7−

8− @using-api no (because of query limit)

9− @results RSS (using search portal)

10− @stable yes (except perhaps for the images)

11− @parse url, title, content, publishedDate, thumbnail

124"""

135

146from datetime import datetime

@@ -20,6 +12,16 @@ from searx.utils import match_language, eval_xpath_getindex

2012from searx.engines.bing import language_aliases

2113from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2214

15+# about

16+about = {

17+ "website": 'https://www.bing.com/news',

18+ "wikidata_id": 'Q2878637',

19+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-news-search-api',

20+ "use_official_api": False,

21+ "require_api_key": False,

22+ "results": 'RSS',

23+}

24+

2325# engine dependent config

2426categories = ['news']

2527paging = True

searx/engines/bing_videos.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Bing (Videos)

3−

4− @website https://www.bing.com/videos

5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search)

6−

7− @using-api no

8− @results HTML

9− @stable no

10− @parse url, title, content, thumbnail

114"""

125

136from json import loads

@@ -18,6 +11,16 @@ from searx.utils import match_language

1811from searx.engines.bing import language_aliases

1912from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2013

14+# about

15+about = {

16+ "website": 'https://www.bing.com/videos',

17+ "wikidata_id": 'Q4914152',

18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-video-search-api',

19+ "use_official_api": False,

20+ "require_api_key": False,

21+ "results": 'HTML',

22+}

23+

2124categories = ['videos']

2225paging = True

2326safesearch = True

searx/engines/btdigg.py+14−8

@@ -1,19 +1,25 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 BTDigg (Videos, Music, Files)

3−

4− @website https://btdig.com

5− @provide-api yes (on demand)

6−

7− @using-api no

8− @results HTML (using search portal)

9− @stable no (HTML can change)

10− @parse url, title, content, seed, leech, magnetlink

114"""

125

136from lxml import html

147from urllib.parse import quote, urljoin

158from searx.utils import extract_text, get_torrent_size

169

10+# about

11+about = {

12+ "website": 'https://btdig.com',

13+ "wikidata_id": 'Q4836698',

14+ "official_api_documentation": {

15+ 'url': 'https://btdig.com/contacts',

16+ 'comment': 'on demand'

17+ },

18+ "use_official_api": False,

19+ "require_api_key": False,

20+ "results": 'HTML',

21+}

22+

1723# engine dependent config

1824categories = ['videos', 'music', 'files']

1925paging = True

searx/engines/command.py+4−15

@@ -1,18 +1,7 @@

1−'''

2−searx is free software: you can redistribute it and/or modify

3−it under the terms of the GNU Affero General Public License as published by

4−the Free Software Foundation, either version 3 of the License, or

5−(at your option) any later version.

6−

7−searx is distributed in the hope that it will be useful,

8−but WITHOUT ANY WARRANTY; without even the implied warranty of

9−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

10−GNU Affero General Public License for more details.

11−

12−You should have received a copy of the GNU Affero General Public License

13−along with searx. If not, see < http://www.gnu.org/licenses/ >.

14−'''

15−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Command (offline)

4+"""

165

176import re

187from os.path import expanduser, isabs, realpath, commonprefix

searx/engines/currency_convert.py+14

@@ -1,5 +1,19 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ currency convert (DuckDuckGo)

4+"""

5+

16import json

27

8+# about

9+about = {

10+ "website": 'https://duckduckgo.com/',

11+ "wikidata_id": 'Q12805',

12+ "official_api_documentation": 'https://duckduckgo.com/api',

13+ "use_official_api": False,

14+ "require_api_key": False,

15+ "results": 'JSONP',

16+}

317

418engine_type = 'online_currency'

519categories = []

searx/engines/dailymotion.py+11−10

@@ -1,15 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Dailymotion (Videos)

3−

4− @website https://www.dailymotion.com

5− @provide-api yes (http://www.dailymotion.com/developer)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, thumbnail, publishedDate, embedded

11−

12− @todo set content-parameter with correct data

134"""

145

156from json import loads

@@ -17,6 +8,16 @@ from datetime import datetime

178from urllib.parse import urlencode

189from searx.utils import match_language, html_to_text

1910

11+# about

12+about = {

13+ "website": 'https://www.dailymotion.com',

14+ "wikidata_id": 'Q769222',

15+ "official_api_documentation": 'https://www.dailymotion.com/developer',

16+ "use_official_api": True,

17+ "require_api_key": False,

18+ "results": 'JSON',

19+}

20+

2021# engine dependent config

2122categories = ['videos']

2223paging = True

searx/engines/deezer.py+11−8

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Deezer (Music)

3−

4− @website https://deezer.com

5− @provide-api yes (http://developers.deezer.com/api/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, embedded

114"""

125

136from json import loads

147from urllib.parse import urlencode

158

9+# about

10+about = {

11+ "website": 'https://deezer.com',

12+ "wikidata_id": 'Q602243',

13+ "official_api_documentation": 'https://developers.deezer.com/',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

18+

1619# engine dependent config

1720categories = ['music']

1821paging = True

searx/engines/deviantart.py+11−10

@@ -1,21 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Deviantart (Images)

3−

4− @website https://www.deviantart.com/

5− @provide-api yes (https://www.deviantart.com/developers/) (RSS)

6−

7− @using-api no (TODO, rewrite to api)

8− @results HTML

9− @stable no (HTML can change)

10− @parse url, title, img_src

11−

12− @todo rewrite to api

134"""

145# pylint: disable=missing-function-docstring

156

167from urllib.parse import urlencode

178from lxml import html

189

10+# about

11+about = {

12+ "website": 'https://www.deviantart.com/',

13+ "wikidata_id": 'Q46523',

14+ "official_api_documentation": 'https://www.deviantart.com/developers/',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1920# engine dependent config

2021categories = ['images']

2122paging = True

searx/engines/dictzone.py+10−7

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Dictzone

3−

4− @website https://dictzone.com/

5− @provide-api no

6− @using-api no

7− @results HTML (using search portal)

8− @stable no (HTML can change)

9− @parse url, title, content

104"""

115

126from urllib.parse import urljoin

137from lxml import html

148from searx.utils import eval_xpath

159

10+# about

11+about = {

12+ "website": 'https://dictzone.com/',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

1619

1720engine_type = 'online_dictionnary'

1821categories = ['general']

searx/engines/digbt.py+10−8

@@ -1,19 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 DigBT (Videos, Music, Files)

3−

4− @website https://digbt.org

5− @provide-api no

6−

7− @using-api no

8− @results HTML (using search portal)

9− @stable no (HTML can change)

10− @parse url, title, content, magnetlink

114"""

125

136from urllib.parse import urljoin

147from lxml import html

158from searx.utils import extract_text, get_torrent_size

169

10+# about

11+about = {

12+ "website": 'https://digbt.org',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

1719

1820categories = ['videos', 'music', 'files']

1921paging = True

searx/engines/digg.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Digg (News, Social media)

3−

4− @website https://digg.com

5− @provide-api no

6−

7− @using-api no

8− @results HTML (using search portal)

9− @stable no (HTML can change)

10− @parse url, title, content, publishedDate, thumbnail

114"""

125# pylint: disable=missing-function-docstring

136

@@ -17,6 +10,16 @@ from datetime import datetime

1710

1811from lxml import html

1912

13+# about

14+about = {

15+ "website": 'https://digg.com',

16+ "wikidata_id": 'Q270478',

17+ "official_api_documentation": None,

18+ "use_official_api": False,

19+ "require_api_key": False,

20+ "results": 'HTML',

21+}

22+

2023# engine dependent config

2124categories = ['news', 'social media']

2225paging = True

searx/engines/doku.py+14−10

@@ -1,18 +1,22 @@

1−# Doku Wiki

2−#

3−# @website https://www.dokuwiki.org/

4−# @provide-api yes

5−# (https://www.dokuwiki.org/devel:xmlrpc)

6−#

7−# @using-api no

8−# @results HTML

9−# @stable yes

10−# @parse (general) url, title, content

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Doku Wiki

4+"""

115

126from urllib.parse import urlencode

137from lxml.html import fromstring

148from searx.utils import extract_text, eval_xpath

159

10+# about

11+about = {

12+ "website": 'https://www.dokuwiki.org/',

13+ "wikidata_id": 'Q851864',

14+ "official_api_documentation": 'https://www.dokuwiki.org/devel:xmlrpc',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1620# engine dependent config

1721categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

1822paging = False

searx/engines/duckduckgo.py+11−11

@@ -1,22 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 DuckDuckGo (Web)

3−

4− @website https://duckduckgo.com/

5− @provide-api yes (https://duckduckgo.com/api),

6− but not all results from search-site

7−

8− @using-api no

9− @results HTML (using search portal)

10− @stable no (HTML can change)

11− @parse url, title, content

12−

13− @todo rewrite to api

144"""

155

166from lxml.html import fromstring

177from json import loads

188from searx.utils import extract_text, match_language, eval_xpath

199

10+# about

11+about = {

12+ "website": 'https://duckduckgo.com/',

13+ "wikidata_id": 'Q12805',

14+ "official_api_documentation": 'https://duckduckgo.com/api',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

2020# engine dependent config

2121categories = ['general']

2222paging = False

searx/engines/duckduckgo_definitions.py+12−8

@@ -1,12 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2−DuckDuckGo (definitions)

3−

4−- `Instant Answer API`_

5−- `DuckDuckGo query`_

6−

7−.. _Instant Answer API: https://duckduckgo.com/api

8−.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1

9−

3+ DuckDuckGo (Instant Answer API)

104"""

115

126import json

@@ -22,6 +16,16 @@ from searx.external_urls import get_external_url, get_earth_coordinates_url, are

2216

2317logger = logger.getChild('duckduckgo_definitions')

2418

19+# about

20+about = {

21+ "website": 'https://duckduckgo.com/',

22+ "wikidata_id": 'Q12805',

23+ "official_api_documentation": 'https://duckduckgo.com/api',

24+ "use_official_api": True,

25+ "require_api_key": False,

26+ "results": 'JSON',

27+}

28+

2529URL = 'https://api.duckduckgo.com/'\

2630 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

2731

searx/engines/duckduckgo_images.py+14−11

@@ -1,16 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 DuckDuckGo (Images)

3−

4− @website https://duckduckgo.com/

5− @provide-api yes (https://duckduckgo.com/api),

6− but images are not supported

7−

8− @using-api no

9− @results JSON (site requires js to get images)

10− @stable no (JSON can change)

11− @parse url, title, img_src

12−

13− @todo avoid extra request

144"""

155

166from json import loads

@@ -20,6 +10,19 @@ from searx.engines.duckduckgo import get_region_code

2010from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

2111from searx.poolrequests import get

2212

13+# about

14+about = {

15+ "website": 'https://duckduckgo.com/',

16+ "wikidata_id": 'Q12805',

17+ "official_api_documentation": {

18+ 'url': 'https://duckduckgo.com/api',

19+ 'comment': 'but images are not supported',

20+ },

21+ "use_official_api": False,

22+ "require_api_key": False,

23+ "results": 'JSON (site requires js to get images)',

24+}

25+

2326# engine dependent config

2427categories = ['images']

2528paging = True

searx/engines/duden.py+11−6

@@ -1,11 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Duden

3− @website https://www.duden.de

4− @provide-api no

5− @using-api no

6− @results HTML (using search portal)

7− @stable no (HTML can change)

8− @parse url, title, content

94"""

105

116import re

@@ -13,6 +8,16 @@ from urllib.parse import quote, urljoin

138from lxml import html

149from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

1510

11+# about

12+about = {

13+ "website": 'https://www.duden.de',

14+ "wikidata_id": 'Q73624591',

15+ "official_api_documentation": None,

16+ "use_official_api": False,

17+ "require_api_key": False,

18+ "results": 'HTML',

19+}

20+

1621categories = ['general']

1722paging = True

1823language_support = False

searx/engines/dummy-offline.py+11−3

@@ -1,11 +1,19 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Dummy Offline

3−

4− @results one result

5− @stable yes

64"""

75

86

7+# about

8+about = {

9+ "wikidata_id": None,

10+ "official_api_documentation": None,

11+ "use_official_api": False,

12+ "require_api_key": False,

13+ "results": 'HTML',

14+}

15+

16+

917def search(query, request_params):

1018 return [{

1119 'result': 'this is what you get',

searx/engines/dummy.py+11−3

@@ -1,10 +1,18 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Dummy

3−

4− @results empty array

5− @stable yes

64"""

75

6+# about

7+about = {

8+ "website": None,

9+ "wikidata_id": None,

10+ "official_api_documentation": None,

11+ "use_official_api": False,

12+ "require_api_key": False,

13+ "results": 'empty array',

14+}

15+

816

917# do search-request

1018def request(query, params):

searx/engines/ebay.py+14−9

@@ -1,17 +1,22 @@

1−# Ebay (Videos, Music, Files)

2−#

3−# @website https://www.ebay.com

4−# @provide-api no (nothing found)

5−#

6−# @using-api no

7−# @results HTML (using search portal)

8−# @stable yes (HTML can change)

9−# @parse url, title, content, price, shipping, source

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Ebay (Videos, Music, Files)

4+"""

105

116from lxml import html

127from searx.engines.xpath import extract_text

138from urllib.parse import quote

149

10+# about

11+about = {

12+ "website": 'https://www.ebay.com',

13+ "wikidata_id": 'Q58024',

14+ "official_api_documentation": 'https://developer.ebay.com/',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1520categories = ['shopping']

1621paging = True

1722

searx/engines/elasticsearch.py+5

@@ -1,3 +1,8 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Elasticsearch

4+"""

5+

16from json import loads, dumps

27from requests.auth import HTTPBasicAuth

38from searx.exceptions import SearxEngineAPIException

searx/engines/etools.py+11−7

@@ -1,18 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 eTools (Web)

3−

4− @website https://www.etools.ch

5− @provide-api no

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, content

104"""

115

126from lxml import html

137from urllib.parse import quote

148from searx.utils import extract_text, eval_xpath

159

10+# about

11+about = {

12+ "website": 'https://www.etools.ch',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1620categories = ['general']

1721paging = False

1822language_support = False

searx/engines/fdroid.py+11−7

@@ -1,18 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 F-Droid (a repository of FOSS applications for Android)

3−

4− @website https://f-droid.org/

5− @provide-api no

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, content

104"""

115

126from urllib.parse import urlencode

137from lxml import html

148from searx.utils import extract_text

159

10+# about

11+about = {

12+ "website": 'https://f-droid.org/',

13+ "wikidata_id": 'Q1386210',

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1620# engine dependent config

1721categories = ['files']

1822paging = True

searx/engines/flickr.py+11−9

@@ -1,21 +1,23 @@

1−#!/usr/bin/env python

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 Flickr (Images)

54

6− @website https://www.flickr.com

7− @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

8−

9− @using-api yes

10− @results JSON

11− @stable yes

12− @parse url, title, thumbnail, img_src

135 More info on api-key : https://www.flickr.com/services/apps/create/

146"""

157

168from json import loads

179from urllib.parse import urlencode

1810

11+# about

12+about = {

13+ "website": 'https://www.flickr.com',

14+ "wikidata_id": 'Q103204',

15+ "official_api_documentation": 'https://secure.flickr.com/services/api/flickr.photos.search.html',

16+ "use_official_api": True,

17+ "require_api_key": True,

18+ "results": 'JSON',

19+}

20+

1921categories = ['images']

2022

2123nb_per_page = 15

searx/engines/flickr_noapi.py+12−11

@@ -1,15 +1,6 @@

1−#!/usr/bin/env python

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

4− Flickr (Images)

5−

6− @website https://www.flickr.com

7− @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

8−

9− @using-api no

10− @results HTML

11− @stable no

12− @parse url, title, thumbnail, img_src

3+ Flickr (Images)

134"""

145

156from json import loads

@@ -21,6 +12,16 @@ from searx.utils import ecma_unescape, html_to_text

2112

2213logger = logger.getChild('flickr-noapi')

2314

15+# about

16+about = {

17+ "website": 'https://www.flickr.com',

18+ "wikidata_id": 'Q103204',

19+ "official_api_documentation": 'https://secure.flickr.com/services/api/flickr.photos.search.html',

20+ "use_official_api": False,

21+ "require_api_key": False,

22+ "results": 'HTML',

23+}

24+

2425categories = ['images']

2526

2627url = 'https://www.flickr.com/'

searx/engines/framalibre.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 FramaLibre (It)

3−

4− @website https://framalibre.org/

5− @provide-api no

6−

7− @using-api no

8− @results HTML

9− @stable no (HTML can change)

10− @parse url, title, content, thumbnail, img_src

114"""

125

136from html import escape

@@ -15,6 +8,16 @@ from urllib.parse import urljoin, urlencode

158from lxml import html

169from searx.utils import extract_text

1710

11+# about

12+about = {

13+ "website": 'https://framalibre.org/',

14+ "wikidata_id": 'Q30213882',

15+ "official_api_documentation": None,

16+ "use_official_api": False,

17+ "require_api_key": False,

18+ "results": 'HTML',

19+}

20+

1821# engine dependent config

1922categories = ['it']

2023paging = True

searx/engines/frinkiac.py+15−8

@@ -1,17 +1,24 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2−Frinkiac (Images)

3−

4−@website https://www.frinkiac.com

5−@provide-api no

6−@using-api no

7−@results JSON

8−@stable no

9−@parse url, title, img_src

3+ Frinkiac (Images)

104"""

115

126from json import loads

137from urllib.parse import urlencode

148

9+# about

10+about = {

11+ "website": 'https://frinkiac.com',

12+ "wikidata_id": 'Q24882614',

13+ "official_api_documentation": {

14+ 'url': None,

15+ 'comment': 'see https://github.com/MitchellAW/CompuGlobal'

16+ },

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'JSON',

20+}

21+

1522categories = ['images']

1623

1724BASE = 'https://frinkiac.com/'

searx/engines/genius.py+12−9

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2−Genius

3−

4− @website https://www.genius.com/

5− @provide-api yes (https://docs.genius.com/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, thumbnail, publishedDate

3+ Genius

114"""

125

136from json import loads

147from urllib.parse import urlencode

158from datetime import datetime

169

10+# about

11+about = {

12+ "website": 'https://genius.com/',

13+ "wikidata_id": 'Q3419343',

14+ "official_api_documentation": 'https://docs.genius.com/',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1720# engine dependent config

1821categories = ['music']

1922paging = True

searx/engines/gentoo.py+11−9

@@ -1,20 +1,22 @@

1−# -*- coding: utf-8 -*-

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 Gentoo Wiki

5−

6− @website https://wiki.gentoo.org

7− @provide-api yes

8− @using-api no

9− @results HTML

10− @stable no (HTML can change)

11− @parse url, title

124"""

135

146from urllib.parse import urlencode, urljoin

157from lxml import html

168from searx.utils import extract_text

179

10+# about

11+about = {

12+ "website": 'https://wiki.gentoo.org/',

13+ "wikidata_id": 'Q1050637',

14+ "official_api_documentation": 'https://wiki.gentoo.org/api.php',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1820# engine dependent config

1921categories = ['it']

2022language_support = True

searx/engines/gigablast.py+10−8

@@ -1,14 +1,6 @@

11# SPDX-License-Identifier: AGPL-3.0-or-later

22"""

33 Gigablast (Web)

4−

5− @website https://gigablast.com

6− @provide-api yes (https://gigablast.com/api.html)

7−

8− @using-api yes

9− @results XML

10− @stable yes

11− @parse url, title, content

124"""

135# pylint: disable=missing-function-docstring, invalid-name

146

@@ -18,6 +10,16 @@ from urllib.parse import urlencode

1810# from searx import logger

1911from searx.poolrequests import get

2012

13+# about

14+about = {

15+ "website": 'https://www.gigablast.com',

16+ "wikidata_id": 'Q3105449',

17+ "official_api_documentation": 'https://gigablast.com/api.html',

18+ "use_official_api": True,

19+ "require_api_key": False,

20+ "results": 'JSON',

21+}

22+

2123# engine dependent config

2224categories = ['general']

2325# gigablast's pagination is totally damaged, don't use it

searx/engines/github.py+12−9

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2− Github (It)

3−

4− @website https://github.com/

5− @provide-api yes (https://developer.github.com/v3/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes (using api)

10− @parse url, title, content

3+ Github (IT)

114"""

125

136from json import loads

147from urllib.parse import urlencode

158

9+# about

10+about = {

11+ "website": 'https://github.com/',

12+ "wikidata_id": 'Q364',

13+ "official_api_documentation": 'https://developer.github.com/v3/',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

18+

1619# engine dependent config

1720categories = ['it']

1821

searx/engines/google.py+14−12

@@ -1,19 +1,11 @@

11# SPDX-License-Identifier: AGPL-3.0-or-later

22"""Google (Web)

33

4−:website: https://www.google.com

5−:provide-api: yes (https://developers.google.com/custom-search/)

6−:using-api: not the offical, since it needs registration to another service

7−:results: HTML

8−:stable: no

9−:parse: url, title, content, number_of_results, answer, suggestion, correction

10−

11−For detailed description of the *REST-full* API see: `Query Parameter

12−Definitions`_.

13−

14−.. _Query Parameter Definitions:

15− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

4+ For detailed description of the *REST-full* API see: `Query Parameter

5+ Definitions`_.

166

7+ .. _Query Parameter Definitions:

8+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

179"""

1810

1911# pylint: disable=invalid-name, missing-function-docstring

@@ -27,6 +19,16 @@ from searx.exceptions import SearxEngineCaptchaException

2719

2820logger = logger.getChild('google engine')

2921

22+# about

23+about = {

24+ "website": 'https://www.google.com',

25+ "wikidata_id": 'Q9366',

26+ "official_api_documentation": 'https://developers.google.com/custom-search/',

27+ "use_official_api": False,

28+ "require_api_key": False,

29+ "results": 'HTML',

30+}

31+

3032# engine dependent config

3133categories = ['general']

3234paging = True

searx/engines/google_images.py+10−12

@@ -1,14 +1,6 @@

11# SPDX-License-Identifier: AGPL-3.0-or-later

22"""Google (Images)

33

4−:website: https://images.google.com (redirected to subdomain www.)

5−:provide-api: yes (https://developers.google.com/custom-search/)

6−:using-api: not the offical, since it needs registration to another service

7−:results: HTML

8−:stable: no

9−:template: images.html

10−:parse: url, title, content, source, thumbnail_src, img_src

11−

124For detailed description of the *REST-full* API see: `Query Parameter

135Definitions`_.

146

@@ -18,10 +10,6 @@ Definitions`_.

1810 ``data:` scheme).::

1911

2012 Header set Content-Security-Policy "img-src 'self' data: ;"

21−

22−.. _Query Parameter Definitions:

23− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions

24−

2513"""

2614

2715from urllib.parse import urlencode, urlparse, unquote

@@ -39,6 +27,16 @@ from searx.engines.google import (

3927

4028logger = logger.getChild('google images')

4129

30+# about

31+about = {

32+ "website": 'https://images.google.com/',

33+ "wikidata_id": 'Q521550',

34+ "official_api_documentation": 'https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions', # NOQA

35+ "use_official_api": False,

36+ "require_api_key": False,

37+ "results": 'HTML',

38+}

39+

4240# engine dependent config

4341

4442categories = ['images']

searx/engines/google_news.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Google (News)

3−

4− @website https://news.google.com

5− @provide-api no

6−

7− @using-api no

8− @results HTML

9− @stable no

10− @parse url, title, content, publishedDate

114"""

125

136from urllib.parse import urlencode

@@ -15,6 +8,16 @@ from lxml import html

158from searx.utils import match_language

169from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import

1710

11+# about

12+about = {

13+ "website": 'https://news.google.com',

14+ "wikidata_id": 'Q12020',

15+ "official_api_documentation": None,

16+ "use_official_api": False,

17+ "require_api_key": False,

18+ "results": 'HTML',

19+}

20+

1821# search-url

1922categories = ['news']

2023paging = True

searx/engines/google_videos.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Google (Videos)

3−

4− @website https://www.google.com

5− @provide-api yes (https://developers.google.com/custom-search/)

6−

7− @using-api no

8− @results HTML

9− @stable no

10− @parse url, title, content, thumbnail

114"""

125

136from datetime import date, timedelta

@@ -16,6 +9,16 @@ from lxml import html

169from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex

1710import re

1811

12+# about

13+about = {

14+ "website": 'https://www.google.com',

15+ "wikidata_id": 'Q219885',

16+ "official_api_documentation": 'https://developers.google.com/custom-search/',

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'HTML',

20+}

21+

1922# engine dependent config

2023categories = ['videos']

2124paging = True

searx/engines/ina.py+13−12

@@ -1,15 +1,7 @@

1−# INA (Videos)

2−#

3−# @website https://www.ina.fr/

4−# @provide-api no

5−#

6−# @using-api no

7−# @results HTML (using search portal)

8−# @stable no (HTML can change)

9−# @parse url, title, content, publishedDate, thumbnail

10−#

11−# @todo set content-parameter with correct data

12−# @todo embedded (needs some md5 from video page)

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ INA (Videos)

4+"""

135

146from json import loads

157from html import unescape

@@ -18,6 +10,15 @@ from lxml import html

1810from dateutil import parser

1911from searx.utils import extract_text

2012

13+# about

14+about = {

15+ "website": 'https://www.ina.fr/',

16+ "wikidata_id": 'Q1665109',

17+ "official_api_documentation": None,

18+ "use_official_api": False,

19+ "require_api_key": False,

20+ "results": 'HTML',

21+}

2122

2223# engine dependent config

2324categories = ['videos']

searx/engines/invidious.py+14−9

@@ -1,17 +1,22 @@

1−# Invidious (Videos)

2−#

3−# @website https://invidio.us/

4−# @provide-api yes (https://github.com/omarroth/invidious/wiki/API)

5−#

6−# @using-api yes

7−# @results JSON

8−# @stable yes

9−# @parse url, title, content, publishedDate, thumbnail, embedded, author, length

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Invidious (Videos)

4+"""

105

116from urllib.parse import quote_plus

127from dateutil import parser

138import time

149

10+# about

11+about = {

12+ "website": 'https://instances.invidio.us/',

13+ "wikidata_id": 'Q79343316',

14+ "official_api_documentation": 'https://github.com/omarroth/invidious/wiki/API',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1520# engine dependent config

1621categories = ["videos", "music"]

1722paging = True

searx/engines/json_engine.py+2

@@ -1,3 +1,5 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+

13from collections.abc import Iterable

24from json import loads

35from urllib.parse import urlencode

searx/engines/kickass.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Kickass Torrent (Videos, Music, Files)

3−

4− @website https://kickass.so

5− @provide-api no (nothing found)

6−

7− @using-api no

8− @results HTML (using search portal)

9− @stable yes (HTML can change)

10− @parse url, title, content, seed, leech, magnetlink

114"""

125

136from lxml import html

@@ -15,6 +8,16 @@ from operator import itemgetter

158from urllib.parse import quote, urljoin

169from searx.utils import extract_text, get_torrent_size, convert_str_to_int

1710

11+# about

12+about = {

13+ "website": 'https://kickass.so',

14+ "wikidata_id": 'Q17062285',

15+ "official_api_documentation": None,

16+ "use_official_api": False,

17+ "require_api_key": False,

18+ "results": 'HTML',

19+}

20+

1821# engine dependent config

1922categories = ['videos', 'music', 'files']

2023paging = True

searx/engines/mediawiki.py+12−11

@@ -1,21 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2− general mediawiki-engine (Web)

3−

4− @website websites built on mediawiki (https://www.mediawiki.org)

5− @provide-api yes (http://www.mediawiki.org/wiki/API:Search)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title

11−

12− @todo content

3+ General mediawiki-engine (Web)

134"""

145

156from json import loads

167from string import Formatter

178from urllib.parse import urlencode, quote

189

10+# about

11+about = {

12+ "website": None,

13+ "wikidata_id": None,

14+ "official_api_documentation": 'http://www.mediawiki.org/wiki/API:Search',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1920# engine dependent config

2021categories = ['general']

2122language_support = True

searx/engines/microsoft_academic.py+12−8

@@ -1,12 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2−Microsoft Academic (Science)

3−

4−@website https://academic.microsoft.com

5−@provide-api yes

6−@using-api no

7−@results JSON

8−@stable no

9−@parse url, title, content

3+ Microsoft Academic (Science)

104"""

115

126from datetime import datetime

@@ -15,6 +9,16 @@ from uuid import uuid4

159from urllib.parse import urlencode

1610from searx.utils import html_to_text

1711

12+# about

13+about = {

14+ "website": 'https://academic.microsoft.com',

15+ "wikidata_id": 'Q28136779',

16+ "official_api_documentation": 'http://ma-graph.org/',

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'JSON',

20+}

21+

1822categories = ['images']

1923paging = True

2024result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}'

searx/engines/mixcloud.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Mixcloud (Music)

3−

4− @website https://http://www.mixcloud.com/

5− @provide-api yes (http://www.mixcloud.com/developers/

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, embedded, publishedDate

114"""

125

136from json import loads

147from dateutil import parser

158from urllib.parse import urlencode

169

10+# about

11+about = {

12+ "website": 'https://www.mixcloud.com/',

13+ "wikidata_id": 'Q6883832',

14+ "official_api_documentation": 'http://www.mixcloud.com/developers/',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1720# engine dependent config

1821categories = ['music']

1922paging = True

searx/engines/not_evil.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 not Evil (Onions)

3−

4− @website http://hss3uro2hsxfogfq.onion

5− @provide-api yes (http://hss3uro2hsxfogfq.onion/api.htm)

6−

7− @using-api no

8− @results HTML

9− @stable no

10− @parse url, title, content

114"""

125

136from urllib.parse import urlencode

147from lxml import html

158from searx.engines.xpath import extract_text

169

10+# about

11+about = {

12+ "website": 'http://hss3uro2hsxfogfq.onion',

13+ "wikidata_id": None,

14+ "official_api_documentation": 'http://hss3uro2hsxfogfq.onion/api.htm',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1720# engine dependent config

1821categories = ['onions']

1922paging = True

searx/engines/nyaa.py+11−7

@@ -1,18 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Nyaa.si (Anime Bittorrent tracker)

3−

4− @website https://nyaa.si/

5− @provide-api no

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, content, seed, leech, torrentfile

104"""

115

126from lxml import html

137from urllib.parse import urlencode

148from searx.utils import extract_text, get_torrent_size, int_or_zero

159

10+# about

11+about = {

12+ "website": 'https://nyaa.si/',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1620# engine dependent config

1721categories = ['files', 'images', 'videos', 'music']

1822paging = True

searx/engines/opensemantic.py+13−9

@@ -1,18 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2−Open Semantic Search

3−

4− @website https://www.opensemanticsearch.org/

5− @provide-api yes (https://www.opensemanticsearch.org/dev)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, publishedDate

3+ Open Semantic Search

114"""

5+

126from dateutil import parser

137from json import loads

148from urllib.parse import quote

159

10+# about

11+about = {

12+ "website": 'https://www.opensemanticsearch.org/',

13+ "wikidata_id": None,

14+ "official_api_documentation": 'https://www.opensemanticsearch.org/dev',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1620base_url = 'http://localhost:8983/solr/opensemanticsearch/'

1721search_string = 'query?q={query}'

1822

searx/engines/openstreetmap.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 OpenStreetMap (Map)

3−

4− @website https://openstreetmap.org/

5− @provide-api yes (http://wiki.openstreetmap.org/wiki/Nominatim)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title

114"""

125

136import re

147from json import loads

158from flask_babel import gettext

169

10+# about

11+about = {

12+ "website": 'https://www.openstreetmap.org/',

13+ "wikidata_id": 'Q936',

14+ "official_api_documentation": 'http://wiki.openstreetmap.org/wiki/Nominatim',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1720# engine dependent config

1821categories = ['map']

1922paging = False

searx/engines/pdbe.py+11−8

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 PDBe (Protein Data Bank in Europe)

3−

4− @website https://www.ebi.ac.uk/pdbe

5− @provide-api yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),

6− unlimited

7− @using-api yes

8− @results python dictionary (from json)

9− @stable yes

10− @parse url, title, content, img_src

114"""

125

136from json import loads

147from flask_babel import gettext

158

9+# about

10+about = {

11+ "website": 'https://www.ebi.ac.uk/pdbe',

12+ "wikidata_id": 'Q55823905',

13+ "official_api_documentation": 'https://www.ebi.ac.uk/pdbe/api/doc/search.html',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

18+

1619categories = ['science']

1720

1821hide_obsolete = False

searx/engines/peertube.py+11−10

@@ -1,15 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 peertube (Videos)

3−

4− @website https://www.peertube.live

5− @provide-api yes (https://docs.joinpeertube.org/api-rest-reference.html)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, thumbnail, publishedDate, embedded

11−

12− @todo implement time range support

134"""

145

156from json import loads

@@ -17,6 +8,16 @@ from datetime import datetime

178from urllib.parse import urlencode

189from searx.utils import html_to_text

1910

11+# about

12+about = {

13+ "website": 'https://joinpeertube.org',

14+ "wikidata_id": 'Q50938515',

15+ "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html',

16+ "use_official_api": True,

17+ "require_api_key": False,

18+ "results": 'JSON',

19+}

20+

2021# engine dependent config

2122categories = ["videos"]

2223paging = True

searx/engines/photon.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Photon (Map)

3−

4− @website https://photon.komoot.de

5− @provide-api yes (https://photon.komoot.de/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title

114"""

125

136from json import loads

147from urllib.parse import urlencode

158from searx.utils import searx_useragent

169

10+# about

11+about = {

12+ "website": 'https://photon.komoot.de',

13+ "wikidata_id": None,

14+ "official_api_documentation": 'https://photon.komoot.de/',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1720# engine dependent config

1821categories = ['map']

1922paging = False

searx/engines/piratebay.py+14−9

@@ -1,12 +1,7 @@

1−# Piratebay (Videos, Music, Files)

2−#

3−# @website https://thepiratebay.org

4−# @provide-api yes (https://apibay.org/)

5−#

6−# @using-api yes

7−# @results JSON

8−# @stable no (the API is not documented nor versioned)

9−# @parse url, title, seed, leech, magnetlink, filesize, publishedDate

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Piratebay (Videos, Music, Files)

4+"""

105

116from json import loads

127from datetime import datetime

@@ -15,6 +10,16 @@ from operator import itemgetter

1510from urllib.parse import quote

1611from searx.utils import get_torrent_size

1712

13+# about

14+about = {

15+ "website": 'https://thepiratebay.org',

16+ "wikidata_id": 'Q22663',

17+ "official_api_documentation": 'https://apibay.org/',

18+ "use_official_api": True,

19+ "require_api_key": False,

20+ "results": 'JSON',

21+}

22+

1823# engine dependent config

1924categories = ["videos", "music", "files"]

2025

searx/engines/pubmed.py+13−9

@@ -1,14 +1,6 @@

1−#!/usr/bin/env python

2−

1+# SPDX-License-Identifier: AGPL-3.0-or-later

32"""

43 PubMed (Scholar publications)

5− @website https://www.ncbi.nlm.nih.gov/pubmed/

6− @provide-api yes (https://www.ncbi.nlm.nih.gov/home/develop/api/)

7− @using-api yes

8− @results XML

9− @stable yes

10− @parse url, title, publishedDate, content

11− More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/

124"""

135

146from flask_babel import gettext

@@ -17,6 +9,18 @@ from datetime import datetime

179from urllib.parse import urlencode

1810from searx.poolrequests import get

1911

12+# about

13+about = {

14+ "website": 'https://www.ncbi.nlm.nih.gov/pubmed/',

15+ "wikidata_id": 'Q1540899',

16+ "official_api_documentation": {

17+ 'url': 'https://www.ncbi.nlm.nih.gov/home/develop/api/',

18+ 'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/'

19+ },

20+ "use_official_api": True,

21+ "require_api_key": False,

22+ "results": 'XML',

23+}

2024

2125categories = ['science']

2226

searx/engines/qwant.py+10−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Qwant (Web, Images, News, Social)

3−

4− @website https://qwant.com/

5− @provide-api not officially (https://api.qwant.com/api/search/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content

114"""

125

136from datetime import datetime

@@ -17,6 +10,15 @@ from searx.utils import html_to_text, match_language

1710from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException

1811from searx.raise_for_httperror import raise_for_httperror

1912

13+# about

14+about = {

15+ "website": 'https://www.qwant.com/',

16+ "wikidata_id": 'Q14657870',

17+ "official_api_documentation": None,

18+ "use_official_api": True,

19+ "require_api_key": False,

20+ "results": 'JSON',

21+}

2022

2123# engine dependent config

2224categories = []

searx/engines/recoll.py+11−6

@@ -1,17 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Recoll (local search engine)

3−

4− @using-api yes

5− @results JSON

6− @stable yes

7− @parse url, content, size, abstract, author, mtype, subtype, time, \

8− filename, label, type, embedded

94"""

105

116from datetime import date, timedelta

127from json import loads

138from urllib.parse import urlencode, quote

149

10+# about

11+about = {

12+ "website": None,

13+ "wikidata_id": 'Q15735774',

14+ "official_api_documentation": 'https://www.lesbonscomptes.com/recoll/',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1520# engine dependent config

1621time_range_support = True

1722

searx/engines/reddit.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Reddit

3−

4− @website https://www.reddit.com/

5− @provide-api yes (https://www.reddit.com/dev/api)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, thumbnail, publishedDate

114"""

125

136import json

147from datetime import datetime

158from urllib.parse import urlencode, urljoin, urlparse

169

10+# about

11+about = {

12+ "website": 'https://www.reddit.com/',

13+ "wikidata_id": 'Q1136',

14+ "official_api_documentation": 'https://www.reddit.com/dev/api',

15+ "use_official_api": True,

16+ "require_api_key": False,

17+ "results": 'JSON',

18+}

19+

1720# engine dependent config

1821categories = ['general', 'images', 'news', 'social media']

1922page_size = 25

searx/engines/scanr_structures.py+11−8

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 ScanR Structures (Science)

3−

4− @website https://scanr.enseignementsup-recherche.gouv.fr

5− @provide-api yes (https://scanr.enseignementsup-recherche.gouv.fr/api/swagger-ui.html)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, img_src

114"""

125

136from json import loads, dumps

147from searx.utils import html_to_text

158

9+# about

10+about = {

11+ "website": 'https://scanr.enseignementsup-recherche.gouv.fr',

12+ "wikidata_id": 'Q44105684',

13+ "official_api_documentation": 'https://scanr.enseignementsup-recherche.gouv.fr/opendata',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

18+

1619# engine dependent config

1720categories = ['science']

1821paging = True

searx/engines/searchcode_code.py+11−9

@@ -1,18 +1,20 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2− Searchcode (It)

3−

4− @website https://searchcode.com/

5− @provide-api yes (https://searchcode.com/api/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content

3+ Searchcode (IT)

114"""

125

136from json import loads

147from urllib.parse import urlencode

158

9+# about

10+about = {

11+ "website": 'https://searchcode.com/',

12+ "wikidata_id": None,

13+ "official_api_documentation": 'https://searchcode.com/api/',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

1618

1719# engine dependent config

1820categories = ['it']

searx/engines/searx_engine.py+10−8

@@ -1,18 +1,20 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Searx (all)

3−

4− @website https://github.com/searx/searx

5− @provide-api yes (https://searx.github.io/searx/dev/search_api.html)

6−

7− @using-api yes

8− @results JSON

9− @stable yes (using api)

10− @parse url, title, content

114"""

125

136from json import loads

147from searx.engines import categories as searx_categories

158

9+# about

10+about = {

11+ "website": 'https://github.com/searx/searx',

12+ "wikidata_id": 'Q17639196',

13+ "official_api_documentation": 'https://searx.github.io/searx/dev/search_api.html',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

1618

1719categories = searx_categories.keys()

1820

searx/engines/sepiasearch.py+14−8

@@ -1,17 +1,23 @@

1−# SepiaSearch (Videos)

2−#

3−# @website https://sepiasearch.org

4−# @provide-api https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api

5−# @using-api yes

6−# @results JSON

7−# @stable yes

8−# @parse url, title, content, publishedDate, thumbnail

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ SepiaSearch (Videos)

4+"""

95

106from json import loads

117from dateutil import parser, relativedelta

128from urllib.parse import urlencode

139from datetime import datetime

1410

11+# about

12+about = {

13+ "website": 'https://sepiasearch.org',

14+ "wikidata_id": None,

15+ "official_api_documentation": "https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api", # NOQA

16+ "use_official_api": True,

17+ "require_api_key": False,

18+ "results": 'JSON',

19+}

20+

1521categories = ['videos']

1622paging = True

1723language_support = True

searx/engines/soundcloud.py+10−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Soundcloud (Music)

3−

4− @website https://soundcloud.com

5− @provide-api yes (https://developers.soundcloud.com/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, publishedDate, embedded

114"""

125

136import re

@@ -18,6 +11,15 @@ from urllib.parse import quote_plus, urlencode

1811from searx import logger

1912from searx.poolrequests import get as http_get

2013

14+# about

15+about = {

16+ "website": 'https://soundcloud.com',

17+ "wikidata_id": 'Q568769',

18+ "official_api_documentation": 'https://developers.soundcloud.com/',

19+ "use_official_api": True,

20+ "require_api_key": False,

21+ "results": 'JSON',

22+}

2123

2224# engine dependent config

2325categories = ['music']

searx/engines/spotify.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Spotify (Music)

3−

4− @website https://spotify.com

5− @provide-api yes (https://developer.spotify.com/web-api/search-item/)

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, title, content, embedded

114"""

125

136from json import loads

@@ -15,6 +8,16 @@ from urllib.parse import urlencode

158import requests

169import base64

1710

11+# about

12+about = {

13+ "website": 'https://www.spotify.com',

14+ "wikidata_id": 'Q689141',

15+ "official_api_documentation": 'https://developer.spotify.com/web-api/search-item/',

16+ "use_official_api": True,

17+ "require_api_key": False,

18+ "results": 'JSON',

19+}

20+

1821# engine dependent config

1922categories = ['music']

2023paging = True

searx/engines/stackoverflow.py+12−9

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

2− Stackoverflow (It)

3−

4− @website https://stackoverflow.com/

5− @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)

6−

7− @using-api no

8− @results HTML

9− @stable no (HTML can change)

10− @parse url, title, content

3+ Stackoverflow (IT)

114"""

125

136from urllib.parse import urlencode, urljoin, urlparse

@@ -15,6 +8,16 @@ from lxml import html

158from searx.utils import extract_text

169from searx.exceptions import SearxEngineCaptchaException

1710

11+# about

12+about = {

13+ "website": 'https://stackoverflow.com/',

14+ "wikidata_id": 'Q549037',

15+ "official_api_documentation": 'https://api.stackexchange.com/docs',

16+ "use_official_api": False,

17+ "require_api_key": False,

18+ "results": 'HTML',

19+}

20+

1821# engine dependent config

1922categories = ['it']

2023paging = True

searx/engines/startpage.py+14−11

@@ -1,14 +1,7 @@

1−# Startpage (Web)

2−#

3−# @website https://startpage.com

4−# @provide-api no (nothing found)

5−#

6−# @using-api no

7−# @results HTML

8−# @stable no (HTML can change)

9−# @parse url, title, content

10−#

11−# @todo paging

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Startpage (Web)

4+"""

125

136from lxml import html

147from dateutil import parser

@@ -19,6 +12,16 @@ from babel import Locale

1912from babel.localedata import locale_identifiers

2013from searx.utils import extract_text, eval_xpath, match_language

2114

15+# about

16+about = {

17+ "website": 'https://startpage.com',

18+ "wikidata_id": 'Q2333295',

19+ "official_api_documentation": None,

20+ "use_official_api": False,

21+ "require_api_key": False,

22+ "results": 'HTML',

23+}

24+

2225# engine dependent config

2326categories = ['general']

2427# there is a mechanism to block "bot" search

searx/engines/tokyotoshokan.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Tokyo Toshokan (A BitTorrent Library for Japanese Media)

3−

4− @website https://www.tokyotosho.info/

5− @provide-api no

6− @using-api no

7− @results HTML

8− @stable no (HTML can change)

9− @parse url, title, publishedDate, seed, leech,

10− filesize, magnetlink, content

114"""

125

136import re

@@ -16,6 +9,16 @@ from lxml import html

169from datetime import datetime

1710from searx.utils import extract_text, get_torrent_size, int_or_zero

1811

12+# about

13+about = {

14+ "website": 'https://www.tokyotosho.info/',

15+ "wikidata_id": None,

16+ "official_api_documentation": None,

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'HTML',

20+}

21+

1922# engine dependent config

2023categories = ['files', 'videos', 'music']

2124paging = True

searx/engines/torrentz.py+11−9

@@ -1,14 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Torrentz2.is (BitTorrent meta-search engine)

3−

4− @website https://torrentz2.is/

5− @provide-api no

6−

7− @using-api no

8− @results HTML

9− @stable no (HTML can change, although unlikely,

10− see https://torrentz.is/torrentz.btsearch)

11− @parse url, title, publishedDate, seed, leech, filesize, magnetlink

124"""

135

146import re

@@ -17,6 +9,16 @@ from lxml import html

179from datetime import datetime

1810from searx.utils import extract_text, get_torrent_size

1911

12+# about

13+about = {

14+ "website": 'https://torrentz2.is/',

15+ "wikidata_id": 'Q1156687',

16+ "official_api_documentation": 'https://torrentz.is/torrentz.btsearch',

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'HTML',

20+}

21+

2022# engine dependent config

2123categories = ['files', 'videos', 'music']

2224paging = True

searx/engines/translated.py+11−7

@@ -1,14 +1,18 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 MyMemory Translated

3−

4− @website https://mymemory.translated.net/

5− @provide-api yes (https://mymemory.translated.net/doc/spec.php)

6− @using-api yes

7− @results JSON

8− @stable yes

9− @parse url, title, content

104"""

115

6+# about

7+about = {

8+ "website": 'https://mymemory.translated.net/',

9+ "wikidata_id": None,

10+ "official_api_documentation": 'https://mymemory.translated.net/doc/spec.php',

11+ "use_official_api": True,

12+ "require_api_key": False,

13+ "results": 'JSON',

14+}

15+

1216engine_type = 'online_dictionnary'

1317categories = ['general']

1418url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'

searx/engines/unsplash.py+11−8

@@ -1,18 +1,21 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Unsplash

3−

4− @website https://unsplash.com

5− @provide-api yes (https://unsplash.com/developers)

6−

7− @using-api no

8− @results JSON (using search portal's infiniscroll API)

9− @stable no (JSON format could change any time)

10− @parse url, title, img_src, thumbnail_src

114"""

125

136from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl

147from json import loads

158

9+# about

10+about = {

11+ "website": 'https://unsplash.com',

12+ "wikidata_id": 'Q28233552',

13+ "official_api_documentation": 'https://unsplash.com/developers',

14+ "use_official_api": False,

15+ "require_api_key": False,

16+ "results": 'JSON',

17+}

18+

1619url = 'https://unsplash.com/'

1720search_url = url + 'napi/search/photos?'

1821categories = ['images']

searx/engines/vimeo.py+14−13

@@ -1,21 +1,22 @@

1−# Vimeo (Videos)

2−#

3−# @website https://vimeo.com/

4−# @provide-api yes (http://developer.vimeo.com/api),

5−# they have a maximum count of queries/hour

6−#

7−# @using-api no (TODO, rewrite to api)

8−# @results HTML (using search portal)

9−# @stable no (HTML can change)

10−# @parse url, title, publishedDate, thumbnail, embedded

11−#

12−# @todo rewrite to api

13−# @todo set content-parameter with correct data

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Wikipedia (Web

4+"""

145

156from urllib.parse import urlencode

167from json import loads

178from dateutil import parser

189

10+# about

11+about = {

12+ "website": 'https://vimeo.com/',

13+ "wikidata_id": 'Q156376',

14+ "official_api_documentation": 'http://developer.vimeo.com/api',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1920# engine dependent config

2021categories = ['videos']

2122paging = True

searx/engines/wikidata.py+11−9

@@ -1,14 +1,6 @@

1−# -*- coding: utf-8 -*-

1+# SPDX-License-Identifier: AGPL-3.0-or-later

22"""

33 Wikidata

4−

5− @website https://wikidata.org

6− @provide-api yes (https://query.wikidata.org/)

7−

8− @using-api yes

9− @results JSON

10− @stable yes

11− @parse url, infobox

124"""

135

146

@@ -27,6 +19,16 @@ from searx.engines.wikipedia import _fetch_supported_languages, supported_langua

2719

2820logger = logger.getChild('wikidata')

2921

22+# about

23+about = {

24+ "website": 'https://wikidata.org/',

25+ "wikidata_id": 'Q2013',

26+ "official_api_documentation": 'https://query.wikidata.org/',

27+ "use_official_api": True,

28+ "require_api_key": False,

29+ "results": 'JSON',

30+}

31+

3032# SPARQL

3133SPARQL_ENDPOINT_URL = 'https://query.wikidata.org/sparql'

3234SPARQL_EXPLAIN_URL = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain'

searx/engines/wikipedia.py+11−8

@@ -1,13 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Wikipedia (Web)

3−

4− @website https://en.wikipedia.org/api/rest_v1/

5− @provide-api yes

6−

7− @using-api yes

8− @results JSON

9− @stable yes

10− @parse url, infobox

114"""

125

136from urllib.parse import quote

@@ -16,6 +9,16 @@ from lxml.html import fromstring

169from searx.utils import match_language, searx_useragent

1710from searx.raise_for_httperror import raise_for_httperror

1811

12+# about

13+about = {

14+ "website": 'https://www.wikipedia.org/',

15+ "wikidata_id": 'Q52',

16+ "official_api_documentation": 'https://en.wikipedia.org/api/',

17+ "use_official_api": True,

18+ "require_api_key": False,

19+ "results": 'JSON',

20+}

21+

1922# search-url

2023search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

2124supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'

searx/engines/wolframalpha_api.py+14−9

@@ -1,16 +1,21 @@

1−# Wolfram Alpha (Science)

2−#

3−# @website https://www.wolframalpha.com

4−# @provide-api yes (https://api.wolframalpha.com/v2/)

5−#

6−# @using-api yes

7−# @results XML

8−# @stable yes

9−# @parse url, infobox

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Wolfram|Alpha (Science)

4+"""

105

116from lxml import etree

127from urllib.parse import urlencode

138

9+# about

10+about = {

11+ "website": 'https://www.wolframalpha.com',

12+ "wikidata_id": 'Q207006',

13+ "official_api_documentation": 'https://products.wolframalpha.com/api/',

14+ "use_official_api": True,

15+ "require_api_key": False,

16+ "results": 'XML',

17+}

18+

1419# search-url

1520search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'

1621site_url = 'https://www.wolframalpha.com/input/?{query}'

searx/engines/wolframalpha_noapi.py+14−9

@@ -1,12 +1,7 @@

1−# Wolfram|Alpha (Science)

2−#

3−# @website https://www.wolframalpha.com/

4−# @provide-api yes (https://api.wolframalpha.com/v2/)

5−#

6−# @using-api no

7−# @results JSON

8−# @stable no

9−# @parse url, infobox

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Wolfram|Alpha (Science)

4+"""

105

116from json import loads

127from time import time

@@ -14,6 +9,16 @@ from urllib.parse import urlencode

149

1510from searx.poolrequests import get as http_get

1611

12+# about

13+about = {

14+ "website": 'https://www.wolframalpha.com/',

15+ "wikidata_id": 'Q207006',

16+ "official_api_documentation": 'https://products.wolframalpha.com/api/',

17+ "use_official_api": False,

18+ "require_api_key": False,

19+ "results": 'JSON',

20+}

21+

1722# search-url

1823url = 'https://www.wolframalpha.com/'

1924

searx/engines/www1x.py+11−8

@@ -1,19 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 1x (Images)

3−

4− @website http://1x.com/

5− @provide-api no

6−

7− @using-api no

8− @results HTML

9− @stable no (HTML can change)

10− @parse url, title, thumbnail

114"""

125

136from lxml import html, etree

147from urllib.parse import urlencode, urljoin

158from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

169

10+# about

11+about = {

12+ "website": 'https://1x.com/',

13+ "wikidata_id": None,

14+ "official_api_documentation": None,

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1720# engine dependent config

1821categories = ['images']

1922paging = False

searx/engines/xpath.py+2

@@ -1,3 +1,5 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+

13from lxml import html

24from urllib.parse import urlencode

35from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list

searx/engines/yacy.py+14−13

@@ -1,16 +1,7 @@

1−# Yacy (Web, Images, Videos, Music, Files)

2−#

3−# @website http://yacy.net

4−# @provide-api yes

5−# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)

6−#

7−# @using-api yes

8−# @results JSON

9−# @stable yes

10−# @parse (general) url, title, content, publishedDate

11−# @parse (images) url, title, img_src

12−#

13−# @todo parse video, audio and file results

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Yacy (Web, Images, Videos, Music, Files)

4+"""

145

156from json import loads

167from dateutil import parser

@@ -20,6 +11,16 @@ from requests.auth import HTTPDigestAuth

2011

2112from searx.utils import html_to_text

2213

14+# about

15+about = {

16+ "website": 'https://yacy.net/',

17+ "wikidata_id": 'Q1759675',

18+ "official_api_documentation": 'https://wiki.yacy.net/index.php/Dev:API',

19+ "use_official_api": True,

20+ "require_api_key": False,

21+ "results": 'JSON',

22+}

23+

2324# engine dependent config

2425categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'

2526paging = True

searx/engines/yahoo.py+11−9

@@ -1,20 +1,22 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Yahoo (Web)

3−

4− @website https://search.yahoo.com/web

5− @provide-api yes (https://developer.yahoo.com/boss/search/),

6− $0.80/1000 queries

7−

8− @using-api no (because pricing)

9− @results HTML (using search portal)

10− @stable no (HTML can change)

11− @parse url, title, content, suggestion

124"""

135

146from urllib.parse import unquote, urlencode

157from lxml import html

168from searx.utils import extract_text, extract_url, match_language, eval_xpath

179

10+# about

11+about = {

12+ "website": 'https://search.yahoo.com/',

13+ "wikidata_id": None,

14+ "official_api_documentation": 'https://developer.yahoo.com/api/',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1820# engine dependent config

1921categories = ['general']

2022paging = True

searx/engines/yahoo_news.py+14−10

@@ -1,13 +1,7 @@

1−# Yahoo (News)

2−#

3−# @website https://news.yahoo.com

4−# @provide-api yes (https://developer.yahoo.com/boss/search/)

5−# $0.80/1000 queries

6−#

7−# @using-api no (because pricing)

8−# @results HTML (using search portal)

9−# @stable no (HTML can change)

10−# @parse url, title, content, publishedDate

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Yahoo (News)

4+"""

115

126import re

137from datetime import datetime, timedelta

@@ -18,6 +12,16 @@ from searx.engines.yahoo import _fetch_supported_languages, supported_languages_

1812from dateutil import parser

1913from searx.utils import extract_text, extract_url, match_language

2014

15+# about

16+about = {

17+ "website": 'https://news.yahoo.com',

18+ "wikidata_id": 'Q3044717',

19+ "official_api_documentation": 'https://developer.yahoo.com/api/',

20+ "use_official_api": False,

21+ "require_api_key": False,

22+ "results": 'HTML',

23+}

24+

2125# engine dependent config

2226categories = ['news']

2327paging = True

searx/engines/yandex.py+11−7

@@ -1,12 +1,6 @@

1+# SPDX-License-Identifier: AGPL-3.0-or-later

12"""

23 Yahoo (Web)

3−

4− @website https://yandex.ru/

5− @provide-api ?

6− @using-api no

7− @results HTML (using search portal)

8− @stable no (HTML can change)

9− @parse url, title, content

104"""

115

126from urllib.parse import urlencode, urlparse

@@ -16,6 +10,16 @@ from searx.exceptions import SearxEngineCaptchaException

1610

1711logger = logger.getChild('yandex engine')

1812

13+# about

14+about = {

15+ "website": 'https://yandex.ru/',

16+ "wikidata_id": 'Q5281',

17+ "official_api_documentation": "?",

18+ "use_official_api": False,

19+ "require_api_key": False,

20+ "results": 'HTML',

21+}

22+

1923# engine dependent config

2024categories = ['general']

2125paging = True

searx/engines/yggtorrent.py+14−9

@@ -1,12 +1,7 @@

1−# Yggtorrent (Videos, Music, Files)

2−#

3−# @website https://www2.yggtorrent.si

4−# @provide-api no (nothing found)

5−#

6−# @using-api no

7−# @results HTML (using search portal)

8−# @stable no (HTML can change)

9−# @parse url, title, seed, leech, publishedDate, filesize

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Yggtorrent (Videos, Music, Files)

4+"""

105

116from lxml import html

127from operator import itemgetter

@@ -15,6 +10,16 @@ from urllib.parse import quote

1510from searx.utils import extract_text, get_torrent_size

1611from searx.poolrequests import get as http_get

1712

13+# about

14+about = {

15+ "website": 'https://www2.yggtorrent.si',

16+ "wikidata_id": None,

17+ "official_api_documentation": None,

18+ "use_official_api": False,

19+ "require_api_key": False,

20+ "results": 'HTML',

21+}

22+

1823# engine dependent config

1924categories = ['videos', 'music', 'files']

2025paging = True

searx/engines/youtube_api.py+14−9

@@ -1,18 +1,23 @@

1−# Youtube (Videos)

2−#

3−# @website https://www.youtube.com/

4−# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list)

5−#

6−# @using-api yes

7−# @results JSON

8−# @stable yes

9−# @parse url, title, content, publishedDate, thumbnail, embedded

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Youtube (Videos)

4+"""

105

116from json import loads

127from dateutil import parser

138from urllib.parse import urlencode

149from searx.exceptions import SearxEngineAPIException

1510

11+# about

12+about = {

13+ "website": 'https://www.youtube.com/',

14+ "wikidata_id": 'Q866',

15+ "official_api_documentation": 'https://developers.google.com/youtube/v3/docs/search/list?apix=true',

16+ "use_official_api": True,

17+ "require_api_key": False,

18+ "results": 'JSON',

19+}

20+

1621# engine dependent config

1722categories = ['videos', 'music']

1823paging = False

searx/engines/youtube_noapi.py+14−9

@@ -1,17 +1,22 @@

1−# Youtube (Videos)

2−#

3−# @website https://www.youtube.com/

4−# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list)

5−#

6−# @using-api no

7−# @results HTML

8−# @stable no

9−# @parse url, title, content, publishedDate, thumbnail, embedded

1+# SPDX-License-Identifier: AGPL-3.0-or-later

2+"""

3+ Youtube (Videos)

4+"""

105

116from functools import reduce

127from json import loads

138from urllib.parse import quote_plus

149

10+# about

11+about = {

12+ "website": 'https://www.youtube.com/',

13+ "wikidata_id": 'Q866',

14+ "official_api_documentation": 'https://developers.google.com/youtube/v3/docs/search/list?apix=true',

15+ "use_official_api": False,

16+ "require_api_key": False,

17+ "results": 'HTML',

18+}

19+

1520# engine dependent config

1621categories = ['videos', 'music']

1722paging = True

searx/settings.yml+213

@@ -157,6 +157,13 @@ engines:

157157 timeout : 7.0

158158 disabled : True

159159 shortcut : ai

160+ about:

161+ website: https://archive.is/

162+ wikidata_id: Q13515725

163+ official_api_documentation: http://mementoweb.org/depot/native/archiveis/

164+ use_official_api: false

165+ require_api_key: false

166+ results: HTML

160167

161168 - name : arxiv

162169 engine : arxiv

@@ -201,6 +208,13 @@ engines:

201208 timeout : 4.0

202209 disabled : True

203210 shortcut : bb

211+ about:

212+ website: https://bitbucket.org/

213+ wikidata_id: Q2493781

214+ official_api_documentation: https://developer.atlassian.com/bitbucket

215+ use_official_api: false

216+ require_api_key: false

217+ results: HTML

204218

205219 - name : btdigg

206220 engine : btdigg

@@ -216,6 +230,13 @@ engines:

216230 categories : videos

217231 disabled : True

218232 shortcut : c3tv

233+ about:

234+ website: https://media.ccc.de/

235+ wikidata_id: Q80729951

236+ official_api_documentation: https://github.com/voc/voctoweb

237+ use_official_api: false

238+ require_api_key: false

239+ results: HTML

219240

220241 - name : crossref

221242 engine : json_engine

@@ -226,6 +247,13 @@ engines:

226247 content_query : fullCitation

227248 categories : science

228249 shortcut : cr

250+ about:

251+ website: https://www.crossref.org/

252+ wikidata_id: Q5188229

253+ official_api_documentation: https://github.com/CrossRef/rest-api-doc

254+ use_official_api: false

255+ require_api_key: false

256+ results: JSON

229257

230258 - name : currency

231259 engine : currency_convert

@@ -271,6 +299,13 @@ engines:

271299 categories : general

272300 shortcut : ew

273301 disabled : True

302+ about:

303+ website: https://www.erowid.org/

304+ wikidata_id: Q1430691

305+ official_api_documentation:

306+ use_official_api: false

307+ require_api_key: false

308+ results: HTML

274309

275310# - name : elasticsearch

276311# shortcut : es

@@ -321,6 +356,13 @@ engines:

321356 first_page_num : 1

322357 shortcut : et

323358 disabled : True

359+ about:

360+ website: https://www.etymonline.com/

361+ wikidata_id: Q1188617

362+ official_api_documentation:

363+ use_official_api: false

364+ require_api_key: false

365+ results: HTML

324366

325367# - name : ebay

326368# engine : ebay

@@ -360,6 +402,9 @@ engines:

360402 search_type : title

361403 timeout : 5.0

362404 disabled : True

405+ about:

406+ website: https://directory.fsf.org/

407+ wikidata_id: Q2470288

363408

364409 - name : frinkiac

365410 engine : frinkiac

@@ -394,6 +439,13 @@ engines:

394439 shortcut : gl

395440 timeout : 10.0

396441 disabled : True

442+ about:

443+ website: https://about.gitlab.com/

444+ wikidata_id: Q16639197

445+ official_api_documentation: https://docs.gitlab.com/ee/api/

446+ use_official_api: false

447+ require_api_key: false

448+ results: JSON

397449

398450 - name : github

399451 engine : github

@@ -411,6 +463,13 @@ engines:

411463 categories : it

412464 shortcut : cb

413465 disabled : True

466+ about:

467+ website: https://codeberg.org/

468+ wikidata_id:

469+ official_api_documentation: https://try.gitea.io/api/swagger

470+ use_official_api: false

471+ require_api_key: false

472+ results: JSON

414473

415474 - name : google

416475 engine : google

@@ -441,6 +500,13 @@ engines:

441500 first_page_num : 0

442501 categories : science

443502 shortcut : gos

503+ about:

504+ website: https://scholar.google.com/

505+ wikidata_id: Q494817

506+ official_api_documentation:

507+ use_official_api: false

508+ require_api_key: false

509+ results: HTML

444510

445511 - name : google play apps

446512 engine : xpath

@@ -453,6 +519,13 @@ engines:

453519 categories : files

454520 shortcut : gpa

455521 disabled : True

522+ about:

523+ website: https://play.google.com/

524+ wikidata_id: Q79576

525+ official_api_documentation:

526+ use_official_api: false

527+ require_api_key: false

528+ results: HTML

456529

457530 - name : google play movies

458531 engine : xpath

@@ -465,6 +538,13 @@ engines:

465538 categories : videos

466539 shortcut : gpm

467540 disabled : True

541+ about:

542+ website: https://play.google.com/

543+ wikidata_id: Q79576

544+ official_api_documentation:

545+ use_official_api: false

546+ require_api_key: false

547+ results: HTML

468548

469549 - name : google play music

470550 engine : xpath

@@ -477,6 +557,13 @@ engines:

477557 categories : music

478558 shortcut : gps

479559 disabled : True

560+ about:

561+ website: https://play.google.com/

562+ wikidata_id: Q79576

563+ official_api_documentation:

564+ use_official_api: false

565+ require_api_key: false

566+ results: HTML

480567

481568 - name : geektimes

482569 engine : xpath

@@ -489,6 +576,13 @@ engines:

489576 timeout : 4.0

490577 disabled : True

491578 shortcut : gt

579+ about:

580+ website: https://geektimes.ru/

581+ wikidata_id: Q50572423

582+ official_api_documentation:

583+ use_official_api: false

584+ require_api_key: false

585+ results: HTML

492586

493587 - name : habrahabr

494588 engine : xpath

@@ -501,6 +595,13 @@ engines:

501595 timeout : 4.0

502596 disabled : True

503597 shortcut : habr

598+ about:

599+ website: https://habr.com/

600+ wikidata_id: Q4494434

601+ official_api_documentation: https://habr.com/en/docs/help/api/

602+ use_official_api: false

603+ require_api_key: false

604+ results: HTML

504605

505606 - name : hoogle

506607 engine : json_engine

@@ -513,6 +614,13 @@ engines:

513614 page_size : 20

514615 categories : it

515616 shortcut : ho

617+ about:

618+ website: https://www.haskell.org/

619+ wikidata_id: Q34010

620+ official_api_documentation: https://hackage.haskell.org/api

621+ use_official_api: false

622+ require_api_key: false

623+ results: JSON

516624

517625 - name : ina

518626 engine : ina

@@ -543,6 +651,13 @@ engines:

543651 timeout : 7.0

544652 disabled : True

545653 shortcut : lg

654+ about:

655+ website: http://libgen.rs/

656+ wikidata_id: Q22017206

657+ official_api_documentation:

658+ use_official_api: false

659+ require_api_key: false

660+ results: HTML

546661

547662 - name : lobste.rs

548663 engine : xpath

@@ -555,6 +670,13 @@ engines:

555670 shortcut : lo

556671 timeout : 3.0

557672 disabled: True

673+ about:

674+ website: https://lobste.rs/

675+ wikidata_id: Q60762874

676+ official_api_documentation:

677+ use_official_api: false

678+ require_api_key: false

679+ results: HTML

558680

559681 - name : metager

560682 engine : xpath

@@ -566,6 +688,13 @@ engines:

566688 categories : general

567689 shortcut : mg

568690 disabled : True

691+ about:

692+ website: https://metager.org/

693+ wikidata_id: Q1924645

694+ official_api_documentation:

695+ use_official_api: false

696+ require_api_key: false

697+ results: HTML

569698

570699 - name : microsoft academic

571700 engine : microsoft_academic

@@ -589,6 +718,13 @@ engines:

589718 disabled: True

590719 timeout: 5.0

591720 shortcut : npm

721+ about:

722+ website: https://npms.io/

723+ wikidata_id: Q7067518

724+ official_api_documentation: https://api-docs.npms.io/

725+ use_official_api: false

726+ require_api_key: false

727+ results: JSON

592728

593729# Requires Tor

594730 - name : not evil

@@ -617,6 +753,13 @@ engines:

617753 categories : science

618754 shortcut : oad

619755 timeout: 5.0

756+ about:

757+ website: https://www.openaire.eu/

758+ wikidata_id: Q25106053

759+ official_api_documentation: https://api.openaire.eu/

760+ use_official_api: false

761+ require_api_key: false

762+ results: JSON

620763

621764 - name : openairepublications

622765 engine : json_engine

@@ -629,6 +772,13 @@ engines:

629772 categories : science

630773 shortcut : oap

631774 timeout: 5.0

775+ about:

776+ website: https://www.openaire.eu/

777+ wikidata_id: Q25106053

778+ official_api_documentation: https://api.openaire.eu/

779+ use_official_api: false

780+ require_api_key: false

781+ results: JSON

632782

633783# - name : opensemanticsearch

634784# engine : opensemantic

@@ -650,6 +800,13 @@ engines:

650800 timeout : 4.0

651801 disabled : True

652802 shortcut : or

803+ about:

804+ website: https://openrepos.net/

805+ wikidata_id:

806+ official_api_documentation:

807+ use_official_api: false

808+ require_api_key: false

809+ results: HTML

653810

654811 - name : pdbe

655812 engine : pdbe

@@ -768,6 +925,13 @@ engines:

768925 content_xpath : .//div[@class="search-result-abstract"]

769926 shortcut : se

770927 categories : science

928+ about:

929+ website: https://www.semanticscholar.org/

930+ wikidata_id: Q22908627

931+ official_api_documentation: https://api.semanticscholar.org/

932+ use_official_api: false

933+ require_api_key: false

934+ results: HTML

771935

772936# Spotify needs API credentials

773937# - name : spotify

@@ -876,6 +1040,9 @@ engines:

8761040 number_of_results : 5

8771041 search_type : text

8781042 disabled : True

1043+ about:

1044+ website: https://www.wikibooks.org/

1045+ wikidata_id: Q367

8791046

8801047 - name : wikinews

8811048 engine : mediawiki

@@ -885,6 +1052,9 @@ engines:

8851052 number_of_results : 5

8861053 search_type : text

8871054 disabled : True

1055+ about:

1056+ website: https://www.wikinews.org/

1057+ wikidata_id: Q964

8881058

8891059 - name : wikiquote

8901060 engine : mediawiki

@@ -896,6 +1066,9 @@ engines:

8961066 disabled : True

8971067 additional_tests:

8981068 rosebud: *test_rosebud

1069+ about:

1070+ website: https://www.wikiquote.org/

1071+ wikidata_id: Q369

8991072

9001073 - name : wikisource

9011074 engine : mediawiki

@@ -905,6 +1078,9 @@ engines:

9051078 number_of_results : 5

9061079 search_type : text

9071080 disabled : True

1081+ about:

1082+ website: https://www.wikisource.org/

1083+ wikidata_id: Q263

9081084

9091085 - name : wiktionary

9101086 engine : mediawiki

@@ -914,6 +1090,9 @@ engines:

9141090 number_of_results : 5

9151091 search_type : text

9161092 disabled : True

1093+ about:

1094+ website: https://www.wiktionary.org/

1095+ wikidata_id: Q151

9171096

9181097 - name : wikiversity

9191098 engine : mediawiki

@@ -923,6 +1102,9 @@ engines:

9231102 number_of_results : 5

9241103 search_type : text

9251104 disabled : True

1105+ about:

1106+ website: https://www.wikiversity.org/

1107+ wikidata_id: Q370

9261108

9271109 - name : wikivoyage

9281110 engine : mediawiki

@@ -932,6 +1114,9 @@ engines:

9321114 number_of_results : 5

9331115 search_type : text

9341116 disabled : True

1117+ about:

1118+ website: https://www.wikivoyage.org/

1119+ wikidata_id: Q373

9351120

9361121 - name : wolframalpha

9371122 shortcut : wa

@@ -979,6 +1164,13 @@ engines:

9791164 first_page_num : 0

9801165 page_size : 10

9811166 disabled : True

1167+ about:

1168+ website: https://www.seznam.cz/

1169+ wikidata_id: Q3490485

1170+ official_api_documentation: https://api.sklik.cz/

1171+ use_official_api: false

1172+ require_api_key: false

1173+ results: HTML

9821174

9831175 - name : mojeek

9841176 shortcut: mjk

@@ -993,6 +1185,13 @@ engines:

9931185 first_page_num : 0

9941186 page_size : 10

9951187 disabled : True

1188+ about:

1189+ website: https://www.mojeek.com/

1190+ wikidata_id: Q60747299

1191+ official_api_documentation: https://www.mojeek.com/services/api.html/

1192+ use_official_api: false

1193+ require_api_key: false

1194+ results: HTML

9961195

9971196 - name : naver

9981197 shortcut: nvr

@@ -1007,6 +1206,13 @@ engines:

10071206 first_page_num : 1

10081207 page_size : 10

10091208 disabled : True

1209+ about:

1210+ website: https://www.naver.com/

1211+ wikidata_id: Q485639

1212+ official_api_documentation: https://developers.naver.com/docs/nmt/examples/

1213+ use_official_api: false

1214+ require_api_key: false

1215+ results: HTML

10101216

10111217 - name : rubygems

10121218 shortcut: rbg

@@ -1021,6 +1227,13 @@ engines:

10211227 first_page_num : 1

10221228 categories: it

10231229 disabled : True

1230+ about:

1231+ website: https://rubygems.org/

1232+ wikidata_id: Q1853420

1233+ official_api_documentation: https://guides.rubygems.org/rubygems-org-api/

1234+ use_official_api: false

1235+ require_api_key: false

1236+ results: HTML

10241237

10251238 - name : peertube

10261239 engine: peertube

utils/fetch_engine_descriptions.pynew file+206

@@ -0,0 +1,206 @@

1+#!/usr/bin/env python

2+

3+import sys

4+import json

5+from urllib.parse import quote, urlparse

6+from os.path import realpath, dirname

7+import cld3

8+from lxml.html import fromstring

9+

10+# set path

11+sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))

12+

13+from searx.engines.wikidata import send_wikidata_query

14+from searx.utils import extract_text

15+import searx

16+import searx.search

17+import searx.poolrequests

18+

19+SPARQL_WIKIPEDIA_ARTICLE = """

20+SELECT DISTINCT ?item ?name

21+WHERE {

22+ VALUES ?item { %IDS% }

23+ ?article schema:about ?item ;

24+ schema:inLanguage ?lang ;

25+ schema:name ?name ;

26+ schema:isPartOf [ wikibase:wikiGroup "wikipedia" ] .

27+ FILTER(?lang in (%LANGUAGES_SPARQL%)) .

28+ FILTER (!CONTAINS(?name, ':')) .

29+}

30+"""

31+

32+SPARQL_DESCRIPTION = """

33+SELECT DISTINCT ?item ?itemDescription

34+WHERE {

35+ VALUES ?item { %IDS% }

36+ ?item schema:description ?itemDescription .

37+ FILTER (lang(?itemDescription) in (%LANGUAGES_SPARQL%))

38+}

39+ORDER BY ?itemLang

40+"""

41+

42+LANGUAGES = searx.settings['locales'].keys()

43+LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))

44+IDS = None

45+

46+descriptions = {}

47+wd_to_engine_name = {}

48+

49+

50+def normalize_description(description):

51+ for c in [chr(c) for c in range(0, 31)]:

52+ description = description.replace(c, ' ')

53+ description = ' '.join(description.strip().split())

54+ return description

55+

56+

57+def update_description(engine_name, lang, description, source, replace=True):

58+ if replace or lang not in descriptions[engine_name]:

59+ descriptions[engine_name][lang] = [normalize_description(description), source]

60+

61+

62+def get_wikipedia_summary(language, pageid):

63+ search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'

64+ url = search_url.format(title=quote(pageid), language=language)

65+ try:

66+ response = searx.poolrequests.get(url)

67+ response.raise_for_status()

68+ api_result = json.loads(response.text)

69+ return api_result.get('extract')

70+ except:

71+ return None

72+

73+

74+def detect_language(text):

75+ r = cld3.get_language(str(text)) # pylint: disable=E1101

76+ if r is not None and r.probability >= 0.98 and r.is_reliable:

77+ return r.language

78+ return None

79+

80+

81+def get_website_description(url, lang1, lang2=None):

82+ headers = {

83+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0',

84+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

85+ 'DNT': '1',

86+ 'Upgrade-Insecure-Requests': '1',

87+ 'Sec-GPC': '1',

88+ 'Cache-Control': 'max-age=0',

89+ }

90+ if lang1 is not None:

91+ lang_list = [lang1]

92+ if lang2 is not None:

93+ lang_list.append(lang2)

94+ headers['Accept-Language'] = f'{",".join(lang_list)};q=0.8'

95+ try:

96+ response = searx.poolrequests.get(url, headers=headers, timeout=10)

97+ response.raise_for_status()

98+ except Exception:

99+ return (None, None)

100+

101+ try:

102+ html = fromstring(response.text)

103+ except ValueError:

104+ html = fromstring(response.content)

105+

106+ description = extract_text(html.xpath('/html/head/meta[@name="description"]/@content'))

107+ if not description:

108+ description = extract_text(html.xpath('/html/head/meta[@property="og:description"]/@content'))

109+ if not description:

110+ description = extract_text(html.xpath('/html/head/title'))

111+ lang = extract_text(html.xpath('/html/@lang'))

112+ if lang is None and len(lang1) > 0:

113+ lang = lang1

114+ lang = detect_language(description) or lang or 'en'

115+ lang = lang.split('_')[0]

116+ lang = lang.split('-')[0]

117+ return (lang, description)

118+

119+

120+def initialize():

121+ global descriptions, wd_to_engine_name, IDS

122+ searx.search.initialize()

123+ for engine_name, engine in searx.engines.engines.items():

124+ descriptions[engine_name] = {}

125+ wikidata_id = getattr(engine, "about", {}).get('wikidata_id')

126+ if wikidata_id is not None:

127+ wd_to_engine_name.setdefault(wikidata_id, set()).add(engine_name)

128+

129+ IDS = ' '.join(list(map(lambda wd_id: 'wd:' + wd_id, wd_to_engine_name.keys())))

130+

131+

132+def fetch_wikidata_descriptions():

133+ global IDS

134+ result = send_wikidata_query(SPARQL_DESCRIPTION

135+ .replace('%IDS%', IDS)

136+ .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))

137+ if result is not None:

138+ for binding in result['results']['bindings']:

139+ wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')

140+ lang = binding['itemDescription']['xml:lang']

141+ description = binding['itemDescription']['value']

142+ if ' ' in description: # skip unique word description (like "website")

143+ for engine_name in wd_to_engine_name[wikidata_id]:

144+ update_description(engine_name, lang, description, 'wikidata')

145+

146+

147+def fetch_wikipedia_descriptions():

148+ global IDS

149+ result = send_wikidata_query(SPARQL_WIKIPEDIA_ARTICLE

150+ .replace('%IDS%', IDS)

151+ .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))

152+ if result is not None:

153+ for binding in result['results']['bindings']:

154+ wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')

155+ lang = binding['name']['xml:lang']

156+ pageid = binding['name']['value']

157+ description = get_wikipedia_summary(lang, pageid)

158+ if description is not None and ' ' in description:

159+ for engine_name in wd_to_engine_name[wikidata_id]:

160+ update_description(engine_name, lang, description, 'wikipedia')

161+

162+

163+def normalize_url(url):

164+ url = url.replace('{language}', 'en')

165+ url = urlparse(url)._replace(path='/', params='', query='', fragment='').geturl()

166+ url = url.replace('https://api.', 'https://')

167+ return url

168+

169+

170+def fetch_website_description(engine_name, website):

171+ default_lang, default_description = get_website_description(website, None, None)

172+ if default_lang is None or default_description is None:

173+ return

174+ if default_lang not in descriptions[engine_name]:

175+ descriptions[engine_name][default_lang] = [normalize_description(default_description), website]

176+ for request_lang in ('en-US', 'es-US', 'fr-FR', 'zh', 'ja', 'ru', 'ar', 'ko'):

177+ if request_lang.split('-')[0] not in descriptions[engine_name]:

178+ lang, desc = get_website_description(website, request_lang, request_lang.split('-')[0])

179+ if desc is not None and desc != default_description:

180+ update_description(engine_name, lang, desc, website, replace=False)

181+ else:

182+ break

183+

184+

185+def fetch_website_descriptions():

186+ for engine_name, engine in searx.engines.engines.items():

187+ website = getattr(engine, "about", {}).get('website')

188+ if website is None:

189+ website = normalize_url(getattr(engine, "search_url"))

190+ if website is None:

191+ website = normalize_url(getattr(engine, "base_url"))

192+ if website is not None:

193+ fetch_website_description(engine_name, website)

194+

195+

196+def main():

197+ initialize()

198+ fetch_wikidata_descriptions()

199+ fetch_wikipedia_descriptions()

200+ fetch_website_descriptions()

201+

202+ sys.stdout.write(json.dumps(descriptions, indent=1, separators=(',', ':'), ensure_ascii=False))

203+

204+

205+if __name__ == "__main__":

206+ main()