git.iko.soy/searx
on master help sign in
[enh] engines: add about variable
a4dcfa0Alexandre Flament
searx/engines/1337x.py+14
compare with master
unifiedsplit
@@ -1,7 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
3+ 1337x
4+"""
5+
1from urllib.parse import quote, urljoin
6from urllib.parse import quote, urljoin
2from lxml import html
7from lxml import html
3from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex
8from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex
4
9
10+# about
11+about = {
12+ "website": 'https://1337x.to/',
13+ "wikidata_id": 'Q28134166',
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
5
19
6url = 'https://1337x.to/'
20url = 'https://1337x.to/'
7search_url = url + 'search/{search_term}/{pageno}/'
21search_url = url + 'search/{search_term}/{pageno}/'
searx/engines/acgsou.py+11−7
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)
3 Acgsou (Japanese Animation/Music/Comics Bittorrent tracker)
3−
4− @website https://www.acgsou.com/
5− @provide-api no
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, content, seed, leech, torrentfile
10"""
4"""
11
5
12from urllib.parse import urlencode
6from urllib.parse import urlencode
13from lxml import html
7from lxml import html
14from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex
8from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex
15
9
10+# about
11+about = {
12+ "website": 'https://www.acgsou.com/',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
16# engine dependent config
20# engine dependent config
17categories = ['files', 'images', 'videos', 'music']
21categories = ['files', 'images', 'videos', 'music']
18paging = True
22paging = True
searx/engines/ahmia.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Ahmia (Onions)
3 Ahmia (Onions)
3−
4− @website http://msydqstlz2kzerdg.onion
5− @provides-api no
6−
7− @using-api no
8− @results HTML
9− @stable no
10− @parse url, title, content
11"""
4"""
12
5
13from urllib.parse import urlencode, urlparse, parse_qs
6from urllib.parse import urlencode, urlparse, parse_qs
14from lxml.html import fromstring
7from lxml.html import fromstring
15from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath
8from searx.engines.xpath import extract_url, extract_text, eval_xpath_list, eval_xpath
16
9
10+# about
11+about = {
12+ "website": 'http://msydqstlz2kzerdg.onion',
13+ "wikidata_id": 'Q18693938',
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
17# engine config
20# engine config
18categories = ['onions']
21categories = ['onions']
19paging = True
22paging = True
searx/engines/apkmirror.py+10−7
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 APK Mirror
3 APK Mirror
3−
4− @website https://www.apkmirror.com
5−
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, thumbnail_src
10"""
4"""
11
5
12from urllib.parse import urlencode
6from urllib.parse import urlencode
13from lxml import html
7from lxml import html
14from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
8from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
15
9
10+# about
11+about = {
12+ "website": 'https://www.apkmirror.com',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
16
19
17# engine dependent config
20# engine dependent config
18categories = ['it']
21categories = ['it']
searx/engines/archlinux.py+12−8
@@ -1,20 +1,24 @@
1−# -*- coding: utf-8 -*-
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 Arch Linux Wiki
3 Arch Linux Wiki
5
4
6− @website https://wiki.archlinux.org
7− @provide-api no (Mediawiki provides API, but Arch Wiki blocks access to it
5+ API: Mediawiki provides API, but Arch Wiki blocks access to it
8− @using-api no
9− @results HTML
10− @stable no (HTML can change)
11− @parse url, title
12"""
6"""
13
7
14from urllib.parse import urlencode, urljoin
8from urllib.parse import urlencode, urljoin
15from lxml import html
9from lxml import html
16from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
10from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
17
11
12+# about
13+about = {
14+ "website": 'https://wiki.archlinux.org/',
15+ "wikidata_id": 'Q101445877',
16+ "official_api_documentation": None,
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'HTML',
20+}
21+
18# engine dependent config
22# engine dependent config
19categories = ['it']
23categories = ['it']
20language_support = True
24language_support = True
searx/engines/arxiv.py+10−9
@@ -1,20 +1,21 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 ArXiV (Scientific preprints)
3 ArXiV (Scientific preprints)
5− @website https://arxiv.org
6− @provide-api yes (export.arxiv.org/api/query)
7− @using-api yes
8− @results XML-RSS
9− @stable yes
10− @parse url, title, publishedDate, content
11− More info on api: https://arxiv.org/help/api/user-manual
12"""
4"""
13
5
14from lxml import html
6from lxml import html
15from datetime import datetime
7from datetime import datetime
16from searx.utils import eval_xpath_list, eval_xpath_getindex
8from searx.utils import eval_xpath_list, eval_xpath_getindex
17
9
10+# about
11+about = {
12+ "website": 'https://arxiv.org',
13+ "wikidata_id": 'Q118398',
14+ "official_api_documentation": 'https://arxiv.org/help/api',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'XML-RSS',
18+}
18
19
19categories = ['science']
20categories = ['science']
20paging = True
21paging = True
searx/engines/base.py+10−11
@@ -1,16 +1,6 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 BASE (Scholar publications)
3 BASE (Scholar publications)
5−
6− @website https://base-search.net
7− @provide-api yes with authorization (https://api.base-search.net/)
8−
9− @using-api yes
10− @results XML
11− @stable ?
12− @parse url, title, publishedDate, content
13− More info on api: http://base-search.net/about/download/base_interface.pdf
14"""
4"""
15
5
16from urllib.parse import urlencode
6from urllib.parse import urlencode
@@ -19,6 +9,15 @@ from datetime import datetime
19import re
9import re
20from searx.utils import searx_useragent
10from searx.utils import searx_useragent
21
11
12+# about
13+about = {
14+ "website": 'https://base-search.net',
15+ "wikidata_id": 'Q448335',
16+ "official_api_documentation": 'https://api.base-search.net/',
17+ "use_official_api": True,
18+ "require_api_key": False,
19+ "results": 'XML',
20+}
22
21
23categories = ['science']
22categories = ['science']
24
23
searx/engines/bing.py+11−11
@@ -1,16 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Bing (Web)
3 Bing (Web)
3−
4− @website https://www.bing.com
5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
6− max. 5000 query/month
7−
8− @using-api no (because of query limit)
9− @results HTML (using search portal)
10− @stable no (HTML can change)
11− @parse url, title, content
12−
13− @todo publishedDate
14"""
4"""
15
5
16import re
6import re
@@ -21,6 +11,16 @@ from searx.utils import eval_xpath, extract_text, match_language
21
11
22logger = logger.getChild('bing engine')
12logger = logger.getChild('bing engine')
23
13
14+# about
15+about = {
16+ "website": 'https://www.bing.com',
17+ "wikidata_id": 'Q182496',
18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-web-search-api',
19+ "use_official_api": False,
20+ "require_api_key": False,
21+ "results": 'HTML',
22+}
23+
24# engine dependent config
24# engine dependent config
25categories = ['general']
25categories = ['general']
26paging = True
26paging = True
searx/engines/bing_images.py+11−10
@@ -1,15 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Bing (Images)
3 Bing (Images)
3−
4− @website https://www.bing.com/images
5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
6− max. 5000 query/month
7−
8− @using-api no (because of query limit)
9− @results HTML (using search portal)
10− @stable no (HTML can change)
11− @parse url, title, img_src
12−
13"""
4"""
14
5
15from urllib.parse import urlencode
6from urllib.parse import urlencode
@@ -20,6 +11,16 @@ from searx.utils import match_language
20from searx.engines.bing import language_aliases
11from searx.engines.bing import language_aliases
21from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
12from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
22
13
14+# about
15+about = {
16+ "website": 'https://www.bing.com/images',
17+ "wikidata_id": 'Q182496',
18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-image-search-api',
19+ "use_official_api": False,
20+ "require_api_key": False,
21+ "results": 'HTML',
22+}
23+
23# engine dependent config
24# engine dependent config
24categories = ['images']
25categories = ['images']
25paging = True
26paging = True
searx/engines/bing_news.py+11−9
@@ -1,14 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Bing (News)
3 Bing (News)
3−
4− @website https://www.bing.com/news
5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
6− max. 5000 query/month
7−
8− @using-api no (because of query limit)
9− @results RSS (using search portal)
10− @stable yes (except perhaps for the images)
11− @parse url, title, content, publishedDate, thumbnail
12"""
4"""
13
5
14from datetime import datetime
6from datetime import datetime
@@ -20,6 +12,16 @@ from searx.utils import match_language, eval_xpath_getindex
20from searx.engines.bing import language_aliases
12from searx.engines.bing import language_aliases
21from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
13from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
22
14
15+# about
16+about = {
17+ "website": 'https://www.bing.com/news',
18+ "wikidata_id": 'Q2878637',
19+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-news-search-api',
20+ "use_official_api": False,
21+ "require_api_key": False,
22+ "results": 'RSS',
23+}
24+
23# engine dependent config
25# engine dependent config
24categories = ['news']
26categories = ['news']
25paging = True
27paging = True
searx/engines/bing_videos.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Bing (Videos)
3 Bing (Videos)
3−
4− @website https://www.bing.com/videos
5− @provide-api yes (http://datamarket.azure.com/dataset/bing/search)
6−
7− @using-api no
8− @results HTML
9− @stable no
10− @parse url, title, content, thumbnail
11"""
4"""
12
5
13from json import loads
6from json import loads
@@ -18,6 +11,16 @@ from searx.utils import match_language
18from searx.engines.bing import language_aliases
11from searx.engines.bing import language_aliases
19from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
12from searx.engines.bing import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
20
13
14+# about
15+about = {
16+ "website": 'https://www.bing.com/videos',
17+ "wikidata_id": 'Q4914152',
18+ "official_api_documentation": 'https://www.microsoft.com/en-us/bing/apis/bing-video-search-api',
19+ "use_official_api": False,
20+ "require_api_key": False,
21+ "results": 'HTML',
22+}
23+
21categories = ['videos']
24categories = ['videos']
22paging = True
25paging = True
23safesearch = True
26safesearch = True
searx/engines/btdigg.py+14−8
@@ -1,19 +1,25 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 BTDigg (Videos, Music, Files)
3 BTDigg (Videos, Music, Files)
3−
4− @website https://btdig.com
5− @provide-api yes (on demand)
6−
7− @using-api no
8− @results HTML (using search portal)
9− @stable no (HTML can change)
10− @parse url, title, content, seed, leech, magnetlink
11"""
4"""
12
5
13from lxml import html
6from lxml import html
14from urllib.parse import quote, urljoin
7from urllib.parse import quote, urljoin
15from searx.utils import extract_text, get_torrent_size
8from searx.utils import extract_text, get_torrent_size
16
9
10+# about
11+about = {
12+ "website": 'https://btdig.com',
13+ "wikidata_id": 'Q4836698',
14+ "official_api_documentation": {
15+ 'url': 'https://btdig.com/contacts',
16+ 'comment': 'on demand'
17+ },
18+ "use_official_api": False,
19+ "require_api_key": False,
20+ "results": 'HTML',
21+}
22+
17# engine dependent config
23# engine dependent config
18categories = ['videos', 'music', 'files']
24categories = ['videos', 'music', 'files']
19paging = True
25paging = True
searx/engines/command.py+4−15
@@ -1,18 +1,7 @@
1−'''
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−searx is free software: you can redistribute it and/or modify
2+"""
3−it under the terms of the GNU Affero General Public License as published by
3+ Command (offline)
4−the Free Software Foundation, either version 3 of the License, or
4+"""
5−(at your option) any later version.
6−
7−searx is distributed in the hope that it will be useful,
8−but WITHOUT ANY WARRANTY; without even the implied warranty of
9−MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10−GNU Affero General Public License for more details.
11−
12−You should have received a copy of the GNU Affero General Public License
13−along with searx. If not, see < http://www.gnu.org/licenses/ >.
14−'''
15−
16
5
17import re
6import re
18from os.path import expanduser, isabs, realpath, commonprefix
7from os.path import expanduser, isabs, realpath, commonprefix
searx/engines/currency_convert.py+14
@@ -1,5 +1,19 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
3+ currency convert (DuckDuckGo)
4+"""
5+
1import json
6import json
2
7
8+# about
9+about = {
10+ "website": 'https://duckduckgo.com/',
11+ "wikidata_id": 'Q12805',
12+ "official_api_documentation": 'https://duckduckgo.com/api',
13+ "use_official_api": False,
14+ "require_api_key": False,
15+ "results": 'JSONP',
16+}
3
17
4engine_type = 'online_currency'
18engine_type = 'online_currency'
5categories = []
19categories = []
searx/engines/dailymotion.py+11−10
@@ -1,15 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Dailymotion (Videos)
3 Dailymotion (Videos)
3−
4− @website https://www.dailymotion.com
5− @provide-api yes (http://www.dailymotion.com/developer)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, thumbnail, publishedDate, embedded
11−
12− @todo set content-parameter with correct data
13"""
4"""
14
5
15from json import loads
6from json import loads
@@ -17,6 +8,16 @@ from datetime import datetime
17from urllib.parse import urlencode
8from urllib.parse import urlencode
18from searx.utils import match_language, html_to_text
9from searx.utils import match_language, html_to_text
19
10
11+# about
12+about = {
13+ "website": 'https://www.dailymotion.com',
14+ "wikidata_id": 'Q769222',
15+ "official_api_documentation": 'https://www.dailymotion.com/developer',
16+ "use_official_api": True,
17+ "require_api_key": False,
18+ "results": 'JSON',
19+}
20+
20# engine dependent config
21# engine dependent config
21categories = ['videos']
22categories = ['videos']
22paging = True
23paging = True
searx/engines/deezer.py+11−8
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Deezer (Music)
3 Deezer (Music)
3−
4− @website https://deezer.com
5− @provide-api yes (http://developers.deezer.com/api/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, embedded
11"""
4"""
12
5
13from json import loads
6from json import loads
14from urllib.parse import urlencode
7from urllib.parse import urlencode
15
8
9+# about
10+about = {
11+ "website": 'https://deezer.com',
12+ "wikidata_id": 'Q602243',
13+ "official_api_documentation": 'https://developers.deezer.com/',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
18+
16# engine dependent config
19# engine dependent config
17categories = ['music']
20categories = ['music']
18paging = True
21paging = True
searx/engines/deviantart.py+11−10
@@ -1,21 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Deviantart (Images)
3 Deviantart (Images)
3−
4− @website https://www.deviantart.com/
5− @provide-api yes (https://www.deviantart.com/developers/) (RSS)
6−
7− @using-api no (TODO, rewrite to api)
8− @results HTML
9− @stable no (HTML can change)
10− @parse url, title, img_src
11−
12− @todo rewrite to api
13"""
4"""
14# pylint: disable=missing-function-docstring
5# pylint: disable=missing-function-docstring
15
6
16from urllib.parse import urlencode
7from urllib.parse import urlencode
17from lxml import html
8from lxml import html
18
9
10+# about
11+about = {
12+ "website": 'https://www.deviantart.com/',
13+ "wikidata_id": 'Q46523',
14+ "official_api_documentation": 'https://www.deviantart.com/developers/',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
19# engine dependent config
20# engine dependent config
20categories = ['images']
21categories = ['images']
21paging = True
22paging = True
searx/engines/dictzone.py+10−7
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Dictzone
3 Dictzone
3−
4− @website https://dictzone.com/
5− @provide-api no
6− @using-api no
7− @results HTML (using search portal)
8− @stable no (HTML can change)
9− @parse url, title, content
10"""
4"""
11
5
12from urllib.parse import urljoin
6from urllib.parse import urljoin
13from lxml import html
7from lxml import html
14from searx.utils import eval_xpath
8from searx.utils import eval_xpath
15
9
10+# about
11+about = {
12+ "website": 'https://dictzone.com/',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
16
19
17engine_type = 'online_dictionnary'
20engine_type = 'online_dictionnary'
18categories = ['general']
21categories = ['general']
searx/engines/digbt.py+10−8
@@ -1,19 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 DigBT (Videos, Music, Files)
3 DigBT (Videos, Music, Files)
3−
4− @website https://digbt.org
5− @provide-api no
6−
7− @using-api no
8− @results HTML (using search portal)
9− @stable no (HTML can change)
10− @parse url, title, content, magnetlink
11"""
4"""
12
5
13from urllib.parse import urljoin
6from urllib.parse import urljoin
14from lxml import html
7from lxml import html
15from searx.utils import extract_text, get_torrent_size
8from searx.utils import extract_text, get_torrent_size
16
9
10+# about
11+about = {
12+ "website": 'https://digbt.org',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
17
19
18categories = ['videos', 'music', 'files']
20categories = ['videos', 'music', 'files']
19paging = True
21paging = True
searx/engines/digg.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Digg (News, Social media)
3 Digg (News, Social media)
3−
4− @website https://digg.com
5− @provide-api no
6−
7− @using-api no
8− @results HTML (using search portal)
9− @stable no (HTML can change)
10− @parse url, title, content, publishedDate, thumbnail
11"""
4"""
12# pylint: disable=missing-function-docstring
5# pylint: disable=missing-function-docstring
13
6
@@ -17,6 +10,16 @@ from datetime import datetime
17
10
18from lxml import html
11from lxml import html
19
12
13+# about
14+about = {
15+ "website": 'https://digg.com',
16+ "wikidata_id": 'Q270478',
17+ "official_api_documentation": None,
18+ "use_official_api": False,
19+ "require_api_key": False,
20+ "results": 'HTML',
21+}
22+
20# engine dependent config
23# engine dependent config
21categories = ['news', 'social media']
24categories = ['news', 'social media']
22paging = True
25paging = True
searx/engines/doku.py+14−10
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Doku Wiki
3+ Doku Wiki
2−#
4+"""
3−# @website https://www.dokuwiki.org/
4−# @provide-api yes
5−# (https://www.dokuwiki.org/devel:xmlrpc)
6−#
7−# @using-api no
8−# @results HTML
9−# @stable yes
10−# @parse (general) url, title, content
11
5
12from urllib.parse import urlencode
6from urllib.parse import urlencode
13from lxml.html import fromstring
7from lxml.html import fromstring
14from searx.utils import extract_text, eval_xpath
8from searx.utils import extract_text, eval_xpath
15
9
10+# about
11+about = {
12+ "website": 'https://www.dokuwiki.org/',
13+ "wikidata_id": 'Q851864',
14+ "official_api_documentation": 'https://www.dokuwiki.org/devel:xmlrpc',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
16# engine dependent config
20# engine dependent config
17categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'
21categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'
18paging = False
22paging = False
searx/engines/duckduckgo.py+11−11
@@ -1,22 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 DuckDuckGo (Web)
3 DuckDuckGo (Web)
3−
4− @website https://duckduckgo.com/
5− @provide-api yes (https://duckduckgo.com/api),
6− but not all results from search-site
7−
8− @using-api no
9− @results HTML (using search portal)
10− @stable no (HTML can change)
11− @parse url, title, content
12−
13− @todo rewrite to api
14"""
4"""
15
5
16from lxml.html import fromstring
6from lxml.html import fromstring
17from json import loads
7from json import loads
18from searx.utils import extract_text, match_language, eval_xpath
8from searx.utils import extract_text, match_language, eval_xpath
19
9
10+# about
11+about = {
12+ "website": 'https://duckduckgo.com/',
13+ "wikidata_id": 'Q12805',
14+ "official_api_documentation": 'https://duckduckgo.com/api',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
20# engine dependent config
20# engine dependent config
21categories = ['general']
21categories = ['general']
22paging = False
22paging = False
searx/engines/duckduckgo_definitions.py+12−8
@@ -1,12 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2−DuckDuckGo (definitions)
3−
4−- `Instant Answer API`_
3+ DuckDuckGo (Instant Answer API)
5−- `DuckDuckGo query`_
6−
7−.. _Instant Answer API: https://duckduckgo.com/api
8−.. _DuckDuckGo query: https://api.duckduckgo.com/?q=DuckDuckGo&format=json&pretty=1
9−
10"""
4"""
11
5
12import json
6import json
@@ -22,6 +16,16 @@ from searx.external_urls import get_external_url, get_earth_coordinates_url, are
22
16
23logger = logger.getChild('duckduckgo_definitions')
17logger = logger.getChild('duckduckgo_definitions')
24
18
19+# about
20+about = {
21+ "website": 'https://duckduckgo.com/',
22+ "wikidata_id": 'Q12805',
23+ "official_api_documentation": 'https://duckduckgo.com/api',
24+ "use_official_api": True,
25+ "require_api_key": False,
26+ "results": 'JSON',
27+}
28+
25URL = 'https://api.duckduckgo.com/'\
29URL = 'https://api.duckduckgo.com/'\
26 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
30 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
27
31
searx/engines/duckduckgo_images.py+14−11
@@ -1,16 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 DuckDuckGo (Images)
3 DuckDuckGo (Images)
3−
4− @website https://duckduckgo.com/
5− @provide-api yes (https://duckduckgo.com/api),
6− but images are not supported
7−
8− @using-api no
9− @results JSON (site requires js to get images)
10− @stable no (JSON can change)
11− @parse url, title, img_src
12−
13− @todo avoid extra request
14"""
4"""
15
5
16from json import loads
6from json import loads
@@ -20,6 +10,19 @@ from searx.engines.duckduckgo import get_region_code
20from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
10from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
21from searx.poolrequests import get
11from searx.poolrequests import get
22
12
13+# about
14+about = {
15+ "website": 'https://duckduckgo.com/',
16+ "wikidata_id": 'Q12805',
17+ "official_api_documentation": {
18+ 'url': 'https://duckduckgo.com/api',
19+ 'comment': 'but images are not supported',
20+ },
21+ "use_official_api": False,
22+ "require_api_key": False,
23+ "results": 'JSON (site requires js to get images)',
24+}
25+
23# engine dependent config
26# engine dependent config
24categories = ['images']
27categories = ['images']
25paging = True
28paging = True
searx/engines/duden.py+11−6
@@ -1,11 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Duden
3 Duden
3− @website https://www.duden.de
4− @provide-api no
5− @using-api no
6− @results HTML (using search portal)
7− @stable no (HTML can change)
8− @parse url, title, content
9"""
4"""
10
5
11import re
6import re
@@ -13,6 +8,16 @@ from urllib.parse import quote, urljoin
13from lxml import html
8from lxml import html
14from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
9from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
15
10
11+# about
12+about = {
13+ "website": 'https://www.duden.de',
14+ "wikidata_id": 'Q73624591',
15+ "official_api_documentation": None,
16+ "use_official_api": False,
17+ "require_api_key": False,
18+ "results": 'HTML',
19+}
20+
16categories = ['general']
21categories = ['general']
17paging = True
22paging = True
18language_support = False
23language_support = False
searx/engines/dummy-offline.py+11−3
@@ -1,11 +1,19 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Dummy Offline
3 Dummy Offline
3−
4− @results one result
5− @stable yes
6"""
4"""
7
5
8
6
7+# about
8+about = {
9+ "wikidata_id": None,
10+ "official_api_documentation": None,
11+ "use_official_api": False,
12+ "require_api_key": False,
13+ "results": 'HTML',
14+}
15+
16+
9def search(query, request_params):
17def search(query, request_params):
10 return [{
18 return [{
11 'result': 'this is what you get',
19 'result': 'this is what you get',
searx/engines/dummy.py+11−3
@@ -1,10 +1,18 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Dummy
3 Dummy
3−
4− @results empty array
5− @stable yes
6"""
4"""
7
5
6+# about
7+about = {
8+ "website": None,
9+ "wikidata_id": None,
10+ "official_api_documentation": None,
11+ "use_official_api": False,
12+ "require_api_key": False,
13+ "results": 'empty array',
14+}
15+
8
16
9# do search-request
17# do search-request
10def request(query, params):
18def request(query, params):
searx/engines/ebay.py+14−9
@@ -1,17 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Ebay (Videos, Music, Files)
3+ Ebay (Videos, Music, Files)
2−#
4+"""
3−# @website https://www.ebay.com
4−# @provide-api no (nothing found)
5−#
6−# @using-api no
7−# @results HTML (using search portal)
8−# @stable yes (HTML can change)
9−# @parse url, title, content, price, shipping, source
10
5
11from lxml import html
6from lxml import html
12from searx.engines.xpath import extract_text
7from searx.engines.xpath import extract_text
13from urllib.parse import quote
8from urllib.parse import quote
14
9
10+# about
11+about = {
12+ "website": 'https://www.ebay.com',
13+ "wikidata_id": 'Q58024',
14+ "official_api_documentation": 'https://developer.ebay.com/',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
15categories = ['shopping']
20categories = ['shopping']
16paging = True
21paging = True
17
22
searx/engines/elasticsearch.py+5
@@ -1,3 +1,8 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
3+ Elasticsearch
4+"""
5+
1from json import loads, dumps
6from json import loads, dumps
2from requests.auth import HTTPBasicAuth
7from requests.auth import HTTPBasicAuth
3from searx.exceptions import SearxEngineAPIException
8from searx.exceptions import SearxEngineAPIException
searx/engines/etools.py+11−7
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 eTools (Web)
3 eTools (Web)
3−
4− @website https://www.etools.ch
5− @provide-api no
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, content
10"""
4"""
11
5
12from lxml import html
6from lxml import html
13from urllib.parse import quote
7from urllib.parse import quote
14from searx.utils import extract_text, eval_xpath
8from searx.utils import extract_text, eval_xpath
15
9
10+# about
11+about = {
12+ "website": 'https://www.etools.ch',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
16categories = ['general']
20categories = ['general']
17paging = False
21paging = False
18language_support = False
22language_support = False
searx/engines/fdroid.py+11−7
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 F-Droid (a repository of FOSS applications for Android)
3 F-Droid (a repository of FOSS applications for Android)
3−
4− @website https://f-droid.org/
5− @provide-api no
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, content
10"""
4"""
11
5
12from urllib.parse import urlencode
6from urllib.parse import urlencode
13from lxml import html
7from lxml import html
14from searx.utils import extract_text
8from searx.utils import extract_text
15
9
10+# about
11+about = {
12+ "website": 'https://f-droid.org/',
13+ "wikidata_id": 'Q1386210',
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
16# engine dependent config
20# engine dependent config
17categories = ['files']
21categories = ['files']
18paging = True
22paging = True
searx/engines/flickr.py+11−9
@@ -1,21 +1,23 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 Flickr (Images)
3 Flickr (Images)
5
4
6− @website https://www.flickr.com
7− @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)
8−
9− @using-api yes
10− @results JSON
11− @stable yes
12− @parse url, title, thumbnail, img_src
13 More info on api-key : https://www.flickr.com/services/apps/create/
5 More info on api-key : https://www.flickr.com/services/apps/create/
14"""
6"""
15
7
16from json import loads
8from json import loads
17from urllib.parse import urlencode
9from urllib.parse import urlencode
18
10
11+# about
12+about = {
13+ "website": 'https://www.flickr.com',
14+ "wikidata_id": 'Q103204',
15+ "official_api_documentation": 'https://secure.flickr.com/services/api/flickr.photos.search.html',
16+ "use_official_api": True,
17+ "require_api_key": True,
18+ "results": 'JSON',
19+}
20+
19categories = ['images']
21categories = ['images']
20
22
21nb_per_page = 15
23nb_per_page = 15
searx/engines/flickr_noapi.py+12−11
@@ -1,15 +1,6 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4− Flickr (Images)
3+ Flickr (Images)
5−
6− @website https://www.flickr.com
7− @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)
8−
9− @using-api no
10− @results HTML
11− @stable no
12− @parse url, title, thumbnail, img_src
13"""
4"""
14
5
15from json import loads
6from json import loads
@@ -21,6 +12,16 @@ from searx.utils import ecma_unescape, html_to_text
21
12
22logger = logger.getChild('flickr-noapi')
13logger = logger.getChild('flickr-noapi')
23
14
15+# about
16+about = {
17+ "website": 'https://www.flickr.com',
18+ "wikidata_id": 'Q103204',
19+ "official_api_documentation": 'https://secure.flickr.com/services/api/flickr.photos.search.html',
20+ "use_official_api": False,
21+ "require_api_key": False,
22+ "results": 'HTML',
23+}
24+
24categories = ['images']
25categories = ['images']
25
26
26url = 'https://www.flickr.com/'
27url = 'https://www.flickr.com/'
searx/engines/framalibre.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 FramaLibre (It)
3 FramaLibre (It)
3−
4− @website https://framalibre.org/
5− @provide-api no
6−
7− @using-api no
8− @results HTML
9− @stable no (HTML can change)
10− @parse url, title, content, thumbnail, img_src
11"""
4"""
12
5
13from html import escape
6from html import escape
@@ -15,6 +8,16 @@ from urllib.parse import urljoin, urlencode
15from lxml import html
8from lxml import html
16from searx.utils import extract_text
9from searx.utils import extract_text
17
10
11+# about
12+about = {
13+ "website": 'https://framalibre.org/',
14+ "wikidata_id": 'Q30213882',
15+ "official_api_documentation": None,
16+ "use_official_api": False,
17+ "require_api_key": False,
18+ "results": 'HTML',
19+}
20+
18# engine dependent config
21# engine dependent config
19categories = ['it']
22categories = ['it']
20paging = True
23paging = True
searx/engines/frinkiac.py+15−8
@@ -1,17 +1,24 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2−Frinkiac (Images)
3+ Frinkiac (Images)
3−
4−@website https://www.frinkiac.com
5−@provide-api no
6−@using-api no
7−@results JSON
8−@stable no
9−@parse url, title, img_src
10"""
4"""
11
5
12from json import loads
6from json import loads
13from urllib.parse import urlencode
7from urllib.parse import urlencode
14
8
9+# about
10+about = {
11+ "website": 'https://frinkiac.com',
12+ "wikidata_id": 'Q24882614',
13+ "official_api_documentation": {
14+ 'url': None,
15+ 'comment': 'see https://github.com/MitchellAW/CompuGlobal'
16+ },
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'JSON',
20+}
21+
15categories = ['images']
22categories = ['images']
16
23
17BASE = 'https://frinkiac.com/'
24BASE = 'https://frinkiac.com/'
searx/engines/genius.py+12−9
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2−Genius
3+ Genius
3−
4− @website https://www.genius.com/
5− @provide-api yes (https://docs.genius.com/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, thumbnail, publishedDate
11"""
4"""
12
5
13from json import loads
6from json import loads
14from urllib.parse import urlencode
7from urllib.parse import urlencode
15from datetime import datetime
8from datetime import datetime
16
9
10+# about
11+about = {
12+ "website": 'https://genius.com/',
13+ "wikidata_id": 'Q3419343',
14+ "official_api_documentation": 'https://docs.genius.com/',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['music']
21categories = ['music']
19paging = True
22paging = True
searx/engines/gentoo.py+11−9
@@ -1,20 +1,22 @@
1−# -*- coding: utf-8 -*-
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 Gentoo Wiki
3 Gentoo Wiki
5−
6− @website https://wiki.gentoo.org
7− @provide-api yes
8− @using-api no
9− @results HTML
10− @stable no (HTML can change)
11− @parse url, title
12"""
4"""
13
5
14from urllib.parse import urlencode, urljoin
6from urllib.parse import urlencode, urljoin
15from lxml import html
7from lxml import html
16from searx.utils import extract_text
8from searx.utils import extract_text
17
9
10+# about
11+about = {
12+ "website": 'https://wiki.gentoo.org/',
13+ "wikidata_id": 'Q1050637',
14+ "official_api_documentation": 'https://wiki.gentoo.org/api.php',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
18# engine dependent config
20# engine dependent config
19categories = ['it']
21categories = ['it']
20language_support = True
22language_support = True
searx/engines/gigablast.py+10−8
@@ -1,14 +1,6 @@
1# SPDX-License-Identifier: AGPL-3.0-or-later
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""
2"""
3 Gigablast (Web)
3 Gigablast (Web)
4−
5− @website https://gigablast.com
6− @provide-api yes (https://gigablast.com/api.html)
7−
8− @using-api yes
9− @results XML
10− @stable yes
11− @parse url, title, content
12"""
4"""
13# pylint: disable=missing-function-docstring, invalid-name
5# pylint: disable=missing-function-docstring, invalid-name
14
6
@@ -18,6 +10,16 @@ from urllib.parse import urlencode
18# from searx import logger
10# from searx import logger
19from searx.poolrequests import get
11from searx.poolrequests import get
20
12
13+# about
14+about = {
15+ "website": 'https://www.gigablast.com',
16+ "wikidata_id": 'Q3105449',
17+ "official_api_documentation": 'https://gigablast.com/api.html',
18+ "use_official_api": True,
19+ "require_api_key": False,
20+ "results": 'JSON',
21+}
22+
21# engine dependent config
23# engine dependent config
22categories = ['general']
24categories = ['general']
23# gigablast's pagination is totally damaged, don't use it
25# gigablast's pagination is totally damaged, don't use it
searx/engines/github.py+12−9
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2− Github (It)
3+ Github (IT)
3−
4− @website https://github.com/
5− @provide-api yes (https://developer.github.com/v3/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes (using api)
10− @parse url, title, content
11"""
4"""
12
5
13from json import loads
6from json import loads
14from urllib.parse import urlencode
7from urllib.parse import urlencode
15
8
9+# about
10+about = {
11+ "website": 'https://github.com/',
12+ "wikidata_id": 'Q364',
13+ "official_api_documentation": 'https://developer.github.com/v3/',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
18+
16# engine dependent config
19# engine dependent config
17categories = ['it']
20categories = ['it']
18
21
searx/engines/google.py+14−12
@@ -1,19 +1,11 @@
1# SPDX-License-Identifier: AGPL-3.0-or-later
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Google (Web)
2"""Google (Web)
3
3
4−:website: https://www.google.com
5−:provide-api: yes (https://developers.google.com/custom-search/)
6−:using-api: not the offical, since it needs registration to another service
7−:results: HTML
8−:stable: no
9−:parse: url, title, content, number_of_results, answer, suggestion, correction
10−
11−For detailed description of the *REST-full* API see: `Query Parameter
4+ For detailed description of the *REST-full* API see: `Query Parameter
12−Definitions`_.
5+ Definitions`_.
13−
14−.. _Query Parameter Definitions:
15− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
16
6
7+ .. _Query Parameter Definitions:
8+ https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
17"""
9"""
18
10
19# pylint: disable=invalid-name, missing-function-docstring
11# pylint: disable=invalid-name, missing-function-docstring
@@ -27,6 +19,16 @@ from searx.exceptions import SearxEngineCaptchaException
27
19
28logger = logger.getChild('google engine')
20logger = logger.getChild('google engine')
29
21
22+# about
23+about = {
24+ "website": 'https://www.google.com',
25+ "wikidata_id": 'Q9366',
26+ "official_api_documentation": 'https://developers.google.com/custom-search/',
27+ "use_official_api": False,
28+ "require_api_key": False,
29+ "results": 'HTML',
30+}
31+
30# engine dependent config
32# engine dependent config
31categories = ['general']
33categories = ['general']
32paging = True
34paging = True
searx/engines/google_images.py+10−12
@@ -1,14 +1,6 @@
1# SPDX-License-Identifier: AGPL-3.0-or-later
1# SPDX-License-Identifier: AGPL-3.0-or-later
2"""Google (Images)
2"""Google (Images)
3
3
4−:website: https://images.google.com (redirected to subdomain www.)
5−:provide-api: yes (https://developers.google.com/custom-search/)
6−:using-api: not the offical, since it needs registration to another service
7−:results: HTML
8−:stable: no
9−:template: images.html
10−:parse: url, title, content, source, thumbnail_src, img_src
11−
12For detailed description of the *REST-full* API see: `Query Parameter
4For detailed description of the *REST-full* API see: `Query Parameter
13Definitions`_.
5Definitions`_.
14
6
@@ -18,10 +10,6 @@ Definitions`_.
18 ``data:` scheme).::
10 ``data:` scheme).::
19
11
20 Header set Content-Security-Policy "img-src 'self' data: ;"
12 Header set Content-Security-Policy "img-src 'self' data: ;"
21−
22−.. _Query Parameter Definitions:
23− https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions
24−
25"""
13"""
26
14
27from urllib.parse import urlencode, urlparse, unquote
15from urllib.parse import urlencode, urlparse, unquote
@@ -39,6 +27,16 @@ from searx.engines.google import (
39
27
40logger = logger.getChild('google images')
28logger = logger.getChild('google images')
41
29
30+# about
31+about = {
32+ "website": 'https://images.google.com/',
33+ "wikidata_id": 'Q521550',
34+ "official_api_documentation": 'https://developers.google.com/custom-search/docs/xml_results#WebSearch_Query_Parameter_Definitions', # NOQA
35+ "use_official_api": False,
36+ "require_api_key": False,
37+ "results": 'HTML',
38+}
39+
42# engine dependent config
40# engine dependent config
43
41
44categories = ['images']
42categories = ['images']
searx/engines/google_news.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Google (News)
3 Google (News)
3−
4− @website https://news.google.com
5− @provide-api no
6−
7− @using-api no
8− @results HTML
9− @stable no
10− @parse url, title, content, publishedDate
11"""
4"""
12
5
13from urllib.parse import urlencode
6from urllib.parse import urlencode
@@ -15,6 +8,16 @@ from lxml import html
15from searx.utils import match_language
8from searx.utils import match_language
16from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
9from searx.engines.google import _fetch_supported_languages, supported_languages_url # NOQA # pylint: disable=unused-import
17
10
11+# about
12+about = {
13+ "website": 'https://news.google.com',
14+ "wikidata_id": 'Q12020',
15+ "official_api_documentation": None,
16+ "use_official_api": False,
17+ "require_api_key": False,
18+ "results": 'HTML',
19+}
20+
18# search-url
21# search-url
19categories = ['news']
22categories = ['news']
20paging = True
23paging = True
searx/engines/google_videos.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Google (Videos)
3 Google (Videos)
3−
4− @website https://www.google.com
5− @provide-api yes (https://developers.google.com/custom-search/)
6−
7− @using-api no
8− @results HTML
9− @stable no
10− @parse url, title, content, thumbnail
11"""
4"""
12
5
13from datetime import date, timedelta
6from datetime import date, timedelta
@@ -16,6 +9,16 @@ from lxml import html
16from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
9from searx.utils import extract_text, eval_xpath, eval_xpath_list, eval_xpath_getindex
17import re
10import re
18
11
12+# about
13+about = {
14+ "website": 'https://www.google.com',
15+ "wikidata_id": 'Q219885',
16+ "official_api_documentation": 'https://developers.google.com/custom-search/',
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'HTML',
20+}
21+
19# engine dependent config
22# engine dependent config
20categories = ['videos']
23categories = ['videos']
21paging = True
24paging = True
searx/engines/ina.py+13−12
@@ -1,15 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# INA (Videos)
3+ INA (Videos)
2−#
4+"""
3−# @website https://www.ina.fr/
4−# @provide-api no
5−#
6−# @using-api no
7−# @results HTML (using search portal)
8−# @stable no (HTML can change)
9−# @parse url, title, content, publishedDate, thumbnail
10−#
11−# @todo set content-parameter with correct data
12−# @todo embedded (needs some md5 from video page)
13
5
14from json import loads
6from json import loads
15from html import unescape
7from html import unescape
@@ -18,6 +10,15 @@ from lxml import html
18from dateutil import parser
10from dateutil import parser
19from searx.utils import extract_text
11from searx.utils import extract_text
20
12
13+# about
14+about = {
15+ "website": 'https://www.ina.fr/',
16+ "wikidata_id": 'Q1665109',
17+ "official_api_documentation": None,
18+ "use_official_api": False,
19+ "require_api_key": False,
20+ "results": 'HTML',
21+}
21
22
22# engine dependent config
23# engine dependent config
23categories = ['videos']
24categories = ['videos']
searx/engines/invidious.py+14−9
@@ -1,17 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Invidious (Videos)
3+ Invidious (Videos)
2−#
4+"""
3−# @website https://invidio.us/
4−# @provide-api yes (https://github.com/omarroth/invidious/wiki/API)
5−#
6−# @using-api yes
7−# @results JSON
8−# @stable yes
9−# @parse url, title, content, publishedDate, thumbnail, embedded, author, length
10
5
11from urllib.parse import quote_plus
6from urllib.parse import quote_plus
12from dateutil import parser
7from dateutil import parser
13import time
8import time
14
9
10+# about
11+about = {
12+ "website": 'https://instances.invidio.us/',
13+ "wikidata_id": 'Q79343316',
14+ "official_api_documentation": 'https://github.com/omarroth/invidious/wiki/API',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
15# engine dependent config
20# engine dependent config
16categories = ["videos", "music"]
21categories = ["videos", "music"]
17paging = True
22paging = True
searx/engines/json_engine.py+2
@@ -1,3 +1,5 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
1from collections.abc import Iterable
3from collections.abc import Iterable
2from json import loads
4from json import loads
3from urllib.parse import urlencode
5from urllib.parse import urlencode
searx/engines/kickass.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Kickass Torrent (Videos, Music, Files)
3 Kickass Torrent (Videos, Music, Files)
3−
4− @website https://kickass.so
5− @provide-api no (nothing found)
6−
7− @using-api no
8− @results HTML (using search portal)
9− @stable yes (HTML can change)
10− @parse url, title, content, seed, leech, magnetlink
11"""
4"""
12
5
13from lxml import html
6from lxml import html
@@ -15,6 +8,16 @@ from operator import itemgetter
15from urllib.parse import quote, urljoin
8from urllib.parse import quote, urljoin
16from searx.utils import extract_text, get_torrent_size, convert_str_to_int
9from searx.utils import extract_text, get_torrent_size, convert_str_to_int
17
10
11+# about
12+about = {
13+ "website": 'https://kickass.so',
14+ "wikidata_id": 'Q17062285',
15+ "official_api_documentation": None,
16+ "use_official_api": False,
17+ "require_api_key": False,
18+ "results": 'HTML',
19+}
20+
18# engine dependent config
21# engine dependent config
19categories = ['videos', 'music', 'files']
22categories = ['videos', 'music', 'files']
20paging = True
23paging = True
searx/engines/mediawiki.py+12−11
@@ -1,21 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2− general mediawiki-engine (Web)
3+ General mediawiki-engine (Web)
3−
4− @website websites built on mediawiki (https://www.mediawiki.org)
5− @provide-api yes (http://www.mediawiki.org/wiki/API:Search)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title
11−
12− @todo content
13"""
4"""
14
5
15from json import loads
6from json import loads
16from string import Formatter
7from string import Formatter
17from urllib.parse import urlencode, quote
8from urllib.parse import urlencode, quote
18
9
10+# about
11+about = {
12+ "website": None,
13+ "wikidata_id": None,
14+ "official_api_documentation": 'http://www.mediawiki.org/wiki/API:Search',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
19# engine dependent config
20# engine dependent config
20categories = ['general']
21categories = ['general']
21language_support = True
22language_support = True
searx/engines/microsoft_academic.py+12−8
@@ -1,12 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2−Microsoft Academic (Science)
3+ Microsoft Academic (Science)
3−
4−@website https://academic.microsoft.com
5−@provide-api yes
6−@using-api no
7−@results JSON
8−@stable no
9−@parse url, title, content
10"""
4"""
11
5
12from datetime import datetime
6from datetime import datetime
@@ -15,6 +9,16 @@ from uuid import uuid4
15from urllib.parse import urlencode
9from urllib.parse import urlencode
16from searx.utils import html_to_text
10from searx.utils import html_to_text
17
11
12+# about
13+about = {
14+ "website": 'https://academic.microsoft.com',
15+ "wikidata_id": 'Q28136779',
16+ "official_api_documentation": 'http://ma-graph.org/',
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'JSON',
20+}
21+
18categories = ['images']
22categories = ['images']
19paging = True
23paging = True
20result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}'
24result_url = 'https://academic.microsoft.com/api/search/GetEntityResults?{query}'
searx/engines/mixcloud.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Mixcloud (Music)
3 Mixcloud (Music)
3−
4− @website https://http://www.mixcloud.com/
5− @provide-api yes (http://www.mixcloud.com/developers/
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, embedded, publishedDate
11"""
4"""
12
5
13from json import loads
6from json import loads
14from dateutil import parser
7from dateutil import parser
15from urllib.parse import urlencode
8from urllib.parse import urlencode
16
9
10+# about
11+about = {
12+ "website": 'https://www.mixcloud.com/',
13+ "wikidata_id": 'Q6883832',
14+ "official_api_documentation": 'http://www.mixcloud.com/developers/',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['music']
21categories = ['music']
19paging = True
22paging = True
searx/engines/not_evil.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 not Evil (Onions)
3 not Evil (Onions)
3−
4− @website http://hss3uro2hsxfogfq.onion
5− @provide-api yes (http://hss3uro2hsxfogfq.onion/api.htm)
6−
7− @using-api no
8− @results HTML
9− @stable no
10− @parse url, title, content
11"""
4"""
12
5
13from urllib.parse import urlencode
6from urllib.parse import urlencode
14from lxml import html
7from lxml import html
15from searx.engines.xpath import extract_text
8from searx.engines.xpath import extract_text
16
9
10+# about
11+about = {
12+ "website": 'http://hss3uro2hsxfogfq.onion',
13+ "wikidata_id": None,
14+ "official_api_documentation": 'http://hss3uro2hsxfogfq.onion/api.htm',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['onions']
21categories = ['onions']
19paging = True
22paging = True
searx/engines/nyaa.py+11−7
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Nyaa.si (Anime Bittorrent tracker)
3 Nyaa.si (Anime Bittorrent tracker)
3−
4− @website https://nyaa.si/
5− @provide-api no
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, content, seed, leech, torrentfile
10"""
4"""
11
5
12from lxml import html
6from lxml import html
13from urllib.parse import urlencode
7from urllib.parse import urlencode
14from searx.utils import extract_text, get_torrent_size, int_or_zero
8from searx.utils import extract_text, get_torrent_size, int_or_zero
15
9
10+# about
11+about = {
12+ "website": 'https://nyaa.si/',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
16# engine dependent config
20# engine dependent config
17categories = ['files', 'images', 'videos', 'music']
21categories = ['files', 'images', 'videos', 'music']
18paging = True
22paging = True
searx/engines/opensemantic.py+13−9
@@ -1,18 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2−Open Semantic Search
3+ Open Semantic Search
3−
4− @website https://www.opensemanticsearch.org/
5− @provide-api yes (https://www.opensemanticsearch.org/dev)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, publishedDate
11"""
4"""
5+
12from dateutil import parser
6from dateutil import parser
13from json import loads
7from json import loads
14from urllib.parse import quote
8from urllib.parse import quote
15
9
10+# about
11+about = {
12+ "website": 'https://www.opensemanticsearch.org/',
13+ "wikidata_id": None,
14+ "official_api_documentation": 'https://www.opensemanticsearch.org/dev',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
16base_url = 'http://localhost:8983/solr/opensemanticsearch/'
20base_url = 'http://localhost:8983/solr/opensemanticsearch/'
17search_string = 'query?q={query}'
21search_string = 'query?q={query}'
18
22
searx/engines/openstreetmap.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 OpenStreetMap (Map)
3 OpenStreetMap (Map)
3−
4− @website https://openstreetmap.org/
5− @provide-api yes (http://wiki.openstreetmap.org/wiki/Nominatim)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title
11"""
4"""
12
5
13import re
6import re
14from json import loads
7from json import loads
15from flask_babel import gettext
8from flask_babel import gettext
16
9
10+# about
11+about = {
12+ "website": 'https://www.openstreetmap.org/',
13+ "wikidata_id": 'Q936',
14+ "official_api_documentation": 'http://wiki.openstreetmap.org/wiki/Nominatim',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['map']
21categories = ['map']
19paging = False
22paging = False
searx/engines/pdbe.py+11−8
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 PDBe (Protein Data Bank in Europe)
3 PDBe (Protein Data Bank in Europe)
3−
4− @website https://www.ebi.ac.uk/pdbe
5− @provide-api yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),
6− unlimited
7− @using-api yes
8− @results python dictionary (from json)
9− @stable yes
10− @parse url, title, content, img_src
11"""
4"""
12
5
13from json import loads
6from json import loads
14from flask_babel import gettext
7from flask_babel import gettext
15
8
9+# about
10+about = {
11+ "website": 'https://www.ebi.ac.uk/pdbe',
12+ "wikidata_id": 'Q55823905',
13+ "official_api_documentation": 'https://www.ebi.ac.uk/pdbe/api/doc/search.html',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
18+
16categories = ['science']
19categories = ['science']
17
20
18hide_obsolete = False
21hide_obsolete = False
searx/engines/peertube.py+11−10
@@ -1,15 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 peertube (Videos)
3 peertube (Videos)
3−
4− @website https://www.peertube.live
5− @provide-api yes (https://docs.joinpeertube.org/api-rest-reference.html)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, thumbnail, publishedDate, embedded
11−
12− @todo implement time range support
13"""
4"""
14
5
15from json import loads
6from json import loads
@@ -17,6 +8,16 @@ from datetime import datetime
17from urllib.parse import urlencode
8from urllib.parse import urlencode
18from searx.utils import html_to_text
9from searx.utils import html_to_text
19
10
11+# about
12+about = {
13+ "website": 'https://joinpeertube.org',
14+ "wikidata_id": 'Q50938515',
15+ "official_api_documentation": 'https://docs.joinpeertube.org/api-rest-reference.html',
16+ "use_official_api": True,
17+ "require_api_key": False,
18+ "results": 'JSON',
19+}
20+
20# engine dependent config
21# engine dependent config
21categories = ["videos"]
22categories = ["videos"]
22paging = True
23paging = True
searx/engines/photon.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Photon (Map)
3 Photon (Map)
3−
4− @website https://photon.komoot.de
5− @provide-api yes (https://photon.komoot.de/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title
11"""
4"""
12
5
13from json import loads
6from json import loads
14from urllib.parse import urlencode
7from urllib.parse import urlencode
15from searx.utils import searx_useragent
8from searx.utils import searx_useragent
16
9
10+# about
11+about = {
12+ "website": 'https://photon.komoot.de',
13+ "wikidata_id": None,
14+ "official_api_documentation": 'https://photon.komoot.de/',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['map']
21categories = ['map']
19paging = False
22paging = False
searx/engines/piratebay.py+14−9
@@ -1,12 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Piratebay (Videos, Music, Files)
3+ Piratebay (Videos, Music, Files)
2−#
4+"""
3−# @website https://thepiratebay.org
4−# @provide-api yes (https://apibay.org/)
5−#
6−# @using-api yes
7−# @results JSON
8−# @stable no (the API is not documented nor versioned)
9−# @parse url, title, seed, leech, magnetlink, filesize, publishedDate
10
5
11from json import loads
6from json import loads
12from datetime import datetime
7from datetime import datetime
@@ -15,6 +10,16 @@ from operator import itemgetter
15from urllib.parse import quote
10from urllib.parse import quote
16from searx.utils import get_torrent_size
11from searx.utils import get_torrent_size
17
12
13+# about
14+about = {
15+ "website": 'https://thepiratebay.org',
16+ "wikidata_id": 'Q22663',
17+ "official_api_documentation": 'https://apibay.org/',
18+ "use_official_api": True,
19+ "require_api_key": False,
20+ "results": 'JSON',
21+}
22+
18# engine dependent config
23# engine dependent config
19categories = ["videos", "music", "files"]
24categories = ["videos", "music", "files"]
20
25
searx/engines/pubmed.py+13−9
@@ -1,14 +1,6 @@
1−#!/usr/bin/env python
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−
3"""
2"""
4 PubMed (Scholar publications)
3 PubMed (Scholar publications)
5− @website https://www.ncbi.nlm.nih.gov/pubmed/
6− @provide-api yes (https://www.ncbi.nlm.nih.gov/home/develop/api/)
7− @using-api yes
8− @results XML
9− @stable yes
10− @parse url, title, publishedDate, content
11− More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/
12"""
4"""
13
5
14from flask_babel import gettext
6from flask_babel import gettext
@@ -17,6 +9,18 @@ from datetime import datetime
17from urllib.parse import urlencode
9from urllib.parse import urlencode
18from searx.poolrequests import get
10from searx.poolrequests import get
19
11
12+# about
13+about = {
14+ "website": 'https://www.ncbi.nlm.nih.gov/pubmed/',
15+ "wikidata_id": 'Q1540899',
16+ "official_api_documentation": {
17+ 'url': 'https://www.ncbi.nlm.nih.gov/home/develop/api/',
18+ 'comment': 'More info on api: https://www.ncbi.nlm.nih.gov/books/NBK25501/'
19+ },
20+ "use_official_api": True,
21+ "require_api_key": False,
22+ "results": 'XML',
23+}
20
24
21categories = ['science']
25categories = ['science']
22
26
searx/engines/qwant.py+10−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Qwant (Web, Images, News, Social)
3 Qwant (Web, Images, News, Social)
3−
4− @website https://qwant.com/
5− @provide-api not officially (https://api.qwant.com/api/search/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content
11"""
4"""
12
5
13from datetime import datetime
6from datetime import datetime
@@ -17,6 +10,15 @@ from searx.utils import html_to_text, match_language
17from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
10from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
18from searx.raise_for_httperror import raise_for_httperror
11from searx.raise_for_httperror import raise_for_httperror
19
12
13+# about
14+about = {
15+ "website": 'https://www.qwant.com/',
16+ "wikidata_id": 'Q14657870',
17+ "official_api_documentation": None,
18+ "use_official_api": True,
19+ "require_api_key": False,
20+ "results": 'JSON',
21+}
20
22
21# engine dependent config
23# engine dependent config
22categories = []
24categories = []
searx/engines/recoll.py+11−6
@@ -1,17 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Recoll (local search engine)
3 Recoll (local search engine)
3−
4− @using-api yes
5− @results JSON
6− @stable yes
7− @parse url, content, size, abstract, author, mtype, subtype, time, \
8− filename, label, type, embedded
9"""
4"""
10
5
11from datetime import date, timedelta
6from datetime import date, timedelta
12from json import loads
7from json import loads
13from urllib.parse import urlencode, quote
8from urllib.parse import urlencode, quote
14
9
10+# about
11+about = {
12+ "website": None,
13+ "wikidata_id": 'Q15735774',
14+ "official_api_documentation": 'https://www.lesbonscomptes.com/recoll/',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
15# engine dependent config
20# engine dependent config
16time_range_support = True
21time_range_support = True
17
22
searx/engines/reddit.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Reddit
3 Reddit
3−
4− @website https://www.reddit.com/
5− @provide-api yes (https://www.reddit.com/dev/api)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, thumbnail, publishedDate
11"""
4"""
12
5
13import json
6import json
14from datetime import datetime
7from datetime import datetime
15from urllib.parse import urlencode, urljoin, urlparse
8from urllib.parse import urlencode, urljoin, urlparse
16
9
10+# about
11+about = {
12+ "website": 'https://www.reddit.com/',
13+ "wikidata_id": 'Q1136',
14+ "official_api_documentation": 'https://www.reddit.com/dev/api',
15+ "use_official_api": True,
16+ "require_api_key": False,
17+ "results": 'JSON',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['general', 'images', 'news', 'social media']
21categories = ['general', 'images', 'news', 'social media']
19page_size = 25
22page_size = 25
searx/engines/scanr_structures.py+11−8
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 ScanR Structures (Science)
3 ScanR Structures (Science)
3−
4− @website https://scanr.enseignementsup-recherche.gouv.fr
5− @provide-api yes (https://scanr.enseignementsup-recherche.gouv.fr/api/swagger-ui.html)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, img_src
11"""
4"""
12
5
13from json import loads, dumps
6from json import loads, dumps
14from searx.utils import html_to_text
7from searx.utils import html_to_text
15
8
9+# about
10+about = {
11+ "website": 'https://scanr.enseignementsup-recherche.gouv.fr',
12+ "wikidata_id": 'Q44105684',
13+ "official_api_documentation": 'https://scanr.enseignementsup-recherche.gouv.fr/opendata',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
18+
16# engine dependent config
19# engine dependent config
17categories = ['science']
20categories = ['science']
18paging = True
21paging = True
searx/engines/searchcode_code.py+11−9
@@ -1,18 +1,20 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2− Searchcode (It)
3+ Searchcode (IT)
3−
4− @website https://searchcode.com/
5− @provide-api yes (https://searchcode.com/api/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content
11"""
4"""
12
5
13from json import loads
6from json import loads
14from urllib.parse import urlencode
7from urllib.parse import urlencode
15
8
9+# about
10+about = {
11+ "website": 'https://searchcode.com/',
12+ "wikidata_id": None,
13+ "official_api_documentation": 'https://searchcode.com/api/',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
16
18
17# engine dependent config
19# engine dependent config
18categories = ['it']
20categories = ['it']
searx/engines/searx_engine.py+10−8
@@ -1,18 +1,20 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Searx (all)
3 Searx (all)
3−
4− @website https://github.com/searx/searx
5− @provide-api yes (https://searx.github.io/searx/dev/search_api.html)
6−
7− @using-api yes
8− @results JSON
9− @stable yes (using api)
10− @parse url, title, content
11"""
4"""
12
5
13from json import loads
6from json import loads
14from searx.engines import categories as searx_categories
7from searx.engines import categories as searx_categories
15
8
9+# about
10+about = {
11+ "website": 'https://github.com/searx/searx',
12+ "wikidata_id": 'Q17639196',
13+ "official_api_documentation": 'https://searx.github.io/searx/dev/search_api.html',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
16
18
17categories = searx_categories.keys()
19categories = searx_categories.keys()
18
20
searx/engines/sepiasearch.py+14−8
@@ -1,17 +1,23 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# SepiaSearch (Videos)
3+ SepiaSearch (Videos)
2−#
4+"""
3−# @website https://sepiasearch.org
4−# @provide-api https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api
5−# @using-api yes
6−# @results JSON
7−# @stable yes
8−# @parse url, title, content, publishedDate, thumbnail
9
5
10from json import loads
6from json import loads
11from dateutil import parser, relativedelta
7from dateutil import parser, relativedelta
12from urllib.parse import urlencode
8from urllib.parse import urlencode
13from datetime import datetime
9from datetime import datetime
14
10
11+# about
12+about = {
13+ "website": 'https://sepiasearch.org',
14+ "wikidata_id": None,
15+ "official_api_documentation": "https://framagit.org/framasoft/peertube/search-index/-/tree/master/server/controllers/api", # NOQA
16+ "use_official_api": True,
17+ "require_api_key": False,
18+ "results": 'JSON',
19+}
20+
15categories = ['videos']
21categories = ['videos']
16paging = True
22paging = True
17language_support = True
23language_support = True
searx/engines/soundcloud.py+10−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Soundcloud (Music)
3 Soundcloud (Music)
3−
4− @website https://soundcloud.com
5− @provide-api yes (https://developers.soundcloud.com/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, publishedDate, embedded
11"""
4"""
12
5
13import re
6import re
@@ -18,6 +11,15 @@ from urllib.parse import quote_plus, urlencode
18from searx import logger
11from searx import logger
19from searx.poolrequests import get as http_get
12from searx.poolrequests import get as http_get
20
13
14+# about
15+about = {
16+ "website": 'https://soundcloud.com',
17+ "wikidata_id": 'Q568769',
18+ "official_api_documentation": 'https://developers.soundcloud.com/',
19+ "use_official_api": True,
20+ "require_api_key": False,
21+ "results": 'JSON',
22+}
21
23
22# engine dependent config
24# engine dependent config
23categories = ['music']
25categories = ['music']
searx/engines/spotify.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Spotify (Music)
3 Spotify (Music)
3−
4− @website https://spotify.com
5− @provide-api yes (https://developer.spotify.com/web-api/search-item/)
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, title, content, embedded
11"""
4"""
12
5
13from json import loads
6from json import loads
@@ -15,6 +8,16 @@ from urllib.parse import urlencode
15import requests
8import requests
16import base64
9import base64
17
10
11+# about
12+about = {
13+ "website": 'https://www.spotify.com',
14+ "wikidata_id": 'Q689141',
15+ "official_api_documentation": 'https://developer.spotify.com/web-api/search-item/',
16+ "use_official_api": True,
17+ "require_api_key": False,
18+ "results": 'JSON',
19+}
20+
18# engine dependent config
21# engine dependent config
19categories = ['music']
22categories = ['music']
20paging = True
23paging = True
searx/engines/stackoverflow.py+12−9
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2− Stackoverflow (It)
3+ Stackoverflow (IT)
3−
4− @website https://stackoverflow.com/
5− @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
6−
7− @using-api no
8− @results HTML
9− @stable no (HTML can change)
10− @parse url, title, content
11"""
4"""
12
5
13from urllib.parse import urlencode, urljoin, urlparse
6from urllib.parse import urlencode, urljoin, urlparse
@@ -15,6 +8,16 @@ from lxml import html
15from searx.utils import extract_text
8from searx.utils import extract_text
16from searx.exceptions import SearxEngineCaptchaException
9from searx.exceptions import SearxEngineCaptchaException
17
10
11+# about
12+about = {
13+ "website": 'https://stackoverflow.com/',
14+ "wikidata_id": 'Q549037',
15+ "official_api_documentation": 'https://api.stackexchange.com/docs',
16+ "use_official_api": False,
17+ "require_api_key": False,
18+ "results": 'HTML',
19+}
20+
18# engine dependent config
21# engine dependent config
19categories = ['it']
22categories = ['it']
20paging = True
23paging = True
searx/engines/startpage.py+14−11
@@ -1,14 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Startpage (Web)
3+ Startpage (Web)
2−#
4+"""
3−# @website https://startpage.com
4−# @provide-api no (nothing found)
5−#
6−# @using-api no
7−# @results HTML
8−# @stable no (HTML can change)
9−# @parse url, title, content
10−#
11−# @todo paging
12
5
13from lxml import html
6from lxml import html
14from dateutil import parser
7from dateutil import parser
@@ -19,6 +12,16 @@ from babel import Locale
19from babel.localedata import locale_identifiers
12from babel.localedata import locale_identifiers
20from searx.utils import extract_text, eval_xpath, match_language
13from searx.utils import extract_text, eval_xpath, match_language
21
14
15+# about
16+about = {
17+ "website": 'https://startpage.com',
18+ "wikidata_id": 'Q2333295',
19+ "official_api_documentation": None,
20+ "use_official_api": False,
21+ "require_api_key": False,
22+ "results": 'HTML',
23+}
24+
22# engine dependent config
25# engine dependent config
23categories = ['general']
26categories = ['general']
24# there is a mechanism to block "bot" search
27# there is a mechanism to block "bot" search
searx/engines/tokyotoshokan.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Tokyo Toshokan (A BitTorrent Library for Japanese Media)
3 Tokyo Toshokan (A BitTorrent Library for Japanese Media)
3−
4− @website https://www.tokyotosho.info/
5− @provide-api no
6− @using-api no
7− @results HTML
8− @stable no (HTML can change)
9− @parse url, title, publishedDate, seed, leech,
10− filesize, magnetlink, content
11"""
4"""
12
5
13import re
6import re
@@ -16,6 +9,16 @@ from lxml import html
16from datetime import datetime
9from datetime import datetime
17from searx.utils import extract_text, get_torrent_size, int_or_zero
10from searx.utils import extract_text, get_torrent_size, int_or_zero
18
11
12+# about
13+about = {
14+ "website": 'https://www.tokyotosho.info/',
15+ "wikidata_id": None,
16+ "official_api_documentation": None,
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'HTML',
20+}
21+
19# engine dependent config
22# engine dependent config
20categories = ['files', 'videos', 'music']
23categories = ['files', 'videos', 'music']
21paging = True
24paging = True
searx/engines/torrentz.py+11−9
@@ -1,14 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Torrentz2.is (BitTorrent meta-search engine)
3 Torrentz2.is (BitTorrent meta-search engine)
3−
4− @website https://torrentz2.is/
5− @provide-api no
6−
7− @using-api no
8− @results HTML
9− @stable no (HTML can change, although unlikely,
10− see https://torrentz.is/torrentz.btsearch)
11− @parse url, title, publishedDate, seed, leech, filesize, magnetlink
12"""
4"""
13
5
14import re
6import re
@@ -17,6 +9,16 @@ from lxml import html
17from datetime import datetime
9from datetime import datetime
18from searx.utils import extract_text, get_torrent_size
10from searx.utils import extract_text, get_torrent_size
19
11
12+# about
13+about = {
14+ "website": 'https://torrentz2.is/',
15+ "wikidata_id": 'Q1156687',
16+ "official_api_documentation": 'https://torrentz.is/torrentz.btsearch',
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'HTML',
20+}
21+
20# engine dependent config
22# engine dependent config
21categories = ['files', 'videos', 'music']
23categories = ['files', 'videos', 'music']
22paging = True
24paging = True
searx/engines/translated.py+11−7
@@ -1,14 +1,18 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 MyMemory Translated
3 MyMemory Translated
3−
4− @website https://mymemory.translated.net/
5− @provide-api yes (https://mymemory.translated.net/doc/spec.php)
6− @using-api yes
7− @results JSON
8− @stable yes
9− @parse url, title, content
10"""
4"""
11
5
6+# about
7+about = {
8+ "website": 'https://mymemory.translated.net/',
9+ "wikidata_id": None,
10+ "official_api_documentation": 'https://mymemory.translated.net/doc/spec.php',
11+ "use_official_api": True,
12+ "require_api_key": False,
13+ "results": 'JSON',
14+}
15+
12engine_type = 'online_dictionnary'
16engine_type = 'online_dictionnary'
13categories = ['general']
17categories = ['general']
14url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
18url = 'https://api.mymemory.translated.net/get?q={query}&langpair={from_lang}|{to_lang}{key}'
searx/engines/unsplash.py+11−8
@@ -1,18 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Unsplash
3 Unsplash
3−
4− @website https://unsplash.com
5− @provide-api yes (https://unsplash.com/developers)
6−
7− @using-api no
8− @results JSON (using search portal's infiniscroll API)
9− @stable no (JSON format could change any time)
10− @parse url, title, img_src, thumbnail_src
11"""
4"""
12
5
13from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl
6from urllib.parse import urlencode, urlparse, urlunparse, parse_qsl
14from json import loads
7from json import loads
15
8
9+# about
10+about = {
11+ "website": 'https://unsplash.com',
12+ "wikidata_id": 'Q28233552',
13+ "official_api_documentation": 'https://unsplash.com/developers',
14+ "use_official_api": False,
15+ "require_api_key": False,
16+ "results": 'JSON',
17+}
18+
16url = 'https://unsplash.com/'
19url = 'https://unsplash.com/'
17search_url = url + 'napi/search/photos?'
20search_url = url + 'napi/search/photos?'
18categories = ['images']
21categories = ['images']
searx/engines/vimeo.py+14−13
@@ -1,21 +1,22 @@
1−# Vimeo (Videos)
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2−#
2+"""
3−# @website https://vimeo.com/
3+ Wikipedia (Web
4−# @provide-api yes (http://developer.vimeo.com/api),
4+"""
5−# they have a maximum count of queries/hour
6−#
7−# @using-api no (TODO, rewrite to api)
8−# @results HTML (using search portal)
9−# @stable no (HTML can change)
10−# @parse url, title, publishedDate, thumbnail, embedded
11−#
12−# @todo rewrite to api
13−# @todo set content-parameter with correct data
14
5
15from urllib.parse import urlencode
6from urllib.parse import urlencode
16from json import loads
7from json import loads
17from dateutil import parser
8from dateutil import parser
18
9
10+# about
11+about = {
12+ "website": 'https://vimeo.com/',
13+ "wikidata_id": 'Q156376',
14+ "official_api_documentation": 'http://developer.vimeo.com/api',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
19# engine dependent config
20# engine dependent config
20categories = ['videos']
21categories = ['videos']
21paging = True
22paging = True
searx/engines/wikidata.py+11−9
@@ -1,14 +1,6 @@
1−# -*- coding: utf-8 -*-
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2"""
2"""
3 Wikidata
3 Wikidata
4−
5− @website https://wikidata.org
6− @provide-api yes (https://query.wikidata.org/)
7−
8− @using-api yes
9− @results JSON
10− @stable yes
11− @parse url, infobox
12"""
4"""
13
5
14
6
@@ -27,6 +19,16 @@ from searx.engines.wikipedia import _fetch_supported_languages, supported_langua
27
19
28logger = logger.getChild('wikidata')
20logger = logger.getChild('wikidata')
29
21
22+# about
23+about = {
24+ "website": 'https://wikidata.org/',
25+ "wikidata_id": 'Q2013',
26+ "official_api_documentation": 'https://query.wikidata.org/',
27+ "use_official_api": True,
28+ "require_api_key": False,
29+ "results": 'JSON',
30+}
31+
30# SPARQL
32# SPARQL
31SPARQL_ENDPOINT_URL = 'https://query.wikidata.org/sparql'
33SPARQL_ENDPOINT_URL = 'https://query.wikidata.org/sparql'
32SPARQL_EXPLAIN_URL = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain'
34SPARQL_EXPLAIN_URL = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql?explain'
searx/engines/wikipedia.py+11−8
@@ -1,13 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Wikipedia (Web)
3 Wikipedia (Web)
3−
4− @website https://en.wikipedia.org/api/rest_v1/
5− @provide-api yes
6−
7− @using-api yes
8− @results JSON
9− @stable yes
10− @parse url, infobox
11"""
4"""
12
5
13from urllib.parse import quote
6from urllib.parse import quote
@@ -16,6 +9,16 @@ from lxml.html import fromstring
16from searx.utils import match_language, searx_useragent
9from searx.utils import match_language, searx_useragent
17from searx.raise_for_httperror import raise_for_httperror
10from searx.raise_for_httperror import raise_for_httperror
18
11
12+# about
13+about = {
14+ "website": 'https://www.wikipedia.org/',
15+ "wikidata_id": 'Q52',
16+ "official_api_documentation": 'https://en.wikipedia.org/api/',
17+ "use_official_api": True,
18+ "require_api_key": False,
19+ "results": 'JSON',
20+}
21+
19# search-url
22# search-url
20search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
23search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
21supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
24supported_languages_url = 'https://meta.wikimedia.org/wiki/List_of_Wikipedias'
searx/engines/wolframalpha_api.py+14−9
@@ -1,16 +1,21 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Wolfram Alpha (Science)
3+ Wolfram|Alpha (Science)
2−#
4+"""
3−# @website https://www.wolframalpha.com
4−# @provide-api yes (https://api.wolframalpha.com/v2/)
5−#
6−# @using-api yes
7−# @results XML
8−# @stable yes
9−# @parse url, infobox
10
5
11from lxml import etree
6from lxml import etree
12from urllib.parse import urlencode
7from urllib.parse import urlencode
13
8
9+# about
10+about = {
11+ "website": 'https://www.wolframalpha.com',
12+ "wikidata_id": 'Q207006',
13+ "official_api_documentation": 'https://products.wolframalpha.com/api/',
14+ "use_official_api": True,
15+ "require_api_key": False,
16+ "results": 'XML',
17+}
18+
14# search-url
19# search-url
15search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'
20search_url = 'https://api.wolframalpha.com/v2/query?appid={api_key}&{query}'
16site_url = 'https://www.wolframalpha.com/input/?{query}'
21site_url = 'https://www.wolframalpha.com/input/?{query}'
searx/engines/wolframalpha_noapi.py+14−9
@@ -1,12 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Wolfram|Alpha (Science)
3+ Wolfram|Alpha (Science)
2−#
4+"""
3−# @website https://www.wolframalpha.com/
4−# @provide-api yes (https://api.wolframalpha.com/v2/)
5−#
6−# @using-api no
7−# @results JSON
8−# @stable no
9−# @parse url, infobox
10
5
11from json import loads
6from json import loads
12from time import time
7from time import time
@@ -14,6 +9,16 @@ from urllib.parse import urlencode
14
9
15from searx.poolrequests import get as http_get
10from searx.poolrequests import get as http_get
16
11
12+# about
13+about = {
14+ "website": 'https://www.wolframalpha.com/',
15+ "wikidata_id": 'Q207006',
16+ "official_api_documentation": 'https://products.wolframalpha.com/api/',
17+ "use_official_api": False,
18+ "require_api_key": False,
19+ "results": 'JSON',
20+}
21+
17# search-url
22# search-url
18url = 'https://www.wolframalpha.com/'
23url = 'https://www.wolframalpha.com/'
19
24
searx/engines/www1x.py+11−8
@@ -1,19 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 1x (Images)
3 1x (Images)
3−
4− @website http://1x.com/
5− @provide-api no
6−
7− @using-api no
8− @results HTML
9− @stable no (HTML can change)
10− @parse url, title, thumbnail
11"""
4"""
12
5
13from lxml import html, etree
6from lxml import html, etree
14from urllib.parse import urlencode, urljoin
7from urllib.parse import urlencode, urljoin
15from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
8from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex
16
9
10+# about
11+about = {
12+ "website": 'https://1x.com/',
13+ "wikidata_id": None,
14+ "official_api_documentation": None,
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
17# engine dependent config
20# engine dependent config
18categories = ['images']
21categories = ['images']
19paging = False
22paging = False
searx/engines/xpath.py+2
@@ -1,3 +1,5 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
1from lxml import html
3from lxml import html
2from urllib.parse import urlencode
4from urllib.parse import urlencode
3from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
5from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
searx/engines/yacy.py+14−13
@@ -1,16 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Yacy (Web, Images, Videos, Music, Files)
3+ Yacy (Web, Images, Videos, Music, Files)
2−#
4+"""
3−# @website http://yacy.net
4−# @provide-api yes
5−# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
6−#
7−# @using-api yes
8−# @results JSON
9−# @stable yes
10−# @parse (general) url, title, content, publishedDate
11−# @parse (images) url, title, img_src
12−#
13−# @todo parse video, audio and file results
14
5
15from json import loads
6from json import loads
16from dateutil import parser
7from dateutil import parser
@@ -20,6 +11,16 @@ from requests.auth import HTTPDigestAuth
20
11
21from searx.utils import html_to_text
12from searx.utils import html_to_text
22
13
14+# about
15+about = {
16+ "website": 'https://yacy.net/',
17+ "wikidata_id": 'Q1759675',
18+ "official_api_documentation": 'https://wiki.yacy.net/index.php/Dev:API',
19+ "use_official_api": True,
20+ "require_api_key": False,
21+ "results": 'JSON',
22+}
23+
23# engine dependent config
24# engine dependent config
24categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
25categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
25paging = True
26paging = True
searx/engines/yahoo.py+11−9
@@ -1,20 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Yahoo (Web)
3 Yahoo (Web)
3−
4− @website https://search.yahoo.com/web
5− @provide-api yes (https://developer.yahoo.com/boss/search/),
6− $0.80/1000 queries
7−
8− @using-api no (because pricing)
9− @results HTML (using search portal)
10− @stable no (HTML can change)
11− @parse url, title, content, suggestion
12"""
4"""
13
5
14from urllib.parse import unquote, urlencode
6from urllib.parse import unquote, urlencode
15from lxml import html
7from lxml import html
16from searx.utils import extract_text, extract_url, match_language, eval_xpath
8from searx.utils import extract_text, extract_url, match_language, eval_xpath
17
9
10+# about
11+about = {
12+ "website": 'https://search.yahoo.com/',
13+ "wikidata_id": None,
14+ "official_api_documentation": 'https://developer.yahoo.com/api/',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
18# engine dependent config
20# engine dependent config
19categories = ['general']
21categories = ['general']
20paging = True
22paging = True
searx/engines/yahoo_news.py+14−10
@@ -1,13 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Yahoo (News)
3+ Yahoo (News)
2−#
4+"""
3−# @website https://news.yahoo.com
4−# @provide-api yes (https://developer.yahoo.com/boss/search/)
5−# $0.80/1000 queries
6−#
7−# @using-api no (because pricing)
8−# @results HTML (using search portal)
9−# @stable no (HTML can change)
10−# @parse url, title, content, publishedDate
11
5
12import re
6import re
13from datetime import datetime, timedelta
7from datetime import datetime, timedelta
@@ -18,6 +12,16 @@ from searx.engines.yahoo import _fetch_supported_languages, supported_languages_
18from dateutil import parser
12from dateutil import parser
19from searx.utils import extract_text, extract_url, match_language
13from searx.utils import extract_text, extract_url, match_language
20
14
15+# about
16+about = {
17+ "website": 'https://news.yahoo.com',
18+ "wikidata_id": 'Q3044717',
19+ "official_api_documentation": 'https://developer.yahoo.com/api/',
20+ "use_official_api": False,
21+ "require_api_key": False,
22+ "results": 'HTML',
23+}
24+
21# engine dependent config
25# engine dependent config
22categories = ['news']
26categories = ['news']
23paging = True
27paging = True
searx/engines/yandex.py+11−7
@@ -1,12 +1,6 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
1"""
2"""
2 Yahoo (Web)
3 Yahoo (Web)
3−
4− @website https://yandex.ru/
5− @provide-api ?
6− @using-api no
7− @results HTML (using search portal)
8− @stable no (HTML can change)
9− @parse url, title, content
10"""
4"""
11
5
12from urllib.parse import urlencode, urlparse
6from urllib.parse import urlencode, urlparse
@@ -16,6 +10,16 @@ from searx.exceptions import SearxEngineCaptchaException
16
10
17logger = logger.getChild('yandex engine')
11logger = logger.getChild('yandex engine')
18
12
13+# about
14+about = {
15+ "website": 'https://yandex.ru/',
16+ "wikidata_id": 'Q5281',
17+ "official_api_documentation": "?",
18+ "use_official_api": False,
19+ "require_api_key": False,
20+ "results": 'HTML',
21+}
22+
19# engine dependent config
23# engine dependent config
20categories = ['general']
24categories = ['general']
21paging = True
25paging = True
searx/engines/yggtorrent.py+14−9
@@ -1,12 +1,7 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Yggtorrent (Videos, Music, Files)
3+ Yggtorrent (Videos, Music, Files)
2−#
4+"""
3−# @website https://www2.yggtorrent.si
4−# @provide-api no (nothing found)
5−#
6−# @using-api no
7−# @results HTML (using search portal)
8−# @stable no (HTML can change)
9−# @parse url, title, seed, leech, publishedDate, filesize
10
5
11from lxml import html
6from lxml import html
12from operator import itemgetter
7from operator import itemgetter
@@ -15,6 +10,16 @@ from urllib.parse import quote
15from searx.utils import extract_text, get_torrent_size
10from searx.utils import extract_text, get_torrent_size
16from searx.poolrequests import get as http_get
11from searx.poolrequests import get as http_get
17
12
13+# about
14+about = {
15+ "website": 'https://www2.yggtorrent.si',
16+ "wikidata_id": None,
17+ "official_api_documentation": None,
18+ "use_official_api": False,
19+ "require_api_key": False,
20+ "results": 'HTML',
21+}
22+
18# engine dependent config
23# engine dependent config
19categories = ['videos', 'music', 'files']
24categories = ['videos', 'music', 'files']
20paging = True
25paging = True
searx/engines/youtube_api.py+14−9
@@ -1,18 +1,23 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Youtube (Videos)
3+ Youtube (Videos)
2−#
4+"""
3−# @website https://www.youtube.com/
4−# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list)
5−#
6−# @using-api yes
7−# @results JSON
8−# @stable yes
9−# @parse url, title, content, publishedDate, thumbnail, embedded
10
5
11from json import loads
6from json import loads
12from dateutil import parser
7from dateutil import parser
13from urllib.parse import urlencode
8from urllib.parse import urlencode
14from searx.exceptions import SearxEngineAPIException
9from searx.exceptions import SearxEngineAPIException
15
10
11+# about
12+about = {
13+ "website": 'https://www.youtube.com/',
14+ "wikidata_id": 'Q866',
15+ "official_api_documentation": 'https://developers.google.com/youtube/v3/docs/search/list?apix=true',
16+ "use_official_api": True,
17+ "require_api_key": False,
18+ "results": 'JSON',
19+}
20+
16# engine dependent config
21# engine dependent config
17categories = ['videos', 'music']
22categories = ['videos', 'music']
18paging = False
23paging = False
searx/engines/youtube_noapi.py+14−9
@@ -1,17 +1,22 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+"""
1−# Youtube (Videos)
3+ Youtube (Videos)
2−#
4+"""
3−# @website https://www.youtube.com/
4−# @provide-api yes (https://developers.google.com/apis-explorer/#p/youtube/v3/youtube.search.list)
5−#
6−# @using-api no
7−# @results HTML
8−# @stable no
9−# @parse url, title, content, publishedDate, thumbnail, embedded
10
5
11from functools import reduce
6from functools import reduce
12from json import loads
7from json import loads
13from urllib.parse import quote_plus
8from urllib.parse import quote_plus
14
9
10+# about
11+about = {
12+ "website": 'https://www.youtube.com/',
13+ "wikidata_id": 'Q866',
14+ "official_api_documentation": 'https://developers.google.com/youtube/v3/docs/search/list?apix=true',
15+ "use_official_api": False,
16+ "require_api_key": False,
17+ "results": 'HTML',
18+}
19+
15# engine dependent config
20# engine dependent config
16categories = ['videos', 'music']
21categories = ['videos', 'music']
17paging = True
22paging = True
searx/settings.yml+213
@@ -157,6 +157,13 @@ engines:
157 timeout : 7.0
157 timeout : 7.0
158 disabled : True
158 disabled : True
159 shortcut : ai
159 shortcut : ai
160+ about:
161+ website: https://archive.is/
162+ wikidata_id: Q13515725
163+ official_api_documentation: http://mementoweb.org/depot/native/archiveis/
164+ use_official_api: false
165+ require_api_key: false
166+ results: HTML
160
167
161 - name : arxiv
168 - name : arxiv
162 engine : arxiv
169 engine : arxiv
@@ -201,6 +208,13 @@ engines:
201 timeout : 4.0
208 timeout : 4.0
202 disabled : True
209 disabled : True
203 shortcut : bb
210 shortcut : bb
211+ about:
212+ website: https://bitbucket.org/
213+ wikidata_id: Q2493781
214+ official_api_documentation: https://developer.atlassian.com/bitbucket
215+ use_official_api: false
216+ require_api_key: false
217+ results: HTML
204
218
205 - name : btdigg
219 - name : btdigg
206 engine : btdigg
220 engine : btdigg
@@ -216,6 +230,13 @@ engines:
216 categories : videos
230 categories : videos
217 disabled : True
231 disabled : True
218 shortcut : c3tv
232 shortcut : c3tv
233+ about:
234+ website: https://media.ccc.de/
235+ wikidata_id: Q80729951
236+ official_api_documentation: https://github.com/voc/voctoweb
237+ use_official_api: false
238+ require_api_key: false
239+ results: HTML
219
240
220 - name : crossref
241 - name : crossref
221 engine : json_engine
242 engine : json_engine
@@ -226,6 +247,13 @@ engines:
226 content_query : fullCitation
247 content_query : fullCitation
227 categories : science
248 categories : science
228 shortcut : cr
249 shortcut : cr
250+ about:
251+ website: https://www.crossref.org/
252+ wikidata_id: Q5188229
253+ official_api_documentation: https://github.com/CrossRef/rest-api-doc
254+ use_official_api: false
255+ require_api_key: false
256+ results: JSON
229
257
230 - name : currency
258 - name : currency
231 engine : currency_convert
259 engine : currency_convert
@@ -271,6 +299,13 @@ engines:
271 categories : general
299 categories : general
272 shortcut : ew
300 shortcut : ew
273 disabled : True
301 disabled : True
302+ about:
303+ website: https://www.erowid.org/
304+ wikidata_id: Q1430691
305+ official_api_documentation:
306+ use_official_api: false
307+ require_api_key: false
308+ results: HTML
274
309
275# - name : elasticsearch
310# - name : elasticsearch
276# shortcut : es
311# shortcut : es
@@ -321,6 +356,13 @@ engines:
321 first_page_num : 1
356 first_page_num : 1
322 shortcut : et
357 shortcut : et
323 disabled : True
358 disabled : True
359+ about:
360+ website: https://www.etymonline.com/
361+ wikidata_id: Q1188617
362+ official_api_documentation:
363+ use_official_api: false
364+ require_api_key: false
365+ results: HTML
324
366
325# - name : ebay
367# - name : ebay
326# engine : ebay
368# engine : ebay
@@ -360,6 +402,9 @@ engines:
360 search_type : title
402 search_type : title
361 timeout : 5.0
403 timeout : 5.0
362 disabled : True
404 disabled : True
405+ about:
406+ website: https://directory.fsf.org/
407+ wikidata_id: Q2470288
363
408
364 - name : frinkiac
409 - name : frinkiac
365 engine : frinkiac
410 engine : frinkiac
@@ -394,6 +439,13 @@ engines:
394 shortcut : gl
439 shortcut : gl
395 timeout : 10.0
440 timeout : 10.0
396 disabled : True
441 disabled : True
442+ about:
443+ website: https://about.gitlab.com/
444+ wikidata_id: Q16639197
445+ official_api_documentation: https://docs.gitlab.com/ee/api/
446+ use_official_api: false
447+ require_api_key: false
448+ results: JSON
397
449
398 - name : github
450 - name : github
399 engine : github
451 engine : github
@@ -411,6 +463,13 @@ engines:
411 categories : it
463 categories : it
412 shortcut : cb
464 shortcut : cb
413 disabled : True
465 disabled : True
466+ about:
467+ website: https://codeberg.org/
468+ wikidata_id:
469+ official_api_documentation: https://try.gitea.io/api/swagger
470+ use_official_api: false
471+ require_api_key: false
472+ results: JSON
414
473
415 - name : google
474 - name : google
416 engine : google
475 engine : google
@@ -441,6 +500,13 @@ engines:
441 first_page_num : 0
500 first_page_num : 0
442 categories : science
501 categories : science
443 shortcut : gos
502 shortcut : gos
503+ about:
504+ website: https://scholar.google.com/
505+ wikidata_id: Q494817
506+ official_api_documentation:
507+ use_official_api: false
508+ require_api_key: false
509+ results: HTML
444
510
445 - name : google play apps
511 - name : google play apps
446 engine : xpath
512 engine : xpath
@@ -453,6 +519,13 @@ engines:
453 categories : files
519 categories : files
454 shortcut : gpa
520 shortcut : gpa
455 disabled : True
521 disabled : True
522+ about:
523+ website: https://play.google.com/
524+ wikidata_id: Q79576
525+ official_api_documentation:
526+ use_official_api: false
527+ require_api_key: false
528+ results: HTML
456
529
457 - name : google play movies
530 - name : google play movies
458 engine : xpath
531 engine : xpath
@@ -465,6 +538,13 @@ engines:
465 categories : videos
538 categories : videos
466 shortcut : gpm
539 shortcut : gpm
467 disabled : True
540 disabled : True
541+ about:
542+ website: https://play.google.com/
543+ wikidata_id: Q79576
544+ official_api_documentation:
545+ use_official_api: false
546+ require_api_key: false
547+ results: HTML
468
548
469 - name : google play music
549 - name : google play music
470 engine : xpath
550 engine : xpath
@@ -477,6 +557,13 @@ engines:
477 categories : music
557 categories : music
478 shortcut : gps
558 shortcut : gps
479 disabled : True
559 disabled : True
560+ about:
561+ website: https://play.google.com/
562+ wikidata_id: Q79576
563+ official_api_documentation:
564+ use_official_api: false
565+ require_api_key: false
566+ results: HTML
480
567
481 - name : geektimes
568 - name : geektimes
482 engine : xpath
569 engine : xpath
@@ -489,6 +576,13 @@ engines:
489 timeout : 4.0
576 timeout : 4.0
490 disabled : True
577 disabled : True
491 shortcut : gt
578 shortcut : gt
579+ about:
580+ website: https://geektimes.ru/
581+ wikidata_id: Q50572423
582+ official_api_documentation:
583+ use_official_api: false
584+ require_api_key: false
585+ results: HTML
492
586
493 - name : habrahabr
587 - name : habrahabr
494 engine : xpath
588 engine : xpath
@@ -501,6 +595,13 @@ engines:
501 timeout : 4.0
595 timeout : 4.0
502 disabled : True
596 disabled : True
503 shortcut : habr
597 shortcut : habr
598+ about:
599+ website: https://habr.com/
600+ wikidata_id: Q4494434
601+ official_api_documentation: https://habr.com/en/docs/help/api/
602+ use_official_api: false
603+ require_api_key: false
604+ results: HTML
504
605
505 - name : hoogle
606 - name : hoogle
506 engine : json_engine
607 engine : json_engine
@@ -513,6 +614,13 @@ engines:
513 page_size : 20
614 page_size : 20
514 categories : it
615 categories : it
515 shortcut : ho
616 shortcut : ho
617+ about:
618+ website: https://www.haskell.org/
619+ wikidata_id: Q34010
620+ official_api_documentation: https://hackage.haskell.org/api
621+ use_official_api: false
622+ require_api_key: false
623+ results: JSON
516
624
517 - name : ina
625 - name : ina
518 engine : ina
626 engine : ina
@@ -543,6 +651,13 @@ engines:
543 timeout : 7.0
651 timeout : 7.0
544 disabled : True
652 disabled : True
545 shortcut : lg
653 shortcut : lg
654+ about:
655+ website: http://libgen.rs/
656+ wikidata_id: Q22017206
657+ official_api_documentation:
658+ use_official_api: false
659+ require_api_key: false
660+ results: HTML
546
661
547 - name : lobste.rs
662 - name : lobste.rs
548 engine : xpath
663 engine : xpath
@@ -555,6 +670,13 @@ engines:
555 shortcut : lo
670 shortcut : lo
556 timeout : 3.0
671 timeout : 3.0
557 disabled: True
672 disabled: True
673+ about:
674+ website: https://lobste.rs/
675+ wikidata_id: Q60762874
676+ official_api_documentation:
677+ use_official_api: false
678+ require_api_key: false
679+ results: HTML
558
680
559 - name : metager
681 - name : metager
560 engine : xpath
682 engine : xpath
@@ -566,6 +688,13 @@ engines:
566 categories : general
688 categories : general
567 shortcut : mg
689 shortcut : mg
568 disabled : True
690 disabled : True
691+ about:
692+ website: https://metager.org/
693+ wikidata_id: Q1924645
694+ official_api_documentation:
695+ use_official_api: false
696+ require_api_key: false
697+ results: HTML
569
698
570 - name : microsoft academic
699 - name : microsoft academic
571 engine : microsoft_academic
700 engine : microsoft_academic
@@ -589,6 +718,13 @@ engines:
589 disabled: True
718 disabled: True
590 timeout: 5.0
719 timeout: 5.0
591 shortcut : npm
720 shortcut : npm
721+ about:
722+ website: https://npms.io/
723+ wikidata_id: Q7067518
724+ official_api_documentation: https://api-docs.npms.io/
725+ use_official_api: false
726+ require_api_key: false
727+ results: JSON
592
728
593# Requires Tor
729# Requires Tor
594 - name : not evil
730 - name : not evil
@@ -617,6 +753,13 @@ engines:
617 categories : science
753 categories : science
618 shortcut : oad
754 shortcut : oad
619 timeout: 5.0
755 timeout: 5.0
756+ about:
757+ website: https://www.openaire.eu/
758+ wikidata_id: Q25106053
759+ official_api_documentation: https://api.openaire.eu/
760+ use_official_api: false
761+ require_api_key: false
762+ results: JSON
620
763
621 - name : openairepublications
764 - name : openairepublications
622 engine : json_engine
765 engine : json_engine
@@ -629,6 +772,13 @@ engines:
629 categories : science
772 categories : science
630 shortcut : oap
773 shortcut : oap
631 timeout: 5.0
774 timeout: 5.0
775+ about:
776+ website: https://www.openaire.eu/
777+ wikidata_id: Q25106053
778+ official_api_documentation: https://api.openaire.eu/
779+ use_official_api: false
780+ require_api_key: false
781+ results: JSON
632
782
633# - name : opensemanticsearch
783# - name : opensemanticsearch
634# engine : opensemantic
784# engine : opensemantic
@@ -650,6 +800,13 @@ engines:
650 timeout : 4.0
800 timeout : 4.0
651 disabled : True
801 disabled : True
652 shortcut : or
802 shortcut : or
803+ about:
804+ website: https://openrepos.net/
805+ wikidata_id:
806+ official_api_documentation:
807+ use_official_api: false
808+ require_api_key: false
809+ results: HTML
653
810
654 - name : pdbe
811 - name : pdbe
655 engine : pdbe
812 engine : pdbe
@@ -768,6 +925,13 @@ engines:
768 content_xpath : .//div[@class="search-result-abstract"]
925 content_xpath : .//div[@class="search-result-abstract"]
769 shortcut : se
926 shortcut : se
770 categories : science
927 categories : science
928+ about:
929+ website: https://www.semanticscholar.org/
930+ wikidata_id: Q22908627
931+ official_api_documentation: https://api.semanticscholar.org/
932+ use_official_api: false
933+ require_api_key: false
934+ results: HTML
771
935
772# Spotify needs API credentials
936# Spotify needs API credentials
773# - name : spotify
937# - name : spotify
@@ -876,6 +1040,9 @@ engines:
876 number_of_results : 5
1040 number_of_results : 5
877 search_type : text
1041 search_type : text
878 disabled : True
1042 disabled : True
1043+ about:
1044+ website: https://www.wikibooks.org/
1045+ wikidata_id: Q367
879
1046
880 - name : wikinews
1047 - name : wikinews
881 engine : mediawiki
1048 engine : mediawiki
@@ -885,6 +1052,9 @@ engines:
885 number_of_results : 5
1052 number_of_results : 5
886 search_type : text
1053 search_type : text
887 disabled : True
1054 disabled : True
1055+ about:
1056+ website: https://www.wikinews.org/
1057+ wikidata_id: Q964
888
1058
889 - name : wikiquote
1059 - name : wikiquote
890 engine : mediawiki
1060 engine : mediawiki
@@ -896,6 +1066,9 @@ engines:
896 disabled : True
1066 disabled : True
897 additional_tests:
1067 additional_tests:
898 rosebud: *test_rosebud
1068 rosebud: *test_rosebud
1069+ about:
1070+ website: https://www.wikiquote.org/
1071+ wikidata_id: Q369
899
1072
900 - name : wikisource
1073 - name : wikisource
901 engine : mediawiki
1074 engine : mediawiki
@@ -905,6 +1078,9 @@ engines:
905 number_of_results : 5
1078 number_of_results : 5
906 search_type : text
1079 search_type : text
907 disabled : True
1080 disabled : True
1081+ about:
1082+ website: https://www.wikisource.org/
1083+ wikidata_id: Q263
908
1084
909 - name : wiktionary
1085 - name : wiktionary
910 engine : mediawiki
1086 engine : mediawiki
@@ -914,6 +1090,9 @@ engines:
914 number_of_results : 5
1090 number_of_results : 5
915 search_type : text
1091 search_type : text
916 disabled : True
1092 disabled : True
1093+ about:
1094+ website: https://www.wiktionary.org/
1095+ wikidata_id: Q151
917
1096
918 - name : wikiversity
1097 - name : wikiversity
919 engine : mediawiki
1098 engine : mediawiki
@@ -923,6 +1102,9 @@ engines:
923 number_of_results : 5
1102 number_of_results : 5
924 search_type : text
1103 search_type : text
925 disabled : True
1104 disabled : True
1105+ about:
1106+ website: https://www.wikiversity.org/
1107+ wikidata_id: Q370
926
1108
927 - name : wikivoyage
1109 - name : wikivoyage
928 engine : mediawiki
1110 engine : mediawiki
@@ -932,6 +1114,9 @@ engines:
932 number_of_results : 5
1114 number_of_results : 5
933 search_type : text
1115 search_type : text
934 disabled : True
1116 disabled : True
1117+ about:
1118+ website: https://www.wikivoyage.org/
1119+ wikidata_id: Q373
935
1120
936 - name : wolframalpha
1121 - name : wolframalpha
937 shortcut : wa
1122 shortcut : wa
@@ -979,6 +1164,13 @@ engines:
979 first_page_num : 0
1164 first_page_num : 0
980 page_size : 10
1165 page_size : 10
981 disabled : True
1166 disabled : True
1167+ about:
1168+ website: https://www.seznam.cz/
1169+ wikidata_id: Q3490485
1170+ official_api_documentation: https://api.sklik.cz/
1171+ use_official_api: false
1172+ require_api_key: false
1173+ results: HTML
982
1174
983 - name : mojeek
1175 - name : mojeek
984 shortcut: mjk
1176 shortcut: mjk
@@ -993,6 +1185,13 @@ engines:
993 first_page_num : 0
1185 first_page_num : 0
994 page_size : 10
1186 page_size : 10
995 disabled : True
1187 disabled : True
1188+ about:
1189+ website: https://www.mojeek.com/
1190+ wikidata_id: Q60747299
1191+ official_api_documentation: https://www.mojeek.com/services/api.html/
1192+ use_official_api: false
1193+ require_api_key: false
1194+ results: HTML
996
1195
997 - name : naver
1196 - name : naver
998 shortcut: nvr
1197 shortcut: nvr
@@ -1007,6 +1206,13 @@ engines:
1007 first_page_num : 1
1206 first_page_num : 1
1008 page_size : 10
1207 page_size : 10
1009 disabled : True
1208 disabled : True
1209+ about:
1210+ website: https://www.naver.com/
1211+ wikidata_id: Q485639
1212+ official_api_documentation: https://developers.naver.com/docs/nmt/examples/
1213+ use_official_api: false
1214+ require_api_key: false
1215+ results: HTML
1010
1216
1011 - name : rubygems
1217 - name : rubygems
1012 shortcut: rbg
1218 shortcut: rbg
@@ -1021,6 +1227,13 @@ engines:
1021 first_page_num : 1
1227 first_page_num : 1
1022 categories: it
1228 categories: it
1023 disabled : True
1229 disabled : True
1230+ about:
1231+ website: https://rubygems.org/
1232+ wikidata_id: Q1853420
1233+ official_api_documentation: https://guides.rubygems.org/rubygems-org-api/
1234+ use_official_api: false
1235+ require_api_key: false
1236+ results: HTML
1024
1237
1025 - name : peertube
1238 - name : peertube
1026 engine: peertube
1239 engine: peertube
utils/fetch_engine_descriptions.pynew file+206
@@ -0,0 +1,206 @@
1+#!/usr/bin/env python
2+
3+import sys
4+import json
5+from urllib.parse import quote, urlparse
6+from os.path import realpath, dirname
7+import cld3
8+from lxml.html import fromstring
9+
10+# set path
11+sys.path.append(realpath(dirname(realpath(__file__)) + '/../'))
12+
13+from searx.engines.wikidata import send_wikidata_query
14+from searx.utils import extract_text
15+import searx
16+import searx.search
17+import searx.poolrequests
18+
19+SPARQL_WIKIPEDIA_ARTICLE = """
20+SELECT DISTINCT ?item ?name
21+WHERE {
22+ VALUES ?item { %IDS% }
23+ ?article schema:about ?item ;
24+ schema:inLanguage ?lang ;
25+ schema:name ?name ;
26+ schema:isPartOf [ wikibase:wikiGroup "wikipedia" ] .
27+ FILTER(?lang in (%LANGUAGES_SPARQL%)) .
28+ FILTER (!CONTAINS(?name, ':')) .
29+}
30+"""
31+
32+SPARQL_DESCRIPTION = """
33+SELECT DISTINCT ?item ?itemDescription
34+WHERE {
35+ VALUES ?item { %IDS% }
36+ ?item schema:description ?itemDescription .
37+ FILTER (lang(?itemDescription) in (%LANGUAGES_SPARQL%))
38+}
39+ORDER BY ?itemLang
40+"""
41+
42+LANGUAGES = searx.settings['locales'].keys()
43+LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
44+IDS = None
45+
46+descriptions = {}
47+wd_to_engine_name = {}
48+
49+
50+def normalize_description(description):
51+ for c in [chr(c) for c in range(0, 31)]:
52+ description = description.replace(c, ' ')
53+ description = ' '.join(description.strip().split())
54+ return description
55+
56+
57+def update_description(engine_name, lang, description, source, replace=True):
58+ if replace or lang not in descriptions[engine_name]:
59+ descriptions[engine_name][lang] = [normalize_description(description), source]
60+
61+
62+def get_wikipedia_summary(language, pageid):
63+ search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
64+ url = search_url.format(title=quote(pageid), language=language)
65+ try:
66+ response = searx.poolrequests.get(url)
67+ response.raise_for_status()
68+ api_result = json.loads(response.text)
69+ return api_result.get('extract')
70+ except:
71+ return None
72+
73+
74+def detect_language(text):
75+ r = cld3.get_language(str(text)) # pylint: disable=E1101
76+ if r is not None and r.probability >= 0.98 and r.is_reliable:
77+ return r.language
78+ return None
79+
80+
81+def get_website_description(url, lang1, lang2=None):
82+ headers = {
83+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0',
84+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
85+ 'DNT': '1',
86+ 'Upgrade-Insecure-Requests': '1',
87+ 'Sec-GPC': '1',
88+ 'Cache-Control': 'max-age=0',
89+ }
90+ if lang1 is not None:
91+ lang_list = [lang1]
92+ if lang2 is not None:
93+ lang_list.append(lang2)
94+ headers['Accept-Language'] = f'{",".join(lang_list)};q=0.8'
95+ try:
96+ response = searx.poolrequests.get(url, headers=headers, timeout=10)
97+ response.raise_for_status()
98+ except Exception:
99+ return (None, None)
100+
101+ try:
102+ html = fromstring(response.text)
103+ except ValueError:
104+ html = fromstring(response.content)
105+
106+ description = extract_text(html.xpath('/html/head/meta[@name="description"]/@content'))
107+ if not description:
108+ description = extract_text(html.xpath('/html/head/meta[@property="og:description"]/@content'))
109+ if not description:
110+ description = extract_text(html.xpath('/html/head/title'))
111+ lang = extract_text(html.xpath('/html/@lang'))
112+ if lang is None and len(lang1) > 0:
113+ lang = lang1
114+ lang = detect_language(description) or lang or 'en'
115+ lang = lang.split('_')[0]
116+ lang = lang.split('-')[0]
117+ return (lang, description)
118+
119+
120+def initialize():
121+ global descriptions, wd_to_engine_name, IDS
122+ searx.search.initialize()
123+ for engine_name, engine in searx.engines.engines.items():
124+ descriptions[engine_name] = {}
125+ wikidata_id = getattr(engine, "about", {}).get('wikidata_id')
126+ if wikidata_id is not None:
127+ wd_to_engine_name.setdefault(wikidata_id, set()).add(engine_name)
128+
129+ IDS = ' '.join(list(map(lambda wd_id: 'wd:' + wd_id, wd_to_engine_name.keys())))
130+
131+
132+def fetch_wikidata_descriptions():
133+ global IDS
134+ result = send_wikidata_query(SPARQL_DESCRIPTION
135+ .replace('%IDS%', IDS)
136+ .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
137+ if result is not None:
138+ for binding in result['results']['bindings']:
139+ wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
140+ lang = binding['itemDescription']['xml:lang']
141+ description = binding['itemDescription']['value']
142+ if ' ' in description: # skip unique word description (like "website")
143+ for engine_name in wd_to_engine_name[wikidata_id]:
144+ update_description(engine_name, lang, description, 'wikidata')
145+
146+
147+def fetch_wikipedia_descriptions():
148+ global IDS
149+ result = send_wikidata_query(SPARQL_WIKIPEDIA_ARTICLE
150+ .replace('%IDS%', IDS)
151+ .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
152+ if result is not None:
153+ for binding in result['results']['bindings']:
154+ wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
155+ lang = binding['name']['xml:lang']
156+ pageid = binding['name']['value']
157+ description = get_wikipedia_summary(lang, pageid)
158+ if description is not None and ' ' in description:
159+ for engine_name in wd_to_engine_name[wikidata_id]:
160+ update_description(engine_name, lang, description, 'wikipedia')
161+
162+
163+def normalize_url(url):
164+ url = url.replace('{language}', 'en')
165+ url = urlparse(url)._replace(path='/', params='', query='', fragment='').geturl()
166+ url = url.replace('https://api.', 'https://')
167+ return url
168+
169+
170+def fetch_website_description(engine_name, website):
171+ default_lang, default_description = get_website_description(website, None, None)
172+ if default_lang is None or default_description is None:
173+ return
174+ if default_lang not in descriptions[engine_name]:
175+ descriptions[engine_name][default_lang] = [normalize_description(default_description), website]
176+ for request_lang in ('en-US', 'es-US', 'fr-FR', 'zh', 'ja', 'ru', 'ar', 'ko'):
177+ if request_lang.split('-')[0] not in descriptions[engine_name]:
178+ lang, desc = get_website_description(website, request_lang, request_lang.split('-')[0])
179+ if desc is not None and desc != default_description:
180+ update_description(engine_name, lang, desc, website, replace=False)
181+ else:
182+ break
183+
184+
185+def fetch_website_descriptions():
186+ for engine_name, engine in searx.engines.engines.items():
187+ website = getattr(engine, "about", {}).get('website')
188+ if website is None:
189+ website = normalize_url(getattr(engine, "search_url"))
190+ if website is None:
191+ website = normalize_url(getattr(engine, "base_url"))
192+ if website is not None:
193+ fetch_website_description(engine_name, website)
194+
195+
196+def main():
197+ initialize()
198+ fetch_wikidata_descriptions()
199+ fetch_wikipedia_descriptions()
200+ fetch_website_descriptions()
201+
202+ sys.stdout.write(json.dumps(descriptions, indent=1, separators=(',', ':'), ensure_ascii=False))
203+
204+
205+if __name__ == "__main__":
206+ main()