f77983e · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2602 from MarcAbonce/fix-bing-fetch-languages

Fix fetch_languages for Bing

commitf77983e174ec45913f32482210e36c3cb94cf810 authorAlexandre Flament <alex@al-f.net> dateMon 1 Mar 2021, 09:06 (6 years ago) parent5f6ac3a d6681fd

3 files changed+8−5

compare with master

unifiedsplit

searx/engines/__init__.py+6−2 searx/engines/wikipedia.py+1−1 utils/fetch_languages.py+1−2

searx/engines/__init__.py+6−2

@@ -27,7 +27,7 @@ from searx import settings

2727from searx import logger

2828from searx.data import ENGINES_LANGUAGES

2929from searx.poolrequests import get, get_proxy_cycles

30−from searx.utils import load_module, match_language, get_engine_from_settings

30+from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent

3131

3232

3333logger = logger.getChild('engines')

@@ -131,8 +131,12 @@ def load_engine(engine_data):

131131

132132 # assign language fetching method if auxiliary method exists

133133 if hasattr(engine, '_fetch_supported_languages'):

134+ headers = {

135+ 'User-Agent': gen_useragent(),

136+ 'Accept-Language': 'ja-JP,ja;q=0.8,en-US;q=0.5,en;q=0.3', # bing needs a non-English language

137+ }

134138 setattr(engine, 'fetch_supported_languages',

135− lambda: engine._fetch_supported_languages(get(engine.supported_languages_url)))

139+ lambda: engine._fetch_supported_languages(get(engine.supported_languages_url, headers=headers)))

136140

137141 engine.stats = {

138142 'sent_search_count': 0, # sent search

searx/engines/wikipedia.py+1−1

@@ -106,6 +106,6 @@ def _fetch_supported_languages(resp):

106106 articles = int(td[4].xpath('./a/b')[0].text.replace(',', ''))

107107 # exclude languages with too few articles

108108 if articles >= 100:

109− supported_languages[code] = {"name": name, "english_name": english_name, "articles": articles}

109+ supported_languages[code] = {"name": name, "english_name": english_name}

110110

111111 return supported_languages

utils/fetch_languages.py+1−2

@@ -2,8 +2,7 @@

33# This script generates languages.py from intersecting each engine's supported languages.

44#

5−# Output files (engines_languages.json and languages.py)

6−# are written in current directory to avoid overwriting in case something goes wrong.

5+# Output files: searx/data/engines_languages.json and searx/languages.py

87import json

98from pathlib import Path