8b278cb · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2246 from dalf/mod-searx-data

[mod] Add searx.data module

commit8b278cbfad3f39cc71cb41954d37b30c7d16d947 authorAlexandre Flament <alex@al-f.net> dateWed 7 Oct 2020, 10:38 (6 years ago) parente30dc2f a9dc54b

5 files changed+37−41

compare with master

unifiedsplit

searx/data/__init__.py+21 searx/engines/__init__.py+3−5 searx/engines/currency_convert.py+7−21 searx/external_bang.py+4−8 searx/utils.py+2−7

searx/data/__init__.pynew file+21

@@ -0,0 +1,21 @@

1+import json

2+from pathlib import Path

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader']

6+data_dir = Path(__file__).parent

9+def load(filename):

10+ # add str(...) for Python 3.5

11+ with open(str(data_dir / filename), encoding='utf-8') as fd:

12+ return json.load(fd)

13+

14+

15+def bangs_loader():

16+ return load('bangs.json')

17+

18+

19+ENGINES_LANGUAGES = load('engines_languages.json')

20+CURRENCIES = load('currencies.json')

21+USER_AGENTS = load('useragents.json')

searx/engines/__init__.py+3−5

@@ -19,13 +19,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

1919import sys

2020import threading

2121from os.path import realpath, dirname

22−from io import open

2322from babel.localedata import locale_identifiers

2423from flask_babel import gettext

2524from operator import itemgetter

26−from json import loads

2725from searx import settings

2826from searx import logger

27+from searx.data import ENGINES_LANGUAGES

2928from searx.poolrequests import get

3029from searx.utils import load_module, match_language, get_engine_from_settings

3130

@@ -38,7 +37,6 @@ engines = {}

3837

3938categories = {'general': []}

4039

41−languages = loads(open(engine_dir + '/../data/engines_languages.json', 'r', encoding='utf-8').read())

4240babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]

4341 for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())]

4442

@@ -108,8 +106,8 @@ def load_engine(engine_data):

108106 sys.exit(1)

109107

110108 # assign supported languages from json file

111− if engine_data['name'] in languages:

112− setattr(engine, 'supported_languages', languages[engine_data['name']])

109+ if engine_data['name'] in ENGINES_LANGUAGES:

110+ setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['name']])

113111

114112 # find custom aliases for non standard language codes

115113 if hasattr(engine, 'supported_languages'):

searx/engines/currency_convert.py+7−21

@@ -1,11 +1,11 @@

11import json

22import re

3−import os

43import unicodedata

6−from io import open

75from datetime import datetime

7+from searx.data import CURRENCIES

1010categories = []

1111url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'

@@ -13,8 +13,6 @@ weight = 100

1313

1414parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

1515

16−db = 1

17−

1816

1917def normalize_name(name):

2018 name = name.lower().replace('-', ' ').rstrip('s')

@@ -23,17 +21,17 @@ def normalize_name(name):

2321

2422

2523def name_to_iso4217(name):

26− global db

24+ global CURRENCIES

2725

2826 name = normalize_name(name)

29− currencies = db['names'].get(name, [name])

30− return currencies[0]

27+ currency = CURRENCIES['names'].get(name, [name])

28+ return currency[0]

3129

3230

3331def iso4217_to_name(iso4217, language):

34− global db

32+ global CURRENCIES

3533

36− return db['iso4217'].get(iso4217, {}).get(language, iso4217)

34+ return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)

3735

3836

3937def request(query, params):

@@ -82,15 +80,3 @@ def response(resp):

8280 results.append({'answer': answer, 'url': url})

8381

8482 return results

85−

86−

87−def load():

88− global db

89−

90− current_dir = os.path.dirname(os.path.realpath(__file__))

91− json_data = open(current_dir + "/../data/currencies.json", 'r', encoding='utf-8').read()

92−

93− db = json.loads(json_data)

94−

95−

96−load()

searx/external_bang.py+4−8

@@ -1,7 +1,4 @@

1−import json

2−from os.path import join

3−

4−from searx import searx_dir

1+from searx.data import bangs_loader

63# bangs data coming from the following url convert to json with

74# https://raw.githubusercontent.com/jivesearch/jivesearch/master/bangs/bangs.toml

@@ -9,10 +6,9 @@ from searx import searx_dir

96# NOTE only use the get_bang_url

107

118bangs_data = {}

12−with open(join(searx_dir, 'data/bangs.json'), encoding='utf-8') as json_file:

13− for bang in json.load(json_file)['bang']:

14− for trigger in bang["triggers"]:

15− bangs_data[trigger] = {x: y for x, y in bang.items() if x != "triggers"}

9+for bang in bangs_loader()['bang']:

10+ for trigger in bang["triggers"]:

11+ bangs_data[trigger] = {x: y for x, y in bang.items() if x != "triggers"}

1612

1713

1814def get_bang_url(search_query):

searx/utils.py+2−7

@@ -1,13 +1,10 @@

11# -*- coding: utf-8 -*-

2−import os

32import sys

43import re

5−import json

64import importlib

86from numbers import Number

97from os.path import splitext, join

10−from io import open

118from random import choice

129from html.parser import HTMLParser

1310from urllib.parse import urljoin, urlparse, unquote

@@ -18,6 +15,7 @@ from babel.core import get_global

1815

1916

2017from searx import settings

18+from searx.data import USER_AGENTS

2119from searx.version import VERSION_STRING

2220from searx.languages import language_codes

2321from searx import logger

@@ -31,9 +29,6 @@ blocked_tags = ('script',

3129ecma_unescape4_re = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)

3230ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

3331

34−useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

35− + "/data/useragents.json", 'r', encoding='utf-8').read())

36−

3732xpath_cache = dict()

3833lang_to_lc_cache = dict()

3934

@@ -50,7 +45,7 @@ def gen_useragent(os=None):

5045

5146 See searx/data/useragents.json

5247 """

53− return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

48+ return str(USER_AGENTS['ua'].format(os=os or choice(USER_AGENTS['os']), version=choice(USER_AGENTS['versions'])))

5449

5550

5651class HTMLTextExtractorException(Exception):