a9dc54b · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[mod] Add searx.data module

a9dc54bAlexandre Flament6 years ago

searx/data/__init__.pynew file+21

compare with master

unifiedsplit

@@ -0,0 +1,21 @@

1+import json

2+from pathlib import Path

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader']

6+data_dir = Path(__file__).parent

9+def load(filename):

10+ # add str(...) for Python 3.5

11+ with open(str(data_dir / filename), encoding='utf-8') as fd:

12+ return json.load(fd)

13+

14+

15+def bangs_loader():

16+ return load('bangs.json')

17+

18+

19+ENGINES_LANGUAGES = load('engines_languages.json')

20+CURRENCIES = load('currencies.json')

21+USER_AGENTS = load('useragents.json')

searx/engines/__init__.py+3−5

@@ -19,13 +19,12 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.

19import sys

20import threading

21from os.path import realpath, dirname

22−from io import open

23from babel.localedata import locale_identifiers

22from babel.localedata import locale_identifiers

24from flask_babel import gettext

23from flask_babel import gettext

25from operator import itemgetter

24from operator import itemgetter

26−from json import loads

27from searx import settings

25from searx import settings

28from searx import logger

26from searx import logger

27+from searx.data import ENGINES_LANGUAGES

29from searx.poolrequests import get

28from searx.poolrequests import get

30from searx.utils import load_module, match_language, get_engine_from_settings

29from searx.utils import load_module, match_language, get_engine_from_settings

@@ -38,7 +37,6 @@ engines = {}

39categories = {'general': []}

38categories = {'general': []}

41−languages = loads(open(engine_dir + '/../data/engines_languages.json', 'r', encoding='utf-8').read())

42babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]

40babel_langs = [lang_parts[0] + '-' + lang_parts[-1] if len(lang_parts) > 1 else lang_parts[0]

43 for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())]

41 for lang_parts in (lang_code.split('_') for lang_code in locale_identifiers())]

@@ -108,8 +106,8 @@ def load_engine(engine_data):

108 sys.exit(1)

106 sys.exit(1)

109

107

110 # assign supported languages from json file

108 # assign supported languages from json file

111− if engine_data['name'] in languages:

109+ if engine_data['name'] in ENGINES_LANGUAGES:

112− setattr(engine, 'supported_languages', languages[engine_data['name']])

110+ setattr(engine, 'supported_languages', ENGINES_LANGUAGES[engine_data['name']])

113

111

114 # find custom aliases for non standard language codes

112 # find custom aliases for non standard language codes

115 if hasattr(engine, 'supported_languages'):

113 if hasattr(engine, 'supported_languages'):

searx/engines/currency_convert.py+7−21

@@ -1,11 +1,11 @@

1import json

2import re

3−import os

4import unicodedata

3import unicodedata

6−from io import open

7from datetime import datetime

5from datetime import datetime

7+from searx.data import CURRENCIES

10categories = []

11url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'

@@ -13,8 +13,6 @@ weight = 100

14parser_re = re.compile('.*?(\\d+(?:\\.\\d+)?) ([^.0-9]+) (?:in|to) ([^.0-9]+)', re.I)

16−db = 1

17−

19def normalize_name(name):

17def normalize_name(name):

20 name = name.lower().replace('-', ' ').rstrip('s')

18 name = name.lower().replace('-', ' ').rstrip('s')

@@ -23,17 +21,17 @@ def normalize_name(name):

25def name_to_iso4217(name):

23def name_to_iso4217(name):

26− global db

24+ global CURRENCIES

28 name = normalize_name(name)

26 name = normalize_name(name)

29− currencies = db['names'].get(name, [name])

27+ currency = CURRENCIES['names'].get(name, [name])

30− return currencies[0]

28+ return currency[0]

33def iso4217_to_name(iso4217, language):

31def iso4217_to_name(iso4217, language):

34− global db

32+ global CURRENCIES

36− return db['iso4217'].get(iso4217, {}).get(language, iso4217)

34+ return CURRENCIES['iso4217'].get(iso4217, {}).get(language, iso4217)

39def request(query, params):

37def request(query, params):

@@ -82,15 +80,3 @@ def response(resp):

82 results.append({'answer': answer, 'url': url})

80 results.append({'answer': answer, 'url': url})

84 return results

82 return results

85−

86−

87−def load():

88− global db

89−

90− current_dir = os.path.dirname(os.path.realpath(__file__))

91− json_data = open(current_dir + "/../data/currencies.json", 'r', encoding='utf-8').read()

92−

93− db = json.loads(json_data)

94−

95−

96−load()

searx/external_bang.py+4−8

@@ -1,7 +1,4 @@

1−import json

2−from os.path import join

3−

4−from searx import searx_dir

1+from searx.data import bangs_loader

6# bangs data coming from the following url convert to json with

3# bangs data coming from the following url convert to json with

7# https://raw.githubusercontent.com/jivesearch/jivesearch/master/bangs/bangs.toml

4# https://raw.githubusercontent.com/jivesearch/jivesearch/master/bangs/bangs.toml

@@ -9,10 +6,9 @@ from searx import searx_dir

9# NOTE only use the get_bang_url

6# NOTE only use the get_bang_url

11bangs_data = {}

8bangs_data = {}

12−with open(join(searx_dir, 'data/bangs.json'), encoding='utf-8') as json_file:

9+for bang in bangs_loader()['bang']:

13− for bang in json.load(json_file)['bang']:

10+ for trigger in bang["triggers"]:

14− for trigger in bang["triggers"]:

15− bangs_data[trigger] = {x: y for x, y in bang.items() if x != "triggers"}

11+ bangs_data[trigger] = {x: y for x, y in bang.items() if x != "triggers"}

18def get_bang_url(search_query):

14def get_bang_url(search_query):

searx/utils.py+2−7

@@ -1,13 +1,10 @@

1# -*- coding: utf-8 -*-

2−import os

3import sys

2import sys

4import re

3import re

5−import json

6import importlib

4import importlib

8from numbers import Number

6from numbers import Number

9from os.path import splitext, join

7from os.path import splitext, join

10−from io import open

11from random import choice

8from random import choice

12from html.parser import HTMLParser

9from html.parser import HTMLParser

13from urllib.parse import urljoin, urlparse, unquote

10from urllib.parse import urljoin, urlparse, unquote

@@ -18,6 +15,7 @@ from babel.core import get_global

20from searx import settings

17from searx import settings

18+from searx.data import USER_AGENTS

21from searx.version import VERSION_STRING

19from searx.version import VERSION_STRING

22from searx.languages import language_codes

20from searx.languages import language_codes

23from searx import logger

21from searx import logger

@@ -31,9 +29,6 @@ blocked_tags = ('script',

31ecma_unescape4_re = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)

29ecma_unescape4_re = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)

32ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

30ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)

34−useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))

35− + "/data/useragents.json", 'r', encoding='utf-8').read())

36−

37xpath_cache = dict()

32xpath_cache = dict()

38lang_to_lc_cache = dict()

33lang_to_lc_cache = dict()

@@ -50,7 +45,7 @@ def gen_useragent(os=None):

51 See searx/data/useragents.json

46 See searx/data/useragents.json

52 """

47 """

53− return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

48+ return str(USER_AGENTS['ua'].format(os=os or choice(USER_AGENTS['os']), version=choice(USER_AGENTS['versions'])))

56class HTMLTextExtractorException(Exception):

51class HTMLTextExtractorException(Exception):