d3d50ef · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #2275 from dalf/mod-ahmia-filter

[mod] ahmia_filter.py: minor changes

commitd3d50eff665f03c16adcb26a774b25b4fd5ade08 authorAdam Tauber <asciimoo@gmail.com> dateTue 27 Oct 2020, 21:17 (6 years ago) parentdb703a0 5e70600

2 files changed+10−8

compare with master

unifiedsplit

searx/data/__init__.py+6−1 searx/plugins/ahmia_filter.py+4−7

searx/data/__init__.py+6−1

@@ -2,7 +2,7 @@ import json

22from pathlib import Path

5−__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader']

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader', 'ahmia_blacklist_loader']

66data_dir = Path(__file__).parent

@@ -16,6 +16,11 @@ def bangs_loader():

1616 return load('bangs.json')

1717

1818

19+def ahmia_blacklist_loader():

20+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

21+ return fd.read().split()

22+

23+

1924ENGINES_LANGUAGES = load('engines_languages.json')

2025CURRENCIES = load('currencies.json')

2126USER_AGENTS = load('useragents.json')

searx/plugins/ahmia_filter.py+4−7

@@ -3,9 +3,7 @@

33'''

55from hashlib import md5

6−from os.path import join

7−from urllib.parse import urlparse

8−from searx import searx_dir

6+from searx.data import ahmia_blacklist_loader

108name = "Ahmia blacklist"

119description = "Filter out onion results that appear in Ahmia's blacklist. (See https://ahmia.fi/blacklist)"

@@ -18,15 +16,14 @@ ahmia_blacklist = None

1816def get_ahmia_blacklist():

1917 global ahmia_blacklist

2018 if not ahmia_blacklist:

21− with open(join(join(searx_dir, "data"), "ahmia_blacklist.txt"), 'r') as f:

22− ahmia_blacklist = f.read().split()

19+ ahmia_blacklist = ahmia_blacklist_loader()

2320 return ahmia_blacklist

2421

2522

2623def not_blacklisted(result):

27− if not result.get('is_onion'):

24+ if not result.get('is_onion') or not result.get('parsed_url'):

2825 return True

29− result_hash = md5(urlparse(result.get('url')).hostname.encode()).hexdigest()

26+ result_hash = md5(result['parsed_url'].hostname.encode()).hexdigest()

3027 return result_hash not in get_ahmia_blacklist()

3128

3229