d3d50ef · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

Merge pull request #2275 from dalf/mod-ahmia-filter

d3d50efAdam Tauber6 years ago

searx/data/__init__.py+6−1

compare with master

unifiedsplit

@@ -2,7 +2,7 @@ import json

2from pathlib import Path

5−__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader']

5+__init__ = ['ENGINES_LANGUGAGES', 'CURRENCIES', 'USER_AGENTS', 'bangs_loader', 'ahmia_blacklist_loader']

6data_dir = Path(__file__).parent

@@ -16,6 +16,11 @@ def bangs_loader():

16 return load('bangs.json')

19+def ahmia_blacklist_loader():

20+ with open(str(data_dir / 'ahmia_blacklist.txt'), encoding='utf-8') as fd:

21+ return fd.read().split()

22+

23+

19ENGINES_LANGUAGES = load('engines_languages.json')

24ENGINES_LANGUAGES = load('engines_languages.json')

20CURRENCIES = load('currencies.json')

25CURRENCIES = load('currencies.json')

21USER_AGENTS = load('useragents.json')

26USER_AGENTS = load('useragents.json')

searx/plugins/ahmia_filter.py+4−7

@@ -3,9 +3,7 @@

3'''

5from hashlib import md5

6−from os.path import join

6+from searx.data import ahmia_blacklist_loader

7−from urllib.parse import urlparse

8−from searx import searx_dir

10name = "Ahmia blacklist"

8name = "Ahmia blacklist"

11description = "Filter out onion results that appear in Ahmia's blacklist. (See https://ahmia.fi/blacklist)"

9description = "Filter out onion results that appear in Ahmia's blacklist. (See https://ahmia.fi/blacklist)"

@@ -18,15 +16,14 @@ ahmia_blacklist = None

18def get_ahmia_blacklist():

16def get_ahmia_blacklist():

19 global ahmia_blacklist

17 global ahmia_blacklist

20 if not ahmia_blacklist:

18 if not ahmia_blacklist:

21− with open(join(join(searx_dir, "data"), "ahmia_blacklist.txt"), 'r') as f:

22− ahmia_blacklist = f.read().split()

19+ ahmia_blacklist = ahmia_blacklist_loader()

23 return ahmia_blacklist

20 return ahmia_blacklist

26def not_blacklisted(result):

23def not_blacklisted(result):

27− if not result.get('is_onion'):

24+ if not result.get('is_onion') or not result.get('parsed_url'):

28 return True

25 return True

29− result_hash = md5(urlparse(result.get('url')).hostname.encode()).hexdigest()

26+ result_hash = md5(result['parsed_url'].hostname.encode()).hexdigest()

30 return result_hash not in get_ahmia_blacklist()

27 return result_hash not in get_ahmia_blacklist()