@@ -1,27 +1,21 @@
11# -*- coding: utf-8 -*-
22import os
33import sys
4−import csv
5−import hashlib
6−import hmac
74import re
85import json
96
10−from codecs import getincrementalencoder
117from imp import load_source
128from numbers import Number
139from os.path import splitext, join
14−from io import open, StringIO
10+from io import open
1511from random import choice
1612from html.parser import HTMLParser
1713from lxml.etree import XPath
1814from babel.core import get_global
19−from babel.dates import format_date
2015
2116from searx import settings
2217from searx.version import VERSION_STRING
2318from searx.languages import language_codes
24−from searx import settings
2519from searx import logger
2620
2721
@@ -50,33 +44,6 @@ def gen_useragent(os=None):
5044 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))
5145
5246
53−def highlight_content(content, query):
54−
55− if not content:
56− return None
57− # ignoring html contents
58− # TODO better html content detection
59− if content.find('<') != -1:
60− return content
61−
62− if content.lower().find(query.lower()) > -1:
63− query_regex = '({0})'.format(re.escape(query))
64− content = re.sub(query_regex, '<span class="highlight">\\1</span>',
65− content, flags=re.I | re.U)
66− else:
67− regex_parts = []
68− for chunk in query.split():
69− if len(chunk) == 1:
70− regex_parts.append('\\W+{0}\\W+'.format(re.escape(chunk)))
71− else:
72− regex_parts.append('{0}'.format(re.escape(chunk)))
73− query_regex = '({0})'.format('|'.join(regex_parts))
74− content = re.sub(query_regex, '<span class="highlight">\\1</span>',
75− content, flags=re.I | re.U)
76−
77− return content
78−
79−
8047class HTMLTextExtractorException(Exception):
8148 pass
8249
@@ -139,91 +106,6 @@ def html_to_text(html):
139106 return s.get_text()
140107
141108
142−class UnicodeWriter:
143− """
144− A CSV writer which will write rows to CSV file "f",
145− which is encoded in the given encoding.
146− """
147−
148− def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
149− # Redirect output to a queue
150− self.queue = StringIO()
151− self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
152− self.stream = f
153− self.encoder = getincrementalencoder(encoding)()
154−
155− def writerow(self, row):
156− self.writer.writerow(row)
157− # Fetch UTF-8 output from the queue ...
158− data = self.queue.getvalue()
159− data = data.strip('\x00')
160− # ... and reencode it into the target encoding
161− data = self.encoder.encode(data)
162− # write to the target stream
163− self.stream.write(data.decode())
164− # empty queue
165− self.queue.truncate(0)
166−
167− def writerows(self, rows):
168− for row in rows:
169− self.writerow(row)
170−
171−
172−def get_resources_directory(searx_directory, subdirectory, resources_directory):
173− if not resources_directory:
174− resources_directory = os.path.join(searx_directory, subdirectory)
175− if not os.path.isdir(resources_directory):
176− raise Exception(resources_directory + " is not a directory")
177− return resources_directory
178−
179−
180−def get_themes(templates_path):
181− """Returns available themes list."""
182− themes = os.listdir(templates_path)
183− if '__common__' in themes:
184− themes.remove('__common__')
185− return themes
186−
187−
188−def get_static_files(static_path):
189− static_files = set()
190− static_path_length = len(static_path) + 1
191− for directory, _, files in os.walk(static_path):
192− for filename in files:
193− f = os.path.join(directory[static_path_length:], filename)
194− static_files.add(f)
195− return static_files
196−
197−
198−def get_result_templates(templates_path):
199− result_templates = set()
200− templates_path_length = len(templates_path) + 1
201− for directory, _, files in os.walk(templates_path):
202− if directory.endswith('result_templates'):
203− for filename in files:
204− f = os.path.join(directory[templates_path_length:], filename)
205− result_templates.add(f)
206− return result_templates
207−
208−
209−def format_date_by_locale(date, locale_string):
210− # strftime works only on dates after 1900
211−
212− if date.year <= 1900:
213− return date.isoformat().split('T')[0]
214−
215− if locale_string == 'all':
216− locale_string = settings['ui']['default_locale'] or 'en_US'
217−
218− # to avoid crashing if locale is not supported by babel
219− try:
220− formatted_date = format_date(date, locale=locale_string)
221− except:
222− formatted_date = format_date(date, "YYYY-MM-dd")
223−
224− return formatted_date
225−
226−
227109def dict_subset(d, properties):
228110 result = {}
229111 for k in properties:
@@ -232,14 +114,6 @@ def dict_subset(d, properties):
232114 return result
233115
234116
235−def prettify_url(url, max_length=74):
236− if len(url) > max_length:
237− chunk_len = int(max_length / 2 + 1)
238− return '{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:])
239− else:
240− return url
241−
242−
243117# get element in list or default value
244118def list_get(a_list, index, default=None):
245119 if len(a_list) > index:
@@ -383,17 +257,6 @@ def load_module(filename, module_dir):
383257 return module
384258
385259
386−def new_hmac(secret_key, url):
387− try:
388− secret_key_bytes = bytes(secret_key, 'utf-8')
389− except TypeError as err:
390− if isinstance(secret_key, bytes):
391− secret_key_bytes = secret_key
392− else:
393− raise err
394− return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest()
395−
396−
397260def to_string(obj):
398261 if isinstance(obj, str):
399262 return obj