@@ -39,12 +39,17 @@ lang_to_lc_cache = dict()
3939
4040
4141def searx_useragent():
42+ """Return the searx User Agent"""
4243 return 'searx/{searx_version} {suffix}'.format(
4344 searx_version=VERSION_STRING,
4445 suffix=settings['outgoing'].get('useragent_suffix', ''))
4546
4647
4748def gen_useragent(os=None):
49+ """Return a random browser User Agent
50+
51+ See searx/data/useragents.json
52+ """
4853 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))
4954
5055
@@ -99,24 +104,40 @@ class HTMLTextExtractor(HTMLParser):
99104 return ''.join(self.result).strip()
100105
101106
102−def html_to_text(html):
103− html = html.replace('\n', ' ')
104− html = ' '.join(html.split())
107+def html_to_text(html_str):
108+ """Extract text from a HTML string
109+
110+ Args:
111+ * html_str (str): string HTML
112+
113+ Returns:
114+ * str: extracted text
115+
116+ Examples:
117+ >>> html_to_text('Example <span id="42">#2</span>')
118+ 'Example #2'
119+
120+ >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')
121+ 'Example'
122+ """
123+ html_str = html_str.replace('\n', ' ')
124+ html_str = ' '.join(html_str.split())
105125 s = HTMLTextExtractor()
106126 try:
107− s.feed(html)
127+ s.feed(html_str)
108128 except HTMLTextExtractorException:
109− logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)
129+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str)
110130 return s.get_text()
111131
112132
113133def extract_text(xpath_results):
114− '''
115− if xpath_results is list, extract the text from each result and concat the list
116− if xpath_results is a xml element, extract all the text node from it
117− ( text_content() method from lxml )
118− if xpath_results is a string element, then it's already done
119− '''
134+ """Extract text from a lxml result
135+
136+ * if xpath_results is list, extract the text from each result and concat the list
137+ * if xpath_results is a xml element, extract all the text node from it
138+ ( text_content() method from lxml )
139+ * if xpath_results is a string element, then it's already done
140+ """
120141 if type(xpath_results) == list:
121142 # it's list of result : concat everything using recursive call
122143 result = ''
@@ -135,7 +156,58 @@ def extract_text(xpath_results):
135156 return ' '.join(text.split())
136157
137158
159+def normalize_url(url):
160+ parsed_url = urlparse(url)
161+
162+ # add a / at this end of the url if there is no path
163+ if not parsed_url.netloc:
164+ raise Exception('Cannot parse url')
165+ if not parsed_url.path:
166+ url += '/'
167+
168+ # FIXME : hack for yahoo
169+ if parsed_url.hostname == 'search.yahoo.com'\
170+ and parsed_url.path.startswith('/r'):
171+ p = parsed_url.path
172+ mark = p.find('/**')
173+ if mark != -1:
174+ return unquote(p[mark + 3:]).decode()
175+
176+ return url
177+
178+
138179def extract_url(xpath_results, search_url):
180+ """Extract and normalize URL from lxml Element
181+
182+ Args:
183+ * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)
184+ * search_url (str): Base URL
185+
186+ Example:
187+ >>> def f(s, search_url):
188+ >>> return searx.utils.extract_url(html.fromstring(s), search_url)
189+ >>> f('<span id="42">https://example.com</span>', 'http://example.com/')
190+ 'https://example.com/'
191+ >>> f('https://example.com', 'http://example.com/')
192+ 'https://example.com/'
193+ >>> f('//example.com', 'http://example.com/')
194+ 'http://example.com/'
195+ >>> f('//example.com', 'https://example.com/')
196+ 'https://example.com/'
197+ >>> f('/path?a=1', 'https://example.com')
198+ 'https://example.com/path?a=1'
199+ >>> f('', 'https://example.com')
200+ raise lxml.etree.ParserError
201+ >>> searx.utils.extract_url([], 'https://example.com')
202+ raise Exception
203+
204+ Raises:
205+ * Exception
206+ * lxml.etree.ParserError
207+
208+ Returns:
209+ * str: normalized URL
210+ """
139211 if xpath_results == []:
140212 raise Exception('Empty url resultset')
141213 url = extract_text(xpath_results)
@@ -158,27 +230,15 @@ def extract_url(xpath_results, search_url):
158230 return url
159231
160232
161−def normalize_url(url):
162− parsed_url = urlparse(url)
163−
164− # add a / at this end of the url if there is no path
165− if not parsed_url.netloc:
166− raise Exception('Cannot parse url')
167− if not parsed_url.path:
168− url += '/'
169−
170− # FIXME : hack for yahoo
171− if parsed_url.hostname == 'search.yahoo.com'\
172− and parsed_url.path.startswith('/r'):
173− p = parsed_url.path
174− mark = p.find('/**')
175− if mark != -1:
176− return unquote(p[mark + 3:]).decode()
177−
178− return url
179−
180−
181233def dict_subset(d, properties):
234+ """Extract a subset of a dict
235+
236+ Examples:
237+ >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])
238+ {'A': 'a', 'C': 'c'}
239+ >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])
240+ {'A': 'a'}
241+ """
182242 result = {}
183243 for k in properties:
184244 if k in d:
@@ -186,8 +246,19 @@ def dict_subset(d, properties):
186246 return result
187247
188248
189−# get element in list or default value
190249def list_get(a_list, index, default=None):
250+ """Get element in list or default value
251+
252+ Examples:
253+ >>> list_get(['A', 'B', 'C'], 0)
254+ 'A'
255+ >>> list_get(['A', 'B', 'C'], 3)
256+ None
257+ >>> list_get(['A', 'B', 'C'], 3, 'default')
258+ 'default'
259+ >>> list_get(['A', 'B', 'C'], -1)
260+ 'C'
261+ """
191262 if len(a_list) > index:
192263 return a_list[index]
193264 else:
@@ -195,6 +266,21 @@ def list_get(a_list, index, default=None):
195266
196267
197268def get_torrent_size(filesize, filesize_multiplier):
269+ """
270+
271+ Args:
272+ * filesize (str): size
273+ * filesize_multiplier (str): TB, GB, .... TiB, GiB...
274+
275+ Returns:
276+ * int: number of bytes
277+
278+ Example:
279+ >>> get_torrent_size('5', 'GB')
280+ 5368709120
281+ >>> get_torrent_size('3.14', 'MiB')
282+ 3140000
283+ """
198284 try:
199285 filesize = float(filesize)
200286
@@ -221,14 +307,18 @@ def get_torrent_size(filesize, filesize_multiplier):
221307
222308
223309def convert_str_to_int(number_str):
310+ """Convert number_str to int or 0 if number_str is not a number."""
224311 if number_str.isdigit():
225312 return int(number_str)
226313 else:
227314 return 0
228315
229316
230−# convert a variable to integer or return 0 if it's not a number
231317def int_or_zero(num):
318+ """Convert num to int or 0. num can be either a str or a list.
319+ If num is a list, the first element is converted to int (or return 0 if the list is empty).
320+ If num is a str, see convert_str_to_int
321+ """
232322 if isinstance(num, list):
233323 if len(num) < 1:
234324 return 0
@@ -237,6 +327,22 @@ def int_or_zero(num):
237327
238328
239329def is_valid_lang(lang):
330+ """Return language code and name if lang describe a language.
331+
332+ Examples:
333+ >>> is_valid_lang('zz')
334+ False
335+ >>> is_valid_lang('uk')
336+ (True, 'uk', 'ukrainian')
337+ >>> is_valid_lang(b'uk')
338+ (True, 'uk', 'ukrainian')
339+ >>> is_valid_lang('en')
340+ (True, 'en', 'english')
341+ >>> searx.utils.is_valid_lang('Español')
342+ (True, 'es', 'spanish')
343+ >>> searx.utils.is_valid_lang('Spanish')
344+ (True, 'es', 'spanish')
345+ """
240346 if isinstance(lang, bytes):
241347 lang = lang.decode()
242348 is_abbr = (len(lang) == 2)
@@ -264,8 +370,8 @@ def _get_lang_to_lc_dict(lang_list):
264370 return value
265371
266372
267−# auxiliary function to match lang_code in lang_list
268373def _match_language(lang_code, lang_list=[], custom_aliases={}):
374+ """auxiliary function to match lang_code in lang_list"""
269375 # replace language code with a custom alias if necessary
270376 if lang_code in custom_aliases:
271377 lang_code = custom_aliases[lang_code]
@@ -287,8 +393,8 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):
287393 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)
288394
289395
290−# get the language code from lang_list that best matches locale_code
291396def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):
397+ """get the language code from lang_list that best matches locale_code"""
292398 # try to get language from given locale_code
293399 language = _match_language(locale_code, lang_list, custom_aliases)
294400 if language:
@@ -330,6 +436,7 @@ def load_module(filename, module_dir):
330436
331437
332438def to_string(obj):
439+ """Convert obj to its string representation."""
333440 if isinstance(obj, str):
334441 return obj
335442 if isinstance(obj, Number):
@@ -341,13 +448,19 @@ def to_string(obj):
341448
342449
343450def ecma_unescape(s):
344− """
345− python implementation of the unescape javascript function
451+ """Python implementation of the unescape javascript function
346452
347453 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string
348454 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape
455+
456+ Examples:
457+ >>> ecma_unescape('%u5409')
458+ '吉'
459+ >>> ecma_unescape('%20')
460+ ' '
461+ >>> ecma_unescape('%F3')
462+ 'ó'
349463 """
350− # s = unicode(s)
351464 # "%u5409" becomes "吉"
352465 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)
353466 # "%20" becomes " ", "%F3" becomes "ó"
@@ -371,6 +484,11 @@ def get_engine_from_settings(name):
371484
372485
373486def get_xpath(xpath_str):
487+ """Return cached compiled XPath
488+
489+ There is no thread lock.
490+ Worst case scenario, xpath_str is compiled more than one time.
491+ """
374492 result = xpath_cache.get(xpath_str, None)
375493 if result is None:
376494 result = XPath(xpath_str)
@@ -379,5 +497,6 @@ def get_xpath(xpath_str):
379497
380498
381499def eval_xpath(element, xpath_str):
500+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""
382501 xpath = get_xpath(xpath_str)
383502 return xpath(element)