c1d10bd · searx · git.iko.soy

git.iko.soy/searx

on master help sign in

[mod] searx/utils.py: add docstring

c1d10bdAlexandre Flament6 years ago

searx/utils.py+157−38

compare with master

unifiedsplit

@@ -39,12 +39,17 @@ lang_to_lc_cache = dict()

41def searx_useragent():

42+ """Return the searx User Agent"""

42 return 'searx/{searx_version} {suffix}'.format(

43 return 'searx/{searx_version} {suffix}'.format(

43 searx_version=VERSION_STRING,

44 searx_version=VERSION_STRING,

44 suffix=settings['outgoing'].get('useragent_suffix', ''))

45 suffix=settings['outgoing'].get('useragent_suffix', ''))

47def gen_useragent(os=None):

48def gen_useragent(os=None):

49+ """Return a random browser User Agent

50+

51+ See searx/data/useragents.json

52+ """

48 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

53 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

@@ -99,24 +104,40 @@ class HTMLTextExtractor(HTMLParser):

99 return ''.join(self.result).strip()

104 return ''.join(self.result).strip()

100

105

101

106

102−def html_to_text(html):

107+def html_to_text(html_str):

108+ """Extract text from a HTML string

109+

110+ Args:

111+ * html_str (str): string HTML

112+

113+ Returns:

114+ * str: extracted text

115+

116+ Examples:

117+ >>> html_to_text('Example <span id="42">#2</span>')

118+ 'Example #2'

119+

120+ >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')

121+ 'Example'

122+ """

103− html = html.replace('\n', ' ')

123+ html_str = html_str.replace('\n', ' ')

104− html = ' '.join(html.split())

124+ html_str = ' '.join(html_str.split())

105 s = HTMLTextExtractor()

125 s = HTMLTextExtractor()

106 try:

126 try:

107− s.feed(html)

127+ s.feed(html_str)

108 except HTMLTextExtractorException:

128 except HTMLTextExtractorException:

109− logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)

129+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str)

110 return s.get_text()

130 return s.get_text()

111

131

112

132

113def extract_text(xpath_results):

133def extract_text(xpath_results):

114− '''

134+ """Extract text from a lxml result

135+

115− if xpath_results is list, extract the text from each result and concat the list

136+ * if xpath_results is list, extract the text from each result and concat the list

116− if xpath_results is a xml element, extract all the text node from it

137+ * if xpath_results is a xml element, extract all the text node from it

117− ( text_content() method from lxml )

138+ ( text_content() method from lxml )

118− if xpath_results is a string element, then it's already done

139+ * if xpath_results is a string element, then it's already done

119− '''

140+ """

120 if type(xpath_results) == list:

141 if type(xpath_results) == list:

121 # it's list of result : concat everything using recursive call

142 # it's list of result : concat everything using recursive call

122 result = ''

143 result = ''

@@ -135,7 +156,58 @@ def extract_text(xpath_results):

135 return ' '.join(text.split())

156 return ' '.join(text.split())

136

157

137

158

159+def normalize_url(url):

160+ parsed_url = urlparse(url)

161+

162+ # add a / at this end of the url if there is no path

163+ if not parsed_url.netloc:

164+ raise Exception('Cannot parse url')

165+ if not parsed_url.path:

166+ url += '/'

167+

168+ # FIXME : hack for yahoo

169+ if parsed_url.hostname == 'search.yahoo.com'\

170+ and parsed_url.path.startswith('/r'):

171+ p = parsed_url.path

172+ mark = p.find('/**')

173+ if mark != -1:

174+ return unquote(p[mark + 3:]).decode()

175+

176+ return url

177+

178+

138def extract_url(xpath_results, search_url):

179def extract_url(xpath_results, search_url):

180+ """Extract and normalize URL from lxml Element

181+

182+ Args:

183+ * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)

184+ * search_url (str): Base URL

185+

186+ Example:

187+ >>> def f(s, search_url):

188+ >>> return searx.utils.extract_url(html.fromstring(s), search_url)

189+ >>> f('<span id="42">https://example.com</span>', 'http://example.com/')

190+ 'https://example.com/'

191+ >>> f('https://example.com', 'http://example.com/')

192+ 'https://example.com/'

193+ >>> f('//example.com', 'http://example.com/')

194+ 'http://example.com/'

195+ >>> f('//example.com', 'https://example.com/')

196+ 'https://example.com/'

197+ >>> f('/path?a=1', 'https://example.com')

198+ 'https://example.com/path?a=1'

199+ >>> f('', 'https://example.com')

200+ raise lxml.etree.ParserError

201+ >>> searx.utils.extract_url([], 'https://example.com')

202+ raise Exception

203+

204+ Raises:

205+ * Exception

206+ * lxml.etree.ParserError

207+

208+ Returns:

209+ * str: normalized URL

210+ """

139 if xpath_results == []:

211 if xpath_results == []:

140 raise Exception('Empty url resultset')

212 raise Exception('Empty url resultset')

141 url = extract_text(xpath_results)

213 url = extract_text(xpath_results)

@@ -158,27 +230,15 @@ def extract_url(xpath_results, search_url):

158 return url

230 return url

159

231

160

232

161−def normalize_url(url):

162− parsed_url = urlparse(url)

163−

164− # add a / at this end of the url if there is no path

165− if not parsed_url.netloc:

166− raise Exception('Cannot parse url')

167− if not parsed_url.path:

168− url += '/'

169−

170− # FIXME : hack for yahoo

171− if parsed_url.hostname == 'search.yahoo.com'\

172− and parsed_url.path.startswith('/r'):

173− p = parsed_url.path

174− mark = p.find('/**')

175− if mark != -1:

176− return unquote(p[mark + 3:]).decode()

177−

178− return url

179−

180−

181def dict_subset(d, properties):

233def dict_subset(d, properties):

234+ """Extract a subset of a dict

235+

236+ Examples:

237+ >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])

238+ {'A': 'a', 'C': 'c'}

239+ >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

240+ {'A': 'a'}

241+ """

182 result = {}

242 result = {}

183 for k in properties:

243 for k in properties:

184 if k in d:

244 if k in d:

@@ -186,8 +246,19 @@ def dict_subset(d, properties):

186 return result

246 return result

187

247

188

248

189−# get element in list or default value

190def list_get(a_list, index, default=None):

249def list_get(a_list, index, default=None):

250+ """Get element in list or default value

251+

252+ Examples:

253+ >>> list_get(['A', 'B', 'C'], 0)

254+ 'A'

255+ >>> list_get(['A', 'B', 'C'], 3)

256+ None

257+ >>> list_get(['A', 'B', 'C'], 3, 'default')

258+ 'default'

259+ >>> list_get(['A', 'B', 'C'], -1)

260+ 'C'

261+ """

191 if len(a_list) > index:

262 if len(a_list) > index:

192 return a_list[index]

263 return a_list[index]

193 else:

264 else:

@@ -195,6 +266,21 @@ def list_get(a_list, index, default=None):

195

266

196

267

197def get_torrent_size(filesize, filesize_multiplier):

268def get_torrent_size(filesize, filesize_multiplier):

269+ """

270+

271+ Args:

272+ * filesize (str): size

273+ * filesize_multiplier (str): TB, GB, .... TiB, GiB...

274+

275+ Returns:

276+ * int: number of bytes

277+

278+ Example:

279+ >>> get_torrent_size('5', 'GB')

280+ 5368709120

281+ >>> get_torrent_size('3.14', 'MiB')

282+ 3140000

283+ """

198 try:

284 try:

199 filesize = float(filesize)

285 filesize = float(filesize)

200

286

@@ -221,14 +307,18 @@ def get_torrent_size(filesize, filesize_multiplier):

221

307

222

308

223def convert_str_to_int(number_str):

309def convert_str_to_int(number_str):

310+ """Convert number_str to int or 0 if number_str is not a number."""

224 if number_str.isdigit():

311 if number_str.isdigit():

225 return int(number_str)

312 return int(number_str)

226 else:

313 else:

227 return 0

314 return 0

228

315

229

316

230−# convert a variable to integer or return 0 if it's not a number

231def int_or_zero(num):

317def int_or_zero(num):

318+ """Convert num to int or 0. num can be either a str or a list.

319+ If num is a list, the first element is converted to int (or return 0 if the list is empty).

320+ If num is a str, see convert_str_to_int

321+ """

232 if isinstance(num, list):

322 if isinstance(num, list):

233 if len(num) < 1:

323 if len(num) < 1:

234 return 0

324 return 0

@@ -237,6 +327,22 @@ def int_or_zero(num):

237

327

238

328

239def is_valid_lang(lang):

329def is_valid_lang(lang):

330+ """Return language code and name if lang describe a language.

331+

332+ Examples:

333+ >>> is_valid_lang('zz')

334+ False

335+ >>> is_valid_lang('uk')

336+ (True, 'uk', 'ukrainian')

337+ >>> is_valid_lang(b'uk')

338+ (True, 'uk', 'ukrainian')

339+ >>> is_valid_lang('en')

340+ (True, 'en', 'english')

341+ >>> searx.utils.is_valid_lang('Español')

342+ (True, 'es', 'spanish')

343+ >>> searx.utils.is_valid_lang('Spanish')

344+ (True, 'es', 'spanish')

345+ """

240 if isinstance(lang, bytes):

346 if isinstance(lang, bytes):

241 lang = lang.decode()

347 lang = lang.decode()

242 is_abbr = (len(lang) == 2)

348 is_abbr = (len(lang) == 2)

@@ -264,8 +370,8 @@ def _get_lang_to_lc_dict(lang_list):

264 return value

370 return value

265

371

266

372

267−# auxiliary function to match lang_code in lang_list

268def _match_language(lang_code, lang_list=[], custom_aliases={}):

373def _match_language(lang_code, lang_list=[], custom_aliases={}):

374+ """auxiliary function to match lang_code in lang_list"""

269 # replace language code with a custom alias if necessary

375 # replace language code with a custom alias if necessary

270 if lang_code in custom_aliases:

376 if lang_code in custom_aliases:

271 lang_code = custom_aliases[lang_code]

377 lang_code = custom_aliases[lang_code]

@@ -287,8 +393,8 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):

287 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)

393 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)

288

394

289

395

290−# get the language code from lang_list that best matches locale_code

291def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):

396def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):

397+ """get the language code from lang_list that best matches locale_code"""

292 # try to get language from given locale_code

398 # try to get language from given locale_code

293 language = _match_language(locale_code, lang_list, custom_aliases)

399 language = _match_language(locale_code, lang_list, custom_aliases)

294 if language:

400 if language:

@@ -330,6 +436,7 @@ def load_module(filename, module_dir):

330

436

331

437

332def to_string(obj):

438def to_string(obj):

439+ """Convert obj to its string representation."""

333 if isinstance(obj, str):

440 if isinstance(obj, str):

334 return obj

441 return obj

335 if isinstance(obj, Number):

442 if isinstance(obj, Number):

@@ -341,13 +448,19 @@ def to_string(obj):

341

448

342

449

343def ecma_unescape(s):

450def ecma_unescape(s):

344− """

345− python implementation of the unescape javascript function

451+ """Python implementation of the unescape javascript function

346

452

347 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string

453 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string

348 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape

454 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape

455+

456+ Examples:

457+ >>> ecma_unescape('%u5409')

458+ '吉'

459+ >>> ecma_unescape('%20')

460+ ' '

461+ >>> ecma_unescape('%F3')

462+ 'ó'

349 """

463 """

350− # s = unicode(s)

351 # "%u5409" becomes "吉"

464 # "%u5409" becomes "吉"

352 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)

465 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)

353 # "%20" becomes " ", "%F3" becomes "ó"

466 # "%20" becomes " ", "%F3" becomes "ó"

@@ -371,6 +484,11 @@ def get_engine_from_settings(name):

371

484

372

485

373def get_xpath(xpath_str):

486def get_xpath(xpath_str):

487+ """Return cached compiled XPath

488+

489+ There is no thread lock.

490+ Worst case scenario, xpath_str is compiled more than one time.

491+ """

374 result = xpath_cache.get(xpath_str, None)

492 result = xpath_cache.get(xpath_str, None)

375 if result is None:

493 if result is None:

376 result = XPath(xpath_str)

494 result = XPath(xpath_str)

@@ -379,5 +497,6 @@ def get_xpath(xpath_str):

379

497

380

498

381def eval_xpath(element, xpath_str):

499def eval_xpath(element, xpath_str):

500+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""

382 xpath = get_xpath(xpath_str)

501 xpath = get_xpath(xpath_str)

383 return xpath(element)

502 return xpath(element)