b728cb6 · searx · git.iko.soy

searx/engines/1337x.py+1−2

compare with master

unifiedsplit

@@ -1,7 +1,6 @@

1from urllib.parse import quote, urljoin

2from lxml import html

3−from searx.engines.xpath import extract_text

3+from searx.utils import extract_text, get_torrent_size

4−from searx.utils import get_torrent_size

5

4

6

5

7url = 'https://1337x.to/'

6url = 'https://1337x.to/'

searx/engines/acgsou.py+1−2

@@ -11,8 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml import html

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, get_torrent_size, int_or_zero

15−from searx.utils import get_torrent_size, int_or_zero

16

15

17# engine dependent config

16# engine dependent config

18categories = ['files', 'images', 'videos', 'music']

17categories = ['files', 'images', 'videos', 'music']

searx/engines/apkmirror.py+1−1

@@ -11,7 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml import html

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text

15

16

17# engine dependent config

searx/engines/archlinux.py+1−1

@@ -13,7 +13,7 @@

13

14from urllib.parse import urlencode, urljoin

15from lxml import html

16−from searx.engines.xpath import extract_text

16+from searx.utils import extract_text

17

18# engine dependent config

19categories = ['it']

searx/engines/bing.py+1−2

@@ -17,8 +17,7 @@ import re

17from urllib.parse import urlencode

18from lxml import html

19from searx import logger, utils

20−from searx.engines.xpath import extract_text

21−from searx.utils import match_language, gen_useragent, eval_xpath

20+from searx.utils import extract_text, match_language, gen_useragent, eval_xpath

22

21

23logger = logger.getChild('bing engine')

22logger = logger.getChild('bing engine')

24

23

searx/engines/btdigg.py+1−2

@@ -13,8 +13,7 @@

13from lxml import html

14from operator import itemgetter

15from urllib.parse import quote, urljoin

16−from searx.engines.xpath import extract_text

16+from searx.utils import extract_text, get_torrent_size

17−from searx.utils import get_torrent_size

18

17

19# engine dependent config

18# engine dependent config

20categories = ['videos', 'music', 'files']

19categories = ['videos', 'music', 'files']

searx/engines/deviantart.py+1−1

@@ -15,7 +15,7 @@

15from lxml import html

16import re

17from urllib.parse import urlencode

18−from searx.engines.xpath import extract_text

18+from searx.utils import extract_text

19

20

21# engine dependent config

searx/engines/digbt.py+1−2

@@ -12,8 +12,7 @@

12

13from urllib.parse import urljoin

14from lxml import html

15−from searx.engines.xpath import extract_text

15+from searx.utils import extract_text, get_torrent_size

16−from searx.utils import get_torrent_size

17

16

18

17

19categories = ['videos', 'music', 'files']

18categories = ['videos', 'music', 'files']

searx/engines/doku.py+1−2

@@ -11,8 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml.html import fromstring

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, eval_xpath

15−from searx.utils import eval_xpath

16

15

17# engine dependent config

16# engine dependent config

18categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

17categories = ['general'] # TODO , 'images', 'music', 'videos', 'files'

searx/engines/duckduckgo.py+1−2

@@ -16,9 +16,8 @@

16from lxml.html import fromstring

17from json import loads

18from urllib.parse import urlencode

19−from searx.engines.xpath import extract_text

20from searx.poolrequests import get

19from searx.poolrequests import get

21−from searx.utils import match_language, eval_xpath

20+from searx.utils import extract_text, match_language, eval_xpath

22

21

23# engine dependent config

22# engine dependent config

24categories = ['general']

23categories = ['general']

searx/engines/duckduckgo_definitions.py+1−2

@@ -13,9 +13,8 @@ import json

13from urllib.parse import urlencode

14from lxml import html

15from re import compile

16−from searx.engines.xpath import extract_text

17from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases

16from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url, language_aliases

18−from searx.utils import html_to_text, match_language

17+from searx.utils import extract_text, html_to_text, match_language

19

18

20url = 'https://api.duckduckgo.com/'\

19url = 'https://api.duckduckgo.com/'\

21 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

20 + '?{query}&format=json&pretty=0&no_redirect=1&d=1'

searx/engines/duckduckgo_images.py+1−1

@@ -15,12 +15,12 @@

15

16from json import loads

17from urllib.parse import urlencode

18−from searx.engines.xpath import extract_text

19from searx.engines.duckduckgo import (

18from searx.engines.duckduckgo import (

20 _fetch_supported_languages, supported_languages_url,

19 _fetch_supported_languages, supported_languages_url,

21 get_region_code, language_aliases

20 get_region_code, language_aliases

22)

21)

23from searx.poolrequests import get

22from searx.poolrequests import get

23+from searx.utils import extract_text

24

25# engine dependent config

26categories = ['images']

searx/engines/duden.py+1−2

@@ -11,8 +11,7 @@

11from lxml import html, etree

12import re

13from urllib.parse import quote, urljoin

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, eval_xpath

15−from searx.utils import eval_xpath

16from searx import logger

15from searx import logger

17

16

18categories = ['general']

17categories = ['general']

searx/engines/etools.py+1−2

@@ -11,8 +11,7 @@

11

12from lxml import html

13from urllib.parse import quote

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, eval_xpath

15−from searx.utils import eval_xpath

16

15

17categories = ['general']

16categories = ['general']

18paging = False

17paging = False

searx/engines/fdroid.py+1−1

@@ -11,7 +11,7 @@

11

12from urllib.parse import urlencode

13from lxml import html

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text

15

16# engine dependent config

17categories = ['files']

searx/engines/framalibre.py+1−1

@@ -13,7 +13,7 @@

13from html import escape

14from urllib.parse import urljoin, urlencode

15from lxml import html

16−from searx.engines.xpath import extract_text

16+from searx.utils import extract_text

17

18# engine dependent config

19categories = ['it']

searx/engines/gentoo.py+1−1

@@ -13,7 +13,7 @@

13

14from urllib.parse import urlencode, urljoin

15from lxml import html

16−from searx.engines.xpath import extract_text

16+from searx.utils import extract_text

17

18# engine dependent config

19categories = ['it']

searx/engines/google.py+1−2

@@ -21,9 +21,8 @@ Definitions`_.

21from urllib.parse import urlencode, urlparse

22from lxml import html

23from flask_babel import gettext

24−from searx.engines.xpath import extract_text

25from searx import logger

24from searx import logger

26−from searx.utils import match_language, eval_xpath

25+from searx.utils import match_language, extract_text, eval_xpath

27

26

28logger = logger.getChild('google engine')

27logger = logger.getChild('google engine')

29

28

searx/engines/google_images.py+1−2

@@ -28,8 +28,7 @@ from urllib.parse import urlencode, urlparse, unquote

28from lxml import html

29from flask_babel import gettext

30from searx import logger

31−from searx.utils import eval_xpath

31+from searx.utils import extract_text, eval_xpath

32−from searx.engines.xpath import extract_text

33

32

34# pylint: disable=unused-import

33# pylint: disable=unused-import

35from searx.engines.google import (

34from searx.engines.google import (

searx/engines/google_videos.py+1−1

@@ -14,7 +14,7 @@ from datetime import date, timedelta

14from json import loads

15from urllib.parse import urlencode

16from lxml import html

17−from searx.engines.xpath import extract_text

17+from searx.utils import extract_text

18import re

19

20# engine dependent config

searx/engines/ina.py+1−1

@@ -16,7 +16,7 @@ from urllib.parse import urlencode

16from lxml import html

17from dateutil import parser

18from html.parser import HTMLParser

19−from searx.engines.xpath import extract_text

19+from searx.utils import extract_text

20

21

22# engine dependent config

searx/engines/kickass.py+1−2

@@ -13,8 +13,7 @@

13from lxml import html

14from operator import itemgetter

15from urllib.parse import quote, urljoin

16−from searx.engines.xpath import extract_text

17−from searx.utils import get_torrent_size, convert_str_to_int

16+from searx.utils import extract_text, get_torrent_size, convert_str_to_int

18

17

19# engine dependent config

18# engine dependent config

20categories = ['videos', 'music', 'files']

19categories = ['videos', 'music', 'files']

searx/engines/nyaa.py+1−2

@@ -11,8 +11,7 @@

11

12from lxml import html

13from urllib.parse import urlencode

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, get_torrent_size, int_or_zero

15−from searx.utils import get_torrent_size, int_or_zero

16

15

17# engine dependent config

16# engine dependent config

18categories = ['files', 'images', 'videos', 'music']

17categories = ['files', 'images', 'videos', 'music']

searx/engines/piratebay.py+1−2

@@ -13,8 +13,7 @@ from datetime import datetime

13from operator import itemgetter

14

15from urllib.parse import quote, urljoin

16−from searx.engines.xpath import extract_text

16+from searx.utils import extract_text, get_torrent_size

17−from searx.utils import get_torrent_size

18

17

19# engine dependent config

18# engine dependent config

20categories = ["videos", "music", "files"]

19categories = ["videos", "music", "files"]

searx/engines/seedpeer.py+1−1

@@ -12,7 +12,7 @@ from lxml import html

12from json import loads

13from operator import itemgetter

14from urllib.parse import quote, urljoin

15−from searx.engines.xpath import extract_text

15+from searx.utils import extract_text

16

17

18url = 'https://seedpeer.me/'

searx/engines/stackoverflow.py+1−1

@@ -12,7 +12,7 @@

12

13from urllib.parse import urlencode, urljoin

14from lxml import html

15−from searx.engines.xpath import extract_text

15+from searx.utils import extract_text

16

17# engine dependent config

18categories = ['it']

searx/engines/startpage.py+1−2

@@ -17,9 +17,8 @@ import re

17from unicodedata import normalize, combining

18from babel import Locale

19from babel.localedata import locale_identifiers

20−from searx.engines.xpath import extract_text

21from searx.languages import language_codes

20from searx.languages import language_codes

22−from searx.utils import eval_xpath, match_language

21+from searx.utils import extract_text, eval_xpath, match_language

23

22

24# engine dependent config

23# engine dependent config

25categories = ['general']

24categories = ['general']

searx/engines/tokyotoshokan.py+1−2

@@ -13,9 +13,8 @@

13import re

14from urllib.parse import urlencode

15from lxml import html

16−from searx.engines.xpath import extract_text

17from datetime import datetime

16from datetime import datetime

18−from searx.utils import get_torrent_size, int_or_zero

17+from searx.utils import extract_text, get_torrent_size, int_or_zero

19

18

20# engine dependent config

19# engine dependent config

21categories = ['files', 'videos', 'music']

20categories = ['files', 'videos', 'music']

searx/engines/torrentz.py+1−2

@@ -15,8 +15,7 @@ import re

15from urllib.parse import urlencode

16from lxml import html

17from datetime import datetime

18−from searx.engines.xpath import extract_text

18+from searx.utils import extract_text, get_torrent_size

19−from searx.utils import get_torrent_size

20

19

21# engine dependent config

20# engine dependent config

22categories = ['files', 'videos', 'music']

21categories = ['files', 'videos', 'music']

searx/engines/twitter.py+1−1

@@ -15,7 +15,7 @@

15from urllib.parse import urlencode, urljoin

16from lxml import html

17from datetime import datetime

18−from searx.engines.xpath import extract_text

18+from searx.utils import extract_text

19

20# engine dependent config

21categories = ['social media']

searx/engines/wikidata.py+1−2

@@ -13,9 +13,8 @@

13

14from searx import logger

15from searx.poolrequests import get

16−from searx.engines.xpath import extract_text

17from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

16from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url

18−from searx.utils import match_language, eval_xpath

17+from searx.utils import extract_text, match_language, eval_xpath

19

18

20from urllib.parse import urlencode

19from urllib.parse import urlencode

21from json import loads

20from json import loads

searx/engines/www1x.py+1−1

@@ -12,7 +12,7 @@

12

13from lxml import html

14from urllib.parse import urlencode, urljoin

15−from searx.engines.xpath import extract_text

15+from searx.utils import extract_text

16

17# engine dependent config

18categories = ['images']

searx/engines/xpath.py+2−73

@@ -1,7 +1,6 @@

1−from urllib.parse import unquote, urlencode, urljoin, urlparse

2from lxml import html

1from lxml import html

3−from lxml.etree import _ElementStringResult, _ElementUnicodeResult

2+from urllib.parse import urlencode

4−from searx.utils import html_to_text, eval_xpath

3+from searx.utils import extract_text, extract_url, eval_xpath

5

4

6search_url = None

5search_url = None

7url_xpath = None

6url_xpath = None

@@ -21,76 +20,6 @@ page_size = 1

21first_page_num = 1

20first_page_num = 1

22

21

23

22

24−'''

25−if xpath_results is list, extract the text from each result and concat the list

26−if xpath_results is a xml element, extract all the text node from it

27− ( text_content() method from lxml )

28−if xpath_results is a string element, then it's already done

29−'''

30−

31−

32−def extract_text(xpath_results):

33− if type(xpath_results) == list:

34− # it's list of result : concat everything using recursive call

35− result = ''

36− for e in xpath_results:

37− result = result + extract_text(e)

38− return result.strip()

39− elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:

40− # it's a string

41− return ''.join(xpath_results)

42− else:

43− # it's a element

44− text = html.tostring(

45− xpath_results, encoding='unicode', method='text', with_tail=False

46− )

47− text = text.strip().replace('\n', ' ')

48− return ' '.join(text.split())

49−

50−

51−def extract_url(xpath_results, search_url):

52− if xpath_results == []:

53− raise Exception('Empty url resultset')

54− url = extract_text(xpath_results)

55−

56− if url.startswith('//'):

57− # add http or https to this kind of url //example.com/

58− parsed_search_url = urlparse(search_url)

59− url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

60− elif url.startswith('/'):

61− # fix relative url to the search engine

62− url = urljoin(search_url, url)

63−

64− # fix relative urls that fall through the crack

65− if '://' not in url:

66− url = urljoin(search_url, url)

67−

68− # normalize url

69− url = normalize_url(url)

70−

71− return url

72−

73−

74−def normalize_url(url):

75− parsed_url = urlparse(url)

76−

77− # add a / at this end of the url if there is no path

78− if not parsed_url.netloc:

79− raise Exception('Cannot parse url')

80− if not parsed_url.path:

81− url += '/'

82−

83− # FIXME : hack for yahoo

84− if parsed_url.hostname == 'search.yahoo.com'\

85− and parsed_url.path.startswith('/r'):

86− p = parsed_url.path

87− mark = p.find('/**')

88− if mark != -1:

89− return unquote(p[mark + 3:]).decode()

90−

91− return url

92−

93−

94def request(query, params):

23def request(query, params):

95 query = urlencode({'q': query})[2:]

24 query = urlencode({'q': query})[2:]

96

25

searx/engines/yahoo.py+1−2

@@ -13,8 +13,7 @@

13

14from urllib.parse import unquote, urlencode

15from lxml import html

16−from searx.engines.xpath import extract_text, extract_url

16+from searx.utils import extract_text, extract_url, match_language, eval_xpath

17−from searx.utils import match_language, eval_xpath

18

17

19# engine dependent config

18# engine dependent config

20categories = ['general']

19categories = ['general']

searx/engines/yahoo_news.py+1−2

@@ -13,12 +13,11 @@ import re

13from datetime import datetime, timedelta

14from urllib.parse import urlencode

15from lxml import html

16−from searx.engines.xpath import extract_text, extract_url

17from searx.engines.yahoo import (

16from searx.engines.yahoo import (

18 parse_url, _fetch_supported_languages, supported_languages_url, language_aliases

17 parse_url, _fetch_supported_languages, supported_languages_url, language_aliases

19)

18)

20from dateutil import parser

19from dateutil import parser

21−from searx.utils import match_language

20+from searx.utils import extract_text, extract_url, match_language

22

21

23# engine dependent config

22# engine dependent config

24categories = ['news']

23categories = ['news']

searx/engines/yggtorrent.py+1−2

@@ -12,8 +12,7 @@ from lxml import html

12from operator import itemgetter

13from datetime import datetime

14from urllib.parse import quote

15−from searx.engines.xpath import extract_text

15+from searx.utils import extract_text, get_torrent_size

16−from searx.utils import get_torrent_size

17from searx.poolrequests import get as http_get

16from searx.poolrequests import get as http_get

18

17

19# engine dependent config

18# engine dependent config

searx/engines/youtube_noapi.py+1−2

@@ -11,8 +11,7 @@

11from functools import reduce

12from json import loads

13from urllib.parse import quote_plus

14−from searx.engines.xpath import extract_text

14+from searx.utils import extract_text, list_get

15−from searx.utils import list_get

16

15

17# engine dependent config

16# engine dependent config

18categories = ['videos', 'music']

17categories = ['videos', 'music']

searx/utils.py+219−13

@@ -10,9 +10,13 @@ from os.path import splitext, join

10from io import open

11from random import choice

12from html.parser import HTMLParser

13+from urllib.parse import urljoin, urlparse, unquote

14+

15+from lxml import html

13−from lxml.etree import XPath

16+from lxml.etree import XPath, _ElementStringResult, _ElementUnicodeResult

14from babel.core import get_global

17from babel.core import get_global

15

18

19+

16from searx import settings

20from searx import settings

17from searx.version import VERSION_STRING

21from searx.version import VERSION_STRING

18from searx.languages import language_codes

22from searx.languages import language_codes

@@ -35,12 +39,17 @@ lang_to_lc_cache = dict()

35

39

36

40

37def searx_useragent():

41def searx_useragent():

42+ """Return the searx User Agent"""

38 return 'searx/{searx_version} {suffix}'.format(

43 return 'searx/{searx_version} {suffix}'.format(

39 searx_version=VERSION_STRING,

44 searx_version=VERSION_STRING,

40 suffix=settings['outgoing'].get('useragent_suffix', ''))

45 suffix=settings['outgoing'].get('useragent_suffix', ''))

41

46

42

47

43def gen_useragent(os=None):

48def gen_useragent(os=None):

49+ """Return a random browser User Agent

50+

51+ See searx/data/useragents.json

52+ """

44 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

53 return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))

45

54

46

55

@@ -95,18 +104,156 @@ class HTMLTextExtractor(HTMLParser):

95 return ''.join(self.result).strip()

104 return ''.join(self.result).strip()

96

105

97

106

98−def html_to_text(html):

107+def html_to_text(html_str):

108+ """Extract text from a HTML string

109+

110+ Args:

111+ * html_str (str): string HTML

112+

113+ Returns:

114+ * str: extracted text

115+

116+ Examples:

117+ >>> html_to_text('Example <span id="42">#2</span>')

118+ 'Example #2'

119+

120+ >>> html_to_text('<style>.span { color: red; }</style><span>Example</span>')

121+ 'Example'

122+ """

99− html = html.replace('\n', ' ')

123+ html_str = html_str.replace('\n', ' ')

100− html = ' '.join(html.split())

124+ html_str = ' '.join(html_str.split())

101 s = HTMLTextExtractor()

125 s = HTMLTextExtractor()

102 try:

126 try:

103− s.feed(html)

127+ s.feed(html_str)

104 except HTMLTextExtractorException:

128 except HTMLTextExtractorException:

105− logger.debug("HTMLTextExtractor: invalid HTML\n%s", html)

129+ logger.debug("HTMLTextExtractor: invalid HTML\n%s", html_str)

106 return s.get_text()

130 return s.get_text()

107

131

108

132

133+def extract_text(xpath_results):

134+ """Extract text from a lxml result

135+

136+ * if xpath_results is list, extract the text from each result and concat the list

137+ * if xpath_results is a xml element, extract all the text node from it

138+ ( text_content() method from lxml )

139+ * if xpath_results is a string element, then it's already done

140+ """

141+ if type(xpath_results) == list:

142+ # it's list of result : concat everything using recursive call

143+ result = ''

144+ for e in xpath_results:

145+ result = result + extract_text(e)

146+ return result.strip()

147+ elif type(xpath_results) in [_ElementStringResult, _ElementUnicodeResult]:

148+ # it's a string

149+ return ''.join(xpath_results)

150+ else:

151+ # it's a element

152+ text = html.tostring(

153+ xpath_results, encoding='unicode', method='text', with_tail=False

154+ )

155+ text = text.strip().replace('\n', ' ')

156+ return ' '.join(text.split())

157+

158+

159+def normalize_url(url, base_url):

160+ """Normalize URL: add protocol, join URL with base_url, add trailing slash if there is no path

161+

162+ Args:

163+ * url (str): Relative URL

164+ * base_url (str): Base URL, it must be an absolute URL.

165+

166+ Example:

167+ >>> normalize_url('https://example.com', 'http://example.com/')

168+ 'https://example.com/'

169+ >>> normalize_url('//example.com', 'http://example.com/')

170+ 'http://example.com/'

171+ >>> normalize_url('//example.com', 'https://example.com/')

172+ 'https://example.com/'

173+ >>> normalize_url('/path?a=1', 'https://example.com')

174+ 'https://example.com/path?a=1'

175+ >>> normalize_url('', 'https://example.com')

176+ 'https://example.com/'

177+ >>> normalize_url('/test', '/path')

178+ raise Exception

179+

180+ Raises:

181+ * lxml.etree.ParserError

182+

183+ Returns:

184+ * str: normalized URL

185+ """

186+ if url.startswith('//'):

187+ # add http or https to this kind of url //example.com/

188+ parsed_search_url = urlparse(base_url)

189+ url = '{0}:{1}'.format(parsed_search_url.scheme or 'http', url)

190+ elif url.startswith('/'):

191+ # fix relative url to the search engine

192+ url = urljoin(base_url, url)

193+

194+ # fix relative urls that fall through the crack

195+ if '://' not in url:

196+ url = urljoin(base_url, url)

197+

198+ parsed_url = urlparse(url)

199+

200+ # add a / at this end of the url if there is no path

201+ if not parsed_url.netloc:

202+ raise Exception('Cannot parse url')

203+ if not parsed_url.path:

204+ url += '/'

205+

206+ return url

207+

208+

209+def extract_url(xpath_results, base_url):

210+ """Extract and normalize URL from lxml Element

211+

212+ Args:

213+ * xpath_results (Union[List[html.HtmlElement], html.HtmlElement]): lxml Element(s)

214+ * base_url (str): Base URL

215+

216+ Example:

217+ >>> def f(s, search_url):

218+ >>> return searx.utils.extract_url(html.fromstring(s), search_url)

219+ >>> f('<span id="42">https://example.com</span>', 'http://example.com/')

220+ 'https://example.com/'

221+ >>> f('https://example.com', 'http://example.com/')

222+ 'https://example.com/'

223+ >>> f('//example.com', 'http://example.com/')

224+ 'http://example.com/'

225+ >>> f('//example.com', 'https://example.com/')

226+ 'https://example.com/'

227+ >>> f('/path?a=1', 'https://example.com')

228+ 'https://example.com/path?a=1'

229+ >>> f('', 'https://example.com')

230+ raise lxml.etree.ParserError

231+ >>> searx.utils.extract_url([], 'https://example.com')

232+ raise Exception

233+

234+ Raises:

235+ * Exception

236+ * lxml.etree.ParserError

237+

238+ Returns:

239+ * str: normalized URL

240+ """

241+ if xpath_results == []:

242+ raise Exception('Empty url resultset')

243+

244+ url = extract_text(xpath_results)

245+ return normalize_url(url, base_url)

246+

247+

109def dict_subset(d, properties):

248def dict_subset(d, properties):

249+ """Extract a subset of a dict

250+

251+ Examples:

252+ >>> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'C'])

253+ {'A': 'a', 'C': 'c'}

254+ >>> >> dict_subset({'A': 'a', 'B': 'b', 'C': 'c'}, ['A', 'D'])

255+ {'A': 'a'}

256+ """

110 result = {}

257 result = {}

111 for k in properties:

258 for k in properties:

112 if k in d:

259 if k in d:

@@ -114,8 +261,19 @@ def dict_subset(d, properties):

114 return result

261 return result

115

262

116

263

117−# get element in list or default value

118def list_get(a_list, index, default=None):

264def list_get(a_list, index, default=None):

265+ """Get element in list or default value

266+

267+ Examples:

268+ >>> list_get(['A', 'B', 'C'], 0)

269+ 'A'

270+ >>> list_get(['A', 'B', 'C'], 3)

271+ None

272+ >>> list_get(['A', 'B', 'C'], 3, 'default')

273+ 'default'

274+ >>> list_get(['A', 'B', 'C'], -1)

275+ 'C'

276+ """

119 if len(a_list) > index:

277 if len(a_list) > index:

120 return a_list[index]

278 return a_list[index]

121 else:

279 else:

@@ -123,6 +281,21 @@ def list_get(a_list, index, default=None):

123

281

124

282

125def get_torrent_size(filesize, filesize_multiplier):

283def get_torrent_size(filesize, filesize_multiplier):

284+ """

285+

286+ Args:

287+ * filesize (str): size

288+ * filesize_multiplier (str): TB, GB, .... TiB, GiB...

289+

290+ Returns:

291+ * int: number of bytes

292+

293+ Example:

294+ >>> get_torrent_size('5', 'GB')

295+ 5368709120

296+ >>> get_torrent_size('3.14', 'MiB')

297+ 3140000

298+ """

126 try:

299 try:

127 filesize = float(filesize)

300 filesize = float(filesize)

128

301

@@ -149,14 +322,18 @@ def get_torrent_size(filesize, filesize_multiplier):

149

322

150

323

151def convert_str_to_int(number_str):

324def convert_str_to_int(number_str):

325+ """Convert number_str to int or 0 if number_str is not a number."""

152 if number_str.isdigit():

326 if number_str.isdigit():

153 return int(number_str)

327 return int(number_str)

154 else:

328 else:

155 return 0

329 return 0

156

330

157

331

158−# convert a variable to integer or return 0 if it's not a number

159def int_or_zero(num):

332def int_or_zero(num):

333+ """Convert num to int or 0. num can be either a str or a list.

334+ If num is a list, the first element is converted to int (or return 0 if the list is empty).

335+ If num is a str, see convert_str_to_int

336+ """

160 if isinstance(num, list):

337 if isinstance(num, list):

161 if len(num) < 1:

338 if len(num) < 1:

162 return 0

339 return 0

@@ -165,6 +342,22 @@ def int_or_zero(num):

165

342

166

343

167def is_valid_lang(lang):

344def is_valid_lang(lang):

345+ """Return language code and name if lang describe a language.

346+

347+ Examples:

348+ >>> is_valid_lang('zz')

349+ False

350+ >>> is_valid_lang('uk')

351+ (True, 'uk', 'ukrainian')

352+ >>> is_valid_lang(b'uk')

353+ (True, 'uk', 'ukrainian')

354+ >>> is_valid_lang('en')

355+ (True, 'en', 'english')

356+ >>> searx.utils.is_valid_lang('Español')

357+ (True, 'es', 'spanish')

358+ >>> searx.utils.is_valid_lang('Spanish')

359+ (True, 'es', 'spanish')

360+ """

168 if isinstance(lang, bytes):

361 if isinstance(lang, bytes):

169 lang = lang.decode()

362 lang = lang.decode()

170 is_abbr = (len(lang) == 2)

363 is_abbr = (len(lang) == 2)

@@ -192,8 +385,8 @@ def _get_lang_to_lc_dict(lang_list):

192 return value

385 return value

193

386

194

387

195−# auxiliary function to match lang_code in lang_list

196def _match_language(lang_code, lang_list=[], custom_aliases={}):

388def _match_language(lang_code, lang_list=[], custom_aliases={}):

389+ """auxiliary function to match lang_code in lang_list"""

197 # replace language code with a custom alias if necessary

390 # replace language code with a custom alias if necessary

198 if lang_code in custom_aliases:

391 if lang_code in custom_aliases:

199 lang_code = custom_aliases[lang_code]

392 lang_code = custom_aliases[lang_code]

@@ -215,8 +408,8 @@ def _match_language(lang_code, lang_list=[], custom_aliases={}):

215 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)

408 return _get_lang_to_lc_dict(lang_list).get(lang_code, None)

216

409

217

410

218−# get the language code from lang_list that best matches locale_code

219def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):

411def match_language(locale_code, lang_list=[], custom_aliases={}, fallback='en-US'):

412+ """get the language code from lang_list that best matches locale_code"""

220 # try to get language from given locale_code

413 # try to get language from given locale_code

221 language = _match_language(locale_code, lang_list, custom_aliases)

414 language = _match_language(locale_code, lang_list, custom_aliases)

222 if language:

415 if language:

@@ -258,6 +451,7 @@ def load_module(filename, module_dir):

258

451

259

452

260def to_string(obj):

453def to_string(obj):

454+ """Convert obj to its string representation."""

261 if isinstance(obj, str):

455 if isinstance(obj, str):

262 return obj

456 return obj

263 if isinstance(obj, Number):

457 if isinstance(obj, Number):

@@ -269,13 +463,19 @@ def to_string(obj):

269

463

270

464

271def ecma_unescape(s):

465def ecma_unescape(s):

272− """

273− python implementation of the unescape javascript function

466+ """Python implementation of the unescape javascript function

274

467

275 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string

468 https://www.ecma-international.org/ecma-262/6.0/#sec-unescape-string

276 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape

469 https://developer.mozilla.org/fr/docs/Web/JavaScript/Reference/Objets_globaux/unescape

470+

471+ Examples:

472+ >>> ecma_unescape('%u5409')

473+ '吉'

474+ >>> ecma_unescape('%20')

475+ ' '

476+ >>> ecma_unescape('%F3')

477+ 'ó'

277 """

478 """

278− # s = unicode(s)

279 # "%u5409" becomes "吉"

479 # "%u5409" becomes "吉"

280 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)

480 s = ecma_unescape4_re.sub(lambda e: chr(int(e.group(1), 16)), s)

281 # "%20" becomes " ", "%F3" becomes "ó"

481 # "%20" becomes " ", "%F3" becomes "ó"

@@ -299,6 +499,11 @@ def get_engine_from_settings(name):

299

499

300

500

301def get_xpath(xpath_str):

501def get_xpath(xpath_str):

502+ """Return cached compiled XPath

503+

504+ There is no thread lock.

505+ Worst case scenario, xpath_str is compiled more than one time.

506+ """

302 result = xpath_cache.get(xpath_str, None)

507 result = xpath_cache.get(xpath_str, None)

303 if result is None:

508 if result is None:

304 result = XPath(xpath_str)

509 result = XPath(xpath_str)

@@ -307,5 +512,6 @@ def get_xpath(xpath_str):

307

512

308

513

309def eval_xpath(element, xpath_str):

514def eval_xpath(element, xpath_str):

515+ """Equivalent of element.xpath(xpath_str) but compile xpath_str once for all."""

310 xpath = get_xpath(xpath_str)

516 xpath = get_xpath(xpath_str)

311 return xpath(element)

517 return xpath(element)

tests/unit/test_utils.py+45−4

@@ -1,4 +1,7 @@

1# -*- coding: utf-8 -*-

2+import lxml.etree

3+from lxml import html

4+

2from searx.testing import SearxTestCase

5from searx.testing import SearxTestCase

3from searx import utils

6from searx import utils

4

7

@@ -16,7 +19,30 @@ class TestUtils(SearxTestCase):

16 self.assertTrue(utils.searx_useragent().startswith('searx'))

19 self.assertTrue(utils.searx_useragent().startswith('searx'))

17

20

18 def test_html_to_text(self):

21 def test_html_to_text(self):

19− html = """

22+ html_str = """

23+ <a href="/testlink" class="link_access_account">

24+ <style>

25+ .toto {

26+ color: red;

27+ }

28+ </style>

29+ <span class="toto">

30+ <span>

31+ <img src="test.jpg" />

32+ </span>

33+ </span>

34+ <span class="titi">

35+ Test text

36+ </span>

37+ <script>value='dummy';</script>

38+ </a>

39+ """

40+ self.assertIsInstance(utils.html_to_text(html_str), str)

41+ self.assertIsNotNone(utils.html_to_text(html_str))

42+ self.assertEqual(utils.html_to_text(html_str), "Test text")

43+

44+ def test_extract_text(self):

45+ html_str = """

20 <a href="/testlink" class="link_access_account">

46 <a href="/testlink" class="link_access_account">

21 <span class="toto">

47 <span class="toto">

22 <span>

48 <span>

@@ -28,9 +54,24 @@ class TestUtils(SearxTestCase):

28 </span>

54 </span>

29 </a>

55 </a>

30 """

56 """

31− self.assertIsInstance(utils.html_to_text(html), str)

57+ dom = html.fromstring(html_str)

32− self.assertIsNotNone(utils.html_to_text(html))

58+ self.assertEqual(utils.extract_text(dom), 'Test text')

59+ self.assertEqual(utils.extract_text(dom.xpath('//span')), 'Test text')

60+ self.assertEqual(utils.extract_text(dom.xpath('//img/@src')), 'test.jpg')

33− self.assertEqual(utils.html_to_text(html), "Test text")

61+ self.assertEqual(utils.extract_text(dom.xpath('//unexistingtag')), '')

62+

63+ def test_extract_url(self):

64+ def f(html_str, search_url):

65+ return utils.extract_url(html.fromstring(html_str), search_url)

66+ self.assertEqual(f('<span id="42">https://example.com</span>', 'http://example.com/'), 'https://example.com/')

67+ self.assertEqual(f('https://example.com', 'http://example.com/'), 'https://example.com/')

68+ self.assertEqual(f('//example.com', 'http://example.com/'), 'http://example.com/')

69+ self.assertEqual(f('//example.com', 'https://example.com/'), 'https://example.com/')

70+ self.assertEqual(f('/path?a=1', 'https://example.com'), 'https://example.com/path?a=1')

71+ with self.assertRaises(lxml.etree.ParserError):

72+ f('', 'https://example.com')

73+ with self.assertRaises(Exception):

74+ utils.extract_url([], 'https://example.com')

34

75

35 def test_html_to_text_invalid(self):

76 def test_html_to_text_invalid(self):

36 html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'

77 html = '<p><b>Lorem ipsum</i>dolor sit amet</p>'