ad72803 · searx · git.iko.soy

git.iko.soy/searx

[mod] xpath, 1337x, acgsou, apkmirror, archlinux, arxiv: use eval_xpath_* functions

commitad72803ed98760b2a95690dea3b95eacc670427c authorAlexandre Flament <alex@al-f.net> dateThu 26 Nov 2020, 15:49 (6 years ago) parentde887c6

6 files changed+51−54

compare with master

unifiedsplit

searx/engines/1337x.py+7−7 searx/engines/acgsou.py+12−15 searx/engines/apkmirror.py+5−4 searx/engines/archlinux.py+3−3 searx/engines/arxiv.py+9−11 searx/engines/xpath.py+15−14

searx/engines/1337x.py+7−7

@@ -1,6 +1,6 @@

11from urllib.parse import quote, urljoin

22from lxml import html

3−from searx.utils import extract_text, get_torrent_size

3+from searx.utils import extract_text, get_torrent_size, eval_xpath, eval_xpath_list, eval_xpath_getindex

66url = 'https://1337x.to/'

@@ -20,12 +20,12 @@ def response(resp):

2020

2121 dom = html.fromstring(resp.text)

2222

23− for result in dom.xpath('//table[contains(@class, "table-list")]/tbody//tr'):

24− href = urljoin(url, result.xpath('./td[contains(@class, "name")]/a[2]/@href')[0])

25− title = extract_text(result.xpath('./td[contains(@class, "name")]/a[2]'))

26− seed = extract_text(result.xpath('.//td[contains(@class, "seeds")]'))

27− leech = extract_text(result.xpath('.//td[contains(@class, "leeches")]'))

28− filesize_info = extract_text(result.xpath('.//td[contains(@class, "size")]/text()'))

23+ for result in eval_xpath_list(dom, '//table[contains(@class, "table-list")]/tbody//tr'):

24+ href = urljoin(url, eval_xpath_getindex(result, './td[contains(@class, "name")]/a[2]/@href', 0))

25+ title = extract_text(eval_xpath(result, './td[contains(@class, "name")]/a[2]'))

26+ seed = extract_text(eval_xpath(result, './/td[contains(@class, "seeds")]'))

27+ leech = extract_text(eval_xpath(result, './/td[contains(@class, "leeches")]'))

28+ filesize_info = extract_text(eval_xpath(result, './/td[contains(@class, "size")]/text()'))

2929 filesize, filesize_multiplier = filesize_info.split()

3030 filesize = get_torrent_size(filesize, filesize_multiplier)

3131

searx/engines/acgsou.py+12−15

@@ -11,7 +11,7 @@

1111

1212from urllib.parse import urlencode

1313from lxml import html

14−from searx.utils import extract_text, get_torrent_size

14+from searx.utils import extract_text, get_torrent_size, eval_xpath_list, eval_xpath_getindex

1515

1616# engine dependent config

1717categories = ['files', 'images', 'videos', 'music']

@@ -37,29 +37,26 @@ def request(query, params):

3737def response(resp):

3838 results = []

3939 dom = html.fromstring(resp.text)

40− for result in dom.xpath(xpath_results):

40+ for result in eval_xpath_list(dom, xpath_results):

4141 # defaults

4242 filesize = 0

4343 magnet_link = "magnet:?xt=urn:btih:{}&tr=http://tracker.acgsou.com:2710/announce"

4444

45− try:

46− category = extract_text(result.xpath(xpath_category)[0])

47− except:

48− pass

49−

50− page_a = result.xpath(xpath_title)[0]

45+ category = extract_text(eval_xpath_getindex(result, xpath_category, 0, default=[]))

46+ page_a = eval_xpath_getindex(result, xpath_title, 0)

5147 title = extract_text(page_a)

5248 href = base_url + page_a.attrib.get('href')

5349

5450 magnet_link = magnet_link.format(page_a.attrib.get('href')[5:-5])

5551

56− try:

57− filesize_info = result.xpath(xpath_filesize)[0]

58− filesize = filesize_info[:-2]

59− filesize_multiplier = filesize_info[-2:]

60− filesize = get_torrent_size(filesize, filesize_multiplier)

61− except:

62− pass

52+ filesize_info = eval_xpath_getindex(result, xpath_filesize, 0, default=None)

53+ if filesize_info:

54+ try:

55+ filesize = filesize_info[:-2]

56+ filesize_multiplier = filesize_info[-2:]

57+ filesize = get_torrent_size(filesize, filesize_multiplier)

58+ except:

59+ pass

6360 # I didn't add download/seed/leech count since as I figured out they are generated randomly everytime

6461 content = 'Category: "{category}".'

6562 content = content.format(category=category)

searx/engines/apkmirror.py+5−4

@@ -11,7 +11,7 @@

1111

1212from urllib.parse import urlencode

1313from lxml import html

14−from searx.utils import extract_text

14+from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

1515

1616

1717# engine dependent config

@@ -42,12 +42,13 @@ def response(resp):

4242 dom = html.fromstring(resp.text)

4343

4444 # parse results

45− for result in dom.xpath('.//div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):

45+ for result in eval_xpath_list(dom, './/div[@id="content"]/div[@class="listWidget"]/div[@class="appRow"]'):

4646

47− link = result.xpath('.//h5/a')[0]

47+ link = eval_xpath_getindex(result, './/h5/a', 0)

4848 url = base_url + link.attrib.get('href') + '#downloads'

4949 title = extract_text(link)

50− thumbnail_src = base_url + result.xpath('.//img')[0].attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')

50+ thumbnail_src = base_url\

51+ + eval_xpath_getindex(result, './/img', 0).attrib.get('src').replace('&w=32&h=32', '&w=64&h=64')

5152

5253 res = {

5354 'url': url,

searx/engines/archlinux.py+3−3

@@ -13,7 +13,7 @@

1313

1414from urllib.parse import urlencode, urljoin

1515from lxml import html

16−from searx.utils import extract_text

16+from searx.utils import extract_text, eval_xpath_list, eval_xpath_getindex

1717

1818# engine dependent config

1919categories = ['it']

@@ -131,8 +131,8 @@ def response(resp):

131131 dom = html.fromstring(resp.text)

132132

133133 # parse results

134− for result in dom.xpath(xpath_results):

135− link = result.xpath(xpath_link)[0]

134+ for result in eval_xpath_list(dom, xpath_results):

135+ link = eval_xpath_getindex(result, xpath_link, 0)

136136 href = urljoin(base_url, link.attrib.get('href'))

137137 title = extract_text(link)

138138

searx/engines/arxiv.py+9−11

@@ -13,6 +13,7 @@

1313

1414from lxml import html

1515from datetime import datetime

16+from searx.utils import eval_xpath_list, eval_xpath_getindex

1617

1718

1819categories = ['science']

@@ -42,29 +43,26 @@ def response(resp):

4243 results = []

4344

4445 dom = html.fromstring(resp.content)

45− search_results = dom.xpath('//entry')

4646

47− for entry in search_results:

48− title = entry.xpath('.//title')[0].text

47+ for entry in eval_xpath_list(dom, '//entry'):

48+ title = eval_xpath_getindex(entry, './/title', 0).text

4949

50− url = entry.xpath('.//id')[0].text

50+ url = eval_xpath_getindex(entry, './/id', 0).text

5151

5252 content_string = '{doi_content}{abstract_content}'

5353

54− abstract = entry.xpath('.//summary')[0].text

54+ abstract = eval_xpath_getindex(entry, './/summary', 0).text

5555

5656 # If a doi is available, add it to the snipppet

57− try:

58− doi_content = entry.xpath('.//link[@title="doi"]')[0].text

59− content = content_string.format(doi_content=doi_content, abstract_content=abstract)

60− except:

61− content = content_string.format(doi_content="", abstract_content=abstract)

57+ doi_element = eval_xpath_getindex(entry, './/link[@title="doi"]', 0, default=None)

58+ doi_content = doi_element.text if doi_element is not None else ''

59+ content = content_string.format(doi_content=doi_content, abstract_content=abstract)

6260

6361 if len(content) > 300:

6462 content = content[0:300] + "..."

6563 # TODO: center snippet on query term

6664

67− publishedDate = datetime.strptime(entry.xpath('.//published')[0].text, '%Y-%m-%dT%H:%M:%SZ')

65+ publishedDate = datetime.strptime(eval_xpath_getindex(entry, './/published', 0).text, '%Y-%m-%dT%H:%M:%SZ')

6866

6967 res_dict = {'url': url,

7068 'title': title,

searx/engines/xpath.py+15−14

@@ -1,6 +1,6 @@

11from lxml import html

22from urllib.parse import urlencode

3−from searx.utils import extract_text, extract_url, eval_xpath

3+from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list

55search_url = None

66url_xpath = None

@@ -42,21 +42,22 @@ def response(resp):

4242 is_onion = True if 'onions' in categories else False

4343

4444 if results_xpath:

45− for result in eval_xpath(dom, results_xpath):

46− url = extract_url(eval_xpath(result, url_xpath), search_url)

47− title = extract_text(eval_xpath(result, title_xpath))

48− content = extract_text(eval_xpath(result, content_xpath))

45+ for result in eval_xpath_list(dom, results_xpath):

46+ url = extract_url(eval_xpath_list(result, url_xpath, min_len=1), search_url)

47+ title = extract_text(eval_xpath_list(result, title_xpath, min_len=1))

48+ content = extract_text(eval_xpath_list(result, content_xpath, min_len=1))

4949 tmp_result = {'url': url, 'title': title, 'content': content}

5050

5151 # add thumbnail if available

5252 if thumbnail_xpath:

53− thumbnail_xpath_result = eval_xpath(result, thumbnail_xpath)

53+ thumbnail_xpath_result = eval_xpath_list(result, thumbnail_xpath)

5454 if len(thumbnail_xpath_result) > 0:

5555 tmp_result['img_src'] = extract_url(thumbnail_xpath_result, search_url)

5656

5757 # add alternative cached url if available

5858 if cached_xpath:

59− tmp_result['cached_url'] = cached_url + extract_text(result.xpath(cached_xpath))

59+ tmp_result['cached_url'] = cached_url\

60+ + extract_text(eval_xpath_list(result, cached_xpath, min_len=1))

6061

6162 if is_onion:

6263 tmp_result['is_onion'] = True

@@ -66,19 +67,19 @@ def response(resp):

6667 if cached_xpath:

6768 for url, title, content, cached in zip(

6869 (extract_url(x, search_url) for

69− x in dom.xpath(url_xpath)),

70− map(extract_text, dom.xpath(title_xpath)),

71− map(extract_text, dom.xpath(content_xpath)),

72− map(extract_text, dom.xpath(cached_xpath))

70+ x in eval_xpath_list(dom, url_xpath)),

71+ map(extract_text, eval_xpath_list(dom, title_xpath)),

72+ map(extract_text, eval_xpath_list(dom, content_xpath)),

73+ map(extract_text, eval_xpath_list(dom, cached_xpath))

7374 ):

7475 results.append({'url': url, 'title': title, 'content': content,

7576 'cached_url': cached_url + cached, 'is_onion': is_onion})

7677 else:

7778 for url, title, content in zip(

7879 (extract_url(x, search_url) for

79− x in dom.xpath(url_xpath)),

80− map(extract_text, dom.xpath(title_xpath)),

81− map(extract_text, dom.xpath(content_xpath))

80+ x in eval_xpath_list(dom, url_xpath)),

81+ map(extract_text, eval_xpath_list(dom, title_xpath)),

82+ map(extract_text, eval_xpath_list(dom, content_xpath))

8283 ):

8384 results.append({'url': url, 'title': title, 'content': content, 'is_onion': is_onion})

8485