@@ -0,0 +1,121 @@
1+# SPDX-License-Identifier: AGPL-3.0-or-later
2+
3+"""
4+ xPath Flex Engine (xpath_flex)
5+
6+ Aims to be a more flexible engine that covers xpath use cases for all categories
7+ (as it allows to override the result template).
8+ The engine configuration expects you to specify a 'field_definition' dictionary with the following properties:
9+ * field_name: the template field that should be set by this entry.
10+ * xpath: a xpath expression to apply
11+ * (optional) single_element: if True, will execute the xpath expression to expect a single node, not a list
12+ * (optional) extract: a string specifying an extraction/conversion of the value, the following is supported:
13+ "url": extracts an url using base search url
14+ "boolean": will convert the value passed to the template to True if the xpath matches any node
15+ "boolean_negate": will convert the value passed to the template to True if the xpath does NOT match a node
16+ by default, text will be extracted from the found node(s)
17+
18+ an example would look like this (for an example shopping site with paging):
19+
20+ - name : xpath-flex-example
21+ engine : xpath_flex
22+ paging : True
23+ search_url : https://myl-shopping.site/search?q={query}&p={pageno}
24+ template : products.html
25+ results_xpath : //div[@class="listing--container"]/div[@class="listing"]/div[contains(@class,"product--box")]
26+ field_definition:
27+ - field_name: url
28+ xpath: (.//a[contains(@class,"product--image")])/@href
29+ extract: url
30+ - field_name: title
31+ xpath: (.//a[contains(@class,"product--image")])/@title
32+ - field_name: content
33+ xpath: .//div[@class="product--description"]/text()
34+ - field_name: price
35+ xpath: .//div[@class="product--price"]/span/text()
36+ - field_name: thumbnail
37+ xpath: substring-before( (.//span[@class="image--media"]/img)/@srcset, ", ")
38+ extract: url
39+ single_element: True
40+ - field_name: has_stock
41+ xpath: .//a[contains(@class,"buynow")][not(contains(@class,"is--disabled"))]
42+ extract: boolean
43+ single_element: True
44+
45+"""
46+
47+from lxml import html
48+from urllib.parse import urlencode
49+from searx import logger
50+from searx.utils import extract_text, extract_url, eval_xpath, eval_xpath_list
51+
52+logger = logger.getChild('xpath_general engine')
53+
54+search_url = None
55+paging = False
56+results_xpath = ''
57+soft_max_redirects = 0
58+template = 'default.html'
59+unresolvable_value = '' # will be set if expression cannot be resolved
60+default_field_settings = {'single_element': False}
61+
62+field_definition = {}
63+
64+# parameters for engines with paging support
65+#
66+# number of results on each page
67+# (only needed if the site requires not a page number, but an offset)
68+page_size = 1
69+# number of the first page (usually 0 or 1)
70+first_page_num = 1
71+
72+
73+def request(query, params):
74+ query = urlencode({'q': query})[2:]
75+
76+ fp = {'query': query}
77+ if paging and search_url.find('{pageno}') >= 0:
78+ fp['pageno'] = (params['pageno'] - 1) * page_size + first_page_num
79+
80+ params['url'] = search_url.format(**fp)
81+ params['query'] = query
82+ params['soft_max_redirects'] = soft_max_redirects
83+
84+ return params
85+
86+
87+def response(resp):
88+ results = []
89+ dom = html.fromstring(resp.text)
90+
91+ for result in eval_xpath_list(dom, results_xpath):
92+
93+ single_result = {
94+ 'template': template
95+ }
96+
97+ for single_field in field_definition:
98+ single_field = {**default_field_settings, **single_field}
99+ try:
100+ if single_field['single_element']:
101+ node = eval_xpath(result, single_field['xpath'])
102+ else:
103+ node = eval_xpath_list(result, single_field['xpath'])
104+
105+ if 'extract' in single_field and single_field['extract'] == 'url':
106+ value = extract_url(node, search_url)
107+ elif 'extract' in single_field and single_field['extract'] == 'boolean':
108+ value = (isinstance(node, list) and len(node) > 0)
109+ elif 'extract' in single_field and single_field['extract'] == 'boolean_negate':
110+ value = (isinstance(node, list) and len(node) < 1)
111+ else:
112+ value = extract_text(node)
113+
114+ single_result[single_field['field_name']] = value
115+ except Exception as e:
116+ logger.warning('error in resolving field %s:\n%s', single_field['field_name'], e)
117+ single_result[single_field['field_name']] = unresolvable_value
118+
119+ results.append(single_result)
120+
121+ return results