git.iko.soy/searx
on master help sign in
[mod] result.py: merge infobox URL and attributes when the same label or the same entity
382fdedAlexandre Flament
searx/results.py+26−8
compare with master
unifiedsplit
@@ -20,6 +20,18 @@ def result_content_len(content):
20
20
21
21
22def compare_urls(url_a, url_b):
22def compare_urls(url_a, url_b):
23+ """Lazy compare between two URL.
24+ "www.example.com" and "example.com" are equals.
25+ "www.example.com/path/" and "www.example.com/path" are equals.
26+ "https://www.example.com/" and "http://www.example.com/" are equals.
27+
28+ Args:
29+ url_a (ParseResult): first URL
30+ url_b (ParseResult): second URL
31+
32+ Returns:
33+ bool: True if url_a and url_b are equals
34+ """
23 # ignore www. in comparison
35 # ignore www. in comparison
24 if url_a.netloc.startswith('www.'):
36 if url_a.netloc.startswith('www.'):
25 host_a = url_a.netloc.replace('www.', '', 1)
37 host_a = url_a.netloc.replace('www.', '', 1)
@@ -68,8 +80,10 @@ def merge_two_infoboxes(infobox1, infobox2):
68 for url2 in infobox2.get('urls', []):
80 for url2 in infobox2.get('urls', []):
69 unique_url = True
81 unique_url = True
70 parsed_url2 = urlparse(url2.get('url', ''))
82 parsed_url2 = urlparse(url2.get('url', ''))
83+ entity_url2 = url2.get('entity')
71 for url1 in urls1:
84 for url1 in urls1:
85+ if (entity_url2 is not None and url1.get('entity') == entity_url2)\
72− if compare_urls(urlparse(url1.get('url', '')), parsed_url2):
86+ or compare_urls(urlparse(url1.get('url', '')), parsed_url2):
73 unique_url = False
87 unique_url = False
74 break
88 break
75 if unique_url:
89 if unique_url:
@@ -86,18 +100,22 @@ def merge_two_infoboxes(infobox1, infobox2):
86 infobox1['img_src'] = img2
100 infobox1['img_src'] = img2
87
101
88 if 'attributes' in infobox2:
102 if 'attributes' in infobox2:
89− attributes1 = infobox1.get('attributes', None)
103+ attributes1 = infobox1.get('attributes')
90 if attributes1 is None:
104 if attributes1 is None:
91− attributes1 = []
92− infobox1['attributes'] = attributes1
105+ infobox1['attributes'] = attributes1 = []
93
106
94 attributeSet = set()
107 attributeSet = set()
95− for attribute in infobox1.get('attributes', []):
108+ for attribute in attributes1:
96− if attribute.get('label', None) not in attributeSet:
109+ label = attribute.get('label')
110+ if label not in attributeSet:
97− attributeSet.add(attribute.get('label', None))
111+ attributeSet.add(label)
112+ entity = attribute.get('entity')
113+ if entity not in attributeSet:
114+ attributeSet.add(entity)
98
115
99 for attribute in infobox2.get('attributes', []):
116 for attribute in infobox2.get('attributes', []):
100− if attribute.get('label', None) not in attributeSet:
117+ if attribute.get('label') not in attributeSet\
118+ and attribute.get('entity') not in attributeSet:
101 attributes1.append(attribute)
119 attributes1.append(attribute)
102
120
103 if 'content' in infobox2:
121 if 'content' in infobox2: