@@ -20,6 +20,18 @@ def result_content_len(content):
2020
2121
2222def compare_urls(url_a, url_b):
23+ """Lazy compare between two URL.
24+ "www.example.com" and "example.com" are equals.
25+ "www.example.com/path/" and "www.example.com/path" are equals.
26+ "https://www.example.com/" and "http://www.example.com/" are equals.
27+
28+ Args:
29+ url_a (ParseResult): first URL
30+ url_b (ParseResult): second URL
31+
32+ Returns:
33+ bool: True if url_a and url_b are equals
34+ """
2335 # ignore www. in comparison
2436 if url_a.netloc.startswith('www.'):
2537 host_a = url_a.netloc.replace('www.', '', 1)
@@ -68,8 +80,10 @@ def merge_two_infoboxes(infobox1, infobox2):
6880 for url2 in infobox2.get('urls', []):
6981 unique_url = True
7082 parsed_url2 = urlparse(url2.get('url', ''))
83+ entity_url2 = url2.get('entity')
7184 for url1 in urls1:
72− if compare_urls(urlparse(url1.get('url', '')), parsed_url2):
85+ if (entity_url2 is not None and url1.get('entity') == entity_url2)\
86+ or compare_urls(urlparse(url1.get('url', '')), parsed_url2):
7387 unique_url = False
7488 break
7589 if unique_url:
@@ -86,18 +100,22 @@ def merge_two_infoboxes(infobox1, infobox2):
86100 infobox1['img_src'] = img2
87101
88102 if 'attributes' in infobox2:
89− attributes1 = infobox1.get('attributes', None)
103+ attributes1 = infobox1.get('attributes')
90104 if attributes1 is None:
91− attributes1 = []
92− infobox1['attributes'] = attributes1
105+ infobox1['attributes'] = attributes1 = []
93106
94107 attributeSet = set()
95− for attribute in infobox1.get('attributes', []):
96− if attribute.get('label', None) not in attributeSet:
97− attributeSet.add(attribute.get('label', None))
108+ for attribute in attributes1:
109+ label = attribute.get('label')
110+ if label not in attributeSet:
111+ attributeSet.add(label)
112+ entity = attribute.get('entity')
113+ if entity not in attributeSet:
114+ attributeSet.add(entity)
98115
99116 for attribute in infobox2.get('attributes', []):
100− if attribute.get('label', None) not in attributeSet:
117+ if attribute.get('label') not in attributeSet\
118+ and attribute.get('entity') not in attributeSet:
101119 attributes1.append(attribute)
102120
103121 if 'content' in infobox2: