on master help sign in
Merge pull request #2102 from MarcAbonce/wikipedia-api-cleanup

Fix Wikipedia's paragraph extraction

commit1185c06a8732101ad16bb3a8926ff0797ff2c6b9 authorAdam Tauber <asciimoo@gmail.com> dateMon 27 Jul 2020, 14:46 (6 years ago) parent6d1876977b9faa
1 file changed+1−27
searx/engines/wikipedia.py+1−27
searx/engines/wikipedia.py+1−27
@@ -49,29 +49,6 @@ def request(query, params):
4949 return params
5050
5151
52−# get first meaningful paragraph
53−# this should filter out disambiguation pages and notes above first paragraph
54−# "magic numbers" were obtained by fine tuning
55−def extract_first_paragraph(content, title, image):
56− first_paragraph = None
57−
58− failed_attempts = 0
59− for paragraph in content.split('\n'):
60−
61− starts_with_title = paragraph.lower().find(title.lower(), 0, len(title) + 35)
62− length = len(paragraph)
63−
64− if length >= 200 or (starts_with_title >= 0 and (image or length >= 150)):
65− first_paragraph = paragraph
66− break
67−
68− failed_attempts += 1
69− if failed_attempts > 3:
70− return None
71−
72− return first_paragraph
73−
74−
7552# get response from search-request
7653def response(resp):
7754 results = []
@@ -97,10 +74,7 @@ def response(resp):
9774 if image:
9875 image = image.get('source')
9976
100− extract = page.get('extract')
101−
102− summary = extract_first_paragraph(extract, title, image)
103− summary = summary.replace('() ', '')
77+ summary = page.get('extract', '').split('\n')[0].replace('()', '')
10478
10579 # link to wikipedia article
10680 wikipedia_link = base_url.format(language=url_lang(resp.search_params['language'])) \