362c849 · searx · git.iko.soy

git.iko.soy/searx

[fix][mod] wikidata date handling refactor - fixes #387

commit362c849797e2e6f0e232642c23744c47a75cdfd4 authorAdam Tauber <asciimoo@gmail.com> dateMon 7 Sep 2015, 22:39 (11 years ago) parent7580852

2 files changed+36−26

compare with master

unifiedsplit

searx/engines/wikidata.py+26−8 searx/utils.py+10−18

searx/engines/wikidata.py+26−8

@@ -1,8 +1,15 @@

11import json

2−from urllib import urlencode

3+from searx import logger

34from searx.poolrequests import get

45from searx.utils import format_date_by_locale

7+from datetime import datetime

8+from dateutil.parser import parse as dateutil_parse

9+from urllib import urlencode

10+

11+

12+logger = logger.getChild('wikidata')

613result_count = 1

714wikidata_host = 'https://www.wikidata.org'

815wikidata_api = wikidata_host + '/w/api.php'

@@ -164,14 +171,12 @@ def getDetail(jsonresponse, wikidata_id, language, locale):

164171 if postal_code is not None:

165172 attributes.append({'label': 'Postal code(s)', 'value': postal_code})

166173

167− date_of_birth = get_time(claims, 'P569', None)

174+ date_of_birth = get_time(claims, 'P569', locale, None)

168175 if date_of_birth is not None:

169− date_of_birth = format_date_by_locale(date_of_birth[8:], locale)

170176 attributes.append({'label': 'Date of birth', 'value': date_of_birth})

171177

172− date_of_death = get_time(claims, 'P570', None)

178+ date_of_death = get_time(claims, 'P570', locale, None)

173179 if date_of_death is not None:

174− date_of_death = format_date_by_locale(date_of_death[8:], locale)

175180 attributes.append({'label': 'Date of death', 'value': date_of_death})

176181

177182 if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:

@@ -229,7 +234,7 @@ def get_string(claims, propertyName, defaultValue=None):

229234 return result[0]

230235

231236

232−def get_time(claims, propertyName, defaultValue=None):

237+def get_time(claims, propertyName, locale, defaultValue=None):

233238 propValue = claims.get(propertyName, {})

234239 if len(propValue) == 0:

235240 return defaultValue

@@ -244,9 +249,22 @@ def get_time(claims, propertyName, defaultValue=None):

244249 result.append(value.get('time', ''))

245250

246251 if len(result) == 0:

247− return defaultValue

252+ date_string = defaultValue

248253 else:

249− return ', '.join(result)

254+ date_string = ', '.join(result)

255+

256+ try:

257+ parsed_date = datetime.strptime(date_string, "+%Y-%m-%dT%H:%M:%SZ")

258+ except:

259+ if date_string.startswith('-'):

260+ return date_string.split('T')[0]

261+ try:

262+ parsed_date = dateutil_parse(date_string, fuzzy=False, default=False)

263+ except:

264+ logger.debug('could not parse date %s', date_string)

265+ return date_string.split('T')[0]

266+

267+ return format_date_by_locale(parsed_date, locale)

250268

251269

252270def get_geolink(claims, propertyName, defaultValue=''):

searx/utils.py+10−18

@@ -1,11 +1,10 @@

11# import htmlentitydefs

2−import locale

3−import dateutil.parser

42import cStringIO

53import csv

64import os

75import re

7+from babel.dates import format_date

98from codecs import getincrementalencoder

109from HTMLParser import HTMLParser

1110from random import choice

@@ -195,23 +194,16 @@ def get_result_templates(base_path):

195194 return result_templates

196195

197196

198−def format_date_by_locale(date_string, locale_string):

197+def format_date_by_locale(date, locale_string):

199198 # strftime works only on dates after 1900

200− parsed_date = dateutil.parser.parse(date_string)

201− if parsed_date.year <= 1900:

202− return parsed_date.isoformat().split('T')[0]

203−

204− orig_locale = locale.getlocale()[0]

205− try:

206− locale.setlocale(locale.LC_ALL, locale_string)

207− except:

208− logger.warning('cannot set locale: {0}'.format(locale_string))

209− formatted_date = parsed_date.strftime(locale.nl_langinfo(locale.D_FMT))

210− try:

211− locale.setlocale(locale.LC_ALL, orig_locale)

212− except:

213− logger.warning('cannot set original locale: {0}'.format(orig_locale))

214− return formatted_date

199+

200+ if date.year <= 1900:

201+ return date.isoformat().split('T')[0]

202+

203+ if locale_string == 'all':

204+ locale_string = settings['ui']['default_locale'] or 'en_US'

205+

206+ return format_date(date, locale=locale_string)

215207

216208

217209def dict_subset(d, properties):