on master help sign in
Merge pull request #2595 from dalf/update-wikidata-units

[mod] update wikidata_units.json and fetch_wikidata_units.py

commit5f4a085fc44ed64499d1aa5c2e29d2f861037b5f authorAlexandre Flament <alex@al-f.net> dateTue 23 Feb 2021, 17:22 (6 years ago) parent94c932093d1da4
2 files changed+170−63
searx/data/wikidata_units.json+144−46 utils/fetch_wikidata_units.py+26−17
searx/data/wikidata_units.json+144−46
@@ -1,7 +1,67 @@
11{
22 "Q199": "1",
3+ "Q100036106": "int nmi",
34 "Q100149279": "°We",
45 "Q100995": "lb",
6+ "Q101194838": "GHz/V",
7+ "Q101463141": "ym²",
8+ "Q101463237": "zm²",
9+ "Q101463321": "am²",
10+ "Q101463409": "fm²",
11+ "Q101463496": "pm²",
12+ "Q101463679": "hm²",
13+ "Q101464050": "Mm²",
14+ "Q101464215": "Gm²",
15+ "Q101464369": "Tm²",
16+ "Q101464499": "Pm²",
17+ "Q101464624": "Em²",
18+ "Q101464753": "Zm²",
19+ "Q101464875": "Ym²",
20+ "Q101515060": "g/J",
21+ "Q101875087": "cd/cm²",
22+ "Q101877596": "g/ml",
23+ "Q101879174": "dm/s",
24+ "Q102068844": "cm⁻³",
25+ "Q102129339": "min⁻¹",
26+ "Q102129428": "h⁻¹",
27+ "Q102129592": "d⁻¹",
28+ "Q102130673": "ym/s",
29+ "Q102130674": "zm/s",
30+ "Q102130677": "am/s",
31+ "Q102130679": "fm/s",
32+ "Q102130681": "pm/s",
33+ "Q102130684": "nm/s",
34+ "Q102130686": "μm/s",
35+ "Q102130688": "mm/s",
36+ "Q102130690": "dam/s",
37+ "Q102130692": "hm/s",
38+ "Q102130694": "Mm/s",
39+ "Q102130696": "Gm/s",
40+ "Q102130698": "Tm/s",
41+ "Q102130700": "Pm/s",
42+ "Q102130702": "Em/s",
43+ "Q102130704": "Zm/s",
44+ "Q102130706": "Ym/s",
45+ "Q102130743": "ym/s²",
46+ "Q102130745": "zm/s²",
47+ "Q102130747": "am/s²",
48+ "Q102130748": "fm/s²",
49+ "Q102130751": "pm/s²",
50+ "Q102130753": "nm/s²",
51+ "Q102130755": "μm/s²",
52+ "Q102130756": "mm/s²",
53+ "Q102130758": "dm/s²",
54+ "Q102130759": "dam/s²",
55+ "Q102130761": "hm/s²",
56+ "Q102130762": "km/s²",
57+ "Q102130765": "Mm/s²",
58+ "Q102130767": "Gm/s²",
59+ "Q102130769": "Tm/s²",
60+ "Q102130771": "Pm/s²",
61+ "Q102130773": "Em/s²",
62+ "Q102130775": "Zm/s²",
63+ "Q102130777": "Ym/s²",
64+ "Q102178883": "dm³/h",
565 "Q1022113": "cm³",
666 "Q102573": "Bq",
767 "Q103246": "Sv",
@@ -9,12 +69,16 @@
969 "Q10380431": "TJ",
1070 "Q1040401": "das",
1171 "Q1040427": "hs",
12− "Q1042866": "Zibit",
72+ "Q104117265": "Bi",
73+ "Q1042866": "Zib",
74+ "Q104907398": "μN m",
75+ "Q104907399": "mN m",
1376 "Q1050958": "inHg",
1477 "Q1051665": "m/s²",
1578 "Q1052397": "rad",
1679 "Q1054140": "Mm",
1780 "Q10543042": "Ym",
81+ "Q105519288": "B SPL",
1882 "Q1057069": "hg",
1983 "Q1063756": "rad/s",
2084 "Q1063786": "in²",
@@ -22,33 +86,34 @@
2286 "Q1066138": "Ps",
2387 "Q1067722": "Fg",
2488 "Q1069725": "p.",
89+ "Q1072404": "K",
2590 "Q1084321": "Tb/s",
2691 "Q1086691": "fg",
2792 "Q1091257": "tex",
2893 "Q1092296": "a",
29− "Q1104069": "CAD$",
94+ "Q1104069": "$",
3095 "Q11061003": "μm²",
3196 "Q11061005": "nm²",
3297 "Q1131660": "st",
3398 "Q1137675": "cr",
34− "Q1140444": "Zbit",
35− "Q1140577": "Ybit",
36− "Q1152074": "Pbit",
37− "Q1152323": "Tbit",
99+ "Q1140444": "Zb",
100+ "Q1140577": "Yb",
101+ "Q1152074": "Pb",
102+ "Q1152323": "Tb",
38103 "Q1165799": "mil",
39104 "Q11776930": "Mg",
40105 "Q11830636": "psf",
41106 "Q11929860": "kpc",
42107 "Q1194225": "lbf",
43− "Q1194580": "Mibit",
44− "Q1195111": "Ebit",
108+ "Q1194580": "Mib",
109+ "Q1195111": "Eb",
45110 "Q1196837": "ω_P",
46111 "Q1197459": "Ms",
47112 "Q11982285": "Em³",
48113 "Q11982288": "Zm³",
49114 "Q11982289": "Tm³",
50115 "Q12011178": "Zs",
51− "Q1204894": "Gibit",
116+ "Q1204894": "Gib",
52117 "Q12257695": "Eb/s",
53118 "Q12257696": "EB/s",
54119 "Q12261466": "kB/s",
@@ -59,7 +124,7 @@
59124 "Q12269308": "Zb/s",
60125 "Q12269309": "ZB/s",
61126 "Q1247300": "cm H₂O",
62− "Q12714022": "sh cwt",
127+ "Q12714022": "cwt",
63128 "Q12789864": "GeV",
64129 "Q12874593": "W h",
65130 "Q128822": "kn",
@@ -71,9 +136,9 @@
71136 "Q1323615": "oz t",
72137 "Q132643": "kr",
73138 "Q13400897": "g",
74− "Q13479685": "mm wg",
75− "Q1351253": "Eibit",
76− "Q1351334": "Pibit",
139+ "Q13479685": "mm H2O",
140+ "Q1351253": "Eib",
141+ "Q1351334": "Pib",
77142 "Q13542672": "Ry",
78143 "Q13548586": "THz",
79144 "Q13582667": "kgf/cm²",
@@ -88,13 +153,15 @@
88153 "Q14158377": "A_P",
89154 "Q14623803": "MDa",
90155 "Q14623804": "kDa",
91− "Q1472674": "Sv",
156+ "Q1472674": "S",
92157 "Q14754979": "Zg",
93158 "Q14786969": "MJ",
159+ "Q14850704": "℧",
94160 "Q14913554": "Ys",
95161 "Q14914907": "th",
96162 "Q14916719": "Gpc",
97163 "Q14923662": "Pm³",
164+ "Q1501273": "HU",
98165 "Q1511773": "LSd",
99166 "Q15120301": "l atm",
100167 "Q1542309": "xu",
@@ -110,7 +177,7 @@
110177 "Q163354": "H",
111178 "Q1640501": "hyl",
112179 "Q1645498": "μg",
113− "Q16859309": "lb·ft",
180+ "Q16859309": "lb ft",
114181 "Q169893": "S",
115182 "Q170804": "Wb",
116183 "Q17093295": "m/h",
@@ -140,7 +207,7 @@
140207 "Q182429": "m/s",
141208 "Q1826195": "dl",
142209 "Q18413919": "cm/s",
143− "Q184172": "FF",
210+ "Q184172": "F",
144211 "Q185078": "a",
145212 "Q185153": "erg",
146213 "Q185648": "Torr",
@@ -171,11 +238,10 @@
171238 "Q2029519": "hl",
172239 "Q203567": "₦",
173240 "Q2042279": "m H₂O",
174− "Q204737": "៛",
175241 "Q2051195": "GWh",
176242 "Q2055118": "ppb",
177243 "Q2064166": "fc",
178− "Q206600": "ރ",
244+ "Q206600": "MRF",
179245 "Q20706220": "cmm",
180246 "Q20706221": "dmm",
181247 "Q2080811": "vol%",
@@ -196,9 +262,11 @@
196262 "Q21075844": "ml/l",
197263 "Q21077820": "mg/m³",
198264 "Q21091747": "mg/kg",
199− "Q211256": "mph",
265+ "Q211256": "mi/h",
266+ "Q21154419": "PD",
200267 "Q211580": "BTU (th)",
201268 "Q212120": "A h",
269+ "Q213005": "G$",
202270 "Q2140397": "in³",
203271 "Q214377": "ell",
204272 "Q2143992": "kHz",
@@ -211,7 +279,7 @@
211279 "Q215571": "N m",
212280 "Q21604951": "g/m³",
213281 "Q2165290": "yd³",
214− "Q216880": "kp",
282+ "Q216880": "kgf",
215283 "Q217208": "a",
216284 "Q2175964": "dm³",
217285 "Q218593": "in",
@@ -229,11 +297,14 @@
229297 "Q229354": "Ci",
230298 "Q232291": "mi²",
231299 "Q2332346": "ml",
300+ "Q235729": "y (365 days)",
301+ "Q23808021": "oz (ap.)",
232302 "Q23823681": "TW",
233303 "Q23925410": "gal (UK)",
234304 "Q23925413": "gal (US)",
235305 "Q23931040": "dam²",
236306 "Q23931103": "nmi²",
307+ "Q240468": "syr£",
237308 "Q2414435": "$b.",
238309 "Q242988": "Lib$",
239310 "Q2438073": "ag",
@@ -252,7 +323,7 @@
252323 "Q25511288": "mb",
253324 "Q2553708": "MV",
254325 "Q2554092": "kV",
255− "Q259502": "AU$",
326+ "Q259502": "A$",
256327 "Q260126": "rem",
257328 "Q2612219": "Pg",
258329 "Q261247": "ct",
@@ -306,8 +377,11 @@
306377 "Q30001831": "aV",
307378 "Q30001832": "aW",
308379 "Q30001833": "aWb",
309− "Q3013059": "kyr",
310− "Q3194304": "kbit",
380+ "Q3013059": "ka",
381+ "Q304479": "tr",
382+ "Q305896": "DPI",
383+ "Q31889818": "ppq",
384+ "Q3194304": "kb",
311385 "Q3207456": "mW",
312386 "Q321017": "R",
313387 "Q3221356": "ym",
@@ -330,10 +404,10 @@
330404 "Q3312063": "fL",
331405 "Q3320608": "kW",
332406 "Q3331719": "dm²",
333− "Q3332689": "ToR",
334− "Q3332814": "Mbit",
407+ "Q3332689": "RT",
408+ "Q3332814": "Mb",
335409 "Q3396758": "daa",
336− "Q3414243": "rps",
410+ "Q3414243": "qps",
337411 "Q3421309": "R_J",
338412 "Q3495543": "mbar",
339413 "Q355198": "px",
@@ -343,11 +417,11 @@
343417 "Q376660": "nat",
344418 "Q37732658": "°R",
345419 "Q3773454": "Mpc",
346− "Q3815076": "Kibit",
420+ "Q3815076": "Kib",
347421 "Q3833309": "£",
348422 "Q3858002": "mA h",
349423 "Q3867152": "ft/s²",
350− "Q389062": "Tibit",
424+ "Q389062": "Tib",
351425 "Q3902688": "pl",
352426 "Q3902709": "ps",
353427 "Q39360235": "US lea",
@@ -359,7 +433,7 @@
359433 "Q39462789": "µin²",
360434 "Q39467934": "kgf/m²",
361435 "Q39469927": "N/m²",
362− "Q39617688": "cwt long",
436+ "Q39617688": "cwt",
363437 "Q39617818": "t lb",
364438 "Q39628023": "y",
365439 "Q39699418": "cm/s²",
@@ -367,14 +441,14 @@
367441 "Q39709980": "bd",
368442 "Q39710113": "bhp EDR",
369443 "Q3972226": "kL",
370− "Q4041686": "iwg",
444+ "Q4041686": "in H20",
371445 "Q4068266": "Ʒ",
372446 "Q4176683": "aC",
373− "Q420266": "oz. fl.",
447+ "Q420266": "fl oz",
374448 "Q42319606": "people/m²",
375449 "Q4243638": "km³",
376450 "Q4456994": "mF",
377− "Q469356": "tn. sh.",
451+ "Q469356": "T",
378452 "Q476572": "Ha",
379453 "Q482798": "yd",
380454 "Q483261": "Da",
@@ -390,15 +464,18 @@
390464 "Q514845": "pz",
391465 "Q5195628": "hm³",
392466 "Q5198770": "dam³",
393− "Q524410": "byr",
467+ "Q524410": "Ga",
468+ "Q5299480": "DPCm",
394469 "Q53393488": "PHz",
395470 "Q53393490": "EHz",
396471 "Q53393494": "ZHz",
397472 "Q53393498": "YHz",
398473 "Q53393659": "ML",
399474 "Q53393664": "GL",
475+ "Q53393669": "El",
400476 "Q53393674": "ZL",
401477 "Q53393678": "YL",
478+ "Q53393768": "zl",
402479 "Q53393771": "yL",
403480 "Q53393868": "GJ",
404481 "Q53393886": "PJ",
@@ -492,7 +569,7 @@
492569 "Q54083813": "Zkat",
493570 "Q5409016": "MVA",
494571 "Q5465723": "ft-pdl",
495− "Q549389": "bit/s",
572+ "Q549389": "b/s",
496573 "Q550341": "V A",
497574 "Q552299": "ch",
498575 "Q55442349": "U/L",
@@ -523,6 +600,8 @@
523600 "Q6170164": "yg",
524601 "Q6171168": "zg",
525602 "Q61756607": "yd",
603+ "Q61771602": "ft",
604+ "Q61771670": "in",
526605 "Q61793198": "rd",
527606 "Q61794766": "ch (US survey)",
528607 "Q61994988": "Wth",
@@ -534,13 +613,12 @@
534613 "Q6414556": "kip",
535614 "Q648908": "bya",
536615 "Q64996135": "gal (US)/min",
537− "Q65028392": "mm/yr",
616+ "Q65028392": "mm/a",
538617 "Q651336": "M_J",
539618 "Q6517513": "dag",
540619 "Q667419": "UK t",
541620 "Q681996": "M⊕",
542621 "Q685662": "p_P",
543− "Q6859652": "mm Hg",
544622 "Q686163": "$",
545623 "Q68725821": "°Rø",
546624 "Q68726230": "°De",
@@ -582,20 +660,23 @@
582660 "Q70444514": "Ymol",
583661 "Q70444609": "Pmol",
584662 "Q712226": "km²",
663+ "Q717310": "Mg",
585664 "Q72081071": "MeV",
586665 "Q723733": "ms",
587666 "Q730251": "ft·lbf",
588667 "Q732707": "MHz",
589668 "Q73408": "K",
590669 "Q7350781": "Mb/s",
670+ "Q7398951": "PPI",
591671 "Q743895": "bpm",
592672 "Q748716": "ft/s",
593673 "Q750178": "‱",
674+ "Q752079": "RT",
594675 "Q752197": "kJ/mol",
595676 "Q7672057": "TU",
596677 "Q777017": "dBm",
597678 "Q78754556": "rot",
598− "Q78756901": "rev",
679+ "Q78756901": "r",
599680 "Q78757683": "windings",
600681 "Q79726": "kB",
601682 "Q79735": "MB",
@@ -637,14 +718,16 @@
637718 "Q848856": "dam",
638719 "Q851872": "o",
639720 "Q854546": "Gm",
640− "Q855161": "Yibit",
721+ "Q855161": "Yib",
641722 "Q856240": "ft³/min",
642723 "Q857027": "ft²",
643724 "Q85854198": "MN",
644− "Q864818": "abA",
645725 "Q87262709": "kΩ",
646726 "Q87416053": "MΩ",
647727 "Q88296091": "tsp",
728+ "Q89473028": "bu (UK)",
729+ "Q89662131": "pt (UK)",
730+ "Q901492": "ph",
648731 "Q9026416": "MWth",
649732 "Q9048643": "nl",
650733 "Q905912": "L",
@@ -653,7 +736,9 @@
653736 "Q911730": "nx",
654737 "Q914151": "P_P",
655738 "Q915169": "F_P",
656− "Q93318": "nmi",
739+ "Q93318": "M",
740+ "Q93678895": "gill (US)",
741+ "Q93679498": "gill (UK)",
657742 "Q940052": "q",
658743 "Q94076025": "dalm",
659744 "Q94076717": "dakat",
@@ -664,6 +749,7 @@
664749 "Q94415255": "GC",
665750 "Q94415438": "Yrad",
666751 "Q94415526": "YC",
752+ "Q94415561": "krad",
667753 "Q94415782": "Mrad",
668754 "Q94416260": "GN",
669755 "Q94416535": "cN",
@@ -943,6 +1029,7 @@
9431029 "Q96106385": "h°C",
9441030 "Q96106393": "M°C",
9451031 "Q96236286": "G°C",
1032+ "Q96312779": "μas",
9461033 "Q97059641": "p°C",
9471034 "Q97059652": "T°C",
9481035 "Q97143826": "P°C",
@@ -953,9 +1040,21 @@
9531040 "Q97143843": "z°C",
9541041 "Q97143849": "Y°C",
9551042 "Q97143851": "a°C",
1043+ "Q98492214": "den",
9561044 "Q98538634": "eV/m²",
9571045 "Q98635536": "eV/m",
9581046 "Q98642859": "eV m²/kg",
1047+ "Q98793302": "qt (UK)",
1048+ "Q98793408": "liq qt (US)",
1049+ "Q98793687": "dry qt (US)",
1050+ "Q99476928": "gf",
1051+ "Q99487704": "ppt",
1052+ "Q99490009": "BTU (IT)",
1053+ "Q99490479": "BTU (39 °F)",
1054+ "Q99490986": "BTU (59 °F)",
1055+ "Q99491193": "BTU (60 °F)",
1056+ "Q99491447": "BTU (mean)",
1057+ "Q99492167": "m Hg",
9591058 "Q11229": "%",
9601059 "Q11570": "kg",
9611060 "Q11573": "m",
@@ -965,8 +1064,7 @@
9651064 "Q12129": "pc",
9661065 "Q12438": "N",
9671066 "Q16068": "DM",
968− "Q1811": "ua",
969− "Q20764": "Myr",
1067+ "Q20764": "Ma",
9701068 "Q2101": "e",
9711069 "Q25235": "h",
9721070 "Q25236": "W",
@@ -979,25 +1077,25 @@
9791077 "Q25517": "m³",
9801078 "Q33680": "rad",
9811079 "Q35852": "ha",
982− "Q36384": "equiv",
1080+ "Q36384": "Eq",
9831081 "Q3710": "ft",
9841082 "Q39274": "Sv",
9851083 "Q39369": "Hz",
9861084 "Q41509": "mol",
9871085 "Q41803": "g",
9881086 "Q42289": "°F",
989− "Q4406": "TV$",
1087+ "Q4406": "$T",
9901088 "Q44395": "Pa",
991− "Q4587": "Le",
9921089 "Q4588": "WS$",
9931090 "Q4592": "F$",
9941091 "Q4596": "Rs",
9951092 "Q4597": "$",
9961093 "Q47083": "Ω",
9971094 "Q48013": "oz",
1095+ "Q4917": "US$",
9981096 "Q50094": "Np",
9991097 "Q50098": "B",
1000− "Q531": "ly",
1098+ "Q531": "l.y.",
10011099 "Q5329": "dB",
10021100 "Q573": "d",
10031101 "Q577": "a",
utils/fetch_wikidata_units.py+26−17
@@ -12,31 +12,40 @@ from searx import searx_dir
1212from searx.engines.wikidata import send_wikidata_query
1313
1414
15+# the response contains duplicate ?item with the different ?symbol
16+# "ORDER BY ?item DESC(?rank) ?symbol" provides a deterministic result
17+# even if a ?item has different ?symbol of the same rank.
18+# A deterministic result
19+# see:
20+# * https://www.wikidata.org/wiki/Help:Ranking
21+# * https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format ("Statement representation" section)
22+# * https://w.wiki/32BT
23+# see the result for https://www.wikidata.org/wiki/Q11582
24+# there are multiple symbols the same rank
1525SARQL_REQUEST = """
16−SELECT DISTINCT ?item ?symbol ?P2370 ?P2370Unit ?P2442 ?P2442Unit
26+SELECT DISTINCT ?item ?symbol
1727WHERE
1828{
19−?item wdt:P31/wdt:P279 wd:Q47574.
20−?item wdt:P5061 ?symbol.
21−FILTER(LANG(?symbol) = "en").
29+ ?item wdt:P31/wdt:P279 wd:Q47574 .
30+ ?item p:P5061 ?symbolP .
31+ ?symbolP ps:P5061 ?symbol ;
32+ wikibase:rank ?rank .
33+ FILTER(LANG(?symbol) = "en").
2234}
23−ORDER BY ?item
35+ORDER BY ?item DESC(?rank) ?symbol
2436"""
2537
2638
2739def get_data():
28− def get_key(unit):
29− return unit['item']['value'].replace('http://www.wikidata.org/entity/', '')
30−
31− def get_value(unit):
32− return unit['symbol']['value']
33−
34− result = send_wikidata_query(SARQL_REQUEST)
35− if result is not None:
36− # sort the unit by entity name
37− # so different fetchs keep the file unchanged.
38− list(result['results']['bindings']).sort(key=get_key)
39− return collections.OrderedDict([(get_key(unit), get_value(unit)) for unit in result['results']['bindings']])
40+ results = collections.OrderedDict()
41+ response = send_wikidata_query(SARQL_REQUEST)
42+ for unit in response['results']['bindings']:
43+ name = unit['item']['value'].replace('http://www.wikidata.org/entity/', '')
44+ unit = unit['symbol']['value']
45+ if name not in results:
46+ # ignore duplicate: always use the first one
47+ results[name] = unit
48+ return results
4049
4150
4251def get_wikidata_units_filename():