mirror of
https://github.com/osm-search/Nominatim.git
synced 2024-11-27 10:43:02 +03:00
c8873d34af
The configure() function now receives a Transliterator object instead of the ICU rules. This harmonizes the parameters with the create function.
140 lines
4.4 KiB
Python
140 lines
4.4 KiB
Python
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
# This file is part of Nominatim. (https://nominatim.org)
|
|
#
|
|
# Copyright (C) 2022 by the Nominatim developer community.
|
|
# For a full list of authors see the git log.
|
|
"""
|
|
Parser for configuration for variants.
|
|
"""
|
|
from typing import Any, Iterator, Tuple, List, Optional, Set, NamedTuple
|
|
from collections import defaultdict
|
|
import itertools
|
|
import re
|
|
|
|
from nominatim.config import flatten_config_list
|
|
from nominatim.errors import UsageError
|
|
|
|
class ICUVariant(NamedTuple):
|
|
""" A single replacement rule for variant creation.
|
|
"""
|
|
source: str
|
|
replacement: str
|
|
|
|
|
|
def get_variant_config(in_rules: Any,
|
|
normalizer: Any) -> Tuple[List[Tuple[str, List[str]]], str]:
|
|
""" Convert the variant definition from the configuration into
|
|
replacement sets.
|
|
|
|
Returns a tuple containing the replacement set and the list of characters
|
|
used in the replacements.
|
|
"""
|
|
immediate = defaultdict(list)
|
|
chars: Set[str] = set()
|
|
|
|
if in_rules:
|
|
vset: Set[ICUVariant] = set()
|
|
rules = flatten_config_list(in_rules, 'variants')
|
|
|
|
vmaker = _VariantMaker(normalizer)
|
|
|
|
for section in rules:
|
|
for rule in (section.get('words') or []):
|
|
vset.update(vmaker.compute(rule))
|
|
|
|
# Intermediate reorder by source. Also compute required character set.
|
|
for variant in vset:
|
|
if variant.source[-1] == ' ' and variant.replacement[-1] == ' ':
|
|
replstr = variant.replacement[:-1]
|
|
else:
|
|
replstr = variant.replacement
|
|
immediate[variant.source].append(replstr)
|
|
chars.update(variant.source)
|
|
|
|
return list(immediate.items()), ''.join(chars)
|
|
|
|
|
|
class _VariantMaker:
|
|
""" Generator for all necessary ICUVariants from a single variant rule.
|
|
|
|
All text in rules is normalized to make sure the variants match later.
|
|
"""
|
|
|
|
def __init__(self, normalizer: Any) -> None:
|
|
self.norm = normalizer
|
|
|
|
|
|
def compute(self, rule: Any) -> Iterator[ICUVariant]:
|
|
""" Generator for all ICUVariant tuples from a single variant rule.
|
|
"""
|
|
parts = re.split(r'(\|)?([=-])>', rule)
|
|
if len(parts) != 4:
|
|
raise UsageError(f"Syntax error in variant rule: {rule}")
|
|
|
|
decompose = parts[1] is None
|
|
src_terms = [self._parse_variant_word(t) for t in parts[0].split(',')]
|
|
repl_terms = (self.norm.transliterate(t).strip() for t in parts[3].split(','))
|
|
|
|
# If the source should be kept, add a 1:1 replacement
|
|
if parts[2] == '-':
|
|
for src in src_terms:
|
|
if src:
|
|
for froms, tos in _create_variants(*src, src[0], decompose):
|
|
yield ICUVariant(froms, tos)
|
|
|
|
for src, repl in itertools.product(src_terms, repl_terms):
|
|
if src and repl:
|
|
for froms, tos in _create_variants(*src, repl, decompose):
|
|
yield ICUVariant(froms, tos)
|
|
|
|
|
|
def _parse_variant_word(self, name: str) -> Optional[Tuple[str, str, str]]:
|
|
name = name.strip()
|
|
match = re.fullmatch(r'([~^]?)([^~$^]*)([~$]?)', name)
|
|
if match is None or (match.group(1) == '~' and match.group(3) == '~'):
|
|
raise UsageError(f"Invalid variant word descriptor '{name}'")
|
|
norm_name = self.norm.transliterate(match.group(2)).strip()
|
|
if not norm_name:
|
|
return None
|
|
|
|
return norm_name, match.group(1), match.group(3)
|
|
|
|
|
|
_FLAG_MATCH = {'^': '^ ',
|
|
'$': ' ^',
|
|
'': ' '}
|
|
|
|
|
|
def _create_variants(src: str, preflag: str, postflag: str,
|
|
repl: str, decompose: bool) -> Iterator[Tuple[str, str]]:
|
|
if preflag == '~':
|
|
postfix = _FLAG_MATCH[postflag]
|
|
# suffix decomposition
|
|
src = src + postfix
|
|
repl = repl + postfix
|
|
|
|
yield src, repl
|
|
yield ' ' + src, ' ' + repl
|
|
|
|
if decompose:
|
|
yield src, ' ' + repl
|
|
yield ' ' + src, repl
|
|
elif postflag == '~':
|
|
# prefix decomposition
|
|
prefix = _FLAG_MATCH[preflag]
|
|
src = prefix + src
|
|
repl = prefix + repl
|
|
|
|
yield src, repl
|
|
yield src + ' ', repl + ' '
|
|
|
|
if decompose:
|
|
yield src, repl + ' '
|
|
yield src + ' ', repl
|
|
else:
|
|
prefix = _FLAG_MATCH[preflag]
|
|
postfix = _FLAG_MATCH[postflag]
|
|
|
|
yield prefix + src + postfix, prefix + repl + postfix
|