From 837bdecde8c8bcc5c09356d7a1db9acef14d48b1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 Oct 2023 22:35:18 +0200 Subject: [PATCH] add skeleton code for convert function --- nominatim/cli.py | 1 + nominatim/clicmd/__init__.py | 1 + nominatim/clicmd/args.py | 3 + nominatim/clicmd/convert.py | 95 +++++++++++++++++++++++++++++++ nominatim/tools/convert_sqlite.py | 30 ++++++++++ 5 files changed, 130 insertions(+) create mode 100644 nominatim/clicmd/convert.py create mode 100644 nominatim/tools/convert_sqlite.py diff --git a/nominatim/cli.py b/nominatim/cli.py index 1029ee7a..88a60782 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -206,6 +206,7 @@ def get_set_parser() -> CommandlineParser: parser.add_subcommand('admin', clicmd.AdminFuncs()) parser.add_subcommand('export', clicmd.QueryExport()) + parser.add_subcommand('convert', clicmd.ConvertDB()) parser.add_subcommand('serve', AdminServe()) parser.add_subcommand('search', clicmd.APISearch()) diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py index 235dff0c..c8de68c1 100644 --- a/nominatim/clicmd/__init__.py +++ b/nominatim/clicmd/__init__.py @@ -25,3 +25,4 @@ from nominatim.clicmd.admin import AdminFuncs as AdminFuncs from nominatim.clicmd.freeze import SetupFreeze as SetupFreeze from nominatim.clicmd.special_phrases import ImportSpecialPhrases as ImportSpecialPhrases from nominatim.clicmd.export import QueryExport as QueryExport +from nominatim.clicmd.convert import ConvertDB as ConvertDB diff --git a/nominatim/clicmd/args.py b/nominatim/clicmd/args.py index e632e4c7..eb3a3b61 100644 --- a/nominatim/clicmd/args.py +++ b/nominatim/clicmd/args.py @@ -101,6 +101,9 @@ class NominatimArgs: language: Optional[str] restrict_to_country: Optional[str] + # Arguments to 'convert' + output: Path + # Arguments to 'refresh' postcodes: bool word_tokens: bool diff --git a/nominatim/clicmd/convert.py b/nominatim/clicmd/convert.py new file mode 100644 index 00000000..26b3fb1f --- /dev/null +++ b/nominatim/clicmd/convert.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Implementation of the 'convert' subcommand. +""" +from typing import Set, Any, Union, Optional, Sequence +import argparse +import asyncio +from pathlib import Path + +from nominatim.clicmd.args import NominatimArgs +from nominatim.errors import UsageError + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +class WithAction(argparse.Action): + """ Special action that saves a list of flags, given on the command-line + as `--with-foo` or `--without-foo`. + """ + def __init__(self, option_strings: Sequence[str], dest: Any, + default: bool = True, **kwargs: Any) -> None: + if 'nargs' in kwargs: + raise ValueError("nargs not allowed.") + if option_strings is None: + raise ValueError("Positional parameter not allowed.") + + self.dest_set = kwargs.pop('dest_set') + full_option_strings = [] + for opt in option_strings: + if not opt.startswith('--'): + raise ValueError("short-form options not allowed") + if default: + self.dest_set.add(opt[2:]) + full_option_strings.append(f"--with-{opt[2:]}") + full_option_strings.append(f"--without-{opt[2:]}") + + super().__init__(full_option_strings, argparse.SUPPRESS, nargs=0, **kwargs) + + + def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None) -> None: + assert option_string + if option_string.startswith('--with-'): + self.dest_set.add(option_string[7:]) + if option_string.startswith('--without-'): + self.dest_set.discard(option_string[10:]) + + +class ConvertDB: + """ Convert an existing database into a different format. (EXPERIMENTAL) + + Dump a read-only version of the database in a different format. + At the moment only a SQLite database suitable for reverse lookup + can be created. + """ + + def __init__(self) -> None: + self.options: Set[str] = set() + + def add_args(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument('--format', default='sqlite', + choices=('sqlite', ), + help='Format of the output database (must be sqlite currently)') + parser.add_argument('--output', '-o', required=True, type=Path, + help='File to write the database to.') + group = parser.add_argument_group('Switches to define database layout' + '(currently no effect)') + group.add_argument('--reverse', action=WithAction, dest_set=self.options, default=True, + help='Enable/disable support for reverse and lookup API' + ' (default: enabled)') + group.add_argument('--search', action=WithAction, dest_set=self.options, default=False, + help='Enable/disable support for search API (default: disabled)') + group.add_argument('--details', action=WithAction, dest_set=self.options, default=True, + help='Enable/disable support for details API (default: enabled)') + + + def run(self, args: NominatimArgs) -> int: + if args.output.exists(): + raise UsageError(f"File '{args.output}' already exists. Refusing to overwrite.") + + if args.format == 'sqlite': + from ..tools import convert_sqlite + + asyncio.run(convert_sqlite.convert(args.project_dir, args.output, self.options)) + return 0 + + return 1 diff --git a/nominatim/tools/convert_sqlite.py b/nominatim/tools/convert_sqlite.py new file mode 100644 index 00000000..42977ce8 --- /dev/null +++ b/nominatim/tools/convert_sqlite.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Exporting a Nominatim database to SQlite. +""" +from typing import Set +from pathlib import Path + +import sqlalchemy as sa + +import nominatim.api as napi + +async def convert(project_dir: Path, outfile: Path, options: Set[str]) -> None: + """ Export an existing database to sqlite. The resulting database + will be usable against the Python frontend of Nominatim. + """ + api = napi.NominatimAPIAsync(project_dir) + + try: + outapi = napi.NominatimAPIAsync(project_dir, + {'NOMINATIM_DATABASE_DSN': f"sqlite:dbname={outfile}"}) + + async with api.begin() as inconn, outapi.begin() as outconn: + pass + finally: + await api.close()