2021-02-08 19:23:05 +03:00
|
|
|
"""
|
|
|
|
Implementation of 'refresh' subcommand.
|
|
|
|
"""
|
|
|
|
import logging
|
|
|
|
from pathlib import Path
|
|
|
|
|
2021-04-16 15:20:09 +03:00
|
|
|
from nominatim.db.connection import connect
|
2021-02-08 19:23:05 +03:00
|
|
|
|
|
|
|
# Do not repeat documentation of subcommand classes.
|
|
|
|
# pylint: disable=C0111
|
|
|
|
# Using non-top-level imports to avoid eventually unused imports.
|
|
|
|
# pylint: disable=E0012,C0415
|
|
|
|
|
|
|
|
LOG = logging.getLogger()
|
|
|
|
|
|
|
|
class UpdateRefresh:
|
|
|
|
"""\
|
|
|
|
Recompute auxiliary data used by the indexing process.
|
|
|
|
|
|
|
|
These functions must not be run in parallel with other update commands.
|
|
|
|
"""
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def add_args(parser):
|
|
|
|
group = parser.add_argument_group('Data arguments')
|
|
|
|
group.add_argument('--postcodes', action='store_true',
|
|
|
|
help='Update postcode centroid table')
|
|
|
|
group.add_argument('--word-counts', action='store_true',
|
|
|
|
help='Compute frequency of full-word search terms')
|
|
|
|
group.add_argument('--address-levels', action='store_true',
|
|
|
|
help='Reimport address level configuration')
|
|
|
|
group.add_argument('--functions', action='store_true',
|
|
|
|
help='Update the PL/pgSQL functions in the database')
|
|
|
|
group.add_argument('--wiki-data', action='store_true',
|
|
|
|
help='Update Wikipedia/data importance numbers.')
|
|
|
|
group.add_argument('--importance', action='store_true',
|
|
|
|
help='Recompute place importances (expensive!)')
|
|
|
|
group.add_argument('--website', action='store_true',
|
|
|
|
help='Refresh the directory that serves the scripts for the web API')
|
|
|
|
group = parser.add_argument_group('Arguments for function refresh')
|
|
|
|
group.add_argument('--no-diff-updates', action='store_false', dest='diffs',
|
|
|
|
help='Do not enable code for propagating updates')
|
|
|
|
group.add_argument('--enable-debug-statements', action='store_true',
|
|
|
|
help='Enable debug warning statements in functions')
|
|
|
|
|
|
|
|
@staticmethod
|
|
|
|
def run(args):
|
2021-05-12 20:57:48 +03:00
|
|
|
from ..tools import refresh, postcodes
|
2021-04-22 23:47:34 +03:00
|
|
|
from ..tokenizer import factory as tokenizer_factory
|
2021-05-13 01:14:52 +03:00
|
|
|
from ..indexer.indexer import Indexer
|
2021-02-08 19:23:05 +03:00
|
|
|
|
2021-05-12 20:57:48 +03:00
|
|
|
tokenizer = tokenizer_factory.get_tokenizer_for_db(args.config)
|
|
|
|
|
2021-02-08 19:23:05 +03:00
|
|
|
if args.postcodes:
|
|
|
|
LOG.warning("Update postcodes centroid")
|
2021-05-12 20:57:48 +03:00
|
|
|
postcodes.update_postcodes(args.config.get_libpq_dsn(),
|
|
|
|
args.project_dir, tokenizer)
|
2021-05-13 01:14:52 +03:00
|
|
|
indexer = Indexer(args.config.get_libpq_dsn(), tokenizer,
|
|
|
|
args.threads or 1)
|
|
|
|
indexer.index_postcodes()
|
2021-02-08 19:23:05 +03:00
|
|
|
|
|
|
|
if args.word_counts:
|
|
|
|
LOG.warning('Recompute frequency of full-word search terms')
|
2021-02-24 00:50:23 +03:00
|
|
|
refresh.recompute_word_counts(args.config.get_libpq_dsn(), args.sqllib_dir)
|
2021-02-08 19:23:05 +03:00
|
|
|
|
|
|
|
if args.address_levels:
|
|
|
|
cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)
|
|
|
|
LOG.warning('Updating address levels from %s', cfg)
|
2021-02-23 12:11:21 +03:00
|
|
|
with connect(args.config.get_libpq_dsn()) as conn:
|
|
|
|
refresh.load_address_levels_from_file(conn, cfg)
|
2021-02-08 19:23:05 +03:00
|
|
|
|
|
|
|
if args.functions:
|
|
|
|
LOG.warning('Create functions')
|
2021-02-23 12:11:21 +03:00
|
|
|
with connect(args.config.get_libpq_dsn()) as conn:
|
2021-04-19 11:01:17 +03:00
|
|
|
refresh.create_functions(conn, args.config,
|
2021-02-23 12:11:21 +03:00
|
|
|
args.diffs, args.enable_debug_statements)
|
2021-04-22 23:47:34 +03:00
|
|
|
tokenizer.update_sql_functions(args.config)
|
2021-02-08 19:23:05 +03:00
|
|
|
|
|
|
|
if args.wiki_data:
|
2021-02-25 00:02:13 +03:00
|
|
|
data_path = Path(args.config.WIKIPEDIA_DATA_PATH
|
|
|
|
or args.project_dir)
|
|
|
|
LOG.warning('Import wikipdia article importance from %s', data_path)
|
|
|
|
if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
|
|
|
|
data_path) > 0:
|
|
|
|
LOG.fatal('FATAL: Wikipedia importance dump file not found')
|
|
|
|
return 1
|
|
|
|
|
2021-02-08 19:23:05 +03:00
|
|
|
# Attention: importance MUST come after wiki data import.
|
|
|
|
if args.importance:
|
2021-02-25 00:02:13 +03:00
|
|
|
LOG.warning('Update importance values for database')
|
|
|
|
with connect(args.config.get_libpq_dsn()) as conn:
|
|
|
|
refresh.recompute_importance(conn)
|
|
|
|
|
2021-02-08 19:23:05 +03:00
|
|
|
if args.website:
|
2021-02-19 19:51:06 +03:00
|
|
|
webdir = args.project_dir / 'website'
|
|
|
|
LOG.warning('Setting up website directory at %s', webdir)
|
2021-05-13 00:44:37 +03:00
|
|
|
with connect(args.config.get_libpq_dsn()) as conn:
|
|
|
|
refresh.setup_website(webdir, args.config, conn)
|
2021-02-08 19:23:05 +03:00
|
|
|
return 0
|