From 7957af562d5ce8266b177039783be4dc8bdd7898 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 6 Jul 2022 15:37:27 +0200 Subject: [PATCH] blacken all the code https://black.readthedocs.io/ --- conftest.py | 21 +- docs/conf.py | 148 +- scripts/errorlist.py | 4 +- scripts/glibc_check.py | 21 +- scripts/hash_sizes.py | 19 +- setup.py | 142 +- setup_docs.py | 380 +- src/borg/__init__.py | 6 +- src/borg/__main__.py | 7 +- src/borg/archive.py | 816 +-- src/borg/archiver.py | 4193 ++++++++------ src/borg/cache.py | 533 +- src/borg/constants.py | 38 +- src/borg/crypto/file_integrity.py | 41 +- src/borg/crypto/key.py | 294 +- src/borg/crypto/keymanager.py | 92 +- src/borg/crypto/nonces.py | 8 +- src/borg/fuse.py | 213 +- src/borg/fuse_impl.py | 10 +- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/checks.py | 13 +- src/borg/helpers/datastruct.py | 5 +- src/borg/helpers/errors.py | 2 + src/borg/helpers/fs.py | 94 +- src/borg/helpers/manifest.py | 91 +- src/borg/helpers/misc.py | 103 +- src/borg/helpers/msgpack.py | 100 +- src/borg/helpers/parseformat.py | 548 +- src/borg/helpers/passphrase.py | 66 +- src/borg/helpers/process.py | 88 +- src/borg/helpers/progress.py | 29 +- src/borg/helpers/time.py | 38 +- src/borg/helpers/yes.py | 55 +- src/borg/locking.py | 39 +- src/borg/logger.py | 64 +- src/borg/lrucache.py | 4 +- src/borg/nanorst.py | 128 +- src/borg/patterns.py | 84 +- src/borg/platform/__init__.py | 1 + src/borg/platform/base.py | 37 +- src/borg/platform/xattr.py | 11 +- src/borg/platformflags.py | 8 +- src/borg/remote.py | 590 +- src/borg/repository.py | 530 +- src/borg/selftest.py | 23 +- src/borg/shellpattern.py | 2 +- src/borg/testsuite/__init__.py | 67 +- src/borg/testsuite/archive.py | 153 +- src/borg/testsuite/archiver.py | 4829 +++++++++-------- src/borg/testsuite/benchmark.py | 53 +- src/borg/testsuite/cache.py | 196 +- src/borg/testsuite/checksums.py | 26 +- src/borg/testsuite/chunker.py | 122 +- src/borg/testsuite/chunker_pytest.py | 91 +- src/borg/testsuite/chunker_slow.py | 7 +- src/borg/testsuite/compress.py | 99 +- src/borg/testsuite/crypto.py | 241 +- .../testsuite/efficient_collection_queue.py | 24 +- src/borg/testsuite/file_integrity.py | 109 +- src/borg/testsuite/hashindex.py | 68 +- src/borg/testsuite/helpers.py | 872 +-- src/borg/testsuite/item.py | 75 +- src/borg/testsuite/key.py | 233 +- src/borg/testsuite/locking.py | 88 +- src/borg/testsuite/logger.py | 15 +- src/borg/testsuite/lrucache.py | 33 +- src/borg/testsuite/nanorst.py | 20 +- src/borg/testsuite/nonces.py | 29 +- src/borg/testsuite/patterns.py | 649 ++- src/borg/testsuite/platform.py | 134 +- src/borg/testsuite/remote.py | 44 +- src/borg/testsuite/repository.py | 453 +- src/borg/testsuite/shellpattern.py | 187 +- src/borg/testsuite/version.py | 76 +- src/borg/testsuite/xattr.py | 52 +- src/borg/upgrade.py | 73 +- src/borg/version.py | 14 +- src/borg/xattr.py | 32 +- 78 files changed, 10568 insertions(+), 8337 deletions(-) diff --git a/conftest.py b/conftest.py index 6166b4759..1c08cbbda 100644 --- a/conftest.py +++ b/conftest.py @@ -3,8 +3,8 @@ import pytest # needed to get pretty assertion failures in unit tests: -if hasattr(pytest, 'register_assert_rewrite'): - pytest.register_assert_rewrite('borg.testsuite') +if hasattr(pytest, "register_assert_rewrite"): + pytest.register_assert_rewrite("borg.testsuite") import borg.cache # noqa: E402 @@ -21,11 +21,10 @@ @pytest.fixture(autouse=True) def clean_env(tmpdir_factory, monkeypatch): # avoid that we access / modify the user's normal .config / .cache directory: - monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home'))) - monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home'))) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir_factory.mktemp("xdg-config-home"))) + monkeypatch.setenv("XDG_CACHE_HOME", str(tmpdir_factory.mktemp("xdg-cache-home"))) # also avoid to use anything from the outside environment: - keys = [key for key in os.environ - if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )] + keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)] for key in keys: monkeypatch.delenv(key, raising=False) # Speed up tests @@ -41,7 +40,7 @@ def pytest_report_header(config, startdir): "symlinks": are_symlinks_supported(), "hardlinks": are_hardlinks_supported(), "atime/mtime": is_utime_fully_supported(), - "modes": "BORG_TESTS_IGNORE_MODES" not in os.environ + "modes": "BORG_TESTS_IGNORE_MODES" not in os.environ, } enabled = [] disabled = [] @@ -60,9 +59,11 @@ def __init__(self, request): self.org_cache_wipe_cache = borg.cache.LocalCache.wipe_cache def wipe_should_not_be_called(*a, **kw): - raise AssertionError("Cache wipe was triggered, if this is part of the test add " - "@pytest.mark.allow_cache_wipe") - if 'allow_cache_wipe' not in request.keywords: + raise AssertionError( + "Cache wipe was triggered, if this is part of the test add " "@pytest.mark.allow_cache_wipe" + ) + + if "allow_cache_wipe" not in request.keywords: borg.cache.LocalCache.wipe_cache = wipe_should_not_be_called request.addfinalizer(self.undo) diff --git a/docs/conf.py b/docs/conf.py index a4b1caf2f..fd13fed41 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,84 +13,85 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. import sys, os -sys.path.insert(0, os.path.abspath('../src')) + +sys.path.insert(0, os.path.abspath("../src")) from borg import __version__ as sw_version # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'Borg - Deduplicating Archiver' -copyright = '2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)' +project = "Borg - Deduplicating Archiver" +copyright = "2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -split_char = '+' if '+' in sw_version else '-' +split_char = "+" if "+" in sw_version else "-" version = sw_version.split(split_char)[0] # The full version, including alpha/beta/rc tags. release = version -suppress_warnings = ['image.nonlocal_uri'] +suppress_warnings = ["image.nonlocal_uri"] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -today_fmt = '%Y-%m-%d' +today_fmt = "%Y-%m-%d" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # The Borg docs contain no or very little Python docs. # Thus, the primary domain is rst. -primary_domain = 'rst' +primary_domain = "rst" # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -100,79 +101,73 @@ import guzzle_sphinx_theme html_theme_path = guzzle_sphinx_theme.html_theme_path() -html_theme = 'guzzle_sphinx_theme' +html_theme = "guzzle_sphinx_theme" def set_rst_settings(app): - app.env.settings.update({ - 'field_name_limit': 0, - 'option_limit': 0, - }) + app.env.settings.update({"field_name_limit": 0, "option_limit": 0}) def setup(app): - app.add_css_file('css/borg.css') - app.connect('builder-inited', set_rst_settings) + app.add_css_file("css/borg.css") + app.connect("builder-inited", set_rst_settings) + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme_options = { - 'project_nav_name': 'Borg %s' % version, -} +html_theme_options = {"project_nav_name": "Borg %s" % version} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = ['_themes'] +# html_theme_path = ['_themes'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/logo.svg' +html_logo = "_static/logo.svg" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -html_favicon = '_static/favicon.ico' +html_favicon = "_static/favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['borg_theme'] +html_static_path = ["borg_theme"] -html_extra_path = ['../src/borg/paperkey.html'] +html_extra_path = ["../src/borg/paperkey.html"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -html_last_updated_fmt = '%Y-%m-%d' +html_last_updated_fmt = "%Y-%m-%d" # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. html_use_smartypants = True -smartquotes_action = 'qe' # no D in there means "do not transform -- and ---" +smartquotes_action = "qe" # no D in there means "do not transform -- and ---" # Custom sidebar templates, maps document names to template names. -html_sidebars = { - '**': ['logo-text.html', 'searchbox.html', 'globaltoc.html'], -} +html_sidebars = {"**": ["logo-text.html", "searchbox.html", "globaltoc.html"]} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. html_use_index = False # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False @@ -186,57 +181,45 @@ def setup(app): # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'borgdoc' +htmlhelp_basename = "borgdoc" # -- Options for LaTeX output -------------------------------------------------- # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('book', 'Borg.tex', 'Borg Documentation', - 'The Borg Collective', 'manual'), -] +latex_documents = [("book", "Borg.tex", "Borg Documentation", "The Borg Collective", "manual")] # The name of an image file (relative to this directory) to place at the top of # the title page. -latex_logo = '_static/logo.pdf' +latex_logo = "_static/logo.pdf" -latex_elements = { - 'papersize': 'a4paper', - 'pointsize': '10pt', - 'figure_align': 'H', -} +latex_elements = {"papersize": "a4paper", "pointsize": "10pt", "figure_align": "H"} # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -latex_show_urls = 'footnote' +latex_show_urls = "footnote" # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -latex_appendices = [ - 'support', - 'resources', - 'changes', - 'authors', -] +latex_appendices = ["support", "resources", "changes", "authors"] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -244,21 +227,24 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('usage', 'borg', - 'BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.', - ['The Borg Collective (see AUTHORS file)'], - 1), + ( + "usage", + "borg", + "BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.", + ["The Borg Collective (see AUTHORS file)"], + 1, + ) ] extensions = [ - 'sphinx.ext.extlinks', - 'sphinx.ext.autodoc', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', + "sphinx.ext.extlinks", + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", ] extlinks = { - 'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'), - 'targz_url': ('https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz' % version, None), + "issue": ("https://github.com/borgbackup/borg/issues/%s", "#"), + "targz_url": ("https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz" % version, None), } diff --git a/scripts/errorlist.py b/scripts/errorlist.py index c4a0e7a0e..6eae5056b 100755 --- a/scripts/errorlist.py +++ b/scripts/errorlist.py @@ -10,5 +10,5 @@ for cls in sorted(classes, key=lambda cls: (cls.__module__, cls.__qualname__)): if cls is ErrorWithTraceback: continue - print(' ', cls.__qualname__) - print(indent(cls.__doc__, ' ' * 8)) + print(" ", cls.__qualname__) + print(indent(cls.__doc__, " " * 8)) diff --git a/scripts/glibc_check.py b/scripts/glibc_check.py index 5f8344960..8c1585e6e 100755 --- a/scripts/glibc_check.py +++ b/scripts/glibc_check.py @@ -13,11 +13,11 @@ verbose = True objdump = "objdump -T %s" -glibc_re = re.compile(r'GLIBC_([0-9]\.[0-9]+)') +glibc_re = re.compile(r"GLIBC_([0-9]\.[0-9]+)") def parse_version(v): - major, minor = v.split('.') + major, minor = v.split(".") return int(major), int(minor) @@ -32,11 +32,9 @@ def main(): overall_versions = set() for filename in filenames: try: - output = subprocess.check_output(objdump % filename, shell=True, - stderr=subprocess.STDOUT) + output = subprocess.check_output(objdump % filename, shell=True, stderr=subprocess.STDOUT) output = output.decode() - versions = {parse_version(match.group(1)) - for match in glibc_re.finditer(output)} + versions = {parse_version(match.group(1)) for match in glibc_re.finditer(output)} requires_glibc = max(versions) overall_versions.add(requires_glibc) if verbose: @@ -50,14 +48,15 @@ def main(): if verbose: if ok: - print("The binaries work with the given glibc %s." % - format_version(given)) + print("The binaries work with the given glibc %s." % format_version(given)) else: - print("The binaries do not work with the given glibc %s. " - "Minimum is: %s" % (format_version(given), format_version(wanted))) + print( + "The binaries do not work with the given glibc %s. " + "Minimum is: %s" % (format_version(given), format_version(wanted)) + ) return ok -if __name__ == '__main__': +if __name__ == "__main__": ok = main() sys.exit(0 if ok else 1) diff --git a/scripts/hash_sizes.py b/scripts/hash_sizes.py index 68e6e160a..19a33916d 100644 --- a/scripts/hash_sizes.py +++ b/scripts/hash_sizes.py @@ -23,11 +23,11 @@ # which growth factor to use when growing a hashtable of size < upto # grow fast (*2.0) at the start so we do not have to resize too often (expensive). # grow slow (*1.1) for huge hash tables (do not jump too much in memory usage) - Policy(256*K, 2.0), - Policy(2*M, 1.7), - Policy(16*M, 1.4), - Policy(128*M, 1.2), - Policy(2*G-1, 1.1), + Policy(256 * K, 2.0), + Policy(2 * M, 1.7), + Policy(16 * M, 1.4), + Policy(128 * M, 1.2), + Policy(2 * G - 1, 1.1), ] @@ -92,12 +92,15 @@ def main(): sizes.append(p) i = int(i * grow_factor) - print("""\ + print( + """\ static int hash_sizes[] = { %s }; -""" % ', '.join(str(size) for size in sizes)) +""" + % ", ".join(str(size) for size in sizes) + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index f2efad233..d79c8100a 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,11 @@ sys.path += [os.path.dirname(__file__)] import setup_docs -is_win32 = sys.platform.startswith('win32') -is_openbsd = sys.platform.startswith('openbsd') +is_win32 = sys.platform.startswith("win32") +is_openbsd = sys.platform.startswith("openbsd") # Number of threads to use for cythonize, not used on windows -cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != 'spawn' else None +cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None # How the build process finds the system libs: # @@ -38,27 +38,23 @@ # 3. otherwise raise a fatal error. # Are we building on ReadTheDocs? -on_rtd = os.environ.get('READTHEDOCS') +on_rtd = os.environ.get("READTHEDOCS") # Extra cflags for all extensions, usually just warnings we want to explicitly enable -cflags = [ - '-Wall', - '-Wextra', - '-Wpointer-arith', -] +cflags = ["-Wall", "-Wextra", "-Wpointer-arith"] -compress_source = 'src/borg/compress.pyx' -crypto_ll_source = 'src/borg/crypto/low_level.pyx' -chunker_source = 'src/borg/chunker.pyx' -hashindex_source = 'src/borg/hashindex.pyx' -item_source = 'src/borg/item.pyx' -checksums_source = 'src/borg/checksums.pyx' -platform_posix_source = 'src/borg/platform/posix.pyx' -platform_linux_source = 'src/borg/platform/linux.pyx' -platform_syncfilerange_source = 'src/borg/platform/syncfilerange.pyx' -platform_darwin_source = 'src/borg/platform/darwin.pyx' -platform_freebsd_source = 'src/borg/platform/freebsd.pyx' -platform_windows_source = 'src/borg/platform/windows.pyx' +compress_source = "src/borg/compress.pyx" +crypto_ll_source = "src/borg/crypto/low_level.pyx" +chunker_source = "src/borg/chunker.pyx" +hashindex_source = "src/borg/hashindex.pyx" +item_source = "src/borg/item.pyx" +checksums_source = "src/borg/checksums.pyx" +platform_posix_source = "src/borg/platform/posix.pyx" +platform_linux_source = "src/borg/platform/linux.pyx" +platform_syncfilerange_source = "src/borg/platform/syncfilerange.pyx" +platform_darwin_source = "src/borg/platform/darwin.pyx" +platform_freebsd_source = "src/borg/platform/freebsd.pyx" +platform_windows_source = "src/borg/platform/windows.pyx" cython_sources = [ compress_source, @@ -67,7 +63,6 @@ hashindex_source, item_source, checksums_source, - platform_posix_source, platform_linux_source, platform_syncfilerange_source, @@ -79,19 +74,20 @@ if cythonize: Sdist = sdist else: + class Sdist(sdist): def __init__(self, *args, **kwargs): - raise Exception('Cython is required to run sdist') + raise Exception("Cython is required to run sdist") - cython_c_files = [fn.replace('.pyx', '.c') for fn in cython_sources] + cython_c_files = [fn.replace(".pyx", ".c") for fn in cython_sources] if not on_rtd and not all(os.path.exists(path) for path in cython_c_files): - raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') + raise ImportError("The GIT version of Borg needs Cython. Install Cython or use a released version.") def rm(file): try: os.unlink(file) - print('rm', file) + print("rm", file) except FileNotFoundError: pass @@ -107,19 +103,19 @@ def finalize_options(self): def run(self): for source in cython_sources: - genc = source.replace('.pyx', '.c') + genc = source.replace(".pyx", ".c") rm(genc) - compiled_glob = source.replace('.pyx', '.cpython*') + compiled_glob = source.replace(".pyx", ".cpython*") for compiled in sorted(glob(compiled_glob)): rm(compiled) cmdclass = { - 'build_ext': build_ext, - 'build_usage': setup_docs.build_usage, - 'build_man': setup_docs.build_man, - 'sdist': Sdist, - 'clean2': Clean, + "build_ext": build_ext, + "build_usage": setup_docs.build_usage, + "build_man": setup_docs.build_man, + "sdist": Sdist, + "clean2": Clean, } @@ -137,16 +133,18 @@ def members_appended(*ds): try: import pkgconfig as pc except ImportError: - print('Warning: can not import pkgconfig python package.') + print("Warning: can not import pkgconfig python package.") pc = None - def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir='lib'): + def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir="lib"): system_prefix = os.environ.get(prefix_env_var) if system_prefix: print(f"Detected and preferring {lib_pkg_name} [via {prefix_env_var}]") - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, lib_subdir)], - libraries=[lib_name]) + return dict( + include_dirs=[os.path.join(system_prefix, "include")], + library_dirs=[os.path.join(system_prefix, lib_subdir)], + libraries=[lib_name], + ) if pc and pc.installed(lib_pkg_name, pc_version): print(f"Detected and preferring {lib_pkg_name} [via pkg-config]") @@ -158,16 +156,13 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s crypto_ldflags = [] if is_win32: - crypto_ext_lib = lib_ext_kwargs( - pc, 'BORG_OPENSSL_PREFIX', 'libcrypto', 'libcrypto', '>=1.1.1', lib_subdir='') + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="") elif is_openbsd: # use openssl (not libressl) because we need AES-OCB and CHACHA20-POLY1305 via EVP api - crypto_ext_lib = lib_ext_kwargs( - pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libecrypto11', '>=1.1.1') - crypto_ldflags += ['-Wl,-rpath=/usr/local/lib/eopenssl11'] + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libecrypto11", ">=1.1.1") + crypto_ldflags += ["-Wl,-rpath=/usr/local/lib/eopenssl11"] else: - crypto_ext_lib = lib_ext_kwargs( - pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libcrypto', '>=1.1.1') + crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1") crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source]), @@ -178,57 +173,60 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s compress_ext_kwargs = members_appended( dict(sources=[compress_source]), - lib_ext_kwargs(pc, 'BORG_LIBLZ4_PREFIX', 'lz4', 'liblz4', '>= 1.7.0'), - lib_ext_kwargs(pc, 'BORG_LIBZSTD_PREFIX', 'zstd', 'libzstd', '>= 1.3.0'), + lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"), + lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"), dict(extra_compile_args=cflags), ) checksums_ext_kwargs = members_appended( dict(sources=[checksums_source]), - lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'), + lib_ext_kwargs(pc, "BORG_LIBXXHASH_PREFIX", "xxhash", "libxxhash", ">= 0.7.3"), dict(extra_compile_args=cflags), ) ext_modules += [ - Extension('borg.crypto.low_level', **crypto_ext_kwargs), - Extension('borg.compress', **compress_ext_kwargs), - Extension('borg.hashindex', [hashindex_source], extra_compile_args=cflags), - Extension('borg.item', [item_source], extra_compile_args=cflags), - Extension('borg.chunker', [chunker_source], extra_compile_args=cflags), - Extension('borg.checksums', **checksums_ext_kwargs), + Extension("borg.crypto.low_level", **crypto_ext_kwargs), + Extension("borg.compress", **compress_ext_kwargs), + Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags), + Extension("borg.item", [item_source], extra_compile_args=cflags), + Extension("borg.chunker", [chunker_source], extra_compile_args=cflags), + Extension("borg.checksums", **checksums_ext_kwargs), ] - posix_ext = Extension('borg.platform.posix', [platform_posix_source], extra_compile_args=cflags) - linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'], extra_compile_args=cflags) - syncfilerange_ext = Extension('borg.platform.syncfilerange', [platform_syncfilerange_source], extra_compile_args=cflags) - freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source], extra_compile_args=cflags) - darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source], extra_compile_args=cflags) - windows_ext = Extension('borg.platform.windows', [platform_windows_source], extra_compile_args=cflags) + posix_ext = Extension("borg.platform.posix", [platform_posix_source], extra_compile_args=cflags) + linux_ext = Extension("borg.platform.linux", [platform_linux_source], libraries=["acl"], extra_compile_args=cflags) + syncfilerange_ext = Extension( + "borg.platform.syncfilerange", [platform_syncfilerange_source], extra_compile_args=cflags + ) + freebsd_ext = Extension("borg.platform.freebsd", [platform_freebsd_source], extra_compile_args=cflags) + darwin_ext = Extension("borg.platform.darwin", [platform_darwin_source], extra_compile_args=cflags) + windows_ext = Extension("borg.platform.windows", [platform_windows_source], extra_compile_args=cflags) if not is_win32: ext_modules.append(posix_ext) else: ext_modules.append(windows_ext) - if sys.platform == 'linux': + if sys.platform == "linux": ext_modules.append(linux_ext) ext_modules.append(syncfilerange_ext) - elif sys.platform.startswith('freebsd'): + elif sys.platform.startswith("freebsd"): ext_modules.append(freebsd_ext) - elif sys.platform == 'darwin': + elif sys.platform == "darwin": ext_modules.append(darwin_ext) # sometimes there's no need to cythonize # this breaks chained commands like 'clean sdist' - cythonizing = len(sys.argv) > 1 and sys.argv[1] not in ( - ('clean', 'clean2', 'egg_info', '--help-commands', '--version')) and '--help' not in sys.argv[1:] + cythonizing = ( + len(sys.argv) > 1 + and sys.argv[1] not in (("clean", "clean2", "egg_info", "--help-commands", "--version")) + and "--help" not in sys.argv[1:] + ) if cythonize and cythonizing: - cython_opts = dict( - compiler_directives={'language_level': '3str'}, - ) + cython_opts = dict(compiler_directives={"language_level": "3str"}) if not is_win32: # compile .pyx extensions to .c in parallel, does not work on windows - cython_opts['nthreads'] = cpu_threads + cython_opts["nthreads"] = cpu_threads # generate C code from Cython for ALL supported platforms, so we have them in the sdist. # the sdist does not require Cython at install time, so we need all as C. @@ -237,8 +235,4 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s ext_modules = cythonize(ext_modules, **cython_opts) -setup( - cmdclass=cmdclass, - ext_modules=ext_modules, - long_description=setup_docs.long_desc_from_readme() -) +setup(cmdclass=cmdclass, ext_modules=ext_modules, long_description=setup_docs.long_desc_from_readme()) diff --git a/setup_docs.py b/setup_docs.py index 7d985ebff..c568a70a2 100644 --- a/setup_docs.py +++ b/setup_docs.py @@ -12,36 +12,34 @@ def long_desc_from_readme(): - with open('README.rst') as fd: + with open("README.rst") as fd: long_description = fd.read() # remove header, but have one \n before first headline - start = long_description.find('What is BorgBackup?') + start = long_description.find("What is BorgBackup?") assert start >= 0 - long_description = '\n' + long_description[start:] + long_description = "\n" + long_description[start:] # remove badges - long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description) + long_description = re.compile(r"^\.\. start-badges.*^\.\. end-badges", re.M | re.S).sub("", long_description) # remove unknown directives - long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description) + long_description = re.compile(r"^\.\. highlight:: \w+$", re.M).sub("", long_description) return long_description def format_metavar(option): - if option.nargs in ('*', '...'): - return '[%s...]' % option.metavar - elif option.nargs == '?': - return '[%s]' % option.metavar + if option.nargs in ("*", "..."): + return "[%s...]" % option.metavar + elif option.nargs == "?": + return "[%s]" % option.metavar elif option.nargs is None: return option.metavar else: - raise ValueError(f'Can\'t format metavar {option.metavar}, unknown nargs {option.nargs}!') + raise ValueError(f"Can't format metavar {option.metavar}, unknown nargs {option.nargs}!") class build_usage(Command): description = "generate usage for each command" - user_options = [ - ('output=', 'O', 'output directory'), - ] + user_options = [("output=", "O", "output directory")] def initialize_options(self): pass @@ -50,17 +48,19 @@ def finalize_options(self): pass def run(self): - print('generating usage docs') + print("generating usage docs") import borg - borg.doc_mode = 'build_man' - if not os.path.exists('docs/usage'): - os.mkdir('docs/usage') + + borg.doc_mode = "build_man" + if not os.path.exists("docs/usage"): + os.mkdir("docs/usage") # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver - parser = Archiver(prog='borg').build_parser() + + parser = Archiver(prog="borg").build_parser() # borgfs has a separate man page to satisfy debian's "every program from a package # must have a man page" requirement, but it doesn't need a separate HTML docs page - #borgfs_parser = Archiver(prog='borgfs').build_parser() + # borgfs_parser = Archiver(prog='borgfs').build_parser() self.generate_level("", parser, Archiver) @@ -68,7 +68,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: - if action.choices is not None and 'SubParsersAction' in str(action.__class__): + if action.choices is not None and "SubParsersAction" in str(action.__class__): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser @@ -76,32 +76,37 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): choices.update(extra_choices) if prefix and not choices: return - print('found commands: %s' % list(choices.keys())) + print("found commands: %s" % list(choices.keys())) for command, parser in sorted(choices.items()): - if command.startswith('debug'): - print('skipping', command) + if command.startswith("debug"): + print("skipping", command) continue - print('generating help for %s' % command) + print("generating help for %s" % command) if self.generate_level(command + " ", parser, Archiver): continue - with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: + with open("docs/usage/%s.rst.inc" % command.replace(" ", "_"), "w") as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") - if command == 'help': + if command == "help": for topic in Archiver.helptext: - params = {"topic": topic, - "underline": '~' * len('borg help ' + topic)} + params = {"topic": topic, "underline": "~" * len("borg help " + topic)} doc.write(".. _borg_{topic}:\n\n".format(**params)) doc.write("borg help {topic}\n{underline}\n\n".format(**params)) doc.write(Archiver.helptext[topic]) else: - params = {"command": command, - "command_": command.replace(' ', '_'), - "underline": '-' * len('borg ' + command)} + params = { + "command": command, + "command_": command.replace(" ", "_"), + "underline": "-" * len("borg " + command), + } doc.write(".. _borg_{command_}:\n\n".format(**params)) - doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params)) + doc.write( + "borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format( + **params + ) + ) self.write_usage(parser, doc) epilog = parser.epilog parser.epilog = None @@ -109,21 +114,21 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): doc.write("\n\nDescription\n~~~~~~~~~~~\n") doc.write(epilog) - if 'create' in choices: - common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] - with open('docs/usage/common-options.rst.inc', 'w') as doc: + if "create" in choices: + common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0] + with open("docs/usage/common-options.rst.inc", "w") as doc: self.write_options_group(common_options, doc, False, base_indent=0) return is_subcommand def write_usage(self, parser, fp): if any(len(o.option_strings) for o in parser._actions): - fp.write(' [options]') + fp.write(" [options]") for option in parser._actions: if option.option_strings: continue - fp.write(' ' + format_metavar(option)) - fp.write('\n\n') + fp.write(" " + format_metavar(option)) + fp.write("\n\n") def write_options(self, parser, fp): def is_positional_group(group): @@ -134,58 +139,58 @@ def is_positional_group(group): def html_write(s): for line in s.splitlines(): - fp.write(' ' + line + '\n') + fp.write(" " + line + "\n") rows = [] for group in parser._action_groups: - if group.title == 'Common options': + if group.title == "Common options": # (no of columns used, columns, ...) - rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`')) + rows.append((1, ".. class:: borg-common-opt-ref\n\n:ref:`common_options`")) else: if not group._group_actions: continue - group_header = '**%s**' % group.title + group_header = "**%s**" % group.title if group.description: - group_header += ' — ' + group.description + group_header += " — " + group.description rows.append((1, group_header)) if is_positional_group(group): for option in group._group_actions: - rows.append((3, '', '``%s``' % option.metavar, option.help or '')) + rows.append((3, "", "``%s``" % option.metavar, option.help or "")) else: for option in group._group_actions: if option.metavar: - option_fmt = '``%s ' + option.metavar + '``' + option_fmt = "``%s " + option.metavar + "``" else: - option_fmt = '``%s``' - option_str = ', '.join(option_fmt % s for s in option.option_strings) - option_desc = textwrap.dedent((option.help or '') % option.__dict__) - rows.append((3, '', option_str, option_desc)) + option_fmt = "``%s``" + option_str = ", ".join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or "") % option.__dict__) + rows.append((3, "", option_str, option_desc)) - fp.write('.. only:: html\n\n') + fp.write(".. only:: html\n\n") table = io.StringIO() - table.write('.. class:: borg-options-table\n\n') + table.write(".. class:: borg-options-table\n\n") self.rows_to_table(rows, table.write) - fp.write(textwrap.indent(table.getvalue(), ' ' * 4)) + fp.write(textwrap.indent(table.getvalue(), " " * 4)) # LaTeX output: # Regular rST option lists (irregular column widths) latex_options = io.StringIO() for group in parser._action_groups: - if group.title == 'Common options': - latex_options.write('\n\n:ref:`common_options`\n') - latex_options.write(' |') + if group.title == "Common options": + latex_options.write("\n\n:ref:`common_options`\n") + latex_options.write(" |") else: self.write_options_group(group, latex_options) - fp.write('\n.. only:: latex\n\n') - fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4)) + fp.write("\n.. only:: latex\n\n") + fp.write(textwrap.indent(latex_options.getvalue(), " " * 4)) def rows_to_table(self, rows, write): def write_row_separator(): - write('+') + write("+") for column_width in column_widths: - write('-' * (column_width + 1)) - write('+') - write('\n') + write("-" * (column_width + 1)) + write("+") + write("\n") # Find column count and width column_count = max(columns for columns, *_ in rows) @@ -201,22 +206,22 @@ def write_row_separator(): # where each cell contains no newline. rowspanning_cells = [] original_cells = list(original_cells) - while any('\n' in cell for cell in original_cells): + while any("\n" in cell for cell in original_cells): cell_bloc = [] for i, cell in enumerate(original_cells): - pre, _, original_cells[i] = cell.partition('\n') + pre, _, original_cells[i] = cell.partition("\n") cell_bloc.append(pre) rowspanning_cells.append(cell_bloc) rowspanning_cells.append(original_cells) for cells in rowspanning_cells: for i, column_width in enumerate(column_widths): if i < columns: - write('| ') + write("| ") write(cells[i].ljust(column_width)) else: - write(' ') - write(''.ljust(column_width)) - write('|\n') + write(" ") + write("".ljust(column_width)) + write("|\n") write_row_separator() # This bit of JavaScript kills the that is invariably inserted by docutils, @@ -224,7 +229,9 @@ def write_row_separator(): # with CSS alone. # Since this is HTML-only output, it would be possible to just generate a directly, # but then we'd lose rST formatting. - write(textwrap.dedent(""" + write( + textwrap.dedent( + """ .. raw:: html - """)) + """ + ) + ) def write_options_group(self, group, fp, with_title=True, base_indent=4): def is_positional_group(group): return any(not o.option_strings for o in group._group_actions) - indent = ' ' * base_indent + indent = " " * base_indent if is_positional_group(group): for option in group._group_actions: - fp.write(option.metavar + '\n') - fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n') + fp.write(option.metavar + "\n") + fp.write(textwrap.indent(option.help or "", " " * base_indent) + "\n") return if not group._group_actions: return if with_title: - fp.write('\n\n') - fp.write(group.title + '\n') + fp.write("\n\n") + fp.write(group.title + "\n") opts = OrderedDict() for option in group._group_actions: if option.metavar: - option_fmt = '%s ' + option.metavar + option_fmt = "%s " + option.metavar else: - option_fmt = '%s' - option_str = ', '.join(option_fmt % s for s in option.option_strings) - option_desc = textwrap.dedent((option.help or '') % option.__dict__) - opts[option_str] = textwrap.indent(option_desc, ' ' * 4) + option_fmt = "%s" + option_str = ", ".join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or "") % option.__dict__) + opts[option_str] = textwrap.indent(option_desc, " " * 4) padding = len(max(opts)) + 1 for option, desc in opts.items(): - fp.write(indent + option.ljust(padding) + desc + '\n') + fp.write(indent + option.ljust(padding) + desc + "\n") class build_man(Command): - description = 'build man pages' + description = "build man pages" user_options = [] see_also = { - 'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'), - 'recreate': ('patterns', 'placeholders', 'compression'), - 'list': ('info', 'diff', 'prune', 'patterns'), - 'info': ('list', 'diff'), - 'rcreate': ('rdelete', 'rlist', 'check', 'key-import', 'key-export', 'key-change-passphrase'), - 'key-import': ('key-export', ), - 'key-export': ('key-import', ), - 'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page - 'umount': ('mount', ), - 'extract': ('mount', ), - 'delete': ('compact', ), - 'prune': ('compact', ), + "create": ("delete", "prune", "check", "patterns", "placeholders", "compression"), + "recreate": ("patterns", "placeholders", "compression"), + "list": ("info", "diff", "prune", "patterns"), + "info": ("list", "diff"), + "rcreate": ("rdelete", "rlist", "check", "key-import", "key-export", "key-change-passphrase"), + "key-import": ("key-export",), + "key-export": ("key-import",), + "mount": ("umount", "extract"), # Would be cooler if these two were on the same page + "umount": ("mount",), + "extract": ("mount",), + "delete": ("compact",), + "prune": ("compact",), } - rst_prelude = textwrap.dedent(""" + rst_prelude = textwrap.dedent( + """ .. role:: ref(title) .. |project_name| replace:: Borg - """) + """ + ) usage_group = { - 'break-lock': 'lock', - 'with-lock': 'lock', - - 'key_change-passphrase': 'key', - 'key_change-location': 'key', - 'key_export': 'key', - 'key_import': 'key', - 'key_migrate-to-repokey': 'key', - - 'export-tar': 'tar', - 'import-tar': 'tar', - - 'benchmark_crud': 'benchmark', - 'benchmark_cpu': 'benchmark', - - 'umount': 'mount', + "break-lock": "lock", + "with-lock": "lock", + "key_change-passphrase": "key", + "key_change-location": "key", + "key_export": "key", + "key_import": "key", + "key_migrate-to-repokey": "key", + "export-tar": "tar", + "import-tar": "tar", + "benchmark_crud": "benchmark", + "benchmark_cpu": "benchmark", + "umount": "mount", } def initialize_options(self): @@ -323,16 +330,18 @@ def finalize_options(self): pass def run(self): - print('building man pages (in docs/man)', file=sys.stderr) + print("building man pages (in docs/man)", file=sys.stderr) import borg - borg.doc_mode = 'build_man' - os.makedirs('docs/man', exist_ok=True) + + borg.doc_mode = "build_man" + os.makedirs("docs/man", exist_ok=True) # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver - parser = Archiver(prog='borg').build_parser() - borgfs_parser = Archiver(prog='borgfs').build_parser() - self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser}) + parser = Archiver(prog="borg").build_parser() + borgfs_parser = Archiver(prog="borgfs").build_parser() + + self.generate_level("", parser, Archiver, {"borgfs": borgfs_parser}) self.build_topic_pages(Archiver) self.build_intro_page() @@ -340,7 +349,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): is_subcommand = False choices = {} for action in parser._actions: - if action.choices is not None and 'SubParsersAction' in str(action.__class__): + if action.choices is not None and "SubParsersAction" in str(action.__class__): is_subcommand = True for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser @@ -350,50 +359,50 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): return for command, parser in sorted(choices.items()): - if command.startswith('debug') or command == 'help': + if command.startswith("debug") or command == "help": continue if command == "borgfs": man_title = command else: - man_title = 'borg-' + command.replace(' ', '-') - print('building man page', man_title + '(1)', file=sys.stderr) + man_title = "borg-" + command.replace(" ", "-") + print("building man page", man_title + "(1)", file=sys.stderr) - is_intermediary = self.generate_level(command + ' ', parser, Archiver) + is_intermediary = self.generate_level(command + " ", parser, Archiver) doc, write = self.new_doc() self.write_man_header(write, man_title, parser.description) - self.write_heading(write, 'SYNOPSIS') + self.write_heading(write, "SYNOPSIS") if is_intermediary: - subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0] + subparsers = [action for action in parser._actions if "SubParsersAction" in str(action.__class__)][0] for subcommand in subparsers.choices: - write('| borg', '[common options]', command, subcommand, '...') - self.see_also.setdefault(command, []).append(f'{command}-{subcommand}') + write("| borg", "[common options]", command, subcommand, "...") + self.see_also.setdefault(command, []).append(f"{command}-{subcommand}") else: if command == "borgfs": - write(command, end='') + write(command, end="") else: - write('borg', '[common options]', command, end='') + write("borg", "[common options]", command, end="") self.write_usage(write, parser) - write('\n') + write("\n") - description, _, notes = parser.epilog.partition('\n.. man NOTES') + description, _, notes = parser.epilog.partition("\n.. man NOTES") if description: - self.write_heading(write, 'DESCRIPTION') + self.write_heading(write, "DESCRIPTION") write(description) if not is_intermediary: - self.write_heading(write, 'OPTIONS') - write('See `borg-common(1)` for common options of Borg commands.') + self.write_heading(write, "OPTIONS") + write("See `borg-common(1)` for common options of Borg commands.") write() self.write_options(write, parser) self.write_examples(write, command) if notes: - self.write_heading(write, 'NOTES') + self.write_heading(write, "NOTES") write(notes) self.write_see_also(write, man_title) @@ -401,14 +410,14 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): self.gen_man_page(man_title, doc.getvalue()) # Generate the borg-common(1) man page with the common options. - if 'create' in choices: + if "create" in choices: doc, write = self.new_doc() - man_title = 'borg-common' - self.write_man_header(write, man_title, 'Common options of Borg commands') + man_title = "borg-common" + self.write_man_header(write, man_title, "Common options of Borg commands") - common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0] + common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0] - self.write_heading(write, 'SYNOPSIS') + self.write_heading(write, "SYNOPSIS") self.write_options_group(write, common_options) self.write_see_also(write, man_title) self.gen_man_page(man_title, doc.getvalue()) @@ -418,20 +427,20 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None): def build_topic_pages(self, Archiver): for topic, text in Archiver.helptext.items(): doc, write = self.new_doc() - man_title = 'borg-' + topic - print('building man page', man_title + '(1)', file=sys.stderr) + man_title = "borg-" + topic + print("building man page", man_title + "(1)", file=sys.stderr) - self.write_man_header(write, man_title, 'Details regarding ' + topic) - self.write_heading(write, 'DESCRIPTION') + self.write_man_header(write, man_title, "Details regarding " + topic) + self.write_heading(write, "DESCRIPTION") write(text) self.gen_man_page(man_title, doc.getvalue()) def build_intro_page(self): doc, write = self.new_doc() - man_title = 'borg' - print('building man page borg(1)', file=sys.stderr) + man_title = "borg" + print("building man page borg(1)", file=sys.stderr) - with open('docs/man_intro.rst') as fd: + with open("docs/man_intro.rst") as fd: man_intro = fd.read() self.write_man_header(write, man_title, "deduplicating and encrypting backup tool") @@ -446,9 +455,10 @@ def new_doc(self): def printer(self, fd): def write(*args, **kwargs): print(*args, file=fd, **kwargs) + return write - def write_heading(self, write, header, char='-', double_sided=False): + def write_heading(self, write, header, char="-", double_sided=False): write() if double_sided: write(char * len(header)) @@ -457,43 +467,43 @@ def write_heading(self, write, header, char='-', double_sided=False): write() def write_man_header(self, write, title, description): - self.write_heading(write, title, '=', double_sided=True) + self.write_heading(write, title, "=", double_sided=True) self.write_heading(write, description, double_sided=True) # man page metadata - write(':Author: The Borg Collective') - write(':Date:', datetime.utcnow().date().isoformat()) - write(':Manual section: 1') - write(':Manual group: borg backup tool') + write(":Author: The Borg Collective") + write(":Date:", datetime.utcnow().date().isoformat()) + write(":Manual section: 1") + write(":Manual group: borg backup tool") write() def write_examples(self, write, command): - command = command.replace(' ', '_') - with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd: + command = command.replace(" ", "_") + with open("docs/usage/%s.rst" % self.usage_group.get(command, command)) as fd: usage = fd.read() - usage_include = '.. include:: %s.rst.inc' % command + usage_include = ".. include:: %s.rst.inc" % command begin = usage.find(usage_include) - end = usage.find('.. include', begin + 1) + end = usage.find(".. include", begin + 1) # If a command has a dedicated anchor, it will occur before the command's include. - if 0 < usage.find('.. _', begin + 1) < end: - end = usage.find('.. _', begin + 1) + if 0 < usage.find(".. _", begin + 1) < end: + end = usage.find(".. _", begin + 1) examples = usage[begin:end] - examples = examples.replace(usage_include, '') - examples = examples.replace('Examples\n~~~~~~~~', '') - examples = examples.replace('Miscellaneous Help\n------------------', '') - examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.') - examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight - examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE) + examples = examples.replace(usage_include, "") + examples = examples.replace("Examples\n~~~~~~~~", "") + examples = examples.replace("Miscellaneous Help\n------------------", "") + examples = examples.replace("``docs/misc/prune-example.txt``:", "``docs/misc/prune-example.txt``.") + examples = examples.replace(".. highlight:: none\n", "") # we don't support highlight + examples = re.sub("^(~+)$", lambda matches: "+" * len(matches.group(0)), examples, flags=re.MULTILINE) examples = examples.strip() if examples: - self.write_heading(write, 'EXAMPLES', '-') + self.write_heading(write, "EXAMPLES", "-") write(examples) def write_see_also(self, write, man_title): - see_also = self.see_also.get(man_title.replace('borg-', ''), ()) - see_also = ['`borg-%s(1)`' % s for s in see_also] - see_also.insert(0, '`borg-common(1)`') - self.write_heading(write, 'SEE ALSO') - write(', '.join(see_also)) + see_also = self.see_also.get(man_title.replace("borg-", ""), ()) + see_also = ["`borg-%s(1)`" % s for s in see_also] + see_also.insert(0, "`borg-common(1)`") + self.write_heading(write, "SEE ALSO") + write(", ".join(see_also)) def gen_man_page(self, name, rst): from docutils.writers import manpage @@ -502,29 +512,29 @@ def gen_man_page(self, name, rst): from docutils.parsers.rst import roles def issue(name, rawtext, text, lineno, inliner, options={}, content=[]): - return [inline(rawtext, '#' + text)], [] + return [inline(rawtext, "#" + text)], [] - roles.register_local_role('issue', issue) + roles.register_local_role("issue", issue) # We give the source_path so that docutils can find relative includes # as-if the document where located in the docs/ directory. - man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer()) - with open('docs/man/%s.1' % name, 'wb') as fd: + man_page = publish_string(source=rst, source_path="docs/%s.rst" % name, writer=manpage.Writer()) + with open("docs/man/%s.1" % name, "wb") as fd: fd.write(man_page) def write_usage(self, write, parser): if any(len(o.option_strings) for o in parser._actions): - write(' [options] ', end='') + write(" [options] ", end="") for option in parser._actions: if option.option_strings: continue - write(format_metavar(option), end=' ') + write(format_metavar(option), end=" ") def write_options(self, write, parser): for group in parser._action_groups: - if group.title == 'Common options' or not group._group_actions: + if group.title == "Common options" or not group._group_actions: continue - title = 'arguments' if group.title == 'positional arguments' else group.title - self.write_heading(write, title, '+') + title = "arguments" if group.title == "positional arguments" else group.title + self.write_heading(write, title, "+") self.write_options_group(write, group) def write_options_group(self, write, group): @@ -534,19 +544,19 @@ def is_positional_group(group): if is_positional_group(group): for option in group._group_actions: write(option.metavar) - write(textwrap.indent(option.help or '', ' ' * 4)) + write(textwrap.indent(option.help or "", " " * 4)) return opts = OrderedDict() for option in group._group_actions: if option.metavar: - option_fmt = '%s ' + option.metavar + option_fmt = "%s " + option.metavar else: - option_fmt = '%s' - option_str = ', '.join(option_fmt % s for s in option.option_strings) - option_desc = textwrap.dedent((option.help or '') % option.__dict__) - opts[option_str] = textwrap.indent(option_desc, ' ' * 4) + option_fmt = "%s" + option_str = ", ".join(option_fmt % s for s in option.option_strings) + option_desc = textwrap.dedent((option.help or "") % option.__dict__) + opts[option_str] = textwrap.indent(option_desc, " " * 4) padding = len(max(opts)) + 1 diff --git a/src/borg/__init__.py b/src/borg/__init__.py index b2597288e..d05eff6e0 100644 --- a/src/borg/__init__.py +++ b/src/borg/__init__.py @@ -9,11 +9,13 @@ # assert that all semver components are integers # this is mainly to show errors when people repackage poorly # and setuptools_scm determines a 0.1.dev... version -assert all(isinstance(v, int) for v in __version_tuple__), \ +assert all(isinstance(v, int) for v in __version_tuple__), ( """\ broken borgbackup version metadata: %r version metadata is obtained dynamically on installation via setuptools_scm, please ensure your git repo has the correct tags or you provide the version using SETUPTOOLS_SCM_PRETEND_VERSION in your build script. -""" % __version__ +""" + % __version__ +) diff --git a/src/borg/__main__.py b/src/borg/__main__.py index 73a2187de..a1dbbdc34 100644 --- a/src/borg/__main__.py +++ b/src/borg/__main__.py @@ -5,11 +5,12 @@ # containing the dll is not in the search path. The dll is shipped # with python in the "DLLs" folder, so let's add this folder # to the path. The folder is always in sys.path, get it from there. -if sys.platform.startswith('win32'): +if sys.platform.startswith("win32"): # Keep it an iterable to support multiple folder which contain "DLLs". - dll_path = (p for p in sys.path if 'DLLs' in os.path.normpath(p).split(os.path.sep)) - os.environ['PATH'] = os.pathsep.join(dll_path) + os.pathsep + os.environ['PATH'] + dll_path = (p for p in sys.path if "DLLs" in os.path.normpath(p).split(os.path.sep)) + os.environ["PATH"] = os.pathsep.join(dll_path) + os.pathsep + os.environ["PATH"] from borg.archiver import main + main() diff --git a/src/borg/archive.py b/src/borg/archive.py index 172f43cae..d346a425f 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -49,11 +49,10 @@ from .remote import cache_if_remote from .repository import Repository, LIST_SCAN_LIMIT -has_link = hasattr(os, 'link') +has_link = hasattr(os, "link") class Statistics: - def __init__(self, output_json=False, iec=False): self.output_json = output_json self.iec = iec @@ -73,7 +72,7 @@ def update(self, size, unique, part=False): def __add__(self, other): if not isinstance(other, Statistics): - raise TypeError('can only add Statistics objects') + raise TypeError("can only add Statistics objects") stats = Statistics(self.output_json, self.iec) stats.osize = self.osize + other.osize stats.usize = self.usize + other.usize @@ -88,34 +87,37 @@ def __str__(self): Number of files: {stats.nfiles} Original size: {stats.osize_fmt} Deduplicated size: {stats.usize_fmt} -""".format(stats=self) +""".format( + stats=self + ) def __repr__(self): return "<{cls} object at {hash:#x} ({self.osize}, {self.usize})>".format( - cls=type(self).__name__, hash=id(self), self=self) + cls=type(self).__name__, hash=id(self), self=self + ) def as_dict(self): return { - 'original_size': FileSize(self.osize, iec=self.iec), - 'deduplicated_size': FileSize(self.usize, iec=self.iec), - 'nfiles': self.nfiles, + "original_size": FileSize(self.osize, iec=self.iec), + "deduplicated_size": FileSize(self.usize, iec=self.iec), + "nfiles": self.nfiles, } def as_raw_dict(self): return { - 'size': self.osize, - 'nfiles': self.nfiles, - 'size_parts': self.osize_parts, - 'nfiles_parts': self.nfiles_parts, + "size": self.osize, + "nfiles": self.nfiles, + "size_parts": self.osize_parts, + "nfiles_parts": self.nfiles_parts, } @classmethod def from_raw_dict(cls, **kw): self = cls() - self.osize = kw['size'] - self.nfiles = kw['nfiles'] - self.osize_parts = kw['size_parts'] - self.nfiles_parts = kw['nfiles_parts'] + self.osize = kw["size"] + self.nfiles = kw["nfiles"] + self.osize_parts = kw["size_parts"] + self.nfiles_parts = kw["nfiles_parts"] return self @property @@ -133,30 +135,26 @@ def show_progress(self, item=None, final=False, stream=None, dt=None): if self.output_json: if not final: data = self.as_dict() - data['path'] = remove_surrogates(item.path if item else '') + data["path"] = remove_surrogates(item.path if item else "") else: data = {} - data.update({ - 'time': time.time(), - 'type': 'archive_progress', - 'finished': final, - }) + data.update({"time": time.time(), "type": "archive_progress", "finished": final}) msg = json.dumps(data) - end = '\n' + end = "\n" else: columns, lines = get_terminal_size() if not final: - msg = '{0.osize_fmt} O {0.usize_fmt} U {0.nfiles} N '.format(self) - path = remove_surrogates(item.path) if item else '' + msg = "{0.osize_fmt} O {0.usize_fmt} U {0.nfiles} N ".format(self) + path = remove_surrogates(item.path) if item else "" space = columns - swidth(msg) if space < 12: - msg = '' + msg = "" space = columns - swidth(msg) if space >= 8: msg += ellipsis_truncate(path, space) else: - msg = ' ' * columns - end = '\r' + msg = " " * columns + end = "\r" print(msg, end=end, file=stream or sys.stderr, flush=True) @@ -181,6 +179,7 @@ class BackupOSError(Exception): Any unwrapped IO error is critical and aborts execution (for example repository IO failure). """ + def __init__(self, op, os_error): self.op = op self.os_error = os_error @@ -190,15 +189,15 @@ def __init__(self, op, os_error): def __str__(self): if self.op: - return f'{self.op}: {self.os_error}' + return f"{self.op}: {self.os_error}" else: return str(self.os_error) class BackupIO: - op = '' + op = "" - def __call__(self, op=''): + def __call__(self, op=""): self.op = op return self @@ -214,7 +213,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def backup_io_iter(iterator): - backup_io.op = 'read' + backup_io.op = "read" while True: with backup_io: try: @@ -242,16 +241,16 @@ def stat_update_check(st_old, st_curr): # are not duplicate in a short timeframe, this check is redundant and solved by the ino check: if stat.S_IFMT(st_old.st_mode) != stat.S_IFMT(st_curr.st_mode): # in this case, we dispatched to wrong handler - abort - raise BackupError('file type changed (race condition), skipping file') + raise BackupError("file type changed (race condition), skipping file") if st_old.st_ino != st_curr.st_ino: # in this case, the hardlinks-related code in create_helper has the wrong inode - abort! - raise BackupError('file inode changed (race condition), skipping file') + raise BackupError("file inode changed (race condition), skipping file") # looks ok, we are still dealing with the same thing - return current stat: return st_curr @contextmanager -def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='open'): +def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op="open"): with backup_io(op): fd = os_open(path=path, parent_fd=parent_fd, name=name, flags=flags, noatime=noatime) try: @@ -263,7 +262,6 @@ def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='op class DownloadPipeline: - def __init__(self, repository, key): self.repository = repository self.key = key @@ -284,7 +282,7 @@ def unpack_many(self, ids, *, filter=None, preload=False): unpacker.feed(data) items = [Item(internal_dict=item) for item in unpacker] for item in items: - if 'chunks' in item: + if "chunks" in item: item.chunks = [ChunkListEntry(*e) for e in item.chunks] if filter: @@ -292,8 +290,8 @@ def unpack_many(self, ids, *, filter=None, preload=False): if preload: for item in items: - if 'chunks' in item: - hlid = item.get('hlid', None) + if "chunks" in item: + hlid = item.get("hlid", None) if hlid is None: preload_chunks = True else: @@ -342,11 +340,11 @@ def flush(self, flush=False): # with CH_ALLOC (and CH_HOLE, for completeness) here. chunks = [] for chunk in self.chunker.chunkify(self.buffer): - alloc = chunk.meta['allocation'] + alloc = chunk.meta["allocation"] if alloc == CH_DATA: data = bytes(chunk.data) elif alloc in (CH_ALLOC, CH_HOLE): - data = zeros[:chunk.meta['size']] + data = zeros[: chunk.meta["size"]] else: raise ValueError("chunk allocation has unsupported value of %r" % alloc) chunks.append(data) @@ -364,7 +362,6 @@ def is_full(self): class CacheChunkBuffer(ChunkBuffer): - def __init__(self, cache, key, stats, chunker_params=ITEMS_CHUNKER_PARAMS): super().__init__(key, chunker_params) self.cache = cache @@ -380,14 +377,14 @@ def get_item_uid_gid(item, *, numeric, uid_forced=None, gid_forced=None, uid_def if uid_forced is not None: uid = uid_forced else: - uid = None if numeric else user2uid(item.get('user')) + uid = None if numeric else user2uid(item.get("user")) uid = item.uid if uid is None else uid if uid < 0: uid = uid_default if gid_forced is not None: gid = gid_forced else: - gid = None if numeric else group2gid(item.get('group')) + gid = None if numeric else group2gid(item.get("group")) gid = item.gid if gid is None else gid if gid < 0: gid = gid_default @@ -395,7 +392,6 @@ def get_item_uid_gid(item, *, numeric, uid_forced=None, gid_forced=None, uid_def class Archive: - class DoesNotExist(Error): """Archive {} does not exist""" @@ -405,11 +401,30 @@ class AlreadyExists(Error): class IncompatibleFilesystemEncodingError(Error): """Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable.""" - def __init__(self, repository, key, manifest, name, cache=None, create=False, - checkpoint_interval=1800, numeric_ids=False, noatime=False, noctime=False, - noflags=False, noacls=False, noxattrs=False, - progress=False, chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None, - consider_part_files=False, log_json=False, iec=False): + def __init__( + self, + repository, + key, + manifest, + name, + cache=None, + create=False, + checkpoint_interval=1800, + numeric_ids=False, + noatime=False, + noctime=False, + noflags=False, + noacls=False, + noxattrs=False, + progress=False, + chunker_params=CHUNKER_PARAMS, + start=None, + start_monotonic=None, + end=None, + consider_part_files=False, + log_json=False, + iec=False, + ): self.cwd = os.getcwd() self.key = key self.repository = repository @@ -428,7 +443,9 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False, self.noflags = noflags self.noacls = noacls self.noxattrs = noxattrs - assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.' + assert (start is None) == ( + start_monotonic is None + ), "Logic error: if start is given, start_monotonic must be given as well and vice versa." if start is None: start = datetime.utcnow() start_monotonic = time.monotonic() @@ -447,7 +464,7 @@ def __init__(self, repository, key, manifest, name, cache=None, create=False, raise self.AlreadyExists(name) i = 0 while True: - self.checkpoint_name = '{}.checkpoint{}'.format(name, i and ('.%d' % i) or '') + self.checkpoint_name = "{}.checkpoint{}".format(name, i and (".%d" % i) or "") if self.checkpoint_name not in manifest.archives: break i += 1 @@ -461,14 +478,14 @@ def _load_meta(self, id): data = self.key.decrypt(id, self.repository.get(id)) metadata = ArchiveItem(internal_dict=msgpack.unpackb(data)) if metadata.version not in (1, 2): # legacy: still need to read v1 archives - raise Exception('Unknown archive metadata version') + raise Exception("Unknown archive metadata version") return metadata def load(self, id): self.id = id self.metadata = self._load_meta(self.id) self.name = self.metadata.name - self.comment = self.metadata.get('comment', '') + self.comment = self.metadata.get("comment", "") @property def ts(self): @@ -480,7 +497,7 @@ def ts(self): def ts_end(self): """Timestamp of archive creation (end) in UTC""" # fall back to time if there is no time_end present in metadata - ts = self.metadata.get('time_end') or self.metadata.time + ts = self.metadata.get("time_end") or self.metadata.time return parse_timestamp(ts) @property @@ -509,30 +526,30 @@ def info(self): start = self.ts end = self.ts_end info = { - 'name': self.name, - 'id': self.fpr, - 'start': OutputTimestamp(start), - 'end': OutputTimestamp(end), - 'duration': (end - start).total_seconds(), - 'stats': stats.as_dict(), - 'limits': { - 'max_archive_size': self._archive_csize() / MAX_DATA_SIZE, - }, + "name": self.name, + "id": self.fpr, + "start": OutputTimestamp(start), + "end": OutputTimestamp(end), + "duration": (end - start).total_seconds(), + "stats": stats.as_dict(), + "limits": {"max_archive_size": self._archive_csize() / MAX_DATA_SIZE}, } if self.create: - info['command_line'] = sys.argv + info["command_line"] = sys.argv else: - info.update({ - 'command_line': self.metadata.cmdline, - 'hostname': self.metadata.hostname, - 'username': self.metadata.username, - 'comment': self.metadata.get('comment', ''), - 'chunker_params': self.metadata.get('chunker_params', ''), - }) + info.update( + { + "command_line": self.metadata.cmdline, + "hostname": self.metadata.hostname, + "username": self.metadata.username, + "comment": self.metadata.get("comment", ""), + "chunker_params": self.metadata.get("chunker_params", ""), + } + ) return info def __str__(self): - return '''\ + return """\ Repository: {location} Archive name: {0.name} Archive fingerprint: {0.fpr} @@ -540,19 +557,19 @@ def __str__(self): Time (end): {end} Duration: {0.duration} Utilization of max. archive size: {csize_max:.0%} -'''.format( +""".format( self, start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)), end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)), csize_max=self._archive_csize() / MAX_DATA_SIZE, - location=self.repository._location.canonical_path() -) + location=self.repository._location.canonical_path(), + ) def __repr__(self): - return 'Archive(%r)' % self.name + return "Archive(%r)" % self.name def item_filter(self, item, filter=None): - if not self.consider_part_files and 'part' in item: + if not self.consider_part_files and "part" in item: # this is a part(ial) file, we usually don't want to consider it. return False return filter(item) if filter else True @@ -560,8 +577,9 @@ def item_filter(self, item, filter=None): def iter_items(self, filter=None, preload=False): # note: when calling this with preload=True, later fetch_many() must be called with # is_preloaded=True or the RemoteRepository code will leak memory! - for item in self.pipeline.unpack_many(self.metadata.items, preload=preload, - filter=lambda item: self.item_filter(item, filter)): + for item in self.pipeline.unpack_many( + self.metadata.items, preload=preload, filter=lambda item: self.item_filter(item, filter) + ): yield item def add_item(self, item, show_progress=True, stats=None): @@ -591,34 +609,37 @@ def save(self, name=None, comment=None, timestamp=None, stats=None, additional_m self.start = start self.end = end metadata = { - 'version': 2, - 'name': name, - 'comment': comment or '', - 'items': self.items_buffer.chunks, - 'cmdline': sys.argv, - 'hostname': hostname, - 'username': getuser(), - 'time': start.strftime(ISO_FORMAT), - 'time_end': end.strftime(ISO_FORMAT), - 'chunker_params': self.chunker_params, + "version": 2, + "name": name, + "comment": comment or "", + "items": self.items_buffer.chunks, + "cmdline": sys.argv, + "hostname": hostname, + "username": getuser(), + "time": start.strftime(ISO_FORMAT), + "time_end": end.strftime(ISO_FORMAT), + "chunker_params": self.chunker_params, } if stats is not None: - metadata.update({ - 'size': stats.osize, - 'nfiles': stats.nfiles, - 'size_parts': stats.osize_parts, - 'nfiles_parts': stats.nfiles_parts}) + metadata.update( + { + "size": stats.osize, + "nfiles": stats.nfiles, + "size_parts": stats.osize_parts, + "nfiles_parts": stats.nfiles_parts, + } + ) metadata.update(additional_metadata or {}) metadata = ArchiveItem(metadata) - data = self.key.pack_and_authenticate_metadata(metadata.as_dict(), context=b'archive') + data = self.key.pack_and_authenticate_metadata(metadata.as_dict(), context=b"archive") self.id = self.key.id_hash(data) try: self.cache.add_chunk(self.id, data, self.stats) except IntegrityError as err: err_msg = str(err) # hack to avoid changing the RPC protocol by introducing new (more specific) exception class - if 'More than allowed put data' in err_msg: - raise Error('%s - archive too big (issue #1473)!' % err_msg) + if "More than allowed put data" in err_msg: + raise Error("%s - archive too big (issue #1473)!" % err_msg) else: raise while self.repository.async_response(wait=True) is not None: @@ -632,6 +653,7 @@ def calc_stats(self, cache, want_unique=True): if not want_unique: unique_size = 0 else: + def add(id): entry = cache.chunks[id] archive_index.add(id, 1, entry.size) @@ -640,10 +662,12 @@ def add(id): sync = CacheSynchronizer(archive_index) add(self.id) # we must escape any % char in the archive name, because we use it in a format string, see #6500 - arch_name_escd = self.name.replace('%', '%%') - pi = ProgressIndicatorPercent(total=len(self.metadata.items), - msg='Calculating statistics for archive %s ... %%3.0f%%%%' % arch_name_escd, - msgid='archive.calc_stats') + arch_name_escd = self.name.replace("%", "%%") + pi = ProgressIndicatorPercent( + total=len(self.metadata.items), + msg="Calculating statistics for archive %s ... %%3.0f%%%%" % arch_name_escd, + msgid="archive.calc_stats", + ) for id, chunk in zip(self.metadata.items, self.repository.get_many(self.metadata.items)): pi.show(increase=1) add(id) @@ -665,17 +689,17 @@ def add(id): def extract_helper(self, item, path, hlm, *, dry_run=False): hardlink_set = False # Hard link? - if 'hlid' in item: + if "hlid" in item: link_target = hlm.retrieve(id=item.hlid) if link_target is not None and has_link: if not dry_run: # another hardlink to same inode (same hlid) was extracted previously, just link to it - with backup_io('link'): + with backup_io("link"): os.link(link_target, path, follow_symlinks=False) hardlink_set = True yield hardlink_set if not hardlink_set: - if 'hlid' in item and has_link: + if "hlid" in item and has_link: # Update entry with extracted item path, so that following hardlinks don't extract twice. # We have hardlinking support, so we will hardlink not extract. hlm.remember(id=item.hlid, info=path) @@ -684,8 +708,18 @@ def extract_helper(self, item, path, hlm, *, dry_run=False): # In this case, we *want* to extract twice, because there is no other way. pass - def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sparse=False, - hlm=None, stripped_components=0, original_path=None, pi=None): + def extract_item( + self, + item, + restore_attrs=True, + dry_run=False, + stdout=False, + sparse=False, + hlm=None, + stripped_components=0, + original_path=None, + pi=None, + ): """ Extract archive item. @@ -699,14 +733,14 @@ def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sp :param original_path: 'path' key as stored in archive :param pi: ProgressIndicatorPercent (or similar) for file extraction progress (in bytes) """ - has_damaged_chunks = 'chunks_healthy' in item + has_damaged_chunks = "chunks_healthy" in item if dry_run or stdout: - with self.extract_helper(item, '', hlm, dry_run=dry_run or stdout) as hardlink_set: + with self.extract_helper(item, "", hlm, dry_run=dry_run or stdout) as hardlink_set: if not hardlink_set: # it does not really set hardlinks due to dry_run, but we need to behave same # as non-dry_run concerning fetching preloaded chunks from the pipeline or # it would get stuck. - if 'chunks' in item: + if "chunks" in item: item_chunks_size = 0 for data in self.pipeline.fetch_many([c.id for c in item.chunks], is_preloaded=True): if pi: @@ -716,19 +750,22 @@ def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sp item_chunks_size += len(data) if stdout: sys.stdout.buffer.flush() - if 'size' in item: + if "size" in item: item_size = item.size if item_size != item_chunks_size: - raise BackupError('Size inconsistency detected: size {}, chunks size {}'.format( - item_size, item_chunks_size)) + raise BackupError( + "Size inconsistency detected: size {}, chunks size {}".format( + item_size, item_chunks_size + ) + ) if has_damaged_chunks: - raise BackupError('File has damaged (all-zero) chunks. Try running borg check --repair.') + raise BackupError("File has damaged (all-zero) chunks. Try running borg check --repair.") return original_path = original_path or item.path dest = self.cwd - if item.path.startswith(('/', '../')): - raise Exception('Path should be relative and local') + if item.path.startswith(("/", "../")): + raise Exception("Path should be relative and local") path = os.path.join(dest, item.path) # Attempt to remove existing files, ignore errors on failure try: @@ -749,36 +786,37 @@ def make_parent(path): mode = item.mode if stat.S_ISREG(mode): - with backup_io('makedirs'): + with backup_io("makedirs"): make_parent(path) with self.extract_helper(item, path, hlm) as hardlink_set: if hardlink_set: return - with backup_io('open'): - fd = open(path, 'wb') + with backup_io("open"): + fd = open(path, "wb") with fd: ids = [c.id for c in item.chunks] for data in self.pipeline.fetch_many(ids, is_preloaded=True): if pi: pi.show(increase=len(data), info=[remove_surrogates(item.path)]) - with backup_io('write'): + with backup_io("write"): if sparse and zeros.startswith(data): # all-zero chunk: create a hole in a sparse file fd.seek(len(data), 1) else: fd.write(data) - with backup_io('truncate_and_attrs'): + with backup_io("truncate_and_attrs"): pos = item_chunks_size = fd.tell() fd.truncate(pos) fd.flush() self.restore_attrs(path, item, fd=fd.fileno()) - if 'size' in item: + if "size" in item: item_size = item.size if item_size != item_chunks_size: - raise BackupError('Size inconsistency detected: size {}, chunks size {}'.format( - item_size, item_chunks_size)) + raise BackupError( + "Size inconsistency detected: size {}, chunks size {}".format(item_size, item_chunks_size) + ) if has_damaged_chunks: - raise BackupError('File has damaged (all-zero) chunks. Try running borg check --repair.') + raise BackupError("File has damaged (all-zero) chunks. Try running borg check --repair.") return with backup_io: # No repository access beyond this point. @@ -815,7 +853,7 @@ def make_parent(path): os.mknod(path, item.mode, item.rdev) self.restore_attrs(path, item) else: - raise Exception('Unknown archive item type %r' % item.mode) + raise Exception("Unknown archive item type %r" % item.mode) def restore_attrs(self, path, item, symlink=False, fd=None): """ @@ -823,7 +861,7 @@ def restore_attrs(self, path, item, symlink=False, fd=None): Does not access the repository. """ - backup_io.op = 'attrs' + backup_io.op = "attrs" uid, gid = get_item_uid_gid(item, numeric=self.numeric_ids) # This code is a bit of a mess due to os specific differences if not is_win32: @@ -849,12 +887,12 @@ def restore_attrs(self, path, item, symlink=False, fd=None): if not symlink: os.chmod(path, item.mode) mtime = item.mtime - if 'atime' in item: + if "atime" in item: atime = item.atime else: # old archives only had mtime in item metadata atime = mtime - if 'birthtime' in item: + if "birthtime" in item: birthtime = item.birthtime try: # This should work on FreeBSD, NetBSD, and Darwin and be harmless on other platforms. @@ -879,11 +917,11 @@ def restore_attrs(self, path, item, symlink=False, fd=None): if not self.noxattrs: # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include # the Linux capabilities in the "security.capability" attribute. - warning = xattr.set_all(fd or path, item.get('xattrs', {}), follow_symlinks=False) + warning = xattr.set_all(fd or path, item.get("xattrs", {}), follow_symlinks=False) if warning: set_ec(EXIT_WARNING) # bsdflags include the immutable flag and need to be set last: - if not self.noflags and 'bsdflags' in item: + if not self.noflags and "bsdflags" in item: try: set_flags(path, item.bsdflags, fd=fd) except OSError: @@ -904,7 +942,7 @@ def rename(self, name): raise self.AlreadyExists(name) oldname = self.name self.name = name - self.set_meta('name', name) + self.set_meta("name", name) del self.manifest.archives[oldname] def delete(self, stats, progress=False, forced=False): @@ -937,7 +975,9 @@ def chunk_decref(id, stats, part=False): try: unpacker = msgpack.Unpacker(use_list=False) items_ids = self.metadata.items - pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", msgid='archive.delete') + pi = ProgressIndicatorPercent( + total=len(items_ids), msg="Decrementing references %3.0f%%", msgid="archive.delete" + ) for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): if progress: pi.show(i) @@ -947,8 +987,8 @@ def chunk_decref(id, stats, part=False): try: for item in unpacker: item = Item(internal_dict=item) - if 'chunks' in item: - part = not self.consider_part_files and 'part' in item + if "chunks" in item: + part = not self.consider_part_files and "part" in item for chunk_id, size in item.chunks: chunk_decref(chunk_id, stats, part=part) except (TypeError, ValueError): @@ -974,8 +1014,8 @@ def chunk_decref(id, stats, part=False): # so there is nothing pending when we return and our caller wants to commit. pass if error: - logger.warning('forced deletion succeeded, but the deleted archive was corrupted.') - logger.warning('borg check --repair is required to free all space.') + logger.warning("forced deletion succeeded, but the deleted archive was corrupted.") + logger.warning("borg check --repair is required to free all space.") @staticmethod def compare_archives_iter(archive1, archive2, matcher=None, can_compare_chunk_ids=False): @@ -987,17 +1027,20 @@ def compare_archives_iter(archive1, archive2, matcher=None, can_compare_chunk_id """ def compare_items(item1, item2): - return ItemDiff(item1, item2, - archive1.pipeline.fetch_many([c.id for c in item1.get('chunks', [])]), - archive2.pipeline.fetch_many([c.id for c in item2.get('chunks', [])]), - can_compare_chunk_ids=can_compare_chunk_ids) + return ItemDiff( + item1, + item2, + archive1.pipeline.fetch_many([c.id for c in item1.get("chunks", [])]), + archive2.pipeline.fetch_many([c.id for c in item2.get("chunks", [])]), + can_compare_chunk_ids=can_compare_chunk_ids, + ) orphans_archive1 = OrderedDict() orphans_archive2 = OrderedDict() for item1, item2 in zip_longest( - archive1.iter_items(lambda item: matcher.match(item.path)), - archive2.iter_items(lambda item: matcher.match(item.path)), + archive1.iter_items(lambda item: matcher.match(item.path)), + archive2.iter_items(lambda item: matcher.match(item.path)), ): if item1 and item2 and item1.path == item2.path: yield (item1.path, compare_items(item1, item2)) @@ -1036,42 +1079,37 @@ def __init__(self, *, noatime, noctime, nobirthtime, numeric_ids, noflags, noacl self.nobirthtime = nobirthtime def stat_simple_attrs(self, st): - attrs = dict( - mode=st.st_mode, - uid=st.st_uid, - gid=st.st_gid, - mtime=safe_ns(st.st_mtime_ns), - ) + attrs = dict(mode=st.st_mode, uid=st.st_uid, gid=st.st_gid, mtime=safe_ns(st.st_mtime_ns)) # borg can work with archives only having mtime (older attic archives do not have # atime/ctime). it can be useful to omit atime/ctime, if they change without the # file content changing - e.g. to get better metadata deduplication. if not self.noatime: - attrs['atime'] = safe_ns(st.st_atime_ns) + attrs["atime"] = safe_ns(st.st_atime_ns) if not self.noctime: - attrs['ctime'] = safe_ns(st.st_ctime_ns) - if not self.nobirthtime and hasattr(st, 'st_birthtime'): + attrs["ctime"] = safe_ns(st.st_ctime_ns) + if not self.nobirthtime and hasattr(st, "st_birthtime"): # sadly, there's no stat_result.st_birthtime_ns - attrs['birthtime'] = safe_ns(int(st.st_birthtime * 10**9)) + attrs["birthtime"] = safe_ns(int(st.st_birthtime * 10**9)) if not self.numeric_ids: user = uid2user(st.st_uid) if user is not None: - attrs['user'] = user + attrs["user"] = user group = gid2group(st.st_gid) if group is not None: - attrs['group'] = group + attrs["group"] = group return attrs def stat_ext_attrs(self, st, path, fd=None): attrs = {} - with backup_io('extended stat'): + with backup_io("extended stat"): flags = 0 if self.noflags else get_flags(path, st, fd=fd) xattrs = {} if self.noxattrs else xattr.get_all(fd or path, follow_symlinks=False) if not self.noacls: acl_get(path, attrs, st, self.numeric_ids, fd=fd) if xattrs: - attrs['xattrs'] = StableDict(xattrs) + attrs["xattrs"] = StableDict(xattrs) if flags: - attrs['bsdflags'] = flags + attrs["bsdflags"] = flags return attrs def stat_attrs(self, st, path, fd=None): @@ -1088,12 +1126,12 @@ def stat_attrs(self, st, path, fd=None): def cached_hash(chunk, id_hash): - allocation = chunk.meta['allocation'] + allocation = chunk.meta["allocation"] if allocation == CH_DATA: data = chunk.data chunk_id = id_hash(data) elif allocation in (CH_HOLE, CH_ALLOC): - size = chunk.meta['size'] + size = chunk.meta["size"] assert size <= len(zeros) data = memoryview(zeros)[:size] try: @@ -1102,16 +1140,14 @@ def cached_hash(chunk, id_hash): chunk_id = id_hash(data) zero_chunk_ids[(id_hash, size)] = chunk_id else: - raise ValueError('unexpected allocation type') + raise ValueError("unexpected allocation type") return chunk_id, data class ChunksProcessor: # Processes an iterator of chunks for an Item - def __init__(self, *, key, cache, - add_item, write_checkpoint, - checkpoint_interval, rechunkify): + def __init__(self, *, key, cache, add_item, write_checkpoint, checkpoint_interval, rechunkify): self.key = key self.cache = cache self.add_item = add_item @@ -1128,7 +1164,7 @@ def write_part_file(self, item, from_chunk, number): # for borg recreate, we already have a size member in the source item (giving the total file size), # but we consider only a part of the file here, thus we must recompute the size from the chunks: item.get_size(memorize=True, from_chunks=True) - item.path += '.borg_part_%d' % number + item.path += ".borg_part_%d" % number item.part = number number += 1 self.add_item(item, show_progress=False) @@ -1137,19 +1173,24 @@ def write_part_file(self, item, from_chunk, number): def maybe_checkpoint(self, item, from_chunk, part_number, forced=False): sig_int_triggered = sig_int and sig_int.action_triggered() - if forced or sig_int_triggered or \ - self.checkpoint_interval and time.monotonic() - self.last_checkpoint > self.checkpoint_interval: + if ( + forced + or sig_int_triggered + or self.checkpoint_interval + and time.monotonic() - self.last_checkpoint > self.checkpoint_interval + ): if sig_int_triggered: - logger.info('checkpoint requested: starting checkpoint creation...') + logger.info("checkpoint requested: starting checkpoint creation...") from_chunk, part_number = self.write_part_file(item, from_chunk, part_number) self.last_checkpoint = time.monotonic() if sig_int_triggered: sig_int.action_completed() - logger.info('checkpoint requested: finished checkpoint creation!') + logger.info("checkpoint requested: finished checkpoint creation!") return from_chunk, part_number def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None): if not chunk_processor: + def chunk_processor(chunk): chunk_id, data = cached_hash(chunk, self.key.id_hash) chunk_entry = cache.add_chunk(chunk_id, data, stats, wait=False) @@ -1159,7 +1200,7 @@ def chunk_processor(chunk): item.chunks = [] # if we rechunkify, we'll get a fundamentally different chunks list, thus we need # to get rid of .chunks_healthy, as it might not correspond to .chunks any more. - if self.rechunkify and 'chunks_healthy' in item: + if self.rechunkify and "chunks_healthy" in item: del item.chunks_healthy from_chunk = 0 part_number = 1 @@ -1188,10 +1229,21 @@ class FilesystemObjectProcessors: # write_checkpoint should then be in the item buffer, # and process_file becomes a callback passed to __init__. - def __init__(self, *, metadata_collector, cache, key, - add_item, process_file_chunks, - chunker_params, show_progress, sparse, - log_json, iec, file_status_printer=None): + def __init__( + self, + *, + metadata_collector, + cache, + key, + add_item, + process_file_chunks, + chunker_params, + show_progress, + sparse, + log_json, + iec, + file_status_printer=None, + ): self.metadata_collector = metadata_collector self.cache = cache self.key = key @@ -1212,7 +1264,7 @@ def create_helper(self, path, st, status=None, hardlinkable=True): hardlinked = hardlinkable and st.st_nlink > 1 update_map = False if hardlinked: - status = 'h' # hardlink + status = "h" # hardlink nothing = object() chunks = self.hlm.retrieve(id=(st.st_ino, st.st_dev), default=nothing) if chunks is nothing: @@ -1225,29 +1277,28 @@ def create_helper(self, path, st, status=None, hardlinkable=True): if update_map: # remember the hlid of this fs object and if the item has chunks, # also remember them, so we do not have to re-chunk a hardlink. - chunks = item.chunks if 'chunks' in item else None + chunks = item.chunks if "chunks" in item else None self.hlm.remember(id=(st.st_ino, st.st_dev), info=chunks) def process_dir_with_fd(self, *, path, fd, st): - with self.create_helper(path, st, 'd', hardlinkable=False) as (item, status, hardlinked): + with self.create_helper(path, st, "d", hardlinkable=False) as (item, status, hardlinked): item.update(self.metadata_collector.stat_attrs(st, path, fd=fd)) return status def process_dir(self, *, path, parent_fd, name, st): - with self.create_helper(path, st, 'd', hardlinkable=False) as (item, status, hardlinked): - with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir, - noatime=True, op='dir_open') as fd: + with self.create_helper(path, st, "d", hardlinkable=False) as (item, status, hardlinked): + with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir, noatime=True, op="dir_open") as fd: # fd is None for directories on windows, in that case a race condition check is not possible. if fd is not None: - with backup_io('fstat'): + with backup_io("fstat"): st = stat_update_check(st, os.fstat(fd)) item.update(self.metadata_collector.stat_attrs(st, path, fd=fd)) return status def process_fifo(self, *, path, parent_fd, name, st): - with self.create_helper(path, st, 'f') as (item, status, hardlinked): # fifo + with self.create_helper(path, st, "f") as (item, status, hardlinked): # fifo with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_normal, noatime=True) as fd: - with backup_io('fstat'): + with backup_io("fstat"): st = stat_update_check(st, os.fstat(fd)) item.update(self.metadata_collector.stat_attrs(st, path, fd=fd)) return status @@ -1255,23 +1306,23 @@ def process_fifo(self, *, path, parent_fd, name, st): def process_dev(self, *, path, parent_fd, name, st, dev_type): with self.create_helper(path, st, dev_type) as (item, status, hardlinked): # char/block device # looks like we can not work fd-based here without causing issues when trying to open/close the device - with backup_io('stat'): + with backup_io("stat"): st = stat_update_check(st, os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)) item.rdev = st.st_rdev item.update(self.metadata_collector.stat_attrs(st, path)) return status def process_symlink(self, *, path, parent_fd, name, st): - with self.create_helper(path, st, 's', hardlinkable=True) as (item, status, hardlinked): + with self.create_helper(path, st, "s", hardlinkable=True) as (item, status, hardlinked): fname = name if name is not None and parent_fd is not None else path - with backup_io('readlink'): + with backup_io("readlink"): source = os.readlink(fname, dir_fd=parent_fd) item.source = source item.update(self.metadata_collector.stat_attrs(st, path)) # can't use FD here? return status def process_pipe(self, *, path, cache, fd, mode, user, group): - status = 'i' # stdin (or other pipe) + status = "i" # stdin (or other pipe) self.print_file_status(status, path) status = None # we already printed the status uid = user2uid(user) @@ -1284,9 +1335,13 @@ def process_pipe(self, *, path, cache, fd, mode, user, group): item = Item( path=path, mode=mode & 0o107777 | 0o100000, # forcing regular file mode - uid=uid, user=user, - gid=gid, group=group, - mtime=t, atime=t, ctime=t, + uid=uid, + user=user, + gid=gid, + group=group, + mtime=t, + atime=t, + ctime=t, ) self.process_file_chunks(item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd))) item.get_size(memorize=True) @@ -1297,7 +1352,7 @@ def process_pipe(self, *, path, cache, fd, mode, user, group): def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal): with self.create_helper(path, st, None) as (item, status, hardlinked): # no status yet with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags, noatime=True) as fd: - with backup_io('fstat'): + with backup_io("fstat"): st = stat_update_check(st, os.fstat(fd)) item.update(self.metadata_collector.stat_simple_attrs(st)) is_special_file = is_special(st.st_mode) @@ -1306,7 +1361,7 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal): # so it can be extracted / accessed in FUSE mount like a regular file. # this needs to be done early, so that part files also get the patched mode. item.mode = stat.S_IFREG | stat.S_IMODE(item.mode) - if 'chunks' in item: # create_helper might have put chunks from a previous hardlink there + if "chunks" in item: # create_helper might have put chunks from a previous hardlink there [cache.chunk_incref(id_, self.stats) for id_, _ in item.chunks] else: # normal case, no "2nd+" hardlink if not is_special_file: @@ -1324,32 +1379,40 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal): # Make sure all ids are available for id_ in ids: if not cache.seen_chunk(id_): - status = 'M' # cache said it is unmodified, but we lost a chunk: process file like modified + status = ( + "M" # cache said it is unmodified, but we lost a chunk: process file like modified + ) break else: chunks = [cache.chunk_incref(id_, self.stats) for id_ in ids] - status = 'U' # regular file, unchanged + status = "U" # regular file, unchanged else: - status = 'M' if known else 'A' # regular file, modified or added + status = "M" if known else "A" # regular file, modified or added self.print_file_status(status, path) status = None # we already printed the status # Only chunkify the file if needed if chunks is not None: item.chunks = chunks else: - with backup_io('read'): - self.process_file_chunks(item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(None, fd))) + with backup_io("read"): + self.process_file_chunks( + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(self.chunker.chunkify(None, fd)), + ) if is_win32: changed_while_backup = False # TODO else: - with backup_io('fstat2'): + with backup_io("fstat2"): st2 = os.fstat(fd) # special files: # - fifos change naturally, because they are fed from the other side. no problem. # - blk/chr devices don't change ctime anyway. changed_while_backup = not is_special_file and st.st_ctime_ns != st2.st_ctime_ns if changed_while_backup: - status = 'C' # regular file changed while we backed it up, might be inconsistent/corrupt! + status = "C" # regular file changed while we backed it up, might be inconsistent/corrupt! if not is_special_file and not changed_while_backup: # we must not memorize special files, because the contents of e.g. a # block or char device will change without its mtime/size/inode changing. @@ -1363,10 +1426,19 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal): class TarfileObjectProcessors: - def __init__(self, *, cache, key, - add_item, process_file_chunks, - chunker_params, show_progress, - log_json, iec, file_status_printer=None): + def __init__( + self, + *, + cache, + key, + add_item, + process_file_chunks, + chunker_params, + show_progress, + log_json, + iec, + file_status_printer=None, + ): self.cache = cache self.key = key self.add_item = add_item @@ -1381,17 +1453,23 @@ def __init__(self, *, cache, key, @contextmanager def create_helper(self, tarinfo, status=None, type=None): ph = tarinfo.pax_headers - if ph and 'BORG.item.version' in ph: - assert ph['BORG.item.version'] == '1' - meta_bin = base64.b64decode(ph['BORG.item.meta']) + if ph and "BORG.item.version" in ph: + assert ph["BORG.item.version"] == "1" + meta_bin = base64.b64decode(ph["BORG.item.meta"]) meta_dict = msgpack.unpackb(meta_bin, object_hook=StableDict) item = Item(internal_dict=meta_dict) else: + def s_to_ns(s): return safe_ns(int(float(s) * 1e9)) - item = Item(path=make_path_safe(tarinfo.name), mode=tarinfo.mode | type, - uid=tarinfo.uid, gid=tarinfo.gid, mtime=s_to_ns(tarinfo.mtime)) + item = Item( + path=make_path_safe(tarinfo.name), + mode=tarinfo.mode | type, + uid=tarinfo.uid, + gid=tarinfo.gid, + mtime=s_to_ns(tarinfo.mtime), + ) if tarinfo.uname: item.user = tarinfo.uname if tarinfo.gname: @@ -1399,7 +1477,7 @@ def s_to_ns(s): if ph: # note: for mtime this is a bit redundant as it is already done by tarfile module, # but we just do it in our way to be consistent for sure. - for name in 'atime', 'ctime', 'mtime': + for name in "atime", "ctime", "mtime": if name in ph: ns = s_to_ns(ph[name]) setattr(item, name, ns) @@ -1440,8 +1518,9 @@ def process_file(self, *, tarinfo, status, type, tar): self.print_file_status(status, tarinfo.name) status = None # we already printed the status fd = tar.extractfile(tarinfo) - self.process_file_chunks(item, self.cache, self.stats, self.show_progress, - backup_io_iter(self.chunker.chunkify(fd))) + self.process_file_chunks( + item, self.cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd)) + ) item.get_size(memorize=True, from_chunks=True) self.stats.nfiles += 1 # we need to remember ALL files, see HardLinkManager.__doc__ @@ -1454,9 +1533,9 @@ def valid_msgpacked_dict(d, keys_serialized): d_len = len(d) if d_len == 0: return False - if d[0] & 0xf0 == 0x80: # object is a fixmap (up to 15 elements) + if d[0] & 0xF0 == 0x80: # object is a fixmap (up to 15 elements) offs = 1 - elif d[0] == 0xde: # object is a map16 (up to 2^16-1 elements) + elif d[0] == 0xDE: # object is a map16 (up to 2^16-1 elements) offs = 3 else: # object is not a map (dict) @@ -1465,9 +1544,9 @@ def valid_msgpacked_dict(d, keys_serialized): if d_len <= offs: return False # is the first dict key a bytestring? - if d[offs] & 0xe0 == 0xa0: # key is a small bytestring (up to 31 chars) + if d[offs] & 0xE0 == 0xA0: # key is a small bytestring (up to 31 chars) pass - elif d[offs] in (0xd9, 0xda, 0xdb): # key is a str8, str16 or str32 + elif d[offs] in (0xD9, 0xDA, 0xDB): # key is a str8, str16 or str32 pass else: # key is not a bytestring @@ -1478,8 +1557,8 @@ def valid_msgpacked_dict(d, keys_serialized): class RobustUnpacker: - """A restartable/robust version of the streaming msgpack unpacker - """ + """A restartable/robust version of the streaming msgpack unpacker""" + def __init__(self, validator, item_keys): super().__init__() self.item_keys = [msgpack.packb(name) for name in item_keys] @@ -1503,7 +1582,7 @@ def __iter__(self): def __next__(self): if self._resync: - data = b''.join(self._buffered_data) + data = b"".join(self._buffered_data) while self._resync: if not data: raise StopIteration @@ -1528,13 +1607,13 @@ def __next__(self): class ArchiveChecker: - def __init__(self): self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, first=0, last=0, sort_by='', glob=None, - verify_data=False, save_space=False): + def check( + self, repository, repair=False, first=0, last=0, sort_by="", glob=None, verify_data=False, save_space=False + ): """Perform a set of checks on 'repository' :param repair: enable repair mode, write updated or corrected data into repository @@ -1543,13 +1622,13 @@ def check(self, repository, repair=False, first=0, last=0, sort_by='', glob=None :param verify_data: integrity verification of data referenced by archives :param save_space: Repository.commit(save_space) """ - logger.info('Starting archive consistency check...') + logger.info("Starting archive consistency check...") self.check_all = not any((first, last, glob)) self.repair = repair self.repository = repository self.init_chunks() if not self.chunks: - logger.error('Repository contains no apparent data at all, cannot continue check/repair.') + logger.error("Repository contains no apparent data at all, cannot continue check/repair.") return False self.key = self.identify_key(repository) if verify_data: @@ -1562,7 +1641,7 @@ def check(self, repository, repair=False, first=0, last=0, sort_by='', glob=None try: self.manifest, _ = Manifest.load(repository, (Manifest.Operation.CHECK,), key=self.key) except IntegrityErrorBase as exc: - logger.error('Repository manifest is corrupted: %s', exc) + logger.error("Repository manifest is corrupted: %s", exc) self.error_found = True del self.chunks[Manifest.MANIFEST_ID] self.manifest = self.rebuild_manifest() @@ -1570,14 +1649,13 @@ def check(self, repository, repair=False, first=0, last=0, sort_by='', glob=None self.orphan_chunks_check() self.finish(save_space=save_space) if self.error_found: - logger.error('Archive consistency check complete, problems found.') + logger.error("Archive consistency check complete, problems found.") else: - logger.info('Archive consistency check complete, no problems found.') + logger.info("Archive consistency check complete, no problems found.") return self.repair or not self.error_found def init_chunks(self): - """Fetch a list of all object keys from repository - """ + """Fetch a list of all object keys from repository""" # Explicitly set the initial usable hash table capacity to avoid performance issues # due to hash table "resonance". # Since reconstruction of archive items can add some new chunks, add 10 % headroom. @@ -1602,13 +1680,14 @@ def identify_key(self, repository): return key_factory(repository, cdata) def verify_data(self): - logger.info('Starting cryptographic data integrity verification...') + logger.info("Starting cryptographic data integrity verification...") chunks_count_index = len(self.chunks) chunks_count_segments = 0 errors = 0 defect_chunks = [] - pi = ProgressIndicatorPercent(total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01, - msgid='check.verify_data') + pi = ProgressIndicatorPercent( + total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01, msgid="check.verify_data" + ) marker = None while True: chunk_ids = self.repository.scan(limit=100, marker=marker) @@ -1626,7 +1705,7 @@ def verify_data(self): except (Repository.ObjectNotFound, IntegrityErrorBase) as err: self.error_found = True errors += 1 - logger.error('chunk %s: %s', bin_to_hex(chunk_id), err) + logger.error("chunk %s: %s", bin_to_hex(chunk_id), err) if isinstance(err, IntegrityErrorBase): defect_chunks.append(chunk_id) # as the exception killed our generator, make a new one for remaining chunks: @@ -1639,13 +1718,14 @@ def verify_data(self): except IntegrityErrorBase as integrity_error: self.error_found = True errors += 1 - logger.error('chunk %s, integrity error: %s', bin_to_hex(chunk_id), integrity_error) + logger.error("chunk %s, integrity error: %s", bin_to_hex(chunk_id), integrity_error) defect_chunks.append(chunk_id) pi.finish() if chunks_count_index != chunks_count_segments: - logger.error('Repo/Chunks index object count vs. segment files object count mismatch.') - logger.error('Repo/Chunks index: %d objects != segment files: %d objects', - chunks_count_index, chunks_count_segments) + logger.error("Repo/Chunks index object count vs. segment files object count mismatch.") + logger.error( + "Repo/Chunks index: %d objects != segment files: %d objects", chunks_count_index, chunks_count_segments + ) if defect_chunks: if self.repair: # if we kill the defect chunk here, subsequent actions within this "borg check" @@ -1653,8 +1733,10 @@ def verify_data(self): # chunks and flag the files as "repaired". # if another backup is done later and the missing chunks get backupped again, # a "borg check" afterwards can heal all files where this chunk was missing. - logger.warning('Found defect chunks. They will be deleted now, so affected files can ' - 'get repaired now and maybe healed later.') + logger.warning( + "Found defect chunks. They will be deleted now, so affected files can " + "get repaired now and maybe healed later." + ) for defect_chunk in defect_chunks: # remote repo (ssh): retry might help for strange network / NIC / RAM errors # as the chunk will be retransmitted from remote server. @@ -1668,29 +1750,35 @@ def verify_data(self): # failed twice -> get rid of this chunk del self.chunks[defect_chunk] self.repository.delete(defect_chunk) - logger.debug('chunk %s deleted.', bin_to_hex(defect_chunk)) + logger.debug("chunk %s deleted.", bin_to_hex(defect_chunk)) else: - logger.warning('chunk %s not deleted, did not consistently fail.', bin_to_hex(defect_chunk)) + logger.warning("chunk %s not deleted, did not consistently fail.", bin_to_hex(defect_chunk)) else: - logger.warning('Found defect chunks. With --repair, they would get deleted, so affected ' - 'files could get repaired then and maybe healed later.') + logger.warning( + "Found defect chunks. With --repair, they would get deleted, so affected " + "files could get repaired then and maybe healed later." + ) for defect_chunk in defect_chunks: - logger.debug('chunk %s is defect.', bin_to_hex(defect_chunk)) + logger.debug("chunk %s is defect.", bin_to_hex(defect_chunk)) log = logger.error if errors else logger.info - log('Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.', - chunks_count_segments, errors) + log( + "Finished cryptographic data integrity verification, verified %d chunks with %d integrity errors.", + chunks_count_segments, + errors, + ) def rebuild_manifest(self): """Rebuild the manifest object if it is missing Iterates through all objects in the repository looking for archive metadata blocks. """ + def valid_archive(obj): if not isinstance(obj, dict): return False return REQUIRED_ARCHIVE_KEYS.issubset(obj) - logger.info('Rebuilding missing manifest, this might take some time...') + logger.info("Rebuilding missing manifest, this might take some time...") # as we have lost the manifest, we do not know any more what valid item keys we had. # collecting any key we encounter in a damaged repo seems unwise, thus we just use # the hardcoded list from the source code. thus, it is not recommended to rebuild a @@ -1698,20 +1786,21 @@ def valid_archive(obj): # within this repository (assuming that newer borg versions support more item keys). manifest = Manifest(self.key, self.repository) archive_keys_serialized = [msgpack.packb(name) for name in ARCHIVE_KEYS] - pi = ProgressIndicatorPercent(total=len(self.chunks), msg="Rebuilding manifest %6.2f%%", step=0.01, - msgid='check.rebuild_manifest') + pi = ProgressIndicatorPercent( + total=len(self.chunks), msg="Rebuilding manifest %6.2f%%", step=0.01, msgid="check.rebuild_manifest" + ) for chunk_id, _ in self.chunks.iteritems(): pi.show() cdata = self.repository.get(chunk_id) try: data = self.key.decrypt(chunk_id, cdata) except IntegrityErrorBase as exc: - logger.error('Skipping corrupted chunk: %s', exc) + logger.error("Skipping corrupted chunk: %s", exc) self.error_found = True continue if not valid_msgpacked_dict(data, archive_keys_serialized): continue - if b'cmdline' not in data or b'\xa7version\x02' not in data: + if b"cmdline" not in data or b"\xa7version\x02" not in data: continue try: archive = msgpack.unpackb(data) @@ -1721,22 +1810,22 @@ def valid_archive(obj): if valid_archive(archive): archive = ArchiveItem(internal_dict=archive) name = archive.name - logger.info('Found archive %s', name) + logger.info("Found archive %s", name) if name in manifest.archives: i = 1 while True: - new_name = '%s.%d' % (name, i) + new_name = "%s.%d" % (name, i) if new_name not in manifest.archives: break i += 1 - logger.warning('Duplicate archive name %s, storing as %s', name, new_name) + logger.warning("Duplicate archive name %s, storing as %s", name, new_name) name = new_name manifest.archives[name] = (chunk_id, archive.time) pi.finish() - logger.info('Manifest rebuild complete.') + logger.info("Manifest rebuild complete.") return manifest - def rebuild_refcounts(self, first=0, last=0, sort_by='', glob=None): + def rebuild_refcounts(self, first=0, last=0, sort_by="", glob=None): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected @@ -1769,6 +1858,7 @@ def verify_file_chunks(archive_name, item): Missing file chunks will be replaced with new chunks of the same length containing all zeros. If a previously missing file chunk re-appears, the replacement chunk is replaced by the correct one. """ + def replacement_chunk(size): chunk = Chunk(None, allocation=CH_ALLOC, size=size) chunk_id, data = cached_hash(chunk, self.key.id_hash) @@ -1778,12 +1868,12 @@ def replacement_chunk(size): offset = 0 chunk_list = [] chunks_replaced = False - has_chunks_healthy = 'chunks_healthy' in item + has_chunks_healthy = "chunks_healthy" in item chunks_current = item.chunks chunks_healthy = item.chunks_healthy if has_chunks_healthy else chunks_current if has_chunks_healthy and len(chunks_current) != len(chunks_healthy): # should never happen, but there was issue #3218. - logger.warning(f'{archive_name}: {item.path}: Invalid chunks_healthy metadata removed!') + logger.warning(f"{archive_name}: {item.path}: Invalid chunks_healthy metadata removed!") del item.chunks_healthy has_chunks_healthy = False chunks_healthy = chunks_current @@ -1792,23 +1882,32 @@ def replacement_chunk(size): if chunk_id not in self.chunks: # a chunk of the healthy list is missing if chunk_current == chunk_healthy: - logger.error('{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). ' - 'Replacing with all-zero chunk.'.format( - archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id))) + logger.error( + "{}: {}: New missing file chunk detected (Byte {}-{}, Chunk {}). " + "Replacing with all-zero chunk.".format( + archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) + ) + ) self.error_found = chunks_replaced = True chunk_id, size, cdata = replacement_chunk(size) add_reference(chunk_id, size, cdata) else: - logger.info('{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). ' - 'It has an all-zero replacement chunk already.'.format( - archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id))) + logger.info( + "{}: {}: Previously missing file chunk is still missing (Byte {}-{}, Chunk {}). " + "It has an all-zero replacement chunk already.".format( + archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) + ) + ) chunk_id, size = chunk_current if chunk_id in self.chunks: add_reference(chunk_id, size) else: - logger.warning('{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). ' - 'Generating new replacement chunk.'.format( - archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id))) + logger.warning( + "{}: {}: Missing all-zero replacement chunk detected (Byte {}-{}, Chunk {}). " + "Generating new replacement chunk.".format( + archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) + ) + ) self.error_found = chunks_replaced = True chunk_id, size, cdata = replacement_chunk(size) add_reference(chunk_id, size, cdata) @@ -1817,8 +1916,11 @@ def replacement_chunk(size): # normal case, all fine. add_reference(chunk_id, size) else: - logger.info('{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).'.format( - archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id))) + logger.info( + "{}: {}: Healed previously missing file chunk! (Byte {}-{}, Chunk {}).".format( + archive_name, item.path, offset, offset + size, bin_to_hex(chunk_id) + ) + ) add_reference(chunk_id, size) mark_as_possibly_superseded(chunk_current[0]) # maybe orphaned the all-zero replacement chunk chunk_list.append([chunk_id, size]) # list-typed element as chunks_healthy is list-of-lists @@ -1827,16 +1929,19 @@ def replacement_chunk(size): # if this is first repair, remember the correct chunk IDs, so we can maybe heal the file later item.chunks_healthy = item.chunks if has_chunks_healthy and chunk_list == chunks_healthy: - logger.info(f'{archive_name}: {item.path}: Completely healed previously damaged file!') + logger.info(f"{archive_name}: {item.path}: Completely healed previously damaged file!") del item.chunks_healthy item.chunks = chunk_list - if 'size' in item: + if "size" in item: item_size = item.size item_chunks_size = item.get_size(from_chunks=True) if item_size != item_chunks_size: # just warn, but keep the inconsistency, so that borg extract can warn about it. - logger.warning('{}: {}: size inconsistency detected: size {}, chunks size {}'.format( - archive_name, item.path, item_size, item_chunks_size)) + logger.warning( + "{}: {}: size inconsistency detected: size {}, chunks size {}".format( + archive_name, item.path, item_size, item_chunks_size + ) + ) def robust_iterator(archive): """Iterates through all archive items @@ -1845,8 +1950,9 @@ def robust_iterator(archive): """ item_keys = self.manifest.item_keys required_item_keys = REQUIRED_ITEM_KEYS - unpacker = RobustUnpacker(lambda item: isinstance(item, StableDict) and 'path' in item, - self.manifest.item_keys) + unpacker = RobustUnpacker( + lambda item: isinstance(item, StableDict) and "path" in item, self.manifest.item_keys + ) _state = 0 def missing_chunk_detector(chunk_id): @@ -1857,29 +1963,29 @@ def missing_chunk_detector(chunk_id): def report(msg, chunk_id, chunk_no): cid = bin_to_hex(chunk_id) - msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see "debug dump-archive-items" + msg += " [chunk: %06d_%s]" % (chunk_no, cid) # see "debug dump-archive-items" self.error_found = True logger.error(msg) def list_keys_safe(keys): - return ', '.join(k.decode(errors='replace') if isinstance(k, bytes) else str(k) for k in keys) + return ", ".join(k.decode(errors="replace") if isinstance(k, bytes) else str(k) for k in keys) def valid_item(obj): if not isinstance(obj, StableDict): - return False, 'not a dictionary' + return False, "not a dictionary" keys = set(obj) if not required_item_keys.issubset(keys): - return False, 'missing required keys: ' + list_keys_safe(required_item_keys - keys) + return False, "missing required keys: " + list_keys_safe(required_item_keys - keys) if not keys.issubset(item_keys): - return False, 'invalid keys: ' + list_keys_safe(keys - item_keys) - return True, '' + return False, "invalid keys: " + list_keys_safe(keys - item_keys) + return True, "" i = 0 for state, items in groupby(archive.items, missing_chunk_detector): items = list(items) if state % 2: for chunk_id in items: - report('item metadata chunk missing', chunk_id, i) + report("item metadata chunk missing", chunk_id, i) i += 1 continue if state > 0: @@ -1893,42 +1999,47 @@ def valid_item(obj): if valid: yield Item(internal_dict=item) else: - report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i) + report( + "Did not get expected metadata dict when unpacking item metadata (%s)" % reason, + chunk_id, + i, + ) except IntegrityError as integrity_error: # key.decrypt() detected integrity issues. # maybe the repo gave us a valid cdata, but not for the chunk_id we wanted. # or the authentication of cdata failed, meaning the encrypted data was corrupted. report(str(integrity_error), chunk_id, i) except msgpack.UnpackException: - report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i) + report("Unpacker crashed while unpacking item metadata, trying to resync...", chunk_id, i) unpacker.resync() except Exception: - report('Exception while decrypting or unpacking item metadata', chunk_id, i) + report("Exception while decrypting or unpacking item metadata", chunk_id, i) raise i += 1 - sort_by = sort_by.split(',') + sort_by = sort_by.split(",") if any((first, last, glob)): archive_infos = self.manifest.archives.list(sort_by=sort_by, glob=glob, first=first, last=last) if glob and not archive_infos: - logger.warning('--glob-archives %s does not match any archives', glob) + logger.warning("--glob-archives %s does not match any archives", glob) if first and len(archive_infos) < first: - logger.warning('--first %d archives: only found %d archives', first, len(archive_infos)) + logger.warning("--first %d archives: only found %d archives", first, len(archive_infos)) if last and len(archive_infos) < last: - logger.warning('--last %d archives: only found %d archives', last, len(archive_infos)) + logger.warning("--last %d archives: only found %d archives", last, len(archive_infos)) else: archive_infos = self.manifest.archives.list(sort_by=sort_by) num_archives = len(archive_infos) - pi = ProgressIndicatorPercent(total=num_archives, msg='Checking archives %3.1f%%', step=0.1, - msgid='check.rebuild_refcounts') + pi = ProgressIndicatorPercent( + total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_refcounts" + ) with cache_if_remote(self.repository) as repository: for i, info in enumerate(archive_infos): pi.show(i) - logger.info(f'Analyzing archive {info.name} ({i + 1}/{num_archives})') + logger.info(f"Analyzing archive {info.name} ({i + 1}/{num_archives})") archive_id = info.id if archive_id not in self.chunks: - logger.error('Archive metadata block %s is missing!', bin_to_hex(archive_id)) + logger.error("Archive metadata block %s is missing!", bin_to_hex(archive_id)) self.error_found = True del self.manifest.archives[info.name] continue @@ -1937,17 +2048,17 @@ def valid_item(obj): try: data = self.key.decrypt(archive_id, cdata) except IntegrityError as integrity_error: - logger.error('Archive metadata block %s is corrupted: %s', bin_to_hex(archive_id), integrity_error) + logger.error("Archive metadata block %s is corrupted: %s", bin_to_hex(archive_id), integrity_error) self.error_found = True del self.manifest.archives[info.name] continue archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) if archive.version != 2: - raise Exception('Unknown archive metadata version') + raise Exception("Unknown archive metadata version") items_buffer = ChunkBuffer(self.key) items_buffer.write_chunk = add_callback for item in robust_iterator(archive): - if 'chunks' in item: + if "chunks" in item: verify_file_chunks(info.name, item) items_buffer.add(item) items_buffer.flush(flush=True) @@ -1966,22 +2077,23 @@ def orphan_chunks_check(self): unused = {id_ for id_, entry in self.chunks.iteritems() if entry.refcount == 0} orphaned = unused - self.possibly_superseded if orphaned: - logger.error(f'{len(orphaned)} orphaned objects found!') + logger.error(f"{len(orphaned)} orphaned objects found!") self.error_found = True if self.repair and unused: - logger.info('Deleting %d orphaned and %d superseded objects...' % ( - len(orphaned), len(self.possibly_superseded))) + logger.info( + "Deleting %d orphaned and %d superseded objects..." % (len(orphaned), len(self.possibly_superseded)) + ) for id_ in unused: self.repository.delete(id_) - logger.info('Finished deleting orphaned/superseded objects.') + logger.info("Finished deleting orphaned/superseded objects.") else: - logger.info('Orphaned objects check skipped (needs all archives checked).') + logger.info("Orphaned objects check skipped (needs all archives checked).") def finish(self, save_space=False): if self.repair: - logger.info('Writing Manifest.') + logger.info("Writing Manifest.") self.manifest.write() - logger.info('Committing repo.') + logger.info("Committing repo.") self.repository.commit(compact=False, save_space=save_space) @@ -1992,13 +2104,29 @@ def __init__(self, metadata=None): @staticmethod def is_temporary_archive(archive_name): - return archive_name.endswith('.recreate') + return archive_name.endswith(".recreate") - def __init__(self, repository, manifest, key, cache, matcher, - exclude_caches=False, exclude_if_present=None, keep_exclude_tags=False, - chunker_params=None, compression=None, recompress=False, always_recompress=False, - dry_run=False, stats=False, progress=False, file_status_printer=None, - timestamp=None, checkpoint_interval=1800): + def __init__( + self, + repository, + manifest, + key, + cache, + matcher, + exclude_caches=False, + exclude_if_present=None, + keep_exclude_tags=False, + chunker_params=None, + compression=None, + recompress=False, + always_recompress=False, + dry_run=False, + stats=False, + progress=False, + file_status_printer=None, + timestamp=None, + checkpoint_interval=1800, + ): self.repository = repository self.key = key self.manifest = manifest @@ -2011,11 +2139,11 @@ def __init__(self, repository, manifest, key, cache, matcher, self.rechunkify = chunker_params is not None if self.rechunkify: - logger.debug('Rechunking archives to %s', chunker_params) + logger.debug("Rechunking archives to %s", chunker_params) self.chunker_params = chunker_params or CHUNKER_PARAMS self.recompress = recompress self.always_recompress = always_recompress - self.compression = compression or CompressionSpec('none') + self.compression = compression or CompressionSpec("none") self.seen_chunks = set() self.timestamp = timestamp @@ -2043,10 +2171,10 @@ def process_items(self, archive, target): for item in archive.iter_items(): if not matcher.match(item.path): - self.print_file_status('x', item.path) + self.print_file_status("x", item.path) continue if self.dry_run: - self.print_file_status('-', item.path) + self.print_file_status("-", item.path) else: self.process_item(archive, target, item) if self.progress: @@ -2054,7 +2182,7 @@ def process_items(self, archive, target): def process_item(self, archive, target, item): status = file_status(item.mode) - if 'chunks' in item: + if "chunks" in item: self.print_file_status(status, item.path) status = None self.process_chunks(archive, target, item) @@ -2103,7 +2231,7 @@ def save(self, archive, target, comment=None, replace_original=True): if self.dry_run: return if comment is None: - comment = archive.metadata.get('comment', '') + comment = archive.metadata.get("comment", "") # Keep for the statistics if necessary if self.stats: @@ -2111,36 +2239,37 @@ def save(self, archive, target, comment=None, replace_original=True): if self.timestamp is None: additional_metadata = { - 'time': archive.metadata.time, - 'time_end': archive.metadata.get('time_end') or archive.metadata.time, - 'cmdline': archive.metadata.cmdline, + "time": archive.metadata.time, + "time_end": archive.metadata.get("time_end") or archive.metadata.time, + "cmdline": archive.metadata.cmdline, # but also remember recreate metadata: - 'recreate_cmdline': sys.argv, + "recreate_cmdline": sys.argv, } else: additional_metadata = { - 'cmdline': archive.metadata.cmdline, + "cmdline": archive.metadata.cmdline, # but also remember recreate metadata: - 'recreate_cmdline': sys.argv, + "recreate_cmdline": sys.argv, } - target.save(comment=comment, timestamp=self.timestamp, - stats=target.stats, additional_metadata=additional_metadata) + target.save( + comment=comment, timestamp=self.timestamp, stats=target.stats, additional_metadata=additional_metadata + ) if replace_original: archive.delete(Statistics(), progress=self.progress) target.rename(archive.name) if self.stats: target.start = _start target.end = datetime.utcnow() - log_multi(str(target), - str(target.stats)) + log_multi(str(target), str(target.stats)) def matcher_add_tagged_dirs(self, archive): """Add excludes to the matcher created by exclude_cache and exclude_if_present.""" + def exclude(dir, tag_item): if self.keep_exclude_tags: tag_files.append(PathPrefixPattern(tag_item.path, recurse_dir=False)) - tagged_dirs.append(FnmatchPattern(dir + '/', recurse_dir=False)) + tagged_dirs.append(FnmatchPattern(dir + "/", recurse_dir=False)) else: tagged_dirs.append(PathPrefixPattern(dir, recurse_dir=False)) @@ -2149,7 +2278,8 @@ def exclude(dir, tag_item): tagged_dirs = [] for item in archive.iter_items( - filter=lambda item: os.path.basename(item.path) == CACHE_TAG_NAME or matcher.match(item.path)): + filter=lambda item: os.path.basename(item.path) == CACHE_TAG_NAME or matcher.match(item.path) + ): dir, tag_file = os.path.split(item.path) if tag_file in self.exclude_if_present: exclude(dir, item) @@ -2162,27 +2292,41 @@ def exclude(dir, tag_item): def create_target(self, archive, target_name=None): """Create target archive.""" - target_name = target_name or archive.name + '.recreate' + target_name = target_name or archive.name + ".recreate" target = self.create_target_archive(target_name) # If the archives use the same chunker params, then don't rechunkify - source_chunker_params = tuple(archive.metadata.get('chunker_params', [])) + source_chunker_params = tuple(archive.metadata.get("chunker_params", [])) if len(source_chunker_params) == 4 and isinstance(source_chunker_params[0], int): # this is a borg < 1.2 chunker_params tuple, no chunker algo specified, but we only had buzhash: - source_chunker_params = (CH_BUZHASH, ) + source_chunker_params + source_chunker_params = (CH_BUZHASH,) + source_chunker_params target.recreate_rechunkify = self.rechunkify and source_chunker_params != target.chunker_params if target.recreate_rechunkify: - logger.debug('Rechunking archive from %s to %s', source_chunker_params or '(unknown)', target.chunker_params) + logger.debug( + "Rechunking archive from %s to %s", source_chunker_params or "(unknown)", target.chunker_params + ) target.process_file_chunks = ChunksProcessor( - cache=self.cache, key=self.key, - add_item=target.add_item, write_checkpoint=target.write_checkpoint, - checkpoint_interval=self.checkpoint_interval, rechunkify=target.recreate_rechunkify).process_file_chunks + cache=self.cache, + key=self.key, + add_item=target.add_item, + write_checkpoint=target.write_checkpoint, + checkpoint_interval=self.checkpoint_interval, + rechunkify=target.recreate_rechunkify, + ).process_file_chunks target.chunker = get_chunker(*target.chunker_params, seed=self.key.chunk_seed) return target def create_target_archive(self, name): - target = Archive(self.repository, self.key, self.manifest, name, create=True, - progress=self.progress, chunker_params=self.chunker_params, cache=self.cache, - checkpoint_interval=self.checkpoint_interval) + target = Archive( + self.repository, + self.key, + self.manifest, + name, + create=True, + progress=self.progress, + chunker_params=self.chunker_params, + cache=self.cache, + checkpoint_interval=self.checkpoint_interval, + ) return target def open_archive(self, name, **kwargs): diff --git a/src/borg/archiver.py b/src/borg/archiver.py index c4831b875..ced4ce037 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -82,7 +82,12 @@ from .helpers import iter_separated from .helpers import get_tar_filter from .nanorst import rst_to_terminal - from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern + from .patterns import ( + ArgparsePatternAction, + ArgparseExcludeFileAction, + ArgparsePatternFileAction, + parse_exclude_pattern, + ) from .patterns import PatternMatcher from .item import Item from .platform import get_flags, get_process_id, SyncFile @@ -114,17 +119,30 @@ def argument(args, str_or_bool): return str_or_bool -def get_repository(location, *, create, exclusive, lock_wait, lock, append_only, - make_parent_dirs, storage_quota, args): - if location.proto == 'ssh': - repository = RemoteRepository(location, create=create, exclusive=exclusive, - lock_wait=lock_wait, lock=lock, append_only=append_only, - make_parent_dirs=make_parent_dirs, args=args) +def get_repository(location, *, create, exclusive, lock_wait, lock, append_only, make_parent_dirs, storage_quota, args): + if location.proto == "ssh": + repository = RemoteRepository( + location, + create=create, + exclusive=exclusive, + lock_wait=lock_wait, + lock=lock, + append_only=append_only, + make_parent_dirs=make_parent_dirs, + args=args, + ) else: - repository = Repository(location.path, create=create, exclusive=exclusive, - lock_wait=lock_wait, lock=lock, append_only=append_only, - make_parent_dirs=make_parent_dirs, storage_quota=storage_quota) + repository = Repository( + location.path, + create=create, + exclusive=exclusive, + lock_wait=lock_wait, + lock=lock, + append_only=append_only, + make_parent_dirs=make_parent_dirs, + storage_quota=storage_quota, + ) return repository @@ -136,16 +154,25 @@ def compat_check(*, create, manifest, key, cache, compatibility, decorator_name) raise AssertionError(f"{decorator_name} decorator compatibility argument must be of type tuple") else: if compatibility is not None: - raise AssertionError(f"{decorator_name} called with compatibility argument, " - f"but would not check {compatibility!r}") + raise AssertionError( + f"{decorator_name} called with compatibility argument, " f"but would not check {compatibility!r}" + ) if create: compatibility = Manifest.NO_OPERATION_CHECK return compatibility -def with_repository(fake=False, invert_fake=False, create=False, lock=True, - exclusive=False, manifest=True, cache=False, secure=True, - compatibility=None): +def with_repository( + fake=False, + invert_fake=False, + create=False, + lock=True, + exclusive=False, + manifest=True, + cache=False, + secure=True, + compatibility=None, +): """ Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …) @@ -160,8 +187,14 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True, :param compatibility: mandatory if not create and (manifest or cache), specifies mandatory feature categories to check """ # Note: with_repository decorator does not have a "key" argument (yet?) - compatibility = compat_check(create=create, manifest=manifest, key=manifest, cache=cache, - compatibility=compatibility, decorator_name='with_repository') + compatibility = compat_check( + create=create, + manifest=manifest, + key=manifest, + cache=cache, + compatibility=compatibility, + decorator_name="with_repository", + ) # To process the `--bypass-lock` option if specified, we need to # modify `lock` inside `wrapper`. Therefore we cannot use the @@ -175,41 +208,57 @@ def with_repository(fake=False, invert_fake=False, create=False, lock=True, def decorator(method): @functools.wraps(method) def wrapper(self, args, **kwargs): - location = getattr(args, 'location') + location = getattr(args, "location") if not location.valid: # location always must be given - raise Error('missing repository, please use --repo or BORG_REPO env var!') - lock = getattr(args, 'lock', _lock) - append_only = getattr(args, 'append_only', False) - storage_quota = getattr(args, 'storage_quota', None) - make_parent_dirs = getattr(args, 'make_parent_dirs', False) + raise Error("missing repository, please use --repo or BORG_REPO env var!") + lock = getattr(args, "lock", _lock) + append_only = getattr(args, "append_only", False) + storage_quota = getattr(args, "storage_quota", None) + make_parent_dirs = getattr(args, "make_parent_dirs", False) if argument(args, fake) ^ invert_fake: return method(self, args, repository=None, **kwargs) - repository = get_repository(location, create=create, exclusive=argument(args, exclusive), - lock_wait=self.lock_wait, lock=lock, append_only=append_only, - make_parent_dirs=make_parent_dirs, storage_quota=storage_quota, - args=args) + repository = get_repository( + location, + create=create, + exclusive=argument(args, exclusive), + lock_wait=self.lock_wait, + lock=lock, + append_only=append_only, + make_parent_dirs=make_parent_dirs, + storage_quota=storage_quota, + args=args, + ) with repository: - if repository.version not in (2, ): - raise Error("This borg version only accepts version 2 repos for -r/--repo. " - "You can use 'borg transfer' to copy archives from old to new repos.") + if repository.version not in (2,): + raise Error( + "This borg version only accepts version 2 repos for -r/--repo. " + "You can use 'borg transfer' to copy archives from old to new repos." + ) if manifest or cache: - kwargs['manifest'], kwargs['key'] = Manifest.load(repository, compatibility) - if 'compression' in args: - kwargs['key'].compressor = args.compression.compressor + kwargs["manifest"], kwargs["key"] = Manifest.load(repository, compatibility) + if "compression" in args: + kwargs["key"].compressor = args.compression.compressor if secure: - assert_secure(repository, kwargs['manifest'], self.lock_wait) + assert_secure(repository, kwargs["manifest"], self.lock_wait) if cache: - with Cache(repository, kwargs['key'], kwargs['manifest'], - progress=getattr(args, 'progress', False), lock_wait=self.lock_wait, - cache_mode=getattr(args, 'files_cache_mode', FILES_CACHE_MODE_DISABLED), - consider_part_files=getattr(args, 'consider_part_files', False), - iec=getattr(args, 'iec', False)) as cache_: + with Cache( + repository, + kwargs["key"], + kwargs["manifest"], + progress=getattr(args, "progress", False), + lock_wait=self.lock_wait, + cache_mode=getattr(args, "files_cache_mode", FILES_CACHE_MODE_DISABLED), + consider_part_files=getattr(args, "consider_part_files", False), + iec=getattr(args, "iec", False), + ) as cache_: return method(self, args, repository=repository, cache=cache_, **kwargs) else: return method(self, args, repository=repository, **kwargs) + return wrapper + return decorator @@ -220,66 +269,94 @@ def with_other_repository(manifest=False, key=False, cache=False, compatibility= the repository at the "other location" is intended to get used as a **source** (== read operations). """ - compatibility = compat_check(create=False, manifest=manifest, key=key, cache=cache, - compatibility=compatibility, decorator_name='with_other_repository') + compatibility = compat_check( + create=False, + manifest=manifest, + key=key, + cache=cache, + compatibility=compatibility, + decorator_name="with_other_repository", + ) def decorator(method): @functools.wraps(method) def wrapper(self, args, **kwargs): - location = getattr(args, 'other_location') + location = getattr(args, "other_location") if not location.valid: # nothing to do return method(self, args, **kwargs) - repository = get_repository(location, create=False, exclusive=True, - lock_wait=self.lock_wait, lock=True, append_only=False, - make_parent_dirs=False, storage_quota=None, - args=args) + repository = get_repository( + location, + create=False, + exclusive=True, + lock_wait=self.lock_wait, + lock=True, + append_only=False, + make_parent_dirs=False, + storage_quota=None, + args=args, + ) with repository: - if repository.version not in (1, 2, ): + if repository.version not in (1, 2): raise Error("This borg version only accepts version 1 or 2 repos for --other-repo.") - kwargs['other_repository'] = repository + kwargs["other_repository"] = repository if manifest or key or cache: manifest_, key_ = Manifest.load(repository, compatibility) assert_secure(repository, manifest_, self.lock_wait) if manifest: - kwargs['other_manifest'] = manifest_ + kwargs["other_manifest"] = manifest_ if key: - kwargs['other_key'] = key_ + kwargs["other_key"] = key_ if cache: - with Cache(repository, key_, manifest_, - progress=False, lock_wait=self.lock_wait, - cache_mode=getattr(args, 'files_cache_mode', FILES_CACHE_MODE_DISABLED), - consider_part_files=getattr(args, 'consider_part_files', False), - iec=getattr(args, 'iec', False)) as cache_: - kwargs['other_cache'] = cache_ + with Cache( + repository, + key_, + manifest_, + progress=False, + lock_wait=self.lock_wait, + cache_mode=getattr(args, "files_cache_mode", FILES_CACHE_MODE_DISABLED), + consider_part_files=getattr(args, "consider_part_files", False), + iec=getattr(args, "iec", False), + ) as cache_: + kwargs["other_cache"] = cache_ return method(self, args, **kwargs) else: return method(self, args, **kwargs) + return wrapper + return decorator def with_archive(method): @functools.wraps(method) def wrapper(self, args, repository, key, manifest, **kwargs): - archive_name = getattr(args, 'name', None) + archive_name = getattr(args, "name", None) assert archive_name is not None - archive = Archive(repository, key, manifest, archive_name, - numeric_ids=getattr(args, 'numeric_ids', False), - noflags=getattr(args, 'nobsdflags', False) or getattr(args, 'noflags', False), - noacls=getattr(args, 'noacls', False), - noxattrs=getattr(args, 'noxattrs', False), - cache=kwargs.get('cache'), - consider_part_files=args.consider_part_files, log_json=args.log_json, iec=args.iec) + archive = Archive( + repository, + key, + manifest, + archive_name, + numeric_ids=getattr(args, "numeric_ids", False), + noflags=getattr(args, "nobsdflags", False) or getattr(args, "noflags", False), + noacls=getattr(args, "noacls", False), + noxattrs=getattr(args, "noxattrs", False), + cache=kwargs.get("cache"), + consider_part_files=args.consider_part_files, + log_json=args.log_json, + iec=args.iec, + ) return method(self, args, repository=repository, manifest=manifest, key=key, archive=archive, **kwargs) + return wrapper def parse_storage_quota(storage_quota): parsed = parse_file_size(storage_quota) - if parsed < parse_file_size('10M'): - raise argparse.ArgumentTypeError('quota is too small (%s). At least 10M are required.' % storage_quota) + if parsed < parse_file_size("10M"): + raise argparse.ArgumentTypeError("quota is too small (%s). At least 10M are required." % storage_quota) return parsed @@ -288,23 +365,23 @@ def get_func(args): # func is used at the leaf parsers of the argparse parser tree, # fallback_func at next level towards the root, # fallback2_func at the 2nd next level (which is root in our case). - for name in 'func', 'fallback_func', 'fallback2_func': + for name in "func", "fallback_func", "fallback2_func": func = getattr(args, name, None) if func is not None: return func - raise Exception('expected func attributes not found') + raise Exception("expected func attributes not found") class Highlander(argparse.Action): """make sure some option is only given once""" + def __call__(self, parser, namespace, values, option_string=None): if getattr(namespace, self.dest, None) != self.default: - raise argparse.ArgumentError(self, 'There can be only one.') + raise argparse.ArgumentError(self, "There can be only one.") setattr(namespace, self.dest, values) class Archiver: - def __init__(self, lock_wait=None, prog=None): self.exit_code = EXIT_SUCCESS self.lock_wait = lock_wait @@ -324,13 +401,12 @@ def print_file_status(self, status, path): # if we get called with status == None, the final file status was already printed if self.output_list and status is not None and (self.output_filter is None or status in self.output_filter): if self.log_json: - print(json.dumps({ - 'type': 'file_status', - 'status': status, - 'path': remove_surrogates(path), - }), file=sys.stderr) + print( + json.dumps({"type": "file_status", "status": status, "path": remove_surrogates(path)}), + file=sys.stderr, + ) else: - logging.getLogger('borg.output.list').info("%1s %s", status, remove_surrogates(path)) + logging.getLogger("borg.output.list").info("%1s %s", status, remove_surrogates(path)) @staticmethod def build_matcher(inclexcl_patterns, include_paths): @@ -351,9 +427,9 @@ def do_serve(self, args): @with_other_repository(manifest=True, key=True, compatibility=(Manifest.Operation.READ,)) @with_repository(exclusive=True, manifest=True, cache=True, compatibility=(Manifest.Operation.WRITE,)) - def do_transfer(self, args, *, - repository, manifest, key, cache, - other_repository=None, other_manifest=None, other_key=None): + def do_transfer( + self, args, *, repository, manifest, key, cache, other_repository=None, other_manifest=None, other_key=None + ): """archives transfer from other repository, optionally upgrade data format""" dry_run = args.dry_run args.consider_checkpoints = True @@ -362,10 +438,11 @@ def do_transfer(self, args, *, return EXIT_SUCCESS from . import upgrade as upgrade_mod + try: - UpgraderCls = getattr(upgrade_mod, f'Upgrader{args.upgrader}') + UpgraderCls = getattr(upgrade_mod, f"Upgrader{args.upgrader}") except AttributeError: - self.print_error(f'No such upgrader: {args.upgrader}') + self.print_error(f"No such upgrader: {args.upgrader}") return EXIT_ERROR upgrader = UpgraderCls(cache=cache) @@ -382,7 +459,7 @@ def do_transfer(self, args, *, archive = Archive(repository, key, manifest, name, cache=cache, create=True) if not dry_run else None upgrader.new_archive(archive=archive) for item in other_archive.iter_items(): - if 'chunks' in item: + if "chunks" in item: chunks = [] for chunk_id, size in item.chunks: refcount = cache.seen_chunk(chunk_id, size) @@ -392,8 +469,9 @@ def do_transfer(self, args, *, # keep compressed payload same, avoid decompression / recompression data = other_key.decrypt(chunk_id, cdata, decompress=False) data = upgrader.upgrade_compressed_chunk(chunk=data) - chunk_entry = cache.add_chunk(chunk_id, data, archive.stats, wait=False, - compress=False, size=size) + chunk_entry = cache.add_chunk( + chunk_id, data, archive.stats, wait=False, compress=False, size=size + ) cache.repository.async_response(wait=False) chunks.append(chunk_entry) transfer_size += size @@ -410,18 +488,23 @@ def do_transfer(self, args, *, if not dry_run: additional_metadata = upgrader.upgrade_archive_metadata(metadata=other_archive.metadata) archive.save(stats=archive.stats, additional_metadata=additional_metadata) - print(f"{name}: finished. " - f"transfer_size: {format_file_size(transfer_size)} " - f"present_size: {format_file_size(present_size)}") + print( + f"{name}: finished. " + f"transfer_size: {format_file_size(transfer_size)} " + f"present_size: {format_file_size(present_size)}" + ) else: - print(f"{name}: completed" if transfer_size == 0 else - f"{name}: incomplete, " - f"transfer_size: {format_file_size(transfer_size)} " - f"present_size: {format_file_size(present_size)}") + print( + f"{name}: completed" + if transfer_size == 0 + else f"{name}: incomplete, " + f"transfer_size: {format_file_size(transfer_size)} " + f"present_size: {format_file_size(present_size)}" + ) return EXIT_SUCCESS @with_repository(create=True, exclusive=True, manifest=False) - @with_other_repository(key=True, compatibility=(Manifest.Operation.READ, )) + @with_other_repository(key=True, compatibility=(Manifest.Operation.READ,)) def do_rcreate(self, args, repository, *, other_repository=None, other_key=None): """Create a new, empty repository""" path = args.location.canonical_path() @@ -439,34 +522,42 @@ def do_rcreate(self, args, repository, *, other_repository=None, other_key=None) pass if key.tam_required: tam_file = tam_required_file(repository) - open(tam_file, 'w').close() + open(tam_file, "w").close() - if key.NAME != 'plaintext': + if key.NAME != "plaintext": logger.warning( - '\n' - 'IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!\n' - 'If you used a repokey mode, the key is stored in the repo, but you should back it up separately.\n' + "\n" + "IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!\n" + "If you used a repokey mode, the key is stored in the repo, but you should back it up separately.\n" 'Use "borg key export" to export the key, optionally in printable format.\n' - 'Write down the passphrase. Store both at safe place(s).\n') + "Write down the passphrase. Store both at safe place(s).\n" + ) return self.exit_code @with_repository(exclusive=True, manifest=False) def do_check(self, args, repository): """Check repository consistency""" if args.repair: - msg = ("This is a potentially dangerous function.\n" - "check --repair might lead to data loss (for kinds of corruption it is not\n" - "capable of dealing with). BE VERY CAREFUL!\n" - "\n" - "Type 'YES' if you understand this and want to continue: ") - if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", - truish=('YES', ), retry=False, - env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): + msg = ( + "This is a potentially dangerous function.\n" + "check --repair might lead to data loss (for kinds of corruption it is not\n" + "capable of dealing with). BE VERY CAREFUL!\n" + "\n" + "Type 'YES' if you understand this and want to continue: " + ) + if not yes( + msg, + false_msg="Aborting.", + invalid_msg="Invalid answer, aborting.", + truish=("YES",), + retry=False, + env_var_override="BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", + ): return EXIT_ERROR - if args.repo_only and any( - (args.verify_data, args.first, args.last, args.glob_archives)): - self.print_error("--repository-only contradicts --first, --last, -a / --glob-archives " - " and --verify-data arguments.") + if args.repo_only and any((args.verify_data, args.first, args.last, args.glob_archives)): + self.print_error( + "--repository-only contradicts --first, --last, -a / --glob-archives " " and --verify-data arguments." + ) return EXIT_ERROR if args.repair and args.max_duration: self.print_error("--repair does not allow --max-duration argument.") @@ -481,33 +572,40 @@ def do_check(self, args, repository): if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space, max_duration=args.max_duration): return EXIT_WARNING - if not args.repo_only and not ArchiveChecker().check(repository, repair=args.repair, - first=args.first, last=args.last, sort_by=args.sort_by or 'ts', glob=args.glob_archives, - verify_data=args.verify_data, save_space=args.save_space): + if not args.repo_only and not ArchiveChecker().check( + repository, + repair=args.repair, + first=args.first, + last=args.last, + sort_by=args.sort_by or "ts", + glob=args.glob_archives, + verify_data=args.verify_data, + save_space=args.save_space, + ): return EXIT_WARNING return EXIT_SUCCESS @with_repository(compatibility=(Manifest.Operation.CHECK,)) def do_change_passphrase(self, args, repository, manifest, key): """Change repository key file passphrase""" - if not hasattr(key, 'change_passphrase'): - print('This repository is not encrypted, cannot change the passphrase.') + if not hasattr(key, "change_passphrase"): + print("This repository is not encrypted, cannot change the passphrase.") return EXIT_ERROR key.change_passphrase() - logger.info('Key updated') - if hasattr(key, 'find_key'): + logger.info("Key updated") + if hasattr(key, "find_key"): # print key location to make backing it up easier - logger.info('Key location: %s', key.find_key()) + logger.info("Key location: %s", key.find_key()) return EXIT_SUCCESS @with_repository(exclusive=True, manifest=True, cache=True, compatibility=(Manifest.Operation.CHECK,)) def do_change_location(self, args, repository, manifest, key, cache): """Change repository key location""" - if not hasattr(key, 'change_passphrase'): - print('This repository is not encrypted, cannot change the key location.') + if not hasattr(key, "change_passphrase"): + print("This repository is not encrypted, cannot change the key location.") return EXIT_ERROR - if args.key_mode == 'keyfile': + if args.key_mode == "keyfile": if isinstance(key, AESOCBRepoKey): key_new = AESOCBKeyfileKey(repository) elif isinstance(key, CHPORepoKey): @@ -519,7 +617,7 @@ def do_change_location(self, args, repository, manifest, key, cache): else: print("Change not needed or not supported.") return EXIT_WARNING - if args.key_mode == 'repokey': + if args.key_mode == "repokey": if isinstance(key, AESOCBKeyfileKey): key_new = AESOCBRepoKey(repository) elif isinstance(key, CHPOKeyfileKey): @@ -532,8 +630,16 @@ def do_change_location(self, args, repository, manifest, key, cache): print("Change not needed or not supported.") return EXIT_WARNING - for name in ('repository_id', 'enc_key', 'enc_hmac_key', 'id_key', 'chunk_seed', - 'tam_required', 'sessionid', 'cipher'): + for name in ( + "repository_id", + "enc_key", + "enc_hmac_key", + "id_key", + "chunk_seed", + "tam_required", + "sessionid", + "cipher", + ): value = getattr(key, name) setattr(key_new, name, value) @@ -552,20 +658,20 @@ def do_change_location(self, args, repository, manifest, key, cache): cache.key = key_new cache.commit() - loc = key_new.find_key() if hasattr(key_new, 'find_key') else None + loc = key_new.find_key() if hasattr(key_new, "find_key") else None if args.keep: - logger.info(f'Key copied to {loc}') + logger.info(f"Key copied to {loc}") else: key.remove(key.target) # remove key from current location - logger.info(f'Key moved to {loc}') + logger.info(f"Key moved to {loc}") return EXIT_SUCCESS @with_repository(exclusive=True, compatibility=(Manifest.Operation.CHECK,)) def do_change_algorithm(self, args, repository, manifest, key): """Change repository key algorithm""" - if not hasattr(key, 'change_passphrase'): - print('This repository is not encrypted, cannot change the algorithm.') + if not hasattr(key, "change_passphrase"): + print("This repository is not encrypted, cannot change the algorithm.") return EXIT_ERROR key.save(key.target, key._passphrase, algorithm=KEY_ALGORITHMS[args.algorithm]) return EXIT_SUCCESS @@ -601,7 +707,7 @@ def do_key_import(self, args, repository): if not args.path: self.print_error("input file to import key from expected") return EXIT_ERROR - if args.path != '-' and not os.path.exists(args.path): + if args.path != "-" and not os.path.exists(args.path): self.print_error("input file does not exist: " + args.path) return EXIT_ERROR manager.import_keyfile(args) @@ -609,38 +715,43 @@ def do_key_import(self, args, repository): def do_benchmark_crud(self, args): """Benchmark Create, Read, Update, Delete for archives.""" + def measurement_run(repo, path): - compression = '--compression=none' + compression = "--compression=none" # measure create perf (without files cache to always have it chunking) t_start = time.monotonic() - rc = self.do_create(self.parse_args([f'--repo={repo}', 'create', compression, '--files-cache=disabled', - 'borg-benchmark-crud1', path])) + rc = self.do_create( + self.parse_args( + [f"--repo={repo}", "create", compression, "--files-cache=disabled", "borg-benchmark-crud1", path] + ) + ) t_end = time.monotonic() dt_create = t_end - t_start assert rc == 0 # now build files cache - rc1 = self.do_create(self.parse_args([f'--repo={repo}', 'create', compression, - 'borg-benchmark-crud2', path])) - rc2 = self.do_delete(self.parse_args([f'--repo={repo}', 'delete', '-a', 'borg-benchmark-crud2'])) + rc1 = self.do_create( + self.parse_args([f"--repo={repo}", "create", compression, "borg-benchmark-crud2", path]) + ) + rc2 = self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud2"])) assert rc1 == rc2 == 0 # measure a no-change update (archive1 is still present) t_start = time.monotonic() - rc1 = self.do_create(self.parse_args([f'--repo={repo}', 'create', compression, - 'borg-benchmark-crud3', path])) + rc1 = self.do_create( + self.parse_args([f"--repo={repo}", "create", compression, "borg-benchmark-crud3", path]) + ) t_end = time.monotonic() dt_update = t_end - t_start - rc2 = self.do_delete(self.parse_args([f'--repo={repo}', 'delete', '-a', 'borg-benchmark-crud3'])) + rc2 = self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud3"])) assert rc1 == rc2 == 0 # measure extraction (dry-run: without writing result to disk) t_start = time.monotonic() - rc = self.do_extract(self.parse_args([f'--repo={repo}', 'extract', 'borg-benchmark-crud1', - '--dry-run'])) + rc = self.do_extract(self.parse_args([f"--repo={repo}", "extract", "borg-benchmark-crud1", "--dry-run"])) t_end = time.monotonic() dt_extract = t_end - t_start assert rc == 0 # measure archive deletion (of LAST present archive with the data) t_start = time.monotonic() - rc = self.do_delete(self.parse_args([f'--repo={repo}', 'delete', '-a', 'borg-benchmark-crud1'])) + rc = self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud1"])) t_end = time.monotonic() dt_delete = t_end - t_start assert rc == 0 @@ -649,11 +760,11 @@ def measurement_run(repo, path): @contextmanager def test_files(path, count, size, random): try: - path = os.path.join(path, 'borg-test-data') + path = os.path.join(path, "borg-test-data") os.makedirs(path) - z_buff = None if random else memoryview(zeros)[:size] if size <= len(zeros) else b'\0' * size + z_buff = None if random else memoryview(zeros)[:size] if size <= len(zeros) else b"\0" * size for i in range(count): - fname = os.path.join(path, 'file_%d' % i) + fname = os.path.join(path, "file_%d" % i) data = z_buff if not random else os.urandom(size) with SyncFile(fname, binary=True) as fd: # used for posix_fadvise's sake fd.write(data) @@ -661,19 +772,16 @@ def test_files(path, count, size, random): finally: shutil.rmtree(path) - if '_BORG_BENCHMARK_CRUD_TEST' in os.environ: - tests = [ - ('Z-TEST', 1, 1, False), - ('R-TEST', 1, 1, True), - ] + if "_BORG_BENCHMARK_CRUD_TEST" in os.environ: + tests = [("Z-TEST", 1, 1, False), ("R-TEST", 1, 1, True)] else: tests = [ - ('Z-BIG', 10, 100000000, False), - ('R-BIG', 10, 100000000, True), - ('Z-MEDIUM', 1000, 1000000, False), - ('R-MEDIUM', 1000, 1000000, True), - ('Z-SMALL', 10000, 10000, False), - ('R-SMALL', 10000, 10000, True), + ("Z-BIG", 10, 100000000, False), + ("R-BIG", 10, 100000000, True), + ("Z-MEDIUM", 1000, 1000000, False), + ("R-MEDIUM", 1000, 1000000, True), + ("Z-SMALL", 10000, 10000, False), + ("R-SMALL", 10000, 10000, True), ] for msg, count, size, random in tests: @@ -681,25 +789,27 @@ def test_files(path, count, size, random): dt_create, dt_update, dt_extract, dt_delete = measurement_run(args.location.canonical_path(), path) total_size_MB = count * size / 1e06 file_size_formatted = format_file_size(size) - content = 'random' if random else 'all-zero' - fmt = '%s-%-10s %9.2f MB/s (%d * %s %s files: %.2fs)' - print(fmt % ('C', msg, total_size_MB / dt_create, count, file_size_formatted, content, dt_create)) - print(fmt % ('R', msg, total_size_MB / dt_extract, count, file_size_formatted, content, dt_extract)) - print(fmt % ('U', msg, total_size_MB / dt_update, count, file_size_formatted, content, dt_update)) - print(fmt % ('D', msg, total_size_MB / dt_delete, count, file_size_formatted, content, dt_delete)) + content = "random" if random else "all-zero" + fmt = "%s-%-10s %9.2f MB/s (%d * %s %s files: %.2fs)" + print(fmt % ("C", msg, total_size_MB / dt_create, count, file_size_formatted, content, dt_create)) + print(fmt % ("R", msg, total_size_MB / dt_extract, count, file_size_formatted, content, dt_extract)) + print(fmt % ("U", msg, total_size_MB / dt_update, count, file_size_formatted, content, dt_update)) + print(fmt % ("D", msg, total_size_MB / dt_delete, count, file_size_formatted, content, dt_delete)) return 0 def do_benchmark_cpu(self, args): """Benchmark CPU bound operations.""" from timeit import timeit - random_10M = os.urandom(10*1000*1000) + + random_10M = os.urandom(10 * 1000 * 1000) key_256 = os.urandom(32) key_128 = os.urandom(16) key_96 = os.urandom(12) import io from .chunker import get_chunker + print("Chunkers =======================================================") size = "1GB" @@ -716,16 +826,15 @@ def chunkit(chunker_name, *args, **kwargs): print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") from .checksums import crc32, xxh64 + print("Non-cryptographic checksums / hashes ===========================") size = "1GB" - tests = [ - ("xxh64", lambda: xxh64(random_10M)), - ("crc32 (zlib)", lambda: crc32(random_10M)), - ] + tests = [("xxh64", lambda: xxh64(random_10M)), ("crc32 (zlib)", lambda: crc32(random_10M))] for spec, func in tests: print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") from .crypto.low_level import hmac_sha256, blake2b_256 + print("Cryptographic hashes / MACs ====================================") size = "1GB" for spec, func in [ @@ -736,18 +845,33 @@ def chunkit(chunker_name, *args, **kwargs): from .crypto.low_level import AES256_CTR_BLAKE2b, AES256_CTR_HMAC_SHA256 from .crypto.low_level import AES256_OCB, CHACHA20_POLY1305 + print("Encryption =====================================================") size = "1GB" tests = [ - ("aes-256-ctr-hmac-sha256", lambda: AES256_CTR_HMAC_SHA256( - key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(random_10M, header=b'X')), - ("aes-256-ctr-blake2b", lambda: AES256_CTR_BLAKE2b( - key_256*4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt(random_10M, header=b'X')), - ("aes-256-ocb", lambda: AES256_OCB( - key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b'X')), - ("chacha20-poly1305", lambda: CHACHA20_POLY1305( - key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b'X')), + ( + "aes-256-ctr-hmac-sha256", + lambda: AES256_CTR_HMAC_SHA256(key_256, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt( + random_10M, header=b"X" + ), + ), + ( + "aes-256-ctr-blake2b", + lambda: AES256_CTR_BLAKE2b(key_256 * 4, key_256, iv=key_128, header_len=1, aad_offset=1).encrypt( + random_10M, header=b"X" + ), + ), + ( + "aes-256-ocb", + lambda: AES256_OCB(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt(random_10M, header=b"X"), + ), + ( + "chacha20-poly1305", + lambda: CHACHA20_POLY1305(key_256, iv=key_96, header_len=1, aad_offset=1).encrypt( + random_10M, header=b"X" + ), + ), ] for spec, func in tests: print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") @@ -755,27 +879,28 @@ def chunkit(chunker_name, *args, **kwargs): print("KDFs (slow is GOOD, use argon2!) ===============================") count = 5 for spec, func in [ - ("pbkdf2", lambda: FlexiKey.pbkdf2('mypassphrase', b'salt'*8, PBKDF2_ITERATIONS, 32)), - ("argon2", lambda: FlexiKey.argon2('mypassphrase', 64, b'S' * ARGON2_SALT_BYTES, **ARGON2_ARGS)), + ("pbkdf2", lambda: FlexiKey.pbkdf2("mypassphrase", b"salt" * 8, PBKDF2_ITERATIONS, 32)), + ("argon2", lambda: FlexiKey.argon2("mypassphrase", 64, b"S" * ARGON2_SALT_BYTES, **ARGON2_ARGS)), ]: print(f"{spec:<24} {count:<10} {timeit(func, number=count):.3f}s") from .compress import CompressionSpec + print("Compression ====================================================") for spec in [ - 'lz4', - 'zstd,1', - 'zstd,3', - 'zstd,5', - 'zstd,10', - 'zstd,16', - 'zstd,22', - 'zlib,0', - 'zlib,6', - 'zlib,9', - 'lzma,0', - 'lzma,6', - 'lzma,9', + "lz4", + "zstd,1", + "zstd,3", + "zstd,5", + "zstd,10", + "zstd,16", + "zstd,22", + "zlib,0", + "zlib,6", + "zlib,9", + "lzma,0", + "lzma,6", + "lzma,9", ]: compressor = CompressionSpec(spec).compressor size = "0.1GB" @@ -783,14 +908,14 @@ def chunkit(chunker_name, *args, **kwargs): print("msgpack ========================================================") item = Item(path="/foo/bar/baz", mode=660, mtime=1234567) - items = [item.as_dict(), ] * 1000 + items = [item.as_dict()] * 1000 size = "100k Items" spec = "msgpack" print(f"{spec:<12} {size:<10} {timeit(lambda: msgpack.packb(items), number=100):.3f}s") return 0 - @with_repository(fake='dry_run', exclusive=True, compatibility=(Manifest.Operation.WRITE,)) + @with_repository(fake="dry_run", exclusive=True, compatibility=(Manifest.Operation.WRITE,)) def do_create(self, args, repository, manifest=None, key=None): """Create new archive""" matcher = PatternMatcher(fallback=True) @@ -811,7 +936,7 @@ def create_inner(archive, cache, fso): skip_inodes.add((st.st_ino, st.st_dev)) except OSError: pass - logger.debug('Processing files ...') + logger.debug("Processing files ...") if args.content_from_command: path = args.stdin_name mode = args.stdin_mode @@ -822,93 +947,115 @@ def create_inner(archive, cache, fso): try: proc = subprocess.Popen(args.paths, stdout=subprocess.PIPE) except (FileNotFoundError, PermissionError) as e: - self.print_error('Failed to execute command: %s', e) + self.print_error("Failed to execute command: %s", e) return self.exit_code - status = fso.process_pipe(path=path, cache=cache, fd=proc.stdout, mode=mode, user=user, group=group) + status = fso.process_pipe( + path=path, cache=cache, fd=proc.stdout, mode=mode, user=user, group=group + ) rc = proc.wait() if rc != 0: - self.print_error('Command %r exited with status %d', args.paths[0], rc) + self.print_error("Command %r exited with status %d", args.paths[0], rc) return self.exit_code except BackupOSError as e: - self.print_error('%s: %s', path, e) + self.print_error("%s: %s", path, e) return self.exit_code else: - status = '-' + status = "-" self.print_file_status(status, path) elif args.paths_from_command or args.paths_from_stdin: - paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else '\n' + paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n" if args.paths_from_command: try: proc = subprocess.Popen(args.paths, stdout=subprocess.PIPE) except (FileNotFoundError, PermissionError) as e: - self.print_error('Failed to execute command: %s', e) + self.print_error("Failed to execute command: %s", e) return self.exit_code pipe_bin = proc.stdout else: # args.paths_from_stdin == True pipe_bin = sys.stdin.buffer - pipe = TextIOWrapper(pipe_bin, errors='surrogateescape') + pipe = TextIOWrapper(pipe_bin, errors="surrogateescape") for path in iter_separated(pipe, paths_sep): path = os.path.normpath(path) try: - with backup_io('stat'): + with backup_io("stat"): st = os_stat(path=path, parent_fd=None, name=None, follow_symlinks=False) - status = self._process_any(path=path, parent_fd=None, name=None, st=st, fso=fso, - cache=cache, read_special=args.read_special, dry_run=dry_run) + status = self._process_any( + path=path, + parent_fd=None, + name=None, + st=st, + fso=fso, + cache=cache, + read_special=args.read_special, + dry_run=dry_run, + ) except (BackupOSError, BackupError) as e: - self.print_warning('%s: %s', path, e) - status = 'E' - if status == 'C': - self.print_warning('%s: file changed while we backed it up', path) + self.print_warning("%s: %s", path, e) + status = "E" + if status == "C": + self.print_warning("%s: file changed while we backed it up", path) self.print_file_status(status, path) if args.paths_from_command: rc = proc.wait() if rc != 0: - self.print_error('Command %r exited with status %d', args.paths[0], rc) + self.print_error("Command %r exited with status %d", args.paths[0], rc) return self.exit_code else: for path in args.paths: - if path == '-': # stdin + if path == "-": # stdin path = args.stdin_name mode = args.stdin_mode user = args.stdin_user group = args.stdin_group if not dry_run: try: - status = fso.process_pipe(path=path, cache=cache, fd=sys.stdin.buffer, mode=mode, user=user, group=group) + status = fso.process_pipe( + path=path, cache=cache, fd=sys.stdin.buffer, mode=mode, user=user, group=group + ) except BackupOSError as e: - status = 'E' - self.print_warning('%s: %s', path, e) + status = "E" + self.print_warning("%s: %s", path, e) else: - status = '-' + status = "-" self.print_file_status(status, path) continue path = os.path.normpath(path) - parent_dir = os.path.dirname(path) or '.' + parent_dir = os.path.dirname(path) or "." name = os.path.basename(path) try: # note: for path == '/': name == '' and parent_dir == '/'. # the empty name will trigger a fall-back to path-based processing in os_stat and os_open. - with OsOpen(path=parent_dir, flags=flags_root, noatime=True, op='open_root') as parent_fd: + with OsOpen(path=parent_dir, flags=flags_root, noatime=True, op="open_root") as parent_fd: try: st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False) except OSError as e: - self.print_warning('%s: %s', path, e) + self.print_warning("%s: %s", path, e) continue if args.one_file_system: restrict_dev = st.st_dev else: restrict_dev = None - self._rec_walk(path=path, parent_fd=parent_fd, name=name, - fso=fso, cache=cache, matcher=matcher, - exclude_caches=args.exclude_caches, exclude_if_present=args.exclude_if_present, - keep_exclude_tags=args.keep_exclude_tags, skip_inodes=skip_inodes, - restrict_dev=restrict_dev, read_special=args.read_special, dry_run=dry_run) + self._rec_walk( + path=path, + parent_fd=parent_fd, + name=name, + fso=fso, + cache=cache, + matcher=matcher, + exclude_caches=args.exclude_caches, + exclude_if_present=args.exclude_if_present, + keep_exclude_tags=args.keep_exclude_tags, + skip_inodes=skip_inodes, + restrict_dev=restrict_dev, + read_special=args.read_special, + dry_run=dry_run, + ) # if we get back here, we've finished recursing into , # we do not ever want to get back in there (even if path is given twice as recursion root) skip_inodes.add((st.st_ino, st.st_dev)) except (BackupOSError, BackupError) as e: # this comes from OsOpen, self._rec_walk has own exception handler - self.print_warning('%s: %s', path, e) + self.print_warning("%s: %s", path, e) continue if not dry_run: if args.progress: @@ -923,13 +1070,9 @@ def create_inner(archive, cache, fso): args.stats |= args.json if args.stats: if args.json: - json_print(basic_json_data(manifest, cache=cache, extra={ - 'archive': archive, - })) + json_print(basic_json_data(manifest, cache=cache, extra={"archive": archive})) else: - log_multi(str(archive), - str(archive.stats), - logger=logging.getLogger('borg.output.stats')) + log_multi(str(archive), str(archive.stats), logger=logging.getLogger("borg.output.stats")) self.output_filter = args.output_filter self.output_list = args.output_list @@ -942,25 +1085,64 @@ def create_inner(archive, cache, fso): t0_monotonic = time.monotonic() logger.info('Creating archive at "%s"' % args.location.processed) if not dry_run: - with Cache(repository, key, manifest, progress=args.progress, - lock_wait=self.lock_wait, permit_adhoc_cache=args.no_cache_sync, - cache_mode=args.files_cache_mode, iec=args.iec) as cache: - archive = Archive(repository, key, manifest, args.name, cache=cache, - create=True, checkpoint_interval=args.checkpoint_interval, - numeric_ids=args.numeric_ids, noatime=not args.atime, noctime=args.noctime, - progress=args.progress, - chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic, - log_json=args.log_json, iec=args.iec) - metadata_collector = MetadataCollector(noatime=not args.atime, noctime=args.noctime, - noflags=args.noflags, noacls=args.noacls, noxattrs=args.noxattrs, - numeric_ids=args.numeric_ids, nobirthtime=args.nobirthtime) - cp = ChunksProcessor(cache=cache, key=key, - add_item=archive.add_item, write_checkpoint=archive.write_checkpoint, - checkpoint_interval=args.checkpoint_interval, rechunkify=False) - fso = FilesystemObjectProcessors(metadata_collector=metadata_collector, cache=cache, key=key, - process_file_chunks=cp.process_file_chunks, add_item=archive.add_item, - chunker_params=args.chunker_params, show_progress=args.progress, sparse=args.sparse, - log_json=args.log_json, iec=args.iec, file_status_printer=self.print_file_status) + with Cache( + repository, + key, + manifest, + progress=args.progress, + lock_wait=self.lock_wait, + permit_adhoc_cache=args.no_cache_sync, + cache_mode=args.files_cache_mode, + iec=args.iec, + ) as cache: + archive = Archive( + repository, + key, + manifest, + args.name, + cache=cache, + create=True, + checkpoint_interval=args.checkpoint_interval, + numeric_ids=args.numeric_ids, + noatime=not args.atime, + noctime=args.noctime, + progress=args.progress, + chunker_params=args.chunker_params, + start=t0, + start_monotonic=t0_monotonic, + log_json=args.log_json, + iec=args.iec, + ) + metadata_collector = MetadataCollector( + noatime=not args.atime, + noctime=args.noctime, + noflags=args.noflags, + noacls=args.noacls, + noxattrs=args.noxattrs, + numeric_ids=args.numeric_ids, + nobirthtime=args.nobirthtime, + ) + cp = ChunksProcessor( + cache=cache, + key=key, + add_item=archive.add_item, + write_checkpoint=archive.write_checkpoint, + checkpoint_interval=args.checkpoint_interval, + rechunkify=False, + ) + fso = FilesystemObjectProcessors( + metadata_collector=metadata_collector, + cache=cache, + key=key, + process_file_chunks=cp.process_file_chunks, + add_item=archive.add_item, + chunker_params=args.chunker_params, + show_progress=args.progress, + sparse=args.sparse, + log_json=args.log_json, + iec=args.iec, + file_status_printer=self.print_file_status, + ) create_inner(archive, cache, fso) else: create_inner(None, None, None) @@ -972,7 +1154,7 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d """ if dry_run: - return '-' + return "-" elif stat.S_ISREG(st.st_mode): return fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, cache=cache) elif stat.S_ISDIR(st.st_mode): @@ -988,28 +1170,32 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d else: special = is_special(st_target.st_mode) if special: - return fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st_target, - cache=cache, flags=flags_special_follow) + return fso.process_file( + path=path, parent_fd=parent_fd, name=name, st=st_target, cache=cache, flags=flags_special_follow + ) else: return fso.process_symlink(path=path, parent_fd=parent_fd, name=name, st=st) elif stat.S_ISFIFO(st.st_mode): if not read_special: return fso.process_fifo(path=path, parent_fd=parent_fd, name=name, st=st) else: - return fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, - cache=cache, flags=flags_special) + return fso.process_file( + path=path, parent_fd=parent_fd, name=name, st=st, cache=cache, flags=flags_special + ) elif stat.S_ISCHR(st.st_mode): if not read_special: - return fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type='c') + return fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c") else: - return fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, - cache=cache, flags=flags_special) + return fso.process_file( + path=path, parent_fd=parent_fd, name=name, st=st, cache=cache, flags=flags_special + ) elif stat.S_ISBLK(st.st_mode): if not read_special: - return fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type='b') + return fso.process_dev(path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b") else: - return fso.process_file(path=path, parent_fd=parent_fd, name=name, st=st, - cache=cache, flags=flags_special) + return fso.process_file( + path=path, parent_fd=parent_fd, name=name, st=st, cache=cache, flags=flags_special + ) elif stat.S_ISSOCK(st.st_mode): # Ignore unix sockets return @@ -1020,12 +1206,26 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d # Ignore Solaris event ports return else: - self.print_warning('Unknown file type: %s', path) + self.print_warning("Unknown file type: %s", path) return - def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher, - exclude_caches, exclude_if_present, keep_exclude_tags, - skip_inodes, restrict_dev, read_special, dry_run): + def _rec_walk( + self, + *, + path, + parent_fd, + name, + fso, + cache, + matcher, + exclude_caches, + exclude_if_present, + keep_exclude_tags, + skip_inodes, + restrict_dev, + read_special, + dry_run, + ): """ Process *path* (or, preferably, parent_fd/name) recursively according to the various parameters. @@ -1039,10 +1239,10 @@ def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher, try: recurse_excluded_dir = False if matcher.match(path): - with backup_io('stat'): + with backup_io("stat"): st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False) else: - self.print_file_status('x', path) + self.print_file_status("x", path) # get out here as quickly as possible: # we only need to continue if we shall recurse into an excluded directory. # if we shall not recurse, then do not even touch (stat()) the item, it @@ -1050,7 +1250,7 @@ def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher, if not matcher.recurse_dir: return recurse_excluded_dir = True - with backup_io('stat'): + with backup_io("stat"): st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False) if not stat.S_ISDIR(st.st_mode): return @@ -1064,22 +1264,31 @@ def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher, if self.exclude_nodump: # Ignore if nodump flag is set - with backup_io('flags'): + with backup_io("flags"): if get_flags(path=path, st=st) & stat.UF_NODUMP: - self.print_file_status('x', path) + self.print_file_status("x", path) return if not stat.S_ISDIR(st.st_mode): # directories cannot go in this branch because they can be excluded based on tag # files they might contain - status = self._process_any(path=path, parent_fd=parent_fd, name=name, st=st, fso=fso, cache=cache, - read_special=read_special, dry_run=dry_run) + status = self._process_any( + path=path, + parent_fd=parent_fd, + name=name, + st=st, + fso=fso, + cache=cache, + read_special=read_special, + dry_run=dry_run, + ) else: - with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir, - noatime=True, op='dir_open') as child_fd: + with OsOpen( + path=path, parent_fd=parent_fd, name=name, flags=flags_dir, noatime=True, op="dir_open" + ) as child_fd: # child_fd is None for directories on windows, in that case a race condition check is not possible. if child_fd is not None: - with backup_io('fstat'): + with backup_io("fstat"): st = stat_update_check(st, os.fstat(child_fd)) if recurse: tag_names = dir_is_tagged(path, exclude_caches, exclude_if_present) @@ -1093,43 +1302,67 @@ def _rec_walk(self, *, path, parent_fd, name, fso, cache, matcher, for tag_name in tag_names: tag_path = os.path.join(path, tag_name) self._rec_walk( - path=tag_path, parent_fd=child_fd, name=tag_name, fso=fso, cache=cache, - matcher=matcher, exclude_caches=exclude_caches, exclude_if_present=exclude_if_present, - keep_exclude_tags=keep_exclude_tags, skip_inodes=skip_inodes, - restrict_dev=restrict_dev, read_special=read_special, dry_run=dry_run) - self.print_file_status('x', path) + path=tag_path, + parent_fd=child_fd, + name=tag_name, + fso=fso, + cache=cache, + matcher=matcher, + exclude_caches=exclude_caches, + exclude_if_present=exclude_if_present, + keep_exclude_tags=keep_exclude_tags, + skip_inodes=skip_inodes, + restrict_dev=restrict_dev, + read_special=read_special, + dry_run=dry_run, + ) + self.print_file_status("x", path) return if not recurse_excluded_dir and not dry_run: status = fso.process_dir_with_fd(path=path, fd=child_fd, st=st) if recurse: - with backup_io('scandir'): + with backup_io("scandir"): entries = helpers.scandir_inorder(path=path, fd=child_fd) for dirent in entries: normpath = os.path.normpath(os.path.join(path, dirent.name)) self._rec_walk( - path=normpath, parent_fd=child_fd, name=dirent.name, fso=fso, cache=cache, matcher=matcher, - exclude_caches=exclude_caches, exclude_if_present=exclude_if_present, - keep_exclude_tags=keep_exclude_tags, skip_inodes=skip_inodes, restrict_dev=restrict_dev, - read_special=read_special, dry_run=dry_run) + path=normpath, + parent_fd=child_fd, + name=dirent.name, + fso=fso, + cache=cache, + matcher=matcher, + exclude_caches=exclude_caches, + exclude_if_present=exclude_if_present, + keep_exclude_tags=keep_exclude_tags, + skip_inodes=skip_inodes, + restrict_dev=restrict_dev, + read_special=read_special, + dry_run=dry_run, + ) except (BackupOSError, BackupError) as e: - self.print_warning('%s: %s', path, e) - status = 'E' - if status == 'C': - self.print_warning('%s: file changed while we backed it up', path) + self.print_warning("%s: %s", path, e) + status = "E" + if status == "C": + self.print_warning("%s: file changed while we backed it up", path) if not recurse_excluded_dir: self.print_file_status(status, path) @staticmethod def build_filter(matcher, strip_components): if strip_components: + def item_filter(item): matched = matcher.match(item.path) and os.sep.join(item.path.split(os.sep)[strip_components:]) return matched + else: + def item_filter(item): matched = matcher.match(item.path) return matched + return item_filter @with_repository(compatibility=(Manifest.Operation.READ,)) @@ -1137,10 +1370,14 @@ def item_filter(item): def do_extract(self, args, repository, manifest, key, archive): """Extract archive contents""" # be restrictive when restoring files, restore permissions later - if sys.getfilesystemencoding() == 'ascii': - logger.warning('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.') - if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): - logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') + if sys.getfilesystemencoding() == "ascii": + logger.warning( + 'Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.' + ) + if sys.platform.startswith(("linux", "freebsd", "netbsd", "openbsd", "darwin")): + logger.warning( + "Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8" + ) matcher = self.build_matcher(args.patterns, args.paths) @@ -1155,8 +1392,8 @@ def do_extract(self, args, repository, manifest, key, archive): filter = self.build_filter(matcher, strip_components) if progress: - pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1, msgid='extract') - pi.output('Calculating total archive size for the progress indicator (might take long for large archives)') + pi = ProgressIndicatorPercent(msg="%5.1f%% Extracting: %s", step=0.1, msgid="extract") + pi.output("Calculating total archive size for the progress indicator (might take long for large archives)") extracted_size = sum(item.get_size() for item in archive.iter_items(filter)) pi.total = extracted_size else: @@ -1172,9 +1409,9 @@ def do_extract(self, args, repository, manifest, key, archive): try: archive.extract_item(dir_item, stdout=stdout) except BackupOSError as e: - self.print_warning('%s: %s', remove_surrogates(dir_item.path), e) + self.print_warning("%s: %s", remove_surrogates(dir_item.path), e) if output_list: - logging.getLogger('borg.output.list').info(remove_surrogates(item.path)) + logging.getLogger("borg.output.list").info(remove_surrogates(item.path)) try: if dry_run: archive.extract_item(item, dry_run=True, hlm=hlm, pi=pi) @@ -1183,24 +1420,32 @@ def do_extract(self, args, repository, manifest, key, archive): dirs.append(item) archive.extract_item(item, stdout=stdout, restore_attrs=False) else: - archive.extract_item(item, stdout=stdout, sparse=sparse, hlm=hlm, - stripped_components=strip_components, original_path=orig_path, pi=pi) + archive.extract_item( + item, + stdout=stdout, + sparse=sparse, + hlm=hlm, + stripped_components=strip_components, + original_path=orig_path, + pi=pi, + ) except (BackupOSError, BackupError) as e: - self.print_warning('%s: %s', remove_surrogates(orig_path), e) + self.print_warning("%s: %s", remove_surrogates(orig_path), e) if pi: pi.finish() if not args.dry_run: - pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%', - msgid='extract.permissions') + pi = ProgressIndicatorPercent( + total=len(dirs), msg="Setting directory permissions %3.0f%%", msgid="extract.permissions" + ) while dirs: pi.show() dir_item = dirs.pop(-1) try: archive.extract_item(dir_item, stdout=stdout) except BackupOSError as e: - self.print_warning('%s: %s', remove_surrogates(dir_item.path), e) + self.print_warning("%s: %s", remove_surrogates(dir_item.path), e) for pattern in matcher.get_unmatched_include_patterns(): self.print_warning("Include pattern '%s' never matched.", pattern) if pi: @@ -1232,10 +1477,10 @@ def do_export_tar(self, args, repository, manifest, key, archive): # that it has to be installed -- hardly a problem, considering that # the decompressor must be installed as well to make use of the exported tarball! - filter = get_tar_filter(args.tarfile, decompress=False) if args.tar_filter == 'auto' else args.tar_filter + filter = get_tar_filter(args.tarfile, decompress=False) if args.tar_filter == "auto" else args.tar_filter - tarstream = dash_open(args.tarfile, 'wb') - tarstream_close = args.tarfile != '-' + tarstream = dash_open(args.tarfile, "wb") + tarstream_close = args.tarfile != "-" with create_filter_process(filter, stream=tarstream, stream_close=tarstream_close, inbound=False) as _stream: self._export_tar(args, archive, _stream) @@ -1255,11 +1500,11 @@ def _export_tar(self, args, archive, tarstream): # The | (pipe) symbol instructs tarfile to use a streaming mode of operation # where it never seeks on the passed fileobj. tar_format = dict(GNU=tarfile.GNU_FORMAT, PAX=tarfile.PAX_FORMAT, BORG=tarfile.PAX_FORMAT)[args.tar_format] - tar = tarfile.open(fileobj=tarstream, mode='w|', format=tar_format) + tar = tarfile.open(fileobj=tarstream, mode="w|", format=tar_format) if progress: - pi = ProgressIndicatorPercent(msg='%5.1f%% Processing: %s', step=0.1, msgid='extract') - pi.output('Calculating size') + pi = ProgressIndicatorPercent(msg="%5.1f%% Processing: %s", step=0.1, msgid="extract") + pi.output("Calculating size") extracted_size = sum(item.get_size() for item in archive.iter_items(filter)) pi.total = extracted_size else: @@ -1269,12 +1514,12 @@ def item_content_stream(item): """ Return a file-like object that reads from the chunks of *item*. """ - chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _ in item.chunks], - is_preloaded=True) + chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _ in item.chunks], is_preloaded=True) if pi: info = [remove_surrogates(item.path)] - return ChunkIteratorFileWrapper(chunk_iterator, - lambda read_bytes: pi.show(increase=len(read_bytes), info=info)) + return ChunkIteratorFileWrapper( + chunk_iterator, lambda read_bytes: pi.show(increase=len(read_bytes), info=info) + ) else: return ChunkIteratorFileWrapper(chunk_iterator) @@ -1293,8 +1538,8 @@ def item_to_tarinfo(item, original_path): tarinfo.mode = stat.S_IMODE(item.mode) tarinfo.uid = item.uid tarinfo.gid = item.gid - tarinfo.uname = item.get('user', '') - tarinfo.gname = item.get('group', '') + tarinfo.uname = item.get("user", "") + tarinfo.gname = item.get("group", "") # The linkname in tar has 2 uses: # for symlinks it means the destination, while for hardlinks it refers to the file. # Since hardlinks in tar have a different type code (LNKTYPE) the format might @@ -1305,7 +1550,7 @@ def item_to_tarinfo(item, original_path): modebits = stat.S_IFMT(item.mode) if modebits == stat.S_IFREG: tarinfo.type = tarfile.REGTYPE - if 'hlid' in item: + if "hlid" in item: linkname = hlm.retrieve(id=item.hlid) if linkname is not None: # the first hardlink was already added to the archive, add a tar-hardlink reference to it. @@ -1334,7 +1579,9 @@ def item_to_tarinfo(item, original_path): elif modebits == stat.S_IFIFO: tarinfo.type = tarfile.FIFOTYPE else: - self.print_warning('%s: unsupported file type %o for tar export', remove_surrogates(item.path), modebits) + self.print_warning( + "%s: unsupported file type %o for tar export", remove_surrogates(item.path), modebits + ) set_ec(EXIT_WARNING) return None, stream return tarinfo, stream @@ -1360,16 +1607,16 @@ def item_to_paxheaders(format, item): ph = {} # note: for mtime this is a bit redundant as it is already done by tarfile module, # but we just do it in our way to be consistent for sure. - for name in 'atime', 'ctime', 'mtime': + for name in "atime", "ctime", "mtime": if hasattr(item, name): ns = getattr(item, name) ph[name] = str(ns / 1e9) - if format == 'BORG': # BORG format additions - ph['BORG.item.version'] = '1' + if format == "BORG": # BORG format additions + ph["BORG.item.version"] = "1" # BORG.item.meta - just serialize all metadata we have: meta_bin = msgpack.packb(item.as_dict()) meta_text = base64.b64encode(meta_bin).decode() - ph['BORG.item.meta'] = meta_text + ph["BORG.item.meta"] = meta_text return ph for item in archive.iter_items(filter, preload=True): @@ -1378,10 +1625,10 @@ def item_to_paxheaders(format, item): item.path = os.sep.join(orig_path.split(os.sep)[strip_components:]) tarinfo, stream = item_to_tarinfo(item, orig_path) if tarinfo: - if args.tar_format in ('BORG', 'PAX'): + if args.tar_format in ("BORG", "PAX"): tarinfo.pax_headers = item_to_paxheaders(args.tar_format, item) if output_list: - logging.getLogger('borg.output.list').info(remove_surrogates(orig_path)) + logging.getLogger("borg.output.list").info(remove_surrogates(orig_path)) tar.addfile(tarinfo, stream) if pi: @@ -1403,20 +1650,23 @@ def print_json_output(diff, path): print(json.dumps({"path": path, "changes": [j for j, str in diff]})) def print_text_output(diff, path): - print("{:<19} {}".format(' '.join([str for j, str in diff]), path)) + print("{:<19} {}".format(" ".join([str for j, str in diff]), path)) print_output = print_json_output if args.json_lines else print_text_output archive1 = archive - archive2 = Archive(repository, key, manifest, args.other_name, - consider_part_files=args.consider_part_files) + archive2 = Archive(repository, key, manifest, args.other_name, consider_part_files=args.consider_part_files) - can_compare_chunk_ids = archive1.metadata.get('chunker_params', False) == archive2.metadata.get( - 'chunker_params', True) or args.same_chunker_params + can_compare_chunk_ids = ( + archive1.metadata.get("chunker_params", False) == archive2.metadata.get("chunker_params", True) + or args.same_chunker_params + ) if not can_compare_chunk_ids: - self.print_warning('--chunker-params might be different between archives, diff will be slow.\n' - 'If you know for certain that they are the same, pass --same-chunker-params ' - 'to override this check.') + self.print_warning( + "--chunker-params might be different between archives, diff will be slow.\n" + "If you know for certain that they are the same, pass --same-chunker-params " + "to override this check." + ) matcher = self.build_matcher(args.patterns, args.paths) @@ -1460,12 +1710,16 @@ def do_rdelete(self, args, repository): try: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) n_archives = len(manifest.archives) - msg.append(f"You requested to completely DELETE the following repository " - f"*including* {n_archives} archives it contains:") + msg.append( + f"You requested to completely DELETE the following repository " + f"*including* {n_archives} archives it contains:" + ) except NoManifestError: n_archives = None - msg.append("You requested to completely DELETE the following repository " - "*including* all archives it may contain:") + msg.append( + "You requested to completely DELETE the following repository " + "*including* all archives it may contain:" + ) msg.append(DASHES) msg.append(f"Repository ID: {id}") @@ -1477,19 +1731,27 @@ def do_rdelete(self, args, repository): if n_archives is not None: if n_archives > 0: - for archive_info in manifest.archives.list(sort_by=['ts']): + for archive_info in manifest.archives.list(sort_by=["ts"]): msg.append(format_archive(archive_info)) else: msg.append("This repository seems to not have any archives.") else: - msg.append("This repository seems to have no manifest, so we can't " - "tell anything about its contents.") + msg.append( + "This repository seems to have no manifest, so we can't " + "tell anything about its contents." + ) msg.append(DASHES) msg.append("Type 'YES' if you understand this and want to continue: ") - msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", invalid_msg='Invalid answer, aborting.', truish=('YES',), - retry=False, env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): + msg = "\n".join(msg) + if not yes( + msg, + false_msg="Aborting.", + invalid_msg="Invalid answer, aborting.", + truish=("YES",), + retry=False, + env_var_override="BORG_DELETE_I_KNOW_WHAT_I_AM_DOING", + ): self.exit_code = EXIT_ERROR return self.exit_code if not dry_run: @@ -1517,41 +1779,42 @@ def do_delete(self, args, repository): if not archive_names: return self.exit_code if args.glob_archives is None and args.first == 0 and args.last == 0: - self.print_error("Aborting: if you really want to delete all archives, please use -a '*' " - "or just delete the whole repository (might be much faster).") + self.print_error( + "Aborting: if you really want to delete all archives, please use -a '*' " + "or just delete the whole repository (might be much faster)." + ) return EXIT_ERROR if args.forced == 2: deleted = False - logger_list = logging.getLogger('borg.output.list') + logger_list = logging.getLogger("borg.output.list") for i, archive_name in enumerate(archive_names, 1): try: current_archive = manifest.archives.pop(archive_name) except KeyError: self.exit_code = EXIT_WARNING - logger.warning(f'Archive {archive_name} not found ({i}/{len(archive_names)}).') + logger.warning(f"Archive {archive_name} not found ({i}/{len(archive_names)}).") else: deleted = True if self.output_list: - msg = 'Would delete: {} ({}/{})' if dry_run else 'Deleted archive: {} ({}/{})' - logger_list.info(msg.format(format_archive(current_archive), - i, len(archive_names))) + msg = "Would delete: {} ({}/{})" if dry_run else "Deleted archive: {} ({}/{})" + logger_list.info(msg.format(format_archive(current_archive), i, len(archive_names))) if dry_run: - logger.info('Finished dry-run.') + logger.info("Finished dry-run.") elif deleted: manifest.write() # note: might crash in compact() after committing the repo repository.commit(compact=False) logger.warning('Done. Run "borg check --repair" to clean up the mess.') else: - logger.warning('Aborted.') + logger.warning("Aborted.") return self.exit_code stats = Statistics(iec=args.iec) with Cache(repository, key, manifest, progress=args.progress, lock_wait=self.lock_wait, iec=args.iec) as cache: - msg_delete = 'Would delete archive: {} ({}/{})' if dry_run else 'Deleting archive: {} ({}/{})' - msg_not_found = 'Archive {} not found ({}/{}).' - logger_list = logging.getLogger('borg.output.list') + msg_delete = "Would delete archive: {} ({}/{})" if dry_run else "Deleting archive: {} ({}/{})" + msg_not_found = "Archive {} not found ({}/{})." + logger_list = logging.getLogger("borg.output.list") delete_count = 0 for i, archive_name in enumerate(archive_names, 1): try: @@ -1563,8 +1826,14 @@ def do_delete(self, args, repository): logger_list.info(msg_delete.format(format_archive(archive_info), i, len(archive_names))) if not dry_run: - archive = Archive(repository, key, manifest, archive_name, cache=cache, - consider_part_files=args.consider_part_files) + archive = Archive( + repository, + key, + manifest, + archive_name, + cache=cache, + consider_part_files=args.consider_part_files, + ) archive.delete(stats, progress=args.progress, forced=args.forced) delete_count += 1 if delete_count > 0: @@ -1573,8 +1842,7 @@ def do_delete(self, args, repository): repository.commit(compact=False, save_space=args.save_space) cache.commit() if args.stats: - log_multi(str(stats), - logger=logging.getLogger('borg.output.stats')) + log_multi(str(stats), logger=logging.getLogger("borg.output.stats")) return self.exit_code @@ -1583,12 +1851,13 @@ def do_mount(self, args): # Perform these checks before opening the repository and asking for a passphrase. from .fuse_impl import llfuse, BORG_FUSE_IMPL + if llfuse is None: - self.print_error('borg mount not available: no FUSE support, BORG_FUSE_IMPL=%s.' % BORG_FUSE_IMPL) + self.print_error("borg mount not available: no FUSE support, BORG_FUSE_IMPL=%s." % BORG_FUSE_IMPL) return self.exit_code if not os.path.isdir(args.mountpoint) or not os.access(args.mountpoint, os.R_OK | os.W_OK | os.X_OK): - self.print_error('%s: Mountpoint must be a writable directory' % args.mountpoint) + self.print_error("%s: Mountpoint must be a writable directory" % args.mountpoint) return self.exit_code return self._do_mount(args) @@ -1623,8 +1892,9 @@ def do_list(self, args, repository, manifest, key): format = "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}" def _list_inner(cache): - archive = Archive(repository, key, manifest, args.name, cache=cache, - consider_part_files=args.consider_part_files) + archive = Archive( + repository, key, manifest, args.name, cache=cache, consider_part_files=args.consider_part_files + ) formatter = ItemFormatter(archive, format, json_lines=args.json_lines) for item in archive.iter_items(lambda item: matcher.match(item.path)): @@ -1659,32 +1929,30 @@ def do_rlist(self, args, repository, manifest, key): sys.stdout.write(formatter.format_item(archive_info)) if args.json: - json_print(basic_json_data(manifest, extra={ - 'archives': output_data - })) + json_print(basic_json_data(manifest, extra={"archives": output_data})) return self.exit_code @with_repository(cache=True, compatibility=(Manifest.Operation.READ,)) def do_rinfo(self, args, repository, manifest, key, cache): """Show repository infos""" - info = basic_json_data(manifest, cache=cache, extra={ - 'security_dir': cache.security_manager.dir, - }) + info = basic_json_data(manifest, cache=cache, extra={"security_dir": cache.security_manager.dir}) if args.json: json_print(info) else: - encryption = 'Encrypted: ' - if key.NAME in ('plaintext', 'authenticated'): - encryption += 'No' + encryption = "Encrypted: " + if key.NAME in ("plaintext", "authenticated"): + encryption += "No" else: - encryption += 'Yes (%s)' % key.NAME - if key.NAME.startswith('key file'): - encryption += '\nKey file: %s' % key.find_key() - info['encryption'] = encryption + encryption += "Yes (%s)" % key.NAME + if key.NAME.startswith("key file"): + encryption += "\nKey file: %s" % key.find_key() + info["encryption"] = encryption - print(textwrap.dedent(""" + print( + textwrap.dedent( + """ Repository ID: {id} Location: {location} Repository version: {version} @@ -1692,20 +1960,26 @@ def do_rinfo(self, args, repository, manifest, key, cache): {encryption} Cache: {cache.path} Security dir: {security_dir} - """).strip().format( - id=bin_to_hex(repository.id), - location=repository._location.canonical_path(), - version=repository.version, - append_only=repository.append_only, - **info)) + """ + ) + .strip() + .format( + id=bin_to_hex(repository.id), + location=repository._location.canonical_path(), + version=repository.version, + append_only=repository.append_only, + **info, + ) + ) print(str(cache)) return self.exit_code @with_repository(cache=True, compatibility=(Manifest.Operation.READ,)) def do_info(self, args, repository, manifest, key, cache): """Show archive details such as disk space used""" + def format_cmdline(cmdline): - return remove_surrogates(' '.join(shlex.quote(x) for x in cmdline)) + return remove_surrogates(" ".join(shlex.quote(x) for x in cmdline)) args.consider_checkpoints = True archive_names = tuple(x.name for x in manifest.archives.list_considering(args)) @@ -1713,15 +1987,24 @@ def format_cmdline(cmdline): output_data = [] for i, archive_name in enumerate(archive_names, 1): - archive = Archive(repository, key, manifest, archive_name, cache=cache, - consider_part_files=args.consider_part_files, iec=args.iec) + archive = Archive( + repository, + key, + manifest, + archive_name, + cache=cache, + consider_part_files=args.consider_part_files, + iec=args.iec, + ) info = archive.info() if args.json: output_data.append(info) else: - info['duration'] = format_timedelta(timedelta(seconds=info['duration'])) - info['command_line'] = format_cmdline(info['command_line']) - print(textwrap.dedent(""" + info["duration"] = format_timedelta(timedelta(seconds=info["duration"])) + info["command_line"] = format_cmdline(info["command_line"]) + print( + textwrap.dedent( + """ Archive name: {name} Archive fingerprint: {id} Comment: {comment} @@ -1735,33 +2018,41 @@ def format_cmdline(cmdline): Number of files: {stats[nfiles]} Original size: {stats[original_size]} Deduplicated size: {stats[deduplicated_size]} - """).strip().format(**info)) + """ + ) + .strip() + .format(**info) + ) if self.exit_code: break if not args.json and len(archive_names) - i: print() if args.json: - json_print(basic_json_data(manifest, cache=cache, extra={ - 'archives': output_data, - })) + json_print(basic_json_data(manifest, cache=cache, extra={"archives": output_data})) return self.exit_code @with_repository(exclusive=True, compatibility=(Manifest.Operation.DELETE,)) def do_prune(self, args, repository, manifest, key): """Prune repository archives according to specified rules""" - if not any((args.secondly, args.minutely, args.hourly, args.daily, - args.weekly, args.monthly, args.yearly, args.within)): - self.print_error('At least one of the "keep-within", "keep-last", ' - '"keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", ' - '"keep-weekly", "keep-monthly" or "keep-yearly" settings must be specified.') + if not any( + (args.secondly, args.minutely, args.hourly, args.daily, args.weekly, args.monthly, args.yearly, args.within) + ): + self.print_error( + 'At least one of the "keep-within", "keep-last", ' + '"keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", ' + '"keep-weekly", "keep-monthly" or "keep-yearly" settings must be specified.' + ) return self.exit_code - checkpoint_re = r'\.checkpoint(\.\d+)?' - archives_checkpoints = manifest.archives.list(glob=args.glob_archives, - consider_checkpoints=True, - match_end=r'(%s)?\Z' % checkpoint_re, - sort_by=['ts'], reverse=True) - is_checkpoint = re.compile(r'(%s)\Z' % checkpoint_re).search + checkpoint_re = r"\.checkpoint(\.\d+)?" + archives_checkpoints = manifest.archives.list( + glob=args.glob_archives, + consider_checkpoints=True, + match_end=r"(%s)?\Z" % checkpoint_re, + sort_by=["ts"], + reverse=True, + ) + is_checkpoint = re.compile(r"(%s)\Z" % checkpoint_re).search checkpoints = [arch for arch in archives_checkpoints if is_checkpoint(arch.name)] # keep the latest checkpoint, if there is no later non-checkpoint archive if archives_checkpoints and checkpoints and archives_checkpoints[0] is checkpoints[0]: @@ -1791,41 +2082,41 @@ def do_prune(self, args, repository, manifest, key): to_delete = (set(archives) | checkpoints) - (set(keep) | set(keep_checkpoints)) stats = Statistics(iec=args.iec) with Cache(repository, key, manifest, lock_wait=self.lock_wait, iec=args.iec) as cache: - list_logger = logging.getLogger('borg.output.list') + list_logger = logging.getLogger("borg.output.list") # set up counters for the progress display to_delete_len = len(to_delete) archives_deleted = 0 - pi = ProgressIndicatorPercent(total=len(to_delete), msg='Pruning archives %3.0f%%', msgid='prune') + pi = ProgressIndicatorPercent(total=len(to_delete), msg="Pruning archives %3.0f%%", msgid="prune") for archive in archives_checkpoints: if archive in to_delete: pi.show() if args.dry_run: - log_message = 'Would prune:' + log_message = "Would prune:" else: archives_deleted += 1 - log_message = 'Pruning archive (%d/%d):' % (archives_deleted, to_delete_len) - archive = Archive(repository, key, manifest, archive.name, cache, - consider_part_files=args.consider_part_files) + log_message = "Pruning archive (%d/%d):" % (archives_deleted, to_delete_len) + archive = Archive( + repository, key, manifest, archive.name, cache, consider_part_files=args.consider_part_files + ) archive.delete(stats, forced=args.forced) else: if is_checkpoint(archive.name): - log_message = 'Keeping checkpoint archive:' + log_message = "Keeping checkpoint archive:" else: - log_message = 'Keeping archive (rule: {rule} #{num}):'.format( + log_message = "Keeping archive (rule: {rule} #{num}):".format( rule=kept_because[archive.id][0], num=kept_because[archive.id][1] ) if args.output_list: - list_logger.info("{message:<40} {archive}".format( - message=log_message, archive=format_archive(archive) - )) + list_logger.info( + "{message:<40} {archive}".format(message=log_message, archive=format_archive(archive)) + ) pi.finish() if to_delete and not args.dry_run: manifest.write() repository.commit(compact=False, save_space=args.save_space) cache.commit() if args.stats: - log_multi(str(stats), - logger=logging.getLogger('borg.output.stats')) + log_multi(str(stats), logger=logging.getLogger("borg.output.stats")) return self.exit_code @with_repository(cache=True, exclusive=True, compatibility=(Manifest.Operation.CHECK,)) @@ -1834,28 +2125,40 @@ def do_recreate(self, args, repository, manifest, key, cache): matcher = self.build_matcher(args.patterns, args.paths) self.output_list = args.output_list self.output_filter = args.output_filter - recompress = args.recompress != 'never' - always_recompress = args.recompress == 'always' + recompress = args.recompress != "never" + always_recompress = args.recompress == "always" - recreater = ArchiveRecreater(repository, manifest, key, cache, matcher, - exclude_caches=args.exclude_caches, exclude_if_present=args.exclude_if_present, - keep_exclude_tags=args.keep_exclude_tags, chunker_params=args.chunker_params, - compression=args.compression, recompress=recompress, always_recompress=always_recompress, - progress=args.progress, stats=args.stats, - file_status_printer=self.print_file_status, - checkpoint_interval=args.checkpoint_interval, - dry_run=args.dry_run, timestamp=args.timestamp) + recreater = ArchiveRecreater( + repository, + manifest, + key, + cache, + matcher, + exclude_caches=args.exclude_caches, + exclude_if_present=args.exclude_if_present, + keep_exclude_tags=args.keep_exclude_tags, + chunker_params=args.chunker_params, + compression=args.compression, + recompress=recompress, + always_recompress=always_recompress, + progress=args.progress, + stats=args.stats, + file_status_printer=self.print_file_status, + checkpoint_interval=args.checkpoint_interval, + dry_run=args.dry_run, + timestamp=args.timestamp, + ) archive_names = tuple(archive.name for archive in manifest.archives.list_considering(args)) if args.target is not None and len(archive_names) != 1: - self.print_error('--target: Need to specify single archive') + self.print_error("--target: Need to specify single archive") return self.exit_code for name in archive_names: if recreater.is_temporary_archive(name): continue - print('Processing', name) + print("Processing", name) if not recreater.recreate(name, args.comment, args.target): - logger.info('Skipped archive %s: Nothing to do. Archive was not processed.', name) + logger.info("Skipped archive %s: Nothing to do. Archive was not processed.", name) if not args.dry_run: manifest.write() repository.commit(compact=False) @@ -1868,10 +2171,10 @@ def do_import_tar(self, args, repository, manifest, key, cache): self.output_filter = args.output_filter self.output_list = args.output_list - filter = get_tar_filter(args.tarfile, decompress=True) if args.tar_filter == 'auto' else args.tar_filter + filter = get_tar_filter(args.tarfile, decompress=True) if args.tar_filter == "auto" else args.tar_filter - tarstream = dash_open(args.tarfile, 'rb') - tarstream_close = args.tarfile != '-' + tarstream = dash_open(args.tarfile, "rb") + tarstream_close = args.tarfile != "-" with create_filter_process(filter, stream=tarstream, stream_close=tarstream_close, inbound=True) as _stream: self._import_tar(args, repository, manifest, key, cache, _stream) @@ -1882,46 +2185,66 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): t0 = datetime.utcnow() t0_monotonic = time.monotonic() - archive = Archive(repository, key, manifest, args.name, cache=cache, - create=True, checkpoint_interval=args.checkpoint_interval, - progress=args.progress, - chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic, - log_json=args.log_json) - cp = ChunksProcessor(cache=cache, key=key, - add_item=archive.add_item, write_checkpoint=archive.write_checkpoint, - checkpoint_interval=args.checkpoint_interval, rechunkify=False) - tfo = TarfileObjectProcessors(cache=cache, key=key, - process_file_chunks=cp.process_file_chunks, add_item=archive.add_item, - chunker_params=args.chunker_params, show_progress=args.progress, - log_json=args.log_json, iec=args.iec, - file_status_printer=self.print_file_status) + archive = Archive( + repository, + key, + manifest, + args.name, + cache=cache, + create=True, + checkpoint_interval=args.checkpoint_interval, + progress=args.progress, + chunker_params=args.chunker_params, + start=t0, + start_monotonic=t0_monotonic, + log_json=args.log_json, + ) + cp = ChunksProcessor( + cache=cache, + key=key, + add_item=archive.add_item, + write_checkpoint=archive.write_checkpoint, + checkpoint_interval=args.checkpoint_interval, + rechunkify=False, + ) + tfo = TarfileObjectProcessors( + cache=cache, + key=key, + process_file_chunks=cp.process_file_chunks, + add_item=archive.add_item, + chunker_params=args.chunker_params, + show_progress=args.progress, + log_json=args.log_json, + iec=args.iec, + file_status_printer=self.print_file_status, + ) - tar = tarfile.open(fileobj=tarstream, mode='r|') + tar = tarfile.open(fileobj=tarstream, mode="r|") while True: tarinfo = tar.next() if not tarinfo: break if tarinfo.isreg(): - status = tfo.process_file(tarinfo=tarinfo, status='A', type=stat.S_IFREG, tar=tar) + status = tfo.process_file(tarinfo=tarinfo, status="A", type=stat.S_IFREG, tar=tar) archive.stats.nfiles += 1 elif tarinfo.isdir(): - status = tfo.process_dir(tarinfo=tarinfo, status='d', type=stat.S_IFDIR) + status = tfo.process_dir(tarinfo=tarinfo, status="d", type=stat.S_IFDIR) elif tarinfo.issym(): - status = tfo.process_symlink(tarinfo=tarinfo, status='s', type=stat.S_IFLNK) + status = tfo.process_symlink(tarinfo=tarinfo, status="s", type=stat.S_IFLNK) elif tarinfo.islnk(): # tar uses a hardlink model like: the first instance of a hardlink is stored as a regular file, # later instances are special entries referencing back to the first instance. - status = tfo.process_hardlink(tarinfo=tarinfo, status='h', type=stat.S_IFREG) + status = tfo.process_hardlink(tarinfo=tarinfo, status="h", type=stat.S_IFREG) elif tarinfo.isblk(): - status = tfo.process_dev(tarinfo=tarinfo, status='b', type=stat.S_IFBLK) + status = tfo.process_dev(tarinfo=tarinfo, status="b", type=stat.S_IFBLK) elif tarinfo.ischr(): - status = tfo.process_dev(tarinfo=tarinfo, status='c', type=stat.S_IFCHR) + status = tfo.process_dev(tarinfo=tarinfo, status="c", type=stat.S_IFCHR) elif tarinfo.isfifo(): - status = tfo.process_fifo(tarinfo=tarinfo, status='f', type=stat.S_IFIFO) + status = tfo.process_fifo(tarinfo=tarinfo, status="f", type=stat.S_IFIFO) else: - status = 'E' - self.print_warning('%s: Unsupported tarinfo type %s', tarinfo.name, tarinfo.type) + status = "E" + self.print_warning("%s: Unsupported tarinfo type %s", tarinfo.name, tarinfo.type) self.print_file_status(status, tarinfo.name) # This does not close the fileobj (tarstream) we passed to it -- a side effect of the | mode. @@ -1934,13 +2257,9 @@ def _import_tar(self, args, repository, manifest, key, cache, tarstream): args.stats |= args.json if args.stats: if args.json: - json_print(basic_json_data(archive.manifest, cache=archive.cache, extra={ - 'archive': archive, - })) + json_print(basic_json_data(archive.manifest, cache=archive.cache, extra={"archive": archive})) else: - log_multi(str(archive), - str(archive.stats), - logger=logging.getLogger('borg.output.stats')) + log_multi(str(archive), str(archive.stats), logger=logging.getLogger("borg.output.stats")) @with_repository(manifest=False, exclusive=True) def do_with_lock(self, args, repository): @@ -1957,7 +2276,7 @@ def do_with_lock(self, args, repository): # usually, a 0 byte (open for writing) segment file would be visible in the filesystem here. # we write and close this file, to rather have a valid segment file on disk, before invoking the subprocess. # we can only do this for local repositories (with .io), though: - if hasattr(repository, 'io'): + if hasattr(repository, "io"): repository.io.close_segment() env = prepare_subprocess_env(system=True) try: @@ -1987,81 +2306,87 @@ def do_config(self, args, repository): """get, set, and delete values in a repository or cache config file""" def repo_validate(section, name, value=None, check_value=True): - if section not in ['repository', ]: - raise ValueError('Invalid section') - if name in ['segments_per_dir', 'last_segment_checked', ]: + if section not in ["repository"]: + raise ValueError("Invalid section") + if name in ["segments_per_dir", "last_segment_checked"]: if check_value: try: int(value) except ValueError: - raise ValueError('Invalid value') from None - elif name in ['max_segment_size', 'additional_free_space', 'storage_quota', ]: + raise ValueError("Invalid value") from None + elif name in ["max_segment_size", "additional_free_space", "storage_quota"]: if check_value: try: parse_file_size(value) except ValueError: - raise ValueError('Invalid value') from None - if name == 'storage_quota': - if parse_file_size(value) < parse_file_size('10M'): - raise ValueError('Invalid value: storage_quota < 10M') - elif name == 'max_segment_size': + raise ValueError("Invalid value") from None + if name == "storage_quota": + if parse_file_size(value) < parse_file_size("10M"): + raise ValueError("Invalid value: storage_quota < 10M") + elif name == "max_segment_size": if parse_file_size(value) >= MAX_SEGMENT_SIZE_LIMIT: - raise ValueError('Invalid value: max_segment_size >= %d' % MAX_SEGMENT_SIZE_LIMIT) - elif name in ['append_only', ]: - if check_value and value not in ['0', '1']: - raise ValueError('Invalid value') - elif name in ['id', ]: + raise ValueError("Invalid value: max_segment_size >= %d" % MAX_SEGMENT_SIZE_LIMIT) + elif name in ["append_only"]: + if check_value and value not in ["0", "1"]: + raise ValueError("Invalid value") + elif name in ["id"]: if check_value: try: bin_id = unhexlify(value) except: - raise ValueError('Invalid value, must be 64 hex digits') from None + raise ValueError("Invalid value, must be 64 hex digits") from None if len(bin_id) != 32: - raise ValueError('Invalid value, must be 64 hex digits') + raise ValueError("Invalid value, must be 64 hex digits") else: - raise ValueError('Invalid name') + raise ValueError("Invalid name") def cache_validate(section, name, value=None, check_value=True): - if section not in ['cache', ]: - raise ValueError('Invalid section') - if name in ['previous_location', ]: + if section not in ["cache"]: + raise ValueError("Invalid section") + if name in ["previous_location"]: if check_value: Location(value) else: - raise ValueError('Invalid name') + raise ValueError("Invalid name") def list_config(config): default_values = { - 'version': '1', - 'segments_per_dir': str(DEFAULT_SEGMENTS_PER_DIR), - 'max_segment_size': str(MAX_SEGMENT_SIZE_LIMIT), - 'additional_free_space': '0', - 'storage_quota': repository.storage_quota, - 'append_only': repository.append_only + "version": "1", + "segments_per_dir": str(DEFAULT_SEGMENTS_PER_DIR), + "max_segment_size": str(MAX_SEGMENT_SIZE_LIMIT), + "additional_free_space": "0", + "storage_quota": repository.storage_quota, + "append_only": repository.append_only, } - print('[repository]') - for key in ['version', 'segments_per_dir', 'max_segment_size', - 'storage_quota', 'additional_free_space', 'append_only', - 'id']: - value = config.get('repository', key, fallback=False) + print("[repository]") + for key in [ + "version", + "segments_per_dir", + "max_segment_size", + "storage_quota", + "additional_free_space", + "append_only", + "id", + ]: + value = config.get("repository", key, fallback=False) if value is None: value = default_values.get(key) if value is None: - raise Error('The repository config is missing the %s key which has no default value' % key) - print(f'{key} = {value}') - for key in ['last_segment_checked', ]: - value = config.get('repository', key, fallback=None) + raise Error("The repository config is missing the %s key which has no default value" % key) + print(f"{key} = {value}") + for key in ["last_segment_checked"]: + value = config.get("repository", key, fallback=None) if value is None: continue - print(f'{key} = {value}') + print(f"{key} = {value}") if not args.list: if args.name is None: - self.print_error('No config key name was provided.') + self.print_error("No config key name was provided.") return self.exit_code try: - section, name = args.name.split('.') + section, name = args.name.split(".") except ValueError: section = args.cache and "cache" or "repository" name = args.name @@ -2112,22 +2437,21 @@ def do_debug_info(self, args): print(sysinfo()) # Additional debug information - print('CRC implementation:', crc32.__name__) - print('Process ID:', get_process_id()) + print("CRC implementation:", crc32.__name__) + print("Process ID:", get_process_id()) return EXIT_SUCCESS @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) def do_debug_dump_archive_items(self, args, repository, manifest, key): """dump (decrypted, decompressed) archive items metadata (not: data)""" - archive = Archive(repository, key, manifest, args.name, - consider_part_files=args.consider_part_files) + archive = Archive(repository, key, manifest, args.name, consider_part_files=args.consider_part_files) for i, item_id in enumerate(archive.metadata.items): data = key.decrypt(item_id, repository.get(item_id)) - filename = '%06d_%s.items' % (i, bin_to_hex(item_id)) - print('Dumping', filename) - with open(filename, 'wb') as fd: + filename = "%06d_%s.items" % (i, bin_to_hex(item_id)) + print("Dumping", filename) + with open(filename, "wb") as fd: fd.write(data) - print('Done.') + print("Done.") return EXIT_SUCCESS @with_repository(compatibility=Manifest.NO_OPERATION_CHECK) @@ -2142,27 +2466,27 @@ def do_debug_dump_archive(self, args, repository, manifest, key): indent = 4 def do_indent(d): - return textwrap.indent(json.dumps(d, indent=indent), prefix=' ' * indent) + return textwrap.indent(json.dumps(d, indent=indent), prefix=" " * indent) def output(fd): # this outputs megabytes of data for a modest sized archive, so some manual streaming json output - fd.write('{\n') + fd.write("{\n") fd.write(' "_name": ' + json.dumps(args.name) + ",\n") fd.write(' "_manifest_entry":\n') fd.write(do_indent(prepare_dump_dict(archive_meta_orig))) - fd.write(',\n') + fd.write(",\n") - data = key.decrypt(archive_meta_orig['id'], repository.get(archive_meta_orig['id'])) + data = key.decrypt(archive_meta_orig["id"], repository.get(archive_meta_orig["id"])) archive_org_dict = msgpack.unpackb(data, object_hook=StableDict) fd.write(' "_meta":\n') fd.write(do_indent(prepare_dump_dict(archive_org_dict))) - fd.write(',\n') + fd.write(",\n") fd.write(' "_items": [\n') unpacker = msgpack.Unpacker(use_list=False, object_hook=StableDict) first = True - for item_id in archive_org_dict['items']: + for item_id in archive_org_dict["items"]: data = key.decrypt(item_id, repository.get(item_id)) unpacker.feed(data) for item in unpacker: @@ -2170,13 +2494,13 @@ def output(fd): if first: first = False else: - fd.write(',\n') + fd.write(",\n") fd.write(do_indent(item)) - fd.write('\n') - fd.write(' ]\n}\n') + fd.write("\n") + fd.write(" ]\n}\n") - with dash_open(args.path, 'w') as fd: + with dash_open(args.path, "w") as fd: output(fd) return EXIT_SUCCESS @@ -2188,7 +2512,7 @@ def do_debug_dump_manifest(self, args, repository, manifest, key): meta = prepare_dump_dict(msgpack.unpackb(data, object_hook=StableDict)) - with dash_open(args.path, 'w') as fd: + with dash_open(args.path, "w") as fd: json.dump(meta, fd, indent=4) return EXIT_SUCCESS @@ -2201,14 +2525,14 @@ def decrypt_dump(i, id, cdata, tag=None, segment=None, offset=None): if cdata is not None: data = key.decrypt(id, cdata) else: - data = b'' - tag_str = '' if tag is None else '_' + tag - segment_str = '_' + str(segment) if segment is not None else '' - offset_str = '_' + str(offset) if offset is not None else '' - id_str = '_' + bin_to_hex(id) if id is not None else '' - filename = '%08d%s%s%s%s.obj' % (i, segment_str, offset_str, tag_str, id_str) - print('Dumping', filename) - with open(filename, 'wb') as fd: + data = b"" + tag_str = "" if tag is None else "_" + tag + segment_str = "_" + str(segment) if segment is not None else "" + offset_str = "_" + str(offset) if offset is not None else "" + id_str = "_" + bin_to_hex(id) if id is not None else "" + filename = "%08d%s%s%s%s.obj" % (i, segment_str, offset_str, tag_str, id_str) + print("Dumping", filename) + with open(filename, "wb") as fd: fd.write(data) if args.ghost: @@ -2222,11 +2546,11 @@ def decrypt_dump(i, id, cdata, tag=None, segment=None, offset=None): i = 0 for id, cdata, tag, segment, offset in repository.scan_low_level(segment=args.segment, offset=args.offset): if tag == TAG_PUT: - decrypt_dump(i, id, cdata, tag='put', segment=segment, offset=offset) + decrypt_dump(i, id, cdata, tag="put", segment=segment, offset=offset) elif tag == TAG_DELETE: - decrypt_dump(i, id, None, tag='del', segment=segment, offset=offset) + decrypt_dump(i, id, None, tag="del", segment=segment, offset=offset) elif tag == TAG_COMMIT: - decrypt_dump(i, None, None, tag='commit', segment=segment, offset=offset) + decrypt_dump(i, None, None, tag="commit", segment=segment, offset=offset) i += 1 else: # set up the key without depending on a manifest obj @@ -2244,7 +2568,7 @@ def decrypt_dump(i, id, cdata, tag=None, segment=None, offset=None): cdata = repository.get(id) decrypt_dump(i, id, cdata) i += 1 - print('Done.') + print("Done.") return EXIT_SUCCESS @with_repository(manifest=False) @@ -2253,33 +2577,37 @@ def do_debug_search_repo_objs(self, args, repository): context = 32 def print_finding(info, wanted, data, offset): - before = data[offset - context:offset] - after = data[offset + len(wanted):offset + len(wanted) + context] - print('{}: {} {} {} == {!r} {!r} {!r}'.format(info, before.hex(), wanted.hex(), after.hex(), - before, wanted, after)) + before = data[offset - context : offset] + after = data[offset + len(wanted) : offset + len(wanted) + context] + print( + "{}: {} {} {} == {!r} {!r} {!r}".format( + info, before.hex(), wanted.hex(), after.hex(), before, wanted, after + ) + ) wanted = args.wanted try: - if wanted.startswith('hex:'): + if wanted.startswith("hex:"): wanted = unhexlify(wanted[4:]) - elif wanted.startswith('str:'): + elif wanted.startswith("str:"): wanted = wanted[4:].encode() else: - raise ValueError('unsupported search term') + raise ValueError("unsupported search term") except (ValueError, UnicodeEncodeError): wanted = None if not wanted: - self.print_error('search term needs to be hex:123abc or str:foobar style') + self.print_error("search term needs to be hex:123abc or str:foobar style") return EXIT_ERROR from .crypto.key import key_factory + # set up the key without depending on a manifest obj ids = repository.list(limit=1, marker=None) cdata = repository.get(ids[0]) key = key_factory(repository, cdata) marker = None - last_data = b'' + last_data = b"" last_id = None i = 0 while True: @@ -2292,11 +2620,11 @@ def print_finding(info, wanted, data, offset): data = key.decrypt(id, cdata) # try to locate wanted sequence crossing the border of last_data and data - boundary_data = last_data[-(len(wanted) - 1):] + data[:len(wanted) - 1] + boundary_data = last_data[-(len(wanted) - 1) :] + data[: len(wanted) - 1] if wanted in boundary_data: - boundary_data = last_data[-(len(wanted) - 1 + context):] + data[:len(wanted) - 1 + context] + boundary_data = last_data[-(len(wanted) - 1 + context) :] + data[: len(wanted) - 1 + context] offset = boundary_data.find(wanted) - info = '%d %s | %s' % (i, last_id.hex(), id.hex()) + info = "%d %s | %s" % (i, last_id.hex(), id.hex()) print_finding(info, wanted, boundary_data, offset) # try to locate wanted sequence in data @@ -2309,8 +2637,8 @@ def print_finding(info, wanted, data, offset): last_id, last_data = id, data i += 1 if i % 10000 == 0: - print('%d objects processed.' % i) - print('Done.') + print("%d objects processed." % i) + print("Done.") return EXIT_SUCCESS @with_repository(manifest=False) @@ -2362,7 +2690,7 @@ def do_debug_delete_obj(self, args, repository): print("object %s not found." % hex_id) if modified: repository.commit(compact=False) - print('Done.') + print("Done.") return EXIT_SUCCESS @with_repository(manifest=False, exclusive=True, cache=True, compatibility=Manifest.NO_OPERATION_CHECK) @@ -2391,9 +2719,9 @@ def do_debug_dump_hints(self, args, repository): segments=repository.segments, compact=repository.compact, storage_quota_use=repository.storage_quota_use, - shadow_index={hexlify(k).decode(): v for k, v in repository.shadow_index.items()} + shadow_index={hexlify(k).decode(): v for k, v in repository.shadow_index.items()}, ) - with dash_open(args.path, 'w') as fd: + with dash_open(args.path, "w") as fd: json.dump(hints, fd, indent=4) finally: repository.rollback() @@ -2402,6 +2730,7 @@ def do_debug_dump_hints(self, args, repository): def do_debug_convert_profile(self, args): """convert Borg profile to Python profile""" import marshal + with args.output, args.input: marshal.dump(msgpack.unpack(args.input, use_list=False, raw=False), args.output) return EXIT_SUCCESS @@ -2414,7 +2743,8 @@ def do_break_lock(self, args, repository): return self.exit_code helptext = collections.OrderedDict() - helptext['patterns'] = textwrap.dedent(''' + helptext["patterns"] = textwrap.dedent( + """ When specifying one or more file paths in a Borg command that supports patterns for the respective option or argument, you can apply the patterns described here to include only desired files and/or exclude @@ -2647,8 +2977,10 @@ def do_break_lock(self, args, repository): - home/bobby/junk This allows you to share the same patterns between multiple repositories - without needing to specify them on the command line.\n\n''') - helptext['placeholders'] = textwrap.dedent(''' + without needing to specify them on the command line.\n\n""" + ) + helptext["placeholders"] = textwrap.dedent( + """ Repository URLs, ``--name``, ``-a`` / ``--glob-archives``, ``--comment`` and ``--remote-path`` values support these placeholders: @@ -2705,8 +3037,10 @@ def do_break_lock(self, args, repository): especially when using the now/utcnow placeholders, since systemd performs its own %-based variable replacement even in quoted text. To avoid interference from systemd, double all percent signs (``{hostname}-{now:%Y-%m-%d_%H:%M:%S}`` - becomes ``{hostname}-{now:%%Y-%%m-%%d_%%H:%%M:%%S}``).\n\n''') - helptext['compression'] = textwrap.dedent(''' + becomes ``{hostname}-{now:%%Y-%%m-%%d_%%H:%%M:%%S}``).\n\n""" + ) + helptext["compression"] = textwrap.dedent( + """ It is no problem to mix different compression methods in one repo, deduplication is done on the source data chunks (not on the compressed or encrypted data). @@ -2791,7 +3125,8 @@ def do_break_lock(self, args, repository): borg create --compression auto,lzma ... borg create --compression obfuscate,3,none ... borg create --compression obfuscate,3,auto,zstd,10 ... - borg create --compression obfuscate,2,zstd,6 ...\n\n''') + borg create --compression obfuscate,2,zstd,6 ...\n\n""" + ) def do_help(self, parser, commands, args): if not args.topic: @@ -2808,11 +3143,11 @@ def do_help(self, parser, commands, args): commands[args.topic].print_help() else: msg_lines = [] - msg_lines += ['No help available on %s.' % args.topic] - msg_lines += ['Try one of the following:'] - msg_lines += [' Commands: %s' % ', '.join(sorted(commands.keys()))] - msg_lines += [' Topics: %s' % ', '.join(sorted(self.helptext.keys()))] - parser.error('\n'.join(msg_lines)) + msg_lines += ["No help available on %s." % args.topic] + msg_lines += ["Try one of the following:"] + msg_lines += [" Commands: %s" % ", ".join(sorted(commands.keys()))] + msg_lines += [" Topics: %s" % ", ".join(sorted(self.helptext.keys()))] + parser.error("\n".join(msg_lines)) return self.exit_code def do_subcommand_help(self, parser, args): @@ -2894,29 +3229,31 @@ def add_common_group(self, parser, suffix, provide_defaults=False): assert suffix in self.suffix_precedence def add_argument(*args, **kwargs): - if 'dest' in kwargs: - kwargs.setdefault('action', 'store') - assert kwargs['action'] in ('help', 'store_const', 'store_true', 'store_false', 'store', 'append') - is_append = kwargs['action'] == 'append' + if "dest" in kwargs: + kwargs.setdefault("action", "store") + assert kwargs["action"] in ("help", "store_const", "store_true", "store_false", "store", "append") + is_append = kwargs["action"] == "append" if is_append: - self.append_options.add(kwargs['dest']) - assert kwargs['default'] == [], 'The default is explicitly constructed as an empty list in resolve()' + self.append_options.add(kwargs["dest"]) + assert ( + kwargs["default"] == [] + ), "The default is explicitly constructed as an empty list in resolve()" else: - self.common_options.setdefault(suffix, set()).add(kwargs['dest']) - kwargs['dest'] += suffix + self.common_options.setdefault(suffix, set()).add(kwargs["dest"]) + kwargs["dest"] += suffix if not provide_defaults: # Interpolate help now, in case the %(default)d (or so) is mentioned, # to avoid producing incorrect help output. # Assumption: Interpolated output can safely be interpolated again, # which should always be the case. # Note: We control all inputs. - kwargs['help'] = kwargs['help'] % kwargs + kwargs["help"] = kwargs["help"] % kwargs if not is_append: - kwargs['default'] = self.default_sentinel + kwargs["default"] = self.default_sentinel common_group.add_argument(*args, **kwargs) - common_group = parser.add_argument_group('Common options') + common_group = parser.add_argument_group("Common options") self.define_common_options(add_argument) def resolve(self, args: argparse.Namespace): # Namespace has "in" but otherwise is not like a dict. @@ -2965,13 +3302,13 @@ def build_parser(self): # e.g. through "borg ... --help", define a substitution for the reference here. # It will replace the entire :ref:`foo` verbatim. rst_plain_text_references = { - 'a_status_oddity': '"I am seeing ‘A’ (added) status for a unchanged file!?"', - 'separate_compaction': '"Separate compaction"', - 'list_item_flags': '"Item flags"', - 'borg_patterns': '"borg help patterns"', - 'borg_placeholders': '"borg help placeholders"', - 'key_files': 'Internals -> Data structures and file formats -> Key files', - 'borg_key_export': 'borg key export --help', + "a_status_oddity": '"I am seeing ‘A’ (added) status for a unchanged file!?"', + "separate_compaction": '"Separate compaction"', + "list_item_flags": '"Item flags"', + "borg_patterns": '"borg help patterns"', + "borg_placeholders": '"borg help placeholders"', + "key_files": "Internals -> Data structures and file formats -> Key files", + "borg_key_export": "borg key export --help", } def process_epilog(epilog): @@ -2979,166 +3316,323 @@ def process_epilog(epilog): try: mode = borg.doc_mode except AttributeError: - mode = 'command-line' - if mode in ('command-line', 'build_usage'): - epilog = [line for line in epilog if not line.startswith('.. man')] - epilog = '\n'.join(epilog) - if mode == 'command-line': + mode = "command-line" + if mode in ("command-line", "build_usage"): + epilog = [line for line in epilog if not line.startswith(".. man")] + epilog = "\n".join(epilog) + if mode == "command-line": epilog = rst_to_terminal(epilog, rst_plain_text_references) return epilog def define_common_options(add_common_option): - add_common_option('-h', '--help', action='help', help='show this help message and exit') - add_common_option('--critical', dest='log_level', - action='store_const', const='critical', default='warning', - help='work on log level CRITICAL') - add_common_option('--error', dest='log_level', - action='store_const', const='error', default='warning', - help='work on log level ERROR') - add_common_option('--warning', dest='log_level', - action='store_const', const='warning', default='warning', - help='work on log level WARNING (default)') - add_common_option('--info', '-v', '--verbose', dest='log_level', - action='store_const', const='info', default='warning', - help='work on log level INFO') - add_common_option('--debug', dest='log_level', - action='store_const', const='debug', default='warning', - help='enable debug output, work on log level DEBUG') - add_common_option('--debug-topic', metavar='TOPIC', dest='debug_topics', action='append', default=[], - help='enable TOPIC debugging (can be specified multiple times). ' - 'The logger path is borg.debug. if TOPIC is not fully qualified.') - add_common_option('-p', '--progress', dest='progress', action='store_true', - help='show progress information') - add_common_option('--iec', dest='iec', action='store_true', - help='format using IEC units (1KiB = 1024B)') - add_common_option('--log-json', dest='log_json', action='store_true', - help='Output one JSON object per log line instead of formatted text.') - add_common_option('--lock-wait', metavar='SECONDS', dest='lock_wait', type=int, default=1, - help='wait at most SECONDS for acquiring a repository/cache lock (default: %(default)d).') - add_common_option('--bypass-lock', dest='lock', action='store_false', - default=argparse.SUPPRESS, # only create args attribute if option is specified - help='Bypass locking mechanism') - add_common_option('--show-version', dest='show_version', action='store_true', - help='show/log the borg version') - add_common_option('--show-rc', dest='show_rc', action='store_true', - help='show/log the return code (rc)') - add_common_option('--umask', metavar='M', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, - help='set umask to M (local only, default: %(default)04o)') - add_common_option('--remote-path', metavar='PATH', dest='remote_path', - help='use PATH as borg executable on the remote (default: "borg")') - add_common_option('--upload-ratelimit', metavar='RATE', dest='upload_ratelimit', type=int, - help='set network upload rate limit in kiByte/s (default: 0=unlimited)') - add_common_option('--upload-buffer', metavar='UPLOAD_BUFFER', dest='upload_buffer', type=int, - help='set network upload buffer size in MiB. (default: 0=no buffer)') - add_common_option('--consider-part-files', dest='consider_part_files', action='store_true', - help='treat part files like normal files (e.g. to list/extract them)') - add_common_option('--debug-profile', metavar='FILE', dest='debug_profile', default=None, - help='Write execution profile in Borg format into FILE. For local use a Python-' - 'compatible file can be generated by suffixing FILE with ".pyprof".') - add_common_option('--rsh', metavar='RSH', dest='rsh', - help="Use this command to connect to the 'borg serve' process (default: 'ssh')") - add_common_option('-r', '--repo', metavar='REPO', dest='location', - type=location_validator(other=False), default=Location(other=False), - help="repository to use") + add_common_option("-h", "--help", action="help", help="show this help message and exit") + add_common_option( + "--critical", + dest="log_level", + action="store_const", + const="critical", + default="warning", + help="work on log level CRITICAL", + ) + add_common_option( + "--error", + dest="log_level", + action="store_const", + const="error", + default="warning", + help="work on log level ERROR", + ) + add_common_option( + "--warning", + dest="log_level", + action="store_const", + const="warning", + default="warning", + help="work on log level WARNING (default)", + ) + add_common_option( + "--info", + "-v", + "--verbose", + dest="log_level", + action="store_const", + const="info", + default="warning", + help="work on log level INFO", + ) + add_common_option( + "--debug", + dest="log_level", + action="store_const", + const="debug", + default="warning", + help="enable debug output, work on log level DEBUG", + ) + add_common_option( + "--debug-topic", + metavar="TOPIC", + dest="debug_topics", + action="append", + default=[], + help="enable TOPIC debugging (can be specified multiple times). " + "The logger path is borg.debug. if TOPIC is not fully qualified.", + ) + add_common_option( + "-p", "--progress", dest="progress", action="store_true", help="show progress information" + ) + add_common_option("--iec", dest="iec", action="store_true", help="format using IEC units (1KiB = 1024B)") + add_common_option( + "--log-json", + dest="log_json", + action="store_true", + help="Output one JSON object per log line instead of formatted text.", + ) + add_common_option( + "--lock-wait", + metavar="SECONDS", + dest="lock_wait", + type=int, + default=1, + help="wait at most SECONDS for acquiring a repository/cache lock (default: %(default)d).", + ) + add_common_option( + "--bypass-lock", + dest="lock", + action="store_false", + default=argparse.SUPPRESS, # only create args attribute if option is specified + help="Bypass locking mechanism", + ) + add_common_option( + "--show-version", dest="show_version", action="store_true", help="show/log the borg version" + ) + add_common_option("--show-rc", dest="show_rc", action="store_true", help="show/log the return code (rc)") + add_common_option( + "--umask", + metavar="M", + dest="umask", + type=lambda s: int(s, 8), + default=UMASK_DEFAULT, + help="set umask to M (local only, default: %(default)04o)", + ) + add_common_option( + "--remote-path", + metavar="PATH", + dest="remote_path", + help='use PATH as borg executable on the remote (default: "borg")', + ) + add_common_option( + "--upload-ratelimit", + metavar="RATE", + dest="upload_ratelimit", + type=int, + help="set network upload rate limit in kiByte/s (default: 0=unlimited)", + ) + add_common_option( + "--upload-buffer", + metavar="UPLOAD_BUFFER", + dest="upload_buffer", + type=int, + help="set network upload buffer size in MiB. (default: 0=no buffer)", + ) + add_common_option( + "--consider-part-files", + dest="consider_part_files", + action="store_true", + help="treat part files like normal files (e.g. to list/extract them)", + ) + add_common_option( + "--debug-profile", + metavar="FILE", + dest="debug_profile", + default=None, + help="Write execution profile in Borg format into FILE. For local use a Python-" + 'compatible file can be generated by suffixing FILE with ".pyprof".', + ) + add_common_option( + "--rsh", + metavar="RSH", + dest="rsh", + help="Use this command to connect to the 'borg serve' process (default: 'ssh')", + ) + add_common_option( + "-r", + "--repo", + metavar="REPO", + dest="location", + type=location_validator(other=False), + default=Location(other=False), + help="repository to use", + ) def define_exclude_and_patterns(add_option, *, tag_files=False, strip_components=False): - add_option('-e', '--exclude', metavar='PATTERN', dest='patterns', - type=parse_exclude_pattern, action='append', - help='exclude paths matching PATTERN') - add_option('--exclude-from', metavar='EXCLUDEFILE', action=ArgparseExcludeFileAction, - help='read exclude patterns from EXCLUDEFILE, one per line') - add_option('--pattern', metavar='PATTERN', action=ArgparsePatternAction, - help='include/exclude paths matching PATTERN') - add_option('--patterns-from', metavar='PATTERNFILE', action=ArgparsePatternFileAction, - help='read include/exclude patterns from PATTERNFILE, one per line') + add_option( + "-e", + "--exclude", + metavar="PATTERN", + dest="patterns", + type=parse_exclude_pattern, + action="append", + help="exclude paths matching PATTERN", + ) + add_option( + "--exclude-from", + metavar="EXCLUDEFILE", + action=ArgparseExcludeFileAction, + help="read exclude patterns from EXCLUDEFILE, one per line", + ) + add_option( + "--pattern", + metavar="PATTERN", + action=ArgparsePatternAction, + help="include/exclude paths matching PATTERN", + ) + add_option( + "--patterns-from", + metavar="PATTERNFILE", + action=ArgparsePatternFileAction, + help="read include/exclude patterns from PATTERNFILE, one per line", + ) if tag_files: - add_option('--exclude-caches', dest='exclude_caches', action='store_true', - help='exclude directories that contain a CACHEDIR.TAG file ' - '(http://www.bford.info/cachedir/spec.html)') - add_option('--exclude-if-present', metavar='NAME', dest='exclude_if_present', - action='append', type=str, - help='exclude directories that are tagged by containing a filesystem object with ' - 'the given NAME') - add_option('--keep-exclude-tags', dest='keep_exclude_tags', - action='store_true', - help='if tag objects are specified with ``--exclude-if-present``, ' - 'don\'t omit the tag objects themselves from the backup archive') + add_option( + "--exclude-caches", + dest="exclude_caches", + action="store_true", + help="exclude directories that contain a CACHEDIR.TAG file " + "(http://www.bford.info/cachedir/spec.html)", + ) + add_option( + "--exclude-if-present", + metavar="NAME", + dest="exclude_if_present", + action="append", + type=str, + help="exclude directories that are tagged by containing a filesystem object with " "the given NAME", + ) + add_option( + "--keep-exclude-tags", + dest="keep_exclude_tags", + action="store_true", + help="if tag objects are specified with ``--exclude-if-present``, " + "don't omit the tag objects themselves from the backup archive", + ) if strip_components: - add_option('--strip-components', metavar='NUMBER', dest='strip_components', type=int, default=0, - help='Remove the specified number of leading path elements. ' - 'Paths with fewer elements will be silently skipped.') + add_option( + "--strip-components", + metavar="NUMBER", + dest="strip_components", + type=int, + default=0, + help="Remove the specified number of leading path elements. " + "Paths with fewer elements will be silently skipped.", + ) def define_exclusion_group(subparser, **kwargs): - exclude_group = subparser.add_argument_group('Exclusion options') + exclude_group = subparser.add_argument_group("Exclusion options") define_exclude_and_patterns(exclude_group.add_argument, **kwargs) return exclude_group def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): - filters_group = subparser.add_argument_group('Archive filters', - 'Archive filters can be applied to repository targets.') + filters_group = subparser.add_argument_group( + "Archive filters", "Archive filters can be applied to repository targets." + ) group = filters_group.add_mutually_exclusive_group() - group.add_argument('-a', '--glob-archives', metavar='GLOB', dest='glob_archives', - type=GlobSpec, action=Highlander, - help='only consider archive names matching the glob. ' - 'sh: rules apply, see "borg help patterns".') + group.add_argument( + "-a", + "--glob-archives", + metavar="GLOB", + dest="glob_archives", + type=GlobSpec, + action=Highlander, + help="only consider archive names matching the glob. " 'sh: rules apply, see "borg help patterns".', + ) if sort_by: - sort_by_default = 'timestamp' - filters_group.add_argument('--sort-by', metavar='KEYS', dest='sort_by', - type=SortBySpec, default=sort_by_default, - help='Comma-separated list of sorting keys; valid keys are: {}; default is: {}' - .format(', '.join(AI_HUMAN_SORT_KEYS), sort_by_default)) + sort_by_default = "timestamp" + filters_group.add_argument( + "--sort-by", + metavar="KEYS", + dest="sort_by", + type=SortBySpec, + default=sort_by_default, + help="Comma-separated list of sorting keys; valid keys are: {}; default is: {}".format( + ", ".join(AI_HUMAN_SORT_KEYS), sort_by_default + ), + ) if first_last: group = filters_group.add_mutually_exclusive_group() - group.add_argument('--first', metavar='N', dest='first', default=0, type=positive_int_validator, - help='consider first N archives after other filters were applied') - group.add_argument('--last', metavar='N', dest='last', default=0, type=positive_int_validator, - help='consider last N archives after other filters were applied') + group.add_argument( + "--first", + metavar="N", + dest="first", + default=0, + type=positive_int_validator, + help="consider first N archives after other filters were applied", + ) + group.add_argument( + "--last", + metavar="N", + dest="last", + default=0, + type=positive_int_validator, + help="consider last N archives after other filters were applied", + ) return filters_group def define_borg_mount(parser): parser.set_defaults(func=self.do_mount) - parser.add_argument('--consider-checkpoints', action='store_true', dest='consider_checkpoints', - help='Show checkpoint archives in the repository contents list (default: hidden).') - parser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, - help='where to mount filesystem') - parser.add_argument('-f', '--foreground', dest='foreground', - action='store_true', - help='stay in foreground, do not daemonize') - parser.add_argument('-o', dest='options', type=str, action=Highlander, - help='Extra mount options') - parser.add_argument('--numeric-ids', dest='numeric_ids', action='store_true', - help='use numeric user and group identifiers from archive(s)') + parser.add_argument( + "--consider-checkpoints", + action="store_true", + dest="consider_checkpoints", + help="Show checkpoint archives in the repository contents list (default: hidden).", + ) + parser.add_argument("mountpoint", metavar="MOUNTPOINT", type=str, help="where to mount filesystem") + parser.add_argument( + "-f", + "--foreground", + dest="foreground", + action="store_true", + help="stay in foreground, do not daemonize", + ) + parser.add_argument("-o", dest="options", type=str, action=Highlander, help="Extra mount options") + parser.add_argument( + "--numeric-ids", + dest="numeric_ids", + action="store_true", + help="use numeric user and group identifiers from archive(s)", + ) define_archive_filters_group(parser) - parser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to extract; patterns are supported') + parser.add_argument( + "paths", metavar="PATH", nargs="*", type=str, help="paths to extract; patterns are supported" + ) define_exclusion_group(parser, strip_components=True) - parser = argparse.ArgumentParser(prog=self.prog, description='Borg - Deduplicated Backups', - add_help=False) + parser = argparse.ArgumentParser(prog=self.prog, description="Borg - Deduplicated Backups", add_help=False) # paths and patterns must have an empty list as default everywhere - parser.set_defaults(fallback2_func=functools.partial(self.do_maincommand_help, parser), - paths=[], patterns=[]) - parser.common_options = self.CommonOptions(define_common_options, - suffix_precedence=('_maincommand', '_midcommand', '_subcommand')) - parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, - help='show version number and exit') - parser.common_options.add_common_group(parser, '_maincommand', provide_defaults=True) + parser.set_defaults(fallback2_func=functools.partial(self.do_maincommand_help, parser), paths=[], patterns=[]) + parser.common_options = self.CommonOptions( + define_common_options, suffix_precedence=("_maincommand", "_midcommand", "_subcommand") + ) + parser.add_argument( + "-V", "--version", action="version", version="%(prog)s " + __version__, help="show version number and exit" + ) + parser.common_options.add_common_group(parser, "_maincommand", provide_defaults=True) common_parser = argparse.ArgumentParser(add_help=False, prog=self.prog) common_parser.set_defaults(paths=[], patterns=[]) - parser.common_options.add_common_group(common_parser, '_subcommand') + parser.common_options.add_common_group(common_parser, "_subcommand") mid_common_parser = argparse.ArgumentParser(add_help=False, prog=self.prog) mid_common_parser.set_defaults(paths=[], patterns=[]) - parser.common_options.add_common_group(mid_common_parser, '_midcommand') + parser.common_options.add_common_group(mid_common_parser, "_midcommand") # borg mount - mount_epilog = process_epilog(""" + mount_epilog = process_epilog( + """ This command mounts an archive as a FUSE filesystem. This can be useful for browsing an archive or restoring individual files. When restoring, take into account that the current FUSE implementation does not support @@ -3195,31 +3689,37 @@ def define_borg_mount(parser): When running in the foreground ^C/SIGINT unmounts cleanly, but other signals or crashes do not. - """) + """ + ) - if parser.prog == 'borgfs': + if parser.prog == "borgfs": parser.description = self.do_mount.__doc__ parser.epilog = mount_epilog parser.formatter_class = argparse.RawDescriptionHelpFormatter - parser.help = 'mount repository' + parser.help = "mount repository" define_borg_mount(parser) return parser - subparsers = parser.add_subparsers(title='required arguments', metavar='') + subparsers = parser.add_subparsers(title="required arguments", metavar="") # borg benchmark benchmark_epilog = process_epilog("These commands do various benchmarks.") - subparser = subparsers.add_parser('benchmark', parents=[mid_common_parser], add_help=False, - description='benchmark command', - epilog=benchmark_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='benchmark command') + subparser = subparsers.add_parser( + "benchmark", + parents=[mid_common_parser], + add_help=False, + description="benchmark command", + epilog=benchmark_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="benchmark command", + ) - benchmark_parsers = subparser.add_subparsers(title='required arguments', metavar='') + benchmark_parsers = subparser.add_subparsers(title="required arguments", metavar="") subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser)) - bench_crud_epilog = process_epilog(""" + bench_crud_epilog = process_epilog( + """ This command benchmarks borg CRUD (create, read, update, delete) operations. It creates input data below the given PATH and backups this data into the given REPO. @@ -3257,17 +3757,23 @@ def define_borg_mount(parser): Please note that there might be quite some variance in these measurements. Try multiple measurements and having a otherwise idle machine (and network, if you use it). - """) - subparser = benchmark_parsers.add_parser('crud', parents=[common_parser], add_help=False, - description=self.do_benchmark_crud.__doc__, - epilog=bench_crud_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='benchmarks borg CRUD (create, extract, update, delete).') + """ + ) + subparser = benchmark_parsers.add_parser( + "crud", + parents=[common_parser], + add_help=False, + description=self.do_benchmark_crud.__doc__, + epilog=bench_crud_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="benchmarks borg CRUD (create, extract, update, delete).", + ) subparser.set_defaults(func=self.do_benchmark_crud) - subparser.add_argument('path', metavar='PATH', help='path were to create benchmark input data') + subparser.add_argument("path", metavar="PATH", help="path were to create benchmark input data") - bench_cpu_epilog = process_epilog(""" + bench_cpu_epilog = process_epilog( + """ This command benchmarks misc. CPU bound borg operations. It creates input data in memory, runs the operation and then displays throughput. @@ -3275,29 +3781,41 @@ def define_borg_mount(parser): - an otherwise as idle as possible machine - enough free memory so there will be no slow down due to paging activity - """) - subparser = benchmark_parsers.add_parser('cpu', parents=[common_parser], add_help=False, - description=self.do_benchmark_cpu.__doc__, - epilog=bench_cpu_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='benchmarks borg CPU bound operations.') + """ + ) + subparser = benchmark_parsers.add_parser( + "cpu", + parents=[common_parser], + add_help=False, + description=self.do_benchmark_cpu.__doc__, + epilog=bench_cpu_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="benchmarks borg CPU bound operations.", + ) subparser.set_defaults(func=self.do_benchmark_cpu) # borg break-lock - break_lock_epilog = process_epilog(""" + break_lock_epilog = process_epilog( + """ This command breaks the repository and cache locks. Please use carefully and only while no borg process (on any machine) is trying to access the Cache or the Repository. - """) - subparser = subparsers.add_parser('break-lock', parents=[common_parser], add_help=False, - description=self.do_break_lock.__doc__, - epilog=break_lock_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='break repository and cache locks') + """ + ) + subparser = subparsers.add_parser( + "break-lock", + parents=[common_parser], + add_help=False, + description=self.do_break_lock.__doc__, + epilog=break_lock_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="break repository and cache locks", + ) subparser.set_defaults(func=self.do_break_lock) # borg check - check_epilog = process_epilog(""" + check_epilog = process_epilog( + """ The check command verifies the consistency of a repository and the corresponding archives. check --repair is a potentially dangerous function and might lead to data loss @@ -3370,31 +3888,49 @@ def define_borg_mount(parser): which will detect (accidental) corruption. For encrypted repositories it is tamper-resistant as well, unless the attacker has access to the keys. It is also very slow. - """) - subparser = subparsers.add_parser('check', parents=[common_parser], add_help=False, - description=self.do_check.__doc__, - epilog=check_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='verify repository') + """ + ) + subparser = subparsers.add_parser( + "check", + parents=[common_parser], + add_help=False, + description=self.do_check.__doc__, + epilog=check_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="verify repository", + ) subparser.set_defaults(func=self.do_check) - subparser.add_argument('--repository-only', dest='repo_only', action='store_true', - help='only perform repository checks') - subparser.add_argument('--archives-only', dest='archives_only', action='store_true', - help='only perform archives checks') - subparser.add_argument('--verify-data', dest='verify_data', action='store_true', - help='perform cryptographic archive data integrity verification ' - '(conflicts with ``--repository-only``)') - subparser.add_argument('--repair', dest='repair', action='store_true', - help='attempt to repair any inconsistencies found') - subparser.add_argument('--save-space', dest='save_space', action='store_true', - help='work slower, but using less space') - subparser.add_argument('--max-duration', metavar='SECONDS', dest='max_duration', - type=int, default=0, - help='do only a partial repo check for max. SECONDS seconds (Default: unlimited)') + subparser.add_argument( + "--repository-only", dest="repo_only", action="store_true", help="only perform repository checks" + ) + subparser.add_argument( + "--archives-only", dest="archives_only", action="store_true", help="only perform archives checks" + ) + subparser.add_argument( + "--verify-data", + dest="verify_data", + action="store_true", + help="perform cryptographic archive data integrity verification " "(conflicts with ``--repository-only``)", + ) + subparser.add_argument( + "--repair", dest="repair", action="store_true", help="attempt to repair any inconsistencies found" + ) + subparser.add_argument( + "--save-space", dest="save_space", action="store_true", help="work slower, but using less space" + ) + subparser.add_argument( + "--max-duration", + metavar="SECONDS", + dest="max_duration", + type=int, + default=0, + help="do only a partial repo check for max. SECONDS seconds (Default: unlimited)", + ) define_archive_filters_group(subparser) # borg compact - compact_epilog = process_epilog(""" + compact_epilog = process_epilog( + """ This command frees repository space by compacting segments. Use this regularly to avoid running out of space - you do not need to use this @@ -3412,19 +3948,30 @@ def define_borg_mount(parser): When using ``--verbose``, borg will output an estimate of the freed space. See :ref:`separate_compaction` in Additional Notes for more details. - """) - subparser = subparsers.add_parser('compact', parents=[common_parser], add_help=False, - description=self.do_compact.__doc__, - epilog=compact_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='compact segment files / free space in repo') + """ + ) + subparser = subparsers.add_parser( + "compact", + parents=[common_parser], + add_help=False, + description=self.do_compact.__doc__, + epilog=compact_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="compact segment files / free space in repo", + ) subparser.set_defaults(func=self.do_compact) - subparser.add_argument('--threshold', metavar='PERCENT', dest='threshold', - type=int, default=10, - help='set minimum threshold for saved space in PERCENT (Default: 10)') + subparser.add_argument( + "--threshold", + metavar="PERCENT", + dest="threshold", + type=int, + default=10, + help="set minimum threshold for saved space in PERCENT (Default: 10)", + ) # borg config - config_epilog = process_epilog(""" + config_epilog = process_epilog( + """ This command gets and sets options in a local repository or cache config file. For security reasons, this command only works on local repositories. @@ -3438,29 +3985,34 @@ def define_borg_mount(parser): By default, borg config manipulates the repository config file. Using ``--cache`` edits the repository cache's config file instead. - """) - subparser = subparsers.add_parser('config', parents=[common_parser], add_help=False, - description=self.do_config.__doc__, - epilog=config_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='get and set configuration values') + """ + ) + subparser = subparsers.add_parser( + "config", + parents=[common_parser], + add_help=False, + description=self.do_config.__doc__, + epilog=config_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="get and set configuration values", + ) subparser.set_defaults(func=self.do_config) - subparser.add_argument('-c', '--cache', dest='cache', action='store_true', - help='get and set values from the repo cache') + subparser.add_argument( + "-c", "--cache", dest="cache", action="store_true", help="get and set values from the repo cache" + ) group = subparser.add_mutually_exclusive_group() - group.add_argument('-d', '--delete', dest='delete', action='store_true', - help='delete the key from the config file') - group.add_argument('-l', '--list', dest='list', action='store_true', - help='list the configuration of the repo') + group.add_argument( + "-d", "--delete", dest="delete", action="store_true", help="delete the key from the config file" + ) + group.add_argument("-l", "--list", dest="list", action="store_true", help="list the configuration of the repo") - subparser.add_argument('name', metavar='NAME', nargs='?', - help='name of config key') - subparser.add_argument('value', metavar='VALUE', nargs='?', - help='new value for key') + subparser.add_argument("name", metavar="NAME", nargs="?", help="name of config key") + subparser.add_argument("value", metavar="VALUE", nargs="?", help="new value for key") # borg create - create_epilog = process_epilog(""" + create_epilog = process_epilog( + """ This command creates a backup archive containing all files found while recursively traversing all paths specified. Paths are added to the archive as they are given, that means if relative paths are desired, the command has to be run from the correct @@ -3637,277 +4189,473 @@ def define_borg_mount(parser): By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. - """) + """ + ) - subparser = subparsers.add_parser('create', parents=[common_parser], add_help=False, - description=self.do_create.__doc__, - epilog=create_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='create backup') + subparser = subparsers.add_parser( + "create", + parents=[common_parser], + add_help=False, + description=self.do_create.__doc__, + epilog=create_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="create backup", + ) subparser.set_defaults(func=self.do_create) # note: --dry-run and --stats are mutually exclusive, but we do not want to abort when # parsing, but rather proceed with the dry-run, but without stats (see run() method). - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not create a backup archive') - subparser.add_argument('-s', '--stats', dest='stats', action='store_true', - help='print statistics for the created archive') + subparser.add_argument( + "-n", "--dry-run", dest="dry_run", action="store_true", help="do not create a backup archive" + ) + subparser.add_argument( + "-s", "--stats", dest="stats", action="store_true", help="print statistics for the created archive" + ) - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of items (files, dirs, ...)') - subparser.add_argument('--filter', metavar='STATUSCHARS', dest='output_filter', action=Highlander, - help='only display items with the given status characters (see description)') - subparser.add_argument('--json', action='store_true', - help='output stats as JSON. Implies ``--stats``.') - subparser.add_argument('--no-cache-sync', dest='no_cache_sync', action='store_true', - help='experimental: do not synchronize the cache. Implies not using the files cache.') - subparser.add_argument('--stdin-name', metavar='NAME', dest='stdin_name', default='stdin', - help='use NAME in archive for stdin data (default: %(default)r)') - subparser.add_argument('--stdin-user', metavar='USER', dest='stdin_user', default=uid2user(0), - help='set user USER in archive for stdin data (default: %(default)r)') - subparser.add_argument('--stdin-group', metavar='GROUP', dest='stdin_group', default=gid2group(0), - help='set group GROUP in archive for stdin data (default: %(default)r)') - subparser.add_argument('--stdin-mode', metavar='M', dest='stdin_mode', type=lambda s: int(s, 8), default=STDIN_MODE_DEFAULT, - help='set mode to M in archive for stdin data (default: %(default)04o)') - subparser.add_argument('--content-from-command', action='store_true', - help='interpret PATH as command and store its stdout. See also section Reading from' - ' stdin below.') - subparser.add_argument('--paths-from-stdin', action='store_true', - help='read DELIM-separated list of paths to backup from stdin. Will not ' - 'recurse into directories.') - subparser.add_argument('--paths-from-command', action='store_true', - help='interpret PATH as command and treat its output as ``--paths-from-stdin``') - subparser.add_argument('--paths-delimiter', metavar='DELIM', - help='set path delimiter for ``--paths-from-stdin`` and ``--paths-from-command`` (default: \\n) ') + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of items (files, dirs, ...)" + ) + subparser.add_argument( + "--filter", + metavar="STATUSCHARS", + dest="output_filter", + action=Highlander, + help="only display items with the given status characters (see description)", + ) + subparser.add_argument("--json", action="store_true", help="output stats as JSON. Implies ``--stats``.") + subparser.add_argument( + "--no-cache-sync", + dest="no_cache_sync", + action="store_true", + help="experimental: do not synchronize the cache. Implies not using the files cache.", + ) + subparser.add_argument( + "--stdin-name", + metavar="NAME", + dest="stdin_name", + default="stdin", + help="use NAME in archive for stdin data (default: %(default)r)", + ) + subparser.add_argument( + "--stdin-user", + metavar="USER", + dest="stdin_user", + default=uid2user(0), + help="set user USER in archive for stdin data (default: %(default)r)", + ) + subparser.add_argument( + "--stdin-group", + metavar="GROUP", + dest="stdin_group", + default=gid2group(0), + help="set group GROUP in archive for stdin data (default: %(default)r)", + ) + subparser.add_argument( + "--stdin-mode", + metavar="M", + dest="stdin_mode", + type=lambda s: int(s, 8), + default=STDIN_MODE_DEFAULT, + help="set mode to M in archive for stdin data (default: %(default)04o)", + ) + subparser.add_argument( + "--content-from-command", + action="store_true", + help="interpret PATH as command and store its stdout. See also section Reading from" " stdin below.", + ) + subparser.add_argument( + "--paths-from-stdin", + action="store_true", + help="read DELIM-separated list of paths to backup from stdin. Will not " "recurse into directories.", + ) + subparser.add_argument( + "--paths-from-command", + action="store_true", + help="interpret PATH as command and treat its output as ``--paths-from-stdin``", + ) + subparser.add_argument( + "--paths-delimiter", + metavar="DELIM", + help="set path delimiter for ``--paths-from-stdin`` and ``--paths-from-command`` (default: \\n) ", + ) exclude_group = define_exclusion_group(subparser, tag_files=True) - exclude_group.add_argument('--exclude-nodump', dest='exclude_nodump', action='store_true', - help='exclude files flagged NODUMP') + exclude_group.add_argument( + "--exclude-nodump", dest="exclude_nodump", action="store_true", help="exclude files flagged NODUMP" + ) - fs_group = subparser.add_argument_group('Filesystem options') - fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', action='store_true', - help='stay in the same file system and do not store mount points of other file systems. This might behave different from your expectations, see the docs.') - fs_group.add_argument('--numeric-ids', dest='numeric_ids', action='store_true', - help='only store numeric user and group identifiers') - fs_group.add_argument('--atime', dest='atime', action='store_true', - help='do store atime into archive') - fs_group.add_argument('--noctime', dest='noctime', action='store_true', - help='do not store ctime into archive') - fs_group.add_argument('--nobirthtime', dest='nobirthtime', action='store_true', - help='do not store birthtime (creation date) into archive') - fs_group.add_argument('--noflags', dest='noflags', action='store_true', - help='do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive') - fs_group.add_argument('--noacls', dest='noacls', action='store_true', - help='do not read and store ACLs into archive') - fs_group.add_argument('--noxattrs', dest='noxattrs', action='store_true', - help='do not read and store xattrs into archive') - fs_group.add_argument('--sparse', dest='sparse', action='store_true', - help='detect sparse holes in input (supported only by fixed chunker)') - fs_group.add_argument('--files-cache', metavar='MODE', dest='files_cache_mode', action=Highlander, - type=FilesCacheMode, default=FILES_CACHE_MODE_UI_DEFAULT, - help='operate files cache in MODE. default: %s' % FILES_CACHE_MODE_UI_DEFAULT) - fs_group.add_argument('--read-special', dest='read_special', action='store_true', - help='open and read block and char device files as well as FIFOs as if they were ' - 'regular files. Also follows symlinks pointing to these kinds of files.') + fs_group = subparser.add_argument_group("Filesystem options") + fs_group.add_argument( + "-x", + "--one-file-system", + dest="one_file_system", + action="store_true", + help="stay in the same file system and do not store mount points of other file systems. This might behave different from your expectations, see the docs.", + ) + fs_group.add_argument( + "--numeric-ids", + dest="numeric_ids", + action="store_true", + help="only store numeric user and group identifiers", + ) + fs_group.add_argument("--atime", dest="atime", action="store_true", help="do store atime into archive") + fs_group.add_argument("--noctime", dest="noctime", action="store_true", help="do not store ctime into archive") + fs_group.add_argument( + "--nobirthtime", + dest="nobirthtime", + action="store_true", + help="do not store birthtime (creation date) into archive", + ) + fs_group.add_argument( + "--noflags", + dest="noflags", + action="store_true", + help="do not read and store flags (e.g. NODUMP, IMMUTABLE) into archive", + ) + fs_group.add_argument( + "--noacls", dest="noacls", action="store_true", help="do not read and store ACLs into archive" + ) + fs_group.add_argument( + "--noxattrs", dest="noxattrs", action="store_true", help="do not read and store xattrs into archive" + ) + fs_group.add_argument( + "--sparse", + dest="sparse", + action="store_true", + help="detect sparse holes in input (supported only by fixed chunker)", + ) + fs_group.add_argument( + "--files-cache", + metavar="MODE", + dest="files_cache_mode", + action=Highlander, + type=FilesCacheMode, + default=FILES_CACHE_MODE_UI_DEFAULT, + help="operate files cache in MODE. default: %s" % FILES_CACHE_MODE_UI_DEFAULT, + ) + fs_group.add_argument( + "--read-special", + dest="read_special", + action="store_true", + help="open and read block and char device files as well as FIFOs as if they were " + "regular files. Also follows symlinks pointing to these kinds of files.", + ) - archive_group = subparser.add_argument_group('Archive options') - archive_group.add_argument('--comment', dest='comment', metavar='COMMENT', type=CommentSpec, default='', - help='add a comment text to the archive') - archive_group.add_argument('--timestamp', metavar='TIMESTAMP', dest='timestamp', - type=timestamp, default=None, - help='manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). ' - 'Alternatively, give a reference file/directory.') - archive_group.add_argument('-c', '--checkpoint-interval', metavar='SECONDS', dest='checkpoint_interval', - type=int, default=1800, - help='write checkpoint every SECONDS seconds (Default: 1800)') - archive_group.add_argument('--chunker-params', metavar='PARAMS', dest='chunker_params', - type=ChunkerParams, default=CHUNKER_PARAMS, action=Highlander, - help='specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, ' - 'HASH_MASK_BITS, HASH_WINDOW_SIZE). default: %s,%d,%d,%d,%d' % CHUNKER_PARAMS) - archive_group.add_argument('-C', '--compression', metavar='COMPRESSION', dest='compression', - type=CompressionSpec, default=CompressionSpec('lz4'), - help='select compression algorithm, see the output of the ' - '"borg help compression" command for details.') + archive_group = subparser.add_argument_group("Archive options") + archive_group.add_argument( + "--comment", + dest="comment", + metavar="COMMENT", + type=CommentSpec, + default="", + help="add a comment text to the archive", + ) + archive_group.add_argument( + "--timestamp", + metavar="TIMESTAMP", + dest="timestamp", + type=timestamp, + default=None, + help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). " + "Alternatively, give a reference file/directory.", + ) + archive_group.add_argument( + "-c", + "--checkpoint-interval", + metavar="SECONDS", + dest="checkpoint_interval", + type=int, + default=1800, + help="write checkpoint every SECONDS seconds (Default: 1800)", + ) + archive_group.add_argument( + "--chunker-params", + metavar="PARAMS", + dest="chunker_params", + type=ChunkerParams, + default=CHUNKER_PARAMS, + action=Highlander, + help="specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, " + "HASH_MASK_BITS, HASH_WINDOW_SIZE). default: %s,%d,%d,%d,%d" % CHUNKER_PARAMS, + ) + archive_group.add_argument( + "-C", + "--compression", + metavar="COMPRESSION", + dest="compression", + type=CompressionSpec, + default=CompressionSpec("lz4"), + help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + ) - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to archive') + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument("paths", metavar="PATH", nargs="*", type=str, help="paths to archive") # borg debug - debug_epilog = process_epilog(""" + debug_epilog = process_epilog( + """ These commands are not intended for normal use and potentially very dangerous if used incorrectly. They exist to improve debugging capabilities without direct system access, e.g. in case you ever run into some severe malfunction. Use them only if you know - what you are doing or if a trusted developer tells you what to do.""") + what you are doing or if a trusted developer tells you what to do.""" + ) - subparser = subparsers.add_parser('debug', parents=[mid_common_parser], add_help=False, - description='debugging command (not intended for normal use)', - epilog=debug_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='debugging command (not intended for normal use)') + subparser = subparsers.add_parser( + "debug", + parents=[mid_common_parser], + add_help=False, + description="debugging command (not intended for normal use)", + epilog=debug_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="debugging command (not intended for normal use)", + ) - debug_parsers = subparser.add_subparsers(title='required arguments', metavar='') + debug_parsers = subparser.add_subparsers(title="required arguments", metavar="") subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser)) - debug_info_epilog = process_epilog(""" + debug_info_epilog = process_epilog( + """ This command displays some system information that might be useful for bug reports and debugging problems. If a traceback happens, this information is already appended at the end of the traceback. - """) - subparser = debug_parsers.add_parser('info', parents=[common_parser], add_help=False, - description=self.do_debug_info.__doc__, - epilog=debug_info_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='show system infos for debugging / bug reports (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "info", + parents=[common_parser], + add_help=False, + description=self.do_debug_info.__doc__, + epilog=debug_info_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="show system infos for debugging / bug reports (debug)", + ) subparser.set_defaults(func=self.do_debug_info) - debug_dump_archive_items_epilog = process_epilog(""" + debug_dump_archive_items_epilog = process_epilog( + """ This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. - """) - subparser = debug_parsers.add_parser('dump-archive-items', parents=[common_parser], add_help=False, - description=self.do_debug_dump_archive_items.__doc__, - epilog=debug_dump_archive_items_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='dump archive items (metadata) (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "dump-archive-items", + parents=[common_parser], + add_help=False, + description=self.do_debug_dump_archive_items.__doc__, + epilog=debug_dump_archive_items_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="dump archive items (metadata) (debug)", + ) subparser.set_defaults(func=self.do_debug_dump_archive_items) - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") - debug_dump_archive_epilog = process_epilog(""" + debug_dump_archive_epilog = process_epilog( + """ This command dumps all metadata of an archive in a decoded form to a file. - """) - subparser = debug_parsers.add_parser('dump-archive', parents=[common_parser], add_help=False, - description=self.do_debug_dump_archive.__doc__, - epilog=debug_dump_archive_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='dump decoded archive metadata (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "dump-archive", + parents=[common_parser], + add_help=False, + description=self.do_debug_dump_archive.__doc__, + epilog=debug_dump_archive_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="dump decoded archive metadata (debug)", + ) subparser.set_defaults(func=self.do_debug_dump_archive) - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('path', metavar='PATH', type=str, - help='file to dump data into') + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into") - debug_dump_manifest_epilog = process_epilog(""" + debug_dump_manifest_epilog = process_epilog( + """ This command dumps manifest metadata of a repository in a decoded form to a file. - """) - subparser = debug_parsers.add_parser('dump-manifest', parents=[common_parser], add_help=False, - description=self.do_debug_dump_manifest.__doc__, - epilog=debug_dump_manifest_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='dump decoded repository metadata (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "dump-manifest", + parents=[common_parser], + add_help=False, + description=self.do_debug_dump_manifest.__doc__, + epilog=debug_dump_manifest_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="dump decoded repository metadata (debug)", + ) subparser.set_defaults(func=self.do_debug_dump_manifest) - subparser.add_argument('path', metavar='PATH', type=str, - help='file to dump data into') + subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into") - debug_dump_repo_objs_epilog = process_epilog(""" + debug_dump_repo_objs_epilog = process_epilog( + """ This command dumps raw (but decrypted and decompressed) repo objects to files. - """) - subparser = debug_parsers.add_parser('dump-repo-objs', parents=[common_parser], add_help=False, - description=self.do_debug_dump_repo_objs.__doc__, - epilog=debug_dump_repo_objs_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='dump repo objects (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "dump-repo-objs", + parents=[common_parser], + add_help=False, + description=self.do_debug_dump_repo_objs.__doc__, + epilog=debug_dump_repo_objs_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="dump repo objects (debug)", + ) subparser.set_defaults(func=self.do_debug_dump_repo_objs) - subparser.add_argument('--ghost', dest='ghost', action='store_true', - help='dump all segment file contents, including deleted/uncommitted objects and commits.') - subparser.add_argument('--segment', metavar='SEG', dest='segment', default=None, type=positive_int_validator, - help='used together with --ghost: limit processing to given segment.') - subparser.add_argument('--offset', metavar='OFFS', dest='offset', default=None, type=positive_int_validator, - help='used together with --ghost: limit processing to given offset.') + subparser.add_argument( + "--ghost", + dest="ghost", + action="store_true", + help="dump all segment file contents, including deleted/uncommitted objects and commits.", + ) + subparser.add_argument( + "--segment", + metavar="SEG", + dest="segment", + default=None, + type=positive_int_validator, + help="used together with --ghost: limit processing to given segment.", + ) + subparser.add_argument( + "--offset", + metavar="OFFS", + dest="offset", + default=None, + type=positive_int_validator, + help="used together with --ghost: limit processing to given offset.", + ) - debug_search_repo_objs_epilog = process_epilog(""" + debug_search_repo_objs_epilog = process_epilog( + """ This command searches raw (but decrypted and decompressed) repo objects for a specific bytes sequence. - """) - subparser = debug_parsers.add_parser('search-repo-objs', parents=[common_parser], add_help=False, - description=self.do_debug_search_repo_objs.__doc__, - epilog=debug_search_repo_objs_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='search repo objects (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "search-repo-objs", + parents=[common_parser], + add_help=False, + description=self.do_debug_search_repo_objs.__doc__, + epilog=debug_search_repo_objs_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="search repo objects (debug)", + ) subparser.set_defaults(func=self.do_debug_search_repo_objs) - subparser.add_argument('wanted', metavar='WANTED', type=str, - help='term to search the repo for, either 0x1234abcd hex term or a string') + subparser.add_argument( + "wanted", + metavar="WANTED", + type=str, + help="term to search the repo for, either 0x1234abcd hex term or a string", + ) - debug_get_obj_epilog = process_epilog(""" + debug_get_obj_epilog = process_epilog( + """ This command gets an object from the repository. - """) - subparser = debug_parsers.add_parser('get-obj', parents=[common_parser], add_help=False, - description=self.do_debug_get_obj.__doc__, - epilog=debug_get_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='get object from repository (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "get-obj", + parents=[common_parser], + add_help=False, + description=self.do_debug_get_obj.__doc__, + epilog=debug_get_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="get object from repository (debug)", + ) subparser.set_defaults(func=self.do_debug_get_obj) - subparser.add_argument('id', metavar='ID', type=str, - help='hex object ID to get from the repo') - subparser.add_argument('path', metavar='PATH', type=str, - help='file to write object data into') + subparser.add_argument("id", metavar="ID", type=str, help="hex object ID to get from the repo") + subparser.add_argument("path", metavar="PATH", type=str, help="file to write object data into") - debug_put_obj_epilog = process_epilog(""" + debug_put_obj_epilog = process_epilog( + """ This command puts objects into the repository. - """) - subparser = debug_parsers.add_parser('put-obj', parents=[common_parser], add_help=False, - description=self.do_debug_put_obj.__doc__, - epilog=debug_put_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='put object to repository (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "put-obj", + parents=[common_parser], + add_help=False, + description=self.do_debug_put_obj.__doc__, + epilog=debug_put_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="put object to repository (debug)", + ) subparser.set_defaults(func=self.do_debug_put_obj) - subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, - help='file(s) to read and create object(s) from') + subparser.add_argument( + "paths", metavar="PATH", nargs="+", type=str, help="file(s) to read and create object(s) from" + ) - debug_delete_obj_epilog = process_epilog(""" + debug_delete_obj_epilog = process_epilog( + """ This command deletes objects from the repository. - """) - subparser = debug_parsers.add_parser('delete-obj', parents=[common_parser], add_help=False, - description=self.do_debug_delete_obj.__doc__, - epilog=debug_delete_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='delete object from repository (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "delete-obj", + parents=[common_parser], + add_help=False, + description=self.do_debug_delete_obj.__doc__, + epilog=debug_delete_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="delete object from repository (debug)", + ) subparser.set_defaults(func=self.do_debug_delete_obj) - subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, - help='hex object ID(s) to delete from the repo') + subparser.add_argument( + "ids", metavar="IDs", nargs="+", type=str, help="hex object ID(s) to delete from the repo" + ) - debug_refcount_obj_epilog = process_epilog(""" + debug_refcount_obj_epilog = process_epilog( + """ This command displays the reference count for objects from the repository. - """) - subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False, - description=self.do_debug_refcount_obj.__doc__, - epilog=debug_refcount_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='show refcount for object from repository (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "refcount-obj", + parents=[common_parser], + add_help=False, + description=self.do_debug_refcount_obj.__doc__, + epilog=debug_refcount_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="show refcount for object from repository (debug)", + ) subparser.set_defaults(func=self.do_debug_refcount_obj) - subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, - help='hex object ID(s) to show refcounts for') + subparser.add_argument("ids", metavar="IDs", nargs="+", type=str, help="hex object ID(s) to show refcounts for") - debug_dump_hints_epilog = process_epilog(""" + debug_dump_hints_epilog = process_epilog( + """ This command dumps the repository hints data. - """) - subparser = debug_parsers.add_parser('dump-hints', parents=[common_parser], add_help=False, - description=self.do_debug_dump_hints.__doc__, - epilog=debug_dump_hints_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='dump repo hints (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "dump-hints", + parents=[common_parser], + add_help=False, + description=self.do_debug_dump_hints.__doc__, + epilog=debug_dump_hints_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="dump repo hints (debug)", + ) subparser.set_defaults(func=self.do_debug_dump_hints) - subparser.add_argument('path', metavar='PATH', type=str, - help='file to dump data into') + subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into") - debug_convert_profile_epilog = process_epilog(""" + debug_convert_profile_epilog = process_epilog( + """ Convert a Borg profile to a Python cProfile compatible profile. - """) - subparser = debug_parsers.add_parser('convert-profile', parents=[common_parser], add_help=False, - description=self.do_debug_convert_profile.__doc__, - epilog=debug_convert_profile_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='convert Borg profile to Python profile (debug)') + """ + ) + subparser = debug_parsers.add_parser( + "convert-profile", + parents=[common_parser], + add_help=False, + description=self.do_debug_convert_profile.__doc__, + epilog=debug_convert_profile_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="convert Borg profile to Python profile (debug)", + ) subparser.set_defaults(func=self.do_debug_convert_profile) - subparser.add_argument('input', metavar='INPUT', type=argparse.FileType('rb'), - help='Borg profile') - subparser.add_argument('output', metavar='OUTPUT', type=argparse.FileType('wb'), - help='Output file') + subparser.add_argument("input", metavar="INPUT", type=argparse.FileType("rb"), help="Borg profile") + subparser.add_argument("output", metavar="OUTPUT", type=argparse.FileType("wb"), help="Output file") # borg rdelete - rdelete_epilog = process_epilog(""" + rdelete_epilog = process_epilog( + """ This command deletes the complete repository. When you delete a complete repository, the security info and local cache for it @@ -3916,27 +4664,45 @@ def define_borg_mount(parser): ``--keep-security-info`` option. Always first use ``--dry-run --list`` to see what would be deleted. - """) - subparser = subparsers.add_parser('rdelete', parents=[common_parser], add_help=False, - description=self.do_rdelete.__doc__, - epilog=rdelete_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='delete repository') + """ + ) + subparser = subparsers.add_parser( + "rdelete", + parents=[common_parser], + add_help=False, + description=self.do_rdelete.__doc__, + epilog=rdelete_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="delete repository", + ) subparser.set_defaults(func=self.do_rdelete) - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not change repository') - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of archives') - subparser.add_argument('--force', dest='forced', action='count', default=0, - help='force deletion of corrupted archives, ' - 'use ``--force --force`` in case ``--force`` does not work.') - subparser.add_argument('--cache-only', dest='cache_only', action='store_true', - help='delete only the local cache for the given repository') - subparser.add_argument('--keep-security-info', dest='keep_security_info', action='store_true', - help='keep the local security info when deleting a repository') + subparser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository") + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of archives" + ) + subparser.add_argument( + "--force", + dest="forced", + action="count", + default=0, + help="force deletion of corrupted archives, " "use ``--force --force`` in case ``--force`` does not work.", + ) + subparser.add_argument( + "--cache-only", + dest="cache_only", + action="store_true", + help="delete only the local cache for the given repository", + ) + subparser.add_argument( + "--keep-security-info", + dest="keep_security_info", + action="store_true", + help="keep the local security info when deleting a repository", + ) # borg delete - delete_epilog = process_epilog(""" + delete_epilog = process_epilog( + """ This command deletes archives from the repository. Important: When deleting archives, repository disk space is **not** freed until @@ -3954,34 +4720,58 @@ def define_borg_mount(parser): see :ref:`borg_patterns`). Always first use ``--dry-run --list`` to see what would be deleted. - """) - subparser = subparsers.add_parser('delete', parents=[common_parser], add_help=False, - description=self.do_delete.__doc__, - epilog=delete_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='delete archive') + """ + ) + subparser = subparsers.add_parser( + "delete", + parents=[common_parser], + add_help=False, + description=self.do_delete.__doc__, + epilog=delete_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="delete archive", + ) subparser.set_defaults(func=self.do_delete) - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not change repository') - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of archives') - subparser.add_argument('--consider-checkpoints', action='store_true', dest='consider_checkpoints', - help='consider checkpoint archives for deletion (default: not considered).') - subparser.add_argument('-s', '--stats', dest='stats', action='store_true', - help='print statistics for the deleted archive') - subparser.add_argument('--cache-only', dest='cache_only', action='store_true', - help='delete only the local cache for the given repository') - subparser.add_argument('--force', dest='forced', action='count', default=0, - help='force deletion of corrupted archives, ' - 'use ``--force --force`` in case ``--force`` does not work.') - subparser.add_argument('--keep-security-info', dest='keep_security_info', action='store_true', - help='keep the local security info when deleting a repository') - subparser.add_argument('--save-space', dest='save_space', action='store_true', - help='work slower, but using less space') + subparser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository") + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of archives" + ) + subparser.add_argument( + "--consider-checkpoints", + action="store_true", + dest="consider_checkpoints", + help="consider checkpoint archives for deletion (default: not considered).", + ) + subparser.add_argument( + "-s", "--stats", dest="stats", action="store_true", help="print statistics for the deleted archive" + ) + subparser.add_argument( + "--cache-only", + dest="cache_only", + action="store_true", + help="delete only the local cache for the given repository", + ) + subparser.add_argument( + "--force", + dest="forced", + action="count", + default=0, + help="force deletion of corrupted archives, " "use ``--force --force`` in case ``--force`` does not work.", + ) + subparser.add_argument( + "--keep-security-info", + dest="keep_security_info", + action="store_true", + help="keep the local security info when deleting a repository", + ) + subparser.add_argument( + "--save-space", dest="save_space", action="store_true", help="work slower, but using less space" + ) define_archive_filters_group(subparser) # borg transfer - transfer_epilog = process_epilog(""" + transfer_epilog = process_epilog( + """ This command transfers archives from one repository to another repository. Optionally, it can also upgrade the transferred data. @@ -4008,25 +4798,42 @@ def define_borg_mount(parser): borg --repo=DST_REPO transfer --other-repo=SRC_REPO --upgrader=From12To20 - """) - subparser = subparsers.add_parser('transfer', parents=[common_parser], add_help=False, - description=self.do_transfer.__doc__, - epilog=transfer_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='transfer of archives from another repository') + """ + ) + subparser = subparsers.add_parser( + "transfer", + parents=[common_parser], + add_help=False, + description=self.do_transfer.__doc__, + epilog=transfer_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="transfer of archives from another repository", + ) subparser.set_defaults(func=self.do_transfer) - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not change repository, just check') - subparser.add_argument('--other-repo', metavar='SRC_REPOSITORY', dest='other_location', - type=location_validator(other=True), default=Location(other=True), - help='transfer archives from the other repository') - subparser.add_argument('--upgrader', metavar='UPGRADER', dest='upgrader', - type=str, default='NoOp', - help='use the upgrader to convert transferred data (default: no conversion)') + subparser.add_argument( + "-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository, just check" + ) + subparser.add_argument( + "--other-repo", + metavar="SRC_REPOSITORY", + dest="other_location", + type=location_validator(other=True), + default=Location(other=True), + help="transfer archives from the other repository", + ) + subparser.add_argument( + "--upgrader", + metavar="UPGRADER", + dest="upgrader", + type=str, + default="NoOp", + help="use the upgrader to convert transferred data (default: no conversion)", + ) define_archive_filters_group(subparser) # borg diff - diff_epilog = process_epilog(""" + diff_epilog = process_epilog( + """ This command finds differences (file contents, user/group/mode) between archives. A repository location and an archive name must be specified for REPO::ARCHIVE1. @@ -4043,33 +4850,46 @@ def define_borg_mount(parser): Note that the chunker params changed from Borg 0.xx to 1.0. For more help on include/exclude patterns, see the :ref:`borg_patterns` command output. - """) - subparser = subparsers.add_parser('diff', parents=[common_parser], add_help=False, - description=self.do_diff.__doc__, - epilog=diff_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='find differences in archive contents') + """ + ) + subparser = subparsers.add_parser( + "diff", + parents=[common_parser], + add_help=False, + description=self.do_diff.__doc__, + epilog=diff_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="find differences in archive contents", + ) subparser.set_defaults(func=self.do_diff) - subparser.add_argument('--numeric-ids', dest='numeric_ids', action='store_true', - help='only consider numeric user and group identifiers') - subparser.add_argument('--same-chunker-params', dest='same_chunker_params', action='store_true', - help='Override check of chunker parameters.') - subparser.add_argument('--sort', dest='sort', action='store_true', - help='Sort the output lines by file path.') - subparser.add_argument('--json-lines', action='store_true', - help='Format output as JSON Lines. ') - subparser.add_argument('name', metavar='ARCHIVE1', - type=archivename_validator(), - help='ARCHIVE1 name') - subparser.add_argument('other_name', metavar='ARCHIVE2', - type=archivename_validator(), - help='ARCHIVE2 name') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths of items inside the archives to compare; patterns are supported') + subparser.add_argument( + "--numeric-ids", + dest="numeric_ids", + action="store_true", + help="only consider numeric user and group identifiers", + ) + subparser.add_argument( + "--same-chunker-params", + dest="same_chunker_params", + action="store_true", + help="Override check of chunker parameters.", + ) + subparser.add_argument("--sort", dest="sort", action="store_true", help="Sort the output lines by file path.") + subparser.add_argument("--json-lines", action="store_true", help="Format output as JSON Lines. ") + subparser.add_argument("name", metavar="ARCHIVE1", type=archivename_validator(), help="ARCHIVE1 name") + subparser.add_argument("other_name", metavar="ARCHIVE2", type=archivename_validator(), help="ARCHIVE2 name") + subparser.add_argument( + "paths", + metavar="PATH", + nargs="*", + type=str, + help="paths of items inside the archives to compare; patterns are supported", + ) define_exclusion_group(subparser) # borg export-tar - export_tar_epilog = process_epilog(""" + export_tar_epilog = process_epilog( + """ This command creates a tarball from an archive. When giving '-' as the output FILE, Borg will write a tar stream to standard output. @@ -4111,30 +4931,42 @@ def define_borg_mount(parser): ``--progress`` can be slower than no progress display, since it makes one additional pass over the archive metadata. - """) - subparser = subparsers.add_parser('export-tar', parents=[common_parser], add_help=False, - description=self.do_export_tar.__doc__, - epilog=export_tar_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='create tarball from archive') + """ + ) + subparser = subparsers.add_parser( + "export-tar", + parents=[common_parser], + add_help=False, + description=self.do_export_tar.__doc__, + epilog=export_tar_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="create tarball from archive", + ) subparser.set_defaults(func=self.do_export_tar) - subparser.add_argument('--tar-filter', dest='tar_filter', default='auto', - help='filter program to pipe data through') - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of items (files, dirs, ...)') - subparser.add_argument('--tar-format', metavar='FMT', dest='tar_format', default='GNU', - choices=('BORG', 'PAX', 'GNU'), - help='select tar format: BORG, PAX or GNU') - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('tarfile', metavar='FILE', - help='output tar file. "-" to write to stdout instead.') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to extract; patterns are supported') + subparser.add_argument( + "--tar-filter", dest="tar_filter", default="auto", help="filter program to pipe data through" + ) + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of items (files, dirs, ...)" + ) + subparser.add_argument( + "--tar-format", + metavar="FMT", + dest="tar_format", + default="GNU", + choices=("BORG", "PAX", "GNU"), + help="select tar format: BORG, PAX or GNU", + ) + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument("tarfile", metavar="FILE", help='output tar file. "-" to write to stdout instead.') + subparser.add_argument( + "paths", metavar="PATH", nargs="*", type=str, help="paths to extract; patterns are supported" + ) define_exclusion_group(subparser, strip_components=True) # borg extract - extract_epilog = process_epilog(""" + extract_epilog = process_epilog( + """ This command extracts the contents of an archive. By default the entire archive is extracted but a subset of files and directories can be selected by passing a list of ``PATHs`` as arguments. The file selection can further @@ -4157,46 +4989,60 @@ def define_borg_mount(parser): When parent directories are not extracted (because of using file/directory selection or any other reason), borg can not restore parent directories' metadata, e.g. owner, group, permission, etc. - """) - subparser = subparsers.add_parser('extract', parents=[common_parser], add_help=False, - description=self.do_extract.__doc__, - epilog=extract_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='extract archive contents') + """ + ) + subparser = subparsers.add_parser( + "extract", + parents=[common_parser], + add_help=False, + description=self.do_extract.__doc__, + epilog=extract_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="extract archive contents", + ) subparser.set_defaults(func=self.do_extract) - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of items (files, dirs, ...)') - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not actually change any files') - subparser.add_argument('--numeric-ids', dest='numeric_ids', action='store_true', - help='only obey numeric user and group identifiers') - subparser.add_argument('--noflags', dest='noflags', action='store_true', - help='do not extract/set flags (e.g. NODUMP, IMMUTABLE)') - subparser.add_argument('--noacls', dest='noacls', action='store_true', - help='do not extract/set ACLs') - subparser.add_argument('--noxattrs', dest='noxattrs', action='store_true', - help='do not extract/set xattrs') - subparser.add_argument('--stdout', dest='stdout', action='store_true', - help='write all extracted data to stdout') - subparser.add_argument('--sparse', dest='sparse', action='store_true', - help='create holes in output sparse file from all-zero chunks') - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to extract; patterns are supported') + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of items (files, dirs, ...)" + ) + subparser.add_argument( + "-n", "--dry-run", dest="dry_run", action="store_true", help="do not actually change any files" + ) + subparser.add_argument( + "--numeric-ids", + dest="numeric_ids", + action="store_true", + help="only obey numeric user and group identifiers", + ) + subparser.add_argument( + "--noflags", dest="noflags", action="store_true", help="do not extract/set flags (e.g. NODUMP, IMMUTABLE)" + ) + subparser.add_argument("--noacls", dest="noacls", action="store_true", help="do not extract/set ACLs") + subparser.add_argument("--noxattrs", dest="noxattrs", action="store_true", help="do not extract/set xattrs") + subparser.add_argument( + "--stdout", dest="stdout", action="store_true", help="write all extracted data to stdout" + ) + subparser.add_argument( + "--sparse", + dest="sparse", + action="store_true", + help="create holes in output sparse file from all-zero chunks", + ) + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument( + "paths", metavar="PATH", nargs="*", type=str, help="paths to extract; patterns are supported" + ) define_exclusion_group(subparser, strip_components=True) # borg help - subparser = subparsers.add_parser('help', parents=[common_parser], add_help=False, - description='Extra help') - subparser.add_argument('--epilog-only', dest='epilog_only', action='store_true') - subparser.add_argument('--usage-only', dest='usage_only', action='store_true') + subparser = subparsers.add_parser("help", parents=[common_parser], add_help=False, description="Extra help") + subparser.add_argument("--epilog-only", dest="epilog_only", action="store_true") + subparser.add_argument("--usage-only", dest="usage_only", action="store_true") subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices)) - subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', - help='additional help on TOPIC') + subparser.add_argument("topic", metavar="TOPIC", type=str, nargs="?", help="additional help on TOPIC") # borg rinfo - rinfo_epilog = process_epilog(""" + rinfo_epilog = process_epilog( + """ This command displays detailed information about the repository. Please note that the deduplicated sizes of the individual archives do not add @@ -4207,18 +5053,23 @@ def define_borg_mount(parser): = unique chunks of this archive. All archives / deduplicated size = amount of data stored in the repo = all chunks in the repository. - """) - subparser = subparsers.add_parser('rinfo', parents=[common_parser], add_help=False, - description=self.do_rinfo.__doc__, - epilog=rinfo_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='show repository information') + """ + ) + subparser = subparsers.add_parser( + "rinfo", + parents=[common_parser], + add_help=False, + description=self.do_rinfo.__doc__, + epilog=rinfo_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="show repository information", + ) subparser.set_defaults(func=self.do_rinfo) - subparser.add_argument('--json', action='store_true', - help='format output as JSON') + subparser.add_argument("--json", action="store_true", help="format output as JSON") # borg info - info_epilog = process_epilog(""" + info_epilog = process_epilog( + """ This command displays detailed information about the specified archive. Please note that the deduplicated sizes of the individual archives do not add @@ -4234,19 +5085,24 @@ def define_borg_mount(parser): The size of an archive relative to this limit depends on a number of factors, mainly the number of files, the lengths of paths and other metadata stored for files. This is shown as *utilization of maximum supported archive size*. - """) - subparser = subparsers.add_parser('info', parents=[common_parser], add_help=False, - description=self.do_info.__doc__, - epilog=info_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='show repository or archive information') + """ + ) + subparser = subparsers.add_parser( + "info", + parents=[common_parser], + add_help=False, + description=self.do_info.__doc__, + epilog=info_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="show repository or archive information", + ) subparser.set_defaults(func=self.do_info) - subparser.add_argument('--json', action='store_true', - help='format output as JSON') + subparser.add_argument("--json", action="store_true", help="format output as JSON") define_archive_filters_group(subparser) # borg rcreate - rcreate_epilog = process_epilog(""" + rcreate_epilog = process_epilog( + """ This command creates a new, empty repository. A repository is a filesystem directory containing the deduplicated data from zero or more archives. @@ -4350,40 +5206,75 @@ def define_borg_mount(parser): If you do **not** want to encrypt the contents of your backups, but still want to detect malicious tampering use an `authenticated` mode. It's like `repokey` minus encryption. - """) - subparser = subparsers.add_parser('rcreate', parents=[common_parser], add_help=False, - description=self.do_rcreate.__doc__, epilog=rcreate_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='create a new, empty repository') + """ + ) + subparser = subparsers.add_parser( + "rcreate", + parents=[common_parser], + add_help=False, + description=self.do_rcreate.__doc__, + epilog=rcreate_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="create a new, empty repository", + ) subparser.set_defaults(func=self.do_rcreate) - subparser.add_argument('--other-repo', metavar='SRC_REPOSITORY', dest='other_location', - type=location_validator(other=True), default=Location(other=True), - help='reuse the key material from the other repository') - subparser.add_argument('-e', '--encryption', metavar='MODE', dest='encryption', required=True, - choices=key_argument_names(), - help='select encryption key mode **(required)**') - subparser.add_argument('--append-only', dest='append_only', action='store_true', - help='create an append-only mode repository. Note that this only affects ' - 'the low level structure of the repository, and running `delete` ' - 'or `prune` will still be allowed. See :ref:`append_only_mode` in ' - 'Additional Notes for more details.') - subparser.add_argument('--storage-quota', metavar='QUOTA', dest='storage_quota', default=None, - type=parse_storage_quota, - help='Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.') - subparser.add_argument('--make-parent-dirs', dest='make_parent_dirs', action='store_true', - help='create the parent directories of the repository directory, if they are missing.') + subparser.add_argument( + "--other-repo", + metavar="SRC_REPOSITORY", + dest="other_location", + type=location_validator(other=True), + default=Location(other=True), + help="reuse the key material from the other repository", + ) + subparser.add_argument( + "-e", + "--encryption", + metavar="MODE", + dest="encryption", + required=True, + choices=key_argument_names(), + help="select encryption key mode **(required)**", + ) + subparser.add_argument( + "--append-only", + dest="append_only", + action="store_true", + help="create an append-only mode repository. Note that this only affects " + "the low level structure of the repository, and running `delete` " + "or `prune` will still be allowed. See :ref:`append_only_mode` in " + "Additional Notes for more details.", + ) + subparser.add_argument( + "--storage-quota", + metavar="QUOTA", + dest="storage_quota", + default=None, + type=parse_storage_quota, + help="Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.", + ) + subparser.add_argument( + "--make-parent-dirs", + dest="make_parent_dirs", + action="store_true", + help="create the parent directories of the repository directory, if they are missing.", + ) # borg key - subparser = subparsers.add_parser('key', parents=[mid_common_parser], add_help=False, - description="Manage a keyfile or repokey of a repository", - epilog="", - formatter_class=argparse.RawDescriptionHelpFormatter, - help='manage repository key') + subparser = subparsers.add_parser( + "key", + parents=[mid_common_parser], + add_help=False, + description="Manage a keyfile or repokey of a repository", + epilog="", + formatter_class=argparse.RawDescriptionHelpFormatter, + help="manage repository key", + ) - key_parsers = subparser.add_subparsers(title='required arguments', metavar='') + key_parsers = subparser.add_subparsers(title="required arguments", metavar="") subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser)) - key_export_epilog = process_epilog(""" + key_export_epilog = process_epilog( + """ If repository encryption is used, the repository is inaccessible without the key. This command allows one to backup this essential key. Note that the backup produced does not include the passphrase itself @@ -4417,21 +5308,34 @@ def define_borg_mount(parser): borg key export --qr-html /path/to/repo encrypted-key-backup.html - """) - subparser = key_parsers.add_parser('export', parents=[common_parser], add_help=False, - description=self.do_key_export.__doc__, - epilog=key_export_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='export repository key for backup') + """ + ) + subparser = key_parsers.add_parser( + "export", + parents=[common_parser], + add_help=False, + description=self.do_key_export.__doc__, + epilog=key_export_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="export repository key for backup", + ) subparser.set_defaults(func=self.do_key_export) - subparser.add_argument('path', metavar='PATH', nargs='?', type=str, - help='where to store the backup') - subparser.add_argument('--paper', dest='paper', action='store_true', - help='Create an export suitable for printing and later type-in') - subparser.add_argument('--qr-html', dest='qr', action='store_true', - help='Create an html file suitable for printing and later type-in or qr scan') + subparser.add_argument("path", metavar="PATH", nargs="?", type=str, help="where to store the backup") + subparser.add_argument( + "--paper", + dest="paper", + action="store_true", + help="Create an export suitable for printing and later type-in", + ) + subparser.add_argument( + "--qr-html", + dest="qr", + action="store_true", + help="Create an html file suitable for printing and later type-in or qr scan", + ) - key_import_epilog = process_epilog(""" + key_import_epilog = process_epilog( + """ This command restores a key previously backed up with the export command. If the ``--paper`` option is given, the import will be an interactive @@ -4446,19 +5350,30 @@ def define_borg_mount(parser): associated with the repository. If a key file is found in ``$BORG_KEYS_DIR``, ``borg key import`` overwrites it; otherwise, ``borg key import`` creates a new key file in ``$BORG_KEYS_DIR``. - """) - subparser = key_parsers.add_parser('import', parents=[common_parser], add_help=False, - description=self.do_key_import.__doc__, - epilog=key_import_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='import repository key from backup') + """ + ) + subparser = key_parsers.add_parser( + "import", + parents=[common_parser], + add_help=False, + description=self.do_key_import.__doc__, + epilog=key_import_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="import repository key from backup", + ) subparser.set_defaults(func=self.do_key_import) - subparser.add_argument('path', metavar='PATH', nargs='?', type=str, - help='path to the backup (\'-\' to read from stdin)') - subparser.add_argument('--paper', dest='paper', action='store_true', - help='interactively import from a backup done with ``--paper``') + subparser.add_argument( + "path", metavar="PATH", nargs="?", type=str, help="path to the backup ('-' to read from stdin)" + ) + subparser.add_argument( + "--paper", + dest="paper", + action="store_true", + help="interactively import from a backup done with ``--paper``", + ) - change_passphrase_epilog = process_epilog(""" + change_passphrase_epilog = process_epilog( + """ The key files used for repository encryption are optionally passphrase protected. This command can be used to change this passphrase. @@ -4466,15 +5381,21 @@ def define_borg_mount(parser): secret protected by it (like e.g. encryption/MAC keys or chunker seed). Thus, changing the passphrase after passphrase and borg key got compromised does not protect future (nor past) backups to the same repository. - """) - subparser = key_parsers.add_parser('change-passphrase', parents=[common_parser], add_help=False, - description=self.do_change_passphrase.__doc__, - epilog=change_passphrase_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='change repository passphrase') + """ + ) + subparser = key_parsers.add_parser( + "change-passphrase", + parents=[common_parser], + add_help=False, + description=self.do_change_passphrase.__doc__, + epilog=change_passphrase_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="change repository passphrase", + ) subparser.set_defaults(func=self.do_change_passphrase) - change_location_epilog = process_epilog(""" + change_location_epilog = process_epilog( + """ Change the location of a borg key. The key can be stored at different locations: - keyfile: locally, usually in the home directory @@ -4484,20 +5405,32 @@ def define_borg_mount(parser): This command does NOT change the crypto algorithms, just the key location, thus you must ONLY give the key location (keyfile or repokey). - """) - subparser = key_parsers.add_parser('change-location', parents=[common_parser], add_help=False, - description=self.do_change_location.__doc__, - epilog=change_location_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='change key location') + """ + ) + subparser = key_parsers.add_parser( + "change-location", + parents=[common_parser], + add_help=False, + description=self.do_change_location.__doc__, + epilog=change_location_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="change key location", + ) subparser.set_defaults(func=self.do_change_location) - subparser.add_argument('key_mode', metavar='KEY_LOCATION', choices=('repokey', 'keyfile'), - help='select key location') - subparser.add_argument('--keep', dest='keep', action='store_true', - help='keep the key also at the current location (default: remove it)') + subparser.add_argument( + "key_mode", metavar="KEY_LOCATION", choices=("repokey", "keyfile"), help="select key location" + ) + subparser.add_argument( + "--keep", + dest="keep", + action="store_true", + help="keep the key also at the current location (default: remove it)", + ) # borg list - list_epilog = process_epilog(""" + list_epilog = ( + process_epilog( + """ This command lists the contents of an archive. For more help on include/exclude patterns, see the :ref:`borg_patterns` command output. @@ -4526,36 +5459,57 @@ def define_borg_mount(parser): The following keys are always available: - """) + BaseFormatter.keys_help() + textwrap.dedent(""" + """ + ) + + BaseFormatter.keys_help() + + textwrap.dedent( + """ Keys available only when listing files in an archive: - """) + ItemFormatter.keys_help() - subparser = subparsers.add_parser('list', parents=[common_parser], add_help=False, - description=self.do_list.__doc__, - epilog=list_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='list archive contents') + """ + ) + + ItemFormatter.keys_help() + ) + subparser = subparsers.add_parser( + "list", + parents=[common_parser], + add_help=False, + description=self.do_list.__doc__, + epilog=list_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="list archive contents", + ) subparser.set_defaults(func=self.do_list) - subparser.add_argument('--short', dest='short', action='store_true', - help='only print file/directory names, nothing else') - subparser.add_argument('--format', metavar='FORMAT', dest='format', - help='specify format for file listing ' - '(default: "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")') - subparser.add_argument('--json-lines', action='store_true', - help='Format output as JSON Lines. ' - 'The form of ``--format`` is ignored, ' - 'but keys used in it are added to the JSON output. ' - 'Some keys are always present. Note: JSON can only represent text. ' - 'A "bpath" key is therefore not available.') - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to list; patterns are supported') + subparser.add_argument( + "--short", dest="short", action="store_true", help="only print file/directory names, nothing else" + ) + subparser.add_argument( + "--format", + metavar="FORMAT", + dest="format", + help="specify format for file listing " + '(default: "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}")', + ) + subparser.add_argument( + "--json-lines", + action="store_true", + help="Format output as JSON Lines. " + "The form of ``--format`` is ignored, " + "but keys used in it are added to the JSON output. " + "Some keys are always present. Note: JSON can only represent text. " + 'A "bpath" key is therefore not available.', + ) + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument( + "paths", metavar="PATH", nargs="*", type=str, help="paths to list; patterns are supported" + ) define_exclusion_group(subparser) # borg rlist - rlist_epilog = process_epilog(""" + rlist_epilog = ( + process_epilog( + """ This command lists the archives contained in a repository. .. man NOTES @@ -4584,41 +5538,68 @@ def define_borg_mount(parser): The following keys are always available: - """) + BaseFormatter.keys_help() + textwrap.dedent(""" + """ + ) + + BaseFormatter.keys_help() + + textwrap.dedent( + """ Keys available only when listing archives in a repository: - """) + ArchiveFormatter.keys_help() - subparser = subparsers.add_parser('rlist', parents=[common_parser], add_help=False, - description=self.do_rlist.__doc__, - epilog=rlist_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='list repository contents') + """ + ) + + ArchiveFormatter.keys_help() + ) + subparser = subparsers.add_parser( + "rlist", + parents=[common_parser], + add_help=False, + description=self.do_rlist.__doc__, + epilog=rlist_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="list repository contents", + ) subparser.set_defaults(func=self.do_rlist) - subparser.add_argument('--consider-checkpoints', action='store_true', dest='consider_checkpoints', - help='Show checkpoint archives in the repository contents list (default: hidden).') - subparser.add_argument('--short', dest='short', action='store_true', - help='only print the archive names, nothing else') - subparser.add_argument('--format', metavar='FORMAT', dest='format', - help='specify format for archive listing ' - '(default: "{archive:<36} {time} [{id}]{NL}")') - subparser.add_argument('--json', action='store_true', - help='Format output as JSON. ' - 'The form of ``--format`` is ignored, ' - 'but keys used in it are added to the JSON output. ' - 'Some keys are always present. Note: JSON can only represent text. ' - 'A "barchive" key is therefore not available.') + subparser.add_argument( + "--consider-checkpoints", + action="store_true", + dest="consider_checkpoints", + help="Show checkpoint archives in the repository contents list (default: hidden).", + ) + subparser.add_argument( + "--short", dest="short", action="store_true", help="only print the archive names, nothing else" + ) + subparser.add_argument( + "--format", + metavar="FORMAT", + dest="format", + help="specify format for archive listing " '(default: "{archive:<36} {time} [{id}]{NL}")', + ) + subparser.add_argument( + "--json", + action="store_true", + help="Format output as JSON. " + "The form of ``--format`` is ignored, " + "but keys used in it are added to the JSON output. " + "Some keys are always present. Note: JSON can only represent text. " + 'A "barchive" key is therefore not available.', + ) define_archive_filters_group(subparser) - subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False, - description=self.do_mount.__doc__, - epilog=mount_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='mount repository') + subparser = subparsers.add_parser( + "mount", + parents=[common_parser], + add_help=False, + description=self.do_mount.__doc__, + epilog=mount_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="mount repository", + ) define_borg_mount(subparser) # borg prune - prune_epilog = process_epilog(""" + prune_epilog = process_epilog( + """ The prune command prunes a repository by deleting all archives not matching any of the specified retention options. @@ -4672,44 +5653,72 @@ def define_borg_mount(parser): deleted - the "Deleted data" deduplicated size there is most interesting as that is how much your repository will shrink. Please note that the "All archives" stats refer to the state after pruning. - """) - subparser = subparsers.add_parser('prune', parents=[common_parser], add_help=False, - description=self.do_prune.__doc__, - epilog=prune_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='prune archives') + """ + ) + subparser = subparsers.add_parser( + "prune", + parents=[common_parser], + add_help=False, + description=self.do_prune.__doc__, + epilog=prune_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="prune archives", + ) subparser.set_defaults(func=self.do_prune) - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not change repository') - subparser.add_argument('--force', dest='forced', action='store_true', - help='force pruning of corrupted archives, ' - 'use ``--force --force`` in case ``--force`` does not work.') - subparser.add_argument('-s', '--stats', dest='stats', action='store_true', - help='print statistics for the deleted archive') - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of archives it keeps/prunes') - subparser.add_argument('--keep-within', metavar='INTERVAL', dest='within', type=interval, - help='keep all archives within this time interval') - subparser.add_argument('--keep-last', '--keep-secondly', dest='secondly', type=int, default=0, - help='number of secondly archives to keep') - subparser.add_argument('--keep-minutely', dest='minutely', type=int, default=0, - help='number of minutely archives to keep') - subparser.add_argument('-H', '--keep-hourly', dest='hourly', type=int, default=0, - help='number of hourly archives to keep') - subparser.add_argument('-d', '--keep-daily', dest='daily', type=int, default=0, - help='number of daily archives to keep') - subparser.add_argument('-w', '--keep-weekly', dest='weekly', type=int, default=0, - help='number of weekly archives to keep') - subparser.add_argument('-m', '--keep-monthly', dest='monthly', type=int, default=0, - help='number of monthly archives to keep') - subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, - help='number of yearly archives to keep') + subparser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository") + subparser.add_argument( + "--force", + dest="forced", + action="store_true", + help="force pruning of corrupted archives, " "use ``--force --force`` in case ``--force`` does not work.", + ) + subparser.add_argument( + "-s", "--stats", dest="stats", action="store_true", help="print statistics for the deleted archive" + ) + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of archives it keeps/prunes" + ) + subparser.add_argument( + "--keep-within", + metavar="INTERVAL", + dest="within", + type=interval, + help="keep all archives within this time interval", + ) + subparser.add_argument( + "--keep-last", + "--keep-secondly", + dest="secondly", + type=int, + default=0, + help="number of secondly archives to keep", + ) + subparser.add_argument( + "--keep-minutely", dest="minutely", type=int, default=0, help="number of minutely archives to keep" + ) + subparser.add_argument( + "-H", "--keep-hourly", dest="hourly", type=int, default=0, help="number of hourly archives to keep" + ) + subparser.add_argument( + "-d", "--keep-daily", dest="daily", type=int, default=0, help="number of daily archives to keep" + ) + subparser.add_argument( + "-w", "--keep-weekly", dest="weekly", type=int, default=0, help="number of weekly archives to keep" + ) + subparser.add_argument( + "-m", "--keep-monthly", dest="monthly", type=int, default=0, help="number of monthly archives to keep" + ) + subparser.add_argument( + "-y", "--keep-yearly", dest="yearly", type=int, default=0, help="number of yearly archives to keep" + ) define_archive_filters_group(subparser, sort_by=False, first_last=False) - subparser.add_argument('--save-space', dest='save_space', action='store_true', - help='work slower, but using less space') + subparser.add_argument( + "--save-space", dest="save_space", action="store_true", help="work slower, but using less space" + ) # borg recreate - recreate_epilog = process_epilog(""" + recreate_epilog = process_epilog( + """ Recreate the contents of existing archives. recreate is a potentially dangerous function and might lead to data loss @@ -4760,131 +5769,219 @@ def define_borg_mount(parser): any more after re-chunking (it is also unlikely it would ever work: due to the change of chunking parameters, the missing chunk likely will never be seen again even if you still have the data that produced it). - """) - subparser = subparsers.add_parser('recreate', parents=[common_parser], add_help=False, - description=self.do_recreate.__doc__, - epilog=recreate_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help=self.do_recreate.__doc__) + """ + ) + subparser = subparsers.add_parser( + "recreate", + parents=[common_parser], + add_help=False, + description=self.do_recreate.__doc__, + epilog=recreate_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help=self.do_recreate.__doc__, + ) subparser.set_defaults(func=self.do_recreate) - subparser.add_argument('--list', dest='output_list', action='store_true', - help='output verbose list of items (files, dirs, ...)') - subparser.add_argument('--filter', metavar='STATUSCHARS', dest='output_filter', action=Highlander, - help='only display items with the given status characters (listed in borg create --help)') - subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', - help='do not change anything') - subparser.add_argument('-s', '--stats', dest='stats', action='store_true', - help='print statistics at end') + subparser.add_argument( + "--list", dest="output_list", action="store_true", help="output verbose list of items (files, dirs, ...)" + ) + subparser.add_argument( + "--filter", + metavar="STATUSCHARS", + dest="output_filter", + action=Highlander, + help="only display items with the given status characters (listed in borg create --help)", + ) + subparser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", help="do not change anything") + subparser.add_argument("-s", "--stats", dest="stats", action="store_true", help="print statistics at end") define_exclusion_group(subparser, tag_files=True) archive_group = define_archive_filters_group(subparser) - archive_group.add_argument('--target', dest='target', metavar='TARGET', default=None, - type=archivename_validator(), - help='create a new archive with the name ARCHIVE, do not replace existing archive ' - '(only applies for a single archive)') - archive_group.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval', - type=int, default=1800, metavar='SECONDS', - help='write checkpoint every SECONDS seconds (Default: 1800)') - archive_group.add_argument('--comment', dest='comment', metavar='COMMENT', type=CommentSpec, default=None, - help='add a comment text to the archive') - archive_group.add_argument('--timestamp', metavar='TIMESTAMP', dest='timestamp', - type=timestamp, default=None, - help='manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). ' - 'alternatively, give a reference file/directory.') - archive_group.add_argument('-C', '--compression', metavar='COMPRESSION', dest='compression', - type=CompressionSpec, default=CompressionSpec('lz4'), - help='select compression algorithm, see the output of the ' - '"borg help compression" command for details.') - archive_group.add_argument('--recompress', metavar='MODE', dest='recompress', nargs='?', - default='never', const='if-different', choices=('never', 'if-different', 'always'), - help='recompress data chunks according to `MODE` and ``--compression``. ' - 'Possible modes are ' - '`if-different`: recompress if current compression is with a different ' - 'compression algorithm or different level; ' - '`always`: recompress unconditionally; and ' - '`never`: do not recompress (use this option to explicitly prevent ' - 'recompression). ' - 'If no MODE is given, `if-different` will be used. ' - 'Not passing --recompress is equivalent to "--recompress never".') - archive_group.add_argument('--chunker-params', metavar='PARAMS', dest='chunker_params', action=Highlander, - type=ChunkerParams, default=CHUNKER_PARAMS, - help='specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, ' - 'HASH_MASK_BITS, HASH_WINDOW_SIZE) or `default` to use the current defaults. ' - 'default: %s,%d,%d,%d,%d' % CHUNKER_PARAMS) + archive_group.add_argument( + "--target", + dest="target", + metavar="TARGET", + default=None, + type=archivename_validator(), + help="create a new archive with the name ARCHIVE, do not replace existing archive " + "(only applies for a single archive)", + ) + archive_group.add_argument( + "-c", + "--checkpoint-interval", + dest="checkpoint_interval", + type=int, + default=1800, + metavar="SECONDS", + help="write checkpoint every SECONDS seconds (Default: 1800)", + ) + archive_group.add_argument( + "--comment", + dest="comment", + metavar="COMMENT", + type=CommentSpec, + default=None, + help="add a comment text to the archive", + ) + archive_group.add_argument( + "--timestamp", + metavar="TIMESTAMP", + dest="timestamp", + type=timestamp, + default=None, + help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). " + "alternatively, give a reference file/directory.", + ) + archive_group.add_argument( + "-C", + "--compression", + metavar="COMPRESSION", + dest="compression", + type=CompressionSpec, + default=CompressionSpec("lz4"), + help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + ) + archive_group.add_argument( + "--recompress", + metavar="MODE", + dest="recompress", + nargs="?", + default="never", + const="if-different", + choices=("never", "if-different", "always"), + help="recompress data chunks according to `MODE` and ``--compression``. " + "Possible modes are " + "`if-different`: recompress if current compression is with a different " + "compression algorithm or different level; " + "`always`: recompress unconditionally; and " + "`never`: do not recompress (use this option to explicitly prevent " + "recompression). " + "If no MODE is given, `if-different` will be used. " + 'Not passing --recompress is equivalent to "--recompress never".', + ) + archive_group.add_argument( + "--chunker-params", + metavar="PARAMS", + dest="chunker_params", + action=Highlander, + type=ChunkerParams, + default=CHUNKER_PARAMS, + help="specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, " + "HASH_MASK_BITS, HASH_WINDOW_SIZE) or `default` to use the current defaults. " + "default: %s,%d,%d,%d,%d" % CHUNKER_PARAMS, + ) - subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to recreate; patterns are supported') + subparser.add_argument( + "paths", metavar="PATH", nargs="*", type=str, help="paths to recreate; patterns are supported" + ) # borg rename - rename_epilog = process_epilog(""" + rename_epilog = process_epilog( + """ This command renames an archive in the repository. This results in a different archive ID. - """) - subparser = subparsers.add_parser('rename', parents=[common_parser], add_help=False, - description=self.do_rename.__doc__, - epilog=rename_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='rename archive') + """ + ) + subparser = subparsers.add_parser( + "rename", + parents=[common_parser], + add_help=False, + description=self.do_rename.__doc__, + epilog=rename_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="rename archive", + ) subparser.set_defaults(func=self.do_rename) - subparser.add_argument('name', metavar='OLDNAME', - type=archivename_validator(), - help='specify the archive name') - subparser.add_argument('newname', metavar='NEWNAME', - type=archivename_validator(), - help='specify the new archive name') + subparser.add_argument("name", metavar="OLDNAME", type=archivename_validator(), help="specify the archive name") + subparser.add_argument( + "newname", metavar="NEWNAME", type=archivename_validator(), help="specify the new archive name" + ) # borg serve - serve_epilog = process_epilog(""" + serve_epilog = process_epilog( + """ This command starts a repository server process. This command is usually not used manually. - """) - subparser = subparsers.add_parser('serve', parents=[common_parser], add_help=False, - description=self.do_serve.__doc__, epilog=serve_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='start repository server process') + """ + ) + subparser = subparsers.add_parser( + "serve", + parents=[common_parser], + add_help=False, + description=self.do_serve.__doc__, + epilog=serve_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="start repository server process", + ) subparser.set_defaults(func=self.do_serve) - subparser.add_argument('--restrict-to-path', metavar='PATH', dest='restrict_to_paths', action='append', - help='restrict repository access to PATH. ' - 'Can be specified multiple times to allow the client access to several directories. ' - 'Access to all sub-directories is granted implicitly; PATH doesn\'t need to directly point to a repository.') - subparser.add_argument('--restrict-to-repository', metavar='PATH', dest='restrict_to_repositories', action='append', - help='restrict repository access. Only the repository located at PATH ' - '(no sub-directories are considered) is accessible. ' - 'Can be specified multiple times to allow the client access to several repositories. ' - 'Unlike ``--restrict-to-path`` sub-directories are not accessible; ' - 'PATH needs to directly point at a repository location. ' - 'PATH may be an empty directory or the last element of PATH may not exist, in which case ' - 'the client may initialize a repository there.') - subparser.add_argument('--append-only', dest='append_only', action='store_true', - help='only allow appending to repository segment files. Note that this only ' - 'affects the low level structure of the repository, and running `delete` ' - 'or `prune` will still be allowed. See :ref:`append_only_mode` in Additional ' - 'Notes for more details.') - subparser.add_argument('--storage-quota', metavar='QUOTA', dest='storage_quota', - type=parse_storage_quota, default=None, - help='Override storage quota of the repository (e.g. 5G, 1.5T). ' - 'When a new repository is initialized, sets the storage quota on the new ' - 'repository as well. Default: no quota.') + subparser.add_argument( + "--restrict-to-path", + metavar="PATH", + dest="restrict_to_paths", + action="append", + help="restrict repository access to PATH. " + "Can be specified multiple times to allow the client access to several directories. " + "Access to all sub-directories is granted implicitly; PATH doesn't need to directly point to a repository.", + ) + subparser.add_argument( + "--restrict-to-repository", + metavar="PATH", + dest="restrict_to_repositories", + action="append", + help="restrict repository access. Only the repository located at PATH " + "(no sub-directories are considered) is accessible. " + "Can be specified multiple times to allow the client access to several repositories. " + "Unlike ``--restrict-to-path`` sub-directories are not accessible; " + "PATH needs to directly point at a repository location. " + "PATH may be an empty directory or the last element of PATH may not exist, in which case " + "the client may initialize a repository there.", + ) + subparser.add_argument( + "--append-only", + dest="append_only", + action="store_true", + help="only allow appending to repository segment files. Note that this only " + "affects the low level structure of the repository, and running `delete` " + "or `prune` will still be allowed. See :ref:`append_only_mode` in Additional " + "Notes for more details.", + ) + subparser.add_argument( + "--storage-quota", + metavar="QUOTA", + dest="storage_quota", + type=parse_storage_quota, + default=None, + help="Override storage quota of the repository (e.g. 5G, 1.5T). " + "When a new repository is initialized, sets the storage quota on the new " + "repository as well. Default: no quota.", + ) # borg umount - umount_epilog = process_epilog(""" + umount_epilog = process_epilog( + """ This command un-mounts a FUSE filesystem that was mounted with ``borg mount``. This is a convenience wrapper that just calls the platform-specific shell command - usually this is either umount or fusermount -u. - """) - subparser = subparsers.add_parser('umount', parents=[common_parser], add_help=False, - description=self.do_umount.__doc__, - epilog=umount_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='umount repository') + """ + ) + subparser = subparsers.add_parser( + "umount", + parents=[common_parser], + add_help=False, + description=self.do_umount.__doc__, + epilog=umount_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="umount repository", + ) subparser.set_defaults(func=self.do_umount) - subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, - help='mountpoint of the filesystem to umount') + subparser.add_argument( + "mountpoint", metavar="MOUNTPOINT", type=str, help="mountpoint of the filesystem to umount" + ) # borg with-lock - with_lock_epilog = process_epilog(""" + with_lock_epilog = process_epilog( + """ This command runs a user-specified command while the repository lock is held. It will first try to acquire the lock (make sure that no other operation is @@ -4898,20 +5995,24 @@ def define_borg_mount(parser): the copy. Thus, before using borg on the copy from a different host, you need to use "borg break-lock" on the copied repository, because Borg is cautious and does not automatically remove stale locks made by a different host. - """) - subparser = subparsers.add_parser('with-lock', parents=[common_parser], add_help=False, - description=self.do_with_lock.__doc__, - epilog=with_lock_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help='run user command with lock held') + """ + ) + subparser = subparsers.add_parser( + "with-lock", + parents=[common_parser], + add_help=False, + description=self.do_with_lock.__doc__, + epilog=with_lock_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="run user command with lock held", + ) subparser.set_defaults(func=self.do_with_lock) - subparser.add_argument('command', metavar='COMMAND', - help='command to run') - subparser.add_argument('args', metavar='ARGS', nargs=argparse.REMAINDER, - help='command arguments') + subparser.add_argument("command", metavar="COMMAND", help="command to run") + subparser.add_argument("args", metavar="ARGS", nargs=argparse.REMAINDER, help="command arguments") # borg import-tar - import_tar_epilog = process_epilog(""" + import_tar_epilog = process_epilog( + """ This command creates a backup archive from a tarball. When giving '-' as path, Borg will read a tar stream from standard input. @@ -4945,51 +6046,93 @@ def define_borg_mount(parser): - UNIX V7 tar - SunOS tar with extended attributes - """) - subparser = subparsers.add_parser('import-tar', parents=[common_parser], add_help=False, - description=self.do_import_tar.__doc__, - epilog=import_tar_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help=self.do_import_tar.__doc__) + """ + ) + subparser = subparsers.add_parser( + "import-tar", + parents=[common_parser], + add_help=False, + description=self.do_import_tar.__doc__, + epilog=import_tar_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help=self.do_import_tar.__doc__, + ) subparser.set_defaults(func=self.do_import_tar) - subparser.add_argument('--tar-filter', dest='tar_filter', default='auto', action=Highlander, - help='filter program to pipe data through') - subparser.add_argument('-s', '--stats', dest='stats', - action='store_true', default=False, - help='print statistics for the created archive') - subparser.add_argument('--list', dest='output_list', - action='store_true', default=False, - help='output verbose list of items (files, dirs, ...)') - subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', action=Highlander, - help='only display items with the given status characters') - subparser.add_argument('--json', action='store_true', - help='output stats as JSON (implies --stats)') + subparser.add_argument( + "--tar-filter", + dest="tar_filter", + default="auto", + action=Highlander, + help="filter program to pipe data through", + ) + subparser.add_argument( + "-s", + "--stats", + dest="stats", + action="store_true", + default=False, + help="print statistics for the created archive", + ) + subparser.add_argument( + "--list", + dest="output_list", + action="store_true", + default=False, + help="output verbose list of items (files, dirs, ...)", + ) + subparser.add_argument( + "--filter", + dest="output_filter", + metavar="STATUSCHARS", + action=Highlander, + help="only display items with the given status characters", + ) + subparser.add_argument("--json", action="store_true", help="output stats as JSON (implies --stats)") - archive_group = subparser.add_argument_group('Archive options') - archive_group.add_argument('--comment', dest='comment', metavar='COMMENT', default='', - help='add a comment text to the archive') - archive_group.add_argument('--timestamp', dest='timestamp', - type=timestamp, default=None, - metavar='TIMESTAMP', - help='manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). ' - 'alternatively, give a reference file/directory.') - archive_group.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval', - type=int, default=1800, metavar='SECONDS', - help='write checkpoint every SECONDS seconds (Default: 1800)') - archive_group.add_argument('--chunker-params', dest='chunker_params', action=Highlander, - type=ChunkerParams, default=CHUNKER_PARAMS, - metavar='PARAMS', - help='specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, ' - 'HASH_MASK_BITS, HASH_WINDOW_SIZE). default: %s,%d,%d,%d,%d' % CHUNKER_PARAMS) - archive_group.add_argument('-C', '--compression', metavar='COMPRESSION', dest='compression', - type=CompressionSpec, default=CompressionSpec('lz4'), - help='select compression algorithm, see the output of the ' - '"borg help compression" command for details.') + archive_group = subparser.add_argument_group("Archive options") + archive_group.add_argument( + "--comment", dest="comment", metavar="COMMENT", default="", help="add a comment text to the archive" + ) + archive_group.add_argument( + "--timestamp", + dest="timestamp", + type=timestamp, + default=None, + metavar="TIMESTAMP", + help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). " + "alternatively, give a reference file/directory.", + ) + archive_group.add_argument( + "-c", + "--checkpoint-interval", + dest="checkpoint_interval", + type=int, + default=1800, + metavar="SECONDS", + help="write checkpoint every SECONDS seconds (Default: 1800)", + ) + archive_group.add_argument( + "--chunker-params", + dest="chunker_params", + action=Highlander, + type=ChunkerParams, + default=CHUNKER_PARAMS, + metavar="PARAMS", + help="specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, " + "HASH_MASK_BITS, HASH_WINDOW_SIZE). default: %s,%d,%d,%d,%d" % CHUNKER_PARAMS, + ) + archive_group.add_argument( + "-C", + "--compression", + metavar="COMPRESSION", + dest="compression", + type=CompressionSpec, + default=CompressionSpec("lz4"), + help="select compression algorithm, see the output of the " '"borg help compression" command for details.', + ) - subparser.add_argument('name', metavar='NAME', type=NameSpec, - help='specify the archive name') - subparser.add_argument('tarfile', metavar='TARFILE', - help='input tar file. "-" to read from stdin instead.') + subparser.add_argument("name", metavar="NAME", type=NameSpec, help="specify the archive name") + subparser.add_argument("tarfile", metavar="TARFILE", help='input tar file. "-" to read from stdin instead.') return parser def get_args(self, argv, cmd): @@ -5003,7 +6146,7 @@ def get_args(self, argv, cmd): client_argv = shlex.split(cmd) # Drop environment variables (do *not* interpret them) before trying to parse # the borg command line. - client_argv = list(itertools.dropwhile(lambda arg: '=' in arg, client_argv)) + client_argv = list(itertools.dropwhile(lambda arg: "=" in arg, client_argv)) client_result = self.parse_args(client_argv[1:]) if client_result.func == result.func: # make sure we only process like normal if the client is executing @@ -5012,21 +6155,11 @@ def get_args(self, argv, cmd): # client is allowed to specify the allowlisted options, # everything else comes from the forced "borg serve" command (or the defaults). # stuff from denylist must never be used from the client. - denylist = { - 'restrict_to_paths', - 'restrict_to_repositories', - 'append_only', - 'storage_quota', - 'umask', - } - allowlist = { - 'debug_topics', - 'lock_wait', - 'log_level', - } + denylist = {"restrict_to_paths", "restrict_to_repositories", "append_only", "storage_quota", "umask"} + allowlist = {"debug_topics", "lock_wait", "log_level"} not_present = object() for attr_name in allowlist: - assert attr_name not in denylist, 'allowlist has denylisted attribute name %s' % attr_name + assert attr_name not in denylist, "allowlist has denylisted attribute name %s" % attr_name value = getattr(client_result, attr_name, not_present) if value is not not_present: # note: it is not possible to specify a allowlisted option via a forced command, @@ -5040,24 +6173,34 @@ def parse_args(self, args=None): if args: args = self.preprocess_args(args) parser = self.build_parser() - args = parser.parse_args(args or ['-h']) + args = parser.parse_args(args or ["-h"]) parser.common_options.resolve(args) func = get_func(args) if func == self.do_create and args.paths and args.paths_from_stdin: - parser.error('Must not pass PATH with ``--paths-from-stdin``.') + parser.error("Must not pass PATH with ``--paths-from-stdin``.") if func == self.do_create and not args.paths: if args.content_from_command or args.paths_from_command: - parser.error('No command given.') + parser.error("No command given.") elif not args.paths_from_stdin: # need at least 1 path but args.paths may also be populated from patterns - parser.error('Need at least one PATH argument.') - if not getattr(args, 'lock', True): # Option --bypass-lock sets args.lock = False - bypass_allowed = {self.do_check, self.do_config, self.do_diff, - self.do_export_tar, self.do_extract, self.do_info, self.do_rinfo, - self.do_list, self.do_rlist, self.do_mount, self.do_umount} + parser.error("Need at least one PATH argument.") + if not getattr(args, "lock", True): # Option --bypass-lock sets args.lock = False + bypass_allowed = { + self.do_check, + self.do_config, + self.do_diff, + self.do_export_tar, + self.do_extract, + self.do_info, + self.do_rinfo, + self.do_list, + self.do_rlist, + self.do_mount, + self.do_umount, + } if func not in bypass_allowed: - raise Error('Not allowed to bypass locking mechanism for chosen command') - if getattr(args, 'timestamp', None): + raise Error("Not allowed to bypass locking mechanism for chosen command") + if getattr(args, "timestamp", None): args.location = args.location.with_timestamp(args.timestamp) return args @@ -5069,26 +6212,26 @@ def prerun_checks(self, logger, is_serve): selftest(logger) def _setup_implied_logging(self, args): - """ turn on INFO level logging for args that imply that they will produce output """ + """turn on INFO level logging for args that imply that they will produce output""" # map of option name to name of logger for that option option_logger = { - 'output_list': 'borg.output.list', - 'show_version': 'borg.output.show-version', - 'show_rc': 'borg.output.show-rc', - 'stats': 'borg.output.stats', - 'progress': 'borg.output.progress', + "output_list": "borg.output.list", + "show_version": "borg.output.show-version", + "show_rc": "borg.output.show-rc", + "stats": "borg.output.stats", + "progress": "borg.output.progress", } for option, logger_name in option_logger.items(): option_set = args.get(option, False) - logging.getLogger(logger_name).setLevel('INFO' if option_set else 'WARN') + logging.getLogger(logger_name).setLevel("INFO" if option_set else "WARN") def _setup_topic_debugging(self, args): """Turn on DEBUG level logging for specified --debug-topics.""" for topic in args.debug_topics: - if '.' not in topic: - topic = 'borg.debug.' + topic - logger.debug('Enabling debug topic %s', topic) - logging.getLogger(topic).setLevel('DEBUG') + if "." not in topic: + topic = "borg.debug." + topic + logger.debug("Enabling debug topic %s", topic) + logging.getLogger(topic).setLevel("DEBUG") def run(self, args): os.umask(args.umask) # early, before opening files @@ -5101,13 +6244,13 @@ def run(self, args): args.progress |= is_serve self._setup_implied_logging(vars(args)) self._setup_topic_debugging(args) - if getattr(args, 'stats', False) and getattr(args, 'dry_run', False): + if getattr(args, "stats", False) and getattr(args, "dry_run", False): # the data needed for --stats is not computed when using --dry-run, so we can't do it. # for ease of scripting, we just ignore --stats when given with --dry-run. logger.warning("Ignoring --stats. It is not supported when using --dry-run.") args.stats = False if args.show_version: - logging.getLogger('borg.output.show-version').info('borgbackup version %s' % __version__) + logging.getLogger("borg.output.show-version").info("borgbackup version %s" % __version__) self.prerun_checks(logger, is_serve) if not is_supported_msgpack(): logger.error("You do not have a supported version of the msgpack python package installed. Terminating.") @@ -5120,10 +6263,11 @@ def run(self, args): # Import only when needed - avoids a further increase in startup time import cProfile import marshal - logger.debug('Writing execution profile to %s', args.debug_profile) + + logger.debug("Writing execution profile to %s", args.debug_profile) # Open the file early, before running the main program, to avoid # a very late crash in case the specified path is invalid. - with open(args.debug_profile, 'wb') as fd: + with open(args.debug_profile, "wb") as fd: profiler = cProfile.Profile() variables = dict(locals()) profiler.enable() @@ -5132,7 +6276,7 @@ def run(self, args): finally: profiler.disable() profiler.snapshot_stats() - if args.debug_profile.endswith('.pyprof'): + if args.debug_profile.endswith(".pyprof"): marshal.dump(profiler.stats, fd) else: # We use msgpack here instead of the marshal module used by cProfile itself, @@ -5153,19 +6297,19 @@ def sig_info_handler(sig_no, stack): # pragma: no cover with signal_handler(sig_no, signal.SIG_IGN): for frame in inspect.getouterframes(stack): func, loc = frame[3], frame[0].f_locals - if func in ('process_file', '_rec_walk', ): # create op - path = loc['path'] + if func in ("process_file", "_rec_walk"): # create op + path = loc["path"] try: - pos = loc['fd'].tell() - total = loc['st'].st_size + pos = loc["fd"].tell() + total = loc["st"].st_size except Exception: pos, total = 0, 0 logger.info(f"{path} {format_file_size(pos)}/{format_file_size(total)}") break - if func in ('extract_item', ): # extract op - path = loc['item'].path + if func in ("extract_item",): # extract op + path = loc["item"].path try: - pos = loc['fd'].tell() + pos = loc["fd"].tell() except Exception: pos = 0 logger.info(f"{path} {format_file_size(pos)}/???") @@ -5173,15 +6317,15 @@ def sig_info_handler(sig_no, stack): # pragma: no cover def sig_trace_handler(sig_no, stack): # pragma: no cover - print('\nReceived SIGUSR2 at %s, dumping trace...' % datetime.now().replace(microsecond=0), file=sys.stderr) + print("\nReceived SIGUSR2 at %s, dumping trace..." % datetime.now().replace(microsecond=0), file=sys.stderr) faulthandler.dump_traceback() def main(): # pragma: no cover # Make sure stdout and stderr have errors='replace' to avoid unicode # issues when print()-ing unicode file names - sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) - sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) + sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, "replace", line_buffering=True) + sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, "replace", line_buffering=True) # If we receive SIGINT (ctrl-c), SIGTERM (kill) or SIGHUP (kill -HUP), # catch them and raise a proper exception that can be handled for an @@ -5192,21 +6336,24 @@ def main(): # pragma: no cover # Register fault handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL. faulthandler.enable() - with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \ - signal_handler('SIGHUP', raising_signal_handler(SigHup)), \ - signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \ - signal_handler('SIGUSR1', sig_info_handler), \ - signal_handler('SIGUSR2', sig_trace_handler), \ - signal_handler('SIGINFO', sig_info_handler): + with signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), signal_handler( + "SIGHUP", raising_signal_handler(SigHup) + ), signal_handler("SIGTERM", raising_signal_handler(SigTerm)), signal_handler( + "SIGUSR1", sig_info_handler + ), signal_handler( + "SIGUSR2", sig_trace_handler + ), signal_handler( + "SIGINFO", sig_info_handler + ): archiver = Archiver() msg = msgid = tb = None tb_log_level = logging.ERROR try: - args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) + args = archiver.get_args(sys.argv, os.environ.get("SSH_ORIGINAL_COMMAND")) except Error as e: msg = e.get_message() tb_log_level = logging.ERROR if e.traceback else logging.DEBUG - tb = f'{traceback.format_exc()}\n{sysinfo()}' + tb = f"{traceback.format_exc()}\n{sysinfo()}" # we might not have logging setup yet, so get out quickly print(msg, file=sys.stderr) if tb_log_level == logging.ERROR: @@ -5222,56 +6369,56 @@ def main(): # pragma: no cover tb = f"{traceback.format_exc()}\n{sysinfo()}" exit_code = e.exit_code except RemoteRepository.RPCError as e: - important = e.exception_class not in ('LockTimeout', ) and e.traceback + important = e.exception_class not in ("LockTimeout",) and e.traceback msgid = e.exception_class tb_log_level = logging.ERROR if important else logging.DEBUG if important: msg = e.exception_full else: msg = e.get_message() - tb = '\n'.join('Borg server: ' + l for l in e.sysinfo.splitlines()) + tb = "\n".join("Borg server: " + l for l in e.sysinfo.splitlines()) tb += "\n" + sysinfo() exit_code = EXIT_ERROR except Exception: - msg = 'Local Exception' - msgid = 'Exception' + msg = "Local Exception" + msgid = "Exception" tb_log_level = logging.ERROR - tb = f'{traceback.format_exc()}\n{sysinfo()}' + tb = f"{traceback.format_exc()}\n{sysinfo()}" exit_code = EXIT_ERROR except KeyboardInterrupt: - msg = 'Keyboard interrupt' + msg = "Keyboard interrupt" tb_log_level = logging.DEBUG - tb = f'{traceback.format_exc()}\n{sysinfo()}' + tb = f"{traceback.format_exc()}\n{sysinfo()}" exit_code = EXIT_SIGNAL_BASE + 2 except SigTerm: - msg = 'Received SIGTERM' - msgid = 'Signal.SIGTERM' + msg = "Received SIGTERM" + msgid = "Signal.SIGTERM" tb_log_level = logging.DEBUG - tb = f'{traceback.format_exc()}\n{sysinfo()}' + tb = f"{traceback.format_exc()}\n{sysinfo()}" exit_code = EXIT_SIGNAL_BASE + 15 except SigHup: - msg = 'Received SIGHUP.' - msgid = 'Signal.SIGHUP' + msg = "Received SIGHUP." + msgid = "Signal.SIGHUP" exit_code = EXIT_SIGNAL_BASE + 1 if msg: logger.error(msg, msgid=msgid) if tb: logger.log(tb_log_level, tb) if args.show_rc: - rc_logger = logging.getLogger('borg.output.show-rc') - exit_msg = 'terminating with %s status, rc %d' + rc_logger = logging.getLogger("borg.output.show-rc") + exit_msg = "terminating with %s status, rc %d" if exit_code == EXIT_SUCCESS: - rc_logger.info(exit_msg % ('success', exit_code)) + rc_logger.info(exit_msg % ("success", exit_code)) elif exit_code == EXIT_WARNING: - rc_logger.warning(exit_msg % ('warning', exit_code)) + rc_logger.warning(exit_msg % ("warning", exit_code)) elif exit_code == EXIT_ERROR: - rc_logger.error(exit_msg % ('error', exit_code)) + rc_logger.error(exit_msg % ("error", exit_code)) elif exit_code >= EXIT_SIGNAL_BASE: - rc_logger.error(exit_msg % ('signal', exit_code)) + rc_logger.error(exit_msg % ("signal", exit_code)) else: - rc_logger.error(exit_msg % ('abnormal', exit_code or 666)) + rc_logger.error(exit_msg % ("abnormal", exit_code or 666)) sys.exit(exit_code) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/borg/cache.py b/src/borg/cache.py index 5addd34e1..a6148c66d 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -10,7 +10,7 @@ logger = create_logger() -files_cache_logger = create_logger('borg.debug.files_cache') +files_cache_logger = create_logger("borg.debug.files_cache") from .constants import CACHE_README, FILES_CACHE_MODE_DISABLED from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer @@ -37,7 +37,7 @@ from .repository import LIST_SCAN_LIMIT # note: cmtime might me either a ctime or a mtime timestamp -FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size cmtime chunk_ids') +FileCacheEntry = namedtuple("FileCacheEntry", "age inode size cmtime chunk_ids") class SecurityManager: @@ -64,9 +64,9 @@ def __init__(self, repository): self.repository = repository self.dir = get_security_dir(repository.id_str) self.cache_dir = cache_dir(repository) - self.key_type_file = os.path.join(self.dir, 'key-type') - self.location_file = os.path.join(self.dir, 'location') - self.manifest_ts_file = os.path.join(self.dir, 'manifest-timestamp') + self.key_type_file = os.path.join(self.dir, "key-type") + self.location_file = os.path.join(self.dir, "location") + self.manifest_ts_file = os.path.join(self.dir, "manifest-timestamp") @staticmethod def destroy(repository, path=None): @@ -76,8 +76,7 @@ def destroy(repository, path=None): shutil.rmtree(path) def known(self): - return all(os.path.exists(f) - for f in (self.key_type_file, self.location_file, self.manifest_ts_file)) + return all(os.path.exists(f) for f in (self.key_type_file, self.location_file, self.manifest_ts_file)) def key_matches(self, key): if not self.known(): @@ -87,14 +86,14 @@ def key_matches(self, key): type = fd.read() return type == str(key.TYPE) except OSError as exc: - logger.warning('Could not read/parse key type file: %s', exc) + logger.warning("Could not read/parse key type file: %s", exc) def save(self, manifest, key): - logger.debug('security: saving state for %s to %s', self.repository.id_str, self.dir) + logger.debug("security: saving state for %s to %s", self.repository.id_str, self.dir) current_location = self.repository._location.canonical_path() - logger.debug('security: current location %s', current_location) - logger.debug('security: key type %s', str(key.TYPE)) - logger.debug('security: manifest timestamp %s', manifest.timestamp) + logger.debug("security: current location %s", current_location) + logger.debug("security: key type %s", str(key.TYPE)) + logger.debug("security: manifest timestamp %s", manifest.timestamp) with SaveFile(self.location_file) as fd: fd.write(current_location) with SaveFile(self.key_type_file) as fd: @@ -107,28 +106,36 @@ def assert_location_matches(self, cache_config=None): try: with open(self.location_file) as fd: previous_location = fd.read() - logger.debug('security: read previous location %r', previous_location) + logger.debug("security: read previous location %r", previous_location) except FileNotFoundError: - logger.debug('security: previous location file %s not found', self.location_file) + logger.debug("security: previous location file %s not found", self.location_file) previous_location = None except OSError as exc: - logger.warning('Could not read previous location file: %s', exc) + logger.warning("Could not read previous location file: %s", exc) previous_location = None if cache_config and cache_config.previous_location and previous_location != cache_config.previous_location: # Reconcile cache and security dir; we take the cache location. previous_location = cache_config.previous_location - logger.debug('security: using previous_location of cache: %r', previous_location) + logger.debug("security: using previous_location of cache: %r", previous_location) repository_location = self.repository._location.canonical_path() if previous_location and previous_location != repository_location: - msg = ("Warning: The repository at location {} was previously located at {}\n".format( - repository_location, previous_location) + - "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", - retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): + msg = ( + "Warning: The repository at location {} was previously located at {}\n".format( + repository_location, previous_location + ) + + "Do you want to continue? [yN] " + ) + if not yes( + msg, + false_msg="Aborting.", + invalid_msg="Invalid answer, aborting.", + retry=False, + env_var_override="BORG_RELOCATED_REPO_ACCESS_IS_OK", + ): raise Cache.RepositoryAccessAborted() # adapt on-disk config immediately if the new location was accepted - logger.debug('security: updating location stored in cache and security dir') + logger.debug("security: updating location stored in cache and security dir") with SaveFile(self.location_file) as fd: fd.write(repository_location) if cache_config: @@ -138,16 +145,16 @@ def assert_no_manifest_replay(self, manifest, key, cache_config=None): try: with open(self.manifest_ts_file) as fd: timestamp = fd.read() - logger.debug('security: read manifest timestamp %r', timestamp) + logger.debug("security: read manifest timestamp %r", timestamp) except FileNotFoundError: - logger.debug('security: manifest timestamp file %s not found', self.manifest_ts_file) - timestamp = '' + logger.debug("security: manifest timestamp file %s not found", self.manifest_ts_file) + timestamp = "" except OSError as exc: - logger.warning('Could not read previous location file: %s', exc) - timestamp = '' + logger.warning("Could not read previous location file: %s", exc) + timestamp = "" if cache_config: - timestamp = max(timestamp, cache_config.timestamp or '') - logger.debug('security: determined newest manifest timestamp as %s', timestamp) + timestamp = max(timestamp, cache_config.timestamp or "") + logger.debug("security: determined newest manifest timestamp as %s", timestamp) # If repository is older than the cache or security dir something fishy is going on if timestamp and timestamp > manifest.timestamp: if isinstance(key, PlaintextKey): @@ -175,30 +182,36 @@ def assert_secure(self, manifest, key, *, cache_config=None, warn_if_unencrypted self._assert_secure(manifest, key, cache_config) else: self._assert_secure(manifest, key) - logger.debug('security: repository checks ok, allowing access') + logger.debug("security: repository checks ok, allowing access") def _assert_secure(self, manifest, key, cache_config=None): self.assert_location_matches(cache_config) self.assert_key_type(key, cache_config) self.assert_no_manifest_replay(manifest, key, cache_config) if not self.known(): - logger.debug('security: remembering previously unknown repository') + logger.debug("security: remembering previously unknown repository") self.save(manifest, key) def assert_access_unknown(self, warn_if_unencrypted, manifest, key): # warn_if_unencrypted=False is only used for initializing a new repository. # Thus, avoiding asking about a repository that's currently initializing. if not key.logically_encrypted and not self.known(): - msg = ("Warning: Attempting to access a previously unknown unencrypted repository!\n" + - "Do you want to continue? [yN] ") - allow_access = not warn_if_unencrypted or yes(msg, false_msg="Aborting.", + msg = ( + "Warning: Attempting to access a previously unknown unencrypted repository!\n" + + "Do you want to continue? [yN] " + ) + allow_access = not warn_if_unencrypted or yes( + msg, + false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", - retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK') + retry=False, + env_var_override="BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK", + ) if allow_access: if warn_if_unencrypted: - logger.debug('security: remembering unknown unencrypted repository (explicitly allowed)') + logger.debug("security: remembering unknown unencrypted repository (explicitly allowed)") else: - logger.debug('security: initializing unencrypted repository') + logger.debug("security: initializing unencrypted repository") self.save(manifest, key) else: raise Cache.CacheInitAbortedError() @@ -214,10 +227,17 @@ def recanonicalize_relative_location(cache_location, repository): repo_location = repository._location.canonical_path() rl = Location(repo_location) cl = Location(cache_location) - if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \ - and \ - cl.path and rl.path and \ - cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]: + if ( + cl.proto == rl.proto + and cl.user == rl.user + and cl.host == rl.host + and cl.port == rl.port + and cl.path + and rl.path + and cl.path.startswith("/~/") + and rl.path.startswith("/./") + and cl.path[3:] == rl.path[3:] + ): # everything is same except the expected change in relative path canonicalization, # update previous_location to avoid warning / user query about changed location: return repo_location @@ -230,19 +250,19 @@ def cache_dir(repository, path=None): def files_cache_name(): - suffix = os.environ.get('BORG_FILES_CACHE_SUFFIX', '') - return 'files.' + suffix if suffix else 'files' + suffix = os.environ.get("BORG_FILES_CACHE_SUFFIX", "") + return "files." + suffix if suffix else "files" def discover_files_cache_name(path): - return [fn for fn in os.listdir(path) if fn == 'files' or fn.startswith('files.')][0] + return [fn for fn in os.listdir(path) if fn == "files" or fn.startswith("files.")][0] class CacheConfig: def __init__(self, repository, path=None, lock_wait=None): self.repository = repository self.path = cache_dir(repository, path) - self.config_path = os.path.join(self.path, 'config') + self.config_path = os.path.join(self.path, "config") self.lock = None self.lock_wait = lock_wait @@ -259,17 +279,17 @@ def exists(self): def create(self): assert not self.exists() config = configparser.ConfigParser(interpolation=None) - config.add_section('cache') - config.set('cache', 'version', '1') - config.set('cache', 'repository', self.repository.id_str) - config.set('cache', 'manifest', '') - config.add_section('integrity') - config.set('integrity', 'manifest', '') + config.add_section("cache") + config.set("cache", "version", "1") + config.set("cache", "repository", self.repository.id_str) + config.set("cache", "manifest", "") + config.add_section("integrity") + config.set("integrity", "manifest", "") with SaveFile(self.config_path) as fd: config.write(fd) def open(self): - self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=self.lock_wait).acquire() + self.lock = Lock(os.path.join(self.path, "lock"), exclusive=True, timeout=self.lock_wait).acquire() self.load() def load(self): @@ -277,15 +297,17 @@ def load(self): with open(self.config_path) as fd: self._config.read_file(fd) self._check_upgrade(self.config_path) - self.id = self._config.get('cache', 'repository') - self.manifest_id = unhexlify(self._config.get('cache', 'manifest')) - self.timestamp = self._config.get('cache', 'timestamp', fallback=None) - self.key_type = self._config.get('cache', 'key_type', fallback=None) - self.ignored_features = set(parse_stringified_list(self._config.get('cache', 'ignored_features', fallback=''))) - self.mandatory_features = set(parse_stringified_list(self._config.get('cache', 'mandatory_features', fallback=''))) + self.id = self._config.get("cache", "repository") + self.manifest_id = unhexlify(self._config.get("cache", "manifest")) + self.timestamp = self._config.get("cache", "timestamp", fallback=None) + self.key_type = self._config.get("cache", "key_type", fallback=None) + self.ignored_features = set(parse_stringified_list(self._config.get("cache", "ignored_features", fallback=""))) + self.mandatory_features = set( + parse_stringified_list(self._config.get("cache", "mandatory_features", fallback="")) + ) try: - self.integrity = dict(self._config.items('integrity')) - if self._config.get('cache', 'manifest') != self.integrity.pop('manifest'): + self.integrity = dict(self._config.items("integrity")) + if self._config.get("cache", "manifest") != self.integrity.pop("manifest"): # The cache config file is updated (parsed with ConfigParser, the state of the ConfigParser # is modified and then written out.), not re-created. # Thus, older versions will leave our [integrity] section alone, making the section's data invalid. @@ -293,30 +315,30 @@ def load(self): # can discern whether an older version interfered by comparing the manifest IDs of this section # and the main [cache] section. self.integrity = {} - logger.warning('Cache integrity data not available: old Borg version modified the cache.') + logger.warning("Cache integrity data not available: old Borg version modified the cache.") except configparser.NoSectionError: - logger.debug('Cache integrity: No integrity data found (files, chunks). Cache is from old version.') + logger.debug("Cache integrity: No integrity data found (files, chunks). Cache is from old version.") self.integrity = {} - previous_location = self._config.get('cache', 'previous_location', fallback=None) + previous_location = self._config.get("cache", "previous_location", fallback=None) if previous_location: self.previous_location = recanonicalize_relative_location(previous_location, self.repository) else: self.previous_location = None - self._config.set('cache', 'previous_location', self.repository._location.canonical_path()) + self._config.set("cache", "previous_location", self.repository._location.canonical_path()) def save(self, manifest=None, key=None): if manifest: - self._config.set('cache', 'manifest', manifest.id_str) - self._config.set('cache', 'timestamp', manifest.timestamp) - self._config.set('cache', 'ignored_features', ','.join(self.ignored_features)) - self._config.set('cache', 'mandatory_features', ','.join(self.mandatory_features)) - if not self._config.has_section('integrity'): - self._config.add_section('integrity') + self._config.set("cache", "manifest", manifest.id_str) + self._config.set("cache", "timestamp", manifest.timestamp) + self._config.set("cache", "ignored_features", ",".join(self.ignored_features)) + self._config.set("cache", "mandatory_features", ",".join(self.mandatory_features)) + if not self._config.has_section("integrity"): + self._config.add_section("integrity") for file, integrity_data in self.integrity.items(): - self._config.set('integrity', file, integrity_data) - self._config.set('integrity', 'manifest', manifest.id_str) + self._config.set("integrity", file, integrity_data) + self._config.set("integrity", "manifest", manifest.id_str) if key: - self._config.set('cache', 'key_type', str(key.TYPE)) + self._config.set("cache", "key_type", str(key.TYPE)) with SaveFile(self.config_path) as fd: self._config.write(fd) @@ -327,20 +349,21 @@ def close(self): def _check_upgrade(self, config_path): try: - cache_version = self._config.getint('cache', 'version') + cache_version = self._config.getint("cache", "version") wanted_version = 1 if cache_version != wanted_version: self.close() - raise Exception('%s has unexpected cache version %d (wanted: %d).' % - (config_path, cache_version, wanted_version)) + raise Exception( + "%s has unexpected cache version %d (wanted: %d)." % (config_path, cache_version, wanted_version) + ) except configparser.NoSectionError: self.close() - raise Exception('%s does not look like a Borg cache.' % config_path) from None + raise Exception("%s does not look like a Borg cache." % config_path) from None class Cache: - """Client Side cache - """ + """Client Side cache""" + class RepositoryIDNotUnique(Error): """Cache is newer than repository - do you have multiple, independently updated repos with same ID?""" @@ -359,29 +382,56 @@ class EncryptionMethodMismatch(Error): @staticmethod def break_lock(repository, path=None): path = cache_dir(repository, path) - Lock(os.path.join(path, 'lock'), exclusive=True).break_lock() + Lock(os.path.join(path, "lock"), exclusive=True).break_lock() @staticmethod def destroy(repository, path=None): """destroy the cache for ``repository`` or at ``path``""" path = path or os.path.join(get_cache_dir(), repository.id_str) - config = os.path.join(path, 'config') + config = os.path.join(path, "config") if os.path.exists(config): os.remove(config) # kill config first shutil.rmtree(path) - def __new__(cls, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True, - progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=FILES_CACHE_MODE_DISABLED, - consider_part_files=False, iec=False): - + def __new__( + cls, + repository, + key, + manifest, + path=None, + sync=True, + warn_if_unencrypted=True, + progress=False, + lock_wait=None, + permit_adhoc_cache=False, + cache_mode=FILES_CACHE_MODE_DISABLED, + consider_part_files=False, + iec=False, + ): def local(): - return LocalCache(repository=repository, key=key, manifest=manifest, path=path, sync=sync, - warn_if_unencrypted=warn_if_unencrypted, progress=progress, iec=iec, - lock_wait=lock_wait, cache_mode=cache_mode, consider_part_files=consider_part_files) + return LocalCache( + repository=repository, + key=key, + manifest=manifest, + path=path, + sync=sync, + warn_if_unencrypted=warn_if_unencrypted, + progress=progress, + iec=iec, + lock_wait=lock_wait, + cache_mode=cache_mode, + consider_part_files=consider_part_files, + ) def adhoc(): - return AdHocCache(repository=repository, key=key, manifest=manifest, lock_wait=lock_wait, iec=iec, - consider_part_files=consider_part_files) + return AdHocCache( + repository=repository, + key=key, + manifest=manifest, + lock_wait=lock_wait, + iec=iec, + consider_part_files=consider_part_files, + ) if not permit_adhoc_cache: return local() @@ -397,9 +447,9 @@ def adhoc(): # Don't nest cache locks if cache_in_sync: # Local cache is in sync, use it - logger.debug('Cache: choosing local cache (in sync)') + logger.debug("Cache: choosing local cache (in sync)") return local() - logger.debug('Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)') + logger.debug("Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)") return adhoc() @@ -417,10 +467,11 @@ def __init__(self, iec=False): def __str__(self): return self.str_format.format(self.format_tuple()) - Summary = namedtuple('Summary', ['total_size', 'unique_size', 'total_unique_chunks', 'total_chunks']) + Summary = namedtuple("Summary", ["total_size", "unique_size", "total_unique_chunks", "total_chunks"]) def stats(self): from .archive import Archive + # XXX: this should really be moved down to `hashindex.pyx` total_size, unique_size, total_unique_chunks, total_chunks = self.chunks.summarize() # the above values have the problem that they do not consider part files, @@ -430,8 +481,9 @@ def stats(self): # so we can just sum up all archives to get the "all archives" stats: total_size = 0 for archive_name in self.manifest.archives: - archive = Archive(self.repository, self.key, self.manifest, archive_name, - consider_part_files=self.consider_part_files) + archive = Archive( + self.repository, self.key, self.manifest, archive_name, consider_part_files=self.consider_part_files + ) stats = archive.calc_stats(self, want_unique=False) total_size += stats.osize stats = self.Summary(total_size, unique_size, total_unique_chunks, total_chunks)._asdict() @@ -439,7 +491,7 @@ def stats(self): def format_tuple(self): stats = self.stats() - for field in ['total_size', 'unique_size']: + for field in ["total_size", "unique_size"]: stats[field] = format_file_size(stats[field], iec=self.iec) return self.Summary(**stats) @@ -449,9 +501,20 @@ class LocalCache(CacheStatsMixin): Persistent, local (client-side) cache. """ - def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True, - progress=False, lock_wait=None, cache_mode=FILES_CACHE_MODE_DISABLED, consider_part_files=False, - iec=False): + def __init__( + self, + repository, + key, + manifest, + path=None, + sync=True, + warn_if_unencrypted=True, + progress=False, + lock_wait=None, + cache_mode=FILES_CACHE_MODE_DISABLED, + consider_part_files=False, + iec=False, + ): """ :param warn_if_unencrypted: print warning if accessing unknown unencrypted repository :param lock_wait: timeout for lock acquisition (int [s] or None [wait forever]) @@ -500,30 +563,32 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() def create(self): - """Create a new empty cache at `self.path` - """ + """Create a new empty cache at `self.path`""" os.makedirs(self.path) - with open(os.path.join(self.path, 'README'), 'w') as fd: + with open(os.path.join(self.path, "README"), "w") as fd: fd.write(CACHE_README) self.cache_config.create() - ChunkIndex().write(os.path.join(self.path, 'chunks')) - os.makedirs(os.path.join(self.path, 'chunks.archive.d')) + ChunkIndex().write(os.path.join(self.path, "chunks")) + os.makedirs(os.path.join(self.path, "chunks.archive.d")) with SaveFile(os.path.join(self.path, files_cache_name()), binary=True): pass # empty file def _do_open(self): self.cache_config.load() - with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=False, - integrity_data=self.cache_config.integrity.get('chunks')) as fd: + with IntegrityCheckedFile( + path=os.path.join(self.path, "chunks"), + write=False, + integrity_data=self.cache_config.integrity.get("chunks"), + ) as fd: self.chunks = ChunkIndex.read(fd) - if 'd' in self.cache_mode: # d(isabled) + if "d" in self.cache_mode: # d(isabled) self.files = None else: self._read_files() def open(self): if not os.path.isdir(self.path): - raise Exception('%s Does not look like a Borg cache' % self.path) + raise Exception("%s Does not look like a Borg cache" % self.path) self.cache_config.open() self.rollback() @@ -535,12 +600,15 @@ def close(self): def _read_files(self): self.files = {} self._newest_cmtime = None - logger.debug('Reading files cache ...') + logger.debug("Reading files cache ...") files_cache_logger.debug("FILES-CACHE-LOAD: starting...") msg = None try: - with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=False, - integrity_data=self.cache_config.integrity.get(files_cache_name())) as fd: + with IntegrityCheckedFile( + path=os.path.join(self.path, files_cache_name()), + write=False, + integrity_data=self.cache_config.integrity.get(files_cache_name()), + ) as fd: u = msgpack.Unpacker(use_list=True) while True: data = fd.read(64 * 1024) @@ -561,43 +629,41 @@ def _read_files(self): msg = "The files cache is corrupted. [%s]" % str(fie) if msg is not None: logger.warning(msg) - logger.warning('Continuing without files cache - expect lower performance.') + logger.warning("Continuing without files cache - expect lower performance.") self.files = {} files_cache_logger.debug("FILES-CACHE-LOAD: finished, %d entries loaded.", len(self.files)) def begin_txn(self): # Initialize transaction snapshot - pi = ProgressIndicatorMessage(msgid='cache.begin_transaction') - txn_dir = os.path.join(self.path, 'txn.tmp') + pi = ProgressIndicatorMessage(msgid="cache.begin_transaction") + txn_dir = os.path.join(self.path, "txn.tmp") os.mkdir(txn_dir) - pi.output('Initializing cache transaction: Reading config') - shutil.copy(os.path.join(self.path, 'config'), txn_dir) - pi.output('Initializing cache transaction: Reading chunks') - shutil.copy(os.path.join(self.path, 'chunks'), txn_dir) - pi.output('Initializing cache transaction: Reading files') + pi.output("Initializing cache transaction: Reading config") + shutil.copy(os.path.join(self.path, "config"), txn_dir) + pi.output("Initializing cache transaction: Reading chunks") + shutil.copy(os.path.join(self.path, "chunks"), txn_dir) + pi.output("Initializing cache transaction: Reading files") try: shutil.copy(os.path.join(self.path, files_cache_name()), txn_dir) except FileNotFoundError: with SaveFile(os.path.join(txn_dir, files_cache_name()), binary=True): pass # empty file - os.rename(os.path.join(self.path, 'txn.tmp'), - os.path.join(self.path, 'txn.active')) + os.rename(os.path.join(self.path, "txn.tmp"), os.path.join(self.path, "txn.active")) self.txn_active = True pi.finish() def commit(self): - """Commit transaction - """ + """Commit transaction""" if not self.txn_active: return self.security_manager.save(self.manifest, self.key) - pi = ProgressIndicatorMessage(msgid='cache.commit') + pi = ProgressIndicatorMessage(msgid="cache.commit") if self.files is not None: if self._newest_cmtime is None: # was never set because no files were modified/added - self._newest_cmtime = 2 ** 63 - 1 # nanoseconds, good until y2262 - ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20)) - pi.output('Saving files cache') + self._newest_cmtime = 2**63 - 1 # nanoseconds, good until y2262 + ttl = int(os.environ.get("BORG_FILES_CACHE_TTL", 20)) + pi.output("Saving files cache") files_cache_logger.debug("FILES-CACHE-SAVE: starting...") with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=True) as fd: entry_count = 0 @@ -606,41 +672,45 @@ def commit(self): # this is to avoid issues with filesystem snapshots and cmtime granularity. # Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet. entry = FileCacheEntry(*msgpack.unpackb(item)) - if entry.age == 0 and timestamp_to_int(entry.cmtime) < self._newest_cmtime or \ - entry.age > 0 and entry.age < ttl: + if ( + entry.age == 0 + and timestamp_to_int(entry.cmtime) < self._newest_cmtime + or entry.age > 0 + and entry.age < ttl + ): msgpack.pack((path_hash, entry), fd) entry_count += 1 files_cache_logger.debug("FILES-CACHE-KILL: removed all old entries with age >= TTL [%d]", ttl) - files_cache_logger.debug("FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime) + files_cache_logger.debug( + "FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime + ) files_cache_logger.debug("FILES-CACHE-SAVE: finished, %d remaining entries saved.", entry_count) self.cache_config.integrity[files_cache_name()] = fd.integrity_data - pi.output('Saving chunks cache') - with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=True) as fd: + pi.output("Saving chunks cache") + with IntegrityCheckedFile(path=os.path.join(self.path, "chunks"), write=True) as fd: self.chunks.write(fd) - self.cache_config.integrity['chunks'] = fd.integrity_data - pi.output('Saving cache config') + self.cache_config.integrity["chunks"] = fd.integrity_data + pi.output("Saving cache config") self.cache_config.save(self.manifest, self.key) - os.rename(os.path.join(self.path, 'txn.active'), - os.path.join(self.path, 'txn.tmp')) - shutil.rmtree(os.path.join(self.path, 'txn.tmp')) + os.rename(os.path.join(self.path, "txn.active"), os.path.join(self.path, "txn.tmp")) + shutil.rmtree(os.path.join(self.path, "txn.tmp")) self.txn_active = False pi.finish() def rollback(self): - """Roll back partial and aborted transactions - """ + """Roll back partial and aborted transactions""" # Remove partial transaction - if os.path.exists(os.path.join(self.path, 'txn.tmp')): - shutil.rmtree(os.path.join(self.path, 'txn.tmp')) + if os.path.exists(os.path.join(self.path, "txn.tmp")): + shutil.rmtree(os.path.join(self.path, "txn.tmp")) # Roll back active transaction - txn_dir = os.path.join(self.path, 'txn.active') + txn_dir = os.path.join(self.path, "txn.active") if os.path.exists(txn_dir): - shutil.copy(os.path.join(txn_dir, 'config'), self.path) - shutil.copy(os.path.join(txn_dir, 'chunks'), self.path) + shutil.copy(os.path.join(txn_dir, "config"), self.path) + shutil.copy(os.path.join(txn_dir, "chunks"), self.path) shutil.copy(os.path.join(txn_dir, discover_files_cache_name(txn_dir)), self.path) - os.rename(txn_dir, os.path.join(self.path, 'txn.tmp')) - if os.path.exists(os.path.join(self.path, 'txn.tmp')): - shutil.rmtree(os.path.join(self.path, 'txn.tmp')) + os.rename(txn_dir, os.path.join(self.path, "txn.tmp")) + if os.path.exists(os.path.join(self.path, "txn.tmp")): + shutil.rmtree(os.path.join(self.path, "txn.tmp")) self.txn_active = False self._do_open() @@ -654,13 +724,13 @@ def sync(self): get removed and a new master chunks index is built by merging all archive indexes. """ - archive_path = os.path.join(self.path, 'chunks.archive.d') + archive_path = os.path.join(self.path, "chunks.archive.d") # Instrumentation processed_item_metadata_bytes = 0 processed_item_metadata_chunks = 0 compact_chunks_archive_saved_space = 0 - def mkpath(id, suffix=''): + def mkpath(id, suffix=""): id_hex = bin_to_hex(id) path = os.path.join(archive_path, id_hex + suffix) return path @@ -670,8 +740,9 @@ def cached_archives(): fns = os.listdir(archive_path) # filenames with 64 hex digits == 256bit, # or compact indices which are 64 hex digits + ".compact" - return {unhexlify(fn) for fn in fns if len(fn) == 64} | \ - {unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith('.compact')} + return {unhexlify(fn) for fn in fns if len(fn) == 64} | { + unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith(".compact") + } else: return set() @@ -685,14 +756,14 @@ def cleanup_outdated(ids): def cleanup_cached_archive(id, cleanup_compact=True): try: os.unlink(mkpath(id)) - os.unlink(mkpath(id) + '.integrity') + os.unlink(mkpath(id) + ".integrity") except FileNotFoundError: pass if not cleanup_compact: return try: - os.unlink(mkpath(id, suffix='.compact')) - os.unlink(mkpath(id, suffix='.compact') + '.integrity') + os.unlink(mkpath(id, suffix=".compact")) + os.unlink(mkpath(id, suffix=".compact") + ".integrity") except FileNotFoundError: pass @@ -703,7 +774,7 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx): chunk_idx.add(archive_id, 1, len(data)) archive = ArchiveItem(internal_dict=msgpack.unpackb(data)) if archive.version not in (1, 2): # legacy - raise Exception('Unknown archive metadata version') + raise Exception("Unknown archive metadata version") sync = CacheSynchronizer(chunk_idx) for item_id, (csize, data) in zip(archive.items, decrypted_repository.get_many(archive.items)): chunk_idx.add(item_id, 1, len(data)) @@ -716,11 +787,12 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx): def write_archive_index(archive_id, chunk_idx): nonlocal compact_chunks_archive_saved_space compact_chunks_archive_saved_space += chunk_idx.compact() - fn = mkpath(archive_id, suffix='.compact') - fn_tmp = mkpath(archive_id, suffix='.tmp') + fn = mkpath(archive_id, suffix=".compact") + fn_tmp = mkpath(archive_id, suffix=".tmp") try: - with DetachedIntegrityCheckedFile(path=fn_tmp, write=True, - filename=bin_to_hex(archive_id) + '.compact') as fd: + with DetachedIntegrityCheckedFile( + path=fn_tmp, write=True, filename=bin_to_hex(archive_id) + ".compact" + ) as fd: chunk_idx.write(fd) except Exception: safe_unlink(fn_tmp) @@ -733,7 +805,7 @@ def read_archive_index(archive_id, archive_name): try: try: # Attempt to load compact index first - with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + '.compact', write=False) as fd: + with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + ".compact", write=False) as fd: archive_chunk_idx = ChunkIndex.read(fd, permit_compact=True) # In case a non-compact index exists, delete it. cleanup_cached_archive(archive_id, cleanup_compact=False) @@ -744,14 +816,14 @@ def read_archive_index(archive_id, archive_name): with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd: archive_chunk_idx = ChunkIndex.read(fd) except FileIntegrityError as fie: - logger.error('Cached archive chunk index of %s is corrupted: %s', archive_name, fie) + logger.error("Cached archive chunk index of %s is corrupted: %s", archive_name, fie) # Delete corrupted index, set warning. A new index must be build. cleanup_cached_archive(archive_id) set_ec(EXIT_WARNING) return None # Convert to compact index. Delete the existing index first. - logger.debug('Found non-compact index for %s, converting to compact.', archive_name) + logger.debug("Found non-compact index for %s, converting to compact.", archive_name) cleanup_cached_archive(archive_id) write_archive_index(archive_id, archive_chunk_idx) return archive_chunk_idx @@ -769,12 +841,16 @@ def get_archive_ids_to_names(archive_ids): return archive_names def create_master_idx(chunk_idx): - logger.info('Synchronizing chunks cache...') + logger.info("Synchronizing chunks cache...") cached_ids = cached_archives() archive_ids = repo_archives() - logger.info('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.', - len(archive_ids), len(cached_ids), - len(cached_ids - archive_ids), len(archive_ids - cached_ids)) + logger.info( + "Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.", + len(archive_ids), + len(cached_ids), + len(cached_ids - archive_ids), + len(archive_ids - cached_ids), + ) # deallocates old hashindex, creates empty hashindex: chunk_idx.clear() cleanup_outdated(cached_ids - archive_ids) @@ -783,9 +859,12 @@ def create_master_idx(chunk_idx): master_index_capacity = len(self.repository) if archive_ids: chunk_idx = None if not self.do_cache else ChunkIndex(usable=master_index_capacity) - pi = ProgressIndicatorPercent(total=len(archive_ids), step=0.1, - msg='%3.0f%% Syncing chunks cache. Processing archive %s', - msgid='cache.sync') + pi = ProgressIndicatorPercent( + total=len(archive_ids), + step=0.1, + msg="%3.0f%% Syncing chunks cache. Processing archive %s", + msgid="cache.sync", + ) archive_ids_to_names = get_archive_ids_to_names(archive_ids) for archive_id, archive_name in archive_ids_to_names.items(): pi.show(info=[remove_surrogates(archive_name)]) @@ -797,31 +876,36 @@ def create_master_idx(chunk_idx): if archive_id not in cached_ids: # Do not make this an else branch; the FileIntegrityError exception handler # above can remove *archive_id* from *cached_ids*. - logger.info('Fetching and building archive index for %s ...', archive_name) + logger.info("Fetching and building archive index for %s ...", archive_name) archive_chunk_idx = ChunkIndex() fetch_and_build_idx(archive_id, decrypted_repository, archive_chunk_idx) logger.info("Merging into master chunks index ...") chunk_idx.merge(archive_chunk_idx) else: chunk_idx = chunk_idx or ChunkIndex(usable=master_index_capacity) - logger.info('Fetching archive index for %s ...', archive_name) + logger.info("Fetching archive index for %s ...", archive_name) fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx) pi.finish() - logger.debug('Cache sync: processed %s (%d chunks) of metadata', - format_file_size(processed_item_metadata_bytes), processed_item_metadata_chunks) - logger.debug('Cache sync: compact chunks.archive.d storage saved %s bytes', - format_file_size(compact_chunks_archive_saved_space)) - logger.info('Done.') + logger.debug( + "Cache sync: processed %s (%d chunks) of metadata", + format_file_size(processed_item_metadata_bytes), + processed_item_metadata_chunks, + ) + logger.debug( + "Cache sync: compact chunks.archive.d storage saved %s bytes", + format_file_size(compact_chunks_archive_saved_space), + ) + logger.info("Done.") return chunk_idx def legacy_cleanup(): """bring old cache dirs into the desired state (cleanup and adapt)""" try: - os.unlink(os.path.join(self.path, 'chunks.archive')) + os.unlink(os.path.join(self.path, "chunks.archive")) except: pass try: - os.unlink(os.path.join(self.path, 'chunks.archive.tmp')) + os.unlink(os.path.join(self.path, "chunks.archive.tmp")) except: pass try: @@ -832,7 +916,7 @@ def legacy_cleanup(): # The cache can be used by a command that e.g. only checks against Manifest.Operation.WRITE, # which does not have to include all flags from Manifest.Operation.READ. # Since the sync will attempt to read archives, check compatibility with Manifest.Operation.READ. - self.manifest.check_repository_compatibility((Manifest.Operation.READ, )) + self.manifest.check_repository_compatibility((Manifest.Operation.READ,)) self.begin_txn() with cache_if_remote(self.repository, decrypted_cache=self.key) as decrypted_repository: @@ -856,15 +940,15 @@ def check_cache_compatibility(self): def wipe_cache(self): logger.warning("Discarding incompatible cache and forcing a cache rebuild") - archive_path = os.path.join(self.path, 'chunks.archive.d') + archive_path = os.path.join(self.path, "chunks.archive.d") if os.path.isdir(archive_path): - shutil.rmtree(os.path.join(self.path, 'chunks.archive.d')) - os.makedirs(os.path.join(self.path, 'chunks.archive.d')) + shutil.rmtree(os.path.join(self.path, "chunks.archive.d")) + os.makedirs(os.path.join(self.path, "chunks.archive.d")) self.chunks = ChunkIndex() with SaveFile(os.path.join(self.path, files_cache_name()), binary=True): pass # empty file - self.cache_config.manifest_id = '' - self.cache_config._config.set('cache', 'manifest', '') + self.cache_config.manifest_id = "" + self.cache_config._config.set("cache", "manifest", "") self.cache_config.ignored_features = set() self.cache_config.mandatory_features = set() @@ -900,8 +984,9 @@ def seen_chunk(self, id, size=None): if size is not None and stored_size is not None and size != stored_size: # we already have a chunk with that id, but different size. # this is either a hash collision (unlikely) or corruption or a bug. - raise Exception("chunk has same id [%r], but different size (stored: %d new: %d)!" % ( - id, stored_size, size)) + raise Exception( + "chunk has same id [%r], but different size (stored: %d new: %d)!" % (id, stored_size, size) + ) return refcount def chunk_incref(self, id, stats, size=None, part=False): @@ -936,32 +1021,32 @@ def file_known_and_unchanged(self, hashed_path, path_hash, st): if not stat.S_ISREG(st.st_mode): return False, None cache_mode = self.cache_mode - if 'd' in cache_mode: # d(isabled) - files_cache_logger.debug('UNKNOWN: files cache disabled') + if "d" in cache_mode: # d(isabled) + files_cache_logger.debug("UNKNOWN: files cache disabled") return False, None # note: r(echunk) does not need the files cache in this method, but the files cache will # be updated and saved to disk to memorize the files. To preserve previous generations in # the cache, this means that it also needs to get loaded from disk first. - if 'r' in cache_mode: # r(echunk) - files_cache_logger.debug('UNKNOWN: rechunking enforced') + if "r" in cache_mode: # r(echunk) + files_cache_logger.debug("UNKNOWN: rechunking enforced") return False, None entry = self.files.get(path_hash) if not entry: - files_cache_logger.debug('UNKNOWN: no file metadata in cache for: %r', hashed_path) + files_cache_logger.debug("UNKNOWN: no file metadata in cache for: %r", hashed_path) return False, None # we know the file! entry = FileCacheEntry(*msgpack.unpackb(entry)) - if 's' in cache_mode and entry.size != st.st_size: - files_cache_logger.debug('KNOWN-CHANGED: file size has changed: %r', hashed_path) + if "s" in cache_mode and entry.size != st.st_size: + files_cache_logger.debug("KNOWN-CHANGED: file size has changed: %r", hashed_path) return True, None - if 'i' in cache_mode and entry.inode != st.st_ino: - files_cache_logger.debug('KNOWN-CHANGED: file inode number has changed: %r', hashed_path) + if "i" in cache_mode and entry.inode != st.st_ino: + files_cache_logger.debug("KNOWN-CHANGED: file inode number has changed: %r", hashed_path) return True, None - if 'c' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns: - files_cache_logger.debug('KNOWN-CHANGED: file ctime has changed: %r', hashed_path) + if "c" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns: + files_cache_logger.debug("KNOWN-CHANGED: file ctime has changed: %r", hashed_path) return True, None - elif 'm' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns: - files_cache_logger.debug('KNOWN-CHANGED: file mtime has changed: %r', hashed_path) + elif "m" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns: + files_cache_logger.debug("KNOWN-CHANGED: file mtime has changed: %r", hashed_path) return True, None # we ignored the inode number in the comparison above or it is still same. # if it is still the same, replacing it in the tuple doesn't change it. @@ -979,21 +1064,26 @@ def memorize_file(self, hashed_path, path_hash, st, ids): return cache_mode = self.cache_mode # note: r(echunk) modes will update the files cache, d(isabled) mode won't - if 'd' in cache_mode: - files_cache_logger.debug('FILES-CACHE-NOUPDATE: files cache disabled') + if "d" in cache_mode: + files_cache_logger.debug("FILES-CACHE-NOUPDATE: files cache disabled") return - if 'c' in cache_mode: - cmtime_type = 'ctime' + if "c" in cache_mode: + cmtime_type = "ctime" cmtime_ns = safe_ns(st.st_ctime_ns) - elif 'm' in cache_mode: - cmtime_type = 'mtime' + elif "m" in cache_mode: + cmtime_type = "mtime" cmtime_ns = safe_ns(st.st_mtime_ns) - entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids) + entry = FileCacheEntry( + age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids + ) self.files[path_hash] = msgpack.packb(entry) self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns) - files_cache_logger.debug('FILES-CACHE-UPDATE: put %r [has %s] <- %r', - entry._replace(chunk_ids='[%d entries]' % len(entry.chunk_ids)), - cmtime_type, hashed_path) + files_cache_logger.debug( + "FILES-CACHE-UPDATE: put %r [has %s] <- %r", + entry._replace(chunk_ids="[%d entries]" % len(entry.chunk_ids)), + cmtime_type, + hashed_path, + ) class AdHocCache(CacheStatsMixin): @@ -1012,8 +1102,9 @@ class AdHocCache(CacheStatsMixin): Unique chunks Total chunks Chunk index: {0.total_unique_chunks:20d} unknown""" - def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False, - iec=False): + def __init__( + self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False, iec=False + ): CacheStatsMixin.__init__(self, iec=iec) self.repository = repository self.key = key @@ -1024,7 +1115,7 @@ def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wai self.security_manager = SecurityManager(repository) self.security_manager.assert_secure(manifest, key, lock_wait=lock_wait) - logger.warning('Note: --no-cache-sync is an experimental feature.') + logger.warning("Note: --no-cache-sync is an experimental feature.") # Public API @@ -1035,7 +1126,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): pass files = None - cache_mode = 'd' + cache_mode = "d" def file_known_and_unchanged(self, hashed_path, path_hash, st): files_cache_logger.debug("UNKNOWN: files cache not implemented") @@ -1045,7 +1136,7 @@ def memorize_file(self, hashed_path, path_hash, st, ids): pass def add_chunk(self, id, chunk, stats, *, overwrite=False, wait=True, compress=True, size=None): - assert not overwrite, 'AdHocCache does not permit overwrites — trying to use it for recreate?' + assert not overwrite, "AdHocCache does not permit overwrites — trying to use it for recreate?" if not self._txn_active: self.begin_txn() if size is None and compress: @@ -1111,8 +1202,9 @@ def begin_txn(self): # Since we're creating an archive, add 10 % from the start. num_chunks = len(self.repository) self.chunks = ChunkIndex(usable=num_chunks * 1.1) - pi = ProgressIndicatorPercent(total=num_chunks, msg='Downloading chunk list... %3.0f%%', - msgid='cache.download_chunks') + pi = ProgressIndicatorPercent( + total=num_chunks, msg="Downloading chunk list... %3.0f%%", msgid="cache.download_chunks" + ) t0 = perf_counter() num_requests = 0 marker = None @@ -1134,7 +1226,12 @@ def begin_txn(self): del self.chunks[self.manifest.MANIFEST_ID] duration = perf_counter() - t0 or 0.01 pi.finish() - logger.debug('AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s', - num_chunks, duration, num_requests, format_file_size(num_chunks * 34 / duration)) + logger.debug( + "AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s", + num_chunks, + duration, + num_requests, + format_file_size(num_chunks * 34 / duration), + ) # Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes. # Protocol overhead is neglected in this calculation. diff --git a/src/borg/constants.py b/src/borg/constants.py index ff7840299..a7406a63f 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -7,7 +7,7 @@ # fmt: on # this is the set of keys that are always present in items: -REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ]) +REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"]) # this set must be kept complete, otherwise rebuild_manifest might malfunction: # fmt: off @@ -19,7 +19,7 @@ # fmt: on # this is the set of keys that are always present in archives: -REQUIRED_ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'time', ]) +REQUIRED_ARCHIVE_KEYS = frozenset(["version", "name", "items", "cmdline", "time"]) # default umask, overridden by --umask, defaults to read/write only for owner UMASK_DEFAULT = 0o077 @@ -28,8 +28,8 @@ # forcing to 0o100XXX later STDIN_MODE_DEFAULT = 0o660 -CACHE_TAG_NAME = 'CACHEDIR.TAG' -CACHE_TAG_CONTENTS = b'Signature: 8a477f597d28d172789f06886806bc55' +CACHE_TAG_NAME = "CACHEDIR.TAG" +CACHE_TAG_CONTENTS = b"Signature: 8a477f597d28d172789f06886806bc55" # A large, but not unreasonably large segment size. Always less than 2 GiB (for legacy file systems). We choose # 500 MiB which means that no indirection from the inode is needed for typical Linux file systems. @@ -48,7 +48,7 @@ MAX_OBJECT_SIZE = MAX_DATA_SIZE + 41 + 8 # see assertion at end of repository module # repo config max_segment_size value must be below this limit to stay within uint32 offsets: -MAX_SEGMENT_SIZE_LIMIT = 2 ** 32 - MAX_OBJECT_SIZE +MAX_SEGMENT_SIZE_LIMIT = 2**32 - MAX_OBJECT_SIZE # have one all-zero bytes object # we use it at all places where we need to detect or create all-zero buffers @@ -71,12 +71,12 @@ CHUNK_MIN_EXP = 19 # 2**19 == 512kiB CHUNK_MAX_EXP = 23 # 2**23 == 8MiB -HASH_WINDOW_SIZE = 0xfff # 4095B +HASH_WINDOW_SIZE = 0xFFF # 4095B HASH_MASK_BITS = 21 # results in ~2MiB chunks statistically # chunker algorithms -CH_BUZHASH = 'buzhash' -CH_FIXED = 'fixed' +CH_BUZHASH = "buzhash" +CH_FIXED = "fixed" # defaults, use --chunker-params to override CHUNKER_PARAMS = (CH_BUZHASH, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) @@ -88,8 +88,8 @@ CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2 # operating mode of the files cache (for fast skipping of unchanged files) -FILES_CACHE_MODE_UI_DEFAULT = 'ctime,size,inode' # default for "borg create" command (CLI UI) -FILES_CACHE_MODE_DISABLED = 'd' # most borg commands do not use the files cache at all (disable) +FILES_CACHE_MODE_UI_DEFAULT = "ctime,size,inode" # default for "borg create" command (CLI UI) +FILES_CACHE_MODE_DISABLED = "d" # most borg commands do not use the files cache at all (disable) # return codes returned by borg command # when borg is killed by signal N, rc = 128 + N @@ -101,30 +101,30 @@ # never use datetime.isoformat(), it is evil. always use one of these: # datetime.strftime(ISO_FORMAT) # output always includes .microseconds # datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds -ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S' -ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f' +ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S" +ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f" -DASHES = '-' * 78 +DASHES = "-" * 78 PBKDF2_ITERATIONS = 100000 # https://www.rfc-editor.org/rfc/rfc9106.html#section-4-6.2 -ARGON2_ARGS = {'time_cost': 3, 'memory_cost': 2**16, 'parallelism': 4, 'type': 'id'} +ARGON2_ARGS = {"time_cost": 3, "memory_cost": 2**16, "parallelism": 4, "type": "id"} ARGON2_SALT_BYTES = 16 # Maps the CLI argument to our internal identifier for the format KEY_ALGORITHMS = { # encrypt-and-MAC, kdf: PBKDF2(HMAC−SHA256), encryption: AES256-CTR, authentication: HMAC-SHA256 - 'pbkdf2': 'sha256', + "pbkdf2": "sha256", # encrypt-then-MAC, kdf: argon2, encryption: chacha20, authentication: poly1305 - 'argon2': 'argon2 chacha20-poly1305', + "argon2": "argon2 chacha20-poly1305", } class KeyBlobStorage: - NO_STORAGE = 'no_storage' - KEYFILE = 'keyfile' - REPO = 'repository' + NO_STORAGE = "no_storage" + KEYFILE = "keyfile" + REPO = "repository" class KeyType: diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py index aa2d78d10..e497b6af5 100644 --- a/src/borg/crypto/file_integrity.py +++ b/src/borg/crypto/file_integrity.py @@ -102,12 +102,12 @@ def hash_length(self, seek_to_end=False): class SHA512FileHashingWrapper(FileHashingWrapper): - ALGORITHM = 'SHA512' + ALGORITHM = "SHA512" FACTORY = hashlib.sha512 class XXH64FileHashingWrapper(FileHashingWrapper): - ALGORITHM = 'XXH64' + ALGORITHM = "XXH64" FACTORY = StreamingXXH64 @@ -125,7 +125,7 @@ class IntegrityCheckedFile(FileLikeWrapper): def __init__(self, path, write, filename=None, override_fd=None, integrity_data=None): self.path = path self.writing = write - mode = 'wb' if write else 'rb' + mode = "wb" if write else "rb" self.file_fd = override_fd or open(path, mode) self.digests = {} @@ -155,7 +155,7 @@ def hash_filename(self, filename=None): # While Borg does not use anything except ASCII in these file names, it's important to use # the same encoding everywhere for portability. Using os.fsencode() would be wrong. filename = os.path.basename(filename or self.path) - self.hasher.update(('%10d' % len(filename)).encode()) + self.hasher.update(("%10d" % len(filename)).encode()) self.hasher.update(filename.encode()) @classmethod @@ -163,44 +163,41 @@ def parse_integrity_data(cls, path: str, data: str): try: integrity_data = json.loads(data) # Provisions for agility now, implementation later, but make sure the on-disk joint is oiled. - algorithm = integrity_data['algorithm'] + algorithm = integrity_data["algorithm"] if algorithm not in SUPPORTED_ALGORITHMS: - logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm) + logger.warning("Cannot verify integrity of %s: Unknown algorithm %r", path, algorithm) return - digests = integrity_data['digests'] + digests = integrity_data["digests"] # Require at least presence of the final digest - digests['final'] + digests["final"] return algorithm, digests except (ValueError, TypeError, KeyError) as e: - logger.warning('Could not parse integrity data for %s: %s', path, e) + logger.warning("Could not parse integrity data for %s: %s", path, e) raise FileIntegrityError(path) def hash_part(self, partname, is_final=False): if not self.writing and not self.digests: return - self.hasher.update(('%10d' % len(partname)).encode()) + self.hasher.update(("%10d" % len(partname)).encode()) self.hasher.update(partname.encode()) self.hasher.hash_length(seek_to_end=is_final) digest = self.hasher.hexdigest() if self.writing: self.digests[partname] = digest - elif self.digests and not compare_digest(self.digests.get(partname, ''), digest): + elif self.digests and not compare_digest(self.digests.get(partname, ""), digest): raise FileIntegrityError(self.path) def __exit__(self, exc_type, exc_val, exc_tb): exception = exc_type is not None if not exception: - self.hash_part('final', is_final=True) + self.hash_part("final", is_final=True) self.hasher.__exit__(exc_type, exc_val, exc_tb) if exception: return if self.writing: - self.store_integrity_data(json.dumps({ - 'algorithm': self.hasher.ALGORITHM, - 'digests': self.digests, - })) + self.store_integrity_data(json.dumps({"algorithm": self.hasher.ALGORITHM, "digests": self.digests})) elif self.digests: - logger.debug('Verified integrity of %s', self.path) + logger.debug("Verified integrity of %s", self.path) def store_integrity_data(self, data: str): self.integrity_data = data @@ -214,12 +211,12 @@ def __init__(self, path, write, filename=None, override_fd=None): self.output_integrity_file = self.integrity_file_path(os.path.join(output_dir, filename)) def load_integrity_data(self, path, integrity_data): - assert not integrity_data, 'Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile' + assert not integrity_data, "Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile" return self.read_integrity_file(self.path) @staticmethod def integrity_file_path(path): - return path + '.integrity' + return path + ".integrity" @classmethod def read_integrity_file(cls, path): @@ -227,11 +224,11 @@ def read_integrity_file(cls, path): with open(cls.integrity_file_path(path)) as fd: return cls.parse_integrity_data(path, fd.read()) except FileNotFoundError: - logger.info('No integrity file found for %s', path) + logger.info("No integrity file found for %s", path) except OSError as e: - logger.warning('Could not read integrity file for %s: %s', path, e) + logger.warning("Could not read integrity file for %s: %s", path, e) raise FileIntegrityError(path) def store_integrity_data(self, data: str): - with open(self.output_integrity_file, 'w') as fd: + with open(self.output_integrity_file, "w") as fd: fd.write(data) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 94f824942..61a2b5385 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -59,9 +59,11 @@ class UnsupportedKeyFormatError(Error): class TAMRequiredError(IntegrityError): - __doc__ = textwrap.dedent(""" + __doc__ = textwrap.dedent( + """ Manifest is unauthenticated, but it is required for this repository. Is somebody attacking you? - """).strip() + """ + ).strip() traceback = False @@ -71,11 +73,12 @@ class TAMInvalid(IntegrityError): def __init__(self): # Error message becomes: "Data integrity error: Manifest authentication did not verify" - super().__init__('Manifest authentication did not verify') + super().__init__("Manifest authentication did not verify") class TAMUnsupportedSuiteError(IntegrityError): """Could not verify manifest: Unsupported suite {!r}; a newer version is needed.""" + traceback = False @@ -110,7 +113,7 @@ def key_factory(repository, manifest_data): def tam_required_file(repository): security_dir = get_security_dir(bin_to_hex(repository.id)) - return os.path.join(security_dir, 'tam_required') + return os.path.join(security_dir, "tam_required") def tam_required(repository): @@ -126,9 +129,10 @@ def uses_same_id_hash(other_key, key): old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey) new_blake2_ids = (Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey) same_ids = ( - isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids) and isinstance(key, new_hmac_sha256_ids) - or - isinstance(other_key, old_blake2_ids + new_blake2_ids) and isinstance(key, new_blake2_ids) + isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids) + and isinstance(key, new_hmac_sha256_ids) + or isinstance(other_key, old_blake2_ids + new_blake2_ids) + and isinstance(key, new_blake2_ids) ) return same_ids @@ -140,10 +144,10 @@ class KeyBase: TYPES_ACCEPTABLE = None # override in subclasses # Human-readable name - NAME = 'UNDEFINED' + NAME = "UNDEFINED" # Name used in command line / API (e.g. borg init --encryption=...) - ARG_NAME = 'UNDEFINED' + ARG_NAME = "UNDEFINED" # Storage type (no key blob storage / keyfile / repo) STORAGE = KeyBlobStorage.NO_STORAGE @@ -167,13 +171,12 @@ def __init__(self, repository): self.target = None # key location file path / repo obj # Some commands write new chunks (e.g. rename) but don't take a --compression argument. This duplicates # the default used by those commands who do take a --compression argument. - self.compressor = Compressor('lz4') + self.compressor = Compressor("lz4") self.decompress = self.compressor.decompress self.tam_required = True def id_hash(self, data): - """Return HMAC hash using the "id" HMAC key - """ + """Return HMAC hash using the "id" HMAC key""" raise NotImplementedError def encrypt(self, id, data, compress=True): @@ -186,83 +189,79 @@ def assert_id(self, id, data): if id and id != Manifest.MANIFEST_ID: id_computed = self.id_hash(data) if not hmac.compare_digest(id_computed, id): - raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id)) + raise IntegrityError("Chunk %s: id verification failed" % bin_to_hex(id)) def assert_type(self, type_byte, id=None): if type_byte not in self.TYPES_ACCEPTABLE: - id_str = bin_to_hex(id) if id is not None else '(unknown)' - raise IntegrityError(f'Chunk {id_str}: Invalid encryption envelope') + id_str = bin_to_hex(id) if id is not None else "(unknown)" + raise IntegrityError(f"Chunk {id_str}: Invalid encryption envelope") def _tam_key(self, salt, context): return hkdf_hmac_sha512( ikm=self.id_key + self.enc_key + self.enc_hmac_key, salt=salt, - info=b'borg-metadata-authentication-' + context, - output_length=64 + info=b"borg-metadata-authentication-" + context, + output_length=64, ) - def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'): + def pack_and_authenticate_metadata(self, metadata_dict, context=b"manifest"): metadata_dict = StableDict(metadata_dict) - tam = metadata_dict['tam'] = StableDict({ - 'type': 'HKDF_HMAC_SHA512', - 'hmac': bytes(64), - 'salt': os.urandom(64), - }) + tam = metadata_dict["tam"] = StableDict({"type": "HKDF_HMAC_SHA512", "hmac": bytes(64), "salt": os.urandom(64)}) packed = msgpack.packb(metadata_dict) - tam_key = self._tam_key(tam['salt'], context) - tam['hmac'] = hmac.digest(tam_key, packed, 'sha512') + tam_key = self._tam_key(tam["salt"], context) + tam["hmac"] = hmac.digest(tam_key, packed, "sha512") return msgpack.packb(metadata_dict) def unpack_and_verify_manifest(self, data, force_tam_not_required=False): """Unpack msgpacked *data* and return (object, did_verify).""" - if data.startswith(b'\xc1' * 4): + if data.startswith(b"\xc1" * 4): # This is a manifest from the future, we can't read it. raise UnsupportedManifestError() tam_required = self.tam_required if force_tam_not_required and tam_required: - logger.warning('Manifest authentication DISABLED.') + logger.warning("Manifest authentication DISABLED.") tam_required = False data = bytearray(data) - unpacker = get_limited_unpacker('manifest') + unpacker = get_limited_unpacker("manifest") unpacker.feed(data) unpacked = unpacker.unpack() - if 'tam' not in unpacked: + if "tam" not in unpacked: if tam_required: raise TAMRequiredError(self.repository._location.canonical_path()) else: - logger.debug('TAM not found and not required') + logger.debug("TAM not found and not required") return unpacked, False - tam = unpacked.pop('tam', None) + tam = unpacked.pop("tam", None) if not isinstance(tam, dict): raise TAMInvalid() - tam_type = tam.get('type', '') - if tam_type != 'HKDF_HMAC_SHA512': + tam_type = tam.get("type", "") + if tam_type != "HKDF_HMAC_SHA512": if tam_required: raise TAMUnsupportedSuiteError(repr(tam_type)) else: - logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type) + logger.debug("Ignoring TAM made with unsupported suite, since TAM is not required: %r", tam_type) return unpacked, False - tam_hmac = tam.get('hmac') - tam_salt = tam.get('salt') + tam_hmac = tam.get("hmac") + tam_salt = tam.get("salt") if not isinstance(tam_salt, (bytes, str)) or not isinstance(tam_hmac, (bytes, str)): raise TAMInvalid() tam_hmac = want_bytes(tam_hmac) # legacy tam_salt = want_bytes(tam_salt) # legacy offset = data.index(tam_hmac) - data[offset:offset + 64] = bytes(64) - tam_key = self._tam_key(tam_salt, context=b'manifest') - calculated_hmac = hmac.digest(tam_key, data, 'sha512') + data[offset : offset + 64] = bytes(64) + tam_key = self._tam_key(tam_salt, context=b"manifest") + calculated_hmac = hmac.digest(tam_key, data, "sha512") if not hmac.compare_digest(calculated_hmac, tam_hmac): raise TAMInvalid() - logger.debug('TAM-verified manifest') + logger.debug("TAM-verified manifest") return unpacked, True class PlaintextKey(KeyBase): TYPE = KeyType.PLAINTEXT TYPES_ACCEPTABLE = {TYPE} - NAME = 'plaintext' - ARG_NAME = 'none' + NAME = "plaintext" + ARG_NAME = "none" STORAGE = KeyBlobStorage.NO_STORAGE chunk_seed = 0 @@ -287,7 +286,7 @@ def id_hash(self, data): def encrypt(self, id, data, compress=True): if compress: data = self.compressor.compress(data) - return b''.join([self.TYPE_STR, data]) + return b"".join([self.TYPE_STR, data]) def decrypt(self, id, data, decompress=True): self.assert_type(data[0], id) @@ -364,8 +363,7 @@ class AESKeyBase(KeyBase): def encrypt(self, id, data, compress=True): if compress: data = self.compressor.compress(data) - next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), - self.cipher.block_count(len(data))) + next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data))) return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv) def decrypt(self, id, data, decompress=True): @@ -395,12 +393,10 @@ def init_from_random_data(self): chunk_seed = bytes_to_int(data[96:100]) # Convert to signed int32 if chunk_seed & 0x80000000: - chunk_seed = chunk_seed - 0xffffffff - 1 + chunk_seed = chunk_seed - 0xFFFFFFFF - 1 self.init_from_given_data( - enc_key=data[0:32], - enc_hmac_key=data[32:64], - id_key=data[64:96], - chunk_seed=chunk_seed) + enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed + ) def init_ciphers(self, manifest_data=None): self.cipher = self.CIPHERSUITE(mac_key=self.enc_hmac_key, enc_key=self.enc_key, header_len=1, aad_offset=1) @@ -418,13 +414,13 @@ def init_ciphers(self, manifest_data=None): class FlexiKey: - FILE_ID = 'BORG_KEY' + FILE_ID = "BORG_KEY" @classmethod def detect(cls, repository, manifest_data): key = cls(repository) target = key.find_key() - prompt = 'Enter passphrase for key %s: ' % target + prompt = "Enter passphrase for key %s: " % target passphrase = Passphrase.env_passphrase() if passphrase is None: passphrase = Passphrase() @@ -449,18 +445,18 @@ def _load(self, key_data, passphrase): data = msgpack.unpackb(data) key = Key(internal_dict=data) if key.version != 1: - raise IntegrityError('Invalid key file header') + raise IntegrityError("Invalid key file header") self.repository_id = key.repository_id self.enc_key = key.enc_key self.enc_hmac_key = key.enc_hmac_key self.id_key = key.id_key self.chunk_seed = key.chunk_seed - self.tam_required = key.get('tam_required', tam_required(self.repository)) + self.tam_required = key.get("tam_required", tam_required(self.repository)) return True return False def decrypt_key_file(self, data, passphrase): - unpacker = get_limited_unpacker('key') + unpacker = get_limited_unpacker("key") unpacker.feed(data) data = unpacker.unpack() encrypted_key = EncryptedKey(internal_dict=data) @@ -468,9 +464,9 @@ def decrypt_key_file(self, data, passphrase): raise UnsupportedKeyFormatError() else: self._encrypted_key_algorithm = encrypted_key.algorithm - if encrypted_key.algorithm == 'sha256': + if encrypted_key.algorithm == "sha256": return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase) - elif encrypted_key.algorithm == 'argon2 chacha20-poly1305': + elif encrypted_key.algorithm == "argon2 chacha20-poly1305": return self.decrypt_key_file_argon2(encrypted_key, passphrase) else: raise UnsupportedKeyFormatError() @@ -479,7 +475,7 @@ def decrypt_key_file(self, data, passphrase): def pbkdf2(passphrase, salt, iterations, output_len_in_bytes): if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1": iterations = 1 - return pbkdf2_hmac('sha256', passphrase.encode('utf-8'), salt, iterations, output_len_in_bytes) + return pbkdf2_hmac("sha256", passphrase.encode("utf-8"), salt, iterations, output_len_in_bytes) @staticmethod def argon2( @@ -489,18 +485,14 @@ def argon2( time_cost: int, memory_cost: int, parallelism: int, - type: Literal['i', 'd', 'id'] + type: Literal["i", "d", "id"], ) -> bytes: if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1": time_cost = 1 parallelism = 1 # 8 is the smallest value that avoids the "Memory cost is too small" exception memory_cost = 8 - type_map = { - 'i': argon2.low_level.Type.I, - 'd': argon2.low_level.Type.D, - 'id': argon2.low_level.Type.ID, - } + type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID} key = argon2.low_level.hash_secret_raw( secret=passphrase.encode("utf-8"), hash_len=output_len_in_bytes, @@ -514,7 +506,7 @@ def argon2( def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase): key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32) - data = AES(key, b'\0'*16).decrypt(encrypted_key.data) + data = AES(key, b"\0" * 16).decrypt(encrypted_key.data) if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash): return data return None @@ -536,44 +528,32 @@ def decrypt_key_file_argon2(self, encrypted_key, passphrase): return None def encrypt_key_file(self, data, passphrase, algorithm): - if algorithm == 'sha256': + if algorithm == "sha256": return self.encrypt_key_file_pbkdf2(data, passphrase) - elif algorithm == 'argon2 chacha20-poly1305': + elif algorithm == "argon2 chacha20-poly1305": return self.encrypt_key_file_argon2(data, passphrase) else: - raise ValueError(f'Unexpected algorithm: {algorithm}') + raise ValueError(f"Unexpected algorithm: {algorithm}") def encrypt_key_file_pbkdf2(self, data, passphrase): salt = os.urandom(32) iterations = PBKDF2_ITERATIONS key = self.pbkdf2(passphrase, salt, iterations, 32) hash = hmac_sha256(key, data) - cdata = AES(key, b'\0'*16).encrypt(data) - enc_key = EncryptedKey( - version=1, - salt=salt, - iterations=iterations, - algorithm='sha256', - hash=hash, - data=cdata, - ) + cdata = AES(key, b"\0" * 16).encrypt(data) + enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata) return msgpack.packb(enc_key.as_dict()) def encrypt_key_file_argon2(self, data, passphrase): salt = os.urandom(ARGON2_SALT_BYTES) - key = self.argon2( - passphrase, - output_len_in_bytes=32, - salt=salt, - **ARGON2_ARGS, - ) + key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS) ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0) encrypted_key = EncryptedKey( version=1, - algorithm='argon2 chacha20-poly1305', + algorithm="argon2 chacha20-poly1305", salt=salt, data=ae_cipher.encrypt(data), - **{'argon2_' + k: v for k, v in ARGON2_ARGS.items()}, + **{"argon2_" + k: v for k, v in ARGON2_ARGS.items()}, ) return msgpack.packb(encrypted_key.as_dict()) @@ -588,7 +568,7 @@ def _save(self, passphrase, algorithm): tam_required=self.tam_required, ) data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm) - key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii'))) + key_data = "\n".join(textwrap.wrap(b2a_base64(data).decode("ascii"))) return key_data def change_passphrase(self, passphrase=None): @@ -612,22 +592,23 @@ def create(cls, repository, args, *, other_key=None): enc_key=other_key.enc_key, enc_hmac_key=other_key.enc_hmac_key, id_key=other_key.id_key, - chunk_seed=other_key.chunk_seed) + chunk_seed=other_key.chunk_seed, + ) passphrase = other_key._passphrase else: key.init_from_random_data() passphrase = Passphrase.new(allow_empty=True) key.init_ciphers() target = key.get_new_target(args) - key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS['argon2']) + key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS["argon2"]) logger.info('Key in "%s" created.' % target) - logger.info('Keep this key safe. Your data will be inaccessible without it.') + logger.info("Keep this key safe. Your data will be inaccessible without it.") return key def sanity_check(self, filename, id): - file_id = self.FILE_ID.encode() + b' ' + file_id = self.FILE_ID.encode() + b" " repo_id = hexlify(id) - with open(filename, 'rb') as fd: + with open(filename, "rb") as fd: # we do the magic / id check in binary mode to avoid stumbling over # decoding errors if somebody has binary files in the keys dir for some reason. if fd.read(len(file_id)) != file_id: @@ -653,7 +634,7 @@ def find_key(self): raise RepoKeyNotFoundError(loc) from None return loc else: - raise TypeError('Unsupported borg key storage type') + raise TypeError("Unsupported borg key storage type") def get_existing_or_new_target(self, args): keyfile = self._find_key_file_from_environment() @@ -683,10 +664,10 @@ def get_new_target(self, args): elif self.STORAGE == KeyBlobStorage.REPO: return self.repository else: - raise TypeError('Unsupported borg key storage type') + raise TypeError("Unsupported borg key storage type") def _find_key_file_from_environment(self): - keyfile = os.environ.get('BORG_KEY_FILE') + keyfile = os.environ.get("BORG_KEY_FILE") if keyfile: return os.path.abspath(keyfile) @@ -696,17 +677,17 @@ def _get_new_target_in_keys_dir(self, args): i = 1 while os.path.exists(path): i += 1 - path = filename + '.%d' % i + path = filename + ".%d" % i return path def load(self, target, passphrase): if self.STORAGE == KeyBlobStorage.KEYFILE: with open(target) as fd: - key_data = ''.join(fd.readlines()[1:]) + key_data = "".join(fd.readlines()[1:]) elif self.STORAGE == KeyBlobStorage.REPO: # While the repository is encrypted, we consider a repokey repository with a blank # passphrase an unencrypted repository. - self.logically_encrypted = passphrase != '' + self.logically_encrypted = passphrase != "" # what we get in target is just a repo location, but we already have the repo obj: target = self.repository @@ -715,9 +696,9 @@ def load(self, target, passphrase): # if we got an empty key, it means there is no key. loc = target._location.canonical_path() raise RepoKeyNotFoundError(loc) from None - key_data = key_data.decode('utf-8') # remote repo: msgpack issue #99, getting bytes + key_data = key_data.decode("utf-8") # remote repo: msgpack issue #99, getting bytes else: - raise TypeError('Unsupported borg key storage type') + raise TypeError("Unsupported borg key storage type") success = self._load(key_data, passphrase) if success: self.target = target @@ -732,31 +713,31 @@ def save(self, target, passphrase, algorithm, create=False): # see issue #6036 raise Error('Aborting because key in "%s" already exists.' % target) with SaveFile(target) as fd: - fd.write(f'{self.FILE_ID} {bin_to_hex(self.repository_id)}\n') + fd.write(f"{self.FILE_ID} {bin_to_hex(self.repository_id)}\n") fd.write(key_data) - fd.write('\n') + fd.write("\n") elif self.STORAGE == KeyBlobStorage.REPO: - self.logically_encrypted = passphrase != '' - key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes + self.logically_encrypted = passphrase != "" + key_data = key_data.encode("utf-8") # remote repo: msgpack issue #99, giving bytes target.save_key(key_data) else: - raise TypeError('Unsupported borg key storage type') + raise TypeError("Unsupported borg key storage type") self.target = target def remove(self, target): if self.STORAGE == KeyBlobStorage.KEYFILE: os.remove(target) elif self.STORAGE == KeyBlobStorage.REPO: - target.save_key(b'') # save empty key (no new api at remote repo necessary) + target.save_key(b"") # save empty key (no new api at remote repo necessary) else: - raise TypeError('Unsupported borg key storage type') + raise TypeError("Unsupported borg key storage type") class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE} TYPE = KeyType.KEYFILE - NAME = 'key file' - ARG_NAME = 'keyfile' + NAME = "key file" + ARG_NAME = "keyfile" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = AES256_CTR_HMAC_SHA256 @@ -764,8 +745,8 @@ class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE} TYPE = KeyType.REPO - NAME = 'repokey' - ARG_NAME = 'repokey' + NAME = "repokey" + ARG_NAME = "repokey" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = AES256_CTR_HMAC_SHA256 @@ -773,8 +754,8 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} TYPE = KeyType.BLAKE2KEYFILE - NAME = 'key file BLAKE2b' - ARG_NAME = 'keyfile-blake2' + NAME = "key file BLAKE2b" + ARG_NAME = "keyfile-blake2" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = AES256_CTR_BLAKE2b @@ -782,8 +763,8 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO} TYPE = KeyType.BLAKE2REPO - NAME = 'repokey BLAKE2b' - ARG_NAME = 'repokey-blake2' + NAME = "repokey BLAKE2b" + ARG_NAME = "repokey-blake2" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = AES256_CTR_BLAKE2b @@ -810,7 +791,7 @@ def init_ciphers(self, manifest_data=None): def encrypt(self, id, data, compress=True): if compress: data = self.compressor.compress(data) - return b''.join([self.TYPE_STR, data]) + return b"".join([self.TYPE_STR, data]) def decrypt(self, id, data, decompress=True): self.assert_type(data[0], id) @@ -825,15 +806,15 @@ def decrypt(self, id, data, decompress=True): class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase): TYPE = KeyType.AUTHENTICATED TYPES_ACCEPTABLE = {TYPE} - NAME = 'authenticated' - ARG_NAME = 'authenticated' + NAME = "authenticated" + ARG_NAME = "authenticated" class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): TYPE = KeyType.BLAKE2AUTHENTICATED TYPES_ACCEPTABLE = {TYPE} - NAME = 'authenticated BLAKE2b' - ARG_NAME = 'authenticated-blake2' + NAME = "authenticated BLAKE2b" + ARG_NAME = "authenticated-blake2" # ------------ new crypto ------------ @@ -862,17 +843,17 @@ class AEADKeyBase(KeyBase): logically_encrypted = True - MAX_IV = 2 ** 48 - 1 + MAX_IV = 2**48 - 1 def encrypt(self, id, data, compress=True): # to encrypt new data in this session we use always self.cipher and self.sessionid if compress: data = self.compressor.compress(data) - reserved = b'\0' + reserved = b"\0" iv = self.cipher.next_iv() if iv > self.MAX_IV: # see the data-structures docs about why the IV range is enough raise IntegrityError("IV overflow, should never happen.") - iv_48bit = iv.to_bytes(6, 'big') + iv_48bit = iv.to_bytes(6, "big") header = self.TYPE_STR + reserved + iv_48bit + self.sessionid return self.cipher.encrypt(data, header=header, iv=iv, aad=id) @@ -881,7 +862,7 @@ def decrypt(self, id, data, decompress=True): self.assert_type(data[0], id) iv_48bit = data[2:8] sessionid = data[8:32] - iv = int.from_bytes(iv_48bit, 'big') + iv = int.from_bytes(iv_48bit, "big") cipher = self._get_cipher(sessionid, iv) try: payload = cipher.decrypt(data, aad=id) @@ -908,27 +889,25 @@ def init_from_random_data(self): chunk_seed = bytes_to_int(data[96:100]) # Convert to signed int32 if chunk_seed & 0x80000000: - chunk_seed = chunk_seed - 0xffffffff - 1 + chunk_seed = chunk_seed - 0xFFFFFFFF - 1 self.init_from_given_data( - enc_key=data[0:32], - enc_hmac_key=data[32:64], - id_key=data[64:96], - chunk_seed=chunk_seed) + enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed + ) def _get_session_key(self, sessionid): assert len(sessionid) == 24 # 192bit key = hkdf_hmac_sha512( ikm=self.enc_key + self.enc_hmac_key, salt=sessionid, - info=b'borg-session-key-' + self.CIPHERSUITE.__name__.encode(), - output_length=32 + info=b"borg-session-key-" + self.CIPHERSUITE.__name__.encode(), + output_length=32, ) return key def _get_cipher(self, sessionid, iv): assert isinstance(iv, int) key = self._get_session_key(sessionid) - cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1+1+6+24, aad_offset=0) + cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1 + 1 + 6 + 24, aad_offset=0) return cipher def init_ciphers(self, manifest_data=None, iv=0): @@ -940,8 +919,8 @@ def init_ciphers(self, manifest_data=None, iv=0): class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO} TYPE = KeyType.AESOCBKEYFILE - NAME = 'key file AES-OCB' - ARG_NAME = 'keyfile-aes-ocb' + NAME = "key file AES-OCB" + ARG_NAME = "keyfile-aes-ocb" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = AES256_OCB @@ -949,8 +928,8 @@ class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO} TYPE = KeyType.AESOCBREPO - NAME = 'repokey AES-OCB' - ARG_NAME = 'repokey-aes-ocb' + NAME = "repokey AES-OCB" + ARG_NAME = "repokey-aes-ocb" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = AES256_OCB @@ -958,8 +937,8 @@ class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO} TYPE = KeyType.CHPOKEYFILE - NAME = 'key file ChaCha20-Poly1305' - ARG_NAME = 'keyfile-chacha20-poly1305' + NAME = "key file ChaCha20-Poly1305" + ARG_NAME = "keyfile-chacha20-poly1305" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = CHACHA20_POLY1305 @@ -967,8 +946,8 @@ class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO} TYPE = KeyType.CHPOREPO - NAME = 'repokey ChaCha20-Poly1305' - ARG_NAME = 'repokey-chacha20-poly1305' + NAME = "repokey ChaCha20-Poly1305" + ARG_NAME = "repokey-chacha20-poly1305" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = CHACHA20_POLY1305 @@ -976,8 +955,8 @@ class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO} TYPE = KeyType.BLAKE2AESOCBKEYFILE - NAME = 'key file BLAKE2b AES-OCB' - ARG_NAME = 'keyfile-blake2-aes-ocb' + NAME = "key file BLAKE2b AES-OCB" + ARG_NAME = "keyfile-blake2-aes-ocb" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = AES256_OCB @@ -985,8 +964,8 @@ class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO} TYPE = KeyType.BLAKE2AESOCBREPO - NAME = 'repokey BLAKE2b AES-OCB' - ARG_NAME = 'repokey-blake2-aes-ocb' + NAME = "repokey BLAKE2b AES-OCB" + ARG_NAME = "repokey-blake2-aes-ocb" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = AES256_OCB @@ -994,8 +973,8 @@ class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO} TYPE = KeyType.BLAKE2CHPOKEYFILE - NAME = 'key file BLAKE2b ChaCha20-Poly1305' - ARG_NAME = 'keyfile-blake2-chacha20-poly1305' + NAME = "key file BLAKE2b ChaCha20-Poly1305" + ARG_NAME = "keyfile-blake2-chacha20-poly1305" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = CHACHA20_POLY1305 @@ -1003,26 +982,33 @@ class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO} TYPE = KeyType.BLAKE2CHPOREPO - NAME = 'repokey BLAKE2b ChaCha20-Poly1305' - ARG_NAME = 'repokey-blake2-chacha20-poly1305' + NAME = "repokey BLAKE2b ChaCha20-Poly1305" + ARG_NAME = "repokey-blake2-chacha20-poly1305" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = CHACHA20_POLY1305 LEGACY_KEY_TYPES = ( # legacy (AES-CTR based) crypto - KeyfileKey, RepoKey, - Blake2KeyfileKey, Blake2RepoKey, + KeyfileKey, + RepoKey, + Blake2KeyfileKey, + Blake2RepoKey, ) AVAILABLE_KEY_TYPES = ( # these are available encryption modes for new repositories # not encrypted modes PlaintextKey, - AuthenticatedKey, Blake2AuthenticatedKey, + AuthenticatedKey, + Blake2AuthenticatedKey, # new crypto - AESOCBKeyfileKey, AESOCBRepoKey, - CHPOKeyfileKey, CHPORepoKey, - Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey, - Blake2CHPOKeyfileKey, Blake2CHPORepoKey, + AESOCBKeyfileKey, + AESOCBRepoKey, + CHPOKeyfileKey, + CHPORepoKey, + Blake2AESOCBKeyfileKey, + Blake2AESOCBRepoKey, + Blake2CHPOKeyfileKey, + Blake2CHPORepoKey, ) diff --git a/src/borg/crypto/keymanager.py b/src/borg/crypto/keymanager.py index 715d451a1..552134096 100644 --- a/src/borg/crypto/keymanager.py +++ b/src/borg/crypto/keymanager.py @@ -53,7 +53,7 @@ def load_keyblob(self): k = CHPOKeyfileKey(self.repository) target = k.find_key() with open(target) as fd: - self.keyblob = ''.join(fd.readlines()[1:]) + self.keyblob = "".join(fd.readlines()[1:]) elif self.keyblob_storage == KeyBlobStorage.REPO: key_data = self.repository.load_key().decode() @@ -70,75 +70,77 @@ def store_keyblob(self, args): self.store_keyfile(target) elif self.keyblob_storage == KeyBlobStorage.REPO: - self.repository.save_key(self.keyblob.encode('utf-8')) + self.repository.save_key(self.keyblob.encode("utf-8")) def get_keyfile_data(self): - data = f'{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n' + data = f"{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n" data += self.keyblob - if not self.keyblob.endswith('\n'): - data += '\n' + if not self.keyblob.endswith("\n"): + data += "\n" return data def store_keyfile(self, target): - with dash_open(target, 'w') as fd: + with dash_open(target, "w") as fd: fd.write(self.get_keyfile_data()) def export(self, path): if path is None: - path = '-' + path = "-" self.store_keyfile(path) def export_qr(self, path): if path is None: - path = '-' + path = "-" - with dash_open(path, 'wb') as fd: + with dash_open(path, "wb") as fd: key_data = self.get_keyfile_data() - html = pkgutil.get_data('borg', 'paperkey.html') - html = html.replace(b'', key_data.encode() + b'') + html = pkgutil.get_data("borg", "paperkey.html") + html = html.replace(b"", key_data.encode() + b"") fd.write(html) def export_paperkey(self, path): if path is None: - path = '-' + path = "-" def grouped(s): - ret = '' + ret = "" i = 0 for ch in s: if i and i % 6 == 0: - ret += ' ' + ret += " " ret += ch i += 1 return ret - export = 'To restore key use borg key import --paper /path/to/repo\n\n' + export = "To restore key use borg key import --paper /path/to/repo\n\n" binary = a2b_base64(self.keyblob) - export += 'BORG PAPER KEY v1\n' + export += "BORG PAPER KEY v1\n" lines = (len(binary) + 17) // 18 repoid = bin_to_hex(self.repository.id)[:18] complete_checksum = sha256_truncated(binary, 12) - export += 'id: {:d} / {} / {} - {}\n'.format(lines, - grouped(repoid), - grouped(complete_checksum), - sha256_truncated((str(lines) + '/' + repoid + '/' + complete_checksum).encode('ascii'), 2)) + export += "id: {:d} / {} / {} - {}\n".format( + lines, + grouped(repoid), + grouped(complete_checksum), + sha256_truncated((str(lines) + "/" + repoid + "/" + complete_checksum).encode("ascii"), 2), + ) idx = 0 while len(binary): idx += 1 binline = binary[:18] - checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2) - export += f'{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n' + checksum = sha256_truncated(idx.to_bytes(2, byteorder="big") + binline, 2) + export += f"{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n" binary = binary[18:] - with dash_open(path, 'w') as fd: + with dash_open(path, "w") as fd: fd.write(export) def import_keyfile(self, args): file_id = CHPOKeyfileKey.FILE_ID - first_line = file_id + ' ' + bin_to_hex(self.repository.id) + '\n' - with dash_open(args.path, 'r') as fd: + first_line = file_id + " " + bin_to_hex(self.repository.id) + "\n" + with dash_open(args.path, "r") as fd: file_first_line = fd.read(len(first_line)) if file_first_line != first_line: if not file_first_line.startswith(file_id): @@ -154,52 +156,52 @@ def import_paperkey(self, args): # imported here because it has global side effects import readline except ImportError: - print('Note: No line editing available due to missing readline support') + print("Note: No line editing available due to missing readline support") repoid = bin_to_hex(self.repository.id)[:18] try: while True: # used for repeating on overall checksum mismatch # id line input while True: - idline = input('id: ').replace(' ', '') - if idline == '': - if yes('Abort import? [yN]:'): + idline = input("id: ").replace(" ", "") + if idline == "": + if yes("Abort import? [yN]:"): raise EOFError() try: - (data, checksum) = idline.split('-') + (data, checksum) = idline.split("-") except ValueError: print("each line must contain exactly one '-', try again") continue try: - (id_lines, id_repoid, id_complete_checksum) = data.split('/') + (id_lines, id_repoid, id_complete_checksum) = data.split("/") except ValueError: print("the id line must contain exactly three '/', try again") continue - if sha256_truncated(data.lower().encode('ascii'), 2) != checksum: - print('line checksum did not match, try same line again') + if sha256_truncated(data.lower().encode("ascii"), 2) != checksum: + print("line checksum did not match, try same line again") continue try: lines = int(id_lines) except ValueError: - print('internal error while parsing length') + print("internal error while parsing length") break if repoid != id_repoid: raise RepoIdMismatch() - result = b'' + result = b"" idx = 1 # body line input while True: - inline = input(f'{idx:2d}: ') - inline = inline.replace(' ', '') - if inline == '': - if yes('Abort import? [yN]:'): + inline = input(f"{idx:2d}: ") + inline = inline.replace(" ", "") + if inline == "": + if yes("Abort import? [yN]:"): raise EOFError() try: - (data, checksum) = inline.split('-') + (data, checksum) = inline.split("-") except ValueError: print("each line must contain exactly one '-', try again") continue @@ -208,8 +210,8 @@ def import_paperkey(self, args): except binascii.Error: print("only characters 0-9 and a-f and '-' are valid, try again") continue - if sha256_truncated(idx.to_bytes(2, byteorder='big') + part, 2) != checksum: - print(f'line checksum did not match, try line {idx} again') + if sha256_truncated(idx.to_bytes(2, byteorder="big") + part, 2) != checksum: + print(f"line checksum did not match, try line {idx} again") continue result += part if idx == lines: @@ -217,13 +219,13 @@ def import_paperkey(self, args): idx += 1 if sha256_truncated(result, 12) != id_complete_checksum: - print('The overall checksum did not match, retry or enter a blank line to abort.') + print("The overall checksum did not match, retry or enter a blank line to abort.") continue - self.keyblob = '\n'.join(textwrap.wrap(b2a_base64(result).decode('ascii'))) + '\n' + self.keyblob = "\n".join(textwrap.wrap(b2a_base64(result).decode("ascii"))) + "\n" self.store_keyblob(args) break except EOFError: - print('\n - aborted') + print("\n - aborted") return diff --git a/src/borg/crypto/nonces.py b/src/borg/crypto/nonces.py index 12a597484..d1a9d9fbd 100644 --- a/src/borg/crypto/nonces.py +++ b/src/borg/crypto/nonces.py @@ -18,7 +18,7 @@ def __init__(self, repository, manifest_nonce): self.repository = repository self.end_of_nonce_reservation = None self.manifest_nonce = manifest_nonce - self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), 'nonce') + self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce") def get_local_free_nonce(self): try: @@ -78,7 +78,11 @@ def ensure_reservation(self, nonce, nonce_space_needed): repo_free_nonce = self.get_repo_free_nonce() local_free_nonce = self.get_local_free_nonce() - free_nonce_space = max(x for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) if x is not None) + free_nonce_space = max( + x + for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) + if x is not None + ) reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION assert reservation_end < MAX_REPRESENTABLE_NONCE self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index b6349da7d..9caafd73b 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -20,7 +20,9 @@ def async_wrapper(fn): @functools.wraps(fn) async def wrapper(*args, **kwargs): return fn(*args, **kwargs) + return wrapper + else: trio = None @@ -29,6 +31,7 @@ def async_wrapper(fn): from .logger import create_logger + logger = create_logger() from .crypto.low_level import blake2b_128 @@ -79,7 +82,7 @@ class ItemCache: # to resize it in the first place; that's free). GROW_META_BY = 2 * 1024 * 1024 - indirect_entry_struct = struct.Struct('=cII') + indirect_entry_struct = struct.Struct("=cII") assert indirect_entry_struct.size == 9 def __init__(self, decrypted_repository): @@ -105,7 +108,7 @@ def __init__(self, decrypted_repository): # These are items that span more than one chunk and thus cannot be efficiently cached # by the object cache (self.decrypted_repository), which would require variable-length structures; # possible but not worth the effort, see iter_archive_items. - self.fd = tempfile.TemporaryFile(prefix='borg-tmp') + self.fd = tempfile.TemporaryFile(prefix="borg-tmp") # A small LRU cache for chunks requested by ItemCache.get() from the object cache, # this significantly speeds up directory traversal and similar operations which @@ -123,12 +126,12 @@ def __init__(self, decrypted_repository): def get(self, inode): offset = inode - self.offset if offset < 0: - raise ValueError('ItemCache.get() called with an invalid inode number') - if self.meta[offset] == ord(b'I'): + raise ValueError("ItemCache.get() called with an invalid inode number") + if self.meta[offset] == ord(b"I"): _, chunk_id_relative_offset, chunk_offset = self.indirect_entry_struct.unpack_from(self.meta, offset) chunk_id_offset = offset - chunk_id_relative_offset # bytearray slices are bytearrays as well, explicitly convert to bytes() - chunk_id = bytes(self.meta[chunk_id_offset:chunk_id_offset + 32]) + chunk_id = bytes(self.meta[chunk_id_offset : chunk_id_offset + 32]) chunk = self.chunks.get(chunk_id) if not chunk: csize, chunk = next(self.decrypted_repository.get_many([chunk_id])) @@ -137,12 +140,12 @@ def get(self, inode): unpacker = msgpack.Unpacker() unpacker.feed(data) return Item(internal_dict=next(unpacker)) - elif self.meta[offset] == ord(b'S'): - fd_offset = int.from_bytes(self.meta[offset + 1:offset + 9], 'little') + elif self.meta[offset] == ord(b"S"): + fd_offset = int.from_bytes(self.meta[offset + 1 : offset + 9], "little") self.fd.seek(fd_offset, io.SEEK_SET) return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024))) else: - raise ValueError('Invalid entry type in self.meta') + raise ValueError("Invalid entry type in self.meta") def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=False): unpacker = msgpack.Unpacker() @@ -153,7 +156,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= chunk_begin = 0 # Length of the chunk preceding the current chunk last_chunk_length = 0 - msgpacked_bytes = b'' + msgpacked_bytes = b"" write_offset = self.write_offset meta = self.meta @@ -163,7 +166,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= # Store the chunk ID in the meta-array if write_offset + 32 >= len(meta): self.meta = meta = meta + bytes(self.GROW_META_BY) - meta[write_offset:write_offset + 32] = key + meta[write_offset : write_offset + 32] = key current_id_offset = write_offset write_offset += 32 @@ -182,7 +185,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= # tell() is not helpful for the need_more_data case, but we know it is the remainder # of the data in that case. in the other case, tell() works as expected. length = (len(data) - start) if need_more_data else (unpacker.tell() - stream_offset) - msgpacked_bytes += data[start:start+length] + msgpacked_bytes += data[start : start + length] stream_offset += length if need_more_data: @@ -190,14 +193,14 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= break item = Item(internal_dict=item) - if filter and not filter(item) or not consider_part_files and 'part' in item: - msgpacked_bytes = b'' + if filter and not filter(item) or not consider_part_files and "part" in item: + msgpacked_bytes = b"" continue current_item = msgpacked_bytes current_item_length = len(current_item) current_spans_chunks = stream_offset - current_item_length < chunk_begin - msgpacked_bytes = b'' + msgpacked_bytes = b"" if write_offset + 9 >= len(meta): self.meta = meta = meta + bytes(self.GROW_META_BY) @@ -221,11 +224,11 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= if current_spans_chunks: pos = self.fd.seek(0, io.SEEK_END) self.fd.write(current_item) - meta[write_offset:write_offset + 9] = b'S' + pos.to_bytes(8, 'little') + meta[write_offset : write_offset + 9] = b"S" + pos.to_bytes(8, "little") self.direct_items += 1 else: item_offset = stream_offset - current_item_length - chunk_begin - pack_indirect_into(meta, write_offset, b'I', write_offset - current_id_offset, item_offset) + pack_indirect_into(meta, write_offset, b"I", write_offset - current_id_offset, item_offset) self.indirect_items += 1 inode = write_offset + self.offset write_offset += 9 @@ -236,8 +239,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files= class FuseBackend: - """Virtual filesystem based on archive(s) to provide information to fuse - """ + """Virtual filesystem based on archive(s) to provide information to fuse""" def __init__(self, key, manifest, repository, args, decrypted_repository): self.repository_uncached = repository @@ -307,8 +309,7 @@ def _allocate_inode(self): return self.inode_count def _create_dir(self, parent, mtime=None): - """Create directory - """ + """Create directory""" ino = self._allocate_inode() if mtime is not None: self._items[ino] = Item(internal_dict=self.default_dir.as_dict()) @@ -319,26 +320,31 @@ def _create_dir(self, parent, mtime=None): return ino def find_inode(self, path, prefix=[]): - segments = prefix + path.split(b'/') + segments = prefix + path.split(b"/") inode = 1 for segment in segments: inode = self.contents[inode][segment] return inode def _process_archive(self, archive_name, prefix=[]): - """Build FUSE inode hierarchy from archive metadata - """ + """Build FUSE inode hierarchy from archive metadata""" self.file_versions = {} # for versions mode: original path -> version t0 = time.perf_counter() - archive = Archive(self.repository_uncached, self.key, self._manifest, archive_name, - consider_part_files=self._args.consider_part_files) + archive = Archive( + self.repository_uncached, + self.key, + self._manifest, + archive_name, + consider_part_files=self._args.consider_part_files, + ) strip_components = self._args.strip_components matcher = Archiver.build_matcher(self._args.patterns, self._args.paths) hlm = HardLinkManager(id_type=bytes, info_type=str) # hlid -> path filter = Archiver.build_filter(matcher, strip_components) - for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter, - consider_part_files=self._args.consider_part_files): + for item_inode, item in self.cache.iter_archive_items( + archive.metadata.items, filter=filter, consider_part_files=self._args.consider_part_files + ): if strip_components: item.path = os.sep.join(item.path.split(os.sep)[strip_components:]) path = os.fsencode(item.path) @@ -354,24 +360,24 @@ def _process_archive(self, archive_name, prefix=[]): else: self._items[inode] = item continue - segments = prefix + path.split(b'/') + segments = prefix + path.split(b"/") parent = 1 for segment in segments[:-1]: parent = self._process_inner(segment, parent) self._process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode, hlm) duration = time.perf_counter() - t0 - logger.debug('fuse: _process_archive completed in %.1f s for archive %s', duration, archive.name) + logger.debug("fuse: _process_archive completed in %.1f s for archive %s", duration, archive.name) def _process_leaf(self, name, item, parent, prefix, is_dir, item_inode, hlm): path = item.path del item.path # save some space def file_version(item, path): - if 'chunks' in item: + if "chunks" in item: file_id = blake2b_128(path) current_version, previous_id = self.versions_index.get(file_id, (0, None)) - contents_id = blake2b_128(b''.join(chunk_id for chunk_id, _ in item.chunks)) + contents_id = blake2b_128(b"".join(chunk_id for chunk_id, _ in item.chunks)) if contents_id != previous_id: current_version += 1 @@ -382,14 +388,14 @@ def file_version(item, path): def make_versioned_name(name, version, add_dir=False): if add_dir: # add intermediate directory with same name as filename - path_fname = name.rsplit(b'/', 1) - name += b'/' + path_fname[-1] + path_fname = name.rsplit(b"/", 1) + name += b"/" + path_fname[-1] # keep original extension at end to avoid confusing tools name, ext = os.path.splitext(name) - version_enc = os.fsencode('.%05d' % version) + version_enc = os.fsencode(".%05d" % version) return name + version_enc + ext - if 'hlid' in item: + if "hlid" in item: link_target = hlm.retrieve(id=item.hlid, default=None) if link_target is not None: # Hard link was extracted previously, just link @@ -401,10 +407,10 @@ def make_versioned_name(name, version, add_dir=False): try: inode = self.find_inode(link_target, prefix) except KeyError: - logger.warning('Skipping broken hard link: %s -> %s', path, link_target) + logger.warning("Skipping broken hard link: %s -> %s", path, link_target) return item = self.get_item(inode) - item.nlink = item.get('nlink', 1) + 1 + item.nlink = item.get("nlink", 1) + 1 self._items[inode] = item else: inode = item_inode @@ -439,31 +445,41 @@ def _process_inner(self, name, parent_inode): class FuseOperations(llfuse.Operations, FuseBackend): - """Export archive as a FUSE filesystem - """ + """Export archive as a FUSE filesystem""" def __init__(self, key, repository, manifest, args, decrypted_repository): llfuse.Operations.__init__(self) FuseBackend.__init__(self, key, manifest, repository, args, decrypted_repository) self.decrypted_repository = decrypted_repository - data_cache_capacity = int(os.environ.get('BORG_MOUNT_DATA_CACHE_ENTRIES', os.cpu_count() or 1)) - logger.debug('mount data cache capacity: %d chunks', data_cache_capacity) + data_cache_capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1)) + logger.debug("mount data cache capacity: %d chunks", data_cache_capacity) self.data_cache = LRUCache(capacity=data_cache_capacity, dispose=lambda _: None) self._last_pos = LRUCache(capacity=FILES, dispose=lambda _: None) def sig_info_handler(self, sig_no, stack): - logger.debug('fuse: %d synth inodes, %d edges (%s)', - self.inode_count, len(self.parent), - # getsizeof is the size of the dict itself; key and value are two small-ish integers, - # which are shared due to code structure (this has been verified). - format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count))) - logger.debug('fuse: %d pending archives', len(self.pending_archives)) - logger.debug('fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s', - self.cache.direct_items + self.cache.indirect_items, self.cache.direct_items, self.cache.indirect_items, - format_file_size(sys.getsizeof(self.cache.meta)), - format_file_size(os.stat(self.cache.fd.fileno()).st_size)) - logger.debug('fuse: data cache: %d/%d entries, %s', len(self.data_cache.items()), self.data_cache._capacity, - format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items()))) + logger.debug( + "fuse: %d synth inodes, %d edges (%s)", + self.inode_count, + len(self.parent), + # getsizeof is the size of the dict itself; key and value are two small-ish integers, + # which are shared due to code structure (this has been verified). + format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)), + ) + logger.debug("fuse: %d pending archives", len(self.pending_archives)) + logger.debug( + "fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s", + self.cache.direct_items + self.cache.indirect_items, + self.cache.direct_items, + self.cache.indirect_items, + format_file_size(sys.getsizeof(self.cache.meta)), + format_file_size(os.stat(self.cache.fd.fileno()).st_size), + ) + logger.debug( + "fuse: data cache: %d/%d entries, %s", + len(self.data_cache.items()), + self.data_cache._capacity, + format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())), + ) self.decrypted_repository.log_instrumentation() def mount(self, mountpoint, mount_options, foreground=False): @@ -475,25 +491,25 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0): if option == key: options.pop(idx) return present - if option.startswith(key + '='): + if option.startswith(key + "="): options.pop(idx) - value = option.split('=', 1)[1] + value = option.split("=", 1)[1] if wanted_type is bool: v = value.lower() - if v in ('y', 'yes', 'true', '1'): + if v in ("y", "yes", "true", "1"): return True - if v in ('n', 'no', 'false', '0'): + if v in ("n", "no", "false", "0"): return False - raise ValueError('unsupported value in option: %s' % option) + raise ValueError("unsupported value in option: %s" % option) if wanted_type is int: try: return int(value, base=int_base) except ValueError: - raise ValueError('unsupported value in option: %s' % option) from None + raise ValueError("unsupported value in option: %s" % option) from None try: return wanted_type(value) except ValueError: - raise ValueError('unsupported value in option: %s' % option) from None + raise ValueError("unsupported value in option: %s" % option) from None else: return not_present @@ -502,20 +518,20 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0): # cause security issues if used with allow_other mount option. # When not using allow_other or allow_root, access is limited to the # mounting user anyway. - options = ['fsname=borgfs', 'ro', 'default_permissions'] + options = ["fsname=borgfs", "ro", "default_permissions"] if mount_options: - options.extend(mount_options.split(',')) - ignore_permissions = pop_option(options, 'ignore_permissions', True, False, bool) + options.extend(mount_options.split(",")) + ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool) if ignore_permissions: # in case users have a use-case that requires NOT giving "default_permissions", # this is enabled by the custom "ignore_permissions" mount option which just # removes "default_permissions" again: - pop_option(options, 'default_permissions', True, False, bool) - self.allow_damaged_files = pop_option(options, 'allow_damaged_files', True, False, bool) - self.versions = pop_option(options, 'versions', True, False, bool) - self.uid_forced = pop_option(options, 'uid', None, None, int) - self.gid_forced = pop_option(options, 'gid', None, None, int) - self.umask = pop_option(options, 'umask', 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222 + pop_option(options, "default_permissions", True, False, bool) + self.allow_damaged_files = pop_option(options, "allow_damaged_files", True, False, bool) + self.versions = pop_option(options, "versions", True, False, bool) + self.uid_forced = pop_option(options, "uid", None, None, int) + self.gid_forced = pop_option(options, "gid", None, None, int) + self.umask = pop_option(options, "umask", 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222 dir_uid = self.uid_forced if self.uid_forced is not None else self.default_uid dir_gid = self.gid_forced if self.gid_forced is not None else self.default_gid dir_user = uid2user(dir_uid) @@ -523,8 +539,9 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0): assert isinstance(dir_user, str) assert isinstance(dir_group, str) dir_mode = 0o40755 & ~self.umask - self.default_dir = Item(mode=dir_mode, mtime=int(time.time() * 1e9), - user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid) + self.default_dir = Item( + mode=dir_mode, mtime=int(time.time() * 1e9), user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid + ) self._create_filesystem() llfuse.init(self, mountpoint, options) if not foreground: @@ -533,7 +550,7 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0): else: with daemonizing() as (old_id, new_id): # local repo: the locking process' PID is changing, migrate it: - logger.debug('fuse: mount local repo, going to background: migrating lock.') + logger.debug("fuse: mount local repo, going to background: migrating lock.") self.repository_uncached.migrate_lock(old_id, new_id) # If the file system crashes, we do not want to umount because in that @@ -543,11 +560,10 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0): # mirror. umount = False try: - with signal_handler('SIGUSR1', self.sig_info_handler), \ - signal_handler('SIGINFO', self.sig_info_handler): + with signal_handler("SIGUSR1", self.sig_info_handler), signal_handler("SIGINFO", self.sig_info_handler): signal = fuse_main() # no crash and no signal (or it's ^C and we're in the foreground) -> umount request - umount = (signal is None or (signal == SIGINT and foreground)) + umount = signal is None or (signal == SIGINT and foreground) finally: llfuse.close(umount) @@ -573,19 +589,24 @@ def _getattr(self, inode, ctx=None): entry.entry_timeout = 300 entry.attr_timeout = 300 entry.st_mode = item.mode & ~self.umask - entry.st_nlink = item.get('nlink', 1) - entry.st_uid, entry.st_gid = get_item_uid_gid(item, numeric=self.numeric_ids, - uid_default=self.default_uid, gid_default=self.default_gid, - uid_forced=self.uid_forced, gid_forced=self.gid_forced) - entry.st_rdev = item.get('rdev', 0) + entry.st_nlink = item.get("nlink", 1) + entry.st_uid, entry.st_gid = get_item_uid_gid( + item, + numeric=self.numeric_ids, + uid_default=self.default_uid, + gid_default=self.default_gid, + uid_forced=self.uid_forced, + gid_forced=self.gid_forced, + ) + entry.st_rdev = item.get("rdev", 0) entry.st_size = item.get_size() entry.st_blksize = 512 entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize # note: older archives only have mtime (not atime nor ctime) entry.st_mtime_ns = mtime_ns = item.mtime - entry.st_atime_ns = item.get('atime', mtime_ns) - entry.st_ctime_ns = item.get('ctime', mtime_ns) - entry.st_birthtime_ns = item.get('birthtime', mtime_ns) + entry.st_atime_ns = item.get("atime", mtime_ns) + entry.st_ctime_ns = item.get("ctime", mtime_ns) + entry.st_birthtime_ns = item.get("birthtime", mtime_ns) return entry @async_wrapper @@ -595,22 +616,22 @@ def getattr(self, inode, ctx=None): @async_wrapper def listxattr(self, inode, ctx=None): item = self.get_item(inode) - return item.get('xattrs', {}).keys() + return item.get("xattrs", {}).keys() @async_wrapper def getxattr(self, inode, name, ctx=None): item = self.get_item(inode) try: - return item.get('xattrs', {})[name] or b'' + return item.get("xattrs", {})[name] or b"" except KeyError: raise llfuse.FUSEError(llfuse.ENOATTR) from None @async_wrapper def lookup(self, parent_inode, name, ctx=None): self.check_pending_archive(parent_inode) - if name == b'.': + if name == b".": inode = parent_inode - elif name == b'..': + elif name == b"..": inode = self.parent[parent_inode] else: inode = self.contents[parent_inode].get(name) @@ -622,12 +643,14 @@ def lookup(self, parent_inode, name, ctx=None): def open(self, inode, flags, ctx=None): if not self.allow_damaged_files: item = self.get_item(inode) - if 'chunks_healthy' in item: + if "chunks_healthy" in item: # Processed archive items don't carry the path anymore; for converting the inode # to the path we'd either have to store the inverse of the current structure, # or search the entire archive. So we just don't print it. It's easy to correlate anyway. - logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. ' - 'Mount with allow_damaged_files to read damaged files.') + logger.warning( + "File has damaged (all-zero) chunks. Try running borg check --repair. " + "Mount with allow_damaged_files to read damaged files." + ) raise llfuse.FUSEError(errno.EIO) return llfuse.FileInfo(fh=inode) if has_pyfuse3 else inode @@ -669,7 +692,7 @@ def read(self, fh, offset, size): if offset + n < len(data): # chunk was only partially read, cache it self.data_cache[id] = data - parts.append(data[offset:offset + n]) + parts.append(data[offset : offset + n]) offset = 0 size -= n if not size: @@ -678,12 +701,13 @@ def read(self, fh, offset, size): else: self._last_pos[fh] = (chunk_no, chunk_offset) break - return b''.join(parts) + return b"".join(parts) # note: we can't have a generator (with yield) and not a generator (async) in the same method if has_pyfuse3: + async def readdir(self, fh, off, token): - entries = [(b'.', fh), (b'..', self.parent[fh])] + entries = [(b".", fh), (b"..", self.parent[fh])] entries.extend(self.contents[fh].items()) for i, (name, inode) in enumerate(entries[off:], off): attrs = self._getattr(inode) @@ -691,8 +715,9 @@ async def readdir(self, fh, off, token): break else: + def readdir(self, fh, off): - entries = [(b'.', fh), (b'..', self.parent[fh])] + entries = [(b".", fh), (b"..", self.parent[fh])] entries.extend(self.contents[fh].items()) for i, (name, inode) in enumerate(entries[off:], off): attrs = self._getattr(inode) diff --git a/src/borg/fuse_impl.py b/src/borg/fuse_impl.py index 390ac576f..b63d95794 100644 --- a/src/borg/fuse_impl.py +++ b/src/borg/fuse_impl.py @@ -4,11 +4,11 @@ import os -BORG_FUSE_IMPL = os.environ.get('BORG_FUSE_IMPL', 'pyfuse3,llfuse') +BORG_FUSE_IMPL = os.environ.get("BORG_FUSE_IMPL", "pyfuse3,llfuse") -for FUSE_IMPL in BORG_FUSE_IMPL.split(','): +for FUSE_IMPL in BORG_FUSE_IMPL.split(","): FUSE_IMPL = FUSE_IMPL.strip() - if FUSE_IMPL == 'pyfuse3': + if FUSE_IMPL == "pyfuse3": try: import pyfuse3 as llfuse except ImportError: @@ -17,7 +17,7 @@ has_llfuse = False has_pyfuse3 = True break - elif FUSE_IMPL == 'llfuse': + elif FUSE_IMPL == "llfuse": try: import llfuse except ImportError: @@ -26,7 +26,7 @@ has_llfuse = True has_pyfuse3 = False break - elif FUSE_IMPL == 'none': + elif FUSE_IMPL == "none": pass else: raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index d8a4cda62..67ee948e0 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -24,7 +24,7 @@ # generic mechanism to enable users to invoke workarounds by setting the # BORG_WORKAROUNDS environment variable to a list of comma-separated strings. # see the docs for a list of known workaround strings. -workarounds = tuple(os.environ.get('BORG_WORKAROUNDS', '').split(',')) +workarounds = tuple(os.environ.get("BORG_WORKAROUNDS", "").split(",")) """ The global exit_code variable is used so that modules other than archiver can increase the program exit code if a diff --git a/src/borg/helpers/checks.py b/src/borg/helpers/checks.py index 26a6c0129..74b50a03f 100644 --- a/src/borg/helpers/checks.py +++ b/src/borg/helpers/checks.py @@ -23,15 +23,16 @@ class ExtensionModuleError(Error): def check_extension_modules(): from .. import platform, compress, crypto, item, chunker, hashindex - if hashindex.API_VERSION != '1.2_01': + + if hashindex.API_VERSION != "1.2_01": raise ExtensionModuleError - if chunker.API_VERSION != '1.2_01': + if chunker.API_VERSION != "1.2_01": raise ExtensionModuleError - if compress.API_VERSION != '1.2_02': + if compress.API_VERSION != "1.2_02": raise ExtensionModuleError - if crypto.low_level.API_VERSION != '1.3_01': + if crypto.low_level.API_VERSION != "1.3_01": raise ExtensionModuleError - if item.API_VERSION != '1.2_01': + if item.API_VERSION != "1.2_01": raise ExtensionModuleError - if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.2_05': + if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != "1.2_05": raise ExtensionModuleError diff --git a/src/borg/helpers/datastruct.py b/src/borg/helpers/datastruct.py index 31192a8c7..f647beb14 100644 --- a/src/borg/helpers/datastruct.py +++ b/src/borg/helpers/datastruct.py @@ -3,6 +3,7 @@ class StableDict(dict): """A dict subclass with stable items() ordering""" + def items(self): return sorted(super().items()) @@ -20,8 +21,8 @@ def __init__(self, allocator, size=4096, limit=None): Initialize the buffer: use allocator(size) call to allocate a buffer. Optionally, set the upper for the buffer size. """ - assert callable(allocator), 'must give alloc(size) function as first param' - assert limit is None or size <= limit, 'initial size must be <= limit' + assert callable(allocator), "must give alloc(size) function as first param" + assert limit is None or size <= limit, "initial size must be <= limit" self.allocator = allocator self.limit = limit self.resize(size, init=True) diff --git a/src/borg/helpers/errors.py b/src/borg/helpers/errors.py index f07f7bb98..ae71cc565 100644 --- a/src/borg/helpers/errors.py +++ b/src/borg/helpers/errors.py @@ -5,6 +5,7 @@ class Error(Exception): """Error: {}""" + # Error base class # if we raise such an Error and it is only caught by the uppermost @@ -26,6 +27,7 @@ def get_message(self): class ErrorWithTraceback(Error): """Error: {}""" + # like Error, but show a traceback also traceback = True diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 9d3c55c5e..c12e7a7d3 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -16,6 +16,7 @@ from ..constants import * # NOQA from ..logger import create_logger + logger = create_logger() @@ -50,32 +51,32 @@ def get_base_dir(): - ~$USER, if USER is set - ~ """ - base_dir = os.environ.get('BORG_BASE_DIR') or os.environ.get('HOME') + base_dir = os.environ.get("BORG_BASE_DIR") or os.environ.get("HOME") # os.path.expanduser() behaves differently for '~' and '~someuser' as # parameters: when called with an explicit username, the possibly set # environment variable HOME is no longer respected. So we have to check if # it is set and only expand the user's home directory if HOME is unset. if not base_dir: - base_dir = os.path.expanduser('~%s' % os.environ.get('USER', '')) + base_dir = os.path.expanduser("~%s" % os.environ.get("USER", "")) return base_dir def get_keys_dir(): """Determine where to repository keys and cache""" - keys_dir = os.environ.get('BORG_KEYS_DIR') + keys_dir = os.environ.get("BORG_KEYS_DIR") if keys_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - keys_dir = os.path.join(get_config_dir(), 'keys') + keys_dir = os.path.join(get_config_dir(), "keys") ensure_dir(keys_dir) return keys_dir def get_security_dir(repository_id=None): """Determine where to store local security information.""" - security_dir = os.environ.get('BORG_SECURITY_DIR') + security_dir = os.environ.get("BORG_SECURITY_DIR") if security_dir is None: # note: do not just give this as default to the environment.get(), see issue #5979. - security_dir = os.path.join(get_config_dir(), 'security') + security_dir = os.path.join(get_config_dir(), "security") if repository_id: security_dir = os.path.join(security_dir, repository_id) ensure_dir(security_dir) @@ -85,22 +86,28 @@ def get_security_dir(repository_id=None): def get_cache_dir(): """Determine where to repository keys and cache""" # Get cache home path - cache_home = os.path.join(get_base_dir(), '.cache') + cache_home = os.path.join(get_base_dir(), ".cache") # Try to use XDG_CACHE_HOME instead if BORG_BASE_DIR isn't explicitly set - if not os.environ.get('BORG_BASE_DIR'): - cache_home = os.environ.get('XDG_CACHE_HOME', cache_home) + if not os.environ.get("BORG_BASE_DIR"): + cache_home = os.environ.get("XDG_CACHE_HOME", cache_home) # Use BORG_CACHE_DIR if set, otherwise assemble final path from cache home path - cache_dir = os.environ.get('BORG_CACHE_DIR', os.path.join(cache_home, 'borg')) + cache_dir = os.environ.get("BORG_CACHE_DIR", os.path.join(cache_home, "borg")) # Create path if it doesn't exist yet ensure_dir(cache_dir) cache_tag_fn = os.path.join(cache_dir, CACHE_TAG_NAME) if not os.path.exists(cache_tag_fn): - cache_tag_contents = CACHE_TAG_CONTENTS + textwrap.dedent(""" + cache_tag_contents = ( + CACHE_TAG_CONTENTS + + textwrap.dedent( + """ # This file is a cache directory tag created by Borg. # For information about cache directory tags, see: # http://www.bford.info/cachedir/spec.html - """).encode('ascii') + """ + ).encode("ascii") + ) from ..platform import SaveFile + with SaveFile(cache_tag_fn, binary=True) as fd: fd.write(cache_tag_contents) return cache_dir @@ -109,12 +116,12 @@ def get_cache_dir(): def get_config_dir(): """Determine where to store whole config""" # Get config home path - config_home = os.path.join(get_base_dir(), '.config') + config_home = os.path.join(get_base_dir(), ".config") # Try to use XDG_CONFIG_HOME instead if BORG_BASE_DIR isn't explicitly set - if not os.environ.get('BORG_BASE_DIR'): - config_home = os.environ.get('XDG_CONFIG_HOME', config_home) + if not os.environ.get("BORG_BASE_DIR"): + config_home = os.environ.get("XDG_CONFIG_HOME", config_home) # Use BORG_CONFIG_DIR if set, otherwise assemble final path from config home path - config_dir = os.environ.get('BORG_CONFIG_DIR', os.path.join(config_home, 'borg')) + config_dir = os.environ.get("BORG_CONFIG_DIR", os.path.join(config_home, "borg")) # Create path if it doesn't exist yet ensure_dir(config_dir) return config_dir @@ -130,7 +137,7 @@ def dir_is_cachedir(path): tag_path = os.path.join(path, CACHE_TAG_NAME) try: if os.path.exists(tag_path): - with open(tag_path, 'rb') as tag_file: + with open(tag_path, "rb") as tag_file: tag_data = tag_file.read(len(CACHE_TAG_CONTENTS)) if tag_data == CACHE_TAG_CONTENTS: return True @@ -157,13 +164,12 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): return tag_names -_safe_re = re.compile(r'^((\.\.)?/+)+') +_safe_re = re.compile(r"^((\.\.)?/+)+") def make_path_safe(path): - """Make path safe by making it relative and local - """ - return _safe_re.sub('', path) or '.' + """Make path safe by making it relative and local""" + return _safe_re.sub("", path) or "." class HardLinkManager: @@ -189,6 +195,7 @@ class HardLinkManager: For better hardlink support (including the very first hardlink item for each group of same-target hardlinks), we would need a 2-pass processing, which is not yet implemented. """ + def __init__(self, *, id_type, info_type): self._map = {} self.id_type = id_type @@ -198,21 +205,21 @@ def borg1_hardlinkable(self, mode): # legacy return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode) def borg1_hardlink_master(self, item): # legacy - return item.get('hardlink_master', True) and 'source' not in item and self.borg1_hardlinkable(item.mode) + return item.get("hardlink_master", True) and "source" not in item and self.borg1_hardlinkable(item.mode) def borg1_hardlink_slave(self, item): # legacy - return 'source' in item and self.borg1_hardlinkable(item.mode) + return "source" in item and self.borg1_hardlinkable(item.mode) def hardlink_id_from_path(self, path): """compute a hardlink id from a path""" assert isinstance(path, str) - return hashlib.sha256(path.encode('utf-8', errors='surrogateescape')).digest() + return hashlib.sha256(path.encode("utf-8", errors="surrogateescape")).digest() def hardlink_id_from_inode(self, *, ino, dev): """compute a hardlink id from an inode""" assert isinstance(ino, int) assert isinstance(dev, int) - return hashlib.sha256(f'{ino}/{dev}'.encode()).digest() + return hashlib.sha256(f"{ino}/{dev}".encode()).digest() def remember(self, *, id, info): """ @@ -243,7 +250,7 @@ def scandir_keyfunc(dirent): return (0, dirent.inode()) except OSError as e: # maybe a permission denied error while doing a stat() on the dirent - logger.debug('scandir_inorder: Unable to stat %s: %s', dirent.path, e) + logger.debug("scandir_inorder: Unable to stat %s: %s", dirent.path, e) # order this dirent after all the others lexically by file name # we may not break the whole scandir just because of an exception in one dirent # ignore the exception for now, since another stat will be done later anyways @@ -268,7 +275,7 @@ def secure_erase(path, *, avoid_collateral_damage): If avoid_collateral_damage is False, we always secure erase. If there are hardlinks pointing to the same inode as , they will contain random garbage afterwards. """ - with open(path, 'r+b') as fd: + with open(path, "r+b") as fd: st = os.stat(fd.fileno()) if not (st.st_nlink > 1 and avoid_collateral_damage): fd.write(os.urandom(st.st_size)) @@ -303,7 +310,7 @@ def safe_unlink(path): # no other hardlink! try to recover free space by truncating this file. try: # Do not create *path* if it does not exist, open for truncation in r+b mode (=O_RDWR|O_BINARY). - with open(path, 'r+b') as fd: + with open(path, "r+b") as fd: fd.truncate() except OSError: # truncate didn't work, so we still have the original unlink issue - give up: @@ -314,10 +321,10 @@ def safe_unlink(path): def dash_open(path, mode): - assert '+' not in mode # the streams are either r or w, but never both - if path == '-': - stream = sys.stdin if 'r' in mode else sys.stdout - return stream.buffer if 'b' in mode else stream + assert "+" not in mode # the streams are either r or w, but never both + if path == "-": + stream = sys.stdin if "r" in mode else sys.stdout + return stream.buffer if "b" in mode else stream else: return open(path, mode) @@ -325,17 +332,17 @@ def dash_open(path, mode): def O_(*flags): result = 0 for flag in flags: - result |= getattr(os, 'O_' + flag, 0) + result |= getattr(os, "O_" + flag, 0) return result -flags_base = O_('BINARY', 'NOCTTY', 'RDONLY') -flags_special = flags_base | O_('NOFOLLOW') # BLOCK == wait when reading devices or fifos +flags_base = O_("BINARY", "NOCTTY", "RDONLY") +flags_special = flags_base | O_("NOFOLLOW") # BLOCK == wait when reading devices or fifos flags_special_follow = flags_base # BLOCK == wait when reading symlinked devices or fifos -flags_normal = flags_base | O_('NONBLOCK', 'NOFOLLOW') -flags_noatime = flags_normal | O_('NOATIME') -flags_root = O_('RDONLY') -flags_dir = O_('DIRECTORY', 'RDONLY', 'NOFOLLOW') +flags_normal = flags_base | O_("NONBLOCK", "NOFOLLOW") +flags_noatime = flags_normal | O_("NOATIME") +flags_root = O_("RDONLY") +flags_dir = O_("DIRECTORY", "RDONLY", "NOFOLLOW") def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False): @@ -362,7 +369,7 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False): return None _flags_normal = flags if noatime: - _flags_noatime = _flags_normal | O_('NOATIME') + _flags_noatime = _flags_normal | O_("NOATIME") try: # if we have O_NOATIME, this likely will succeed if we are root or owner of file: fd = os.open(fname, _flags_noatime, dir_fd=parent_fd) @@ -375,7 +382,8 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False): except OSError as exc: # O_NOATIME causes EROFS when accessing a volume shadow copy in WSL1 from . import workarounds - if 'retry_erofs' in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal: + + if "retry_erofs" in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal: fd = os.open(fname, _flags_normal, dir_fd=parent_fd) else: raise @@ -407,6 +415,6 @@ def os_stat(*, path=None, parent_fd=None, name=None, follow_symlinks=False): def umount(mountpoint): env = prepare_subprocess_env(system=True) try: - return subprocess.call(['fusermount', '-u', mountpoint], env=env) + return subprocess.call(["fusermount", "-u", mountpoint], env=env) except FileNotFoundError: - return subprocess.call(['umount', mountpoint], env=env) + return subprocess.call(["umount", mountpoint], env=env) diff --git a/src/borg/helpers/manifest.py b/src/borg/helpers/manifest.py index f2eeaad68..1aeaf91fa 100644 --- a/src/borg/helpers/manifest.py +++ b/src/borg/helpers/manifest.py @@ -9,6 +9,7 @@ from .errors import Error from ..logger import create_logger + logger = create_logger() from .datastruct import StableDict @@ -26,10 +27,10 @@ class MandatoryFeatureUnsupported(Error): """Unsupported repository feature(s) {}. A newer version of borg is required to access this repository.""" -ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts') +ArchiveInfo = namedtuple("ArchiveInfo", "name id ts") -AI_HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields) -AI_HUMAN_SORT_KEYS.remove('ts') +AI_HUMAN_SORT_KEYS = ["timestamp"] + list(ArchiveInfo._fields) +AI_HUMAN_SORT_KEYS.remove("ts") class Archives(abc.MutableMapping): @@ -38,6 +39,7 @@ class Archives(abc.MutableMapping): and we can deal with str keys (and it internally encodes to byte keys) and either str timestamps or datetime timestamps. """ + def __init__(self): # key: str archive name, value: dict('id': bytes_id, 'time': str_iso_ts) self._archives = {} @@ -53,8 +55,8 @@ def __getitem__(self, name): values = self._archives.get(name) if values is None: raise KeyError - ts = parse_timestamp(values['time']) - return ArchiveInfo(name=name, id=values['id'], ts=ts) + ts = parse_timestamp(values["time"]) + return ArchiveInfo(name=name, id=values["id"], ts=ts) def __setitem__(self, name, info): assert isinstance(name, str) @@ -64,13 +66,15 @@ def __setitem__(self, name, info): if isinstance(ts, datetime): ts = ts.replace(tzinfo=None).strftime(ISO_FORMAT) assert isinstance(ts, str) - self._archives[name] = {'id': id, 'time': ts} + self._archives[name] = {"id": id, "time": ts} def __delitem__(self, name): assert isinstance(name, str) del self._archives[name] - def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False): + def list( + self, *, glob=None, match_end=r"\Z", sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False + ): """ Return list of ArchiveInfo instances according to the parameters. @@ -84,17 +88,17 @@ def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=T some callers EXPECT to iterate over all archives in a repo for correct operation. """ if isinstance(sort_by, (str, bytes)): - raise TypeError('sort_by must be a sequence of str') - regex = re.compile(shellpattern.translate(glob or '*', match_end=match_end)) + raise TypeError("sort_by must be a sequence of str") + regex = re.compile(shellpattern.translate(glob or "*", match_end=match_end)) archives = [x for x in self.values() if regex.match(x.name) is not None] if not consider_checkpoints: - archives = [x for x in archives if '.checkpoint' not in x.name] + archives = [x for x in archives if ".checkpoint" not in x.name] for sortkey in reversed(sort_by): archives.sort(key=attrgetter(sortkey)) if first: archives = archives[:first] elif last: - archives = archives[max(len(archives) - last, 0):] + archives = archives[max(len(archives) - last, 0) :] if reverse: archives.reverse() return archives @@ -103,17 +107,25 @@ def list_considering(self, args): """ get a list of archives, considering --first/last/prefix/glob-archives/sort/consider-checkpoints cmdline args """ - name = getattr(args, 'name', None) - consider_checkpoints = getattr(args, 'consider_checkpoints', None) + name = getattr(args, "name", None) + consider_checkpoints = getattr(args, "consider_checkpoints", None) if name is not None: - raise Error('Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints.') - return self.list(sort_by=args.sort_by.split(','), consider_checkpoints=consider_checkpoints, glob=args.glob_archives, first=args.first, last=args.last) + raise Error( + "Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints." + ) + return self.list( + sort_by=args.sort_by.split(","), + consider_checkpoints=consider_checkpoints, + glob=args.glob_archives, + first=args.first, + last=args.last, + ) def set_raw_dict(self, d): """set the dict we get from the msgpack unpacker""" for k, v in d.items(): assert isinstance(k, str) - assert isinstance(v, dict) and 'id' in v and 'time' in v + assert isinstance(v, dict) and "id" in v and "time" in v self._archives[k] = v def get_raw_dict(self): @@ -122,7 +134,6 @@ def get_raw_dict(self): class Manifest: - @enum.unique class Operation(enum.Enum): # The comments here only roughly describe the scope of each feature. In the end, additions need to be @@ -133,25 +144,25 @@ class Operation(enum.Enum): # The READ operation describes which features are needed to safely list and extract the archives in the # repository. - READ = 'read' + READ = "read" # The CHECK operation is for all operations that need either to understand every detail # of the repository (for consistency checks and repairs) or are seldom used functions that just # should use the most restrictive feature set because more fine grained compatibility tracking is # not needed. - CHECK = 'check' + CHECK = "check" # The WRITE operation is for adding archives. Features here ensure that older clients don't add archives # in an old format, or is used to lock out clients that for other reasons can no longer safely add new # archives. - WRITE = 'write' + WRITE = "write" # The DELETE operation is for all operations (like archive deletion) that need a 100% correct reference # count and the need to be able to find all (directly and indirectly) referenced chunks of a given archive. - DELETE = 'delete' + DELETE = "delete" NO_OPERATION_CHECK = tuple() SUPPORTED_REPO_FEATURES = frozenset([]) - MANIFEST_ID = b'\0' * 32 + MANIFEST_ID = b"\0" * 32 def __init__(self, key, repository, item_keys=None): self.archives = Archives() @@ -175,6 +186,7 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False): from ..item import ManifestItem from ..crypto.key import key_factory, tam_required_file, tam_required from ..repository import Repository + try: cdata = repository.get(cls.MANIFEST_ID) except Repository.ObjectNotFound: @@ -183,26 +195,28 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False): key = key_factory(repository, cdata) manifest = cls(key, repository) data = key.decrypt(cls.MANIFEST_ID, cdata) - manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(data, force_tam_not_required=force_tam_not_required) + manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest( + data, force_tam_not_required=force_tam_not_required + ) m = ManifestItem(internal_dict=manifest_dict) manifest.id = key.id_hash(data) - if m.get('version') not in (1, 2): - raise ValueError('Invalid manifest version') + if m.get("version") not in (1, 2): + raise ValueError("Invalid manifest version") manifest.archives.set_raw_dict(m.archives) - manifest.timestamp = m.get('timestamp') + manifest.timestamp = m.get("timestamp") manifest.config = m.config # valid item keys are whatever is known in the repo or every key we know - manifest.item_keys = ITEM_KEYS | frozenset(m.get('item_keys', [])) + manifest.item_keys = ITEM_KEYS | frozenset(m.get("item_keys", [])) if manifest.tam_verified: - manifest_required = manifest.config.get('tam_required', False) + manifest_required = manifest.config.get("tam_required", False) security_required = tam_required(repository) if manifest_required and not security_required: - logger.debug('Manifest is TAM verified and says TAM is required, updating security database...') + logger.debug("Manifest is TAM verified and says TAM is required, updating security database...") file = tam_required_file(repository) - open(file, 'w').close() + open(file, "w").close() if not manifest_required and security_required: - logger.debug('Manifest is TAM verified and says TAM is *not* required, updating security database...') + logger.debug("Manifest is TAM verified and says TAM is *not* required, updating security database...") os.unlink(tam_required_file(repository)) manifest.check_repository_compatibility(operations) return manifest, key @@ -210,32 +224,33 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False): def check_repository_compatibility(self, operations): for operation in operations: assert isinstance(operation, self.Operation) - feature_flags = self.config.get('feature_flags', None) + feature_flags = self.config.get("feature_flags", None) if feature_flags is None: return if operation.value not in feature_flags: continue requirements = feature_flags[operation.value] - if 'mandatory' in requirements: - unsupported = set(requirements['mandatory']) - self.SUPPORTED_REPO_FEATURES + if "mandatory" in requirements: + unsupported = set(requirements["mandatory"]) - self.SUPPORTED_REPO_FEATURES if unsupported: raise MandatoryFeatureUnsupported(list(unsupported)) def get_all_mandatory_features(self): result = {} - feature_flags = self.config.get('feature_flags', None) + feature_flags = self.config.get("feature_flags", None) if feature_flags is None: return result for operation, requirements in feature_flags.items(): - if 'mandatory' in requirements: - result[operation] = set(requirements['mandatory']) + if "mandatory" in requirements: + result[operation] = set(requirements["mandatory"]) return result def write(self): from ..item import ManifestItem + if self.key.tam_required: - self.config['tam_required'] = True + self.config["tam_required"] = True # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly if self.timestamp is None: self.timestamp = datetime.utcnow().strftime(ISO_FORMAT) diff --git a/src/borg/helpers/misc.py b/src/borg/helpers/misc.py index 96eb78993..ae40974db 100644 --- a/src/borg/helpers/misc.py +++ b/src/borg/helpers/misc.py @@ -10,6 +10,7 @@ from operator import attrgetter from ..logger import create_logger + logger = create_logger() from .time import to_localtime @@ -30,15 +31,17 @@ def prune_within(archives, hours, kept_because): return result -PRUNING_PATTERNS = OrderedDict([ - ("secondly", '%Y-%m-%d %H:%M:%S'), - ("minutely", '%Y-%m-%d %H:%M'), - ("hourly", '%Y-%m-%d %H'), - ("daily", '%Y-%m-%d'), - ("weekly", '%G-%V'), - ("monthly", '%Y-%m'), - ("yearly", '%Y'), -]) +PRUNING_PATTERNS = OrderedDict( + [ + ("secondly", "%Y-%m-%d %H:%M:%S"), + ("minutely", "%Y-%m-%d %H:%M"), + ("hourly", "%Y-%m-%d %H"), + ("daily", "%Y-%m-%d"), + ("weekly", "%G-%V"), + ("monthly", "%Y-%m"), + ("yearly", "%Y"), + ] +) def prune_split(archives, rule, n, kept_because=None): @@ -51,7 +54,7 @@ def prune_split(archives, rule, n, kept_because=None): return keep a = None - for a in sorted(archives, key=attrgetter('ts'), reverse=True): + for a in sorted(archives, key=attrgetter("ts"), reverse=True): period = to_localtime(a.ts).strftime(pattern) if period != last: last = period @@ -63,14 +66,14 @@ def prune_split(archives, rule, n, kept_because=None): # Keep oldest archive if we didn't reach the target retention count if a is not None and len(keep) < n and a.id not in kept_because: keep.append(a) - kept_because[a.id] = (rule+"[oldest]", len(keep)) + kept_because[a.id] = (rule + "[oldest]", len(keep)) return keep def sysinfo(): - show_sysinfo = os.environ.get('BORG_SHOW_SYSINFO', 'yes').lower() - if show_sysinfo == 'no': - return '' + show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower() + if show_sysinfo == "no": + return "" python_implementation = platform.python_implementation() python_version = platform.python_version() @@ -80,30 +83,34 @@ def sysinfo(): uname = os.uname() except AttributeError: uname = None - if sys.platform.startswith('linux'): - linux_distribution = ('Unknown Linux', '', '') + if sys.platform.startswith("linux"): + linux_distribution = ("Unknown Linux", "", "") else: linux_distribution = None try: - msgpack_version = '.'.join(str(v) for v in msgpack.version) + msgpack_version = ".".join(str(v) for v in msgpack.version) except: - msgpack_version = 'unknown' + msgpack_version = "unknown" from ..fuse_impl import llfuse, BORG_FUSE_IMPL - llfuse_name = llfuse.__name__ if llfuse else 'None' - llfuse_version = (' %s' % llfuse.__version__) if llfuse else '' - llfuse_info = f'{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]' + + llfuse_name = llfuse.__name__ if llfuse else "None" + llfuse_version = (" %s" % llfuse.__version__) if llfuse else "" + llfuse_info = f"{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]" info = [] if uname is not None: - info.append('Platform: {}'.format(' '.join(uname))) + info.append("Platform: {}".format(" ".join(uname))) if linux_distribution is not None: - info.append('Linux: %s %s %s' % linux_distribution) - info.append('Borg: {} Python: {} {} msgpack: {} fuse: {}'.format( - borg_version, python_implementation, python_version, msgpack_version, llfuse_info)) - info.append('PID: %d CWD: %s' % (os.getpid(), os.getcwd())) - info.append('sys.argv: %r' % sys.argv) - info.append('SSH_ORIGINAL_COMMAND: %r' % os.environ.get('SSH_ORIGINAL_COMMAND')) - info.append('') - return '\n'.join(info) + info.append("Linux: %s %s %s" % linux_distribution) + info.append( + "Borg: {} Python: {} {} msgpack: {} fuse: {}".format( + borg_version, python_implementation, python_version, msgpack_version, llfuse_info + ) + ) + info.append("PID: %d CWD: %s" % (os.getpid(), os.getcwd())) + info.append("sys.argv: %r" % sys.argv) + info.append("SSH_ORIGINAL_COMMAND: %r" % os.environ.get("SSH_ORIGINAL_COMMAND")) + info.append("") + return "\n".join(info) def log_multi(*msgs, level=logging.INFO, logger=logger): @@ -133,7 +140,7 @@ def __init__(self, chunk_iterator, read_callback=None): """ self.chunk_iterator = chunk_iterator self.chunk_offset = 0 - self.chunk = b'' + self.chunk = b"" self.exhausted = False self.read_callback = read_callback @@ -152,11 +159,11 @@ def _refill(self): def _read(self, nbytes): if not nbytes: - return b'' + return b"" remaining = self._refill() will_read = min(remaining, nbytes) self.chunk_offset += will_read - return self.chunk[self.chunk_offset - will_read:self.chunk_offset] + return self.chunk[self.chunk_offset - will_read : self.chunk_offset] def read(self, nbytes): parts = [] @@ -166,7 +173,7 @@ def read(self, nbytes): parts.append(read_data) if self.read_callback: self.read_callback(read_data) - return b''.join(parts) + return b"".join(parts) def open_item(archive, item): @@ -207,7 +214,7 @@ def read(self, n): super().close() except OSError: pass - return '' + return "" def write(self, s): if not self.closed: @@ -225,8 +232,8 @@ def iter_separated(fd, sep=None, read_size=4096): """Iter over chunks of open file ``fd`` delimited by ``sep``. Doesn't trim.""" buf = fd.read(read_size) is_str = isinstance(buf, str) - part = '' if is_str else b'' - sep = sep or ('\n' if is_str else b'\n') + part = "" if is_str else b"" + sep = sep or ("\n" if is_str else b"\n") while len(buf) > 0: part2, *items = buf.split(sep) *full, part = (part + part2, *items) @@ -240,17 +247,17 @@ def iter_separated(fd, sep=None, read_size=4096): def get_tar_filter(fname, decompress): # Note that filter is None if fname is '-'. - if fname.endswith(('.tar.gz', '.tgz')): - filter = 'gzip -d' if decompress else 'gzip' - elif fname.endswith(('.tar.bz2', '.tbz')): - filter = 'bzip2 -d' if decompress else 'bzip2' - elif fname.endswith(('.tar.xz', '.txz')): - filter = 'xz -d' if decompress else 'xz' - elif fname.endswith(('.tar.lz4', )): - filter = 'lz4 -d' if decompress else 'lz4' - elif fname.endswith(('.tar.zstd', )): - filter = 'zstd -d' if decompress else 'zstd' + if fname.endswith((".tar.gz", ".tgz")): + filter = "gzip -d" if decompress else "gzip" + elif fname.endswith((".tar.bz2", ".tbz")): + filter = "bzip2 -d" if decompress else "bzip2" + elif fname.endswith((".tar.xz", ".txz")): + filter = "xz -d" if decompress else "xz" + elif fname.endswith((".tar.lz4",)): + filter = "lz4 -d" if decompress else "lz4" + elif fname.endswith((".tar.zstd",)): + filter = "zstd -d" if decompress else "zstd" else: filter = None - logger.debug('Automatically determined tar filter: %s', filter) + logger.debug("Automatically determined tar filter: %s", filter) return filter diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py index c1e753dff..0ea51d019 100644 --- a/src/borg/helpers/msgpack.py +++ b/src/borg/helpers/msgpack.py @@ -64,7 +64,7 @@ USE_BIN_TYPE = True RAW = False -UNICODE_ERRORS = 'surrogateescape' +UNICODE_ERRORS = "surrogateescape" class PackException(Exception): @@ -76,13 +76,25 @@ class UnpackException(Exception): class Packer(mp_Packer): - def __init__(self, *, default=None, unicode_errors=UNICODE_ERRORS, - use_single_float=False, autoreset=True, use_bin_type=USE_BIN_TYPE, - strict_types=False): + def __init__( + self, + *, + default=None, + unicode_errors=UNICODE_ERRORS, + use_single_float=False, + autoreset=True, + use_bin_type=USE_BIN_TYPE, + strict_types=False + ): assert unicode_errors == UNICODE_ERRORS - super().__init__(default=default, unicode_errors=unicode_errors, - use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type, - strict_types=strict_types) + super().__init__( + default=default, + unicode_errors=unicode_errors, + use_single_float=use_single_float, + autoreset=autoreset, + use_bin_type=use_bin_type, + strict_types=strict_types, + ) def pack(self, obj): try: @@ -108,18 +120,36 @@ def pack(o, stream, *, use_bin_type=USE_BIN_TYPE, unicode_errors=UNICODE_ERRORS, class Unpacker(mp_Unpacker): - def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=RAW, - object_hook=None, object_pairs_hook=None, list_hook=None, - unicode_errors=UNICODE_ERRORS, max_buffer_size=0, - ext_hook=ExtType, - strict_map_key=False): + def __init__( + self, + file_like=None, + *, + read_size=0, + use_list=True, + raw=RAW, + object_hook=None, + object_pairs_hook=None, + list_hook=None, + unicode_errors=UNICODE_ERRORS, + max_buffer_size=0, + ext_hook=ExtType, + strict_map_key=False + ): assert raw == RAW assert unicode_errors == UNICODE_ERRORS - kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, - object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, - ext_hook=ext_hook, - strict_map_key=strict_map_key) + kw = dict( + file_like=file_like, + read_size=read_size, + use_list=use_list, + raw=raw, + object_hook=object_hook, + object_pairs_hook=object_pairs_hook, + list_hook=list_hook, + unicode_errors=unicode_errors, + max_buffer_size=max_buffer_size, + ext_hook=ext_hook, + strict_map_key=strict_map_key, + ) super().__init__(**kw) def unpack(self): @@ -141,28 +171,22 @@ def __next__(self): next = __next__ -def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS, - strict_map_key=False, - **kwargs): +def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs): assert raw == RAW assert unicode_errors == UNICODE_ERRORS try: - kw = dict(raw=raw, unicode_errors=unicode_errors, - strict_map_key=strict_map_key) + kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key) kw.update(kwargs) return mp_unpackb(packed, **kw) except Exception as e: raise UnpackException(e) -def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS, - strict_map_key=False, - **kwargs): +def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs): assert raw == RAW assert unicode_errors == UNICODE_ERRORS try: - kw = dict(raw=raw, unicode_errors=unicode_errors, - strict_map_key=strict_map_key) + kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key) kw.update(kwargs) return mp_unpack(stream, **kw) except Exception as e: @@ -171,32 +195,34 @@ def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS, # msgpacking related utilities ----------------------------------------------- + def is_slow_msgpack(): import msgpack import msgpack.fallback + return msgpack.Packer is msgpack.fallback.Packer def is_supported_msgpack(): # DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py. import msgpack - return (1, 0, 3) <= msgpack.version <= (1, 0, 4) and \ - msgpack.version not in [] # < add bad releases here to deny list + + return (1, 0, 3) <= msgpack.version <= ( + 1, + 0, + 4, + ) and msgpack.version not in [] # < add bad releases here to deny list def get_limited_unpacker(kind): """return a limited Unpacker because we should not trust msgpack data received from remote""" # Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for # unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N). - args = dict(use_list=False, # return tuples, not lists - max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE), - ) - if kind in ('server', 'client'): + args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE)) # return tuples, not lists + if kind in ("server", "client"): pass # nothing special - elif kind in ('manifest', 'key'): - args.update(dict(use_list=True, # default value - object_hook=StableDict, - )) + elif kind in ("manifest", "key"): + args.update(dict(use_list=True, object_hook=StableDict)) # default value else: raise ValueError('kind must be "server", "client", "manifest" or "key"') return Unpacker(**args) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index a9d2ac960..b058ada99 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -15,6 +15,7 @@ from string import Formatter from ..logger import create_logger + logger = create_logger() from .errors import Error @@ -28,34 +29,34 @@ def bin_to_hex(binary): - return hexlify(binary).decode('ascii') + return hexlify(binary).decode("ascii") -def safe_decode(s, coding='utf-8', errors='surrogateescape'): +def safe_decode(s, coding="utf-8", errors="surrogateescape"): """decode bytes to str, with round-tripping "invalid" bytes""" if s is None: return None return s.decode(coding, errors) -def safe_encode(s, coding='utf-8', errors='surrogateescape'): +def safe_encode(s, coding="utf-8", errors="surrogateescape"): """encode str to bytes, with round-tripping "invalid" bytes""" if s is None: return None return s.encode(coding, errors) -def remove_surrogates(s, errors='replace'): +def remove_surrogates(s, errors="replace"): """Replace surrogates generated by fsdecode with '?'""" - return s.encode('utf-8', errors).decode('utf-8') + return s.encode("utf-8", errors).decode("utf-8") def eval_escapes(s): """Evaluate literal escape sequences in a string (eg `\\n` -> `\n`).""" - return s.encode('ascii', 'backslashreplace').decode('unicode-escape') + return s.encode("ascii", "backslashreplace").decode("unicode-escape") -def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'): +def decode_dict(d, keys, encoding="utf-8", errors="surrogateescape"): for key in keys: if isinstance(d.get(key), bytes): d[key] = d[key].decode(encoding, errors) @@ -66,13 +67,13 @@ def positive_int_validator(value): """argparse type for positive integers""" int_value = int(value) if int_value <= 0: - raise argparse.ArgumentTypeError('A positive integer is required: %s' % value) + raise argparse.ArgumentTypeError("A positive integer is required: %s" % value) return int_value def interval(s): """Convert a string representing a valid interval to a number of hours.""" - multiplier = {'H': 1, 'd': 24, 'w': 24 * 7, 'm': 24 * 31, 'y': 24 * 365} + multiplier = {"H": 1, "d": 24, "w": 24 * 7, "m": 24 * 31, "y": 24 * 365} if s.endswith(tuple(multiplier.keys())): number = s[:-1] @@ -80,8 +81,7 @@ def interval(s): else: # range suffixes in ascending multiplier order ranges = [k for k, v in sorted(multiplier.items(), key=lambda t: t[1])] - raise argparse.ArgumentTypeError( - f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}') + raise argparse.ArgumentTypeError(f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}') try: hours = int(number) * multiplier[suffix] @@ -89,17 +89,16 @@ def interval(s): hours = -1 if hours <= 0: - raise argparse.ArgumentTypeError( - 'Unexpected interval number "%s": expected an integer greater than 0' % number) + raise argparse.ArgumentTypeError('Unexpected interval number "%s": expected an integer greater than 0' % number) return hours def ChunkerParams(s): - params = s.strip().split(',') + params = s.strip().split(",") count = len(params) if count == 0: - raise ValueError('no chunker params given') + raise ValueError("no chunker params given") algo = params[0].lower() if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size] block_size = int(params[1]) @@ -110,36 +109,36 @@ def ChunkerParams(s): # or in-memory chunk management. # choose the block (chunk) size wisely: if you have a lot of data and you cut # it into very small chunks, you are asking for trouble! - raise ValueError('block_size must not be less than 64 Bytes') + raise ValueError("block_size must not be less than 64 Bytes") if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE: - raise ValueError('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE) + raise ValueError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE) return algo, block_size, header_size - if algo == 'default' and count == 1: # default + if algo == "default" and count == 1: # default return CHUNKER_PARAMS # this must stay last as it deals with old-style compat mode (no algorithm, 4 params, buzhash): if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size - chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:]) + chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4 :]) if not (chunk_min <= chunk_mask <= chunk_max): - raise ValueError('required: chunk_min <= chunk_mask <= chunk_max') + raise ValueError("required: chunk_min <= chunk_mask <= chunk_max") if chunk_min < 6: # see comment in 'fixed' algo check - raise ValueError('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)') + raise ValueError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)") if chunk_max > 23: - raise ValueError('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)') + raise ValueError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)") return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size - raise ValueError('invalid chunker params') + raise ValueError("invalid chunker params") def FilesCacheMode(s): - ENTRIES_MAP = dict(ctime='c', mtime='m', size='s', inode='i', rechunk='r', disabled='d') - VALID_MODES = ('cis', 'ims', 'cs', 'ms', 'cr', 'mr', 'd', 's') # letters in alpha order - entries = set(s.strip().split(',')) + ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d") + VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s") # letters in alpha order + entries = set(s.strip().split(",")) if not entries <= set(ENTRIES_MAP): - raise ValueError('cache mode must be a comma-separated list of: %s' % ','.join(sorted(ENTRIES_MAP))) + raise ValueError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP))) short_entries = {ENTRIES_MAP[entry] for entry in entries} - mode = ''.join(sorted(short_entries)) + mode = "".join(sorted(short_entries)) if mode not in VALID_MODES: - raise ValueError('cache mode short must be one of: %s' % ','.join(VALID_MODES)) + raise ValueError("cache mode short must be one of: %s" % ",".join(VALID_MODES)) return mode @@ -151,9 +150,9 @@ def partial_format(format, mapping): """ for key, value in mapping.items(): key = re.escape(key) - format = re.sub(fr'(? 0 else '' - fmt = '{0:{1}.{2}f}{3}{4}{5}' +def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False): + sign = "+" if sign and num > 0 else "" + fmt = "{0:{1}.{2}f}{3}{4}{5}" prec = 0 for unit in units[:-1]: if abs(round(num, precision)) < power: @@ -277,27 +271,37 @@ def sizeof_fmt(num, suffix='B', units=None, power=None, sep='', precision=2, sig return fmt.format(num, sign, prec, sep, unit, suffix) -def sizeof_fmt_iec(num, suffix='B', sep='', precision=2, sign=False): - return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign, - units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024) +def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False): + return sizeof_fmt( + num, + suffix=suffix, + sep=sep, + precision=precision, + sign=sign, + units=["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"], + power=1024, + ) -def sizeof_fmt_decimal(num, suffix='B', sep='', precision=2, sign=False): - return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign, - units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000) +def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False): + return sizeof_fmt( + num, + suffix=suffix, + sep=sep, + precision=precision, + sign=sign, + units=["", "k", "M", "G", "T", "P", "E", "Z", "Y"], + power=1000, + ) def format_archive(archive): - return '%-36s %s [%s]' % ( - archive.name, - format_time(to_localtime(archive.ts)), - bin_to_hex(archive.id), - ) + return "%-36s %s [%s]" % (archive.name, format_time(to_localtime(archive.ts)), bin_to_hex(archive.id)) def parse_stringified_list(s): l = re.split(" *, *", s) - return [item for item in l if item != ''] + return [item for item in l if item != ""] class Location: @@ -343,28 +347,42 @@ class Location: """ # regexes for misc. kinds of supported location specifiers: - ssh_re = re.compile(r""" + ssh_re = re.compile( + r""" (?Pssh):// # ssh:// - """ + optional_user_re + host_re + r""" # user@ (optional), host name or address + """ + + optional_user_re + + host_re + + r""" # user@ (optional), host name or address (?::(?P\d+))? # :port (optional) - """ + abs_path_re, re.VERBOSE) # path + """ + + abs_path_re, + re.VERBOSE, + ) # path - file_re = re.compile(r""" + file_re = re.compile( + r""" (?Pfile):// # file:// - """ + file_path_re, re.VERBOSE) # servername/path or path + """ + + file_path_re, + re.VERBOSE, + ) # servername/path or path - local_re = re.compile(local_path_re, re.VERBOSE) # local path + local_re = re.compile(local_path_re, re.VERBOSE) # local path - win_file_re = re.compile(r""" + win_file_re = re.compile( + r""" (?:file://)? # optional file protocol (?P (?:[a-zA-Z]:)? # Drive letter followed by a colon (optional) (?:[^:]+) # Anything which does not contain a :, at least one character ) - """, re.VERBOSE) + """, + re.VERBOSE, + ) - def __init__(self, text='', overrides={}, other=False): - self.repo_env_var = 'BORG_OTHER_REPO' if other else 'BORG_REPO' + def __init__(self, text="", overrides={}, other=False): + self.repo_env_var = "BORG_OTHER_REPO" if other else "BORG_REPO" self.valid = False self.proto = None self.user = None @@ -393,15 +411,15 @@ def parse(self, text, overrides={}): def _parse(self, text): def normpath_special(p): # avoid that normpath strips away our relative path hack and even makes p absolute - relative = p.startswith('/./') + relative = p.startswith("/./") p = os.path.normpath(p) - return ('/.' + p) if relative else p + return ("/." + p) if relative else p if is_win32: m = self.win_file_re.match(text) if m: - self.proto = 'file' - self.path = m.group('path') + self.proto = "file" + self.path = m.group("path") return True # On windows we currently only support windows paths. @@ -409,38 +427,38 @@ def normpath_special(p): m = self.ssh_re.match(text) if m: - self.proto = m.group('proto') - self.user = m.group('user') - self._host = m.group('host') - self.port = m.group('port') and int(m.group('port')) or None - self.path = normpath_special(m.group('path')) + self.proto = m.group("proto") + self.user = m.group("user") + self._host = m.group("host") + self.port = m.group("port") and int(m.group("port")) or None + self.path = normpath_special(m.group("path")) return True m = self.file_re.match(text) if m: - self.proto = m.group('proto') - self.path = normpath_special(m.group('path')) + self.proto = m.group("proto") + self.path = normpath_special(m.group("path")) return True m = self.local_re.match(text) if m: - self.proto = 'file' - self.path = normpath_special(m.group('path')) + self.proto = "file" + self.path = normpath_special(m.group("path")) return True return False def __str__(self): items = [ - 'proto=%r' % self.proto, - 'user=%r' % self.user, - 'host=%r' % self.host, - 'port=%r' % self.port, - 'path=%r' % self.path, + "proto=%r" % self.proto, + "user=%r" % self.user, + "host=%r" % self.host, + "port=%r" % self.port, + "path=%r" % self.path, ] - return ', '.join(items) + return ", ".join(items) def to_key_filename(self): - name = re.sub(r'[^\w]', '_', self.path).strip('_') - if self.proto != 'file': - name = re.sub(r'[^\w]', '_', self.host) + '__' + name + name = re.sub(r"[^\w]", "_", self.path).strip("_") + if self.proto != "file": + name = re.sub(r"[^\w]", "_", self.host) + "__" + name if len(name) > 100: # Limit file names to some reasonable length. Most file systems # limit them to 255 [unit of choice]; due to variations in unicode @@ -455,28 +473,30 @@ def __repr__(self): def host(self): # strip square brackets used for IPv6 addrs if self._host is not None: - return self._host.lstrip('[').rstrip(']') + return self._host.lstrip("[").rstrip("]") def canonical_path(self): - if self.proto == 'file': + if self.proto == "file": return self.path else: - if self.path and self.path.startswith('~'): - path = '/' + self.path # /~/x = path x relative to home dir - elif self.path and not self.path.startswith('/'): - path = '/./' + self.path # /./x = path x relative to cwd + if self.path and self.path.startswith("~"): + path = "/" + self.path # /~/x = path x relative to home dir + elif self.path and not self.path.startswith("/"): + path = "/./" + self.path # /./x = path x relative to cwd else: path = self.path - return 'ssh://{}{}{}{}'.format(f'{self.user}@' if self.user else '', - self._host, # needed for ipv6 addrs - f':{self.port}' if self.port else '', - path) + return "ssh://{}{}{}{}".format( + f"{self.user}@" if self.user else "", + self._host, # needed for ipv6 addrs + f":{self.port}" if self.port else "", + path, + ) def with_timestamp(self, timestamp): - return Location(self.raw, overrides={ - 'now': DatetimeWrapper(timestamp.astimezone(None)), - 'utcnow': DatetimeWrapper(timestamp), - }) + return Location( + self.raw, + overrides={"now": DatetimeWrapper(timestamp.astimezone(None)), "utcnow": DatetimeWrapper(timestamp)}, + ) def location_validator(proto=None, other=False): @@ -486,33 +506,35 @@ def validator(text): except ValueError as err: raise argparse.ArgumentTypeError(str(err)) from None if proto is not None and loc.proto != proto: - if proto == 'file': + if proto == "file": raise argparse.ArgumentTypeError('"%s": Repository must be local' % text) else: raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text) return loc + return validator def archivename_validator(): def validator(text): text = replace_placeholders(text) - if '/' in text or '::' in text or not text: + if "/" in text or "::" in text or not text: raise argparse.ArgumentTypeError('Invalid archive name: "%s"' % text) return text + return validator class BaseFormatter: FIXED_KEYS = { # Formatting aids - 'LF': '\n', - 'SPACE': ' ', - 'TAB': '\t', - 'CR': '\r', - 'NUL': '\0', - 'NEWLINE': os.linesep, - 'NL': os.linesep, + "LF": "\n", + "SPACE": " ", + "TAB": "\t", + "CR": "\r", + "NUL": "\0", + "NEWLINE": os.linesep, + "NL": os.linesep, } def get_item_data(self, item): @@ -523,42 +545,45 @@ def format_item(self, item): @staticmethod def keys_help(): - return "- NEWLINE: OS dependent line separator\n" \ - "- NL: alias of NEWLINE\n" \ - "- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n" \ - "- SPACE\n" \ - "- TAB\n" \ - "- CR\n" \ - "- LF" + return ( + "- NEWLINE: OS dependent line separator\n" + "- NL: alias of NEWLINE\n" + "- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n" + "- SPACE\n" + "- TAB\n" + "- CR\n" + "- LF" + ) class ArchiveFormatter(BaseFormatter): KEY_DESCRIPTIONS = { - 'archive': 'archive name interpreted as text (might be missing non-text characters, see barchive)', - 'name': 'alias of "archive"', - 'barchive': 'verbatim archive name, can contain any character except NUL', - 'comment': 'archive comment interpreted as text (might be missing non-text characters, see bcomment)', - 'bcomment': 'verbatim archive comment, can contain any character except NUL', + "archive": "archive name interpreted as text (might be missing non-text characters, see barchive)", + "name": 'alias of "archive"', + "barchive": "verbatim archive name, can contain any character except NUL", + "comment": "archive comment interpreted as text (might be missing non-text characters, see bcomment)", + "bcomment": "verbatim archive comment, can contain any character except NUL", # *start* is the key used by borg-info for this timestamp, this makes the formats more compatible - 'start': 'time (start) of creation of the archive', - 'time': 'alias of "start"', - 'end': 'time (end) of creation of the archive', - 'command_line': 'command line which was used to create the archive', - 'id': 'internal ID of the archive', - 'hostname': 'hostname of host on which this archive was created', - 'username': 'username of user who created this archive', + "start": "time (start) of creation of the archive", + "time": 'alias of "start"', + "end": "time (end) of creation of the archive", + "command_line": "command line which was used to create the archive", + "id": "internal ID of the archive", + "hostname": "hostname of host on which this archive was created", + "username": "username of user who created this archive", } KEY_GROUPS = ( - ('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'), - ('start', 'time', 'end', 'command_line'), - ('hostname', 'username'), + ("archive", "name", "barchive", "comment", "bcomment", "id"), + ("start", "time", "end", "command_line"), + ("hostname", "username"), ) @classmethod def available_keys(cls): from .manifest import ArchiveInfo - fake_archive_info = ArchiveInfo('archivename', b'\1'*32, datetime(1970, 1, 1, tzinfo=timezone.utc)) - formatter = cls('', None, None, None) + + fake_archive_info = ArchiveInfo("archivename", b"\1" * 32, datetime(1970, 1, 1, tzinfo=timezone.utc)) + formatter = cls("", None, None, None) keys = [] keys.extend(formatter.call_keys.keys()) keys.extend(formatter.get_item_data(fake_archive_info).keys()) @@ -596,12 +621,12 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False): self.format = partial_format(format, static_keys) self.format_keys = {f[1] for f in Formatter().parse(format)} self.call_keys = { - 'hostname': partial(self.get_meta, 'hostname', rs=True), - 'username': partial(self.get_meta, 'username', rs=True), - 'comment': partial(self.get_meta, 'comment', rs=True), - 'bcomment': partial(self.get_meta, 'comment', rs=False), - 'end': self.get_ts_end, - 'command_line': self.get_cmdline, + "hostname": partial(self.get_meta, "hostname", rs=True), + "username": partial(self.get_meta, "username", rs=True), + "comment": partial(self.get_meta, "comment", rs=True), + "bcomment": partial(self.get_meta, "comment", rs=False), + "end": self.get_ts_end, + "command_line": self.get_cmdline, } self.used_call_keys = set(self.call_keys) & self.format_keys if self.json: @@ -611,21 +636,23 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False): self.item_data = static_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n" def get_item_data(self, archive_info): self.name = archive_info.name self.id = archive_info.id item_data = {} item_data.update(self.item_data) - item_data.update({ - 'name': remove_surrogates(archive_info.name), - 'archive': remove_surrogates(archive_info.name), - 'barchive': archive_info.name, - 'id': bin_to_hex(archive_info.id), - 'time': self.format_time(archive_info.ts), - 'start': self.format_time(archive_info.ts), - }) + item_data.update( + { + "name": remove_surrogates(archive_info.name), + "archive": remove_surrogates(archive_info.name), + "barchive": archive_info.name, + "id": bin_to_hex(archive_info.id), + "time": self.format_time(archive_info.ts), + "start": self.format_time(archive_info.ts), + } + ) for key in self.used_call_keys: item_data[key] = self.call_keys[key]() return item_data @@ -635,19 +662,20 @@ def archive(self): """lazy load / update loaded archive""" if self._archive is None or self._archive.id != self.id: from ..archive import Archive + self._archive = Archive(self.repository, self.key, self.manifest, self.name, iec=self.iec) return self._archive def get_meta(self, key, rs): - value = self.archive.metadata.get(key, '') + value = self.archive.metadata.get(key, "") return remove_surrogates(value) if rs else value def get_cmdline(self): - cmdline = map(remove_surrogates, self.archive.metadata.get('cmdline', [])) + cmdline = map(remove_surrogates, self.archive.metadata.get("cmdline", [])) if self.json: return list(cmdline) else: - return ' '.join(map(shlex.quote, cmdline)) + return " ".join(map(shlex.quote, cmdline)) def get_ts_end(self): return self.format_time(self.archive.ts_end) @@ -659,31 +687,29 @@ def format_time(self, ts): class ItemFormatter(BaseFormatter): # we provide the hash algos from python stdlib (except shake_*) and additionally xxh64. # shake_* is not provided because it uses an incompatible .digest() method to support variable length. - hash_algorithms = hashlib.algorithms_guaranteed.union({'xxh64'}).difference({'shake_128', 'shake_256'}) + hash_algorithms = hashlib.algorithms_guaranteed.union({"xxh64"}).difference({"shake_128", "shake_256"}) KEY_DESCRIPTIONS = { - 'bpath': 'verbatim POSIX path, can contain any character except NUL', - 'path': 'path interpreted as text (might be missing non-text characters, see bpath)', - 'source': 'link target for symlinks (identical to linktarget)', - 'hlid': 'hard link identity (same if hardlinking same fs object)', - 'extra': 'prepends {source} with " -> " for soft links and " link to " for hard links', - 'dsize': 'deduplicated size', - 'num_chunks': 'number of chunks in this file', - 'unique_chunks': 'number of unique chunks in this file', - 'xxh64': 'XXH64 checksum of this file (note: this is NOT a cryptographic hash!)', - 'health': 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)', + "bpath": "verbatim POSIX path, can contain any character except NUL", + "path": "path interpreted as text (might be missing non-text characters, see bpath)", + "source": "link target for symlinks (identical to linktarget)", + "hlid": "hard link identity (same if hardlinking same fs object)", + "extra": 'prepends {source} with " -> " for soft links and " link to " for hard links', + "dsize": "deduplicated size", + "num_chunks": "number of chunks in this file", + "unique_chunks": "number of unique chunks in this file", + "xxh64": "XXH64 checksum of this file (note: this is NOT a cryptographic hash!)", + "health": 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)', } KEY_GROUPS = ( - ('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'hlid', 'flags'), - ('size', 'dsize', 'num_chunks', 'unique_chunks'), - ('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'), + ("type", "mode", "uid", "gid", "user", "group", "path", "bpath", "source", "linktarget", "hlid", "flags"), + ("size", "dsize", "num_chunks", "unique_chunks"), + ("mtime", "ctime", "atime", "isomtime", "isoctime", "isoatime"), tuple(sorted(hash_algorithms)), - ('archiveid', 'archivename', 'extra'), - ('health', ) + ("archiveid", "archivename", "extra"), + ("health",), ) - KEYS_REQUIRING_CACHE = ( - 'dsize', 'unique_chunks', - ) + KEYS_REQUIRING_CACHE = ("dsize", "unique_chunks") @classmethod def available_keys(cls): @@ -691,7 +717,8 @@ class FakeArchive: fpr = name = "" from ..item import Item - fake_item = Item(mode=0, path='', user='', group='', mtime=0, uid=0, gid=0) + + fake_item = Item(mode=0, path="", user="", group="", mtime=0, uid=0, gid=0) formatter = cls(FakeArchive, "") keys = [] keys.extend(formatter.call_keys.keys()) @@ -723,13 +750,11 @@ def format_needs_cache(cls, format): def __init__(self, archive, format, *, json_lines=False): from ..checksums import StreamingXXH64 + self.xxh64 = StreamingXXH64 self.archive = archive self.json_lines = json_lines - static_keys = { - 'archivename': archive.name, - 'archiveid': archive.fpr, - } + static_keys = {"archivename": archive.name, "archiveid": archive.fpr} static_keys.update(self.FIXED_KEYS) if self.json_lines: self.item_data = {} @@ -739,23 +764,23 @@ def __init__(self, archive, format, *, json_lines=False): self.format = partial_format(format, static_keys) self.format_keys = {f[1] for f in Formatter().parse(format)} self.call_keys = { - 'size': self.calculate_size, - 'dsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size), - 'num_chunks': self.calculate_num_chunks, - 'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1), - 'isomtime': partial(self.format_iso_time, 'mtime'), - 'isoctime': partial(self.format_iso_time, 'ctime'), - 'isoatime': partial(self.format_iso_time, 'atime'), - 'mtime': partial(self.format_time, 'mtime'), - 'ctime': partial(self.format_time, 'ctime'), - 'atime': partial(self.format_time, 'atime'), + "size": self.calculate_size, + "dsize": partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size), + "num_chunks": self.calculate_num_chunks, + "unique_chunks": partial(self.sum_unique_chunks_metadata, lambda chunk: 1), + "isomtime": partial(self.format_iso_time, "mtime"), + "isoctime": partial(self.format_iso_time, "ctime"), + "isoatime": partial(self.format_iso_time, "atime"), + "mtime": partial(self.format_time, "mtime"), + "ctime": partial(self.format_time, "ctime"), + "atime": partial(self.format_time, "atime"), } for hash_function in self.hash_algorithms: self.call_keys[hash_function] = partial(self.hash_item, hash_function) self.used_call_keys = set(self.call_keys) & self.format_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n" def get_item_data(self, item): item_data = {} @@ -763,30 +788,30 @@ def get_item_data(self, item): mode = stat.filemode(item.mode) item_type = mode[0] - source = item.get('source', '') - extra = '' + source = item.get("source", "") + extra = "" if source: source = remove_surrogates(source) - extra = ' -> %s' % source - hlid = item.get('hlid') - hlid = bin_to_hex(hlid) if hlid else '' - item_data['type'] = item_type - item_data['mode'] = mode - item_data['user'] = item.get('user', str(item.uid)) - item_data['group'] = item.get('group', str(item.gid)) - item_data['uid'] = item.uid - item_data['gid'] = item.gid - item_data['path'] = remove_surrogates(item.path) + extra = " -> %s" % source + hlid = item.get("hlid") + hlid = bin_to_hex(hlid) if hlid else "" + item_data["type"] = item_type + item_data["mode"] = mode + item_data["user"] = item.get("user", str(item.uid)) + item_data["group"] = item.get("group", str(item.gid)) + item_data["uid"] = item.uid + item_data["gid"] = item.gid + item_data["path"] = remove_surrogates(item.path) if self.json_lines: - item_data['healthy'] = 'chunks_healthy' not in item + item_data["healthy"] = "chunks_healthy" not in item else: - item_data['bpath'] = item.path - item_data['extra'] = extra - item_data['health'] = 'broken' if 'chunks_healthy' in item else 'healthy' - item_data['source'] = source - item_data['linktarget'] = source - item_data['hlid'] = hlid - item_data['flags'] = item.get('bsdflags') + item_data["bpath"] = item.path + item_data["extra"] = extra + item_data["health"] = "broken" if "chunks_healthy" in item else "healthy" + item_data["source"] = source + item_data["linktarget"] = source + item_data["hlid"] = hlid + item_data["flags"] = item.get("bsdflags") for key in self.used_call_keys: item_data[key] = self.call_keys[key](item) return item_data @@ -801,21 +826,21 @@ def sum_unique_chunks_metadata(self, metadata_func, item): the metadata needed from the chunk """ chunk_index = self.archive.cache.chunks - chunks = item.get('chunks', []) + chunks = item.get("chunks", []) chunks_counter = Counter(c.id for c in chunks) return sum(metadata_func(c) for c in chunks if chunk_index[c.id].refcount == chunks_counter[c.id]) def calculate_num_chunks(self, item): - return len(item.get('chunks', [])) + return len(item.get("chunks", [])) def calculate_size(self, item): # note: does not support hardlink slaves, they will be size 0 return item.get_size() def hash_item(self, hash_function, item): - if 'chunks' not in item: + if "chunks" not in item: return "" - if hash_function == 'xxh64': + if hash_function == "xxh64": hash = self.xxh64() elif hash_function in self.hash_algorithms: hash = hashlib.new(hash_function) @@ -832,18 +857,18 @@ def format_iso_time(self, key, item): def file_status(mode): if stat.S_ISREG(mode): - return 'A' + return "A" elif stat.S_ISDIR(mode): - return 'd' + return "d" elif stat.S_ISBLK(mode): - return 'b' + return "b" elif stat.S_ISCHR(mode): - return 'c' + return "c" elif stat.S_ISLNK(mode): - return 's' + return "s" elif stat.S_ISFIFO(mode): - return 'f' - return '?' + return "f" + return "?" def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True): @@ -868,7 +893,7 @@ def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comme line = line.rstrip(rstrip) if remove_empty and not line: continue - if remove_comments and line.startswith('#'): + if remove_comments and line.startswith("#"): continue yield line @@ -883,6 +908,7 @@ def swidth_slice(string, max_width): Latin characters are usually one cell wide, many CJK characters are two cells wide. """ from ..platform import swidth + reverse = max_width < 0 max_width = abs(max_width) if reverse: @@ -896,7 +922,7 @@ def swidth_slice(string, max_width): result.append(character) if reverse: result.reverse() - return ''.join(result) + return "".join(result) def ellipsis_truncate(msg, space): @@ -905,15 +931,15 @@ def ellipsis_truncate(msg, space): this_is_a_very_long_string -------> this_is..._string """ from ..platform import swidth - ellipsis_width = swidth('...') + + ellipsis_width = swidth("...") msg_width = swidth(msg) if space < 8: # if there is very little space, just show ... - return '...' + ' ' * (space - ellipsis_width) + return "..." + " " * (space - ellipsis_width) if space < ellipsis_width + msg_width: - return '{}...{}'.format(swidth_slice(msg, space // 2 - ellipsis_width), - swidth_slice(msg, -space // 2)) - return msg + ' ' * (space - msg_width) + return "{}...{}".format(swidth_slice(msg, space // 2 - ellipsis_width), swidth_slice(msg, -space // 2)) + return msg + " " * (space - msg_width) class BorgJsonEncoder(json.JSONEncoder): @@ -922,23 +948,16 @@ def default(self, o): from ..remote import RemoteRepository from ..archive import Archive from ..cache import LocalCache, AdHocCache + if isinstance(o, Repository) or isinstance(o, RemoteRepository): - return { - 'id': bin_to_hex(o.id), - 'location': o._location.canonical_path(), - } + return {"id": bin_to_hex(o.id), "location": o._location.canonical_path()} if isinstance(o, Archive): return o.info() if isinstance(o, LocalCache): - return { - 'path': o.path, - 'stats': o.stats(), - } + return {"path": o.path, "stats": o.stats()} if isinstance(o, AdHocCache): - return { - 'stats': o.stats(), - } - if callable(getattr(o, 'to_json', None)): + return {"stats": o.stats()} + if callable(getattr(o, "to_json", None)): return o.to_json() return super().default(o) @@ -946,17 +965,12 @@ def default(self, o): def basic_json_data(manifest, *, cache=None, extra=None): key = manifest.key data = extra or {} - data.update({ - 'repository': BorgJsonEncoder().default(manifest.repository), - 'encryption': { - 'mode': key.ARG_NAME, - }, - }) - data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc)) - if key.NAME.startswith('key file'): - data['encryption']['keyfile'] = key.find_key() + data.update({"repository": BorgJsonEncoder().default(manifest.repository), "encryption": {"mode": key.ARG_NAME}}) + data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc)) + if key.NAME.startswith("key file"): + data["encryption"]["keyfile"] = key.find_key() if cache: - data['cache'] = cache + data["cache"] = cache return data @@ -975,13 +989,13 @@ def decode_bytes(value): # look nice and chunk ids should mostly show in hex. Use a special # inband signaling character (ASCII DEL) to distinguish between # decoded and hex mode. - if not value.startswith(b'\x7f'): + if not value.startswith(b"\x7f"): try: value = value.decode() return value except UnicodeDecodeError: pass - return '\u007f' + bin_to_hex(value) + return "\u007f" + bin_to_hex(value) def decode_tuple(t): res = [] diff --git a/src/borg/helpers/passphrase.py b/src/borg/helpers/passphrase.py index b53e26ad1..6aa8b4327 100644 --- a/src/borg/helpers/passphrase.py +++ b/src/borg/helpers/passphrase.py @@ -39,7 +39,7 @@ def _env_passphrase(cls, env_var, default=None): @classmethod def env_passphrase(cls, default=None): - passphrase = cls._env_passphrase('BORG_PASSPHRASE', default) + passphrase = cls._env_passphrase("BORG_PASSPHRASE", default) if passphrase is not None: return passphrase passphrase = cls.env_passcommand() @@ -51,7 +51,7 @@ def env_passphrase(cls, default=None): @classmethod def env_passcommand(cls, default=None): - passcommand = os.environ.get('BORG_PASSCOMMAND', None) + passcommand = os.environ.get("BORG_PASSCOMMAND", None) if passcommand is not None: # passcommand is a system command (not inside pyinstaller env) env = prepare_subprocess_env(system=True) @@ -59,21 +59,21 @@ def env_passcommand(cls, default=None): passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True, env=env) except (subprocess.CalledProcessError, FileNotFoundError) as e: raise PasscommandFailure(e) - return cls(passphrase.rstrip('\n')) + return cls(passphrase.rstrip("\n")) @classmethod def fd_passphrase(cls): try: - fd = int(os.environ.get('BORG_PASSPHRASE_FD')) + fd = int(os.environ.get("BORG_PASSPHRASE_FD")) except (ValueError, TypeError): return None - with os.fdopen(fd, mode='r') as f: + with os.fdopen(fd, mode="r") as f: passphrase = f.read() - return cls(passphrase.rstrip('\n')) + return cls(passphrase.rstrip("\n")) @classmethod def env_new_passphrase(cls, default=None): - return cls._env_passphrase('BORG_NEW_PASSPHRASE', default) + return cls._env_passphrase("BORG_NEW_PASSPHRASE", default) @classmethod def getpass(cls, prompt): @@ -83,32 +83,38 @@ def getpass(cls, prompt): if prompt: print() # avoid err msg appearing right of prompt msg = [] - for env_var in 'BORG_PASSPHRASE', 'BORG_PASSCOMMAND': + for env_var in "BORG_PASSPHRASE", "BORG_PASSCOMMAND": env_var_set = os.environ.get(env_var) is not None - msg.append('{} is {}.'.format(env_var, 'set' if env_var_set else 'not set')) - msg.append('Interactive password query failed.') - raise NoPassphraseFailure(' '.join(msg)) from None + msg.append("{} is {}.".format(env_var, "set" if env_var_set else "not set")) + msg.append("Interactive password query failed.") + raise NoPassphraseFailure(" ".join(msg)) from None else: return cls(pw) @classmethod def verification(cls, passphrase): - msg = 'Do you want your passphrase to be displayed for verification? [yN]: ' - if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.', - retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'): - print('Your passphrase (between double-quotes): "%s"' % passphrase, - file=sys.stderr) - print('Make sure the passphrase displayed above is exactly what you wanted.', - file=sys.stderr) + msg = "Do you want your passphrase to be displayed for verification? [yN]: " + if yes( + msg, + retry_msg=msg, + invalid_msg="Invalid answer, try again.", + retry=True, + env_var_override="BORG_DISPLAY_PASSPHRASE", + ): + print('Your passphrase (between double-quotes): "%s"' % passphrase, file=sys.stderr) + print("Make sure the passphrase displayed above is exactly what you wanted.", file=sys.stderr) try: - passphrase.encode('ascii') + passphrase.encode("ascii") except UnicodeEncodeError: - print('Your passphrase (UTF-8 encoding in hex): %s' % - bin_to_hex(passphrase.encode('utf-8')), - file=sys.stderr) - print('As you have a non-ASCII passphrase, it is recommended to keep the ' - 'UTF-8 encoding in hex together with the passphrase at a safe place.', - file=sys.stderr) + print( + "Your passphrase (UTF-8 encoding in hex): %s" % bin_to_hex(passphrase.encode("utf-8")), + file=sys.stderr, + ) + print( + "As you have a non-ASCII passphrase, it is recommended to keep the " + "UTF-8 encoding in hex together with the passphrase at a safe place.", + file=sys.stderr, + ) @classmethod def new(cls, allow_empty=False): @@ -119,17 +125,17 @@ def new(cls, allow_empty=False): if passphrase is not None: return passphrase for retry in range(1, 11): - passphrase = cls.getpass('Enter new passphrase: ') + passphrase = cls.getpass("Enter new passphrase: ") if allow_empty or passphrase: - passphrase2 = cls.getpass('Enter same passphrase again: ') + passphrase2 = cls.getpass("Enter same passphrase again: ") if passphrase == passphrase2: cls.verification(passphrase) - logger.info('Remember your passphrase. Your data will be inaccessible without it.') + logger.info("Remember your passphrase. Your data will be inaccessible without it.") return passphrase else: - print('Passphrases do not match', file=sys.stderr) + print("Passphrases do not match", file=sys.stderr) else: - print('Passphrase must not be blank', file=sys.stderr) + print("Passphrase must not be blank", file=sys.stderr) else: raise PasswordRetriesExceeded diff --git a/src/borg/helpers/process.py b/src/borg/helpers/process.py index 0cedcb7b7..41aa38815 100644 --- a/src/borg/helpers/process.py +++ b/src/borg/helpers/process.py @@ -13,6 +13,7 @@ from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin from ..logger import create_logger + logger = create_logger() from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error @@ -21,6 +22,7 @@ @contextlib.contextmanager def _daemonize(): from ..platform import get_process_id + old_id = get_process_id() pid = os.fork() if pid: @@ -30,13 +32,13 @@ def _daemonize(): except _ExitCodeException as e: exit_code = e.exit_code finally: - logger.debug('Daemonizing: Foreground process (%s, %s, %s) is now dying.' % old_id) + logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id) os._exit(exit_code) os.setsid() pid = os.fork() if pid: os._exit(0) - os.chdir('/') + os.chdir("/") os.close(0) os.close(1) fd = os.open(os.devnull, os.O_RDWR) @@ -78,12 +80,12 @@ def daemonizing(*, timeout=5): with _daemonize() as (old_id, new_id): if new_id is None: # The original / parent process, waiting for a signal to die. - logger.debug('Daemonizing: Foreground process (%s, %s, %s) is waiting for background process...' % old_id) + logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id) exit_code = EXIT_SUCCESS # Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case... - with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \ - signal_handler('SIGHUP', raising_signal_handler(SigHup)), \ - signal_handler('SIGTERM', raising_signal_handler(SigTerm)): + with signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), signal_handler( + "SIGHUP", raising_signal_handler(SigHup) + ), signal_handler("SIGTERM", raising_signal_handler(SigTerm)): try: if timeout > 0: time.sleep(timeout) @@ -96,15 +98,17 @@ def daemonizing(*, timeout=5): exit_code = EXIT_WARNING except KeyboardInterrupt: # Manual termination. - logger.debug('Daemonizing: Foreground process (%s, %s, %s) received SIGINT.' % old_id) + logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id) exit_code = EXIT_SIGNAL_BASE + 2 except BaseException as e: # Just in case... - logger.warning('Daemonizing: Foreground process received an exception while waiting:\n' + - ''.join(traceback.format_exception(e.__class__, e, e.__traceback__))) + logger.warning( + "Daemonizing: Foreground process received an exception while waiting:\n" + + "".join(traceback.format_exception(e.__class__, e, e.__traceback__)) + ) exit_code = EXIT_WARNING else: - logger.warning('Daemonizing: Background process did not respond (timeout). Is it alive?') + logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?") exit_code = EXIT_WARNING finally: # Don't call with-body, but die immediately! @@ -113,22 +117,26 @@ def daemonizing(*, timeout=5): # The background / grandchild process. sig_to_foreground = signal.SIGTERM - logger.debug('Daemonizing: Background process (%s, %s, %s) is starting...' % new_id) + logger.debug("Daemonizing: Background process (%s, %s, %s) is starting..." % new_id) try: yield old_id, new_id except BaseException as e: sig_to_foreground = signal.SIGHUP - logger.warning('Daemonizing: Background process raised an exception while starting:\n' + - ''.join(traceback.format_exception(e.__class__, e, e.__traceback__))) + logger.warning( + "Daemonizing: Background process raised an exception while starting:\n" + + "".join(traceback.format_exception(e.__class__, e, e.__traceback__)) + ) raise e else: - logger.debug('Daemonizing: Background process (%s, %s, %s) has started.' % new_id) + logger.debug("Daemonizing: Background process (%s, %s, %s) has started." % new_id) finally: try: os.kill(old_id[1], sig_to_foreground) except BaseException as e: - logger.error('Daemonizing: Trying to kill the foreground process raised an exception:\n' + - ''.join(traceback.format_exception(e.__class__, e, e.__traceback__))) + logger.error( + "Daemonizing: Trying to kill the foreground process raised an exception:\n" + + "".join(traceback.format_exception(e.__class__, e, e.__traceback__)) + ) class _ExitCodeException(BaseException): @@ -187,7 +195,7 @@ def __init__(self): self._sig_int_triggered = False self._action_triggered = False self._action_done = False - self.ctx = signal_handler('SIGINT', self.handler) + self.ctx = signal_handler("SIGINT", self.handler) def __bool__(self): # this will be True (and stay True) after the first Ctrl-C/SIGINT @@ -228,7 +236,7 @@ def __exit__(self, exception_type, exception_value, traceback): sig_int = SigIntManager() -def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs): +def popen_with_error_handling(cmd_line: str, log_prefix="", **kwargs): """ Handle typical errors raised by subprocess.Popen. Return None if an error occurred, otherwise return the Popen object. @@ -240,27 +248,27 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs): Does not change the exit code. """ - assert not kwargs.get('shell'), 'Sorry pal, shell mode is a no-no' + assert not kwargs.get("shell"), "Sorry pal, shell mode is a no-no" try: command = shlex.split(cmd_line) if not command: - raise ValueError('an empty command line is not permitted') + raise ValueError("an empty command line is not permitted") except ValueError as ve: - logger.error('%s%s', log_prefix, ve) + logger.error("%s%s", log_prefix, ve) return - logger.debug('%scommand line: %s', log_prefix, command) + logger.debug("%scommand line: %s", log_prefix, command) try: return subprocess.Popen(command, **kwargs) except FileNotFoundError: - logger.error('%sexecutable not found: %s', log_prefix, command[0]) + logger.error("%sexecutable not found: %s", log_prefix, command[0]) return except PermissionError: - logger.error('%spermission denied: %s', log_prefix, command[0]) + logger.error("%spermission denied: %s", log_prefix, command[0]) return def is_terminal(fd=sys.stdout): - return hasattr(fd, 'isatty') and fd.isatty() and (not is_win32 or 'ANSICON' in os.environ) + return hasattr(fd, "isatty") and fd.isatty() and (not is_win32 or "ANSICON" in os.environ) def prepare_subprocess_env(system, env=None): @@ -278,8 +286,8 @@ def prepare_subprocess_env(system, env=None): # but we do not want that system binaries (like ssh or other) pick up # (non-matching) libraries from there. # thus we install the original LDLP, before pyinstaller has modified it: - lp_key = 'LD_LIBRARY_PATH' - lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 / v3.2.1 has this + lp_key = "LD_LIBRARY_PATH" + lp_orig = env.get(lp_key + "_ORIG") # pyinstaller >= 20160820 / v3.2.1 has this if lp_orig is not None: env[lp_key] = lp_orig else: @@ -292,12 +300,12 @@ def prepare_subprocess_env(system, env=None): # in this case, we must kill LDLP. # We can recognize this via sys.frozen and sys._MEIPASS being set. lp = env.get(lp_key) - if lp is not None and getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): + if lp is not None and getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): env.pop(lp_key) # security: do not give secrets to subprocess - env.pop('BORG_PASSPHRASE', None) + env.pop("BORG_PASSPHRASE", None) # for information, give borg version to the subprocess - env['BORG_VERSION'] = __version__ + env["BORG_VERSION"] = __version__ return env @@ -314,13 +322,15 @@ def create_filter_process(cmd, stream, stream_close, inbound=True): # communication with the process is a one-way road, i.e. the process can never block # for us to do something while we block on the process for something different. if inbound: - proc = popen_with_error_handling(cmd, stdout=subprocess.PIPE, stdin=filter_stream, - log_prefix='filter-process: ', env=env) + proc = popen_with_error_handling( + cmd, stdout=subprocess.PIPE, stdin=filter_stream, log_prefix="filter-process: ", env=env + ) else: - proc = popen_with_error_handling(cmd, stdin=subprocess.PIPE, stdout=filter_stream, - log_prefix='filter-process: ', env=env) + proc = popen_with_error_handling( + cmd, stdin=subprocess.PIPE, stdout=filter_stream, log_prefix="filter-process: ", env=env + ) if not proc: - raise Error(f'filter {cmd}: process creation failed') + raise Error(f"filter {cmd}: process creation failed") stream = proc.stdout if inbound else proc.stdin # inbound: do not close the pipe (this is the task of the filter process [== writer]) # outbound: close the pipe, otherwise the filter process would not notice when we are done. @@ -331,7 +341,7 @@ def create_filter_process(cmd, stream, stream_close, inbound=True): except Exception: # something went wrong with processing the stream by borg - logger.debug('Exception, killing the filter...') + logger.debug("Exception, killing the filter...") if cmd: proc.kill() borg_succeeded = False @@ -343,11 +353,11 @@ def create_filter_process(cmd, stream, stream_close, inbound=True): stream.close() if cmd: - logger.debug('Done, waiting for filter to die...') + logger.debug("Done, waiting for filter to die...") rc = proc.wait() - logger.debug('filter cmd exited with code %d', rc) + logger.debug("filter cmd exited with code %d", rc) if filter_stream_close: filter_stream.close() if borg_succeeded and rc: # if borg did not succeed, we know that we killed the filter process - raise Error('filter %s failed, rc=%d' % (cmd, rc)) + raise Error("filter %s failed, rc=%d" % (cmd, rc)) diff --git a/src/borg/helpers/progress.py b/src/borg/helpers/progress.py index 312ff2585..11c45c80b 100644 --- a/src/borg/helpers/progress.py +++ b/src/borg/helpers/progress.py @@ -5,6 +5,7 @@ from shutil import get_terminal_size from ..logger import create_logger + logger = create_logger() from .parseformat import ellipsis_truncate @@ -19,7 +20,7 @@ def justify_to_terminal_size(message): class ProgressIndicatorBase: - LOGGER = 'borg.output.progress' + LOGGER = "borg.output.progress" JSON_TYPE = None json = False @@ -43,15 +44,15 @@ def __init__(self, msgid=None): if not self.logger.handlers: self.handler = logging.StreamHandler(stream=sys.stderr) self.handler.setLevel(logging.INFO) - logger = logging.getLogger('borg') + logger = logging.getLogger("borg") # Some special attributes on the borg logger, created by setup_logging # But also be able to work without that try: formatter = logger.formatter - terminator = '\n' if logger.json else '\r' + terminator = "\n" if logger.json else "\r" self.json = logger.json except AttributeError: - terminator = '\r' + terminator = "\r" else: self.handler.setFormatter(formatter) self.handler.terminator = terminator @@ -79,24 +80,20 @@ def output_json(self, *, finished=False, **kwargs): assert self.json if not self.emit: return - kwargs.update(dict( - operation=self.id, - msgid=self.msgid, - type=self.JSON_TYPE, - finished=finished, - time=time.time(), - )) + kwargs.update( + dict(operation=self.id, msgid=self.msgid, type=self.JSON_TYPE, finished=finished, time=time.time()) + ) print(json.dumps(kwargs), file=sys.stderr, flush=True) def finish(self): if self.json: self.output_json(finished=True) else: - self.output('') + self.output("") class ProgressIndicatorMessage(ProgressIndicatorBase): - JSON_TYPE = 'progress_message' + JSON_TYPE = "progress_message" def output(self, msg): if self.json: @@ -106,7 +103,7 @@ def output(self, msg): class ProgressIndicatorPercent(ProgressIndicatorBase): - JSON_TYPE = 'progress_percent' + JSON_TYPE = "progress_percent" def __init__(self, total=0, step=5, start=0, msg="%3.0f%%", msgid=None): """ @@ -150,7 +147,7 @@ def show(self, current=None, increase=1, info=None): # no need to truncate if we're not outputting to a terminal terminal_space = get_terminal_size(fallback=(-1, -1))[0] if terminal_space != -1: - space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [''])) + space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [""])) info[-1] = ellipsis_truncate(info[-1], space) return self.output(self.msg % tuple([pct] + info), justify=False, info=info) @@ -193,7 +190,7 @@ def show(self): return self.output(self.triggered) def output(self, triggered): - print('.', end='', file=self.file, flush=True) + print(".", end="", file=self.file, flush=True) def finish(self): print(file=self.file) diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py index 8b48076f7..a7170764d 100644 --- a/src/borg/helpers/time.py +++ b/src/borg/helpers/time.py @@ -12,7 +12,7 @@ def to_localtime(ts): def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string""" - fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS + fmt = ISO_FORMAT if "." in timestamp else ISO_FORMAT_NO_USECS dt = datetime.strptime(timestamp, fmt) if tzinfo is not None: dt = dt.replace(tzinfo=tzinfo) @@ -27,11 +27,16 @@ def timestamp(s): return datetime.fromtimestamp(ts, tz=timezone.utc) except OSError: # didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support. - for format in ('%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S+00:00', - '%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S', - '%Y-%m-%dT%H:%M', '%Y-%m-%d %H:%M', - '%Y-%m-%d', '%Y-%j', - ): + for format in ( + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S+00:00", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M", + "%Y-%m-%d %H:%M", + "%Y-%m-%d", + "%Y-%j", + ): try: return datetime.strptime(s, format).replace(tzinfo=timezone.utc) except ValueError: @@ -54,7 +59,7 @@ def timestamp(s): # subtract last 48h to avoid any issues that could be caused by tz calculations. # this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit). # msgpack can pack up to uint64. - MAX_S = 2**31-1 - 48*3600 + MAX_S = 2**31 - 1 - 48 * 3600 MAX_NS = MAX_S * 1000000000 else: # nanosecond timestamps will fit into a signed int64. @@ -62,7 +67,7 @@ def timestamp(s): # this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit). # round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S. # msgpack can pack up to uint64. - MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000 + MAX_NS = (2**63 - 1 - 48 * 3600 * 1000000000) // 1000000000 * 1000000000 MAX_S = MAX_NS // 1000000000 @@ -89,11 +94,11 @@ def safe_timestamp(item_timestamp_ns): return datetime.fromtimestamp(t_ns / 1e9) -def format_time(ts: datetime, format_spec=''): +def format_time(ts: datetime, format_spec=""): """ Convert *ts* to a human-friendly format with textual weekday. """ - return ts.strftime('%a, %Y-%m-%d %H:%M:%S' if format_spec == '' else format_spec) + return ts.strftime("%a, %Y-%m-%d %H:%M:%S" if format_spec == "" else format_spec) def isoformat_time(ts: datetime): @@ -105,19 +110,18 @@ def isoformat_time(ts: datetime): def format_timedelta(td): - """Format timedelta in a human friendly format - """ + """Format timedelta in a human friendly format""" ts = td.total_seconds() s = ts % 60 m = int(ts / 60) % 60 h = int(ts / 3600) % 24 - txt = '%.2f seconds' % s + txt = "%.2f seconds" % s if m: - txt = '%d minutes %s' % (m, txt) + txt = "%d minutes %s" % (m, txt) if h: - txt = '%d hours %s' % (h, txt) + txt = "%d hours %s" % (h, txt) if td.days: - txt = '%d days %s' % (td.days, txt) + txt = "%d days %s" % (td.days, txt) return txt @@ -131,7 +135,7 @@ def __format__(self, format_spec): return format_time(self.ts, format_spec=format_spec) def __str__(self): - return f'{self}' + return f"{self}" def isoformat(self): return isoformat_time(self.ts) diff --git a/src/borg/helpers/yes.py b/src/borg/helpers/yes.py index 042abcfc8..745b06a51 100644 --- a/src/borg/helpers/yes.py +++ b/src/borg/helpers/yes.py @@ -4,16 +4,30 @@ import os.path import sys -FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', ) -TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', ) -DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', ) +FALSISH = ("No", "NO", "no", "N", "n", "0") +TRUISH = ("Yes", "YES", "yes", "Y", "y", "1") +DEFAULTISH = ("Default", "DEFAULT", "default", "D", "d", "") -def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, - retry_msg=None, invalid_msg=None, env_msg='{} (from {})', - falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH, - default=False, retry=True, env_var_override=None, ofile=None, input=input, prompt=True, - msgid=None): +def yes( + msg=None, + false_msg=None, + true_msg=None, + default_msg=None, + retry_msg=None, + invalid_msg=None, + env_msg="{} (from {})", + falsish=FALSISH, + truish=TRUISH, + defaultish=DEFAULTISH, + default=False, + retry=True, + env_var_override=None, + ofile=None, + input=input, + prompt=True, + msgid=None, +): """Output (usually a question) and let user input an answer. Qualifies the answer according to falsish, truish and defaultish as True, False or . If it didn't qualify and retry is False (no retries wanted), return the default [which @@ -43,18 +57,15 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, :param input: input function [input from builtins] :return: boolean answer value, True or False """ + def output(msg, msg_type, is_prompt=False, **kwargs): - json_output = getattr(logging.getLogger('borg'), 'json', False) + json_output = getattr(logging.getLogger("borg"), "json", False) if json_output: - kwargs.update(dict( - type='question_%s' % msg_type, - msgid=msgid, - message=msg, - )) + kwargs.update(dict(type="question_%s" % msg_type, msgid=msgid, message=msg)) print(json.dumps(kwargs), file=sys.stderr) else: if is_prompt: - print(msg, file=ofile, end='', flush=True) + print(msg, file=ofile, end="", flush=True) else: print(msg, file=ofile) @@ -66,13 +77,13 @@ def output(msg, msg_type, is_prompt=False, **kwargs): if default not in (True, False): raise ValueError("invalid default value, must be True or False") if msg: - output(msg, 'prompt', is_prompt=True) + output(msg, "prompt", is_prompt=True) while True: answer = None if env_var_override: answer = os.environ.get(env_var_override) if answer is not None and env_msg: - output(env_msg.format(answer, env_var_override), 'env_answer', env_var=env_var_override) + output(env_msg.format(answer, env_var_override), "env_answer", env_var=env_var_override) if answer is None: if not prompt: return default @@ -83,22 +94,22 @@ def output(msg, msg_type, is_prompt=False, **kwargs): answer = truish[0] if default else falsish[0] if answer in defaultish: if default_msg: - output(default_msg, 'accepted_default') + output(default_msg, "accepted_default") return default if answer in truish: if true_msg: - output(true_msg, 'accepted_true') + output(true_msg, "accepted_true") return True if answer in falsish: if false_msg: - output(false_msg, 'accepted_false') + output(false_msg, "accepted_false") return False # if we get here, the answer was invalid if invalid_msg: - output(invalid_msg, 'invalid_answer') + output(invalid_msg, "invalid_answer") if not retry: return default if retry_msg: - output(retry_msg, 'prompt_retry', is_prompt=True) + output(retry_msg, "prompt_retry", is_prompt=True) # in case we used an environment variable and it gave an invalid answer, do not use it again: env_var_override = None diff --git a/src/borg/locking.py b/src/borg/locking.py index ecd6d27ca..e7f656915 100644 --- a/src/borg/locking.py +++ b/src/borg/locking.py @@ -8,8 +8,8 @@ from .helpers import Error, ErrorWithTraceback from .logger import create_logger -ADD, REMOVE = 'add', 'remove' -SHARED, EXCLUSIVE = 'shared', 'exclusive' +ADD, REMOVE = "add", "remove" +SHARED, EXCLUSIVE = "shared", "exclusive" logger = create_logger(__name__) @@ -20,6 +20,7 @@ class TimeoutTimer: It can also compute and optionally execute a reasonable sleep time (e.g. to avoid polling too often or to support thread/process rescheduling). """ + def __init__(self, timeout=None, sleep=None): """ Initialize a timer. @@ -43,8 +44,8 @@ def __init__(self, timeout=None, sleep=None): def __repr__(self): return "<{}: start={!r} end={!r} timeout={!r} sleep={!r}>".format( - self.__class__.__name__, self.start_time, self.end_time, - self.timeout_interval, self.sleep_interval) + self.__class__.__name__, self.start_time, self.end_time, self.timeout_interval, self.sleep_interval + ) def start(self): self.start_time = time.time() @@ -102,6 +103,7 @@ class ExclusiveLock: This makes sure the lock is released again if the block is left, no matter how (e.g. if an exception occurred). """ + def __init__(self, path, timeout=None, sleep=None, id=None): self.timeout = timeout self.sleep = sleep @@ -129,7 +131,7 @@ def acquire(self, timeout=None, sleep=None): unique_base_name = os.path.basename(self.unique_name) temp_path = None try: - temp_path = tempfile.mkdtemp(".tmp", base_name + '.', parent_path) + temp_path = tempfile.mkdtemp(".tmp", base_name + ".", parent_path) temp_unique_name = os.path.join(temp_path, unique_base_name) with open(temp_unique_name, "wb"): pass @@ -192,8 +194,8 @@ def kill_stale_lock(self): else: for name in names: try: - host_pid, thread_str = name.rsplit('-', 1) - host, pid_str = host_pid.rsplit('.', 1) + host_pid, thread_str = name.rsplit("-", 1) + host, pid_str = host_pid.rsplit(".", 1) pid = int(pid_str) thread = int(thread_str) except ValueError: @@ -207,17 +209,19 @@ def kill_stale_lock(self): if not self.kill_stale_locks: if not self.stale_warning_printed: # Log this at warning level to hint the user at the ability - logger.warning("Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name) + logger.warning( + "Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name + ) self.stale_warning_printed = True return False try: os.unlink(os.path.join(self.path, name)) - logger.warning('Killed stale lock %s.', name) + logger.warning("Killed stale lock %s.", name) except OSError as err: if not self.stale_warning_printed: # This error will bubble up and likely result in locking failure - logger.error('Found stale lock %s, but cannot delete due to %s', name, str(err)) + logger.error("Found stale lock %s, but cannot delete due to %s", name, str(err)) self.stale_warning_printed = True return False @@ -228,7 +232,7 @@ def kill_stale_lock(self): # Directory is not empty or doesn't exist any more = we lost the race to somebody else--which is ok. return False # EACCES or EIO or ... = we cannot operate anyway - logger.error('Failed to remove lock dir: %s', str(err)) + logger.error("Failed to remove lock dir: %s", str(err)) return False return True @@ -257,6 +261,7 @@ class LockRoster: Note: you usually should call the methods with an exclusive lock held, to avoid conflicting access by multiple threads/processes/machines. """ + def __init__(self, path, id=None): self.path = path self.id = id or platform.get_process_id() @@ -279,8 +284,9 @@ def load(self): if platform.process_alive(host, pid, thread): elements.add((host, pid, thread)) else: - logger.warning('Removed stale %s roster lock for host %s pid %d thread %d.', - key, host, pid, thread) + logger.warning( + "Removed stale %s roster lock for host %s pid %d thread %d.", key, host, pid, thread + ) data[key] = list(elements) except (FileNotFoundError, ValueError): # no or corrupt/empty roster file? @@ -315,7 +321,7 @@ def modify(self, key, op): elif op == REMOVE: elements.remove(self.id) else: - raise ValueError('Unknown LockRoster op %r' % op) + raise ValueError("Unknown LockRoster op %r" % op) roster[key] = list(list(e) for e in elements) self.save(roster) @@ -354,6 +360,7 @@ class Lock: This makes sure the lock is released again if the block is left, no matter how (e.g. if an exception occurred). """ + def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None): self.path = path self.is_exclusive = exclusive @@ -361,11 +368,11 @@ def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None): self.timeout = timeout self.id = id or platform.get_process_id() # globally keeping track of shared and exclusive lockers: - self._roster = LockRoster(path + '.roster', id=id) + self._roster = LockRoster(path + ".roster", id=id) # an exclusive lock, used for: # - holding while doing roster queries / updates # - holding while the Lock itself is exclusive - self._lock = ExclusiveLock(path + '.exclusive', id=id, timeout=timeout) + self._lock = ExclusiveLock(path + ".exclusive", id=id, timeout=timeout) def __enter__(self): return self.acquire() diff --git a/src/borg/logger.py b/src/borg/logger.py index b24eb7711..492b90a27 100644 --- a/src/borg/logger.py +++ b/src/borg/logger.py @@ -53,7 +53,7 @@ def _log_warning(message, category, filename, lineno, file=None, line=None): logger.warning(msg) -def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info', is_serve=False, json=False): +def setup_logging(stream=None, conf_fname=None, env_var="BORG_LOGGING_CONF", level="info", is_serve=False, json=False): """setup logging module according to the arguments provided if conf_fname is given (or the config file name can be determined via @@ -80,7 +80,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev logging.config.fileConfig(f) configured = True logger = logging.getLogger(__name__) - borg_logger = logging.getLogger('borg') + borg_logger = logging.getLogger("borg") borg_logger.json = json logger.debug(f'using logging configuration read from "{conf_fname}"') warnings.showwarning = _log_warning @@ -88,15 +88,15 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev except Exception as err: # XXX be more precise err_msg = str(err) # if we did not / not successfully load a logging configuration, fallback to this: - logger = logging.getLogger('') + logger = logging.getLogger("") handler = logging.StreamHandler(stream) if is_serve and not json: - fmt = '$LOG %(levelname)s %(name)s Remote: %(message)s' + fmt = "$LOG %(levelname)s %(name)s Remote: %(message)s" else: - fmt = '%(message)s' + fmt = "%(message)s" formatter = JsonFormatter(fmt) if json else logging.Formatter(fmt) handler.setFormatter(formatter) - borg_logger = logging.getLogger('borg') + borg_logger = logging.getLogger("borg") borg_logger.formatter = formatter borg_logger.json = json if configured and logger.handlers: @@ -111,7 +111,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev logger = logging.getLogger(__name__) if err_msg: logger.warning(f'setup_logging for "{conf_fname}" failed with "{err_msg}".') - logger.debug('using builtin fallback logging configuration') + logger.debug("using builtin fallback logging configuration") warnings.showwarning = _log_warning return handler @@ -151,6 +151,7 @@ def create_logger(name=None): be careful not to call any logger methods before the setup_logging() call. If you try, you'll get an exception. """ + class LazyLogger: def __init__(self, name=None): self.__name = name or find_parent_module() @@ -162,49 +163,49 @@ def __logger(self): if not configured: raise Exception("tried to call a logger before setup_logging() was called") self.__real_logger = logging.getLogger(self.__name) - if self.__name.startswith('borg.debug.') and self.__real_logger.level == logging.NOTSET: - self.__real_logger.setLevel('WARNING') + if self.__name.startswith("borg.debug.") and self.__real_logger.level == logging.NOTSET: + self.__real_logger.setLevel("WARNING") return self.__real_logger def getChild(self, suffix): - return LazyLogger(self.__name + '.' + suffix) + return LazyLogger(self.__name + "." + suffix) def setLevel(self, *args, **kw): return self.__logger.setLevel(*args, **kw) def log(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.log(*args, **kw) def exception(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.exception(*args, **kw) def debug(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.debug(*args, **kw) def info(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.info(*args, **kw) def warning(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.warning(*args, **kw) def error(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.error(*args, **kw) def critical(self, *args, **kw): - if 'msgid' in kw: - kw.setdefault('extra', {})['msgid'] = kw.pop('msgid') + if "msgid" in kw: + kw.setdefault("extra", {})["msgid"] = kw.pop("msgid") return self.__logger.critical(*args, **kw) return LazyLogger(name) @@ -212,11 +213,11 @@ def critical(self, *args, **kw): class JsonFormatter(logging.Formatter): RECORD_ATTRIBUTES = ( - 'levelname', - 'name', - 'message', + "levelname", + "name", + "message", # msgid is an attribute we made up in Borg to expose a non-changing handle for log messages - 'msgid', + "msgid", ) # Other attributes that are not very useful but do exist: @@ -229,12 +230,7 @@ class JsonFormatter(logging.Formatter): def format(self, record): super().format(record) - data = { - 'type': 'log_message', - 'time': record.created, - 'message': '', - 'levelname': 'CRITICAL', - } + data = {"type": "log_message", "time": record.created, "message": "", "levelname": "CRITICAL"} for attr in self.RECORD_ATTRIBUTES: value = getattr(record, attr, None) if value: diff --git a/src/borg/lrucache.py b/src/borg/lrucache.py index 4f7f1f829..097fe4c52 100644 --- a/src/borg/lrucache.py +++ b/src/borg/lrucache.py @@ -10,8 +10,8 @@ def __init__(self, capacity, dispose): def __setitem__(self, key, value): assert key not in self._cache, ( - "Unexpected attempt to replace a cached item," - " without first deleting the old item.") + "Unexpected attempt to replace a cached item," " without first deleting the old item." + ) self._lru.append(key) while len(self._lru) > self._capacity: del self[self._lru[0]] diff --git a/src/borg/nanorst.py b/src/borg/nanorst.py index 5700a0b3c..71d9e9470 100644 --- a/src/borg/nanorst.py +++ b/src/borg/nanorst.py @@ -11,18 +11,18 @@ def __init__(self, s): def read(self, n): self.i += n - return self.str[self.i - n:self.i] + return self.str[self.i - n : self.i] def peek(self, n): if n >= 0: - return self.str[self.i:self.i + n] + return self.str[self.i : self.i + n] else: - return self.str[self.i + n - 1:self.i - 1] + return self.str[self.i + n - 1 : self.i - 1] def peekline(self): - out = '' + out = "" i = self.i - while i < len(self.str) and self.str[i] != '\n': + while i < len(self.str) and self.str[i] != "\n": out += self.str[i] i += 1 return out @@ -34,18 +34,18 @@ def readline(self): def process_directive(directive, arguments, out, state_hook): - if directive == 'container' and arguments == 'experimental': - state_hook('text', '**', out) - out.write('++ Experimental ++') - state_hook('**', 'text', out) + if directive == "container" and arguments == "experimental": + state_hook("text", "**", out) + out.write("++ Experimental ++") + state_hook("**", "text", out) else: - state_hook('text', '**', out) + state_hook("text", "**", out) out.write(directive.title()) - out.write(':\n') - state_hook('**', 'text', out) + out.write(":\n") + state_hook("**", "text", out) if arguments: out.write(arguments) - out.write('\n') + out.write("\n") def rst_to_text(text, state_hook=None, references=None): @@ -58,12 +58,12 @@ def rst_to_text(text, state_hook=None, references=None): """ state_hook = state_hook or (lambda old_state, new_state, out: None) references = references or {} - state = 'text' - inline_mode = 'replace' + state = "text" + inline_mode = "replace" text = TextPecker(text) out = io.StringIO() - inline_single = ('*', '`') + inline_single = ("*", "`") while True: char = text.read(1) @@ -71,81 +71,83 @@ def rst_to_text(text, state_hook=None, references=None): break next = text.peek(1) # type: str - if state == 'text': - if char == '\\' and text.peek(1) in inline_single: + if state == "text": + if char == "\\" and text.peek(1) in inline_single: continue - if text.peek(-1) != '\\': + if text.peek(-1) != "\\": if char in inline_single and next != char: state_hook(state, char, out) state = char continue - if char == next == '*': - state_hook(state, '**', out) - state = '**' + if char == next == "*": + state_hook(state, "**", out) + state = "**" text.read(1) continue - if char == next == '`': - state_hook(state, '``', out) - state = '``' + if char == next == "`": + state_hook(state, "``", out) + state = "``" text.read(1) continue - if text.peek(-1).isspace() and char == ':' and text.peek(5) == 'ref:`': + if text.peek(-1).isspace() and char == ":" and text.peek(5) == "ref:`": # translate reference text.read(5) - ref = '' + ref = "" while True: char = text.peek(1) - if char == '`': + if char == "`": text.read(1) break - if char == '\n': + if char == "\n": text.read(1) continue # merge line breaks in :ref:`...\n...` ref += text.read(1) try: out.write(references[ref]) except KeyError: - raise ValueError("Undefined reference in Archiver help: %r — please add reference " - "substitution to 'rst_plain_text_references'" % ref) + raise ValueError( + "Undefined reference in Archiver help: %r — please add reference " + "substitution to 'rst_plain_text_references'" % ref + ) continue - if char == ':' and text.peek(2) == ':\n': # End of line code block + if char == ":" and text.peek(2) == ":\n": # End of line code block text.read(2) - state_hook(state, 'code-block', out) - state = 'code-block' - out.write(':\n') + state_hook(state, "code-block", out) + state = "code-block" + out.write(":\n") continue - if text.peek(-2) in ('\n\n', '') and char == next == '.': + if text.peek(-2) in ("\n\n", "") and char == next == ".": text.read(2) - directive, is_directive, arguments = text.readline().partition('::') + directive, is_directive, arguments = text.readline().partition("::") text.read(1) if not is_directive: # partition: if the separator is not in the text, the leftmost output is the entire input - if directive == 'nanorst: inline-fill': - inline_mode = 'fill' - elif directive == 'nanorst: inline-replace': - inline_mode = 'replace' + if directive == "nanorst: inline-fill": + inline_mode = "fill" + elif directive == "nanorst: inline-replace": + inline_mode = "replace" continue process_directive(directive, arguments.strip(), out, state_hook) continue if state in inline_single and char == state: - state_hook(state, 'text', out) - state = 'text' - if inline_mode == 'fill': - out.write(2 * ' ') + state_hook(state, "text", out) + state = "text" + if inline_mode == "fill": + out.write(2 * " ") continue - if state == '``' and char == next == '`': - state_hook(state, 'text', out) - state = 'text' + if state == "``" and char == next == "`": + state_hook(state, "text", out) + state = "text" text.read(1) - if inline_mode == 'fill': - out.write(4 * ' ') + if inline_mode == "fill": + out.write(4 * " ") continue - if state == '**' and char == next == '*': - state_hook(state, 'text', out) - state = 'text' + if state == "**" and char == next == "*": + state_hook(state, "text", out) + state = "text" text.read(1) continue - if state == 'code-block' and char == next == '\n' and text.peek(5)[1:] != ' ': + if state == "code-block" and char == next == "\n" and text.peek(5)[1:] != " ": # Foo:: # # *stuff* *code* *ignore .. all markup* @@ -153,11 +155,11 @@ def rst_to_text(text, state_hook=None, references=None): # More arcane stuff # # Regular text... - state_hook(state, 'text', out) - state = 'text' + state_hook(state, "text", out) + state = "text" out.write(char) - assert state == 'text', 'Invalid final state %r (This usually indicates unmatched */**)' % state + assert state == "text", "Invalid final state %r (This usually indicates unmatched */**)" % state return out.getvalue() @@ -191,12 +193,12 @@ def __contains__(self, item): def ansi_escapes(old_state, new_state, out): - if old_state == 'text' and new_state in ('*', '`', '``'): - out.write('\033[4m') - if old_state == 'text' and new_state == '**': - out.write('\033[1m') - if old_state in ('*', '`', '``', '**') and new_state == 'text': - out.write('\033[0m') + if old_state == "text" and new_state in ("*", "`", "``"): + out.write("\033[4m") + if old_state == "text" and new_state == "**": + out.write("\033[1m") + if old_state in ("*", "`", "``", "**") and new_state == "text": + out.write("\033[0m") def rst_to_terminal(rst, references=None, destination=sys.stdout): diff --git a/src/borg/patterns.py b/src/borg/patterns.py index 49668ef2b..749430190 100644 --- a/src/borg/patterns.py +++ b/src/borg/patterns.py @@ -75,6 +75,7 @@ class PatternMatcher: *fallback* is a boolean value that *match()* returns if no matching patterns are found. """ + def __init__(self, fallback=None): self._items = [] @@ -96,18 +97,13 @@ def __init__(self, fallback=None): self.include_patterns = [] # TODO: move this info to parse_inclexcl_command and store in PatternBase subclass? - self.is_include_cmd = { - IECommand.Exclude: False, - IECommand.ExcludeNoRecurse: False, - IECommand.Include: True - } + self.is_include_cmd = {IECommand.Exclude: False, IECommand.ExcludeNoRecurse: False, IECommand.Include: True} def empty(self): return not len(self._items) and not len(self._path_full_patterns) def _add(self, pattern, cmd): - """*cmd* is an IECommand value. - """ + """*cmd* is an IECommand value.""" if isinstance(pattern, PathFullPattern): key = pattern.pattern # full, normalized path self._path_full_patterns[key] = cmd @@ -123,8 +119,7 @@ def add(self, patterns, cmd): self._add(pattern, cmd) def add_includepaths(self, include_paths): - """Used to add inclusion-paths from args.paths (from commandline). - """ + """Used to add inclusion-paths from args.paths (from commandline).""" include_patterns = [parse_pattern(p, PathPrefixPattern) for p in include_paths] self.add(include_patterns, IECommand.Include) self.fallback = not include_patterns @@ -135,8 +130,7 @@ def get_unmatched_include_patterns(self): return [p for p in self.include_patterns if p.match_count == 0] def add_inclexcl(self, patterns): - """Add list of patterns (of type CmdTuple) to internal list. - """ + """Add list of patterns (of type CmdTuple) to internal list.""" for pattern, cmd in patterns: self._add(pattern, cmd) @@ -172,12 +166,12 @@ def normalize_path(path): """normalize paths for MacOS (but do nothing on other platforms)""" # HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match. # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. - return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path + return unicodedata.normalize("NFD", path) if sys.platform == "darwin" else path class PatternBase: - """Shared logic for inclusion/exclusion patterns. - """ + """Shared logic for inclusion/exclusion patterns.""" + PREFIX = NotImplemented def __init__(self, pattern, recurse_dir=False): @@ -201,7 +195,7 @@ def match(self, path, normalize=True): return matches def __repr__(self): - return f'{type(self)}({self.pattern})' + return f"{type(self)}({self.pattern})" def __str__(self): return self.pattern_orig @@ -216,6 +210,7 @@ def _match(self, path): class PathFullPattern(PatternBase): """Full match of a path.""" + PREFIX = "pf" def _prepare(self, pattern): @@ -236,6 +231,7 @@ class PathPrefixPattern(PatternBase): If a directory is specified, all paths that start with that path match as well. A trailing slash makes no difference. """ + PREFIX = "pp" def _prepare(self, pattern): @@ -251,13 +247,14 @@ class FnmatchPattern(PatternBase): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ + PREFIX = "fm" def _prepare(self, pattern): if pattern.endswith(os.path.sep): - pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep + pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + "*" + os.path.sep else: - pattern = os.path.normpath(pattern) + os.path.sep + '*' + pattern = os.path.normpath(pattern) + os.path.sep + "*" self.pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed @@ -266,13 +263,14 @@ def _prepare(self, pattern): self.regex = re.compile(fnmatch.translate(self.pattern)) def _match(self, path): - return (self.regex.match(path + os.path.sep) is not None) + return self.regex.match(path + os.path.sep) is not None class ShellPattern(PatternBase): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ + PREFIX = "sh" def _prepare(self, pattern): @@ -287,12 +285,12 @@ def _prepare(self, pattern): self.regex = re.compile(shellpattern.translate(self.pattern)) def _match(self, path): - return (self.regex.match(path + os.path.sep) is not None) + return self.regex.match(path + os.path.sep) is not None class RegexPattern(PatternBase): - """Regular expression to exclude. - """ + """Regular expression to exclude.""" + PREFIX = "re" def _prepare(self, pattern): @@ -301,28 +299,22 @@ def _prepare(self, pattern): def _match(self, path): # Normalize path separators - if os.path.sep != '/': - path = path.replace(os.path.sep, '/') + if os.path.sep != "/": + path = path.replace(os.path.sep, "/") - return (self.regex.search(path) is not None) + return self.regex.search(path) is not None -_PATTERN_CLASSES = { - FnmatchPattern, - PathFullPattern, - PathPrefixPattern, - RegexPattern, - ShellPattern, -} +_PATTERN_CLASSES = {FnmatchPattern, PathFullPattern, PathPrefixPattern, RegexPattern, ShellPattern} _PATTERN_CLASS_BY_PREFIX = {i.PREFIX: i for i in _PATTERN_CLASSES} -CmdTuple = namedtuple('CmdTuple', 'val cmd') +CmdTuple = namedtuple("CmdTuple", "val cmd") class IECommand(Enum): - """A command that an InclExcl file line can represent. - """ + """A command that an InclExcl file line can represent.""" + RootPath = 1 PatternStyle = 2 Include = 3 @@ -343,9 +335,7 @@ def get_pattern_class(prefix): def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True): - """Read pattern from string and return an instance of the appropriate implementation class. - - """ + """Read pattern from string and return an instance of the appropriate implementation class.""" if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum(): (style, pattern) = (pattern[:2], pattern[3:]) cls = get_pattern_class(style) @@ -355,8 +345,7 @@ def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True): def parse_exclude_pattern(pattern_str, fallback=FnmatchPattern): - """Read pattern from string and return an instance of the appropriate implementation class. - """ + """Read pattern from string and return an instance of the appropriate implementation class.""" epattern_obj = parse_pattern(pattern_str, fallback, recurse_dir=False) return CmdTuple(epattern_obj, IECommand.ExcludeNoRecurse) @@ -365,21 +354,20 @@ def parse_inclexcl_command(cmd_line_str, fallback=ShellPattern): """Read a --patterns-from command from string and return a CmdTuple object.""" cmd_prefix_map = { - '-': IECommand.Exclude, - '!': IECommand.ExcludeNoRecurse, - '+': IECommand.Include, - 'R': IECommand.RootPath, - 'r': IECommand.RootPath, - 'P': IECommand.PatternStyle, - 'p': IECommand.PatternStyle, + "-": IECommand.Exclude, + "!": IECommand.ExcludeNoRecurse, + "+": IECommand.Include, + "R": IECommand.RootPath, + "r": IECommand.RootPath, + "P": IECommand.PatternStyle, + "p": IECommand.PatternStyle, } if not cmd_line_str: raise argparse.ArgumentTypeError("A pattern/command must not be empty.") cmd = cmd_prefix_map.get(cmd_line_str[0]) if cmd is None: - raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" % - ', '.join(cmd_prefix_map)) + raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" % ", ".join(cmd_prefix_map)) # remaining text on command-line following the command character remainder_str = cmd_line_str[1:].lstrip() diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py index 98f926230..6692960fe 100644 --- a/src/borg/platform/__init__.py +++ b/src/borg/platform/__init__.py @@ -17,6 +17,7 @@ if not is_win32: from .posix import process_alive, local_pid_alive + # posix swidth implementation works for: linux, freebsd, darwin, openindiana, cygwin from .posix import swidth from .posix import get_errno diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index d6f2aae8f..15b778729 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -20,9 +20,9 @@ are correctly composed into the base functionality. """ -API_VERSION = '1.2_05' +API_VERSION = "1.2_05" -fdatasync = getattr(os, 'fdatasync', os.fsync) +fdatasync = getattr(os, "fdatasync", os.fsync) from .xattr import ENOATTR @@ -86,14 +86,16 @@ def acl_set(path, item, numeric_ids=False, fd=None): def set_flags(path, bsd_flags, fd=None): lchflags(path, bsd_flags) + except ImportError: + def set_flags(path, bsd_flags, fd=None): pass def get_flags(path, st, fd=None): """Return BSD-style file flags for path or stat without following symlinks.""" - return getattr(st, 'st_flags', 0) + return getattr(st, "st_flags", 0) def sync_dir(path): @@ -114,8 +116,8 @@ def sync_dir(path): def safe_fadvise(fd, offset, len, advice): - if hasattr(os, 'posix_fadvise'): - advice = getattr(os, 'POSIX_FADV_' + advice) + if hasattr(os, "posix_fadvise"): + advice = getattr(os, "POSIX_FADV_" + advice) try: os.posix_fadvise(fd, offset, len, advice) except OSError: @@ -158,7 +160,7 @@ def __init__(self, path, *, fd=None, binary=False): that corresponds to path (like from os.open(path, ...) or os.mkstemp(...)) :param binary: whether to open in binary mode, default is False. """ - mode = 'xb' if binary else 'x' # x -> raise FileExists exception in open() if file exists already + mode = "xb" if binary else "x" # x -> raise FileExists exception in open() if file exists already self.path = path if fd is None: self.f = open(path, mode=mode) # python file object @@ -181,15 +183,17 @@ def sync(self): after sync(). """ from .. import platform + self.f.flush() platform.fdatasync(self.fd) # tell the OS that it does not need to cache what we just wrote, # avoids spoiling the cache for the OS and other processes. - safe_fadvise(self.fd, 0, 0, 'DONTNEED') + safe_fadvise(self.fd, 0, 0, "DONTNEED") def close(self): """sync() and close.""" from .. import platform + dirname = None try: dirname = os.path.dirname(self.path) @@ -216,23 +220,26 @@ class SaveFile: Internally used temporary files are created in the target directory and are named -.tmp and cleaned up in normal and error conditions. """ + def __init__(self, path, binary=False): self.binary = binary self.path = path self.dir = os.path.dirname(path) - self.tmp_prefix = os.path.basename(path) + '-' + self.tmp_prefix = os.path.basename(path) + "-" self.tmp_fd = None # OS-level fd self.tmp_fname = None # full path/filename corresponding to self.tmp_fd self.f = None # python-file-like SyncFile def __enter__(self): from .. import platform - self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix='.tmp', dir=self.dir) + + self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix=".tmp", dir=self.dir) self.f = platform.SyncFile(self.tmp_fname, fd=self.tmp_fd, binary=self.binary) return self.f def __exit__(self, exc_type, exc_val, exc_tb): from .. import platform + self.f.close() # this indirectly also closes self.tmp_fd self.tmp_fd = None if exc_type is not None: @@ -246,7 +253,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # thanks to the crappy os.umask api, we can't query the umask without setting it. :-( umask = os.umask(UMASK_DEFAULT) os.umask(umask) - os.chmod(self.tmp_fname, mode=0o666 & ~ umask) + os.chmod(self.tmp_fname, mode=0o666 & ~umask) except OSError: # chmod might fail if the fs does not support it. # this is not harmful, the file will still have permissions for the owner. @@ -270,13 +277,13 @@ def swidth(s): # patched socket.getfqdn() - see https://bugs.python.org/issue5004 -def getfqdn(name=''): +def getfqdn(name=""): """Get fully qualified domain name from name. An empty argument is interpreted as meaning the local host. """ name = name.strip() - if not name or name == '0.0.0.0': + if not name or name == "0.0.0.0": name = socket.gethostname() try: addrs = socket.getaddrinfo(name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME) @@ -296,14 +303,14 @@ def getfqdn(name=''): fqdn = getfqdn(hostname) # some people put the fqdn into /etc/hostname (which is wrong, should be the short hostname) # fix this (do the same as "hostname --short" cli command does internally): -hostname = hostname.split('.')[0] +hostname = hostname.split(".")[0] # uuid.getnode() is problematic in some environments (e.g. OpenVZ, see #3968) where the virtual MAC address # is all-zero. uuid.getnode falls back to returning a random value in that case, which is not what we want. # thus, we offer BORG_HOST_ID where a user can set an own, unique id for each of his hosts. -hostid = os.environ.get('BORG_HOST_ID') +hostid = os.environ.get("BORG_HOST_ID") if not hostid: - hostid = f'{fqdn}@{uuid.getnode()}' + hostid = f"{fqdn}@{uuid.getnode()}" def get_process_id(): diff --git a/src/borg/platform/xattr.py b/src/borg/platform/xattr.py index 74e3d3ab6..ad4f2b7d6 100644 --- a/src/borg/platform/xattr.py +++ b/src/borg/platform/xattr.py @@ -18,7 +18,7 @@ def split_string0(buf): """split a list of zero-terminated strings into python not-zero-terminated bytes""" if isinstance(buf, bytearray): buf = bytes(buf) # use a bytes object, so we return a list of bytes objects - return buf.split(b'\0')[:-1] + return buf.split(b"\0")[:-1] def split_lstring(buf): @@ -27,8 +27,8 @@ def split_lstring(buf): mv = memoryview(buf) while mv: length = mv[0] - result.append(bytes(mv[1:1 + length])) - mv = mv[1 + length:] + result.append(bytes(mv[1 : 1 + length])) + mv = mv[1 + length :] return result @@ -38,6 +38,7 @@ class BufferTooSmallError(Exception): def _check(rv, path=None, detect_buffer_too_small=False): from . import get_errno + if rv < 0: e = get_errno() if detect_buffer_too_small and e == errno.ERANGE: @@ -48,9 +49,9 @@ def _check(rv, path=None, detect_buffer_too_small=False): try: msg = os.strerror(e) except ValueError: - msg = '' + msg = "" if isinstance(path, int): - path = '' % path + path = "" % path raise OSError(e, msg, path) if detect_buffer_too_small and rv >= len(buffer): # freebsd does not error with ERANGE if the buffer is too small, diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py index 8bfea7732..64525b8aa 100644 --- a/src/borg/platformflags.py +++ b/src/borg/platformflags.py @@ -6,7 +6,7 @@ import sys -is_win32 = sys.platform.startswith('win32') -is_linux = sys.platform.startswith('linux') -is_freebsd = sys.platform.startswith('freebsd') -is_darwin = sys.platform.startswith('darwin') +is_win32 = sys.platform.startswith("win32") +is_linux = sys.platform.startswith("linux") +is_freebsd = sys.platform.startswith("freebsd") +is_darwin = sys.platform.startswith("darwin") diff --git a/src/borg/remote.py b/src/borg/remote.py index 22b078dcd..a1196d600 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -38,7 +38,7 @@ RPC_PROTOCOL_VERSION = 2 BORG_VERSION = parse_version(__version__) -MSGID, MSG, ARGS, RESULT = 'i', 'm', 'a', 'r' +MSGID, MSG, ARGS, RESULT = "i", "m", "a", "r" MAX_INFLIGHT = 100 @@ -92,8 +92,8 @@ def __init__(self, data): data = data.decode()[:128] except UnicodeDecodeError: data = data[:128] - data = ['%02X' % byte for byte in data] - data = textwrap.fill(' '.join(data), 16 * 3) + data = ["%02X" % byte for byte in data] + data = textwrap.fill(" ".join(data), 16 * 3) super().__init__(data) @@ -119,47 +119,47 @@ def __init__(self, data): compatMap = { - 'check': ('repair', 'save_space', ), - 'commit': ('save_space', ), - 'rollback': (), - 'destroy': (), - '__len__': (), - 'list': ('limit', 'marker', ), - 'put': ('id', 'data', ), - 'get': ('id', ), - 'delete': ('id', ), - 'save_key': ('keydata', ), - 'load_key': (), - 'break_lock': (), - 'negotiate': ('client_data', ), - 'open': ('path', 'create', 'lock_wait', 'lock', 'exclusive', 'append_only', ), - 'info': (), - 'get_free_nonce': (), - 'commit_nonce_reservation': ('next_unreserved', 'start_nonce', ), + "check": ("repair", "save_space"), + "commit": ("save_space",), + "rollback": (), + "destroy": (), + "__len__": (), + "list": ("limit", "marker"), + "put": ("id", "data"), + "get": ("id",), + "delete": ("id",), + "save_key": ("keydata",), + "load_key": (), + "break_lock": (), + "negotiate": ("client_data",), + "open": ("path", "create", "lock_wait", "lock", "exclusive", "append_only"), + "info": (), + "get_free_nonce": (), + "commit_nonce_reservation": ("next_unreserved", "start_nonce"), } class RepositoryServer: # pragma: no cover rpc_methods = ( - '__len__', - 'check', - 'commit', - 'delete', - 'destroy', - 'get', - 'list', - 'scan', - 'negotiate', - 'open', - 'info', - 'put', - 'rollback', - 'save_key', - 'load_key', - 'break_lock', - 'get_free_nonce', - 'commit_nonce_reservation', - 'inject_exception', + "__len__", + "check", + "commit", + "delete", + "destroy", + "get", + "list", + "scan", + "negotiate", + "open", + "info", + "put", + "rollback", + "save_key", + "load_key", + "break_lock", + "get_free_nonce", + "commit_nonce_reservation", + "inject_exception", ) def __init__(self, restrict_to_paths, restrict_to_repositories, append_only, storage_quota): @@ -172,14 +172,16 @@ def __init__(self, restrict_to_paths, restrict_to_repositories, append_only, sto # (see RepositoryServer.open below). self.append_only = append_only self.storage_quota = storage_quota - self.client_version = parse_version('1.0.8') # fallback version if client is too old to send version information + self.client_version = parse_version( + "1.0.8" + ) # fallback version if client is too old to send version information def positional_to_named(self, method, argv): """Translate from positional protocol to named protocol.""" try: return {name: argv[pos] for pos, name in enumerate(compatMap[method])} except IndexError: - if method == 'open' and len(argv) == 4: + if method == "open" and len(argv) == 4: # borg clients < 1.0.7 use open() with 4 args mapping = compatMap[method][:4] else: @@ -198,7 +200,7 @@ def serve(self): os.set_blocking(stdin_fd, False) os.set_blocking(stdout_fd, True) os.set_blocking(stderr_fd, True) - unpacker = get_limited_unpacker('server') + unpacker = get_limited_unpacker("server") while True: r, w, es = select.select([stdin_fd], [], [], 10) if r: @@ -207,8 +209,12 @@ def serve(self): if self.repository is not None: self.repository.close() else: - os_write(stderr_fd, 'Borg {}: Got connection close before repository was opened.\n' - .format(__version__).encode()) + os_write( + stderr_fd, + "Borg {}: Got connection close before repository was opened.\n".format( + __version__ + ).encode(), + ) return unpacker.feed(data) for unpacked in unpacker: @@ -249,25 +255,34 @@ def serve(self): # for these, except ErrorWithTraceback, which should always display a traceback. pass else: - logging.debug('\n'.join(ex_full)) + logging.debug("\n".join(ex_full)) try: - msg = msgpack.packb({MSGID: msgid, - 'exception_class': e.__class__.__name__, - 'exception_args': e.args, - 'exception_full': ex_full, - 'exception_short': ex_short, - 'exception_trace': ex_trace, - 'sysinfo': sysinfo()}) + msg = msgpack.packb( + { + MSGID: msgid, + "exception_class": e.__class__.__name__, + "exception_args": e.args, + "exception_full": ex_full, + "exception_short": ex_short, + "exception_trace": ex_trace, + "sysinfo": sysinfo(), + } + ) except TypeError: - msg = msgpack.packb({MSGID: msgid, - 'exception_class': e.__class__.__name__, - 'exception_args': [x if isinstance(x, (str, bytes, int)) else None - for x in e.args], - 'exception_full': ex_full, - 'exception_short': ex_short, - 'exception_trace': ex_trace, - 'sysinfo': sysinfo()}) + msg = msgpack.packb( + { + MSGID: msgid, + "exception_class": e.__class__.__name__, + "exception_args": [ + x if isinstance(x, (str, bytes, int)) else None for x in e.args + ], + "exception_full": ex_full, + "exception_short": ex_short, + "exception_trace": ex_trace, + "sysinfo": sysinfo(), + } + ) os_write(stdout_fd, msg) else: @@ -282,11 +297,11 @@ def serve(self): msg = e.get_message() else: tb_log_level = logging.ERROR - msg = '%s Exception in RPC call' % e.__class__.__name__ - tb = f'{traceback.format_exc()}\n{sysinfo()}' + msg = "%s Exception in RPC call" % e.__class__.__name__ + tb = f"{traceback.format_exc()}\n{sysinfo()}" logging.error(msg) logging.log(tb_log_level, tb) - exc = 'Remote Exception (see remote log for the traceback)' + exc = "Remote Exception (see remote log for the traceback)" os_write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: if dictFormat: @@ -304,51 +319,52 @@ def negotiate(self, client_data): # clients since 1.1.0b3 use a dict as client_data # clients since 1.1.0b6 support json log format from server if isinstance(client_data, dict): - self.client_version = client_data['client_version'] - level = logging.getLevelName(logging.getLogger('').level) + self.client_version = client_data["client_version"] + level = logging.getLevelName(logging.getLogger("").level) setup_logging(is_serve=True, json=True, level=level) - logger.debug('Initialized logging system for JSON-based protocol') + logger.debug("Initialized logging system for JSON-based protocol") else: self.client_version = BORG_VERSION # seems to be newer than current version (no known old format) # not a known old format, send newest negotiate this version knows - return {'server_version': BORG_VERSION} + return {"server_version": BORG_VERSION} def _resolve_path(self, path): if isinstance(path, bytes): path = os.fsdecode(path) # Leading slash is always present with URI (ssh://), but not with short-form (who@host:path). - if path.startswith('/~/'): # /~/x = path x relative to home dir + if path.startswith("/~/"): # /~/x = path x relative to home dir path = os.path.join(get_base_dir(), path[3:]) - elif path.startswith('~/'): + elif path.startswith("~/"): path = os.path.join(get_base_dir(), path[2:]) - elif path.startswith('/~'): # /~username/x = relative to "user" home dir + elif path.startswith("/~"): # /~username/x = relative to "user" home dir path = os.path.expanduser(path[1:]) - elif path.startswith('~'): + elif path.startswith("~"): path = os.path.expanduser(path) - elif path.startswith('/./'): # /./x = path x relative to cwd + elif path.startswith("/./"): # /./x = path x relative to cwd path = path[3:] return os.path.realpath(path) - def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False, - make_parent_dirs=False): - logging.debug('Resolving repository path %r', path) + def open( + self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False, make_parent_dirs=False + ): + logging.debug("Resolving repository path %r", path) path = self._resolve_path(path) - logging.debug('Resolved repository path to %r', path) - path_with_sep = os.path.join(path, '') # make sure there is a trailing slash (os.sep) + logging.debug("Resolved repository path to %r", path) + path_with_sep = os.path.join(path, "") # make sure there is a trailing slash (os.sep) if self.restrict_to_paths: # if --restrict-to-path P is given, we make sure that we only operate in/below path P. # for the prefix check, it is important that the compared paths both have trailing slashes, # so that a path /foobar will NOT be accepted with --restrict-to-path /foo option. for restrict_to_path in self.restrict_to_paths: - restrict_to_path_with_sep = os.path.join(os.path.realpath(restrict_to_path), '') # trailing slash + restrict_to_path_with_sep = os.path.join(os.path.realpath(restrict_to_path), "") # trailing slash if path_with_sep.startswith(restrict_to_path_with_sep): break else: raise PathNotAllowed(path) if self.restrict_to_repositories: for restrict_to_repository in self.restrict_to_repositories: - restrict_to_repository_with_sep = os.path.join(os.path.realpath(restrict_to_repository), '') + restrict_to_repository_with_sep = os.path.join(os.path.realpath(restrict_to_repository), "") if restrict_to_repository_with_sep == path_with_sep: break else: @@ -357,32 +373,37 @@ def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, ap # while "borg init --append-only" (=append_only) does, regardless of the --append-only (self.append_only) # flag for serve. append_only = (not create and self.append_only) or append_only - self.repository = Repository(path, create, lock_wait=lock_wait, lock=lock, - append_only=append_only, - storage_quota=self.storage_quota, - exclusive=exclusive, - make_parent_dirs=make_parent_dirs) + self.repository = Repository( + path, + create, + lock_wait=lock_wait, + lock=lock, + append_only=append_only, + storage_quota=self.storage_quota, + exclusive=exclusive, + make_parent_dirs=make_parent_dirs, + ) self.repository.__enter__() # clean exit handled by serve() method return self.repository.id def inject_exception(self, kind): - s1 = 'test string' - s2 = 'test string2' - if kind == 'DoesNotExist': + s1 = "test string" + s2 = "test string2" + if kind == "DoesNotExist": raise Repository.DoesNotExist(s1) - elif kind == 'AlreadyExists': + elif kind == "AlreadyExists": raise Repository.AlreadyExists(s1) - elif kind == 'CheckNeeded': + elif kind == "CheckNeeded": raise Repository.CheckNeeded(s1) - elif kind == 'IntegrityError': + elif kind == "IntegrityError": raise IntegrityError(s1) - elif kind == 'PathNotAllowed': - raise PathNotAllowed('foo') - elif kind == 'ObjectNotFound': + elif kind == "PathNotAllowed": + raise PathNotAllowed("foo") + elif kind == "ObjectNotFound": raise Repository.ObjectNotFound(s1, s2) - elif kind == 'InvalidRPCMethod': + elif kind == "InvalidRPCMethod": raise InvalidRPCMethod(s1) - elif kind == 'divide': + elif kind == "divide": 0 // 0 @@ -409,7 +430,7 @@ def write(self, fd, to_send): self.ratelimit_quota += self.ratelimit self.ratelimit_last = time.monotonic() if len(to_send) > self.ratelimit_quota: - to_send = to_send[:self.ratelimit_quota] + to_send = to_send[: self.ratelimit_quota] written = os.write(fd, to_send) if self.ratelimit: self.ratelimit_quota -= written @@ -436,6 +457,7 @@ def api(*, since, **kwargs_decorator): more desirable behaviour). If False, an exception is generated. E.g. before 'threshold' was introduced in 1.2.0a8, a hardcoded threshold of 0.1 was used in commit(). """ + def decorator(f): @functools.wraps(f) def do_rpc(self, *args, **kwargs): @@ -444,10 +466,10 @@ def do_rpc(self, *args, **kwargs): named = {} # Arguments for the remote process extra = {} # Arguments for the local process for name, param in sig.parameters.items(): - if name == 'self': + if name == "self": continue if name in bound_args.arguments: - if name == 'wait': + if name == "wait": extra[name] = bound_args.arguments[name] else: named[name] = bound_args.arguments[name] @@ -459,18 +481,21 @@ def do_rpc(self, *args, **kwargs): raise self.RPCServerOutdated(f.__name__, format_version(since)) for name, restriction in kwargs_decorator.items(): - if restriction['since'] <= self.server_version: + if restriction["since"] <= self.server_version: continue - if 'previously' in restriction and named[name] == restriction['previously']: + if "previously" in restriction and named[name] == restriction["previously"]: continue - if restriction.get('dontcare', False): + if restriction.get("dontcare", False): continue - raise self.RPCServerOutdated(f"{f.__name__} {name}={named[name]!s}", - format_version(restriction['since'])) + raise self.RPCServerOutdated( + f"{f.__name__} {name}={named[name]!s}", format_version(restriction["since"]) + ) return self.call(f.__name__, named, **extra) + return do_rpc + return decorator @@ -484,32 +509,32 @@ def __init__(self, unpacked): self.unpacked = unpacked def get_message(self): - if 'exception_short' in self.unpacked: - return '\n'.join(self.unpacked['exception_short']) + if "exception_short" in self.unpacked: + return "\n".join(self.unpacked["exception_short"]) else: return self.exception_class @property def traceback(self): - return self.unpacked.get('exception_trace', True) + return self.unpacked.get("exception_trace", True) @property def exception_class(self): - return self.unpacked['exception_class'] + return self.unpacked["exception_class"] @property def exception_full(self): - if 'exception_full' in self.unpacked: - return '\n'.join(self.unpacked['exception_full']) + if "exception_full" in self.unpacked: + return "\n".join(self.unpacked["exception_full"]) else: - return self.get_message() + '\nRemote Exception (see remote log for the traceback)' + return self.get_message() + "\nRemote Exception (see remote log for the traceback)" @property def sysinfo(self): - if 'sysinfo' in self.unpacked: - return self.unpacked['sysinfo'] + if "sysinfo" in self.unpacked: + return self.unpacked["sysinfo"] else: - return '' + return "" class RPCServerOutdated(Error): """Borg server is too old for {}. Required version {}""" @@ -525,15 +550,24 @@ def required_version(self): # If compatibility with 1.0.x is not longer needed, replace all checks of this with True and simplify the code dictFormat = False # outside of __init__ for testing of legacy free protocol - def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock=True, append_only=False, - make_parent_dirs=False, args=None): + def __init__( + self, + location, + create=False, + exclusive=False, + lock_wait=None, + lock=True, + append_only=False, + make_parent_dirs=False, + args=None, + ): self.location = self._location = location self.preload_ids = [] self.msgid = 0 self.rx_bytes = 0 self.tx_bytes = 0 self.to_send = EfficientCollectionQueue(1024 * 1024, bytes) - self.stderr_received = b'' # incomplete stderr line bytes received (no \n yet) + self.stderr_received = b"" # incomplete stderr line bytes received (no \n yet) self.chunkid_to_msgids = {} self.ignore_responses = set() self.responses = {} @@ -541,18 +575,20 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock self.shutdown_time = None self.ratelimit = SleepingBandwidthLimiter(args.upload_ratelimit * 1024 if args and args.upload_ratelimit else 0) self.upload_buffer_size_limit = args.upload_buffer * 1024 * 1024 if args and args.upload_buffer else 0 - self.unpacker = get_limited_unpacker('client') - self.server_version = parse_version('1.0.8') # fallback version if server is too old to send version information + self.unpacker = get_limited_unpacker("client") + self.server_version = parse_version( + "1.0.8" + ) # fallback version if server is too old to send version information self.p = None self._args = args - testing = location.host == '__testsuite__' + testing = location.host == "__testsuite__" # when testing, we invoke and talk to a borg process directly (no ssh). # when not testing, we invoke the system-installed ssh binary to talk to a remote borg. env = prepare_subprocess_env(system=not testing) borg_cmd = self.borg_cmd(args, testing) if not testing: borg_cmd = self.ssh_cmd(location) + borg_cmd - logger.debug('SSH command line: %s', borg_cmd) + logger.debug("SSH command line: %s", borg_cmd) self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() @@ -565,26 +601,30 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock try: try: - version = self.call('negotiate', {'client_data': { - 'client_version': BORG_VERSION, - }}) + version = self.call("negotiate", {"client_data": {"client_version": BORG_VERSION}}) except ConnectionClosed: - raise ConnectionClosedWithHint('Is borg working on the server?') from None + raise ConnectionClosedWithHint("Is borg working on the server?") from None if version == RPC_PROTOCOL_VERSION: self.dictFormat = False - elif isinstance(version, dict) and 'server_version' in version: + elif isinstance(version, dict) and "server_version" in version: self.dictFormat = True - self.server_version = version['server_version'] + self.server_version = version["server_version"] else: - raise Exception('Server insisted on using unsupported protocol version %s' % version) + raise Exception("Server insisted on using unsupported protocol version %s" % version) def do_open(): - self.id = self.open(path=self.location.path, create=create, lock_wait=lock_wait, - lock=lock, exclusive=exclusive, append_only=append_only, - make_parent_dirs=make_parent_dirs) + self.id = self.open( + path=self.location.path, + create=create, + lock_wait=lock_wait, + lock=lock, + exclusive=exclusive, + append_only=append_only, + make_parent_dirs=make_parent_dirs, + ) info = self.info() - self.version = info['version'] - self.append_only = info['append_only'] + self.version = info["version"] + self.append_only = info["append_only"] if self.dictFormat: do_open() @@ -593,7 +633,7 @@ def do_open(): try: do_open() except self.RPCError as err: - if err.exception_class != 'TypeError': + if err.exception_class != "TypeError": raise msg = """\ Please note: @@ -605,8 +645,8 @@ def do_open(): """ # emit this msg in the same way as the 'Remote: ...' lines that show the remote TypeError sys.stderr.write(msg) - self.server_version = parse_version('1.0.6') - compatMap['open'] = ('path', 'create', 'lock_wait', 'lock', ) + self.server_version = parse_version("1.0.6") + compatMap["open"] = ("path", "create", "lock_wait", "lock") # try again with corrected version and compatMap do_open() except Exception: @@ -615,13 +655,13 @@ def do_open(): def __del__(self): if len(self.responses): - logging.debug('still %d cached responses left in RemoteRepository' % (len(self.responses),)) + logging.debug("still %d cached responses left in RemoteRepository" % (len(self.responses),)) if self.p: self.close() - assert False, 'cleanup happened in Repository.__del__' + assert False, "cleanup happened in Repository.__del__" def __repr__(self): - return f'<{self.__class__.__name__} {self.location.canonical_path()}>' + return f"<{self.__class__.__name__} {self.location.canonical_path()}>" def __enter__(self): return self @@ -635,8 +675,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): # in any case, we want to cleanly close the repo, even if the # rollback can not succeed (e.g. because the connection was # already closed) and raised another exception: - logger.debug('RemoteRepository: %s bytes sent, %s bytes received, %d messages sent', - format_file_size(self.tx_bytes), format_file_size(self.rx_bytes), self.msgid) + logger.debug( + "RemoteRepository: %s bytes sent, %s bytes received, %d messages sent", + format_file_size(self.tx_bytes), + format_file_size(self.rx_bytes), + self.msgid, + ) self.close() @property @@ -650,17 +694,17 @@ def borg_cmd(self, args, testing): if args is not None: root_logger = logging.getLogger() if root_logger.isEnabledFor(logging.DEBUG): - opts.append('--debug') + opts.append("--debug") elif root_logger.isEnabledFor(logging.INFO): - opts.append('--info') + opts.append("--info") elif root_logger.isEnabledFor(logging.WARNING): pass # warning is default elif root_logger.isEnabledFor(logging.ERROR): - opts.append('--error') + opts.append("--error") elif root_logger.isEnabledFor(logging.CRITICAL): - opts.append('--critical') + opts.append("--critical") else: - raise ValueError('log level missing, fix this code') + raise ValueError("log level missing, fix this code") # Tell the remote server about debug topics it may need to consider. # Note that debug topics are usable for "spew" or "trace" logs which would @@ -674,31 +718,31 @@ def borg_cmd(self, args, testing): # This is not considered a problem, since this is a debugging feature that # should not be used for regular use. for topic in args.debug_topics: - if '.' not in topic: - topic = 'borg.debug.' + topic - if 'repository' in topic: - opts.append('--debug-topic=%s' % topic) + if "." not in topic: + topic = "borg.debug." + topic + if "repository" in topic: + opts.append("--debug-topic=%s" % topic) - if 'storage_quota' in args and args.storage_quota: - opts.append('--storage-quota=%s' % args.storage_quota) + if "storage_quota" in args and args.storage_quota: + opts.append("--storage-quota=%s" % args.storage_quota) env_vars = [] if testing: - return env_vars + [sys.executable, '-m', 'borg.archiver', 'serve'] + opts + self.extra_test_args + return env_vars + [sys.executable, "-m", "borg.archiver", "serve"] + opts + self.extra_test_args else: # pragma: no cover - remote_path = args.remote_path or os.environ.get('BORG_REMOTE_PATH', 'borg') + remote_path = args.remote_path or os.environ.get("BORG_REMOTE_PATH", "borg") remote_path = replace_placeholders(remote_path) - return env_vars + [remote_path, 'serve'] + opts + return env_vars + [remote_path, "serve"] + opts def ssh_cmd(self, location): """return a ssh command line that can be prefixed to a borg command line""" - rsh = self._args.rsh or os.environ.get('BORG_RSH', 'ssh') + rsh = self._args.rsh or os.environ.get("BORG_RSH", "ssh") args = shlex.split(rsh) if location.port: - args += ['-p', str(location.port)] + args += ["-p", str(location.port)] if location.user: - args.append(f'{location.user}@{location.host}') + args.append(f"{location.user}@{location.host}") else: - args.append('%s' % location.host) + args.append("%s" % location.host) return args def named_to_positional(self, method, kwargs): @@ -709,7 +753,7 @@ def call(self, cmd, args, **kw): return resp def call_many(self, cmd, calls, wait=True, is_preloaded=False, async_wait=True): - if not calls and cmd != 'async_responses': + if not calls and cmd != "async_responses": return def send_buffer(): @@ -732,41 +776,41 @@ def pop_preload_msgid(chunkid): return msgid def handle_error(unpacked): - error = unpacked['exception_class'] - old_server = 'exception_args' not in unpacked - args = unpacked.get('exception_args') + error = unpacked["exception_class"] + old_server = "exception_args" not in unpacked + args = unpacked.get("exception_args") - if error == 'DoesNotExist': + if error == "DoesNotExist": raise Repository.DoesNotExist(self.location.processed) - elif error == 'AlreadyExists': + elif error == "AlreadyExists": raise Repository.AlreadyExists(self.location.processed) - elif error == 'CheckNeeded': + elif error == "CheckNeeded": raise Repository.CheckNeeded(self.location.processed) - elif error == 'IntegrityError': + elif error == "IntegrityError": if old_server: - raise IntegrityError('(not available)') + raise IntegrityError("(not available)") else: raise IntegrityError(args[0]) - elif error == 'AtticRepository': + elif error == "AtticRepository": if old_server: - raise Repository.AtticRepository('(not available)') + raise Repository.AtticRepository("(not available)") else: raise Repository.AtticRepository(args[0]) - elif error == 'PathNotAllowed': + elif error == "PathNotAllowed": if old_server: - raise PathNotAllowed('(unknown)') + raise PathNotAllowed("(unknown)") else: raise PathNotAllowed(args[0]) - elif error == 'ParentPathDoesNotExist': + elif error == "ParentPathDoesNotExist": raise Repository.ParentPathDoesNotExist(args[0]) - elif error == 'ObjectNotFound': + elif error == "ObjectNotFound": if old_server: - raise Repository.ObjectNotFound('(not available)', self.location.processed) + raise Repository.ObjectNotFound("(not available)", self.location.processed) else: raise Repository.ObjectNotFound(args[0], self.location.processed) - elif error == 'InvalidRPCMethod': + elif error == "InvalidRPCMethod": if old_server: - raise InvalidRPCMethod('(not available)') + raise InvalidRPCMethod("(not available)") else: raise InvalidRPCMethod(args[0]) else: @@ -780,14 +824,17 @@ def handle_error(unpacked): if self.shutdown_time and time.monotonic() > self.shutdown_time: # we are shutting this RemoteRepository down already, make sure we do not waste # a lot of time in case a lot of async stuff is coming in or remote is gone or slow. - logger.debug('shutdown_time reached, shutting down with %d waiting_for and %d async_responses.', - len(waiting_for), len(self.async_responses)) + logger.debug( + "shutdown_time reached, shutting down with %d waiting_for and %d async_responses.", + len(waiting_for), + len(self.async_responses), + ) return while waiting_for: try: unpacked = self.responses.pop(waiting_for[0]) waiting_for.pop(0) - if 'exception_class' in unpacked: + if "exception_class" in unpacked: handle_error(unpacked) else: yield unpacked[RESULT] @@ -795,7 +842,7 @@ def handle_error(unpacked): return except KeyError: break - if cmd == 'async_responses': + if cmd == "async_responses": while True: try: msgid, unpacked = self.async_responses.popitem() @@ -807,7 +854,7 @@ def handle_error(unpacked): else: return else: - if 'exception_class' in unpacked: + if "exception_class" in unpacked: handle_error(unpacked) else: yield unpacked[RESULT] @@ -817,7 +864,7 @@ def handle_error(unpacked): w_fds = [] r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) if x: - raise Exception('FD exception occurred') + raise Exception("FD exception occurred") for fd in r: if fd is self.stdout_fd: data = os.read(fd, BUFSIZE) @@ -833,7 +880,7 @@ def handle_error(unpacked): _, msgid, error, res = unpacked if error: # ignore res, because it is only a fixed string anyway. - unpacked = {MSGID: msgid, 'exception_class': error} + unpacked = {MSGID: msgid, "exception_class": error} else: unpacked = {MSGID: msgid, RESULT: res} else: @@ -841,7 +888,7 @@ def handle_error(unpacked): if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) # async methods never return values, but may raise exceptions. - if 'exception_class' in unpacked: + if "exception_class" in unpacked: self.async_responses[msgid] = unpacked else: # we currently do not have async result values except "None", @@ -858,83 +905,95 @@ def handle_error(unpacked): # deal with incomplete lines (may appear due to block buffering) if self.stderr_received: data = self.stderr_received + data - self.stderr_received = b'' + self.stderr_received = b"" lines = data.splitlines(keepends=True) - if lines and not lines[-1].endswith((b'\r', b'\n')): + if lines and not lines[-1].endswith((b"\r", b"\n")): self.stderr_received = lines.pop() # now we have complete lines in and any partial line in self.stderr_received. for line in lines: handle_remote_line(line.decode()) # decode late, avoid partial utf-8 sequences if w: - while (len(self.to_send) <= maximum_to_send) and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: + while ( + (len(self.to_send) <= maximum_to_send) + and (calls or self.preload_ids) + and len(waiting_for) < MAX_INFLIGHT + ): if calls: if is_preloaded: - assert cmd == 'get', "is_preload is only supported for 'get'" - if calls[0]['id'] in self.chunkid_to_msgids: - waiting_for.append(pop_preload_msgid(calls.pop(0)['id'])) + assert cmd == "get", "is_preload is only supported for 'get'" + if calls[0]["id"] in self.chunkid_to_msgids: + waiting_for.append(pop_preload_msgid(calls.pop(0)["id"])) else: args = calls.pop(0) - if cmd == 'get' and args['id'] in self.chunkid_to_msgids: - waiting_for.append(pop_preload_msgid(args['id'])) + if cmd == "get" and args["id"] in self.chunkid_to_msgids: + waiting_for.append(pop_preload_msgid(args["id"])) else: self.msgid += 1 waiting_for.append(self.msgid) if self.dictFormat: self.to_send.push_back(msgpack.packb({MSGID: self.msgid, MSG: cmd, ARGS: args})) else: - self.to_send.push_back(msgpack.packb((1, self.msgid, cmd, self.named_to_positional(cmd, args)))) + self.to_send.push_back( + msgpack.packb((1, self.msgid, cmd, self.named_to_positional(cmd, args))) + ) if not self.to_send and self.preload_ids: chunk_id = self.preload_ids.pop(0) - args = {'id': chunk_id} + args = {"id": chunk_id} self.msgid += 1 self.chunkid_to_msgids.setdefault(chunk_id, []).append(self.msgid) if self.dictFormat: - self.to_send.push_back(msgpack.packb({MSGID: self.msgid, MSG: 'get', ARGS: args})) + self.to_send.push_back(msgpack.packb({MSGID: self.msgid, MSG: "get", ARGS: args})) else: - self.to_send.push_back(msgpack.packb((1, self.msgid, 'get', self.named_to_positional('get', args)))) + self.to_send.push_back( + msgpack.packb((1, self.msgid, "get", self.named_to_positional("get", args))) + ) send_buffer() self.ignore_responses |= set(waiting_for) # we lose order here - @api(since=parse_version('1.0.0'), - append_only={'since': parse_version('1.0.7'), 'previously': False}, - make_parent_dirs={'since': parse_version('1.1.9'), 'previously': False}) - def open(self, path, create=False, lock_wait=None, lock=True, exclusive=False, append_only=False, - make_parent_dirs=False): + @api( + since=parse_version("1.0.0"), + append_only={"since": parse_version("1.0.7"), "previously": False}, + make_parent_dirs={"since": parse_version("1.1.9"), "previously": False}, + ) + def open( + self, path, create=False, lock_wait=None, lock=True, exclusive=False, append_only=False, make_parent_dirs=False + ): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('2.0.0a3')) + @api(since=parse_version("2.0.0a3")) def info(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0'), - max_duration={'since': parse_version('1.2.0a4'), 'previously': 0}) + @api(since=parse_version("1.0.0"), max_duration={"since": parse_version("1.2.0a4"), "previously": 0}) def check(self, repair=False, save_space=False, max_duration=0): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0'), - compact={'since': parse_version('1.2.0a0'), 'previously': True, 'dontcare': True}, - threshold={'since': parse_version('1.2.0a8'), 'previously': 0.1, 'dontcare': True}) + @api( + since=parse_version("1.0.0"), + compact={"since": parse_version("1.2.0a0"), "previously": True, "dontcare": True}, + threshold={"since": parse_version("1.2.0a8"), "previously": 0.1, "dontcare": True}, + ) def commit(self, save_space=False, compact=True, threshold=0.1): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def rollback(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def destroy(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def __len__(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def list(self, limit=None, marker=None): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.1.0b3')) + @api(since=parse_version("1.1.0b3")) def scan(self, limit=None, marker=None): """actual remoting is done via self.call in the @api decorator""" @@ -943,33 +1002,33 @@ def get(self, id): return resp def get_many(self, ids, is_preloaded=False): - yield from self.call_many('get', [{'id': id} for id in ids], is_preloaded=is_preloaded) + yield from self.call_many("get", [{"id": id} for id in ids], is_preloaded=is_preloaded) - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def put(self, id, data, wait=True): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def delete(self, id, wait=True): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def save_key(self, keydata): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def load_key(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def get_free_nonce(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def commit_nonce_reservation(self, next_unreserved, start_nonce): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version('1.0.0')) + @api(since=parse_version("1.0.0")) def break_lock(self): """actual remoting is done via self.call in the @api decorator""" @@ -981,7 +1040,7 @@ def close(self): self.p = None def async_response(self, wait=True): - for resp in self.call_many('async_responses', calls=[], wait=True, async_wait=wait): + for resp in self.call_many("async_responses", calls=[], wait=True, async_wait=wait): return resp def preload(self, ids): @@ -994,72 +1053,72 @@ def handle_remote_line(line): This function is remarkably complex because it handles multiple wire formats. """ - assert line.endswith(('\r', '\n')) - if line.startswith('{'): + assert line.endswith(("\r", "\n")) + if line.startswith("{"): # This format is used by Borg since 1.1.0b6 for new-protocol clients. # It is the same format that is exposed by --log-json. msg = json.loads(line) - if msg['type'] not in ('progress_message', 'progress_percent', 'log_message'): - logger.warning('Dropped remote log message with unknown type %r: %s', msg['type'], line) + if msg["type"] not in ("progress_message", "progress_percent", "log_message"): + logger.warning("Dropped remote log message with unknown type %r: %s", msg["type"], line) return - if msg['type'] == 'log_message': + if msg["type"] == "log_message": # Re-emit log messages on the same level as the remote to get correct log suppression and verbosity. - level = getattr(logging, msg['levelname'], logging.CRITICAL) + level = getattr(logging, msg["levelname"], logging.CRITICAL) assert isinstance(level, int) - target_logger = logging.getLogger(msg['name']) - msg['message'] = 'Remote: ' + msg['message'] + target_logger = logging.getLogger(msg["name"]) + msg["message"] = "Remote: " + msg["message"] # In JSON mode, we manually check whether the log message should be propagated. - if logging.getLogger('borg').json and level >= target_logger.getEffectiveLevel(): - sys.stderr.write(json.dumps(msg) + '\n') + if logging.getLogger("borg").json and level >= target_logger.getEffectiveLevel(): + sys.stderr.write(json.dumps(msg) + "\n") else: - target_logger.log(level, '%s', msg['message']) - elif msg['type'].startswith('progress_'): + target_logger.log(level, "%s", msg["message"]) + elif msg["type"].startswith("progress_"): # Progress messages are a bit more complex. # First of all, we check whether progress output is enabled. This is signalled # through the effective level of the borg.output.progress logger # (also see ProgressIndicatorBase in borg.helpers). - progress_logger = logging.getLogger('borg.output.progress') + progress_logger = logging.getLogger("borg.output.progress") if progress_logger.getEffectiveLevel() == logging.INFO: # When progress output is enabled, we check whether the client is in # --log-json mode, as signalled by the "json" attribute on the "borg" logger. - if logging.getLogger('borg').json: + if logging.getLogger("borg").json: # In --log-json mode we re-emit the progress JSON line as sent by the server, # with the message, if any, prefixed with "Remote: ". - if 'message' in msg: - msg['message'] = 'Remote: ' + msg['message'] - sys.stderr.write(json.dumps(msg) + '\n') - elif 'message' in msg: + if "message" in msg: + msg["message"] = "Remote: " + msg["message"] + sys.stderr.write(json.dumps(msg) + "\n") + elif "message" in msg: # In text log mode we write only the message to stderr and terminate with \r # (carriage return, i.e. move the write cursor back to the beginning of the line) # so that the next message, progress or not, overwrites it. This mirrors the behaviour # of local progress displays. - sys.stderr.write('Remote: ' + msg['message'] + '\r') - elif line.startswith('$LOG '): + sys.stderr.write("Remote: " + msg["message"] + "\r") + elif line.startswith("$LOG "): # This format is used by borg serve 0.xx, 1.0.x and 1.1.0b1..b5. # It prefixed log lines with $LOG as a marker, followed by the log level # and optionally a logger name, then "Remote:" as a separator followed by the original # message. - _, level, msg = line.split(' ', 2) + _, level, msg = line.split(" ", 2) level = getattr(logging, level, logging.CRITICAL) # str -> int - if msg.startswith('Remote:'): + if msg.startswith("Remote:"): # server format: '$LOG Remote: ' logging.log(level, msg.rstrip()) else: # server format '$LOG Remote: ' - logname, msg = msg.split(' ', 1) + logname, msg = msg.split(" ", 1) logging.getLogger(logname).log(level, msg.rstrip()) else: # Plain 1.0.x and older format - re-emit to stderr (mirroring what the 1.0.x # client did) or as a generic log message. # We don't know what priority the line had. - if logging.getLogger('borg').json: - logging.getLogger('').warning('Remote: ' + line.strip()) + if logging.getLogger("borg").json: + logging.getLogger("").warning("Remote: " + line.strip()) else: # In non-JSON mode we circumvent logging to preserve carriage returns (\r) # which are generated by remote progress displays. - sys.stderr.write('Remote: ' + line) + sys.stderr.write("Remote: " + line) class RepositoryNoCache: @@ -1071,6 +1130,7 @@ class RepositoryNoCache: The return value is returned from get()/get_many(). By default, the raw repository data is returned. """ + def __init__(self, repository, transform=None): self.repository = repository self.transform = transform or (lambda key, data: data) @@ -1112,7 +1172,7 @@ def __init__(self, repository, pack=None, unpack=None, transform=None): self.pack = pack or (lambda data: data) self.unpack = unpack or (lambda data: data) self.cache = set() - self.basedir = tempfile.mkdtemp(prefix='borg-cache-') + self.basedir = tempfile.mkdtemp(prefix="borg-cache-") self.query_size_limit() self.size = 0 # Instrumentation @@ -1147,7 +1207,7 @@ def add_entry(self, key, data, cache): packed = self.pack(transformed) file = self.key_filename(key) try: - with open(file, 'wb') as fd: + with open(file, "wb") as fd: fd.write(packed) except OSError as os_error: try: @@ -1167,11 +1227,19 @@ def add_entry(self, key, data, cache): return transformed def log_instrumentation(self): - logger.debug('RepositoryCache: current items %d, size %s / %s, %d hits, %d misses, %d slow misses (+%.1fs), ' - '%d evictions, %d ENOSPC hit', - len(self.cache), format_file_size(self.size), format_file_size(self.size_limit), - self.hits, self.misses, self.slow_misses, self.slow_lat, - self.evictions, self.enospc) + logger.debug( + "RepositoryCache: current items %d, size %s / %s, %d hits, %d misses, %d slow misses (+%.1fs), " + "%d evictions, %d ENOSPC hit", + len(self.cache), + format_file_size(self.size), + format_file_size(self.size_limit), + self.hits, + self.misses, + self.slow_misses, + self.slow_lat, + self.evictions, + self.enospc, + ) def close(self): self.log_instrumentation() @@ -1184,7 +1252,7 @@ def get_many(self, keys, cache=True): for key in keys: if key in self.cache: file = self.key_filename(key) - with open(file, 'rb') as fd: + with open(file, "rb") as fd: self.hits += 1 yield self.unpack(fd.read()) else: @@ -1218,12 +1286,12 @@ def cache_if_remote(repository, *, decrypted_cache=False, pack=None, unpack=None Internally, objects are compressed with LZ4. """ if decrypted_cache and (pack or unpack or transform): - raise ValueError('decrypted_cache and pack/unpack/transform are incompatible') + raise ValueError("decrypted_cache and pack/unpack/transform are incompatible") elif decrypted_cache: key = decrypted_cache # 32 bit csize, 64 bit (8 byte) xxh64 - cache_struct = struct.Struct('=I8s') - compressor = Compressor('lz4') + cache_struct = struct.Struct("=I8s") + compressor = Compressor("lz4") def pack(data): csize, decrypted = data @@ -1232,10 +1300,10 @@ def pack(data): def unpack(data): data = memoryview(data) - csize, checksum = cache_struct.unpack(data[:cache_struct.size]) - compressed = data[cache_struct.size:] + csize, checksum = cache_struct.unpack(data[: cache_struct.size]) + compressed = data[cache_struct.size :] if checksum != xxh64(compressed): - raise IntegrityError('detected corrupted data in metadata cache') + raise IntegrityError("detected corrupted data in metadata cache") return csize, compressor.decompress(compressed) def transform(id_, data): diff --git a/src/borg/repository.py b/src/borg/repository.py index ba193243c..3a05b09ef 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -30,9 +30,9 @@ logger = create_logger(__name__) -MAGIC = b'BORG_SEG' +MAGIC = b"BORG_SEG" MAGIC_LEN = len(MAGIC) -ATTIC_MAGIC = b'ATTICSEG' +ATTIC_MAGIC = b"ATTICSEG" assert len(ATTIC_MAGIC) == MAGIC_LEN TAG_PUT = 0 @@ -172,11 +172,20 @@ class InsufficientFreeSpaceError(Error): class StorageQuotaExceeded(Error): """The storage quota ({}) has been exceeded ({}). Try deleting some archives.""" - def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True, - append_only=False, storage_quota=None, check_segment_magic=True, - make_parent_dirs=False): + def __init__( + self, + path, + create=False, + exclusive=False, + lock_wait=None, + lock=True, + append_only=False, + storage_quota=None, + check_segment_magic=True, + make_parent_dirs=False, + ): self.path = os.path.abspath(path) - self._location = Location('file://%s' % self.path) + self._location = Location("file://%s" % self.path) self.version = None self.io = None # type: LoggedIO self.lock = None @@ -201,7 +210,7 @@ def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=Tru # v2 is the default repo version for borg 2.0 # v1 repos must only be used in a read-only way, e.g. for # --other-repo=V1_REPO with borg init and borg transfer! - self.acceptable_repo_versions = (1, 2, ) + self.acceptable_repo_versions = (1, 2) def __del__(self): if self.lock: @@ -209,7 +218,7 @@ def __del__(self): assert False, "cleanup happened in Repository.__del__" def __repr__(self): - return f'<{self.__class__.__name__} {self.path}>' + return f"<{self.__class__.__name__} {self.path}>" def __enter__(self): if self.do_create: @@ -225,7 +234,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): # The ENOSPC could have originated somewhere else besides the Repository. The cleanup is always safe, unless # EIO or FS corruption ensues, which is why we specifically check for ENOSPC. if self._active_txn and no_space_left_on_device: - logger.warning('No space left on device, cleaning up partial transaction to free space.') + logger.warning("No space left on device, cleaning up partial transaction to free space.") cleanup = True else: cleanup = False @@ -241,12 +250,12 @@ def is_repository(path): """Check whether there is already a Borg repository at *path*.""" try: # Use binary mode to avoid troubles if a README contains some stuff not in our locale - with open(os.path.join(path, 'README'), 'rb') as fd: + with open(os.path.join(path, "README"), "rb") as fd: # Read only the first ~100 bytes (if any), in case some README file we stumble upon is large. readme_head = fd.read(100) # The first comparison captures our current variant (REPOSITORY_README), the second comparison # is an older variant of the README file (used by 1.0.x). - return b'Borg Backup repository' in readme_head or b'Borg repository' in readme_head + return b"Borg Backup repository" in readme_head or b"Borg repository" in readme_head except OSError: # Ignore FileNotFound, PermissionError, ... return False @@ -287,8 +296,7 @@ def check_can_create_repository(self, path): raise self.AlreadyExists(path) def create(self, path): - """Create a new empty repository at `path` - """ + """Create a new empty repository at `path`""" self.check_can_create_repository(path) if self.make_parent_dirs: parent_path = os.path.join(path, os.pardir) @@ -298,35 +306,37 @@ def create(self, path): os.mkdir(path) except FileNotFoundError as err: raise self.ParentPathDoesNotExist(path) from err - with open(os.path.join(path, 'README'), 'w') as fd: + with open(os.path.join(path, "README"), "w") as fd: fd.write(REPOSITORY_README) - os.mkdir(os.path.join(path, 'data')) + os.mkdir(os.path.join(path, "data")) config = ConfigParser(interpolation=None) - config.add_section('repository') + config.add_section("repository") self.version = 2 - config.set('repository', 'version', str(self.version)) - config.set('repository', 'segments_per_dir', str(DEFAULT_SEGMENTS_PER_DIR)) - config.set('repository', 'max_segment_size', str(DEFAULT_MAX_SEGMENT_SIZE)) - config.set('repository', 'append_only', str(int(self.append_only))) + config.set("repository", "version", str(self.version)) + config.set("repository", "segments_per_dir", str(DEFAULT_SEGMENTS_PER_DIR)) + config.set("repository", "max_segment_size", str(DEFAULT_MAX_SEGMENT_SIZE)) + config.set("repository", "append_only", str(int(self.append_only))) if self.storage_quota: - config.set('repository', 'storage_quota', str(self.storage_quota)) + config.set("repository", "storage_quota", str(self.storage_quota)) else: - config.set('repository', 'storage_quota', '0') - config.set('repository', 'additional_free_space', '0') - config.set('repository', 'id', bin_to_hex(os.urandom(32))) + config.set("repository", "storage_quota", "0") + config.set("repository", "additional_free_space", "0") + config.set("repository", "id", bin_to_hex(os.urandom(32))) self.save_config(path, config) def save_config(self, path, config): - config_path = os.path.join(path, 'config') - old_config_path = os.path.join(path, 'config.old') + config_path = os.path.join(path, "config") + old_config_path = os.path.join(path, "config.old") if os.path.isfile(old_config_path): logger.warning("Old config file not securely erased on previous config update") secure_erase(old_config_path, avoid_collateral_damage=True) if os.path.isfile(config_path): - link_error_msg = ("Failed to securely erase old repository config file (hardlinks not supported). " - "Old repokey data, if any, might persist on physical storage.") + link_error_msg = ( + "Failed to securely erase old repository config file (hardlinks not supported). " + "Old repokey data, if any, might persist on physical storage." + ) try: os.link(config_path, old_config_path) except OSError as e: @@ -345,32 +355,34 @@ def save_config(self, path, config): # error is only a problem if we even had a lock if self.do_lock: raise - logger.warning("%s: Failed writing to '%s'. This is expected when working on " - "read-only repositories." % (e.strerror, e.filename)) + logger.warning( + "%s: Failed writing to '%s'. This is expected when working on " + "read-only repositories." % (e.strerror, e.filename) + ) if os.path.isfile(old_config_path): secure_erase(old_config_path, avoid_collateral_damage=True) def save_key(self, keydata): assert self.config - keydata = keydata.decode('utf-8') # remote repo: msgpack issue #99, getting bytes + keydata = keydata.decode("utf-8") # remote repo: msgpack issue #99, getting bytes # note: saving an empty key means that there is no repokey any more - self.config.set('repository', 'key', keydata) + self.config.set("repository", "key", keydata) self.save_config(self.path, self.config) def load_key(self): - keydata = self.config.get('repository', 'key', fallback='').strip() + keydata = self.config.get("repository", "key", fallback="").strip() # note: if we return an empty string, it means there is no repo key - return keydata.encode('utf-8') # remote repo: msgpack issue #99, returning bytes + return keydata.encode("utf-8") # remote repo: msgpack issue #99, returning bytes def get_free_nonce(self): if self.do_lock and not self.lock.got_exclusive_lock(): raise AssertionError("bug in code, exclusive lock should exist here") - nonce_path = os.path.join(self.path, 'nonce') + nonce_path = os.path.join(self.path, "nonce") try: with open(nonce_path) as fd: - return int.from_bytes(unhexlify(fd.read()), byteorder='big') + return int.from_bytes(unhexlify(fd.read()), byteorder="big") except FileNotFoundError: return None @@ -380,30 +392,33 @@ def commit_nonce_reservation(self, next_unreserved, start_nonce): if self.get_free_nonce() != start_nonce: raise Exception("nonce space reservation with mismatched previous state") - nonce_path = os.path.join(self.path, 'nonce') + nonce_path = os.path.join(self.path, "nonce") try: with SaveFile(nonce_path, binary=False) as fd: - fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder='big'))) + fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder="big"))) except PermissionError as e: # error is only a problem if we even had a lock if self.do_lock: raise - logger.warning("%s: Failed writing to '%s'. This is expected when working on " - "read-only repositories." % (e.strerror, e.filename)) + logger.warning( + "%s: Failed writing to '%s'. This is expected when working on " + "read-only repositories." % (e.strerror, e.filename) + ) def destroy(self): - """Destroy the repository at `self.path` - """ + """Destroy the repository at `self.path`""" if self.append_only: raise ValueError(self.path + " is in append-only mode") self.close() - os.remove(os.path.join(self.path, 'config')) # kill config first + os.remove(os.path.join(self.path, "config")) # kill config first shutil.rmtree(self.path) def get_index_transaction_id(self): - indices = sorted(int(fn[6:]) - for fn in os.listdir(self.path) - if fn.startswith('index.') and fn[6:].isdigit() and os.stat(os.path.join(self.path, fn)).st_size != 0) + indices = sorted( + int(fn[6:]) + for fn in os.listdir(self.path) + if fn.startswith("index.") and fn[6:].isdigit() and os.stat(os.path.join(self.path, fn)).st_size != 0 + ) if indices: return indices[-1] else: @@ -434,7 +449,7 @@ def get_transaction_id(self): return self.get_index_transaction_id() def break_lock(self): - Lock(os.path.join(self.path, 'lock')).break_lock() + Lock(os.path.join(self.path, "lock")).break_lock() def migrate_lock(self, old_id, new_id): # note: only needed for local repos @@ -450,39 +465,38 @@ def open(self, path, exclusive, lock_wait=None, lock=True): if not stat.S_ISDIR(st.st_mode): raise self.InvalidRepository(path) if lock: - self.lock = Lock(os.path.join(path, 'lock'), exclusive, timeout=lock_wait).acquire() + self.lock = Lock(os.path.join(path, "lock"), exclusive, timeout=lock_wait).acquire() else: self.lock = None self.config = ConfigParser(interpolation=None) try: - with open(os.path.join(self.path, 'config')) as fd: + with open(os.path.join(self.path, "config")) as fd: self.config.read_file(fd) except FileNotFoundError: self.close() raise self.InvalidRepository(self.path) - if 'repository' not in self.config.sections(): + if "repository" not in self.config.sections(): self.close() - raise self.InvalidRepositoryConfig(path, 'no repository section found') - self.version = self.config.getint('repository', 'version') + raise self.InvalidRepositoryConfig(path, "no repository section found") + self.version = self.config.getint("repository", "version") if self.version not in self.acceptable_repo_versions: self.close() raise self.InvalidRepositoryConfig( - path, - 'repository version %d is not supported by this borg version' % self.version + path, "repository version %d is not supported by this borg version" % self.version ) - self.max_segment_size = parse_file_size(self.config.get('repository', 'max_segment_size')) + self.max_segment_size = parse_file_size(self.config.get("repository", "max_segment_size")) if self.max_segment_size >= MAX_SEGMENT_SIZE_LIMIT: self.close() - raise self.InvalidRepositoryConfig(path, 'max_segment_size >= %d' % MAX_SEGMENT_SIZE_LIMIT) # issue 3592 - self.segments_per_dir = self.config.getint('repository', 'segments_per_dir') - self.additional_free_space = parse_file_size(self.config.get('repository', 'additional_free_space', fallback=0)) + raise self.InvalidRepositoryConfig(path, "max_segment_size >= %d" % MAX_SEGMENT_SIZE_LIMIT) # issue 3592 + self.segments_per_dir = self.config.getint("repository", "segments_per_dir") + self.additional_free_space = parse_file_size(self.config.get("repository", "additional_free_space", fallback=0)) # append_only can be set in the constructor # it shouldn't be overridden (True -> False) here - self.append_only = self.append_only or self.config.getboolean('repository', 'append_only', fallback=False) + self.append_only = self.append_only or self.config.getboolean("repository", "append_only", fallback=False) if self.storage_quota is None: # self.storage_quota is None => no explicit storage_quota was specified, use repository setting. - self.storage_quota = parse_file_size(self.config.get('repository', 'storage_quota', fallback=0)) - self.id = unhexlify(self.config.get('repository', 'id').strip()) + self.storage_quota = parse_file_size(self.config.get("repository", "storage_quota", fallback=0)) + self.id = unhexlify(self.config.get("repository", "id").strip()) self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir) if self.check_segment_magic: # read a segment and check whether we are dealing with a non-upgraded Attic repository @@ -493,11 +507,7 @@ def open(self, path, exclusive, lock_wait=None, lock=True): def info(self): """return some infos about the repo (must be opened first)""" - return dict( - id=self.id, - version=self.version, - append_only=self.append_only, - ) + return dict(id=self.id, version=self.version, append_only=self.append_only) def close(self): if self.lock: @@ -508,8 +518,7 @@ def close(self): self.lock = None def commit(self, save_space=False, compact=True, threshold=0.1): - """Commit transaction - """ + """Commit transaction""" # save_space is not used anymore, but stays for RPC/API compatibility. if self.transaction_doomed: exception = self.transaction_doomed @@ -526,32 +535,32 @@ def commit(self, save_space=False, compact=True, threshold=0.1): self.rollback() def _read_integrity(self, transaction_id, key): - integrity_file = 'integrity.%d' % transaction_id + integrity_file = "integrity.%d" % transaction_id integrity_path = os.path.join(self.path, integrity_file) try: - with open(integrity_path, 'rb') as fd: + with open(integrity_path, "rb") as fd: integrity = msgpack.unpack(fd) except FileNotFoundError: return - if integrity.get('version') != 2: - logger.warning('Unknown integrity data version %r in %s', integrity.get('version'), integrity_file) + if integrity.get("version") != 2: + logger.warning("Unknown integrity data version %r in %s", integrity.get("version"), integrity_file) return return integrity[key] def open_index(self, transaction_id, auto_recover=True): if transaction_id is None: return NSIndex() - index_path = os.path.join(self.path, 'index.%d' % transaction_id) + index_path = os.path.join(self.path, "index.%d" % transaction_id) variant = hashindex_variant(index_path) - integrity_data = self._read_integrity(transaction_id, 'index') + integrity_data = self._read_integrity(transaction_id, "index") try: with IntegrityCheckedFile(index_path, write=False, integrity_data=integrity_data) as fd: - if variant == 'k32_v16': + if variant == "k32_v16": return NSIndex.read(fd) - if variant == 'k32_v8': # legacy + if variant == "k32_v8": # legacy return NSIndex1.read(fd) except (ValueError, OSError, FileIntegrityError) as exc: - logger.warning('Repository index missing or corrupted, trying to recover from: %s', exc) + logger.warning("Repository index missing or corrupted, trying to recover from: %s", exc) os.unlink(index_path) if not auto_recover: raise @@ -583,7 +592,7 @@ def prepare_txn(self, transaction_id, do_cleanup=True): try: self.index = self.open_index(transaction_id, auto_recover=False) except (ValueError, OSError, FileIntegrityError) as exc: - logger.warning('Checking repository transaction due to previous error: %s', exc) + logger.warning("Checking repository transaction due to previous error: %s", exc) self.check_transaction() self.index = self.open_index(transaction_id, auto_recover=False) if transaction_id is None: @@ -594,14 +603,14 @@ def prepare_txn(self, transaction_id, do_cleanup=True): else: if do_cleanup: self.io.cleanup(transaction_id) - hints_path = os.path.join(self.path, 'hints.%d' % transaction_id) - index_path = os.path.join(self.path, 'index.%d' % transaction_id) - integrity_data = self._read_integrity(transaction_id, 'hints') + hints_path = os.path.join(self.path, "hints.%d" % transaction_id) + index_path = os.path.join(self.path, "index.%d" % transaction_id) + integrity_data = self._read_integrity(transaction_id, "hints") try: with IntegrityCheckedFile(hints_path, write=False, integrity_data=integrity_data) as fd: hints = msgpack.unpack(fd) except (msgpack.UnpackException, FileNotFoundError, FileIntegrityError) as e: - logger.warning('Repository hints file missing or corrupted, trying to recover: %s', e) + logger.warning("Repository hints file missing or corrupted, trying to recover: %s", e) if not isinstance(e, FileNotFoundError): os.unlink(hints_path) # index must exist at this point @@ -609,23 +618,23 @@ def prepare_txn(self, transaction_id, do_cleanup=True): self.check_transaction() self.prepare_txn(transaction_id) return - if hints['version'] == 1: - logger.debug('Upgrading from v1 hints.%d', transaction_id) - self.segments = hints['segments'] + if hints["version"] == 1: + logger.debug("Upgrading from v1 hints.%d", transaction_id) + self.segments = hints["segments"] self.compact = FreeSpace() self.storage_quota_use = 0 self.shadow_index = {} - for segment in sorted(hints['compact']): - logger.debug('Rebuilding sparse info for segment %d', segment) + for segment in sorted(hints["compact"]): + logger.debug("Rebuilding sparse info for segment %d", segment) self._rebuild_sparse(segment) - logger.debug('Upgrade to v2 hints complete') - elif hints['version'] != 2: - raise ValueError('Unknown hints file version: %d' % hints['version']) + logger.debug("Upgrade to v2 hints complete") + elif hints["version"] != 2: + raise ValueError("Unknown hints file version: %d" % hints["version"]) else: - self.segments = hints['segments'] - self.compact = FreeSpace(hints['compact']) - self.storage_quota_use = hints.get('storage_quota_use', 0) - self.shadow_index = hints.get('shadow_index', {}) + self.segments = hints["segments"] + self.compact = FreeSpace(hints["compact"]) + self.storage_quota_use = hints.get("storage_quota_use", 0) + self.shadow_index = hints.get("shadow_index", {}) self.log_storage_quota() # Drop uncommitted segments in the shadow index for key, shadowed_segments in self.shadow_index.items(): @@ -639,50 +648,51 @@ def flush_and_sync(fd): os.fsync(fd.fileno()) def rename_tmp(file): - os.rename(file + '.tmp', file) + os.rename(file + ".tmp", file) hints = { - 'version': 2, - 'segments': self.segments, - 'compact': self.compact, - 'storage_quota_use': self.storage_quota_use, - 'shadow_index': self.shadow_index, + "version": 2, + "segments": self.segments, + "compact": self.compact, + "storage_quota_use": self.storage_quota_use, + "shadow_index": self.shadow_index, } integrity = { # Integrity version started at 2, the current hints version. # Thus, integrity version == hints version, for now. - 'version': 2, + "version": 2 } transaction_id = self.io.get_segments_transaction_id() assert transaction_id is not None # Log transaction in append-only mode if self.append_only: - with open(os.path.join(self.path, 'transactions'), 'a') as log: - print('transaction %d, UTC time %s' % ( - transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log) + with open(os.path.join(self.path, "transactions"), "a") as log: + print( + "transaction %d, UTC time %s" % (transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log + ) # Write hints file - hints_name = 'hints.%d' % transaction_id + hints_name = "hints.%d" % transaction_id hints_file = os.path.join(self.path, hints_name) - with IntegrityCheckedFile(hints_file + '.tmp', filename=hints_name, write=True) as fd: + with IntegrityCheckedFile(hints_file + ".tmp", filename=hints_name, write=True) as fd: msgpack.pack(hints, fd) flush_and_sync(fd) - integrity['hints'] = fd.integrity_data + integrity["hints"] = fd.integrity_data # Write repository index - index_name = 'index.%d' % transaction_id + index_name = "index.%d" % transaction_id index_file = os.path.join(self.path, index_name) - with IntegrityCheckedFile(index_file + '.tmp', filename=index_name, write=True) as fd: + with IntegrityCheckedFile(index_file + ".tmp", filename=index_name, write=True) as fd: # XXX: Consider using SyncFile for index write-outs. self.index.write(fd) flush_and_sync(fd) - integrity['index'] = fd.integrity_data + integrity["index"] = fd.integrity_data # Write integrity file, containing checksums of the hints and index files - integrity_name = 'integrity.%d' % transaction_id + integrity_name = "integrity.%d" % transaction_id integrity_file = os.path.join(self.path, integrity_name) - with open(integrity_file + '.tmp', 'wb') as fd: + with open(integrity_file + ".tmp", "wb") as fd: msgpack.pack(integrity, fd) flush_and_sync(fd) @@ -695,9 +705,9 @@ def rename_tmp(file): sync_dir(self.path) # Remove old auxiliary files - current = '.%d' % transaction_id + current = ".%d" % transaction_id for name in os.listdir(self.path): - if not name.startswith(('index.', 'hints.', 'integrity.')): + if not name.startswith(("index.", "hints.", "integrity.")): continue if name.endswith(current): continue @@ -737,9 +747,11 @@ def check_free_space(self): except FileNotFoundError: # looks like self.compact is referring to a non-existent segment file, ignore it. pass - logger.debug('check_free_space: few segments, not requiring a full free segment') + logger.debug("check_free_space: few segments, not requiring a full free segment") compact_working_space = min(compact_working_space, full_segment_size) - logger.debug('check_free_space: calculated working space for compact as %d bytes', compact_working_space) + logger.debug( + "check_free_space: calculated working space for compact as %d bytes", compact_working_space + ) required_free_space += compact_working_space else: # Keep one full worst-case segment free in non-append-only mode @@ -748,12 +760,12 @@ def check_free_space(self): try: free_space = shutil.disk_usage(self.path).free except OSError as os_error: - logger.warning('Failed to check free space before committing: ' + str(os_error)) + logger.warning("Failed to check free space before committing: " + str(os_error)) return - logger.debug(f'check_free_space: required bytes {required_free_space}, free bytes {free_space}') + logger.debug(f"check_free_space: required bytes {required_free_space}, free bytes {free_space}") if free_space < required_free_space: if self.created: - logger.error('Not enough free space to initialize repository at this location.') + logger.error("Not enough free space to initialize repository at this location.") self.destroy() else: self._rollback(cleanup=True) @@ -763,14 +775,16 @@ def check_free_space(self): def log_storage_quota(self): if self.storage_quota: - logger.info('Storage quota: %s out of %s used.', - format_file_size(self.storage_quota_use), format_file_size(self.storage_quota)) + logger.info( + "Storage quota: %s out of %s used.", + format_file_size(self.storage_quota_use), + format_file_size(self.storage_quota), + ) def compact_segments(self, threshold): - """Compact sparse segments by copying data into new segments - """ + """Compact sparse segments by copying data into new segments""" if not self.compact: - logger.debug('nothing to do: compact empty') + logger.debug("nothing to do: compact empty") return quota_use_before = self.storage_quota_use index_transaction_id = self.get_index_transaction_id() @@ -784,22 +798,25 @@ def complete_xfer(intermediate=True): segment = self.io.write_commit(intermediate=intermediate) self.segments.setdefault(segment, 0) self.compact[segment] += LoggedIO.header_fmt.size - logger.debug('complete_xfer: wrote %scommit at segment %d', 'intermediate ' if intermediate else '', segment) + logger.debug( + "complete_xfer: wrote %scommit at segment %d", "intermediate " if intermediate else "", segment + ) # get rid of the old, sparse, unused segments. free space. for segment in unused: - logger.debug('complete_xfer: deleting unused segment %d', segment) + logger.debug("complete_xfer: deleting unused segment %d", segment) count = self.segments.pop(segment) - assert count == 0, 'Corrupted segment reference count - corrupted index or hints' + assert count == 0, "Corrupted segment reference count - corrupted index or hints" self.io.delete_segment(segment) del self.compact[segment] unused = [] - logger.debug('Compaction started (threshold is %i%%).', threshold * 100) - pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1, - msgid='repository.compact_segments') + logger.debug("Compaction started (threshold is %i%%).", threshold * 100) + pi = ProgressIndicatorPercent( + total=len(self.compact), msg="Compacting segments %3.0f%%", step=1, msgid="repository.compact_segments" + ) for segment, freeable_space in sorted(self.compact.items()): if not self.io.segment_exists(segment): - logger.warning('segment %d not found, but listed in compaction data', segment) + logger.warning("segment %d not found, but listed in compaction data", segment) del self.compact[segment] pi.show() continue @@ -808,13 +825,22 @@ def complete_xfer(intermediate=True): # we want to compact if: # - we can free a considerable relative amount of space (freeable_ratio over some threshold) if not (freeable_ratio > threshold): - logger.debug('not compacting segment %d (maybe freeable: %2.2f%% [%d bytes])', - segment, freeable_ratio * 100.0, freeable_space) + logger.debug( + "not compacting segment %d (maybe freeable: %2.2f%% [%d bytes])", + segment, + freeable_ratio * 100.0, + freeable_space, + ) pi.show() continue segments.setdefault(segment, 0) - logger.debug('compacting segment %d with usage count %d (maybe freeable: %2.2f%% [%d bytes])', - segment, segments[segment], freeable_ratio * 100.0, freeable_space) + logger.debug( + "compacting segment %d with usage count %d (maybe freeable: %2.2f%% [%d bytes])", + segment, + segments[segment], + freeable_ratio * 100.0, + freeable_space, + ) for tag, key, offset, data in self.io.iter_objects(segment, include_data=True): if tag == TAG_COMMIT: continue @@ -849,7 +875,9 @@ def complete_xfer(intermediate=True): shadowed_put_exists = key not in self.shadow_index or any( # If the key is in the shadow index and there is any segment with an older PUT of this # key, we have a shadowed put. - shadowed < segment for shadowed in self.shadow_index[key]) + shadowed < segment + for shadowed in self.shadow_index[key] + ) delete_is_not_stable = index_transaction_id is None or segment > index_transaction_id if shadowed_put_exists or delete_is_not_stable: @@ -898,14 +926,14 @@ def complete_xfer(intermediate=True): if not self.shadow_index[key]: # shadowed segments list is empty -> remove it del self.shadow_index[key] - assert segments[segment] == 0, 'Corrupted segment reference count - corrupted index or hints' + assert segments[segment] == 0, "Corrupted segment reference count - corrupted index or hints" unused.append(segment) pi.show() pi.finish() complete_xfer(intermediate=False) quota_use_after = self.storage_quota_use - logger.info('compaction freed about %s repository space.', format_file_size(quota_use_before - quota_use_after)) - logger.debug('compaction completed.') + logger.info("compaction freed about %s repository space.", format_file_size(quota_use_before - quota_use_after)) + logger.debug("compaction completed.") def replay_segments(self, index_transaction_id, segments_transaction_id): # fake an old client, so that in case we do not have an exclusive lock yet, prepare_txn will upgrade the lock: @@ -914,8 +942,9 @@ def replay_segments(self, index_transaction_id, segments_transaction_id): self.prepare_txn(index_transaction_id, do_cleanup=False) try: segment_count = sum(1 for _ in self.io.segment_iterator()) - pi = ProgressIndicatorPercent(total=segment_count, msg='Replaying segments %3.0f%%', - msgid='repository.replay_segments') + pi = ProgressIndicatorPercent( + total=segment_count, msg="Replaying segments %3.0f%%", msgid="repository.replay_segments" + ) for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) if index_transaction_id is not None and segment <= index_transaction_id: @@ -960,7 +989,7 @@ def _update_index(self, segment, objects, report=None): elif tag == TAG_COMMIT: continue else: - msg = f'Unexpected tag {tag} in segment {segment}' + msg = f"Unexpected tag {tag} in segment {segment}" if report is None: raise self.CheckNeeded(msg) else: @@ -1008,56 +1037,57 @@ def report_error(msg): error_found = True logger.error(msg) - logger.info('Starting repository check') + logger.info("Starting repository check") assert not self._active_txn try: transaction_id = self.get_transaction_id() current_index = self.open_index(transaction_id) - logger.debug('Read committed index of transaction %d', transaction_id) + logger.debug("Read committed index of transaction %d", transaction_id) except Exception as exc: transaction_id = self.io.get_segments_transaction_id() current_index = None - logger.debug('Failed to read committed index (%s)', exc) + logger.debug("Failed to read committed index (%s)", exc) if transaction_id is None: - logger.debug('No segments transaction found') + logger.debug("No segments transaction found") transaction_id = self.get_index_transaction_id() if transaction_id is None: - logger.debug('No index transaction found, trying latest segment') + logger.debug("No index transaction found, trying latest segment") transaction_id = self.io.get_latest_segment() if transaction_id is None: - report_error('This repository contains no valid data.') + report_error("This repository contains no valid data.") return False if repair: self.io.cleanup(transaction_id) segments_transaction_id = self.io.get_segments_transaction_id() - logger.debug('Segment transaction is %s', segments_transaction_id) - logger.debug('Determined transaction is %s', transaction_id) + logger.debug("Segment transaction is %s", segments_transaction_id) + logger.debug("Determined transaction is %s", transaction_id) self.prepare_txn(None) # self.index, self.compact, self.segments, self.shadow_index all empty now! segment_count = sum(1 for _ in self.io.segment_iterator()) - logger.debug('Found %d segments', segment_count) + logger.debug("Found %d segments", segment_count) partial = bool(max_duration) assert not (repair and partial) - mode = 'partial' if partial else 'full' + mode = "partial" if partial else "full" if partial: # continue a past partial check (if any) or start one from beginning - last_segment_checked = self.config.getint('repository', 'last_segment_checked', fallback=-1) - logger.info('skipping to segments >= %d', last_segment_checked + 1) + last_segment_checked = self.config.getint("repository", "last_segment_checked", fallback=-1) + logger.info("skipping to segments >= %d", last_segment_checked + 1) else: # start from the beginning and also forget about any potential past partial checks last_segment_checked = -1 - self.config.remove_option('repository', 'last_segment_checked') + self.config.remove_option("repository", "last_segment_checked") self.save_config(self.path, self.config) t_start = time.monotonic() - pi = ProgressIndicatorPercent(total=segment_count, msg='Checking segments %3.1f%%', step=0.1, - msgid='repository.check') + pi = ProgressIndicatorPercent( + total=segment_count, msg="Checking segments %3.1f%%", step=0.1, msgid="repository.check" + ) for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) if segment <= last_segment_checked: continue if segment > transaction_id: continue - logger.debug('checking segment file %s...', filename) + logger.debug("checking segment file %s...", filename) try: objects = list(self.io.iter_objects(segment)) except IntegrityError as err: @@ -1069,35 +1099,35 @@ def report_error(msg): if not partial: self._update_index(segment, objects, report_error) if partial and time.monotonic() > t_start + max_duration: - logger.info('finished partial segment check, last segment checked is %d', segment) - self.config.set('repository', 'last_segment_checked', str(segment)) + logger.info("finished partial segment check, last segment checked is %d", segment) + self.config.set("repository", "last_segment_checked", str(segment)) self.save_config(self.path, self.config) break else: - logger.info('finished segment check at segment %d', segment) - self.config.remove_option('repository', 'last_segment_checked') + logger.info("finished segment check at segment %d", segment) + self.config.remove_option("repository", "last_segment_checked") self.save_config(self.path, self.config) pi.finish() # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found if repair and segments_transaction_id is None: - report_error(f'Adding commit tag to segment {transaction_id}') + report_error(f"Adding commit tag to segment {transaction_id}") self.io.segment = transaction_id + 1 self.io.write_commit() if not partial: - logger.info('Starting repository index check') + logger.info("Starting repository index check") if current_index and not repair: # current_index = "as found on disk" # self.index = "as rebuilt in-memory from segments" if len(current_index) != len(self.index): - report_error('Index object count mismatch.') - logger.error('committed index: %d objects', len(current_index)) - logger.error('rebuilt index: %d objects', len(self.index)) + report_error("Index object count mismatch.") + logger.error("committed index: %d objects", len(current_index)) + logger.error("rebuilt index: %d objects", len(self.index)) else: - logger.info('Index object count match.') - line_format = 'ID: %-64s rebuilt index: %-16s committed index: %-16s' - not_found = '' + logger.info("Index object count match.") + line_format = "ID: %-64s rebuilt index: %-16s committed index: %-16s" + not_found = "" for key, value in self.index.iteritems(): current_value = current_index.get(key, not_found) if current_value != value: @@ -1113,11 +1143,11 @@ def report_error(msg): self.rollback() if error_found: if repair: - logger.info('Finished %s repository check, errors found and repaired.', mode) + logger.info("Finished %s repository check, errors found and repaired.", mode) else: - logger.error('Finished %s repository check, errors found.', mode) + logger.error("Finished %s repository check, errors found.", mode) else: - logger.info('Finished %s repository check, no problems found.', mode) + logger.info("Finished %s repository check, no problems found.", mode) return not error_found or repair def scan_low_level(self, segment=None, offset=None): @@ -1136,18 +1166,19 @@ def scan_low_level(self, segment=None, offset=None): if segment is not None and current_segment > segment: break try: - for tag, key, current_offset, data in self.io.iter_objects(segment=current_segment, - offset=offset or 0, include_data=True): + for tag, key, current_offset, data in self.io.iter_objects( + segment=current_segment, offset=offset or 0, include_data=True + ): if offset is not None and current_offset > offset: break yield key, data, tag, current_segment, current_offset except IntegrityError as err: - logger.error('Segment %d (%s) has IntegrityError(s) [%s] - skipping.' % ( - current_segment, filename, str(err))) + logger.error( + "Segment %d (%s) has IntegrityError(s) [%s] - skipping." % (current_segment, filename, str(err)) + ) def _rollback(self, *, cleanup): - """ - """ + """ """ if cleanup: self.io.cleanup(self.io.get_segments_transaction_id()) self.index = None @@ -1189,7 +1220,7 @@ def scan(self, limit=None, marker=None): if we encounter CRC errors in segment entry headers, rest of segment is skipped. """ if limit is not None and limit < 1: - raise ValueError('please use limit > 0 or limit = None') + raise ValueError("please use limit > 0 or limit = None") if not self.index: transaction_id = self.get_transaction_id() self.index = self.open_index(transaction_id) @@ -1225,7 +1256,7 @@ def get(self, id): if not self.index: self.index = self.open_index(self.get_transaction_id()) try: - in_index = NSIndexEntry(*((self.index[id] + (None, ))[:3])) # legacy: index entriess have no size element + in_index = NSIndexEntry(*((self.index[id] + (None,))[:3])) # legacy: index entriess have no size element return self.io.read(in_index.segment, in_index.offset, id, expected_size=in_index.size) except KeyError: raise self.ObjectNotFound(id, self.path) from None @@ -1259,7 +1290,8 @@ def put(self, id, data, wait=True): self.index[id] = NSIndexEntry(segment, offset, len(data)) if self.storage_quota and self.storage_quota_use > self.storage_quota: self.transaction_doomed = self.StorageQuotaExceeded( - format_file_size(self.storage_quota), format_file_size(self.storage_quota_use)) + format_file_size(self.storage_quota), format_file_size(self.storage_quota_use) + ) raise self.transaction_doomed def delete(self, id, wait=True): @@ -1302,20 +1334,18 @@ def async_response(self, wait=True): """ def preload(self, ids): - """Preload objects (only applies to remote repositories) - """ + """Preload objects (only applies to remote repositories)""" class LoggedIO: - class SegmentFull(Exception): """raised when a segment is full, before opening next""" - header_fmt = struct.Struct(' self.limit): @@ -1438,10 +1464,10 @@ def get_write_fd(self, no_new=False, want_new=False, raise_full=False): self.close_segment() if not self._write_fd: if self.segment % self.segments_per_dir == 0: - dirname = os.path.join(self.path, 'data', str(self.segment // self.segments_per_dir)) + dirname = os.path.join(self.path, "data", str(self.segment // self.segments_per_dir)) if not os.path.exists(dirname): os.mkdir(dirname) - sync_dir(os.path.join(self.path, 'data')) + sync_dir(os.path.join(self.path, "data")) self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True) self._write_fd.write(MAGIC) self.offset = MAGIC_LEN @@ -1459,7 +1485,7 @@ def get_fd(self, segment): now = time.monotonic() def open_fd(): - fd = open(self.segment_filename(segment), 'rb') + fd = open(self.segment_filename(segment), "rb") self.fds[segment] = (now, fd) return fd @@ -1535,13 +1561,13 @@ def iter_objects(self, segment, offset=0, include_data=False, read_data=True): # Repository.scan() calls us with segment > 0 when it continues an ongoing iteration # from a marker position - but then we have checked the magic before already. if fd.read(MAGIC_LEN) != MAGIC: - raise IntegrityError(f'Invalid segment magic [segment {segment}, offset {0}]') + raise IntegrityError(f"Invalid segment magic [segment {segment}, offset {0}]") offset = MAGIC_LEN header = fd.read(self.header_fmt.size) while header: - size, tag, key, data = self._read(fd, header, segment, offset, - (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), - read_data=read_data) + size, tag, key, data = self._read( + fd, header, segment, offset, (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), read_data=read_data + ) if include_data: yield tag, key, offset, data else: @@ -1558,7 +1584,7 @@ def iter_objects(self, segment, offset=0, include_data=False, read_data=True): header = fd.read(self.header_fmt.size) def recover_segment(self, segment, filename): - logger.info('attempting to recover ' + filename) + logger.info("attempting to recover " + filename) if segment in self.fds: del self.fds[segment] if os.path.getsize(filename) < MAGIC_LEN + self.header_fmt.size: @@ -1568,7 +1594,7 @@ def recover_segment(self, segment, filename): fd.write(MAGIC) return with SaveFile(filename, binary=True) as dst_fd: - with open(filename, 'rb') as src_fd: + with open(filename, "rb") as src_fd: # note: file must not be 0 size or mmap() will crash. with mmap.mmap(src_fd.fileno(), 0, access=mmap.ACCESS_READ) as mm: # memoryview context manager is problematic, see https://bugs.python.org/issue35686 @@ -1577,7 +1603,7 @@ def recover_segment(self, segment, filename): try: dst_fd.write(MAGIC) while len(d) >= self.header_fmt.size: - crc, size, tag = self.header_fmt.unpack(d[:self.header_fmt.size]) + crc, size, tag = self.header_fmt.unpack(d[: self.header_fmt.size]) size_invalid = size > MAX_OBJECT_SIZE or size < self.header_fmt.size or size > len(d) if size_invalid or tag > MAX_TAG_ID: d = d[1:] @@ -1585,15 +1611,18 @@ def recover_segment(self, segment, filename): if tag == TAG_PUT2: c_offset = self.HEADER_ID_SIZE + self.ENTRY_HASH_SIZE # skip if header is invalid - if crc32(d[4:c_offset]) & 0xffffffff != crc: + if crc32(d[4:c_offset]) & 0xFFFFFFFF != crc: d = d[1:] continue # skip if content is invalid - if self.entry_hash(d[4:self.HEADER_ID_SIZE], d[c_offset:size]) != d[self.HEADER_ID_SIZE:c_offset]: + if ( + self.entry_hash(d[4 : self.HEADER_ID_SIZE], d[c_offset:size]) + != d[self.HEADER_ID_SIZE : c_offset] + ): d = d[1:] continue elif tag in (TAG_DELETE, TAG_COMMIT, TAG_PUT): - if crc32(d[4:size]) & 0xffffffff != crc: + if crc32(d[4:size]) & 0xFFFFFFFF != crc: d = d[1:] continue else: # tag unknown @@ -1625,12 +1654,14 @@ def read(self, segment, offset, id, read_data=True, *, expected_size=None): header = fd.read(self.header_fmt.size) size, tag, key, data = self._read(fd, header, segment, offset, (TAG_PUT2, TAG_PUT), read_data) if id != key: - raise IntegrityError('Invalid segment entry header, is not for wanted id [segment {}, offset {}]'.format( - segment, offset)) + raise IntegrityError( + "Invalid segment entry header, is not for wanted id [segment {}, offset {}]".format(segment, offset) + ) data_size_from_header = size - header_size(tag) if expected_size is not None and expected_size != data_size_from_header: - raise IntegrityError(f'size from repository index: {expected_size} != ' - f'size from entry header: {data_size_from_header}') + raise IntegrityError( + f"size from repository index: {expected_size} != " f"size from entry header: {data_size_from_header}" + ) return data if read_data else data_size_from_header def _read(self, fd, header, segment, offset, acceptable_tags, read_data=True): @@ -1643,35 +1674,38 @@ def _read(self, fd, header, segment, offset, acceptable_tags, read_data=True): PUT tags, read_data == True: crc32 check (header+data) PUT tags, read_data == False: crc32 check can not be done, all data obtained must be considered informational """ + def check_crc32(wanted, header, *data): result = crc32(memoryview(header)[4:]) # skip first 32 bits of the header, they contain the crc. for d in data: result = crc32(d, result) - if result & 0xffffffff != wanted: - raise IntegrityError(f'Segment entry header checksum mismatch [segment {segment}, offset {offset}]') + if result & 0xFFFFFFFF != wanted: + raise IntegrityError(f"Segment entry header checksum mismatch [segment {segment}, offset {offset}]") # See comment on MAX_TAG_ID for details - assert max(acceptable_tags) <= MAX_TAG_ID, 'Exceeding MAX_TAG_ID will break backwards compatibility' + assert max(acceptable_tags) <= MAX_TAG_ID, "Exceeding MAX_TAG_ID will break backwards compatibility" key = data = None fmt = self.header_fmt try: hdr_tuple = fmt.unpack(header) except struct.error as err: - raise IntegrityError(f'Invalid segment entry header [segment {segment}, offset {offset}]: {err}') from None + raise IntegrityError(f"Invalid segment entry header [segment {segment}, offset {offset}]: {err}") from None crc, size, tag = hdr_tuple length = size - fmt.size # we already read the header if size > MAX_OBJECT_SIZE: # if you get this on an archive made with borg < 1.0.7 and millions of files and # you need to restore it, you can disable this check by using "if False:" above. - raise IntegrityError(f'Invalid segment entry size {size} - too big [segment {segment}, offset {offset}]') + raise IntegrityError(f"Invalid segment entry size {size} - too big [segment {segment}, offset {offset}]") if size < fmt.size: - raise IntegrityError(f'Invalid segment entry size {size} - too small [segment {segment}, offset {offset}]') + raise IntegrityError(f"Invalid segment entry size {size} - too small [segment {segment}, offset {offset}]") if tag not in (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT): - raise IntegrityError(f'Invalid segment entry header, did not get a known tag ' - f'[segment {segment}, offset {offset}]') + raise IntegrityError( + f"Invalid segment entry header, did not get a known tag " f"[segment {segment}, offset {offset}]" + ) if tag not in acceptable_tags: - raise IntegrityError(f'Invalid segment entry header, did not get acceptable tag ' - f'[segment {segment}, offset {offset}]') + raise IntegrityError( + f"Invalid segment entry header, did not get acceptable tag " f"[segment {segment}, offset {offset}]" + ) if tag == TAG_COMMIT: check_crc32(crc, header) # that's all for COMMITs. @@ -1680,8 +1714,10 @@ def check_crc32(wanted, header, *data): key = fd.read(32) length -= 32 if len(key) != 32: - raise IntegrityError(f'Segment entry key short read [segment {segment}, offset {offset}]: ' - f'expected {32}, got {len(key)} bytes') + raise IntegrityError( + f"Segment entry key short read [segment {segment}, offset {offset}]: " + f"expected {32}, got {len(key)} bytes" + ) if tag == TAG_DELETE: check_crc32(crc, header, key) # that's all for DELETEs. @@ -1692,23 +1728,29 @@ def check_crc32(wanted, header, *data): entry_hash = fd.read(self.ENTRY_HASH_SIZE) length -= self.ENTRY_HASH_SIZE if len(entry_hash) != self.ENTRY_HASH_SIZE: - raise IntegrityError(f'Segment entry hash short read [segment {segment}, offset {offset}]: ' - f'expected {self.ENTRY_HASH_SIZE}, got {len(entry_hash)} bytes') + raise IntegrityError( + f"Segment entry hash short read [segment {segment}, offset {offset}]: " + f"expected {self.ENTRY_HASH_SIZE}, got {len(entry_hash)} bytes" + ) check_crc32(crc, header, key, entry_hash) if not read_data: # seek over data oldpos = fd.tell() seeked = fd.seek(length, os.SEEK_CUR) - oldpos if seeked != length: - raise IntegrityError(f'Segment entry data short seek [segment {segment}, offset {offset}]: ' - f'expected {length}, got {seeked} bytes') + raise IntegrityError( + f"Segment entry data short seek [segment {segment}, offset {offset}]: " + f"expected {length}, got {seeked} bytes" + ) else: # read data! data = fd.read(length) if len(data) != length: - raise IntegrityError(f'Segment entry data short read [segment {segment}, offset {offset}]: ' - f'expected {length}, got {len(data)} bytes') + raise IntegrityError( + f"Segment entry data short read [segment {segment}, offset {offset}]: " + f"expected {length}, got {len(data)} bytes" + ) if tag == TAG_PUT2: if self.entry_hash(memoryview(header)[4:], key, data) != entry_hash: - raise IntegrityError(f'Segment entry hash mismatch [segment {segment}, offset {offset}]') + raise IntegrityError(f"Segment entry hash mismatch [segment {segment}, offset {offset}]") elif tag == TAG_PUT: check_crc32(crc, header, key, data) return size, tag, key, data @@ -1717,14 +1759,14 @@ def write_put(self, id, data, raise_full=False): data_size = len(data) if data_size > MAX_DATA_SIZE: # this would push the segment entry size beyond MAX_OBJECT_SIZE. - raise IntegrityError(f'More than allowed put data [{data_size} > {MAX_DATA_SIZE}]') + raise IntegrityError(f"More than allowed put data [{data_size} > {MAX_DATA_SIZE}]") fd = self.get_write_fd(want_new=(id == Manifest.MANIFEST_ID), raise_full=raise_full) size = data_size + self.HEADER_ID_SIZE + self.ENTRY_HASH_SIZE offset = self.offset header = self.header_no_crc_fmt.pack(size, TAG_PUT2) entry_hash = self.entry_hash(header, id, data) - crc = self.crc_fmt.pack(crc32(entry_hash, crc32(id, crc32(header))) & 0xffffffff) - fd.write(b''.join((crc, header, id, entry_hash))) + crc = self.crc_fmt.pack(crc32(entry_hash, crc32(id, crc32(header))) & 0xFFFFFFFF) + fd.write(b"".join((crc, header, id, entry_hash))) fd.write(data) self.offset += size return self.segment, offset @@ -1732,8 +1774,8 @@ def write_put(self, id, data, raise_full=False): def write_delete(self, id, raise_full=False): fd = self.get_write_fd(want_new=(id == Manifest.MANIFEST_ID), raise_full=raise_full) header = self.header_no_crc_fmt.pack(self.HEADER_ID_SIZE, TAG_DELETE) - crc = self.crc_fmt.pack(crc32(id, crc32(header)) & 0xffffffff) - fd.write(b''.join((crc, header, id))) + crc = self.crc_fmt.pack(crc32(id, crc32(header)) & 0xFFFFFFFF) + fd.write(b"".join((crc, header, id))) self.offset += self.HEADER_ID_SIZE return self.segment, self.HEADER_ID_SIZE @@ -1744,8 +1786,8 @@ def write_commit(self, intermediate=False): if intermediate: fd.sync() header = self.header_no_crc_fmt.pack(self.header_fmt.size, TAG_COMMIT) - crc = self.crc_fmt.pack(crc32(header) & 0xffffffff) - fd.write(b''.join((crc, header))) + crc = self.crc_fmt.pack(crc32(header) & 0xFFFFFFFF) + fd.write(b"".join((crc, header))) self.close_segment() return self.segment - 1 # close_segment() increments it diff --git a/src/borg/selftest.py b/src/borg/selftest.py index 1bef011c2..49a9704c2 100644 --- a/src/borg/selftest.py +++ b/src/borg/selftest.py @@ -50,16 +50,16 @@ def test_name(self, test): def log_results(self, logger): for test, failure in self.errors + self.failures + self.unexpectedSuccesses: - logger.error('self test %s FAILED:\n%s', self.test_name(test), failure) + logger.error("self test %s FAILED:\n%s", self.test_name(test), failure) for test, reason in self.skipped: - logger.warning('self test %s skipped: %s', self.test_name(test), reason) + logger.warning("self test %s skipped: %s", self.test_name(test), reason) def successful_test_count(self): return len(self.successes) def selftest(logger): - if os.environ.get('BORG_SELFTEST') == 'disabled': + if os.environ.get("BORG_SELFTEST") == "disabled": logger.debug("borg selftest disabled via BORG_SELFTEST env variable") return selftest_started = time.perf_counter() @@ -69,7 +69,7 @@ def selftest(logger): module = sys.modules[test_case.__module__] # a normal borg user does not have pytest installed, we must not require it in the test modules used here. # note: this only detects the usual toplevel import - assert 'pytest' not in dir(module), "pytest must not be imported in %s" % module.__name__ + assert "pytest" not in dir(module), "pytest must not be imported in %s" % module.__name__ test_suite.addTest(defaultTestLoader.loadTestsFromTestCase(test_case)) test_suite.run(result) result.log_results(logger) @@ -77,12 +77,17 @@ def selftest(logger): count_mismatch = successful_tests != SELFTEST_COUNT if result.wasSuccessful() and count_mismatch: # only print this if all tests succeeded - logger.error("self test count (%d != %d) mismatch, either test discovery is broken or a test was added " - "without updating borg.selftest", - successful_tests, SELFTEST_COUNT) + logger.error( + "self test count (%d != %d) mismatch, either test discovery is broken or a test was added " + "without updating borg.selftest", + successful_tests, + SELFTEST_COUNT, + ) if not result.wasSuccessful() or count_mismatch: - logger.error("self test failed\n" - "Could be a bug either in Borg, the package / distribution you use, your OS or your hardware.") + logger.error( + "self test failed\n" + "Could be a bug either in Borg, the package / distribution you use, your OS or your hardware." + ) sys.exit(2) assert False, "sanity assertion failed: ran beyond sys.exit()" selftest_elapsed = time.perf_counter() - selftest_started diff --git a/src/borg/shellpattern.py b/src/borg/shellpattern.py index c9447704b..7427a4c89 100644 --- a/src/borg/shellpattern.py +++ b/src/borg/shellpattern.py @@ -33,7 +33,7 @@ def translate(pat, match_end=r"\Z"): if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep: # **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands # for the platform-specific path separator - res += fr"(?:[^\{sep}]*\{sep})*" + res += rf"(?:[^\{sep}]*\{sep})*" i += 2 else: # * == wildcard for name parts (does not cross path separator) diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 0ad942482..c98fd5968 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -2,6 +2,7 @@ import filecmp import functools import os + try: import posix except ImportError: @@ -25,14 +26,14 @@ from ..fuse_impl import llfuse, has_pyfuse3, has_llfuse # Does this version of llfuse support ns precision? -have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False +have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, "st_mtime_ns") if llfuse else False try: from pytest import raises except: # noqa raises = None -has_lchflags = hasattr(os, 'lchflags') or sys.platform.startswith('linux') +has_lchflags = hasattr(os, "lchflags") or sys.platform.startswith("linux") try: with tempfile.NamedTemporaryFile() as file: platform.set_flags(file.name, stat.UF_NODUMP) @@ -40,14 +41,14 @@ has_lchflags = False # The mtime get/set precision varies on different OS and Python versions -if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): +if posix and "HAVE_FUTIMENS" in getattr(posix, "_have_functions", []): st_mtime_ns_round = 0 -elif 'HAVE_UTIMES' in sysconfig.get_config_vars(): +elif "HAVE_UTIMES" in sysconfig.get_config_vars(): st_mtime_ns_round = -6 else: st_mtime_ns_round = -9 -if sys.platform.startswith('netbsd'): +if sys.platform.startswith("netbsd"): st_mtime_ns_round = -4 # only >1 microsecond resolution here? @@ -61,8 +62,8 @@ def unopened_tempfile(): def are_symlinks_supported(): with unopened_tempfile() as filepath: try: - os.symlink('somewhere', filepath) - if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == 'somewhere': + os.symlink("somewhere", filepath) + if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == "somewhere": return True except OSError: pass @@ -71,12 +72,12 @@ def are_symlinks_supported(): @functools.lru_cache def are_hardlinks_supported(): - if not hasattr(os, 'link'): + if not hasattr(os, "link"): # some pythons do not have os.link return False with unopened_tempfile() as file1path, unopened_tempfile() as file2path: - open(file1path, 'w').close() + open(file1path, "w").close() try: os.link(file1path, file2path) stat1 = os.stat(file1path) @@ -108,9 +109,9 @@ def is_utime_fully_supported(): with unopened_tempfile() as filepath: # Some filesystems (such as SSHFS) don't support utime on symlinks if are_symlinks_supported(): - os.symlink('something', filepath) + os.symlink("something", filepath) else: - open(filepath, 'w').close() + open(filepath, "w").close() try: os.utime(filepath, (1000, 2000), follow_symlinks=False) new_stats = os.stat(filepath, follow_symlinks=False) @@ -125,14 +126,14 @@ def is_utime_fully_supported(): @functools.lru_cache def is_birthtime_fully_supported(): - if not hasattr(os.stat_result, 'st_birthtime'): + if not hasattr(os.stat_result, "st_birthtime"): return False with unopened_tempfile() as filepath: # Some filesystems (such as SSHFS) don't support utime on symlinks if are_symlinks_supported(): - os.symlink('something', filepath) + os.symlink("something", filepath) else: - open(filepath, 'w').close() + open(filepath, "w").close() try: birthtime, mtime, atime = 946598400, 946684800, 946771200 os.utime(filepath, (atime, birthtime), follow_symlinks=False) @@ -149,7 +150,7 @@ def is_birthtime_fully_supported(): def no_selinux(x): # selinux fails our FUSE tests, thus ignore selinux xattrs - SELINUX_KEY = b'security.selinux' + SELINUX_KEY = b"security.selinux" if isinstance(x, dict): return {k: v for k, v in x.items() if k != SELINUX_KEY} if isinstance(x, list): @@ -157,8 +158,8 @@ def no_selinux(x): class BaseTestCase(unittest.TestCase): - """ - """ + """ """ + assert_in = unittest.TestCase.assertIn assert_not_in = unittest.TestCase.assertNotIn assert_equal = unittest.TestCase.assertEqual @@ -171,9 +172,9 @@ class BaseTestCase(unittest.TestCase): @contextmanager def assert_creates_file(self, path): - assert not os.path.exists(path), f'{path} should not exist' + assert not os.path.exists(path), f"{path} should not exist" yield - assert os.path.exists(path), f'{path} should exist' + assert os.path.exists(path), f"{path} should exist" def assert_dirs_equal(self, dir1, dir2, **kwargs): diff = filecmp.dircmp(dir1, dir2) @@ -191,10 +192,10 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False, s2 = os.stat(path2, follow_symlinks=False) # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev - attrs = ['st_uid', 'st_gid', 'st_rdev'] + attrs = ["st_uid", "st_gid", "st_rdev"] if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our FUSE filesystem - attrs.append('st_nlink') + attrs.append("st_nlink") d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] d1.insert(1, oct(s1.st_mode)) @@ -225,7 +226,9 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False, d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): - self._assert_dirs_equal_cmp(sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns) + self._assert_dirs_equal_cmp( + sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns + ) @contextmanager def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=False, **kwargs): @@ -247,7 +250,7 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal mountpoint = tempfile.mkdtemp() else: os.mkdir(mountpoint) - args = [f'--repo={location}', 'mount', mountpoint] + list(options) + args = [f"--repo={location}", "mount", mountpoint] + list(options) if os_fork: # Do not spawn, but actually (OS) fork. if os.fork() == 0: @@ -264,12 +267,11 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal # This should never be reached, since it daemonizes, # and the grandchild process exits before cmd() returns. # However, just in case... - print('Fatal: borg mount did not daemonize properly. Force exiting.', - file=sys.stderr, flush=True) + print("Fatal: borg mount did not daemonize properly. Force exiting.", file=sys.stderr, flush=True) os._exit(0) else: self.cmd(*args, fork=fork, **kwargs) - if kwargs.get('exit_code', EXIT_SUCCESS) == EXIT_ERROR: + if kwargs.get("exit_code", EXIT_SUCCESS) == EXIT_ERROR: # If argument `exit_code = EXIT_ERROR`, then this call # is testing the behavior of an unsuccessful mount and # we must not continue, as there is no mount to work @@ -292,7 +294,7 @@ def wait_for_mountstate(self, mountpoint, *, mounted, timeout=5): if os.path.ismount(mountpoint) == mounted: return time.sleep(0.1) - message = 'Waiting for {} of {}'.format('mount' if mounted else 'umount', mountpoint) + message = "Waiting for {} of {}".format("mount" if mounted else "umount", mountpoint) raise TimeoutError(message) @contextmanager @@ -308,17 +310,17 @@ def read_only(self, path): tests are running with root privileges. Instead, the folder is rendered immutable with chattr or chflags, respectively. """ - if sys.platform.startswith('linux'): + if sys.platform.startswith("linux"): cmd_immutable = 'chattr +i "%s"' % path cmd_mutable = 'chattr -i "%s"' % path - elif sys.platform.startswith(('darwin', 'freebsd', 'netbsd', 'openbsd')): + elif sys.platform.startswith(("darwin", "freebsd", "netbsd", "openbsd")): cmd_immutable = 'chflags uchg "%s"' % path cmd_mutable = 'chflags nouchg "%s"' % path - elif sys.platform.startswith('sunos'): # openindiana + elif sys.platform.startswith("sunos"): # openindiana cmd_immutable = 'chmod S+vimmutable "%s"' % path cmd_mutable = 'chmod S-vimmutable "%s"' % path else: - message = 'Testing read-only repos is not supported on platform %s' % sys.platform + message = "Testing read-only repos is not supported on platform %s" % sys.platform self.skipTest(message) try: os.system('LD_PRELOAD= chmod -R ugo-w "%s"' % path) @@ -365,12 +367,13 @@ def __exit__(self, *args, **kw): class FakeInputs: """Simulate multiple user inputs, can be used as input() replacement""" + def __init__(self, inputs): self.inputs = inputs def __call__(self, prompt=None): if prompt is not None: - print(prompt, end='') + print(prompt, end="") try: return self.inputs.pop(0) except IndexError: diff --git a/src/borg/testsuite/archive.py b/src/borg/testsuite/archive.py index cf19a5821..4eef5413f 100644 --- a/src/borg/testsuite/archive.py +++ b/src/borg/testsuite/archive.py @@ -33,64 +33,66 @@ def test_stats_basic(stats): def tests_stats_progress(stats, monkeypatch, columns=80): - monkeypatch.setenv('COLUMNS', str(columns)) + monkeypatch.setenv("COLUMNS", str(columns)) out = StringIO() stats.show_progress(stream=out) - s = '20 B O 20 B U 1 N ' - buf = ' ' * (columns - len(s)) + s = "20 B O 20 B U 1 N " + buf = " " * (columns - len(s)) assert out.getvalue() == s + buf + "\r" out = StringIO() - stats.update(10 ** 3, unique=False) - stats.show_progress(item=Item(path='foo'), final=False, stream=out) - s = '1.02 kB O 20 B U 1 N foo' - buf = ' ' * (columns - len(s)) + stats.update(10**3, unique=False) + stats.show_progress(item=Item(path="foo"), final=False, stream=out) + s = "1.02 kB O 20 B U 1 N foo" + buf = " " * (columns - len(s)) assert out.getvalue() == s + buf + "\r" out = StringIO() - stats.show_progress(item=Item(path='foo'*40), final=False, stream=out) - s = '1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo' - buf = ' ' * (columns - len(s)) + stats.show_progress(item=Item(path="foo" * 40), final=False, stream=out) + s = "1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo" + buf = " " * (columns - len(s)) assert out.getvalue() == s + buf + "\r" def test_stats_format(stats): - assert str(stats) == """\ + assert ( + str(stats) + == """\ Number of files: 1 Original size: 20 B Deduplicated size: 20 B """ + ) s = f"{stats.osize_fmt}" assert s == "20 B" # kind of redundant, but id is variable so we can't match reliably - assert repr(stats) == f'' + assert repr(stats) == f"" def test_stats_progress_json(stats): stats.output_json = True out = StringIO() - stats.show_progress(item=Item(path='foo'), stream=out) + stats.show_progress(item=Item(path="foo"), stream=out) result = json.loads(out.getvalue()) - assert result['type'] == 'archive_progress' - assert isinstance(result['time'], float) - assert result['finished'] is False - assert result['path'] == 'foo' - assert result['original_size'] == 20 - assert result['nfiles'] == 1 + assert result["type"] == "archive_progress" + assert isinstance(result["time"], float) + assert result["finished"] is False + assert result["path"] == "foo" + assert result["original_size"] == 20 + assert result["nfiles"] == 1 out = StringIO() stats.show_progress(stream=out, final=True) result = json.loads(out.getvalue()) - assert result['type'] == 'archive_progress' - assert isinstance(result['time'], float) - assert result['finished'] is True # see #6570 - assert 'path' not in result - assert 'original_size' not in result - assert 'nfiles' not in result + assert result["type"] == "archive_progress" + assert isinstance(result["time"], float) + assert result["finished"] is True # see #6570 + assert "path" not in result + assert "original_size" not in result + assert "nfiles" not in result class MockCache: - class MockRepo: def async_response(self, wait=True): pass @@ -105,30 +107,24 @@ def add_chunk(self, id, chunk, stats=None, wait=True): class ArchiveTimestampTestCase(BaseTestCase): - def _test_timestamp_parsing(self, isoformat, expected): repository = Mock() key = PlaintextKey(repository) manifest = Manifest(repository, key) - a = Archive(repository, key, manifest, 'test', create=True) + a = Archive(repository, key, manifest, "test", create=True) a.metadata = ArchiveItem(time=isoformat) self.assert_equal(a.ts, expected) def test_with_microseconds(self): - self._test_timestamp_parsing( - '1970-01-01T00:00:01.000001', - datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc)) + self._test_timestamp_parsing("1970-01-01T00:00:01.000001", datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc)) def test_without_microseconds(self): - self._test_timestamp_parsing( - '1970-01-01T00:00:01', - datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc)) + self._test_timestamp_parsing("1970-01-01T00:00:01", datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc)) class ChunkBufferTestCase(BaseTestCase): - def test(self): - data = [Item(path='p1'), Item(path='p2')] + data = [Item(path="p1"), Item(path="p2")] cache = MockCache() key = PlaintextKey(None) chunks = CacheChunkBuffer(cache, key, None) @@ -144,7 +140,7 @@ def test(self): def test_partial(self): big = "0123456789abcdefghijklmnopqrstuvwxyz" * 25000 - data = [Item(path='full', source=big), Item(path='partial', source=big)] + data = [Item(path="full", source=big), Item(path="partial", source=big)] cache = MockCache() key = PlaintextKey(None) chunks = CacheChunkBuffer(cache, key, None) @@ -165,12 +161,11 @@ def test_partial(self): class RobustUnpackerTestCase(BaseTestCase): - def make_chunks(self, items): - return b''.join(msgpack.packb({'path': item}) for item in items) + return b"".join(msgpack.packb({"path": item}) for item in items) def _validator(self, value): - return isinstance(value, dict) and value.get('path') in ('foo', 'bar', 'boo', 'baz') + return isinstance(value, dict) and value.get("path") in ("foo", "bar", "boo", "baz") def process(self, input): unpacker = RobustUnpacker(validator=self._validator, item_keys=ITEM_KEYS) @@ -185,14 +180,14 @@ def process(self, input): return result def test_extra_garbage_no_sync(self): - chunks = [(False, [self.make_chunks(['foo', 'bar'])]), - (False, [b'garbage'] + [self.make_chunks(['boo', 'baz'])])] + chunks = [ + (False, [self.make_chunks(["foo", "bar"])]), + (False, [b"garbage"] + [self.make_chunks(["boo", "baz"])]), + ] result = self.process(chunks) - self.assert_equal(result, [ - {'path': 'foo'}, {'path': 'bar'}, - 103, 97, 114, 98, 97, 103, 101, - {'path': 'boo'}, - {'path': 'baz'}]) + self.assert_equal( + result, [{"path": "foo"}, {"path": "bar"}, 103, 97, 114, 98, 97, 103, 101, {"path": "boo"}, {"path": "baz"}] + ) def split(self, left, length): parts = [] @@ -202,22 +197,22 @@ def split(self, left, length): return parts def test_correct_stream(self): - chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 2) + chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 2) input = [(False, chunks)] result = self.process(input) - self.assert_equal(result, [{'path': 'foo'}, {'path': 'bar'}, {'path': 'boo'}, {'path': 'baz'}]) + self.assert_equal(result, [{"path": "foo"}, {"path": "bar"}, {"path": "boo"}, {"path": "baz"}]) def test_missing_chunk(self): - chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4) + chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4) input = [(False, chunks[:3]), (True, chunks[4:])] result = self.process(input) - self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}]) + self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}]) def test_corrupt_chunk(self): - chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4) - input = [(False, chunks[:3]), (True, [b'gar', b'bage'] + chunks[3:])] + chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4) + input = [(False, chunks[:3]), (True, [b"gar", b"bage"] + chunks[3:])] result = self.process(input) - self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}]) + self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}]) @pytest.fixture @@ -225,12 +220,17 @@ def item_keys_serialized(): return [msgpack.packb(name) for name in ITEM_KEYS] -@pytest.mark.parametrize('packed', - [b'', b'x', b'foobar', ] + - [msgpack.packb(o) for o in ( - [None, 0, 0.0, False, '', {}, [], ()] + - [42, 23.42, True, b'foobar', {b'foo': b'bar'}, [b'foo', b'bar'], (b'foo', b'bar')] - )]) +@pytest.mark.parametrize( + "packed", + [b"", b"x", b"foobar"] + + [ + msgpack.packb(o) + for o in ( + [None, 0, 0.0, False, "", {}, [], ()] + + [42, 23.42, True, b"foobar", {b"foo": b"bar"}, [b"foo", b"bar"], (b"foo", b"bar")] + ) + ], +) def test_invalid_msgpacked_item(packed, item_keys_serialized): assert not valid_msgpacked_dict(packed, item_keys_serialized) @@ -239,20 +239,25 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized): IK = sorted(list(ITEM_KEYS)) -@pytest.mark.parametrize('packed', - [msgpack.packb(o) for o in [ - {'path': b'/a/b/c'}, # small (different msgpack mapping type!) - OrderedDict((k, b'') for k in IK), # as big (key count) as it gets - OrderedDict((k, b'x' * 1000) for k in IK), # as big (key count and volume) as it gets - ]]) +@pytest.mark.parametrize( + "packed", + [ + msgpack.packb(o) + for o in [ + {"path": b"/a/b/c"}, # small (different msgpack mapping type!) + OrderedDict((k, b"") for k in IK), # as big (key count) as it gets + OrderedDict((k, b"x" * 1000) for k in IK), # as big (key count and volume) as it gets + ] + ], +) def test_valid_msgpacked_items(packed, item_keys_serialized): assert valid_msgpacked_dict(packed, item_keys_serialized) def test_key_length_msgpacked_items(): - key = 'x' * 32 # 31 bytes is the limit for fixstr msgpack type - data = {key: b''} - item_keys_serialized = [msgpack.packb(key), ] + key = "x" * 32 # 31 bytes is the limit for fixstr msgpack type + data = {key: b""} + item_keys_serialized = [msgpack.packb(key)] assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized) @@ -277,7 +282,7 @@ def __next__(self): normal_iterator = Iterator(StopIteration) for _ in backup_io_iter(normal_iterator): - assert False, 'StopIteration handled incorrectly' + assert False, "StopIteration handled incorrectly" def test_get_item_uid_gid(): @@ -288,7 +293,7 @@ def test_get_item_uid_gid(): user0, group0 = uid2user(0), gid2group(0) # this is intentionally a "strange" item, with not matching ids/names. - item = Item(path='filename', uid=1, gid=2, user=user0, group=group0) + item = Item(path="filename", uid=1, gid=2, user=user0, group=group0) uid, gid = get_item_uid_gid(item, numeric=False) # these are found via a name-to-id lookup @@ -306,7 +311,7 @@ def test_get_item_uid_gid(): assert gid == 4 # item metadata broken, has negative ids. - item = Item(path='filename', uid=-1, gid=-2, user=user0, group=group0) + item = Item(path="filename", uid=-1, gid=-2, user=user0, group=group0) uid, gid = get_item_uid_gid(item, numeric=True) # use the uid/gid defaults (which both default to 0). @@ -319,7 +324,7 @@ def test_get_item_uid_gid(): assert gid == 6 # item metadata broken, has negative ids and non-existing user/group names. - item = Item(path='filename', uid=-3, gid=-4, user='udoesnotexist', group='gdoesnotexist') + item = Item(path="filename", uid=-3, gid=-4, user="udoesnotexist", group="gdoesnotexist") uid, gid = get_item_uid_gid(item, numeric=False) # use the uid/gid defaults (which both default to 0). @@ -332,7 +337,7 @@ def test_get_item_uid_gid(): assert gid == 8 # item metadata has valid uid/gid, but non-existing user/group names. - item = Item(path='filename', uid=9, gid=10, user='udoesnotexist', group='gdoesnotexist') + item = Item(path="filename", uid=9, gid=10, user="udoesnotexist", group="gdoesnotexist") uid, gid = get_item_uid_gid(item, numeric=False) # because user/group name does not exist here, use valid numeric ids from item metadata. diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 4a29440d4..63bb8beca 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -53,32 +53,38 @@ from ..repository import Repository from . import has_lchflags, llfuse from . import BaseTestCase, changedir, environment_variable, no_selinux -from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported +from . import ( + are_symlinks_supported, + are_hardlinks_supported, + are_fifos_supported, + is_utime_fully_supported, + is_birthtime_fully_supported, +) from .platform import fakeroot_detected, is_darwin from . import key -RK_ENCRYPTION = '--encryption=repokey-aes-ocb' -KF_ENCRYPTION = '--encryption=keyfile-chacha20-poly1305' +RK_ENCRYPTION = "--encryption=repokey-aes-ocb" +KF_ENCRYPTION = "--encryption=keyfile-chacha20-poly1305" -src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): +def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b"", binary_output=False, **kw): if fork: try: if exe is None: - borg = (sys.executable, '-m', 'borg.archiver') + borg = (sys.executable, "-m", "borg.archiver") elif isinstance(exe, str): - borg = (exe, ) + borg = (exe,) elif not isinstance(exe, tuple): - raise ValueError('exe must be None, a tuple or a str') + raise ValueError("exe must be None, a tuple or a str") output = subprocess.check_output(borg + args, stderr=subprocess.STDOUT, input=input) ret = 0 except subprocess.CalledProcessError as e: output = e.output ret = e.returncode except SystemExit as e: # possibly raised by argparse - output = '' + output = "" ret = e.code if binary_output: return ret, output @@ -91,7 +97,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_outpu sys.stdin.buffer = BytesIO(input) output = BytesIO() # Always use utf-8 here, to simply .decode() below - output_text = sys.stdout = sys.stderr = io.TextIOWrapper(output, encoding='utf-8') + output_text = sys.stdout = sys.stderr = io.TextIOWrapper(output, encoding="utf-8") if archiver is None: archiver = Archiver() archiver.prerun_checks = lambda *args: None @@ -113,50 +119,51 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_outpu def have_gnutar(): - if not shutil.which('tar'): + if not shutil.which("tar"): return False - popen = subprocess.Popen(['tar', '--version'], stdout=subprocess.PIPE) + popen = subprocess.Popen(["tar", "--version"], stdout=subprocess.PIPE) stdout, stderr = popen.communicate() - return b'GNU tar' in stdout + return b"GNU tar" in stdout # check if the binary "borg.exe" is available (for local testing a symlink to virtualenv/bin/borg should do) try: - exec_cmd('help', exe='borg.exe', fork=True) - BORG_EXES = ['python', 'binary', ] + exec_cmd("help", exe="borg.exe", fork=True) + BORG_EXES = ["python", "binary"] except FileNotFoundError: - BORG_EXES = ['python', ] + BORG_EXES = ["python"] @pytest.fixture(params=BORG_EXES) def cmd(request): - if request.param == 'python': + if request.param == "python": exe = None - elif request.param == 'binary': - exe = 'borg.exe' + elif request.param == "binary": + exe = "borg.exe" else: raise ValueError("param must be 'python' or 'binary'") def exec_fn(*args, **kw): return exec_cmd(*args, exe=exe, fork=True, **kw) + return exec_fn def test_return_codes(cmd, tmpdir): - repo = tmpdir.mkdir('repo') - input = tmpdir.mkdir('input') - output = tmpdir.mkdir('output') - input.join('test_file').write('content') - rc, out = cmd('--repo=%s' % str(repo), 'rcreate', '--encryption=none') + repo = tmpdir.mkdir("repo") + input = tmpdir.mkdir("input") + output = tmpdir.mkdir("output") + input.join("test_file").write("content") + rc, out = cmd("--repo=%s" % str(repo), "rcreate", "--encryption=none") assert rc == EXIT_SUCCESS - rc, out = cmd('--repo=%s' % repo, 'create', 'archive', str(input)) + rc, out = cmd("--repo=%s" % repo, "create", "archive", str(input)) assert rc == EXIT_SUCCESS with changedir(str(output)): - rc, out = cmd('--repo=%s' % repo, 'extract', 'archive') + rc, out = cmd("--repo=%s" % repo, "extract", "archive") assert rc == EXIT_SUCCESS - rc, out = cmd('--repo=%s' % repo, 'extract', 'archive', 'does/not/match') + rc, out = cmd("--repo=%s" % repo, "extract", "archive", "does/not/match") assert rc == EXIT_WARNING # pattern did not match - rc, out = cmd('--repo=%s' % repo, 'create', 'archive', str(input)) + rc, out = cmd("--repo=%s" % repo, "create", "archive", str(input)) assert rc == EXIT_ERROR # duplicate archive name @@ -174,7 +181,7 @@ def test_return_codes(cmd, tmpdir): if the directory does not exist, the test will be skipped. """ -DF_MOUNT = '/tmp/borg-mount' +DF_MOUNT = "/tmp/borg-mount" @pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT) @@ -188,24 +195,24 @@ def make_files(dir, count, size, rnd=True): size = random.randint(1, size) for i in range(count): fn = os.path.join(dir, "file%03d" % i) - with open(fn, 'wb') as f: + with open(fn, "wb") as f: data = os.urandom(size) f.write(data) - with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='YES'): + with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING="YES"): mount = DF_MOUNT assert os.path.exists(mount) - repo = os.path.join(mount, 'repo') - input = os.path.join(mount, 'input') - reserve = os.path.join(mount, 'reserve') + repo = os.path.join(mount, "repo") + input = os.path.join(mount, "input") + reserve = os.path.join(mount, "reserve") for j in range(100): shutil.rmtree(repo, ignore_errors=True) shutil.rmtree(input, ignore_errors=True) # keep some space and some inodes in reserve that we can free up later: make_files(reserve, 80, 100000, rnd=False) - rc, out = cmd(f'--repo={repo}', 'rcreate') + rc, out = cmd(f"--repo={repo}", "rcreate") if rc != EXIT_SUCCESS: - print('rcreate', rc, out) + print("rcreate", rc, out) assert rc == EXIT_SUCCESS try: success, i = True, 0 @@ -219,58 +226,58 @@ def make_files(dir, count, size, rnd=True): break raise try: - rc, out = cmd('--repo=%s' % repo, 'create', 'test%03d' % i, input) + rc, out = cmd("--repo=%s" % repo, "create", "test%03d" % i, input) success = rc == EXIT_SUCCESS if not success: - print('create', rc, out) + print("create", rc, out) finally: # make sure repo is not locked - shutil.rmtree(os.path.join(repo, 'lock.exclusive'), ignore_errors=True) - os.remove(os.path.join(repo, 'lock.roster')) + shutil.rmtree(os.path.join(repo, "lock.exclusive"), ignore_errors=True) + os.remove(os.path.join(repo, "lock.roster")) finally: # now some error happened, likely we are out of disk space. # free some space so we can expect borg to be able to work normally: shutil.rmtree(reserve, ignore_errors=True) - rc, out = cmd(f'--repo={repo}', 'rlist') + rc, out = cmd(f"--repo={repo}", "rlist") if rc != EXIT_SUCCESS: - print('rlist', rc, out) - rc, out = cmd(f'--repo={repo}', 'check', '--repair') + print("rlist", rc, out) + rc, out = cmd(f"--repo={repo}", "check", "--repair") if rc != EXIT_SUCCESS: - print('check', rc, out) + print("check", rc, out) assert rc == EXIT_SUCCESS class ArchiverTestCaseBase(BaseTestCase): EXE = None # python source based FORK_DEFAULT = False - prefix = '' + prefix = "" def setUp(self): - os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES' - os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' - os.environ['BORG_PASSPHRASE'] = 'waytooeasyonlyfortests' - os.environ['BORG_SELFTEST'] = 'disabled' + os.environ["BORG_CHECK_I_KNOW_WHAT_I_AM_DOING"] = "YES" + os.environ["BORG_DELETE_I_KNOW_WHAT_I_AM_DOING"] = "YES" + os.environ["BORG_PASSPHRASE"] = "waytooeasyonlyfortests" + os.environ["BORG_SELFTEST"] = "disabled" self.archiver = not self.FORK_DEFAULT and Archiver() or None self.tmpdir = tempfile.mkdtemp() - self.repository_path = os.path.join(self.tmpdir, 'repository') + self.repository_path = os.path.join(self.tmpdir, "repository") self.repository_location = self.prefix + self.repository_path - self.input_path = os.path.join(self.tmpdir, 'input') - self.output_path = os.path.join(self.tmpdir, 'output') - self.keys_path = os.path.join(self.tmpdir, 'keys') - self.cache_path = os.path.join(self.tmpdir, 'cache') - self.exclude_file_path = os.path.join(self.tmpdir, 'excludes') - self.patterns_file_path = os.path.join(self.tmpdir, 'patterns') - os.environ['BORG_KEYS_DIR'] = self.keys_path - os.environ['BORG_CACHE_DIR'] = self.cache_path + self.input_path = os.path.join(self.tmpdir, "input") + self.output_path = os.path.join(self.tmpdir, "output") + self.keys_path = os.path.join(self.tmpdir, "keys") + self.cache_path = os.path.join(self.tmpdir, "cache") + self.exclude_file_path = os.path.join(self.tmpdir, "excludes") + self.patterns_file_path = os.path.join(self.tmpdir, "patterns") + os.environ["BORG_KEYS_DIR"] = self.keys_path + os.environ["BORG_CACHE_DIR"] = self.cache_path os.mkdir(self.input_path) os.chmod(self.input_path, 0o777) # avoid troubles with fakeroot / FUSE os.mkdir(self.output_path) os.mkdir(self.keys_path) os.mkdir(self.cache_path) - with open(self.exclude_file_path, 'wb') as fd: - fd.write(b'input/file2\n# A comment line, then a blank line\n\n') - with open(self.patterns_file_path, 'wb') as fd: - fd.write(b'+input/file_important\n- input/file*\n# A comment line, then a blank line\n\n') + with open(self.exclude_file_path, "wb") as fd: + fd.write(b"input/file2\n# A comment line, then a blank line\n\n") + with open(self.patterns_file_path, "wb") as fd: + fd.write(b"+input/file_important\n- input/file*\n# A comment line, then a blank line\n\n") self._old_wd = os.getcwd() os.chdir(self.tmpdir) @@ -281,9 +288,9 @@ def tearDown(self): setup_logging() def cmd(self, *args, **kw): - exit_code = kw.pop('exit_code', 0) - fork = kw.pop('fork', None) - binary_output = kw.get('binary_output', False) + exit_code = kw.pop("exit_code", 0) + fork = kw.pop("fork", None) + binary_output = kw.get("binary_output", False) if fork is None: fork = self.FORK_DEFAULT ret, output = exec_cmd(*args, fork=fork, exe=self.EXE, archiver=self.archiver, **kw) @@ -293,13 +300,12 @@ def cmd(self, *args, **kw): # if tests are run with the pure-python msgpack, there will be warnings about # this in the output, which would make a lot of tests fail. pp_msg = PURE_PYTHON_MSGPACK_WARNING.encode() if binary_output else PURE_PYTHON_MSGPACK_WARNING - empty = b'' if binary_output else '' - output = empty.join(line for line in output.splitlines(keepends=True) - if pp_msg not in line) + empty = b"" if binary_output else "" + output = empty.join(line for line in output.splitlines(keepends=True) if pp_msg not in line) return output def create_src_archive(self, name): - self.cmd(f'--repo={self.repository_location}', 'create', '--compression=lz4', name, src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "--compression=lz4", name, src_dir) def open_archive(self, name): repository = Repository(self.repository_path, exclusive=True) @@ -312,42 +318,40 @@ def open_repository(self): return Repository(self.repository_path, exclusive=True) def create_regular_file(self, name, size=0, contents=None): - assert not (size != 0 and contents and len(contents) != size), 'size and contents do not match' + assert not (size != 0 and contents and len(contents) != size), "size and contents do not match" filename = os.path.join(self.input_path, name) if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) - with open(filename, 'wb') as fd: + with open(filename, "wb") as fd: if contents is None: - contents = b'X' * size + contents = b"X" * size fd.write(contents) def create_test_files(self, create_hardlinks=True): - """Create a minimal test case including all supported file types - """ + """Create a minimal test case including all supported file types""" # File - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('flagfile', size=1024) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("flagfile", size=1024) # Directory - self.create_regular_file('dir2/file2', size=1024 * 80) + self.create_regular_file("dir2/file2", size=1024 * 80) # File mode - os.chmod('input/file1', 0o4755) + os.chmod("input/file1", 0o4755) # Hard link if are_hardlinks_supported() and create_hardlinks: - os.link(os.path.join(self.input_path, 'file1'), - os.path.join(self.input_path, 'hardlink')) + os.link(os.path.join(self.input_path, "file1"), os.path.join(self.input_path, "hardlink")) # Symlink if are_symlinks_supported(): - os.symlink('somewhere', os.path.join(self.input_path, 'link1')) - self.create_regular_file('fusexattr', size=1) + os.symlink("somewhere", os.path.join(self.input_path, "link1")) + self.create_regular_file("fusexattr", size=1) if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - fn = os.fsencode(os.path.join(self.input_path, 'fusexattr')) + fn = os.fsencode(os.path.join(self.input_path, "fusexattr")) # ironically, due to the way how fakeroot works, comparing FUSE file xattrs to orig file xattrs # will FAIL if fakeroot supports xattrs, thus we only set the xattr if XATTR_FAKEROOT is False. # This is because fakeroot with xattr-support does not propagate xattrs of the underlying file # into "fakeroot space". Because the xattrs exposed by borgfs are these of an underlying file # (from fakeroots point of view) they are invisible to the test process inside the fakeroot. - xattr.setxattr(fn, b'user.foo', b'bar') - xattr.setxattr(fn, b'user.empty', b'') + xattr.setxattr(fn, b"user.foo", b"bar") + xattr.setxattr(fn, b"user.empty", b"") # XXX this always fails for me # ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot # same for newer ubuntu and centos. @@ -356,18 +360,18 @@ def create_test_files(self, create_hardlinks=True): # xattr.setxattr(os.path.join(self.input_path, 'link1'), b'user.foo_symlink', b'bar_symlink', follow_symlinks=False) # FIFO node if are_fifos_supported(): - os.mkfifo(os.path.join(self.input_path, 'fifo1')) + os.mkfifo(os.path.join(self.input_path, "fifo1")) if has_lchflags: - platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) + platform.set_flags(os.path.join(self.input_path, "flagfile"), stat.UF_NODUMP) try: # Block device - os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) + os.mknod("input/bdev", 0o600 | stat.S_IFBLK, os.makedev(10, 20)) # Char device - os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40)) + os.mknod("input/cdev", 0o600 | stat.S_IFCHR, os.makedev(30, 40)) # File mode - os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents + os.chmod("input/dir2", 0o555) # if we take away write perms, we need root to remove contents # File owner - os.chown('input/file1', 100, 200) # raises OSError invalid argument on cygwin + os.chown("input/file1", 100, 200) # raises OSError invalid argument on cygwin have_root = True # we have (fake)root except PermissionError: have_root = False @@ -377,12 +381,12 @@ def create_test_files(self, create_hardlinks=True): raise have_root = False time.sleep(1) # "empty" must have newer timestamp than other files - self.create_regular_file('empty', size=0) + self.create_regular_file("empty", size=0) return have_root class ArchiverTestCase(ArchiverTestCaseBase): - requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') + requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") def get_security_dir(self): repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) @@ -391,61 +395,64 @@ def get_security_dir(self): def test_basic_functionality(self): have_root = self.create_test_files() # fork required to test show-rc output - output = self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION, '--show-version', '--show-rc', fork=True) - self.assert_in('borgbackup version', output) - self.assert_in('terminating with success status, rc 0', output) - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude-nodump', 'test', 'input') - output = self.cmd(f'--repo={self.repository_location}', 'create', '--exclude-nodump', '--stats', 'test.2', 'input') - self.assert_in('Archive name: test.2', output) - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - list_output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--short') - self.assert_in('test', list_output) - self.assert_in('test.2', list_output) + output = self.cmd( + f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION, "--show-version", "--show-rc", fork=True + ) + self.assert_in("borgbackup version", output) + self.assert_in("terminating with success status, rc 0", output) + self.cmd(f"--repo={self.repository_location}", "create", "--exclude-nodump", "test", "input") + output = self.cmd( + f"--repo={self.repository_location}", "create", "--exclude-nodump", "--stats", "test.2", "input" + ) + self.assert_in("Archive name: test.2", output) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + list_output = self.cmd(f"--repo={self.repository_location}", "rlist", "--short") + self.assert_in("test", list_output) + self.assert_in("test.2", list_output) expected = [ - 'input', - 'input/bdev', - 'input/cdev', - 'input/dir2', - 'input/dir2/file2', - 'input/empty', - 'input/file1', - 'input/flagfile', + "input", + "input/bdev", + "input/cdev", + "input/dir2", + "input/dir2/file2", + "input/empty", + "input/file1", + "input/flagfile", ] if are_fifos_supported(): - expected.append('input/fifo1') + expected.append("input/fifo1") if are_symlinks_supported(): - expected.append('input/link1') + expected.append("input/link1") if are_hardlinks_supported(): - expected.append('input/hardlink') + expected.append("input/hardlink") if not have_root: # we could not create these device files without (fake)root - expected.remove('input/bdev') - expected.remove('input/cdev') + expected.remove("input/bdev") + expected.remove("input/cdev") if has_lchflags: # remove the file we did not backup, so input and output become equal - expected.remove('input/flagfile') # this file is UF_NODUMP - os.remove(os.path.join('input', 'flagfile')) - list_output = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--short') + expected.remove("input/flagfile") # this file is UF_NODUMP + os.remove(os.path.join("input", "flagfile")) + list_output = self.cmd(f"--repo={self.repository_location}", "list", "test", "--short") for name in expected: self.assert_in(name, list_output) - self.assert_dirs_equal('input', 'output/input') - info_output = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test') + self.assert_dirs_equal("input", "output/input") + info_output = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test") item_count = 5 if has_lchflags else 6 # one file is UF_NODUMP - self.assert_in('Number of files: %d' % item_count, info_output) + self.assert_in("Number of files: %d" % item_count, info_output) shutil.rmtree(self.cache_path) - info_output2 = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test') + info_output2 = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test") def filter(output): # filter for interesting "info" output, ignore cache rebuilding related stuff - prefixes = ['Name:', 'Fingerprint:', 'Number of files:', 'This archive:', - 'All archives:', 'Chunk index:', ] + prefixes = ["Name:", "Fingerprint:", "Number of files:", "This archive:", "All archives:", "Chunk index:"] result = [] for line in output.splitlines(): for prefix in prefixes: if line.startswith(prefix): result.append(line) - return '\n'.join(result) + return "\n".join(result) # the interesting parts of info_output2 and info_output should be same self.assert_equal(filter(info_output), filter(info_output2)) @@ -453,82 +460,84 @@ def filter(output): @requires_hardlinks def test_create_duplicate_root(self): # setup for #5603 - path_a = os.path.join(self.input_path, 'a') - path_b = os.path.join(self.input_path, 'b') + path_a = os.path.join(self.input_path, "a") + path_b = os.path.join(self.input_path, "b") os.mkdir(path_a) os.mkdir(path_b) - hl_a = os.path.join(path_a, 'hardlink') - hl_b = os.path.join(path_b, 'hardlink') - self.create_regular_file(hl_a, contents=b'123456') + hl_a = os.path.join(path_a, "hardlink") + hl_b = os.path.join(path_b, "hardlink") + self.create_regular_file(hl_a, contents=b"123456") os.link(hl_a, hl_b) - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', 'input') # give input twice! + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "input") # give input twice! # test if created archive has 'input' contents twice: - archive_list = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines') - paths = [json.loads(line)['path'] for line in archive_list.split('\n') if line] + archive_list = self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines") + paths = [json.loads(line)["path"] for line in archive_list.split("\n") if line] # we have all fs items exactly once! - assert sorted(paths) == ['input', 'input/a', 'input/a/hardlink', 'input/b', 'input/b/hardlink'] + assert sorted(paths) == ["input", "input/a", "input/a/hardlink", "input/b", "input/b/hardlink"] def test_init_parent_dirs(self): - parent_path = os.path.join(self.tmpdir, 'parent1', 'parent2') - repository_path = os.path.join(parent_path, 'repository') + parent_path = os.path.join(self.tmpdir, "parent1", "parent2") + repository_path = os.path.join(parent_path, "repository") repository_location = self.prefix + repository_path with pytest.raises(Repository.ParentPathDoesNotExist): # normal borg init does NOT create missing parent dirs - self.cmd(f'--repo={repository_location}', 'rcreate', '--encryption=none') + self.cmd(f"--repo={repository_location}", "rcreate", "--encryption=none") # but if told so, it does: - self.cmd(f'--repo={repository_location}', 'rcreate', '--encryption=none', '--make-parent-dirs') + self.cmd(f"--repo={repository_location}", "rcreate", "--encryption=none", "--make-parent-dirs") assert os.path.exists(parent_path) def test_unix_socket(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) try: sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(os.path.join(self.input_path, 'unix-socket')) + sock.bind(os.path.join(self.input_path, "unix-socket")) except PermissionError as err: if err.errno == errno.EPERM: - pytest.skip('unix sockets disabled or not supported') + pytest.skip("unix sockets disabled or not supported") elif err.errno == errno.EACCES: - pytest.skip('permission denied to create unix sockets') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + pytest.skip("permission denied to create unix sockets") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") sock.close() - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - assert not os.path.exists('input/unix-socket') + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + assert not os.path.exists("input/unix-socket") - @pytest.mark.skipif(not are_symlinks_supported(), reason='symlinks not supported') + @pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") def test_symlink_extract(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - assert os.readlink('input/link1') == 'somewhere' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + assert os.readlink("input/link1") == "somewhere" - @pytest.mark.skipif(not are_symlinks_supported() or not are_hardlinks_supported() or is_darwin, - reason='symlinks or hardlinks or hardlinked symlinks not supported') + @pytest.mark.skipif( + not are_symlinks_supported() or not are_hardlinks_supported() or is_darwin, + reason="symlinks or hardlinks or hardlinked symlinks not supported", + ) def test_hardlinked_symlinks_extract(self): - self.create_regular_file('target', size=1024) - with changedir('input'): - os.symlink('target', 'symlink1') - os.link('symlink1', 'symlink2', follow_symlinks=False) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test') + self.create_regular_file("target", size=1024) + with changedir("input"): + os.symlink("target", "symlink1") + os.link("symlink1", "symlink2", follow_symlinks=False) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test") print(output) - with changedir('input'): - assert os.path.exists('target') - assert os.readlink('symlink1') == 'target' - assert os.readlink('symlink2') == 'target' - st1 = os.stat('symlink1', follow_symlinks=False) - st2 = os.stat('symlink2', follow_symlinks=False) + with changedir("input"): + assert os.path.exists("target") + assert os.readlink("symlink1") == "target" + assert os.readlink("symlink2") == "target" + st1 = os.stat("symlink1", follow_symlinks=False) + st2 = os.stat("symlink2", follow_symlinks=False) assert st1.st_nlink == 2 assert st2.st_nlink == 2 assert st1.st_ino == st2.st_ino assert st1.st_size == st2.st_size - @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') + @pytest.mark.skipif(not is_utime_fully_supported(), reason="cannot properly setup and execute test without utime") def test_atime(self): def has_noatime(some_file): atime_before = os.stat(some_file).st_atime_ns @@ -544,14 +553,14 @@ def has_noatime(some_file): self.create_test_files() atime, mtime = 123456780, 234567890 - have_noatime = has_noatime('input/file1') - os.utime('input/file1', (atime, mtime)) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '--atime', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - sti = os.stat('input/file1') - sto = os.stat('output/input/file1') + have_noatime = has_noatime("input/file1") + os.utime("input/file1", (atime, mtime)) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "--atime", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + sti = os.stat("input/file1") + sto = os.stat("output/input/file1") assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 if have_noatime: assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9 @@ -559,35 +568,39 @@ def has_noatime(some_file): # it touched the input file's atime while backing it up assert sto.st_atime_ns == atime * 1e9 - @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') - @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') + @pytest.mark.skipif(not is_utime_fully_supported(), reason="cannot properly setup and execute test without utime") + @pytest.mark.skipif( + not is_birthtime_fully_supported(), reason="cannot properly setup and execute test without birthtime" + ) def test_birthtime(self): self.create_test_files() birthtime, mtime, atime = 946598400, 946684800, 946771200 - os.utime('input/file1', (atime, birthtime)) - os.utime('input/file1', (atime, mtime)) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - sti = os.stat('input/file1') - sto = os.stat('output/input/file1') + os.utime("input/file1", (atime, birthtime)) + os.utime("input/file1", (atime, mtime)) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + sti = os.stat("input/file1") + sto = os.stat("output/input/file1") assert int(sti.st_birthtime * 1e9) == int(sto.st_birthtime * 1e9) == birthtime * 1e9 assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 - @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') - @pytest.mark.skipif(not is_birthtime_fully_supported(), reason='cannot properly setup and execute test without birthtime') + @pytest.mark.skipif(not is_utime_fully_supported(), reason="cannot properly setup and execute test without utime") + @pytest.mark.skipif( + not is_birthtime_fully_supported(), reason="cannot properly setup and execute test without birthtime" + ) def test_nobirthtime(self): self.create_test_files() birthtime, mtime, atime = 946598400, 946684800, 946771200 - os.utime('input/file1', (atime, birthtime)) - os.utime('input/file1', (atime, mtime)) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '--nobirthtime') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - sti = os.stat('input/file1') - sto = os.stat('output/input/file1') + os.utime("input/file1", (atime, birthtime)) + os.utime("input/file1", (atime, mtime)) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "--nobirthtime") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + sti = os.stat("input/file1") + sto = os.stat("output/input/file1") assert int(sti.st_birthtime * 1e9) == birthtime * 1e9 assert int(sto.st_birthtime * 1e9) == mtime * 1e9 assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 @@ -598,9 +611,9 @@ def _extract_repository_id(self, path): def _set_repository_id(self, path, id): config = ConfigParser(interpolation=None) - config.read(os.path.join(path, 'config')) - config.set('repository', 'id', bin_to_hex(id)) - with open(os.path.join(path, 'config'), 'w') as fd: + config.read(os.path.join(path, "config")) + config.set("repository", "id", bin_to_hex(id)) + with open(os.path.join(path, "config"), "w") as fd: config.write(fd) with Repository(self.repository_path) as repository: return repository.id @@ -610,10 +623,10 @@ def is_sparse(fn, total_size, hole_size): st = os.stat(fn) assert st.st_size == total_size sparse = True - if sparse and hasattr(st, 'st_blocks') and st.st_blocks * 512 >= st.st_size: + if sparse and hasattr(st, "st_blocks") and st.st_blocks * 512 >= st.st_size: sparse = False if sparse and has_seek_hole: - with open(fn, 'rb') as fd: + with open(fn, "rb") as fd: # only check if the first hole is as expected, because the 2nd hole check # is problematic on xfs due to its "dynamic speculative EOF preallocation try: @@ -626,11 +639,11 @@ def is_sparse(fn, total_size, hole_size): sparse = False return sparse - filename = os.path.join(self.input_path, 'sparse') - content = b'foobar' + filename = os.path.join(self.input_path, "sparse") + content = b"foobar" hole_size = 5 * (1 << CHUNK_MAX_EXP) # 5 full chunker buffers total_size = hole_size + len(content) + hole_size - with open(filename, 'wb') as fd: + with open(filename, "wb") as fd: # create a file that has a hole at the beginning and end (if the # OS and filesystem supports sparse files) fd.seek(hole_size, 1) @@ -643,1140 +656,1269 @@ def is_sparse(fn, total_size, hole_size): if sparse_support: # we could create a sparse input file, so creating a backup of it and # extracting it again (as sparse) should also work: - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") with changedir(self.output_path): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--sparse') - self.assert_dirs_equal('input', 'output/input') - filename = os.path.join(self.output_path, 'input', 'sparse') - with open(filename, 'rb') as fd: + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--sparse") + self.assert_dirs_equal("input", "output/input") + filename = os.path.join(self.output_path, "input", "sparse") + with open(filename, "rb") as fd: # check if file contents are as expected - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_equal(fd.read(hole_size), b"\0" * hole_size) self.assert_equal(fd.read(len(content)), content) - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_equal(fd.read(hole_size), b"\0" * hole_size) assert is_sparse(filename, total_size, hole_size) def test_unusual_filenames(self): - filenames = ['normal', 'with some blanks', '(with_parens)', ] + filenames = ["normal", "with some blanks", "(with_parens)"] for filename in filenames: filename = os.path.join(self.input_path, filename) - with open(filename, 'wb'): + with open(filename, "wb"): pass - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") for filename in filenames: - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', os.path.join('input', filename)) - assert os.path.exists(os.path.join('output', 'input', filename)) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", os.path.join("input", filename)) + assert os.path.exists(os.path.join("output", "input", filename)) def test_repository_swap_detection(self): self.create_test_files() - os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + os.environ["BORG_PASSPHRASE"] = "passphrase" + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) repository_id = self._extract_repository_id(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") shutil.rmtree(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") self._set_repository_id(self.repository_path, repository_id) self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.EncryptionMethodMismatch): - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") def test_repository_swap_detection2(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}_unencrypted', 'rcreate', '--encryption=none') - os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd(f'--repo={self.repository_location}_encrypted', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test', 'input') - shutil.rmtree(self.repository_path + '_encrypted') - os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') + self.cmd(f"--repo={self.repository_location}_unencrypted", "rcreate", "--encryption=none") + os.environ["BORG_PASSPHRASE"] = "passphrase" + self.cmd(f"--repo={self.repository_location}_encrypted", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test", "input") + shutil.rmtree(self.repository_path + "_encrypted") + os.rename(self.repository_path + "_unencrypted", self.repository_path + "_encrypted") if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test.2', 'input', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test.2", "input", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.RepositoryAccessAborted): - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test.2', 'input') + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test.2", "input") def test_repository_swap_detection_no_cache(self): self.create_test_files() - os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + os.environ["BORG_PASSPHRASE"] = "passphrase" + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) repository_id = self._extract_repository_id(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") shutil.rmtree(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") self._set_repository_id(self.repository_path, repository_id) self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.EncryptionMethodMismatch): - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") def test_repository_swap_detection2_no_cache(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}_unencrypted', 'rcreate', '--encryption=none') - os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd(f'--repo={self.repository_location}_encrypted', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}_unencrypted', 'rdelete', '--cache-only') - self.cmd(f'--repo={self.repository_location}_encrypted', 'rdelete', '--cache-only') - shutil.rmtree(self.repository_path + '_encrypted') - os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') + self.cmd(f"--repo={self.repository_location}_unencrypted", "rcreate", "--encryption=none") + os.environ["BORG_PASSPHRASE"] = "passphrase" + self.cmd(f"--repo={self.repository_location}_encrypted", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}_unencrypted", "rdelete", "--cache-only") + self.cmd(f"--repo={self.repository_location}_encrypted", "rdelete", "--cache-only") + shutil.rmtree(self.repository_path + "_encrypted") + os.rename(self.repository_path + "_unencrypted", self.repository_path + "_encrypted") if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test.2', 'input', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test.2", "input", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.RepositoryAccessAborted): - self.cmd(f'--repo={self.repository_location}_encrypted', 'create', 'test.2', 'input') + self.cmd(f"--repo={self.repository_location}_encrypted", "create", "test.2", "input") def test_repository_swap_detection_repokey_blank_passphrase(self): # Check that a repokey repo with a blank passphrase is considered like a plaintext repo. self.create_test_files() # User initializes her repository with her passphrase - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") # Attacker replaces it with her own repository, which is encrypted but has no passphrase set shutil.rmtree(self.repository_path) - with environment_variable(BORG_PASSPHRASE=''): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + with environment_variable(BORG_PASSPHRASE=""): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # Delete cache & security database, AKA switch to user perspective - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") shutil.rmtree(self.get_security_dir()) with environment_variable(BORG_PASSPHRASE=None): # This is the part were the user would be tricked, e.g. she assumes that BORG_PASSPHRASE # is set, while it isn't. Previously this raised no warning, # since the repository is, technically, encrypted. if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.CacheInitAbortedError): - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") def test_repository_move(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) security_dir = self.get_security_dir() - os.rename(self.repository_path, self.repository_path + '_new') - with environment_variable(BORG_RELOCATED_REPO_ACCESS_IS_OK='yes'): - self.cmd(f'--repo={self.repository_location}_new', 'rinfo') - with open(os.path.join(security_dir, 'location')) as fd: + os.rename(self.repository_path, self.repository_path + "_new") + with environment_variable(BORG_RELOCATED_REPO_ACCESS_IS_OK="yes"): + self.cmd(f"--repo={self.repository_location}_new", "rinfo") + with open(os.path.join(security_dir, "location")) as fd: location = fd.read() - assert location == Location(self.repository_location + '_new').canonical_path() + assert location == Location(self.repository_location + "_new").canonical_path() # Needs no confirmation anymore - self.cmd(f'--repo={self.repository_location}_new', 'rinfo') + self.cmd(f"--repo={self.repository_location}_new", "rinfo") shutil.rmtree(self.cache_path) - self.cmd(f'--repo={self.repository_location}_new', 'rinfo') + self.cmd(f"--repo={self.repository_location}_new", "rinfo") shutil.rmtree(security_dir) - self.cmd(f'--repo={self.repository_location}_new', 'rinfo') - for file in ('location', 'key-type', 'manifest-timestamp'): + self.cmd(f"--repo={self.repository_location}_new", "rinfo") + for file in ("location", "key-type", "manifest-timestamp"): assert os.path.exists(os.path.join(security_dir, file)) def test_security_dir_compat(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - with open(os.path.join(self.get_security_dir(), 'location'), 'w') as fd: - fd.write('something outdated') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + with open(os.path.join(self.get_security_dir(), "location"), "w") as fd: + fd.write("something outdated") # This is fine, because the cache still has the correct information. security_dir and cache can disagree # if older versions are used to confirm a renamed repository. - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") def test_unknown_unencrypted(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") # Ok: repository is known - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") # Ok: repository is still known (through security_dir) shutil.rmtree(self.cache_path) - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") # Needs confirmation: cache and security dir both gone (eg. another host or rm -rf ~) shutil.rmtree(self.cache_path) shutil.rmtree(self.get_security_dir()) if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'rinfo', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "rinfo", exit_code=EXIT_ERROR) else: with pytest.raises(Cache.CacheInitAbortedError): - self.cmd(f'--repo={self.repository_location}', 'rinfo') - with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") + with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK="yes"): + self.cmd(f"--repo={self.repository_location}", "rinfo") def test_strip_components(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('dir/file') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--strip-components', '3') - assert not os.path.exists('file') - with self.assert_creates_file('file'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--strip-components', '2') - with self.assert_creates_file('dir/file'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--strip-components', '1') - with self.assert_creates_file('input/dir/file'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--strip-components', '0') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("dir/file") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--strip-components", "3") + assert not os.path.exists("file") + with self.assert_creates_file("file"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--strip-components", "2") + with self.assert_creates_file("dir/file"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--strip-components", "1") + with self.assert_creates_file("input/dir/file"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--strip-components", "0") def _extract_hardlinks_setup(self): - os.mkdir(os.path.join(self.input_path, 'dir1')) - os.mkdir(os.path.join(self.input_path, 'dir1/subdir')) + os.mkdir(os.path.join(self.input_path, "dir1")) + os.mkdir(os.path.join(self.input_path, "dir1/subdir")) - self.create_regular_file('source', contents=b'123456') - os.link(os.path.join(self.input_path, 'source'), - os.path.join(self.input_path, 'abba')) - os.link(os.path.join(self.input_path, 'source'), - os.path.join(self.input_path, 'dir1/hardlink')) - os.link(os.path.join(self.input_path, 'source'), - os.path.join(self.input_path, 'dir1/subdir/hardlink')) + self.create_regular_file("source", contents=b"123456") + os.link(os.path.join(self.input_path, "source"), os.path.join(self.input_path, "abba")) + os.link(os.path.join(self.input_path, "source"), os.path.join(self.input_path, "dir1/hardlink")) + os.link(os.path.join(self.input_path, "source"), os.path.join(self.input_path, "dir1/subdir/hardlink")) - self.create_regular_file('dir1/source2') - os.link(os.path.join(self.input_path, 'dir1/source2'), - os.path.join(self.input_path, 'dir1/aaaa')) + self.create_regular_file("dir1/source2") + os.link(os.path.join(self.input_path, "dir1/source2"), os.path.join(self.input_path, "dir1/aaaa")) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") @requires_hardlinks - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_fuse_mount_hardlinks(self): self._extract_hardlinks_setup() - mountpoint = os.path.join(self.tmpdir, 'mountpoint') + mountpoint = os.path.join(self.tmpdir, "mountpoint") # we need to get rid of permissions checking because fakeroot causes issues with it. # On all platforms, borg defaults to "default_permissions" and we need to get rid of it via "ignore_permissions". # On macOS (darwin), we additionally need "defer_permissions" to switch off the checks in osxfuse. - if sys.platform == 'darwin': - ignore_perms = ['-o', 'ignore_permissions,defer_permissions'] + if sys.platform == "darwin": + ignore_perms = ["-o", "ignore_permissions,defer_permissions"] else: - ignore_perms = ['-o', 'ignore_permissions'] - with self.fuse_mount(self.repository_location, mountpoint, '-a', 'test', '--strip-components=2', *ignore_perms), \ - changedir(os.path.join(mountpoint, 'test')): - assert os.stat('hardlink').st_nlink == 2 - assert os.stat('subdir/hardlink').st_nlink == 2 - assert open('subdir/hardlink', 'rb').read() == b'123456' - assert os.stat('aaaa').st_nlink == 2 - assert os.stat('source2').st_nlink == 2 - with self.fuse_mount(self.repository_location, mountpoint, 'input/dir1', '-a', 'test', *ignore_perms), \ - changedir(os.path.join(mountpoint, 'test')): - assert os.stat('input/dir1/hardlink').st_nlink == 2 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 - assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' - assert os.stat('input/dir1/aaaa').st_nlink == 2 - assert os.stat('input/dir1/source2').st_nlink == 2 - with self.fuse_mount(self.repository_location, mountpoint, '-a', 'test', *ignore_perms), \ - changedir(os.path.join(mountpoint, 'test')): - assert os.stat('input/source').st_nlink == 4 - assert os.stat('input/abba').st_nlink == 4 - assert os.stat('input/dir1/hardlink').st_nlink == 4 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 - assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' + ignore_perms = ["-o", "ignore_permissions"] + with self.fuse_mount( + self.repository_location, mountpoint, "-a", "test", "--strip-components=2", *ignore_perms + ), changedir(os.path.join(mountpoint, "test")): + assert os.stat("hardlink").st_nlink == 2 + assert os.stat("subdir/hardlink").st_nlink == 2 + assert open("subdir/hardlink", "rb").read() == b"123456" + assert os.stat("aaaa").st_nlink == 2 + assert os.stat("source2").st_nlink == 2 + with self.fuse_mount( + self.repository_location, mountpoint, "input/dir1", "-a", "test", *ignore_perms + ), changedir(os.path.join(mountpoint, "test")): + assert os.stat("input/dir1/hardlink").st_nlink == 2 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 2 + assert open("input/dir1/subdir/hardlink", "rb").read() == b"123456" + assert os.stat("input/dir1/aaaa").st_nlink == 2 + assert os.stat("input/dir1/source2").st_nlink == 2 + with self.fuse_mount(self.repository_location, mountpoint, "-a", "test", *ignore_perms), changedir( + os.path.join(mountpoint, "test") + ): + assert os.stat("input/source").st_nlink == 4 + assert os.stat("input/abba").st_nlink == 4 + assert os.stat("input/dir1/hardlink").st_nlink == 4 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 4 + assert open("input/dir1/subdir/hardlink", "rb").read() == b"123456" @requires_hardlinks def test_extract_hardlinks1(self): self._extract_hardlinks_setup() - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - assert os.stat('input/source').st_nlink == 4 - assert os.stat('input/abba').st_nlink == 4 - assert os.stat('input/dir1/hardlink').st_nlink == 4 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4 - assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + assert os.stat("input/source").st_nlink == 4 + assert os.stat("input/abba").st_nlink == 4 + assert os.stat("input/dir1/hardlink").st_nlink == 4 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 4 + assert open("input/dir1/subdir/hardlink", "rb").read() == b"123456" @requires_hardlinks def test_extract_hardlinks2(self): self._extract_hardlinks_setup() - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--strip-components', '2') - assert os.stat('hardlink').st_nlink == 2 - assert os.stat('subdir/hardlink').st_nlink == 2 - assert open('subdir/hardlink', 'rb').read() == b'123456' - assert os.stat('aaaa').st_nlink == 2 - assert os.stat('source2').st_nlink == 2 - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', 'input/dir1') - assert os.stat('input/dir1/hardlink').st_nlink == 2 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 - assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456' - assert os.stat('input/dir1/aaaa').st_nlink == 2 - assert os.stat('input/dir1/source2').st_nlink == 2 + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--strip-components", "2") + assert os.stat("hardlink").st_nlink == 2 + assert os.stat("subdir/hardlink").st_nlink == 2 + assert open("subdir/hardlink", "rb").read() == b"123456" + assert os.stat("aaaa").st_nlink == 2 + assert os.stat("source2").st_nlink == 2 + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "input/dir1") + assert os.stat("input/dir1/hardlink").st_nlink == 2 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 2 + assert open("input/dir1/subdir/hardlink", "rb").read() == b"123456" + assert os.stat("input/dir1/aaaa").st_nlink == 2 + assert os.stat("input/dir1/source2").st_nlink == 2 @requires_hardlinks def test_extract_hardlinks_twice(self): # setup for #5603 - path_a = os.path.join(self.input_path, 'a') - path_b = os.path.join(self.input_path, 'b') + path_a = os.path.join(self.input_path, "a") + path_b = os.path.join(self.input_path, "b") os.mkdir(path_a) os.mkdir(path_b) - hl_a = os.path.join(path_a, 'hardlink') - hl_b = os.path.join(path_b, 'hardlink') - self.create_regular_file(hl_a, contents=b'123456') + hl_a = os.path.join(path_a, "hardlink") + hl_b = os.path.join(path_b, "hardlink") + self.create_regular_file(hl_a, contents=b"123456") os.link(hl_a, hl_b) - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', 'input') # give input twice! + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "input") # give input twice! # now test extraction - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") # if issue #5603 happens, extraction gives rc == 1 (triggering AssertionError) and warnings like: # input/a/hardlink: link: [Errno 2] No such file or directory: 'input/a/hardlink' -> 'input/a/hardlink' # input/b/hardlink: link: [Errno 2] No such file or directory: 'input/a/hardlink' -> 'input/b/hardlink' # otherwise, when fixed, the hardlinks should be there and have a link count of 2 - assert os.stat('input/a/hardlink').st_nlink == 2 - assert os.stat('input/b/hardlink').st_nlink == 2 + assert os.stat("input/a/hardlink").st_nlink == 2 + assert os.stat("input/b/hardlink").st_nlink == 2 def test_extract_include_exclude(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('file3', size=1024 * 80) - self.create_regular_file('file4', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude=input/file4', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', 'input/file1', ) - self.assert_equal(sorted(os.listdir('output/input')), ['file1']) - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude=input/file2') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude-from=' + self.exclude_file_path) - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file3", size=1024 * 80) + self.create_regular_file("file4", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "create", "--exclude=input/file4", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "input/file1") + self.assert_equal(sorted(os.listdir("output/input")), ["file1"]) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--exclude=input/file2") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file3"]) + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--exclude-from=" + self.exclude_file_path + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file3"]) def test_extract_include_exclude_regex(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('file3', size=1024 * 80) - self.create_regular_file('file4', size=1024 * 80) - self.create_regular_file('file333', size=1024 * 80) - - # Create with regular expression exclusion for file4 - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude=re:input/file4$', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) - shutil.rmtree('output/input') - - # Extract with regular expression exclusion - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude=re:file3+') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) - shutil.rmtree('output/input') - - # Combine --exclude with fnmatch and regular expression - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude=input/file2', '--exclude=re:file[01]') - self.assert_equal(sorted(os.listdir('output/input')), ['file3', 'file333']) - shutil.rmtree('output/input') - - # Combine --exclude-from and regular expression exclusion - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude-from=' + self.exclude_file_path, - '--exclude=re:file1', '--exclude=re:file(\\d)\\1\\1$') - self.assert_equal(sorted(os.listdir('output/input')), ['file3']) - - def test_extract_include_exclude_regex_from_file(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('file3', size=1024 * 80) - self.create_regular_file('file4', size=1024 * 80) - self.create_regular_file('file333', size=1024 * 80) - self.create_regular_file('aa:something', size=1024 * 80) - - # Create while excluding using mixed pattern styles - with open(self.exclude_file_path, 'wb') as fd: - fd.write(b're:input/file4$\n') - fd.write(b'fm:*aa:*thing\n') - - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude-from=' + self.exclude_file_path, 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) - shutil.rmtree('output/input') - - # Exclude using regular expression - with open(self.exclude_file_path, 'wb') as fd: - fd.write(b're:file3+\n') - - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude-from=' + self.exclude_file_path) - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) - shutil.rmtree('output/input') - - # Mixed exclude pattern styles - with open(self.exclude_file_path, 'wb') as fd: - fd.write(b're:file(\\d)\\1\\1$\n') - fd.write(b'fm:nothingwillmatchthis\n') - fd.write(b'*/file1\n') - fd.write(b're:file2$\n') - - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--exclude-from=' + self.exclude_file_path) - self.assert_equal(sorted(os.listdir('output/input')), ['file3']) - - def test_extract_with_pattern(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.create_regular_file("file1", size=1024 * 80) self.create_regular_file("file2", size=1024 * 80) self.create_regular_file("file3", size=1024 * 80) self.create_regular_file("file4", size=1024 * 80) self.create_regular_file("file333", size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'create', "test", "input") + # Create with regular expression exclusion for file4 + self.cmd(f"--repo={self.repository_location}", "create", "--exclude=re:input/file4$", "test", "input") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333"]) + shutil.rmtree("output/input") + + # Extract with regular expression exclusion + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--exclude=re:file3+") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"]) + shutil.rmtree("output/input") + + # Combine --exclude with fnmatch and regular expression + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", + "extract", + "test", + "--exclude=input/file2", + "--exclude=re:file[01]", + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file3", "file333"]) + shutil.rmtree("output/input") + + # Combine --exclude-from and regular expression exclusion + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", + "extract", + "test", + "--exclude-from=" + self.exclude_file_path, + "--exclude=re:file1", + "--exclude=re:file(\\d)\\1\\1$", + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file3"]) + + def test_extract_include_exclude_regex_from_file(self): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file3", size=1024 * 80) + self.create_regular_file("file4", size=1024 * 80) + self.create_regular_file("file333", size=1024 * 80) + self.create_regular_file("aa:something", size=1024 * 80) + + # Create while excluding using mixed pattern styles + with open(self.exclude_file_path, "wb") as fd: + fd.write(b"re:input/file4$\n") + fd.write(b"fm:*aa:*thing\n") + + self.cmd( + f"--repo={self.repository_location}", "create", "--exclude-from=" + self.exclude_file_path, "test", "input" + ) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333"]) + shutil.rmtree("output/input") + + # Exclude using regular expression + with open(self.exclude_file_path, "wb") as fd: + fd.write(b"re:file3+\n") + + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--exclude-from=" + self.exclude_file_path + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"]) + shutil.rmtree("output/input") + + # Mixed exclude pattern styles + with open(self.exclude_file_path, "wb") as fd: + fd.write(b"re:file(\\d)\\1\\1$\n") + fd.write(b"fm:nothingwillmatchthis\n") + fd.write(b"*/file1\n") + fd.write(b"re:file2$\n") + + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--exclude-from=" + self.exclude_file_path + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file3"]) + + def test_extract_with_pattern(self): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file3", size=1024 * 80) + self.create_regular_file("file4", size=1024 * 80) + self.create_regular_file("file333", size=1024 * 80) + + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") # Extract everything with regular expression with changedir("output"): - self.cmd(f'--repo={self.repository_location}', "extract", "test", "re:.*") + self.cmd(f"--repo={self.repository_location}", "extract", "test", "re:.*") self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333", "file4"]) shutil.rmtree("output/input") # Extract with pattern while also excluding files with changedir("output"): - self.cmd(f'--repo={self.repository_location}', "extract", "--exclude=re:file[34]$", "test", r"re:file\d$") + self.cmd(f"--repo={self.repository_location}", "extract", "--exclude=re:file[34]$", "test", r"re:file\d$") self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"]) shutil.rmtree("output/input") # Combine --exclude with pattern for extraction with changedir("output"): - self.cmd(f'--repo={self.repository_location}', "extract", "--exclude=input/file1", "test", "re:file[12]$") + self.cmd(f"--repo={self.repository_location}", "extract", "--exclude=input/file1", "test", "re:file[12]$") self.assert_equal(sorted(os.listdir("output/input")), ["file2"]) shutil.rmtree("output/input") # Multiple pattern with changedir("output"): - self.cmd(f'--repo={self.repository_location}', "extract", "test", "fm:input/file1", "fm:*file33*", "input/file2") + self.cmd( + f"--repo={self.repository_location}", "extract", "test", "fm:input/file1", "fm:*file33*", "input/file2" + ) self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file333"]) def test_extract_list_output(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file', size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file", size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test') + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test") self.assert_not_in("input/file", output) - shutil.rmtree('output/input') + shutil.rmtree("output/input") - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--info') + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--info") self.assert_not_in("input/file", output) - shutil.rmtree('output/input') + shutil.rmtree("output/input") - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--list') + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--list") self.assert_in("input/file", output) - shutil.rmtree('output/input') + shutil.rmtree("output/input") - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--list', '--info') + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--list", "--info") self.assert_in("input/file", output) def test_extract_progress(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--progress') - assert 'Extracting:' in output + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--progress") + assert "Extracting:" in output def _create_test_caches(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('cache1/%s' % CACHE_TAG_NAME, - contents=CACHE_TAG_CONTENTS + b' extra stuff') - self.create_regular_file('cache2/%s' % CACHE_TAG_NAME, - contents=b'invalid signature') - os.mkdir('input/cache3') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("cache1/%s" % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b" extra stuff") + self.create_regular_file("cache2/%s" % CACHE_TAG_NAME, contents=b"invalid signature") + os.mkdir("input/cache3") if are_hardlinks_supported(): - os.link('input/cache1/%s' % CACHE_TAG_NAME, 'input/cache3/%s' % CACHE_TAG_NAME) + os.link("input/cache1/%s" % CACHE_TAG_NAME, "input/cache3/%s" % CACHE_TAG_NAME) else: - self.create_regular_file('cache3/%s' % CACHE_TAG_NAME, - contents=CACHE_TAG_CONTENTS + b' extra stuff') + self.create_regular_file("cache3/%s" % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b" extra stuff") def test_create_stdin(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - input_data = b'\x00foo\n\nbar\n \n' - self.cmd(f'--repo={self.repository_location}', 'create', 'test', '-', input=input_data) - item = json.loads(self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines')) - assert item['uid'] == 0 - assert item['gid'] == 0 - assert item['size'] == len(input_data) - assert item['path'] == 'stdin' - extracted_data = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--stdout', binary_output=True) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + input_data = b"\x00foo\n\nbar\n \n" + self.cmd(f"--repo={self.repository_location}", "create", "test", "-", input=input_data) + item = json.loads(self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines")) + assert item["uid"] == 0 + assert item["gid"] == 0 + assert item["size"] == len(input_data) + assert item["path"] == "stdin" + extracted_data = self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--stdout", binary_output=True + ) assert extracted_data == input_data def test_create_content_from_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - input_data = 'some test content' - name = 'a/b/c' - self.cmd(f'--repo={self.repository_location}', 'create', '--stdin-name', name, '--content-from-command', - 'test', '--', 'echo', input_data) - item = json.loads(self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines')) - assert item['uid'] == 0 - assert item['gid'] == 0 - assert item['size'] == len(input_data) + 1 # `echo` adds newline - assert item['path'] == name - extracted_data = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--stdout') - assert extracted_data == input_data + '\n' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + input_data = "some test content" + name = "a/b/c" + self.cmd( + f"--repo={self.repository_location}", + "create", + "--stdin-name", + name, + "--content-from-command", + "test", + "--", + "echo", + input_data, + ) + item = json.loads(self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines")) + assert item["uid"] == 0 + assert item["gid"] == 0 + assert item["size"] == len(input_data) + 1 # `echo` adds newline + assert item["path"] == name + extracted_data = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--stdout") + assert extracted_data == input_data + "\n" def test_create_content_from_command_with_failed_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', '--content-from-command', 'test', - '--', 'sh', '-c', 'exit 73;', exit_code=2) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "--content-from-command", + "test", + "--", + "sh", + "-c", + "exit 73;", + exit_code=2, + ) assert output.endswith("Command 'sh' exited with status 73\n") - archive_list = json.loads(self.cmd(f'--repo={self.repository_location}', 'rlist', '--json')) - assert archive_list['archives'] == [] + archive_list = json.loads(self.cmd(f"--repo={self.repository_location}", "rlist", "--json")) + assert archive_list["archives"] == [] def test_create_content_from_command_missing_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test', '--content-from-command', exit_code=2) - assert output.endswith('No command given.\n') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "test", "--content-from-command", exit_code=2) + assert output.endswith("No command given.\n") def test_create_paths_from_stdin(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.create_regular_file("file1", size=1024 * 80) self.create_regular_file("dir1/file2", size=1024 * 80) self.create_regular_file("dir1/file3", size=1024 * 80) self.create_regular_file("file4", size=1024 * 80) - input_data = b'input/file1\0input/dir1\0input/file4' - self.cmd(f'--repo={self.repository_location}', 'create', 'test', - '--paths-from-stdin', '--paths-delimiter', '\\0', input=input_data) - archive_list = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines') - paths = [json.loads(line)['path'] for line in archive_list.split('\n') if line] - assert paths == ['input/file1', 'input/dir1', 'input/file4'] + input_data = b"input/file1\0input/dir1\0input/file4" + self.cmd( + f"--repo={self.repository_location}", + "create", + "test", + "--paths-from-stdin", + "--paths-delimiter", + "\\0", + input=input_data, + ) + archive_list = self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines") + paths = [json.loads(line)["path"] for line in archive_list.split("\n") if line] + assert paths == ["input/file1", "input/dir1", "input/file4"] def test_create_paths_from_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.create_regular_file("file1", size=1024 * 80) self.create_regular_file("file2", size=1024 * 80) self.create_regular_file("file3", size=1024 * 80) self.create_regular_file("file4", size=1024 * 80) - input_data = 'input/file1\ninput/file2\ninput/file3' - self.cmd(f'--repo={self.repository_location}', 'create', '--paths-from-command', 'test', - '--', 'echo', input_data) - archive_list = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines') - paths = [json.loads(line)['path'] for line in archive_list.split('\n') if line] - assert paths == ['input/file1', 'input/file2', 'input/file3'] + input_data = "input/file1\ninput/file2\ninput/file3" + self.cmd( + f"--repo={self.repository_location}", "create", "--paths-from-command", "test", "--", "echo", input_data + ) + archive_list = self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines") + paths = [json.loads(line)["path"] for line in archive_list.split("\n") if line] + assert paths == ["input/file1", "input/file2", "input/file3"] def test_create_paths_from_command_with_failed_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', '--paths-from-command', 'test', - '--', 'sh', '-c', 'exit 73;', exit_code=2) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "--paths-from-command", + "test", + "--", + "sh", + "-c", + "exit 73;", + exit_code=2, + ) assert output.endswith("Command 'sh' exited with status 73\n") - archive_list = json.loads(self.cmd(f'--repo={self.repository_location}', 'rlist', '--json')) - assert archive_list['archives'] == [] + archive_list = json.loads(self.cmd(f"--repo={self.repository_location}", "rlist", "--json")) + assert archive_list["archives"] == [] def test_create_paths_from_command_missing_command(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test', '--paths-from-command', exit_code=2) - assert output.endswith('No command given.\n') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "test", "--paths-from-command", exit_code=2) + assert output.endswith("No command given.\n") def test_create_without_root(self): """test create without a root""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', exit_code=2) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", exit_code=2) def test_create_pattern_root(self): """test create with only a root pattern""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test', '-v', '--list', '--pattern=R input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + output = self.cmd(f"--repo={self.repository_location}", "create", "test", "-v", "--list", "--pattern=R input") self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) def test_create_pattern(self): """test file patterns during create""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('file_important', size=1024 * 80) - output = self.cmd(f'--repo={self.repository_location}', 'create', '-v', '--list', - '--pattern=+input/file_important', '--pattern=-input/file*', - 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file_important", size=1024 * 80) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "-v", + "--list", + "--pattern=+input/file_important", + "--pattern=-input/file*", + "test", + "input", + ) self.assert_in("A input/file_important", output) - self.assert_in('x input/file1', output) - self.assert_in('x input/file2', output) + self.assert_in("x input/file1", output) + self.assert_in("x input/file2", output) def test_create_pattern_file(self): """test file patterns during create""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('otherfile', size=1024 * 80) - self.create_regular_file('file_important', size=1024 * 80) - output = self.cmd(f'--repo={self.repository_location}', 'create', '-v', '--list', - '--pattern=-input/otherfile', '--patterns-from=' + self.patterns_file_path, - 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("otherfile", size=1024 * 80) + self.create_regular_file("file_important", size=1024 * 80) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "-v", + "--list", + "--pattern=-input/otherfile", + "--patterns-from=" + self.patterns_file_path, + "test", + "input", + ) self.assert_in("A input/file_important", output) - self.assert_in('x input/file1', output) - self.assert_in('x input/file2', output) - self.assert_in('x input/otherfile', output) + self.assert_in("x input/file1", output) + self.assert_in("x input/file2", output) + self.assert_in("x input/otherfile", output) def test_create_pattern_exclude_folder_but_recurse(self): """test when patterns exclude a parent folder, but include a child""" - self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') - with open(self.patterns_file_path2, 'wb') as fd: - fd.write(b'+ input/x/b\n- input/x*\n') + self.patterns_file_path2 = os.path.join(self.tmpdir, "patterns2") + with open(self.patterns_file_path2, "wb") as fd: + fd.write(b"+ input/x/b\n- input/x*\n") - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('x/a/foo_a', size=1024 * 80) - self.create_regular_file('x/b/foo_b', size=1024 * 80) - self.create_regular_file('y/foo_y', size=1024 * 80) - output = self.cmd(f'--repo={self.repository_location}', 'create', '-v', '--list', - '--patterns-from=' + self.patterns_file_path2, - 'test', 'input') - self.assert_in('x input/x/a/foo_a', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("x/a/foo_a", size=1024 * 80) + self.create_regular_file("x/b/foo_b", size=1024 * 80) + self.create_regular_file("y/foo_y", size=1024 * 80) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "-v", + "--list", + "--patterns-from=" + self.patterns_file_path2, + "test", + "input", + ) + self.assert_in("x input/x/a/foo_a", output) self.assert_in("A input/x/b/foo_b", output) - self.assert_in('A input/y/foo_y', output) + self.assert_in("A input/y/foo_y", output) def test_create_pattern_exclude_folder_no_recurse(self): """test when patterns exclude a parent folder and, but include a child""" - self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') - with open(self.patterns_file_path2, 'wb') as fd: - fd.write(b'+ input/x/b\n! input/x*\n') + self.patterns_file_path2 = os.path.join(self.tmpdir, "patterns2") + with open(self.patterns_file_path2, "wb") as fd: + fd.write(b"+ input/x/b\n! input/x*\n") - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('x/a/foo_a', size=1024 * 80) - self.create_regular_file('x/b/foo_b', size=1024 * 80) - self.create_regular_file('y/foo_y', size=1024 * 80) - output = self.cmd(f'--repo={self.repository_location}', 'create', '-v', '--list', - '--patterns-from=' + self.patterns_file_path2, - 'test', 'input') - self.assert_not_in('input/x/a/foo_a', output) - self.assert_not_in('input/x/a', output) - self.assert_in('A input/y/foo_y', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("x/a/foo_a", size=1024 * 80) + self.create_regular_file("x/b/foo_b", size=1024 * 80) + self.create_regular_file("y/foo_y", size=1024 * 80) + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "-v", + "--list", + "--patterns-from=" + self.patterns_file_path2, + "test", + "input", + ) + self.assert_not_in("input/x/a/foo_a", output) + self.assert_not_in("input/x/a", output) + self.assert_in("A input/y/foo_y", output) def test_create_pattern_intermediate_folders_first(self): """test that intermediate folders appear first when patterns exclude a parent folder but include a child""" - self.patterns_file_path2 = os.path.join(self.tmpdir, 'patterns2') - with open(self.patterns_file_path2, 'wb') as fd: - fd.write(b'+ input/x/a\n+ input/x/b\n- input/x*\n') + self.patterns_file_path2 = os.path.join(self.tmpdir, "patterns2") + with open(self.patterns_file_path2, "wb") as fd: + fd.write(b"+ input/x/a\n+ input/x/b\n- input/x*\n") - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - self.create_regular_file('x/a/foo_a', size=1024 * 80) - self.create_regular_file('x/b/foo_b', size=1024 * 80) - with changedir('input'): - self.cmd(f'--repo={self.repository_location}', 'create', '--patterns-from=' + self.patterns_file_path2, - 'test', '.') + self.create_regular_file("x/a/foo_a", size=1024 * 80) + self.create_regular_file("x/b/foo_b", size=1024 * 80) + with changedir("input"): + self.cmd( + f"--repo={self.repository_location}", + "create", + "--patterns-from=" + self.patterns_file_path2, + "test", + ".", + ) # list the archive and verify that the "intermediate" folders appear before # their contents - out = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{type} {path}{NL}') + out = self.cmd(f"--repo={self.repository_location}", "list", "test", "--format", "{type} {path}{NL}") out_list = out.splitlines() - self.assert_in('d x/a', out_list) - self.assert_in('d x/b', out_list) + self.assert_in("d x/a", out_list) + self.assert_in("d x/b", out_list) - assert out_list.index('d x/a') < out_list.index('- x/a/foo_a') - assert out_list.index('d x/b') < out_list.index('- x/b/foo_b') + assert out_list.index("d x/a") < out_list.index("- x/a/foo_a") + assert out_list.index("d x/b") < out_list.index("- x/b/foo_b") def test_create_no_cache_sync(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') - create_json = json.loads(self.cmd(f'--repo={self.repository_location}', 'create', - '--no-cache-sync', '--json', '--error', - 'test', 'input')) # ignore experimental warning - info_json = json.loads(self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test', '--json')) - create_stats = create_json['cache']['stats'] - info_stats = info_json['cache']['stats'] + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") + create_json = json.loads( + self.cmd( + f"--repo={self.repository_location}", "create", "--no-cache-sync", "--json", "--error", "test", "input" + ) + ) # ignore experimental warning + info_json = json.loads(self.cmd(f"--repo={self.repository_location}", "info", "-a", "test", "--json")) + create_stats = create_json["cache"]["stats"] + info_stats = info_json["cache"]["stats"] assert create_stats == info_stats - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') - self.cmd(f'--repo={self.repository_location}', 'create', '--no-cache-sync', 'test2', 'input') - self.cmd(f'--repo={self.repository_location}', 'rinfo') - self.cmd(f'--repo={self.repository_location}', 'check') + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") + self.cmd(f"--repo={self.repository_location}", "create", "--no-cache-sync", "test2", "input") + self.cmd(f"--repo={self.repository_location}", "rinfo") + self.cmd(f"--repo={self.repository_location}", "check") def test_extract_pattern_opt(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - self.create_regular_file('file_important', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', - '--pattern=+input/file_important', '--pattern=-input/file*') - self.assert_equal(sorted(os.listdir('output/input')), ['file_important']) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file_important", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + self.cmd( + f"--repo={self.repository_location}", + "extract", + "test", + "--pattern=+input/file_important", + "--pattern=-input/file*", + ) + self.assert_equal(sorted(os.listdir("output/input")), ["file_important"]) def _assert_test_caches(self): - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_equal(sorted(os.listdir('output/input')), ['cache2', 'file1']) - self.assert_equal(sorted(os.listdir('output/input/cache2')), [CACHE_TAG_NAME]) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_equal(sorted(os.listdir("output/input")), ["cache2", "file1"]) + self.assert_equal(sorted(os.listdir("output/input/cache2")), [CACHE_TAG_NAME]) def test_exclude_caches(self): self._create_test_caches() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '--exclude-caches') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "--exclude-caches") self._assert_test_caches() def test_recreate_exclude_caches(self): self._create_test_caches() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--exclude-caches') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test", "--exclude-caches") self._assert_test_caches() def _create_test_tagged(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('tagged1/.NOBACKUP') - self.create_regular_file('tagged2/00-NOBACKUP') - self.create_regular_file('tagged3/.NOBACKUP/file2', size=1024) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("tagged1/.NOBACKUP") + self.create_regular_file("tagged2/00-NOBACKUP") + self.create_regular_file("tagged3/.NOBACKUP/file2", size=1024) def _assert_test_tagged(self): - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_equal(sorted(os.listdir('output/input')), ['file1']) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_equal(sorted(os.listdir("output/input")), ["file1"]) def test_exclude_tagged(self): self._create_test_tagged() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', - '--exclude-if-present', '.NOBACKUP', '--exclude-if-present', '00-NOBACKUP') + self.cmd( + f"--repo={self.repository_location}", + "create", + "test", + "input", + "--exclude-if-present", + ".NOBACKUP", + "--exclude-if-present", + "00-NOBACKUP", + ) self._assert_test_tagged() def test_recreate_exclude_tagged(self): self._create_test_tagged() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--exclude-if-present', '.NOBACKUP', - '--exclude-if-present', '00-NOBACKUP') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd( + f"--repo={self.repository_location}", + "recreate", + "-a", + "test", + "--exclude-if-present", + ".NOBACKUP", + "--exclude-if-present", + "00-NOBACKUP", + ) self._assert_test_tagged() def _create_test_keep_tagged(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file0', size=1024) - self.create_regular_file('tagged1/.NOBACKUP1') - self.create_regular_file('tagged1/file1', size=1024) - self.create_regular_file('tagged2/.NOBACKUP2/subfile1', size=1024) - self.create_regular_file('tagged2/file2', size=1024) - self.create_regular_file('tagged3/%s' % CACHE_TAG_NAME, - contents=CACHE_TAG_CONTENTS + b' extra stuff') - self.create_regular_file('tagged3/file3', size=1024) - self.create_regular_file('taggedall/.NOBACKUP1') - self.create_regular_file('taggedall/.NOBACKUP2/subfile1', size=1024) - self.create_regular_file('taggedall/%s' % CACHE_TAG_NAME, - contents=CACHE_TAG_CONTENTS + b' extra stuff') - self.create_regular_file('taggedall/file4', size=1024) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file0", size=1024) + self.create_regular_file("tagged1/.NOBACKUP1") + self.create_regular_file("tagged1/file1", size=1024) + self.create_regular_file("tagged2/.NOBACKUP2/subfile1", size=1024) + self.create_regular_file("tagged2/file2", size=1024) + self.create_regular_file("tagged3/%s" % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b" extra stuff") + self.create_regular_file("tagged3/file3", size=1024) + self.create_regular_file("taggedall/.NOBACKUP1") + self.create_regular_file("taggedall/.NOBACKUP2/subfile1", size=1024) + self.create_regular_file("taggedall/%s" % CACHE_TAG_NAME, contents=CACHE_TAG_CONTENTS + b" extra stuff") + self.create_regular_file("taggedall/file4", size=1024) def _assert_test_keep_tagged(self): - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_equal(sorted(os.listdir('output/input')), ['file0', 'tagged1', 'tagged2', 'tagged3', 'taggedall']) - self.assert_equal(os.listdir('output/input/tagged1'), ['.NOBACKUP1']) - self.assert_equal(os.listdir('output/input/tagged2'), ['.NOBACKUP2']) - self.assert_equal(os.listdir('output/input/tagged3'), [CACHE_TAG_NAME]) - self.assert_equal(sorted(os.listdir('output/input/taggedall')), - ['.NOBACKUP1', '.NOBACKUP2', CACHE_TAG_NAME, ]) + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_equal(sorted(os.listdir("output/input")), ["file0", "tagged1", "tagged2", "tagged3", "taggedall"]) + self.assert_equal(os.listdir("output/input/tagged1"), [".NOBACKUP1"]) + self.assert_equal(os.listdir("output/input/tagged2"), [".NOBACKUP2"]) + self.assert_equal(os.listdir("output/input/tagged3"), [CACHE_TAG_NAME]) + self.assert_equal(sorted(os.listdir("output/input/taggedall")), [".NOBACKUP1", ".NOBACKUP2", CACHE_TAG_NAME]) def test_exclude_keep_tagged(self): self._create_test_keep_tagged() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '--exclude-if-present', '.NOBACKUP1', - '--exclude-if-present', '.NOBACKUP2', '--exclude-caches', '--keep-exclude-tags') + self.cmd( + f"--repo={self.repository_location}", + "create", + "test", + "input", + "--exclude-if-present", + ".NOBACKUP1", + "--exclude-if-present", + ".NOBACKUP2", + "--exclude-caches", + "--keep-exclude-tags", + ) self._assert_test_keep_tagged() def test_recreate_exclude_keep_tagged(self): self._create_test_keep_tagged() - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--exclude-if-present', '.NOBACKUP1', - '--exclude-if-present', '.NOBACKUP2', '--exclude-caches', '--keep-exclude-tags') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd( + f"--repo={self.repository_location}", + "recreate", + "-a", + "test", + "--exclude-if-present", + ".NOBACKUP1", + "--exclude-if-present", + ".NOBACKUP2", + "--exclude-caches", + "--keep-exclude-tags", + ) self._assert_test_keep_tagged() - @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') + @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") def test_recreate_hardlinked_tags(self): # test for issue #4911 - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.create_regular_file('file1', contents=CACHE_TAG_CONTENTS) # "wrong" filename, but correct tag contents - os.mkdir(os.path.join(self.input_path, 'subdir')) # to make sure the tag is encountered *after* file1 - os.link(os.path.join(self.input_path, 'file1'), - os.path.join(self.input_path, 'subdir', CACHE_TAG_NAME)) # correct tag name, hardlink to file1 - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.create_regular_file("file1", contents=CACHE_TAG_CONTENTS) # "wrong" filename, but correct tag contents + os.mkdir(os.path.join(self.input_path, "subdir")) # to make sure the tag is encountered *after* file1 + os.link( + os.path.join(self.input_path, "file1"), os.path.join(self.input_path, "subdir", CACHE_TAG_NAME) + ) # correct tag name, hardlink to file1 + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") # in the "test" archive, we now have, in this order: # - a regular file item for "file1" # - a hardlink item for "CACHEDIR.TAG" referring back to file1 for its contents - self.cmd(f'--repo={self.repository_location}', 'recreate', 'test', '--exclude-caches', '--keep-exclude-tags') + self.cmd(f"--repo={self.repository_location}", "recreate", "test", "--exclude-caches", "--keep-exclude-tags") # if issue #4911 is present, the recreate will crash with a KeyError for "input/file1" - @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='Linux capabilities test, requires fakeroot >= 1.20.2') + @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason="Linux capabilities test, requires fakeroot >= 1.20.2") def test_extract_capabilities(self): fchown = os.fchown # We need to manually patch chown to get the behaviour Linux has, since fakeroot does not # accurately model the interaction of chown(2) and Linux capabilities, i.e. it does not remove them. def patched_fchown(fd, uid, gid): - xattr.setxattr(fd, b'security.capability', b'', follow_symlinks=False) + xattr.setxattr(fd, b"security.capability", b"", follow_symlinks=False) fchown(fd, uid, gid) # The capability descriptor used here is valid and taken from a /usr/bin/ping - capabilities = b'\x01\x00\x00\x02\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - self.create_regular_file('file') - xattr.setxattr(b'input/file', b'security.capability', capabilities) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - with patch.object(os, 'fchown', patched_fchown): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - assert xattr.getxattr(b'input/file', b'security.capability') == capabilities + capabilities = b"\x01\x00\x00\x02\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + self.create_regular_file("file") + xattr.setxattr(b"input/file", b"security.capability", capabilities) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + with patch.object(os, "fchown", patched_fchown): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + assert xattr.getxattr(b"input/file", b"security.capability") == capabilities - @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='xattr not supported on this system or on this version of' - 'fakeroot') + @pytest.mark.skipif( + not xattr.XATTR_FAKEROOT, reason="xattr not supported on this system or on this version of" "fakeroot" + ) def test_extract_xattrs_errors(self): def patched_setxattr_E2BIG(*args, **kwargs): - raise OSError(errno.E2BIG, 'E2BIG') + raise OSError(errno.E2BIG, "E2BIG") def patched_setxattr_ENOTSUP(*args, **kwargs): - raise OSError(errno.ENOTSUP, 'ENOTSUP') + raise OSError(errno.ENOTSUP, "ENOTSUP") def patched_setxattr_EACCES(*args, **kwargs): - raise OSError(errno.EACCES, 'EACCES') + raise OSError(errno.EACCES, "EACCES") - self.create_regular_file('file') - xattr.setxattr(b'input/file', b'user.attribute', b'value') - self.cmd(f'--repo={self.repository_location}', 'rcreate', '-e' 'none') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - input_abspath = os.path.abspath('input/file') - with patch.object(xattr, 'setxattr', patched_setxattr_E2BIG): - out = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_WARNING) - assert ': when setting extended attribute user.attribute: too big for this filesystem\n' in out + self.create_regular_file("file") + xattr.setxattr(b"input/file", b"user.attribute", b"value") + self.cmd(f"--repo={self.repository_location}", "rcreate", "-e" "none") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + input_abspath = os.path.abspath("input/file") + with patch.object(xattr, "setxattr", patched_setxattr_E2BIG): + out = self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_WARNING) + assert ": when setting extended attribute user.attribute: too big for this filesystem\n" in out os.remove(input_abspath) - with patch.object(xattr, 'setxattr', patched_setxattr_ENOTSUP): - out = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_WARNING) - assert ': when setting extended attribute user.attribute: xattrs not supported on this filesystem\n' in out + with patch.object(xattr, "setxattr", patched_setxattr_ENOTSUP): + out = self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_WARNING) + assert ( + ": when setting extended attribute user.attribute: xattrs not supported on this filesystem\n" in out + ) os.remove(input_abspath) - with patch.object(xattr, 'setxattr', patched_setxattr_EACCES): - out = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_WARNING) - assert ': when setting extended attribute user.attribute: Permission denied\n' in out + with patch.object(xattr, "setxattr", patched_setxattr_EACCES): + out = self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_WARNING) + assert ": when setting extended attribute user.attribute: Permission denied\n" in out assert os.path.isfile(input_abspath) def test_path_normalization(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('dir1/dir2/file', size=1024 * 80) - with changedir('input/dir1/dir2'): - self.cmd(f'--repo={self.repository_location}', 'create', 'test', - '../../../input/dir1/../dir1/dir2/..') - output = self.cmd(f'--repo={self.repository_location}', 'list', 'test') - self.assert_not_in('..', output) - self.assert_in(' input/dir1/dir2/file', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("dir1/dir2/file", size=1024 * 80) + with changedir("input/dir1/dir2"): + self.cmd(f"--repo={self.repository_location}", "create", "test", "../../../input/dir1/../dir1/dir2/..") + output = self.cmd(f"--repo={self.repository_location}", "list", "test") + self.assert_not_in("..", output) + self.assert_in(" input/dir1/dir2/file", output) def test_exclude_normalization(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('file2', size=1024 * 80) - with changedir('input'): - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', '.', '--exclude=file1') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test1') - self.assert_equal(sorted(os.listdir('output')), ['file2']) - with changedir('input'): - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', '.', '--exclude=./file1') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test2') - self.assert_equal(sorted(os.listdir('output')), ['file2']) - self.cmd(f'--repo={self.repository_location}', 'create', 'test3', 'input', '--exclude=input/./file1') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test3') - self.assert_equal(sorted(os.listdir('output/input')), ['file2']) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + with changedir("input"): + self.cmd(f"--repo={self.repository_location}", "create", "test1", ".", "--exclude=file1") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test1") + self.assert_equal(sorted(os.listdir("output")), ["file2"]) + with changedir("input"): + self.cmd(f"--repo={self.repository_location}", "create", "test2", ".", "--exclude=./file1") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test2") + self.assert_equal(sorted(os.listdir("output")), ["file2"]) + self.cmd(f"--repo={self.repository_location}", "create", "test3", "input", "--exclude=input/./file1") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test3") + self.assert_equal(sorted(os.listdir("output/input")), ["file2"]) def test_repeated_files(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', 'input') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "input") def test_overwrite(self): - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('dir2/file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("dir2/file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") # Overwriting regular files and directories should be supported - os.mkdir('output/input') - os.mkdir('output/input/file1') - os.mkdir('output/input/dir2') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - self.assert_dirs_equal('input', 'output/input') + os.mkdir("output/input") + os.mkdir("output/input/file1") + os.mkdir("output/input/dir2") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + self.assert_dirs_equal("input", "output/input") # But non-empty dirs should fail - os.unlink('output/input/file1') - os.mkdir('output/input/file1') - os.mkdir('output/input/file1/dir') - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=1) + os.unlink("output/input/file1") + os.mkdir("output/input/file1") + os.mkdir("output/input/file1/dir") + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=1) def test_rename(self): - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('dir2/file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.2', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'rename', 'test', 'test.3') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.2', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'rename', 'test.2', 'test.4') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.3', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.4', '--dry-run') + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("dir2/file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "extract", "test.2", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "rename", "test", "test.3") + self.cmd(f"--repo={self.repository_location}", "extract", "test.2", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "rename", "test.2", "test.4") + self.cmd(f"--repo={self.repository_location}", "extract", "test.3", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "extract", "test.4", "--dry-run") # Make sure both archives have been renamed with Repository(self.repository_path) as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) self.assert_equal(len(manifest.archives), 2) - self.assert_in('test.3', manifest.archives) - self.assert_in('test.4', manifest.archives) + self.assert_in("test.3", manifest.archives) + self.assert_in("test.4", manifest.archives) def test_info(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - info_repo = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert 'Original size:' in info_repo - info_archive = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test') - assert 'Archive name: test\n' in info_archive - info_archive = self.cmd(f'--repo={self.repository_location}', 'info', '--first', '1') - assert 'Archive name: test\n' in info_archive + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + info_repo = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "Original size:" in info_repo + info_archive = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test") + assert "Archive name: test\n" in info_archive + info_archive = self.cmd(f"--repo={self.repository_location}", "info", "--first", "1") + assert "Archive name: test\n" in info_archive def test_info_json(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - info_repo = json.loads(self.cmd(f'--repo={self.repository_location}', 'rinfo', '--json')) - repository = info_repo['repository'] - assert len(repository['id']) == 64 - assert 'last_modified' in repository - assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise - assert info_repo['encryption']['mode'] == RK_ENCRYPTION[13:] - assert 'keyfile' not in info_repo['encryption'] - cache = info_repo['cache'] - stats = cache['stats'] + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + info_repo = json.loads(self.cmd(f"--repo={self.repository_location}", "rinfo", "--json")) + repository = info_repo["repository"] + assert len(repository["id"]) == 64 + assert "last_modified" in repository + assert datetime.strptime(repository["last_modified"], ISO_FORMAT) # must not raise + assert info_repo["encryption"]["mode"] == RK_ENCRYPTION[13:] + assert "keyfile" not in info_repo["encryption"] + cache = info_repo["cache"] + stats = cache["stats"] assert all(isinstance(o, int) for o in stats.values()) - assert all(key in stats for key in ('total_chunks', 'total_size', 'total_unique_chunks', 'unique_size')) + assert all(key in stats for key in ("total_chunks", "total_size", "total_unique_chunks", "unique_size")) - info_archive = json.loads(self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test', '--json')) - assert info_repo['repository'] == info_archive['repository'] - assert info_repo['cache'] == info_archive['cache'] - archives = info_archive['archives'] + info_archive = json.loads(self.cmd(f"--repo={self.repository_location}", "info", "-a", "test", "--json")) + assert info_repo["repository"] == info_archive["repository"] + assert info_repo["cache"] == info_archive["cache"] + archives = info_archive["archives"] assert len(archives) == 1 archive = archives[0] - assert archive['name'] == 'test' - assert isinstance(archive['command_line'], list) - assert isinstance(archive['duration'], float) - assert len(archive['id']) == 64 - assert 'stats' in archive - assert datetime.strptime(archive['start'], ISO_FORMAT) - assert datetime.strptime(archive['end'], ISO_FORMAT) + assert archive["name"] == "test" + assert isinstance(archive["command_line"], list) + assert isinstance(archive["duration"], float) + assert len(archive["id"]) == 64 + assert "stats" in archive + assert datetime.strptime(archive["start"], ISO_FORMAT) + assert datetime.strptime(archive["end"], ISO_FORMAT) def test_info_json_of_empty_archive(self): """See https://github.com/borgbackup/borg/issues/6120""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - info_repo = json.loads(self.cmd(f'--repo={self.repository_location}', 'info', '--json', '--first=1')) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + info_repo = json.loads(self.cmd(f"--repo={self.repository_location}", "info", "--json", "--first=1")) assert info_repo["archives"] == [] - info_repo = json.loads(self.cmd(f'--repo={self.repository_location}', 'info', '--json', '--last=1')) + info_repo = json.loads(self.cmd(f"--repo={self.repository_location}", "info", "--json", "--last=1")) assert info_repo["archives"] == [] def test_comment(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input', '--comment', 'this is the comment') - self.cmd(f'--repo={self.repository_location}', 'create', 'test3', 'input', '--comment', '"deleted" comment') - self.cmd(f'--repo={self.repository_location}', 'create', 'test4', 'input', '--comment', 'preserved comment') - assert 'Comment: \n' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test1') - assert 'Comment: this is the comment' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test2') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input", "--comment", "this is the comment") + self.cmd(f"--repo={self.repository_location}", "create", "test3", "input", "--comment", '"deleted" comment') + self.cmd(f"--repo={self.repository_location}", "create", "test4", "input", "--comment", "preserved comment") + assert "Comment: \n" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test1") + assert "Comment: this is the comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test2") - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test1', '--comment', 'added comment') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test2', '--comment', 'modified comment') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test3', '--comment', '') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test4', '12345') - assert 'Comment: added comment' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test1') - assert 'Comment: modified comment' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test2') - assert 'Comment: \n' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test3') - assert 'Comment: preserved comment' in self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test4') + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test1", "--comment", "added comment") + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test2", "--comment", "modified comment") + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test3", "--comment", "") + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test4", "12345") + assert "Comment: added comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test1") + assert "Comment: modified comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test2") + assert "Comment: \n" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test3") + assert "Comment: preserved comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test4") def test_delete(self): - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('dir2/file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test.3', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'another_test.1', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'another_test.2', 'input') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.2', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'delete', '--glob-archives', 'another_*') - self.cmd(f'--repo={self.repository_location}', 'delete', '--last', '1') - self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test.2', '--dry-run') - output = self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test.2', '--stats') - self.assert_in('Original size: -', output) # negative size == deleted data + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("dir2/file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test.3", "input") + self.cmd(f"--repo={self.repository_location}", "create", "another_test.1", "input") + self.cmd(f"--repo={self.repository_location}", "create", "another_test.2", "input") + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "extract", "test.2", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "delete", "--glob-archives", "another_*") + self.cmd(f"--repo={self.repository_location}", "delete", "--last", "1") + self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test") + self.cmd(f"--repo={self.repository_location}", "extract", "test.2", "--dry-run") + output = self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test.2", "--stats") + self.assert_in("Original size: -", output) # negative size == deleted data # Make sure all data except the manifest has been deleted with Repository(self.repository_path) as repository: self.assert_equal(len(repository), 1) def test_delete_multiple(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test3', 'input') - self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test1') - self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test2') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test3', '--dry-run') - self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test3') - assert not self.cmd(f'--repo={self.repository_location}', 'rlist') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test3", "input") + self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test1") + self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test2") + self.cmd(f"--repo={self.repository_location}", "extract", "test3", "--dry-run") + self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test3") + assert not self.cmd(f"--repo={self.repository_location}", "rlist") def test_delete_repo(self): - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('dir2/file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test.2', 'input') - os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'no' - self.cmd(f'--repo={self.repository_location}', 'rdelete', exit_code=2) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("dir2/file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test.2", "input") + os.environ["BORG_DELETE_I_KNOW_WHAT_I_AM_DOING"] = "no" + self.cmd(f"--repo={self.repository_location}", "rdelete", exit_code=2) assert os.path.exists(self.repository_path) - os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' - self.cmd(f'--repo={self.repository_location}', 'rdelete') + os.environ["BORG_DELETE_I_KNOW_WHAT_I_AM_DOING"] = "YES" + self.cmd(f"--repo={self.repository_location}", "rdelete") # Make sure the repo is gone self.assertFalse(os.path.exists(self.repository_path)) def test_delete_force(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.create_src_archive("test") with Repository(self.repository_path, exclusive=True) as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - archive = Archive(repository, key, manifest, 'test') + archive = Archive(repository, key, manifest, "test") for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): repository.delete(item.chunks[-1].id) break else: assert False # missed the file repository.commit(compact=False) - output = self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test', '--force') - self.assert_in('deleted archive was corrupted', output) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_not_in('test', output) + output = self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test", "--force") + self.assert_in("deleted archive was corrupted", output) + self.cmd(f"--repo={self.repository_location}", "check", "--repair") + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_not_in("test", output) def test_delete_double_force(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.create_src_archive("test") with Repository(self.repository_path, exclusive=True) as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - archive = Archive(repository, key, manifest, 'test') + archive = Archive(repository, key, manifest, "test") id = archive.metadata.items[0] - repository.put(id, b'corrupted items metadata stream chunk') + repository.put(id, b"corrupted items metadata stream chunk") repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'delete', '-a', 'test', '--force', '--force') - self.cmd(f'--repo={self.repository_location}', 'check', '--repair') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_not_in('test', output) + self.cmd(f"--repo={self.repository_location}", "delete", "-a", "test", "--force", "--force") + self.cmd(f"--repo={self.repository_location}", "check", "--repair") + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_not_in("test", output) def test_corrupted_repository(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--dry-run') - output = self.cmd(f'--repo={self.repository_location}', 'check', '--show-version') - self.assert_in('borgbackup version', output) # implied output even without --info given - self.assert_not_in('Starting repository check', output) # --info not given for root logger + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--dry-run") + output = self.cmd(f"--repo={self.repository_location}", "check", "--show-version") + self.assert_in("borgbackup version", output) # implied output even without --info given + self.assert_not_in("Starting repository check", output) # --info not given for root logger - name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[1] - with open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+b') as fd: + name = sorted(os.listdir(os.path.join(self.tmpdir, "repository", "data", "0")), reverse=True)[1] + with open(os.path.join(self.tmpdir, "repository", "data", "0", name), "r+b") as fd: fd.seek(100) - fd.write(b'XXXX') - output = self.cmd(f'--repo={self.repository_location}', 'check', '--info', exit_code=1) - self.assert_in('Starting repository check', output) # --info given for root logger + fd.write(b"XXXX") + output = self.cmd(f"--repo={self.repository_location}", "check", "--info", exit_code=1) + self.assert_in("Starting repository check", output) # --info given for root logger def test_readonly_check(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'check', '--verify-data', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "check", "--verify-data", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'check', '--verify-data') + self.cmd(f"--repo={self.repository_location}", "check", "--verify-data") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'check', '--verify-data', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "check", "--verify-data", "--bypass-lock") def test_readonly_diff(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('a') - self.create_src_archive('b') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("a") + self.create_src_archive("b") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'diff', 'a', 'b', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "diff", "a", "b", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'diff', 'a', 'b') + self.cmd(f"--repo={self.repository_location}", "diff", "a", "b") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'diff', 'a', 'b', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "diff", "a", "b", "--bypass-lock") def test_readonly_export_tar(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'test.tar', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "export-tar", "test", "test.tar", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'test.tar') + self.cmd(f"--repo={self.repository_location}", "export-tar", "test", "test.tar") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'test.tar', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "export-tar", "test", "test.tar", "--bypass-lock") def test_readonly_extract(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') + self.cmd(f"--repo={self.repository_location}", "extract", "test") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--bypass-lock") def test_readonly_info(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'rinfo', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "rinfo", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'rinfo', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "rinfo", "--bypass-lock") def test_readonly_list(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'rlist', exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "rlist", exit_code=EXIT_ERROR) else: with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo: - self.cmd(f'--repo={self.repository_location}', 'rlist') + self.cmd(f"--repo={self.repository_location}", "rlist") if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - self.cmd(f'--repo={self.repository_location}', 'rlist', '--bypass-lock') + self.cmd(f"--repo={self.repository_location}", "rlist", "--bypass-lock") - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_readonly_mount(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('test') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") with self.read_only(self.repository_path): # verify that command normally doesn't work with read-only repo if self.FORK_DEFAULT: @@ -1788,22 +1930,22 @@ def test_readonly_mount(self): with self.fuse_mount(self.repository_location, fork=False): pass if isinstance(excinfo.value, RemoteRepository.RPCError): - assert excinfo.value.exception_class == 'LockFailed' + assert excinfo.value.exception_class == "LockFailed" # verify that command works with read-only repo when using --bypass-lock - with self.fuse_mount(self.repository_location, None, '--bypass-lock'): + with self.fuse_mount(self.repository_location, None, "--bypass-lock"): pass - @pytest.mark.skipif('BORG_TESTS_IGNORE_MODES' in os.environ, reason='modes unreliable') + @pytest.mark.skipif("BORG_TESTS_IGNORE_MODES" in os.environ, reason="modes unreliable") def test_umask(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") mode = os.stat(self.repository_path).st_mode self.assertEqual(stat.S_IMODE(mode), 0o700) def test_create_dry_run(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '--dry-run', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "--dry-run", "test", "input") # Make sure no archive has been created with Repository(self.repository_path) as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) @@ -1812,7 +1954,7 @@ def test_create_dry_run(self): def add_unknown_feature(self, operation): with Repository(self.repository_path, exclusive=True) as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - manifest.config['feature_flags'] = {operation.value: {'mandatory': ['unknown-feature']}} + manifest.config["feature_flags"] = {operation.value: {"mandatory": ["unknown-feature"]}} manifest.write() repository.commit(compact=False) @@ -1822,68 +1964,68 @@ def cmd_raises_unknown_feature(self, args): else: with pytest.raises(MandatoryFeatureUnsupported) as excinfo: self.cmd(*args) - assert excinfo.value.args == (['unknown-feature'],) + assert excinfo.value.args == (["unknown-feature"],) def test_unknown_feature_on_create(self): - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) self.add_unknown_feature(Manifest.Operation.WRITE) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'create', 'test', 'input']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "create", "test", "input"]) def test_unknown_feature_on_cache_sync(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") self.add_unknown_feature(Manifest.Operation.READ) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'create', 'test', 'input']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "create", "test", "input"]) def test_unknown_feature_on_change_passphrase(self): - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) self.add_unknown_feature(Manifest.Operation.CHECK) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'key', 'change-passphrase']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "key", "change-passphrase"]) def test_unknown_feature_on_read(self): - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") self.add_unknown_feature(Manifest.Operation.READ) - with changedir('output'): - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'extract', 'test']) + with changedir("output"): + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "extract", "test"]) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'rlist']) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'info', '-a', 'test']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "rlist"]) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "info", "-a", "test"]) def test_unknown_feature_on_rename(self): - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") self.add_unknown_feature(Manifest.Operation.CHECK) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'rename', 'test', 'other']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "rename", "test", "other"]) def test_unknown_feature_on_delete(self): - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") self.add_unknown_feature(Manifest.Operation.DELETE) # delete of an archive raises - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'delete', '-a', 'test']) - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}', 'prune', '--keep-daily=3']) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "delete", "-a", "test"]) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}", "prune", "--keep-daily=3"]) # delete of the whole repository ignores features - self.cmd(f'--repo={self.repository_location}', 'rdelete') + self.cmd(f"--repo={self.repository_location}", "rdelete") - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_unknown_feature_on_mount(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") self.add_unknown_feature(Manifest.Operation.READ) - mountpoint = os.path.join(self.tmpdir, 'mountpoint') + mountpoint = os.path.join(self.tmpdir, "mountpoint") os.mkdir(mountpoint) # XXX this might hang if it doesn't raise an error - self.cmd_raises_unknown_feature([f'--repo={self.repository_location}::test', 'mount', mountpoint]) + self.cmd_raises_unknown_feature([f"--repo={self.repository_location}::test", "mount", mountpoint]) @pytest.mark.allow_cache_wipe def test_unknown_mandatory_feature_in_cache(self): if self.prefix: - path_prefix = 'ssh://__testsuite__' + path_prefix = "ssh://__testsuite__" else: - path_prefix = '' + path_prefix = "" - print(self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION)) + print(self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)) with Repository(self.repository_path, exclusive=True) as repository: if path_prefix: @@ -1891,11 +2033,11 @@ def test_unknown_mandatory_feature_in_cache(self): manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) with Cache(repository, key, manifest) as cache: cache.begin_txn() - cache.cache_config.mandatory_features = {'unknown-feature'} + cache.cache_config.mandatory_features = {"unknown-feature"} cache.commit() if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") else: called = False wipe_cache_safe = LocalCache.wipe_cache @@ -1905,8 +2047,8 @@ def wipe_wrapper(*args): called = True wipe_cache_safe(*args) - with patch.object(LocalCache, 'wipe_cache', wipe_wrapper): - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + with patch.object(LocalCache, "wipe_cache", wipe_wrapper): + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") assert called @@ -1918,30 +2060,30 @@ def wipe_wrapper(*args): assert cache.cache_config.mandatory_features == set() def test_progress_on(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test4', 'input', '--progress') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "test4", "input", "--progress") self.assert_in("\r", output) def test_progress_off(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test5', 'input') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "test5", "input") self.assert_not_in("\r", output) def test_file_status(self): """test that various file status show expected results clearly incomplete: only tests for the weird "unchanged" status for now""" - self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file("file1", size=1024 * 80) time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', '--list', 'test', 'input') + self.create_regular_file("file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "--list", "test", "input") self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) # should find first file as unmodified - output = self.cmd(f'--repo={self.repository_location}', 'create', '--list', 'test2', 'input') + output = self.cmd(f"--repo={self.repository_location}", "create", "--list", "test2", "input") self.assert_in("U input/file1", output) # this is expected, although surprising, for why, see: # https://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file @@ -1949,115 +2091,128 @@ def test_file_status(self): def test_file_status_cs_cache_mode(self): """test that a changed file with faked "previous" mtime still gets backed up in ctime,size cache_mode""" - self.create_regular_file('file1', contents=b'123') + self.create_regular_file("file1", contents=b"123") time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=10) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input', - '--list', '--files-cache=ctime,size') + self.create_regular_file("file2", size=10) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd( + f"--repo={self.repository_location}", "create", "test1", "input", "--list", "--files-cache=ctime,size" + ) # modify file1, but cheat with the mtime (and atime) and also keep same size: - st = os.stat('input/file1') - self.create_regular_file('file1', contents=b'321') - os.utime('input/file1', ns=(st.st_atime_ns, st.st_mtime_ns)) + st = os.stat("input/file1") + self.create_regular_file("file1", contents=b"321") + os.utime("input/file1", ns=(st.st_atime_ns, st.st_mtime_ns)) # this mode uses ctime for change detection, so it should find file1 as modified - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input', - '--list', '--files-cache=ctime,size') + output = self.cmd( + f"--repo={self.repository_location}", "create", "test2", "input", "--list", "--files-cache=ctime,size" + ) self.assert_in("M input/file1", output) def test_file_status_ms_cache_mode(self): """test that a chmod'ed file with no content changes does not get chunked again in mtime,size cache_mode""" - self.create_regular_file('file1', size=10) + self.create_regular_file("file1", size=10) time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=10) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', - '--list', '--files-cache=mtime,size', 'test1', 'input') + self.create_regular_file("file2", size=10) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd( + f"--repo={self.repository_location}", "create", "--list", "--files-cache=mtime,size", "test1", "input" + ) # change mode of file1, no content change: - st = os.stat('input/file1') - os.chmod('input/file1', st.st_mode ^ stat.S_IRWXO) # this triggers a ctime change, but mtime is unchanged + st = os.stat("input/file1") + os.chmod("input/file1", st.st_mode ^ stat.S_IRWXO) # this triggers a ctime change, but mtime is unchanged # this mode uses mtime for change detection, so it should find file1 as unmodified - output = self.cmd(f'--repo={self.repository_location}', 'create', - '--list', '--files-cache=mtime,size', 'test2', 'input') + output = self.cmd( + f"--repo={self.repository_location}", "create", "--list", "--files-cache=mtime,size", "test2", "input" + ) self.assert_in("U input/file1", output) def test_file_status_rc_cache_mode(self): """test that files get rechunked unconditionally in rechunk,ctime cache mode""" - self.create_regular_file('file1', size=10) + self.create_regular_file("file1", size=10) time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=10) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', - '--list', '--files-cache=rechunk,ctime', 'test1', 'input') + self.create_regular_file("file2", size=10) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd( + f"--repo={self.repository_location}", "create", "--list", "--files-cache=rechunk,ctime", "test1", "input" + ) # no changes here, but this mode rechunks unconditionally - output = self.cmd(f'--repo={self.repository_location}', 'create', - '--list', '--files-cache=rechunk,ctime', 'test2', 'input') + output = self.cmd( + f"--repo={self.repository_location}", "create", "--list", "--files-cache=rechunk,ctime", "test2", "input" + ) self.assert_in("A input/file1", output) def test_file_status_excluded(self): """test that excluded paths are listed""" - self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file("file1", size=1024 * 80) time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) if has_lchflags: - self.create_regular_file('file3', size=1024 * 80) - platform.set_flags(os.path.join(self.input_path, 'file3'), stat.UF_NODUMP) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'create', '--list', '--exclude-nodump', 'test', 'input') + self.create_regular_file("file3", size=1024 * 80) + platform.set_flags(os.path.join(self.input_path, "file3"), stat.UF_NODUMP) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "create", "--list", "--exclude-nodump", "test", "input") self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) if has_lchflags: self.assert_in("x input/file3", output) # should find second file as excluded - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input', - '--list', '--exclude-nodump', '--exclude', '*/file2') + output = self.cmd( + f"--repo={self.repository_location}", + "create", + "test1", + "input", + "--list", + "--exclude-nodump", + "--exclude", + "*/file2", + ) self.assert_in("U input/file1", output) self.assert_in("x input/file2", output) if has_lchflags: self.assert_in("x input/file3", output) def test_create_json(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - create_info = json.loads(self.cmd(f'--repo={self.repository_location}', 'create', '--json', - 'test', 'input')) + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + create_info = json.loads(self.cmd(f"--repo={self.repository_location}", "create", "--json", "test", "input")) # The usual keys - assert 'encryption' in create_info - assert 'repository' in create_info - assert 'cache' in create_info - assert 'last_modified' in create_info['repository'] + assert "encryption" in create_info + assert "repository" in create_info + assert "cache" in create_info + assert "last_modified" in create_info["repository"] - archive = create_info['archive'] - assert archive['name'] == 'test' - assert isinstance(archive['command_line'], list) - assert isinstance(archive['duration'], float) - assert len(archive['id']) == 64 - assert 'stats' in archive + archive = create_info["archive"] + assert archive["name"] == "test" + assert isinstance(archive["command_line"], list) + assert isinstance(archive["duration"], float) + assert len(archive["id"]) == 64 + assert "stats" in archive def test_create_topical(self): - self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file("file1", size=1024 * 80) time.sleep(1) # file2 must have newer timestamps than file1 - self.create_regular_file('file2', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.create_regular_file("file2", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # no listing by default - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.assert_not_in('file1', output) + output = self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.assert_not_in("file1", output) # shouldn't be listed even if unchanged - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') - self.assert_not_in('file1', output) + output = self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") + self.assert_not_in("file1", output) # should list the file as unchanged - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input', '--list', '--filter=U') - self.assert_in('file1', output) + output = self.cmd(f"--repo={self.repository_location}", "create", "test1", "input", "--list", "--filter=U") + self.assert_in("file1", output) # should *not* list the file as changed - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input', '--list', '--filter=AM') - self.assert_not_in('file1', output) + output = self.cmd(f"--repo={self.repository_location}", "create", "test2", "input", "--list", "--filter=AM") + self.assert_not_in("file1", output) # change the file - self.create_regular_file('file1', size=1024 * 100) + self.create_regular_file("file1", size=1024 * 100) # should list the file as changed - output = self.cmd(f'--repo={self.repository_location}', 'create', 'test3', 'input', '--list', '--filter=AM') - self.assert_in('file1', output) + output = self.cmd(f"--repo={self.repository_location}", "create", "test3", "input", "--list", "--filter=AM") + self.assert_in("file1", output) - @pytest.mark.skipif(not are_fifos_supported(), reason='FIFOs not supported') + @pytest.mark.skipif(not are_fifos_supported(), reason="FIFOs not supported") def test_create_read_special_symlink(self): from threading import Thread @@ -2068,33 +2223,33 @@ def fifo_feeder(fifo_fn, data): finally: os.close(fd) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - data = b'foobar' * 1000 + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + data = b"foobar" * 1000 - fifo_fn = os.path.join(self.input_path, 'fifo') - link_fn = os.path.join(self.input_path, 'link_fifo') + fifo_fn = os.path.join(self.input_path, "fifo") + link_fn = os.path.join(self.input_path, "link_fifo") os.mkfifo(fifo_fn) os.symlink(fifo_fn, link_fn) t = Thread(target=fifo_feeder, args=(fifo_fn, data)) t.start() try: - self.cmd(f'--repo={self.repository_location}', 'create', '--read-special', 'test', 'input/link_fifo') + self.cmd(f"--repo={self.repository_location}", "create", "--read-special", "test", "input/link_fifo") finally: t.join() - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - fifo_fn = 'input/link_fifo' - with open(fifo_fn, 'rb') as f: + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + fifo_fn = "input/link_fifo" + with open(fifo_fn, "rb") as f: extracted_data = f.read() assert extracted_data == data def test_create_read_special_broken_symlink(self): - os.symlink('somewhere does not exist', os.path.join(self.input_path, 'link')) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '--read-special', 'test', 'input') - output = self.cmd(f'--repo={self.repository_location}', 'list', 'test') - assert 'input/link -> somewhere does not exist' in output + os.symlink("somewhere does not exist", os.path.join(self.input_path, "link")) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "--read-special", "test", "input") + output = self.cmd(f"--repo={self.repository_location}", "list", "test") + assert "input/link -> somewhere does not exist" in output # def test_cmdline_compatibility(self): # self.create_regular_file('file1', size=1024 * 80) @@ -2104,42 +2259,42 @@ def test_create_read_special_broken_symlink(self): # self.assert_in('"--old" has been deprecated. Use "--new" instead', output) def test_prune_repository(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', src_dir) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test2", src_dir) # these are not really a checkpoints, but they look like some: - self.cmd(f'--repo={self.repository_location}', 'create', 'test3.checkpoint', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test3.checkpoint.1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test4.checkpoint', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=1') - assert re.search(r'Would prune:\s+test1', output) + self.cmd(f"--repo={self.repository_location}", "create", "test3.checkpoint", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test3.checkpoint.1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test4.checkpoint", src_dir) + output = self.cmd(f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=1") + assert re.search(r"Would prune:\s+test1", output) # must keep the latest non-checkpoint archive: - assert re.search(r'Keeping archive \(rule: daily #1\):\s+test2', output) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output) # must keep the latest checkpoint archive: - assert re.search(r'Keeping checkpoint archive:\s+test4.checkpoint', output) - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--consider-checkpoints') - self.assert_in('test1', output) - self.assert_in('test2', output) - self.assert_in('test3.checkpoint', output) - self.assert_in('test3.checkpoint.1', output) - self.assert_in('test4.checkpoint', output) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=1') - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--consider-checkpoints') - self.assert_not_in('test1', output) + assert re.search(r"Keeping checkpoint archive:\s+test4.checkpoint", output) + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints") + self.assert_in("test1", output) + self.assert_in("test2", output) + self.assert_in("test3.checkpoint", output) + self.assert_in("test3.checkpoint.1", output) + self.assert_in("test4.checkpoint", output) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1") + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints") + self.assert_not_in("test1", output) # the latest non-checkpoint archive must be still there: - self.assert_in('test2', output) + self.assert_in("test2", output) # only the latest checkpoint archive must still be there: - self.assert_not_in('test3.checkpoint', output) - self.assert_not_in('test3.checkpoint.1', output) - self.assert_in('test4.checkpoint', output) + self.assert_not_in("test3.checkpoint", output) + self.assert_not_in("test3.checkpoint.1", output) + self.assert_in("test4.checkpoint", output) # now we supersede the latest checkpoint by a successful backup: - self.cmd(f'--repo={self.repository_location}', 'create', 'test5', src_dir) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=2') - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--consider-checkpoints') + self.cmd(f"--repo={self.repository_location}", "create", "test5", src_dir) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=2") + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints") # all checkpoints should be gone now: - self.assert_not_in('checkpoint', output) + self.assert_not_in("checkpoint", output) # the latest archive must be still there - self.assert_in('test5', output) + self.assert_in("test5", output) # Given a date and time in local tz, create a UTC timestamp string suitable # for create --timestamp command line option @@ -2148,351 +2303,399 @@ def _to_utc_timestamp(self, year, month, day, hour, minute, second): return dtime.astimezone(dateutil.tz.UTC).strftime("%Y-%m-%dT%H:%M:%S") def _create_archive_ts(self, name, y, m, d, H=0, M=0, S=0): - self.cmd(f'--repo={self.repository_location}', 'create', - '--timestamp', self._to_utc_timestamp(y, m, d, H, M, S), name, src_dir) + self.cmd( + f"--repo={self.repository_location}", + "create", + "--timestamp", + self._to_utc_timestamp(y, m, d, H, M, S), + name, + src_dir, + ) # This test must match docs/misc/prune-example.txt def test_prune_repository_example(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # Archives that will be kept, per the example # Oldest archive - self._create_archive_ts('test01', 2015, 1, 1) + self._create_archive_ts("test01", 2015, 1, 1) # 6 monthly archives - self._create_archive_ts('test02', 2015, 6, 30) - self._create_archive_ts('test03', 2015, 7, 31) - self._create_archive_ts('test04', 2015, 8, 31) - self._create_archive_ts('test05', 2015, 9, 30) - self._create_archive_ts('test06', 2015, 10, 31) - self._create_archive_ts('test07', 2015, 11, 30) + self._create_archive_ts("test02", 2015, 6, 30) + self._create_archive_ts("test03", 2015, 7, 31) + self._create_archive_ts("test04", 2015, 8, 31) + self._create_archive_ts("test05", 2015, 9, 30) + self._create_archive_ts("test06", 2015, 10, 31) + self._create_archive_ts("test07", 2015, 11, 30) # 14 daily archives - self._create_archive_ts('test08', 2015, 12, 17) - self._create_archive_ts('test09', 2015, 12, 18) - self._create_archive_ts('test10', 2015, 12, 20) - self._create_archive_ts('test11', 2015, 12, 21) - self._create_archive_ts('test12', 2015, 12, 22) - self._create_archive_ts('test13', 2015, 12, 23) - self._create_archive_ts('test14', 2015, 12, 24) - self._create_archive_ts('test15', 2015, 12, 25) - self._create_archive_ts('test16', 2015, 12, 26) - self._create_archive_ts('test17', 2015, 12, 27) - self._create_archive_ts('test18', 2015, 12, 28) - self._create_archive_ts('test19', 2015, 12, 29) - self._create_archive_ts('test20', 2015, 12, 30) - self._create_archive_ts('test21', 2015, 12, 31) + self._create_archive_ts("test08", 2015, 12, 17) + self._create_archive_ts("test09", 2015, 12, 18) + self._create_archive_ts("test10", 2015, 12, 20) + self._create_archive_ts("test11", 2015, 12, 21) + self._create_archive_ts("test12", 2015, 12, 22) + self._create_archive_ts("test13", 2015, 12, 23) + self._create_archive_ts("test14", 2015, 12, 24) + self._create_archive_ts("test15", 2015, 12, 25) + self._create_archive_ts("test16", 2015, 12, 26) + self._create_archive_ts("test17", 2015, 12, 27) + self._create_archive_ts("test18", 2015, 12, 28) + self._create_archive_ts("test19", 2015, 12, 29) + self._create_archive_ts("test20", 2015, 12, 30) + self._create_archive_ts("test21", 2015, 12, 31) # Additional archives that would be pruned # The second backup of the year - self._create_archive_ts('test22', 2015, 1, 2) + self._create_archive_ts("test22", 2015, 1, 2) # The next older monthly backup - self._create_archive_ts('test23', 2015, 5, 31) + self._create_archive_ts("test23", 2015, 5, 31) # The next older daily backup - self._create_archive_ts('test24', 2015, 12, 16) - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=14', '--keep-monthly=6', '--keep-yearly=1') + self._create_archive_ts("test24", 2015, 12, 16) + output = self.cmd( + f"--repo={self.repository_location}", + "prune", + "--list", + "--dry-run", + "--keep-daily=14", + "--keep-monthly=6", + "--keep-yearly=1", + ) # Prune second backup of the year - assert re.search(r'Would prune:\s+test22', output) + assert re.search(r"Would prune:\s+test22", output) # Prune next older monthly and daily backups - assert re.search(r'Would prune:\s+test23', output) - assert re.search(r'Would prune:\s+test24', output) + assert re.search(r"Would prune:\s+test23", output) + assert re.search(r"Would prune:\s+test24", output) # Must keep the other 21 backups # Yearly is kept as oldest archive - assert re.search(r'Keeping archive \(rule: yearly\[oldest\] #1\):\s+test01', output) + assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #1\):\s+test01", output) for i in range(1, 7): - assert re.search(r'Keeping archive \(rule: monthly #' + str(i) + r'\):\s+test' + ("%02d" % (8-i)), output) + assert re.search(r"Keeping archive \(rule: monthly #" + str(i) + r"\):\s+test" + ("%02d" % (8 - i)), output) for i in range(1, 15): - assert re.search(r'Keeping archive \(rule: daily #' + str(i) + r'\):\s+test' + ("%02d" % (22-i)), output) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') + assert re.search(r"Keeping archive \(rule: daily #" + str(i) + r"\):\s+test" + ("%02d" % (22 - i)), output) + output = self.cmd(f"--repo={self.repository_location}", "rlist") # Nothing pruned after dry run for i in range(1, 25): - self.assert_in('test%02d' % i, output) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=14', '--keep-monthly=6', '--keep-yearly=1') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') + self.assert_in("test%02d" % i, output) + self.cmd( + f"--repo={self.repository_location}", "prune", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1" + ) + output = self.cmd(f"--repo={self.repository_location}", "rlist") # All matching backups plus oldest kept for i in range(1, 22): - self.assert_in('test%02d' % i, output) + self.assert_in("test%02d" % i, output) # Other backups have been pruned for i in range(22, 25): - self.assert_not_in('test%02d' % i, output) + self.assert_not_in("test%02d" % i, output) # With an initial and daily backup, prune daily until oldest is replaced by a monthly backup def test_prune_retain_and_expire_oldest(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # Initial backup - self._create_archive_ts('original_archive', 2020, 9, 1, 11, 15) + self._create_archive_ts("original_archive", 2020, 9, 1, 11, 15) # Archive and prune daily for 30 days for i in range(1, 31): - self._create_archive_ts('september%02d' % i, 2020, 9, i, 12) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=7', '--keep-monthly=1') + self._create_archive_ts("september%02d" % i, 2020, 9, i, 12) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1") # Archive and prune 6 days into the next month for i in range(1, 7): - self._create_archive_ts('october%02d' % i, 2020, 10, i, 12) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=7', '--keep-monthly=1') + self._create_archive_ts("october%02d" % i, 2020, 10, i, 12) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1") # Oldest backup is still retained - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=7', '--keep-monthly=1') - assert re.search(r'Keeping archive \(rule: monthly\[oldest\] #1' + r'\):\s+original_archive', output) + output = self.cmd( + f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1" + ) + assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output) # Archive one more day and prune. - self._create_archive_ts('october07', 2020, 10, 7, 12) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=7', '--keep-monthly=1') + self._create_archive_ts("october07", 2020, 10, 7, 12) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=7", "--keep-monthly=1") # Last day of previous month is retained as monthly, and oldest is expired. - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=7', '--keep-monthly=1') - assert re.search(r'Keeping archive \(rule: monthly #1\):\s+september30', output) - self.assert_not_in('original_archive', output) + output = self.cmd( + f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1" + ) + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+september30", output) + self.assert_not_in("original_archive", output) def test_prune_repository_save_space(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=1') - assert re.search(r'Keeping archive \(rule: daily #1\):\s+test2', output) - assert re.search(r'Would prune:\s+test1', output) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_in('test1', output) - self.assert_in('test2', output) - self.cmd(f'--repo={self.repository_location}', 'prune', '--save-space', '--keep-daily=1') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_not_in('test1', output) - self.assert_in('test2', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test2", src_dir) + output = self.cmd(f"--repo={self.repository_location}", "prune", "--list", "--dry-run", "--keep-daily=1") + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test2", output) + assert re.search(r"Would prune:\s+test1", output) + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_in("test1", output) + self.assert_in("test2", output) + self.cmd(f"--repo={self.repository_location}", "prune", "--save-space", "--keep-daily=1") + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_not_in("test1", output) + self.assert_in("test2", output) def test_prune_repository_prefix(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'foo-2015-08-12-10:00', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'foo-2015-08-12-20:00', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'bar-2015-08-12-10:00', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'bar-2015-08-12-20:00', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=1', '--glob-archives=foo-*') - assert re.search(r'Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00', output) - assert re.search(r'Would prune:\s+foo-2015-08-12-10:00', output) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_in('foo-2015-08-12-10:00', output) - self.assert_in('foo-2015-08-12-20:00', output) - self.assert_in('bar-2015-08-12-10:00', output) - self.assert_in('bar-2015-08-12-20:00', output) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=1', '--glob-archives=foo-*') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_not_in('foo-2015-08-12-10:00', output) - self.assert_in('foo-2015-08-12-20:00', output) - self.assert_in('bar-2015-08-12-10:00', output) - self.assert_in('bar-2015-08-12-20:00', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "foo-2015-08-12-10:00", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "foo-2015-08-12-20:00", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "bar-2015-08-12-10:00", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "bar-2015-08-12-20:00", src_dir) + output = self.cmd( + f"--repo={self.repository_location}", + "prune", + "--list", + "--dry-run", + "--keep-daily=1", + "--glob-archives=foo-*", + ) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output) + assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output) + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_in("foo-2015-08-12-10:00", output) + self.assert_in("foo-2015-08-12-20:00", output) + self.assert_in("bar-2015-08-12-10:00", output) + self.assert_in("bar-2015-08-12-20:00", output) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1", "--glob-archives=foo-*") + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_not_in("foo-2015-08-12-10:00", output) + self.assert_in("foo-2015-08-12-20:00", output) + self.assert_in("bar-2015-08-12-10:00", output) + self.assert_in("bar-2015-08-12-20:00", output) def test_prune_repository_glob(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '2015-08-12-10:00-foo', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', '2015-08-12-20:00-foo', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', '2015-08-12-10:00-bar', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', '2015-08-12-20:00-bar', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'prune', '--list', '--dry-run', '--keep-daily=1', '--glob-archives=2015-*-foo') - assert re.search(r'Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo', output) - assert re.search(r'Would prune:\s+2015-08-12-10:00-foo', output) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_in('2015-08-12-10:00-foo', output) - self.assert_in('2015-08-12-20:00-foo', output) - self.assert_in('2015-08-12-10:00-bar', output) - self.assert_in('2015-08-12-20:00-bar', output) - self.cmd(f'--repo={self.repository_location}', 'prune', '--keep-daily=1', '--glob-archives=2015-*-foo') - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_not_in('2015-08-12-10:00-foo', output) - self.assert_in('2015-08-12-20:00-foo', output) - self.assert_in('2015-08-12-10:00-bar', output) - self.assert_in('2015-08-12-20:00-bar', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-10:00-foo", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-20:00-foo", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-10:00-bar", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "2015-08-12-20:00-bar", src_dir) + output = self.cmd( + f"--repo={self.repository_location}", + "prune", + "--list", + "--dry-run", + "--keep-daily=1", + "--glob-archives=2015-*-foo", + ) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output) + assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output) + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_in("2015-08-12-10:00-foo", output) + self.assert_in("2015-08-12-20:00-foo", output) + self.assert_in("2015-08-12-10:00-bar", output) + self.assert_in("2015-08-12-20:00-bar", output) + self.cmd(f"--repo={self.repository_location}", "prune", "--keep-daily=1", "--glob-archives=2015-*-foo") + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_not_in("2015-08-12-10:00-foo", output) + self.assert_in("2015-08-12-20:00-foo", output) + self.assert_in("2015-08-12-10:00-bar", output) + self.assert_in("2015-08-12-20:00-bar", output) def test_list_prefix(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test-1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'something-else-than-test-1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test-2', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--glob-archives=test-*') - self.assert_in('test-1', output) - self.assert_in('test-2', output) - self.assert_not_in('something-else', output) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test-1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "something-else-than-test-1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test-2", src_dir) + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--glob-archives=test-*") + self.assert_in("test-1", output) + self.assert_in("test-2", output) + self.assert_not_in("something-else", output) def test_list_format(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', src_dir) - output_1 = self.cmd(f'--repo={self.repository_location}', 'list', 'test') - output_2 = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}') - output_3 = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{mtime:%s} {path}{NL}') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", src_dir) + output_1 = self.cmd(f"--repo={self.repository_location}", "list", "test") + output_2 = self.cmd( + f"--repo={self.repository_location}", + "list", + "test", + "--format", + "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}", + ) + output_3 = self.cmd(f"--repo={self.repository_location}", "list", "test", "--format", "{mtime:%s} {path}{NL}") self.assertEqual(output_1, output_2) self.assertNotEqual(output_1, output_3) def test_archives_format(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '--comment', 'comment 1', 'test-1', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', '--comment', 'comment 2', 'test-2', src_dir) - output_1 = self.cmd(f'--repo={self.repository_location}', 'rlist') - output_2 = self.cmd(f'--repo={self.repository_location}', 'rlist', '--format', '{archive:<36} {time} [{id}]{NL}') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "--comment", "comment 1", "test-1", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "--comment", "comment 2", "test-2", src_dir) + output_1 = self.cmd(f"--repo={self.repository_location}", "rlist") + output_2 = self.cmd( + f"--repo={self.repository_location}", "rlist", "--format", "{archive:<36} {time} [{id}]{NL}" + ) self.assertEqual(output_1, output_2) - output_1 = self.cmd(f'--repo={self.repository_location}', 'rlist', '--short') - self.assertEqual(output_1, 'test-1\ntest-2\n') - output_1 = self.cmd(f'--repo={self.repository_location}', 'rlist', '--format', '{barchive}/') - self.assertEqual(output_1, 'test-1/test-2/') - output_3 = self.cmd(f'--repo={self.repository_location}', 'rlist', '--format', '{name} {comment}{NL}') - self.assert_in('test-1 comment 1\n', output_3) - self.assert_in('test-2 comment 2\n', output_3) + output_1 = self.cmd(f"--repo={self.repository_location}", "rlist", "--short") + self.assertEqual(output_1, "test-1\ntest-2\n") + output_1 = self.cmd(f"--repo={self.repository_location}", "rlist", "--format", "{barchive}/") + self.assertEqual(output_1, "test-1/test-2/") + output_3 = self.cmd(f"--repo={self.repository_location}", "rlist", "--format", "{name} {comment}{NL}") + self.assert_in("test-1 comment 1\n", output_3) + self.assert_in("test-2 comment 2\n", output_3) def test_list_hash(self): - self.create_regular_file('empty_file', size=0) - self.create_regular_file('amb', contents=b'a' * 1000000) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - output = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{sha256} {path}{NL}') + self.create_regular_file("empty_file", size=0) + self.create_regular_file("amb", contents=b"a" * 1000000) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + output = self.cmd(f"--repo={self.repository_location}", "list", "test", "--format", "{sha256} {path}{NL}") assert "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0 input/amb" in output assert "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 input/empty_file" in output def test_list_consider_checkpoints(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', src_dir) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", src_dir) # these are not really a checkpoints, but they look like some: - self.cmd(f'--repo={self.repository_location}', 'create', 'test2.checkpoint', src_dir) - self.cmd(f'--repo={self.repository_location}', 'create', 'test3.checkpoint.1', src_dir) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') + self.cmd(f"--repo={self.repository_location}", "create", "test2.checkpoint", src_dir) + self.cmd(f"--repo={self.repository_location}", "create", "test3.checkpoint.1", src_dir) + output = self.cmd(f"--repo={self.repository_location}", "rlist") assert "test1" in output assert "test2.checkpoint" not in output assert "test3.checkpoint.1" not in output - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--consider-checkpoints') + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--consider-checkpoints") assert "test1" in output assert "test2.checkpoint" in output assert "test3.checkpoint.1" in output def test_list_chunk_counts(self): - self.create_regular_file('empty_file', size=0) - self.create_regular_file('two_chunks') - with open(os.path.join(self.input_path, 'two_chunks'), 'wb') as fd: - fd.write(b'abba' * 2000000) - fd.write(b'baab' * 2000000) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - output = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{num_chunks} {unique_chunks} {path}{NL}') + self.create_regular_file("empty_file", size=0) + self.create_regular_file("two_chunks") + with open(os.path.join(self.input_path, "two_chunks"), "wb") as fd: + fd.write(b"abba" * 2000000) + fd.write(b"baab" * 2000000) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + output = self.cmd( + f"--repo={self.repository_location}", "list", "test", "--format", "{num_chunks} {unique_chunks} {path}{NL}" + ) assert "0 0 input/empty_file" in output assert "2 2 input/two_chunks" in output def test_list_size(self): - self.create_regular_file('compressible_file', size=10000) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', '-C', 'lz4', 'test', 'input') - output = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--format', '{size} {path}{NL}') + self.create_regular_file("compressible_file", size=10000) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "-C", "lz4", "test", "input") + output = self.cmd(f"--repo={self.repository_location}", "list", "test", "--format", "{size} {path}{NL}") size, path = output.split("\n")[1].split(" ") assert int(size) == 10000 def test_list_json(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - list_repo = json.loads(self.cmd(f'--repo={self.repository_location}', 'rlist', '--json')) - repository = list_repo['repository'] - assert len(repository['id']) == 64 - assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise - assert list_repo['encryption']['mode'] == RK_ENCRYPTION[13:] - assert 'keyfile' not in list_repo['encryption'] - archive0 = list_repo['archives'][0] - assert datetime.strptime(archive0['time'], ISO_FORMAT) # must not raise + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + list_repo = json.loads(self.cmd(f"--repo={self.repository_location}", "rlist", "--json")) + repository = list_repo["repository"] + assert len(repository["id"]) == 64 + assert datetime.strptime(repository["last_modified"], ISO_FORMAT) # must not raise + assert list_repo["encryption"]["mode"] == RK_ENCRYPTION[13:] + assert "keyfile" not in list_repo["encryption"] + archive0 = list_repo["archives"][0] + assert datetime.strptime(archive0["time"], ISO_FORMAT) # must not raise - list_archive = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines') + list_archive = self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines") items = [json.loads(s) for s in list_archive.splitlines()] assert len(items) == 2 file1 = items[1] - assert file1['path'] == 'input/file1' - assert file1['size'] == 81920 - assert datetime.strptime(file1['mtime'], ISO_FORMAT) # must not raise + assert file1["path"] == "input/file1" + assert file1["size"] == 81920 + assert datetime.strptime(file1["mtime"], ISO_FORMAT) # must not raise - list_archive = self.cmd(f'--repo={self.repository_location}', 'list', 'test', '--json-lines', '--format={sha256}') + list_archive = self.cmd( + f"--repo={self.repository_location}", "list", "test", "--json-lines", "--format={sha256}" + ) items = [json.loads(s) for s in list_archive.splitlines()] assert len(items) == 2 file1 = items[1] - assert file1['path'] == 'input/file1' - assert file1['sha256'] == 'b2915eb69f260d8d3c25249195f2c8f4f716ea82ec760ae929732c0262442b2b' + assert file1["path"] == "input/file1" + assert file1["sha256"] == "b2915eb69f260d8d3c25249195f2c8f4f716ea82ec760ae929732c0262442b2b" def test_log_json(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - log = self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '--log-json', '--list', '--debug') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + log = self.cmd( + f"--repo={self.repository_location}", "create", "test", "input", "--log-json", "--list", "--debug" + ) messages = {} # type -> message, one of each kind for line in log.splitlines(): msg = json.loads(line) - messages[msg['type']] = msg + messages[msg["type"]] = msg - file_status = messages['file_status'] - assert 'status' in file_status - assert file_status['path'].startswith('input') + file_status = messages["file_status"] + assert "status" in file_status + assert file_status["path"].startswith("input") - log_message = messages['log_message'] - assert isinstance(log_message['time'], float) - assert log_message['levelname'] == 'DEBUG' # there should only be DEBUG messages - assert isinstance(log_message['message'], str) + log_message = messages["log_message"] + assert isinstance(log_message["time"], float) + assert log_message["levelname"] == "DEBUG" # there should only be DEBUG messages + assert isinstance(log_message["message"], str) def test_debug_profile(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '--debug-profile=create.prof') - self.cmd('debug', 'convert-profile', 'create.prof', 'create.pyprof') - stats = pstats.Stats('create.pyprof') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "--debug-profile=create.prof") + self.cmd("debug", "convert-profile", "create.prof", "create.pyprof") + stats = pstats.Stats("create.pyprof") stats.strip_dirs() - stats.sort_stats('cumtime') + stats.sort_stats("cumtime") - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input', '--debug-profile=create.pyprof') - stats = pstats.Stats('create.pyprof') # Only do this on trusted data! + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input", "--debug-profile=create.pyprof") + stats = pstats.Stats("create.pyprof") # Only do this on trusted data! stats.strip_dirs() - stats.sort_stats('cumtime') + stats.sort_stats("cumtime") def test_common_options(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - log = self.cmd(f'--repo={self.repository_location}', '--debug', 'create', 'test', 'input') - assert 'security: read previous location' in log + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + log = self.cmd(f"--repo={self.repository_location}", "--debug", "create", "test", "input") + assert "security: read previous location" in log def test_change_passphrase(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + os.environ["BORG_NEW_PASSPHRASE"] = "newpassphrase" # here we have both BORG_PASSPHRASE and BORG_NEW_PASSPHRASE set: - self.cmd(f'--repo={self.repository_location}', 'key', 'change-passphrase') - os.environ['BORG_PASSPHRASE'] = 'newpassphrase' - self.cmd(f'--repo={self.repository_location}', 'rlist') + self.cmd(f"--repo={self.repository_location}", "key", "change-passphrase") + os.environ["BORG_PASSPHRASE"] = "newpassphrase" + self.cmd(f"--repo={self.repository_location}", "rlist") def test_change_location_to_keyfile(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(repokey' in log - self.cmd(f'--repo={self.repository_location}', 'key', 'change-location', 'keyfile') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(key file' in log + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(repokey" in log + self.cmd(f"--repo={self.repository_location}", "key", "change-location", "keyfile") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(key file" in log def test_change_location_to_b2keyfile(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=repokey-blake2-aes-ocb') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(repokey BLAKE2b' in log - self.cmd(f'--repo={self.repository_location}', 'key', 'change-location', 'keyfile') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(key file BLAKE2b' in log + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=repokey-blake2-aes-ocb") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(repokey BLAKE2b" in log + self.cmd(f"--repo={self.repository_location}", "key", "change-location", "keyfile") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(key file BLAKE2b" in log def test_change_location_to_repokey(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(key file' in log - self.cmd(f'--repo={self.repository_location}', 'key', 'change-location', 'repokey') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(repokey' in log + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(key file" in log + self.cmd(f"--repo={self.repository_location}", "key", "change-location", "repokey") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(repokey" in log def test_change_location_to_b2repokey(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=keyfile-blake2-aes-ocb') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(key file BLAKE2b' in log - self.cmd(f'--repo={self.repository_location}', 'key', 'change-location', 'repokey') - log = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert '(repokey BLAKE2b' in log + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=keyfile-blake2-aes-ocb") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(key file BLAKE2b" in log + self.cmd(f"--repo={self.repository_location}", "key", "change-location", "repokey") + log = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "(repokey BLAKE2b" in log def test_break_lock(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'break-lock') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "break-lock") def test_usage(self): self.cmd() - self.cmd('-h') + self.cmd("-h") def test_help(self): - assert 'Borg' in self.cmd('help') - assert 'patterns' in self.cmd('help', 'patterns') - assert 'creates a new, empty repository' in self.cmd('help', 'rcreate') - assert 'positional arguments' not in self.cmd('help', 'rcreate', '--epilog-only') - assert 'creates a new, empty repository' not in self.cmd('help', 'rcreate', '--usage-only') + assert "Borg" in self.cmd("help") + assert "patterns" in self.cmd("help", "patterns") + assert "creates a new, empty repository" in self.cmd("help", "rcreate") + assert "positional arguments" not in self.cmd("help", "rcreate", "--epilog-only") + assert "creates a new, empty repository" not in self.cmd("help", "rcreate", "--usage-only") - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_fuse(self): def has_noatime(some_file): atime_before = os.stat(some_file).st_atime_ns @@ -2505,29 +2708,32 @@ def has_noatime(some_file): noatime_used = flags_noatime != flags_normal return noatime_used and atime_before == atime_after - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.create_test_files() - have_noatime = has_noatime('input/file1') - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude-nodump', '--atime', 'archive', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', '--exclude-nodump', '--atime', 'archive2', 'input') + have_noatime = has_noatime("input/file1") + self.cmd(f"--repo={self.repository_location}", "create", "--exclude-nodump", "--atime", "archive", "input") + self.cmd(f"--repo={self.repository_location}", "create", "--exclude-nodump", "--atime", "archive2", "input") if has_lchflags: # remove the file we did not backup, so input and output become equal - os.remove(os.path.join('input', 'flagfile')) - mountpoint = os.path.join(self.tmpdir, 'mountpoint') + os.remove(os.path.join("input", "flagfile")) + mountpoint = os.path.join(self.tmpdir, "mountpoint") # mount the whole repository, archive contents shall show up in archivename subdirs of mountpoint: with self.fuse_mount(self.repository_location, mountpoint): # flags are not supported by the FUSE mount # we also ignore xattrs here, they are tested separately - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'), - ignore_flags=True, ignore_xattrs=True) - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'), - ignore_flags=True, ignore_xattrs=True) - with self.fuse_mount(self.repository_location, mountpoint, '-a', 'archive'): - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'), - ignore_flags=True, ignore_xattrs=True) + self.assert_dirs_equal( + self.input_path, os.path.join(mountpoint, "archive", "input"), ignore_flags=True, ignore_xattrs=True + ) + self.assert_dirs_equal( + self.input_path, os.path.join(mountpoint, "archive2", "input"), ignore_flags=True, ignore_xattrs=True + ) + with self.fuse_mount(self.repository_location, mountpoint, "-a", "archive"): + self.assert_dirs_equal( + self.input_path, os.path.join(mountpoint, "archive", "input"), ignore_flags=True, ignore_xattrs=True + ) # regular file - in_fn = 'input/file1' - out_fn = os.path.join(mountpoint, 'archive', 'input', 'file1') + in_fn = "input/file1" + out_fn = os.path.join(mountpoint, "archive", "input", "file1") # stat sti1 = os.stat(in_fn) sto1 = os.stat(out_fn) @@ -2543,135 +2749,135 @@ def has_noatime(some_file): # note: there is another hardlink to this, see below assert sti1.st_nlink == sto1.st_nlink == 2 # read - with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f: + with open(in_fn, "rb") as in_f, open(out_fn, "rb") as out_f: assert in_f.read() == out_f.read() # hardlink (to 'input/file1') if are_hardlinks_supported(): - in_fn = 'input/hardlink' - out_fn = os.path.join(mountpoint, 'archive', 'input', 'hardlink') + in_fn = "input/hardlink" + out_fn = os.path.join(mountpoint, "archive", "input", "hardlink") sti2 = os.stat(in_fn) sto2 = os.stat(out_fn) assert sti2.st_nlink == sto2.st_nlink == 2 assert sto1.st_ino == sto2.st_ino # symlink if are_symlinks_supported(): - in_fn = 'input/link1' - out_fn = os.path.join(mountpoint, 'archive', 'input', 'link1') + in_fn = "input/link1" + out_fn = os.path.join(mountpoint, "archive", "input", "link1") sti = os.stat(in_fn, follow_symlinks=False) sto = os.stat(out_fn, follow_symlinks=False) - assert sti.st_size == len('somewhere') - assert sto.st_size == len('somewhere') + assert sti.st_size == len("somewhere") + assert sto.st_size == len("somewhere") assert stat.S_ISLNK(sti.st_mode) assert stat.S_ISLNK(sto.st_mode) assert os.readlink(in_fn) == os.readlink(out_fn) # FIFO if are_fifos_supported(): - out_fn = os.path.join(mountpoint, 'archive', 'input', 'fifo1') + out_fn = os.path.join(mountpoint, "archive", "input", "fifo1") sto = os.stat(out_fn) assert stat.S_ISFIFO(sto.st_mode) # list/read xattrs try: - in_fn = 'input/fusexattr' - out_fn = os.fsencode(os.path.join(mountpoint, 'archive', 'input', 'fusexattr')) + in_fn = "input/fusexattr" + out_fn = os.fsencode(os.path.join(mountpoint, "archive", "input", "fusexattr")) if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert sorted(no_selinux(xattr.listxattr(out_fn))) == [b'user.empty', b'user.foo', ] - assert xattr.getxattr(out_fn, b'user.foo') == b'bar' - assert xattr.getxattr(out_fn, b'user.empty') == b'' + assert sorted(no_selinux(xattr.listxattr(out_fn))) == [b"user.empty", b"user.foo"] + assert xattr.getxattr(out_fn, b"user.foo") == b"bar" + assert xattr.getxattr(out_fn, b"user.empty") == b"" else: assert no_selinux(xattr.listxattr(out_fn)) == [] try: - xattr.getxattr(out_fn, b'user.foo') + xattr.getxattr(out_fn, b"user.foo") except OSError as e: assert e.errno == llfuse.ENOATTR else: assert False, "expected OSError(ENOATTR), but no error was raised" except OSError as err: - if sys.platform.startswith(('nothing_here_now', )) and err.errno == errno.ENOTSUP: + if sys.platform.startswith(("nothing_here_now",)) and err.errno == errno.ENOTSUP: # some systems have no xattr support on FUSE pass else: raise - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_fuse_versions_view(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('test', contents=b'first') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("test", contents=b"first") if are_hardlinks_supported(): - self.create_regular_file('hardlink1', contents=b'123456') - os.link('input/hardlink1', 'input/hardlink2') - os.link('input/hardlink1', 'input/hardlink3') - self.cmd(f'--repo={self.repository_location}', 'create', 'archive1', 'input') - self.create_regular_file('test', contents=b'second') - self.cmd(f'--repo={self.repository_location}', 'create', 'archive2', 'input') - mountpoint = os.path.join(self.tmpdir, 'mountpoint') + self.create_regular_file("hardlink1", contents=b"123456") + os.link("input/hardlink1", "input/hardlink2") + os.link("input/hardlink1", "input/hardlink3") + self.cmd(f"--repo={self.repository_location}", "create", "archive1", "input") + self.create_regular_file("test", contents=b"second") + self.cmd(f"--repo={self.repository_location}", "create", "archive2", "input") + mountpoint = os.path.join(self.tmpdir, "mountpoint") # mount the whole repository, archive contents shall show up in versioned view: - with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions'): - path = os.path.join(mountpoint, 'input', 'test') # filename shows up as directory ... + with self.fuse_mount(self.repository_location, mountpoint, "-o", "versions"): + path = os.path.join(mountpoint, "input", "test") # filename shows up as directory ... files = os.listdir(path) - assert all(f.startswith('test.') for f in files) # ... with files test.xxxxx in there - assert {b'first', b'second'} == {open(os.path.join(path, f), 'rb').read() for f in files} + assert all(f.startswith("test.") for f in files) # ... with files test.xxxxx in there + assert {b"first", b"second"} == {open(os.path.join(path, f), "rb").read() for f in files} if are_hardlinks_supported(): - hl1 = os.path.join(mountpoint, 'input', 'hardlink1', 'hardlink1.00001') - hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') - hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') + hl1 = os.path.join(mountpoint, "input", "hardlink1", "hardlink1.00001") + hl2 = os.path.join(mountpoint, "input", "hardlink2", "hardlink2.00001") + hl3 = os.path.join(mountpoint, "input", "hardlink3", "hardlink3.00001") assert os.stat(hl1).st_ino == os.stat(hl2).st_ino == os.stat(hl3).st_ino - assert open(hl3, 'rb').read() == b'123456' + assert open(hl3, "rb").read() == b"123456" # similar again, but exclude the 1st hardlink: - with self.fuse_mount(self.repository_location, mountpoint, '-o', 'versions', '-e', 'input/hardlink1'): + with self.fuse_mount(self.repository_location, mountpoint, "-o", "versions", "-e", "input/hardlink1"): if are_hardlinks_supported(): - hl2 = os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00001') - hl3 = os.path.join(mountpoint, 'input', 'hardlink3', 'hardlink3.00001') + hl2 = os.path.join(mountpoint, "input", "hardlink2", "hardlink2.00001") + hl3 = os.path.join(mountpoint, "input", "hardlink3", "hardlink3.00001") assert os.stat(hl2).st_ino == os.stat(hl3).st_ino - assert open(hl3, 'rb').read() == b'123456' + assert open(hl3, "rb").read() == b"123456" - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_fuse_allow_damaged_files(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('archive') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("archive") # Get rid of a chunk and repair it - archive, repository = self.open_archive('archive') + archive, repository = self.open_archive("archive") with repository: for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): repository.delete(item.chunks[-1].id) path = item.path # store full path for later break else: assert False # missed the file repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) - mountpoint = os.path.join(self.tmpdir, 'mountpoint') - with self.fuse_mount(self.repository_location, mountpoint, '-a', 'archive'): + mountpoint = os.path.join(self.tmpdir, "mountpoint") + with self.fuse_mount(self.repository_location, mountpoint, "-a", "archive"): with pytest.raises(OSError) as excinfo: - open(os.path.join(mountpoint, 'archive', path)) + open(os.path.join(mountpoint, "archive", path)) assert excinfo.value.errno == errno.EIO - with self.fuse_mount(self.repository_location, mountpoint, '-a', 'archive', '-o', 'allow_damaged_files'): - open(os.path.join(mountpoint, 'archive', path)).close() + with self.fuse_mount(self.repository_location, mountpoint, "-a", "archive", "-o", "allow_damaged_files"): + open(os.path.join(mountpoint, "archive", path)).close() - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_fuse_mount_options(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('arch11') - self.create_src_archive('arch12') - self.create_src_archive('arch21') - self.create_src_archive('arch22') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("arch11") + self.create_src_archive("arch12") + self.create_src_archive("arch21") + self.create_src_archive("arch22") - mountpoint = os.path.join(self.tmpdir, 'mountpoint') - with self.fuse_mount(self.repository_location, mountpoint, '--first=2', '--sort=name'): - assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12'] - with self.fuse_mount(self.repository_location, mountpoint, '--last=2', '--sort=name'): - assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch21', 'arch22'] - with self.fuse_mount(self.repository_location, mountpoint, '--glob-archives=arch1*'): - assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12'] - with self.fuse_mount(self.repository_location, mountpoint, '--glob-archives=arch2*'): - assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch21', 'arch22'] - with self.fuse_mount(self.repository_location, mountpoint, '--glob-archives=arch*'): - assert sorted(os.listdir(os.path.join(mountpoint))) == ['arch11', 'arch12', 'arch21', 'arch22'] - with self.fuse_mount(self.repository_location, mountpoint, '--glob-archives=nope'): + mountpoint = os.path.join(self.tmpdir, "mountpoint") + with self.fuse_mount(self.repository_location, mountpoint, "--first=2", "--sort=name"): + assert sorted(os.listdir(os.path.join(mountpoint))) == ["arch11", "arch12"] + with self.fuse_mount(self.repository_location, mountpoint, "--last=2", "--sort=name"): + assert sorted(os.listdir(os.path.join(mountpoint))) == ["arch21", "arch22"] + with self.fuse_mount(self.repository_location, mountpoint, "--glob-archives=arch1*"): + assert sorted(os.listdir(os.path.join(mountpoint))) == ["arch11", "arch12"] + with self.fuse_mount(self.repository_location, mountpoint, "--glob-archives=arch2*"): + assert sorted(os.listdir(os.path.join(mountpoint))) == ["arch21", "arch22"] + with self.fuse_mount(self.repository_location, mountpoint, "--glob-archives=arch*"): + assert sorted(os.listdir(os.path.join(mountpoint))) == ["arch11", "arch12", "arch21", "arch22"] + with self.fuse_mount(self.repository_location, mountpoint, "--glob-archives=nope"): assert sorted(os.listdir(os.path.join(mountpoint))) == [] - @unittest.skipUnless(llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, "llfuse not installed") def test_migrate_lock_alive(self): """Both old_id and new_id must not be stale during lock migration / daemonization.""" from functools import wraps @@ -2680,7 +2886,7 @@ def test_migrate_lock_alive(self): # Check results are communicated from the borg mount background process # to the pytest process by means of a serialized dict object stored in this file. - assert_data_file = os.path.join(self.tmpdir, 'migrate_lock_assert_data.pickle') + assert_data_file = os.path.join(self.tmpdir, "migrate_lock_assert_data.pickle") # Decorates Lock.migrate_lock() with process_alive() checks before and after. # (We don't want to mix testing code into runtime.) @@ -2689,144 +2895,160 @@ def write_assert_data(migrate_lock): def wrapper(self, old_id, new_id): wrapper.num_calls += 1 assert_data = { - 'num_calls': wrapper.num_calls, - 'old_id': old_id, - 'new_id': new_id, - 'before': { - 'old_id_alive': platform.process_alive(*old_id), - 'new_id_alive': platform.process_alive(*new_id)}, - 'exception': None, - 'exception.extr_tb': None, - 'after': { - 'old_id_alive': None, - 'new_id_alive': None}} + "num_calls": wrapper.num_calls, + "old_id": old_id, + "new_id": new_id, + "before": { + "old_id_alive": platform.process_alive(*old_id), + "new_id_alive": platform.process_alive(*new_id), + }, + "exception": None, + "exception.extr_tb": None, + "after": {"old_id_alive": None, "new_id_alive": None}, + } try: - with open(assert_data_file, 'wb') as _out: + with open(assert_data_file, "wb") as _out: pickle.dump(assert_data, _out) except: pass try: return migrate_lock(self, old_id, new_id) except BaseException as e: - assert_data['exception'] = e - assert_data['exception.extr_tb'] = traceback.extract_tb(e.__traceback__) + assert_data["exception"] = e + assert_data["exception.extr_tb"] = traceback.extract_tb(e.__traceback__) finally: - assert_data['after'].update({ - 'old_id_alive': platform.process_alive(*old_id), - 'new_id_alive': platform.process_alive(*new_id)}) + assert_data["after"].update( + { + "old_id_alive": platform.process_alive(*old_id), + "new_id_alive": platform.process_alive(*new_id), + } + ) try: - with open(assert_data_file, 'wb') as _out: + with open(assert_data_file, "wb") as _out: pickle.dump(assert_data, _out) except: pass + wrapper.num_calls = 0 return wrapper # Decorate borg.locking.Lock.migrate_lock = write_assert_data(borg.locking.Lock.migrate_lock) try: - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.create_src_archive('arch') - mountpoint = os.path.join(self.tmpdir, 'mountpoint') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.create_src_archive("arch") + mountpoint = os.path.join(self.tmpdir, "mountpoint") # In order that the decoration is kept for the borg mount process, we must not spawn, but actually fork; # not to be confused with the forking in borg.helpers.daemonize() which is done as well. with self.fuse_mount(self.repository_location, mountpoint, os_fork=True): pass - with open(assert_data_file, 'rb') as _in: + with open(assert_data_file, "rb") as _in: assert_data = pickle.load(_in) - print(f'\nLock.migrate_lock(): assert_data = {assert_data!r}.', file=sys.stderr, flush=True) - exception = assert_data['exception'] + print(f"\nLock.migrate_lock(): assert_data = {assert_data!r}.", file=sys.stderr, flush=True) + exception = assert_data["exception"] if exception is not None: - extracted_tb = assert_data['exception.extr_tb'] + extracted_tb = assert_data["exception.extr_tb"] print( - 'Lock.migrate_lock() raised an exception:\n', - 'Traceback (most recent call last):\n', + "Lock.migrate_lock() raised an exception:\n", + "Traceback (most recent call last):\n", *traceback.format_list(extracted_tb), *traceback.format_exception(exception.__class__, exception, None), - sep='', end='', file=sys.stderr, flush=True) + sep="", + end="", + file=sys.stderr, + flush=True, + ) - assert assert_data['num_calls'] == 1, "Lock.migrate_lock() must be called exactly once." + assert assert_data["num_calls"] == 1, "Lock.migrate_lock() must be called exactly once." assert exception is None, "Lock.migrate_lock() may not raise an exception." - assert_data_before = assert_data['before'] - assert assert_data_before['old_id_alive'], "old_id must be alive (=must not be stale) when calling Lock.migrate_lock()." - assert assert_data_before['new_id_alive'], "new_id must be alive (=must not be stale) when calling Lock.migrate_lock()." + assert_data_before = assert_data["before"] + assert assert_data_before[ + "old_id_alive" + ], "old_id must be alive (=must not be stale) when calling Lock.migrate_lock()." + assert assert_data_before[ + "new_id_alive" + ], "new_id must be alive (=must not be stale) when calling Lock.migrate_lock()." - assert_data_after = assert_data['after'] - assert assert_data_after['old_id_alive'], "old_id must be alive (=must not be stale) when Lock.migrate_lock() has returned." - assert assert_data_after['new_id_alive'], "new_id must be alive (=must not be stale) when Lock.migrate_lock() has returned." + assert_data_after = assert_data["after"] + assert assert_data_after[ + "old_id_alive" + ], "old_id must be alive (=must not be stale) when Lock.migrate_lock() has returned." + assert assert_data_after[ + "new_id_alive" + ], "new_id must be alive (=must not be stale) when Lock.migrate_lock() has returned." finally: # Undecorate borg.locking.Lock.migrate_lock = borg.locking.Lock.migrate_lock.__wrapped__ def test_debug_dump_archive_items(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'dump-archive-items', 'test') - output_dir = sorted(os.listdir('output')) - assert len(output_dir) > 0 and output_dir[0].startswith('000000_') - assert 'Done.' in output + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "debug", "dump-archive-items", "test") + output_dir = sorted(os.listdir("output")) + assert len(output_dir) > 0 and output_dir[0].startswith("000000_") + assert "Done." in output def test_debug_dump_repo_objs(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'dump-repo-objs') - output_dir = sorted(os.listdir('output')) - assert len(output_dir) > 0 and output_dir[0].startswith('00000000_') - assert 'Done.' in output + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + output = self.cmd(f"--repo={self.repository_location}", "debug", "dump-repo-objs") + output_dir = sorted(os.listdir("output")) + assert len(output_dir) > 0 and output_dir[0].startswith("00000000_") + assert "Done." in output def test_debug_put_get_delete_obj(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - data = b'some data' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + data = b"some data" hexkey = sha256(data).hexdigest() - self.create_regular_file('file', contents=data) - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'put-obj', 'input/file') + self.create_regular_file("file", contents=data) + output = self.cmd(f"--repo={self.repository_location}", "debug", "put-obj", "input/file") assert hexkey in output - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'get-obj', hexkey, 'output/file') + output = self.cmd(f"--repo={self.repository_location}", "debug", "get-obj", hexkey, "output/file") assert hexkey in output - with open('output/file', 'rb') as f: + with open("output/file", "rb") as f: data_read = f.read() assert data == data_read - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'delete-obj', hexkey) + output = self.cmd(f"--repo={self.repository_location}", "debug", "delete-obj", hexkey) assert "deleted" in output - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'delete-obj', hexkey) + output = self.cmd(f"--repo={self.repository_location}", "debug", "delete-obj", hexkey) assert "not found" in output - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'delete-obj', 'invalid') + output = self.cmd(f"--repo={self.repository_location}", "debug", "delete-obj", "invalid") assert "is invalid" in output def test_init_interrupt(self): def raise_eof(*args, **kwargs): raise EOFError - with patch.object(FlexiKey, 'create', raise_eof): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION, exit_code=1) + with patch.object(FlexiKey, "create", raise_eof): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION, exit_code=1) assert not os.path.exists(self.repository_location) def test_init_requires_encryption_option(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', exit_code=2) + self.cmd(f"--repo={self.repository_location}", "rcreate", exit_code=2) def test_init_nested_repositories(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}/nested', 'rcreate', RK_ENCRYPTION, exit_code=2) + self.cmd(f"--repo={self.repository_location}/nested", "rcreate", RK_ENCRYPTION, exit_code=2) else: with pytest.raises(Repository.AlreadyExists): - self.cmd(f'--repo={self.repository_location}/nested', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}/nested", "rcreate", RK_ENCRYPTION) def test_init_refuse_to_overwrite_keyfile(self): """BORG_KEY_FILE=something borg init should quit if "something" already exists. See https://github.com/borgbackup/borg/pull/6046""" - keyfile = os.path.join(self.tmpdir, 'keyfile') + keyfile = os.path.join(self.tmpdir, "keyfile") with environment_variable(BORG_KEY_FILE=keyfile): - self.cmd(f'--repo={self.repository_location}0', 'rcreate', KF_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}0", "rcreate", KF_ENCRYPTION) with open(keyfile) as file: before = file.read() - arg = (f'--repo={self.repository_location}1', 'rcreate', KF_ENCRYPTION) + arg = (f"--repo={self.repository_location}1", "rcreate", KF_ENCRYPTION) if self.FORK_DEFAULT: self.cmd(*arg, exit_code=2) else: @@ -2838,7 +3060,7 @@ def test_init_refuse_to_overwrite_keyfile(self): def check_cache(self): # First run a regular borg check - self.cmd(f'--repo={self.repository_location}', 'check') + self.cmd(f"--repo={self.repository_location}", "check") # Then check that the cache on disk matches exactly what's in the repo. with self.open_repository() as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) @@ -2858,8 +3080,8 @@ def check_cache(self): assert id in seen def test_check_cache(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") with self.open_repository() as repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) with Cache(repository, key, manifest, sync=False) as cache: @@ -2870,177 +3092,206 @@ def test_check_cache(self): self.check_cache() def test_recreate_target_rc(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'recreate', '--target=asdf', exit_code=2) - assert 'Need to specify single archive' in output + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "recreate", "--target=asdf", exit_code=2) + assert "Need to specify single archive" in output def test_recreate_target(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.check_cache() - self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") self.check_cache() - original_archive = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.cmd(f'--repo={self.repository_location}', 'recreate', 'test0', 'input/dir2', - '-e', 'input/dir2/file3', '--target=new-archive') + original_archive = self.cmd(f"--repo={self.repository_location}", "rlist") + self.cmd( + f"--repo={self.repository_location}", + "recreate", + "test0", + "input/dir2", + "-e", + "input/dir2/file3", + "--target=new-archive", + ) self.check_cache() - archives = self.cmd(f'--repo={self.repository_location}', 'rlist') + archives = self.cmd(f"--repo={self.repository_location}", "rlist") assert original_archive in archives - assert 'new-archive' in archives + assert "new-archive" in archives - listing = self.cmd(f'--repo={self.repository_location}', 'list', 'new-archive', '--short') - assert 'file1' not in listing - assert 'dir2/file2' in listing - assert 'dir2/file3' not in listing + listing = self.cmd(f"--repo={self.repository_location}", "list", "new-archive", "--short") + assert "file1" not in listing + assert "dir2/file2" in listing + assert "dir2/file3" not in listing def test_recreate_basic(self): self.create_test_files() - self.create_regular_file('dir2/file3', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', 'test0', 'input/dir2', '-e', 'input/dir2/file3') + self.create_regular_file("dir2/file3", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") + self.cmd(f"--repo={self.repository_location}", "recreate", "test0", "input/dir2", "-e", "input/dir2/file3") self.check_cache() - listing = self.cmd(f'--repo={self.repository_location}', 'list', 'test0', '--short') - assert 'file1' not in listing - assert 'dir2/file2' in listing - assert 'dir2/file3' not in listing + listing = self.cmd(f"--repo={self.repository_location}", "list", "test0", "--short") + assert "file1" not in listing + assert "dir2/file2" in listing + assert "dir2/file3" not in listing - @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') + @pytest.mark.skipif(not are_hardlinks_supported(), reason="hardlinks not supported") def test_recreate_subtree_hardlinks(self): # This is essentially the same problem set as in test_extract_hardlinks self._extract_hardlinks_setup() - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', 'input/dir1') + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input") + self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test", "input/dir1") self.check_cache() - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test') - assert os.stat('input/dir1/hardlink').st_nlink == 2 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 - assert os.stat('input/dir1/aaaa').st_nlink == 2 - assert os.stat('input/dir1/source2').st_nlink == 2 - with changedir('output'): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test2') - assert os.stat('input/dir1/hardlink').st_nlink == 4 + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test") + assert os.stat("input/dir1/hardlink").st_nlink == 2 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 2 + assert os.stat("input/dir1/aaaa").st_nlink == 2 + assert os.stat("input/dir1/source2").st_nlink == 2 + with changedir("output"): + self.cmd(f"--repo={self.repository_location}", "extract", "test2") + assert os.stat("input/dir1/hardlink").st_nlink == 4 def test_recreate_rechunkify(self): - with open(os.path.join(self.input_path, 'large_file'), 'wb') as fd: - fd.write(b'a' * 280) - fd.write(b'b' * 280) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input', '--chunker-params', '7,9,8,128') - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input', '--files-cache=disabled') - list = self.cmd(f'--repo={self.repository_location}', 'list', 'test1', 'input/large_file', - '--format', '{num_chunks} {unique_chunks}') - num_chunks, unique_chunks = map(int, list.split(' ')) + with open(os.path.join(self.input_path, "large_file"), "wb") as fd: + fd.write(b"a" * 280) + fd.write(b"b" * 280) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input", "--chunker-params", "7,9,8,128") + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input", "--files-cache=disabled") + list = self.cmd( + f"--repo={self.repository_location}", + "list", + "test1", + "input/large_file", + "--format", + "{num_chunks} {unique_chunks}", + ) + num_chunks, unique_chunks = map(int, list.split(" ")) # test1 and test2 do not deduplicate assert num_chunks == unique_chunks - self.cmd(f'--repo={self.repository_location}', 'recreate', '--chunker-params', 'default') + self.cmd(f"--repo={self.repository_location}", "recreate", "--chunker-params", "default") self.check_cache() # test1 and test2 do deduplicate after recreate - assert int(self.cmd(f'--repo={self.repository_location}', 'list', 'test1', 'input/large_file', '--format={size}')) - assert not int(self.cmd(f'--repo={self.repository_location}', 'list', 'test1', 'input/large_file', - '--format', '{unique_chunks}')) + assert int( + self.cmd(f"--repo={self.repository_location}", "list", "test1", "input/large_file", "--format={size}") + ) + assert not int( + self.cmd( + f"--repo={self.repository_location}", "list", "test1", "input/large_file", "--format", "{unique_chunks}" + ) + ) def test_recreate_recompress(self): - self.create_regular_file('compressible', size=10000) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input', '-C', 'none') - file_list = self.cmd(f'--repo={self.repository_location}', 'list', 'test', 'input/compressible', - '--format', '{size} {sha256}') - size, sha256_before = file_list.split(' ') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-C', 'lz4', '--recompress') + self.create_regular_file("compressible", size=10000) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "-C", "none") + file_list = self.cmd( + f"--repo={self.repository_location}", "list", "test", "input/compressible", "--format", "{size} {sha256}" + ) + size, sha256_before = file_list.split(" ") + self.cmd(f"--repo={self.repository_location}", "recreate", "-C", "lz4", "--recompress") self.check_cache() - file_list = self.cmd(f'--repo={self.repository_location}', 'list', 'test', 'input/compressible', - '--format', '{size} {sha256}') - size, sha256_after = file_list.split(' ') + file_list = self.cmd( + f"--repo={self.repository_location}", "list", "test", "input/compressible", "--format", "{size} {sha256}" + ) + size, sha256_after = file_list.split(" ") assert sha256_before == sha256_after def test_recreate_timestamp(self): local_timezone = datetime.now(timezone(timedelta(0))).astimezone().tzinfo self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') - self.cmd(f'--repo={self.repository_location}', 'recreate', 'test0', '--timestamp', "1970-01-02T00:00:00", - '--comment', 'test') - info = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test0').splitlines() + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") + self.cmd( + f"--repo={self.repository_location}", + "recreate", + "test0", + "--timestamp", + "1970-01-02T00:00:00", + "--comment", + "test", + ) + info = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test0").splitlines() dtime = datetime(1970, 1, 2) + local_timezone.utcoffset(None) s_time = dtime.strftime("%Y-%m-%d") - assert any([re.search(r'Time \(start\).+ %s' % s_time, item) for item in info]) - assert any([re.search(r'Time \(end\).+ %s' % s_time, item) for item in info]) + assert any([re.search(r"Time \(start\).+ %s" % s_time, item) for item in info]) + assert any([re.search(r"Time \(end\).+ %s" % s_time, item) for item in info]) def test_recreate_dry_run(self): - self.create_regular_file('compressible', size=10000) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - archives_before = self.cmd(f'--repo={self.repository_location}', 'list', 'test') - self.cmd(f'--repo={self.repository_location}', 'recreate', '-n', '-e', 'input/compressible') + self.create_regular_file("compressible", size=10000) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + archives_before = self.cmd(f"--repo={self.repository_location}", "list", "test") + self.cmd(f"--repo={self.repository_location}", "recreate", "-n", "-e", "input/compressible") self.check_cache() - archives_after = self.cmd(f'--repo={self.repository_location}', 'list', 'test') + archives_after = self.cmd(f"--repo={self.repository_location}", "list", "test") assert archives_after == archives_before def test_recreate_skips_nothing_to_do(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - info_before = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test') - self.cmd(f'--repo={self.repository_location}', 'recreate', '--chunker-params', 'default') + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + info_before = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test") + self.cmd(f"--repo={self.repository_location}", "recreate", "--chunker-params", "default") self.check_cache() - info_after = self.cmd(f'--repo={self.repository_location}', 'info', '-a', 'test') + info_after = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test") assert info_before == info_after # includes archive ID def test_with_lock(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - lock_path = os.path.join(self.repository_path, 'lock.exclusive') - cmd = 'python3', '-c', 'import os, sys; sys.exit(42 if os.path.exists("%s") else 23)' % lock_path - self.cmd(f'--repo={self.repository_location}', 'with-lock', *cmd, fork=True, exit_code=42) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + lock_path = os.path.join(self.repository_path, "lock.exclusive") + cmd = "python3", "-c", 'import os, sys; sys.exit(42 if os.path.exists("%s") else 23)' % lock_path + self.cmd(f"--repo={self.repository_location}", "with-lock", *cmd, fork=True, exit_code=42) def test_recreate_list_output(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('file1', size=0) - self.create_regular_file('file2', size=0) - self.create_regular_file('file3', size=0) - self.create_regular_file('file4', size=0) - self.create_regular_file('file5', size=0) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("file1", size=0) + self.create_regular_file("file2", size=0) + self.create_regular_file("file3", size=0) + self.create_regular_file("file4", size=0) + self.create_regular_file("file5", size=0) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") - output = self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--list', '--info', '-e', 'input/file2') + output = self.cmd( + f"--repo={self.repository_location}", "recreate", "-a", "test", "--list", "--info", "-e", "input/file2" + ) self.check_cache() self.assert_in("input/file1", output) self.assert_in("x input/file2", output) - output = self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--list', '-e', 'input/file3') + output = self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test", "--list", "-e", "input/file3") self.check_cache() self.assert_in("input/file1", output) self.assert_in("x input/file3", output) - output = self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '-e', 'input/file4') + output = self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test", "-e", "input/file4") self.check_cache() self.assert_not_in("input/file1", output) self.assert_not_in("x input/file4", output) - output = self.cmd(f'--repo={self.repository_location}', 'recreate', '-a', 'test', '--info', '-e', 'input/file5') + output = self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test", "--info", "-e", "input/file5") self.check_cache() self.assert_not_in("input/file1", output) self.assert_not_in("x input/file5", output) def test_bad_filters(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'delete', '--first', '1', '--last', '1', fork=True, exit_code=2) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd(f"--repo={self.repository_location}", "delete", "--first", "1", "--last", "1", fork=True, exit_code=2) def test_key_export_keyfile(self): - export_file = self.output_path + '/exported' - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + export_file = self.output_path + "/exported" + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) repo_id = self._extract_repository_id(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'key', 'export', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "export", export_file) with open(export_file) as fd: export_contents = fd.read() - assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n') + assert export_contents.startswith("BORG_KEY " + bin_to_hex(repo_id) + "\n") - key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] + key_file = self.keys_path + "/" + os.listdir(self.keys_path)[0] with open(key_file) as fd: key_contents = fd.read() @@ -3049,7 +3300,7 @@ def test_key_export_keyfile(self): os.unlink(key_file) - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file) with open(key_file) as fd: key_contents2 = fd.read() @@ -3057,19 +3308,19 @@ def test_key_export_keyfile(self): assert key_contents2 == key_contents def test_key_import_keyfile_with_borg_key_file(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) - exported_key_file = os.path.join(self.output_path, 'exported') - self.cmd(f'--repo={self.repository_location}', 'key', 'export', exported_key_file) + exported_key_file = os.path.join(self.output_path, "exported") + self.cmd(f"--repo={self.repository_location}", "key", "export", exported_key_file) key_file = os.path.join(self.keys_path, os.listdir(self.keys_path)[0]) with open(key_file) as fd: key_contents = fd.read() os.unlink(key_file) - imported_key_file = os.path.join(self.output_path, 'imported') + imported_key_file = os.path.join(self.output_path, "imported") with environment_variable(BORG_KEY_FILE=imported_key_file): - self.cmd(f'--repo={self.repository_location}', 'key', 'import', exported_key_file) + self.cmd(f"--repo={self.repository_location}", "key", "import", exported_key_file) assert not os.path.isfile(key_file), '"borg key import" should respect BORG_KEY_FILE' with open(imported_key_file) as fd: @@ -3077,15 +3328,15 @@ def test_key_import_keyfile_with_borg_key_file(self): assert imported_key_contents == key_contents def test_key_export_repokey(self): - export_file = self.output_path + '/exported' - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + export_file = self.output_path + "/exported" + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) repo_id = self._extract_repository_id(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'key', 'export', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "export", export_file) with open(export_file) as fd: export_contents = fd.read() - assert export_contents.startswith('BORG_KEY ' + bin_to_hex(repo_id) + '\n') + assert export_contents.startswith("BORG_KEY " + bin_to_hex(repo_id) + "\n") with Repository(self.repository_path) as repository: repo_key = AESOCBRepoKey(repository) @@ -3097,9 +3348,9 @@ def test_key_export_repokey(self): assert repo_key.enc_key == backup_key.enc_key with Repository(self.repository_path) as repository: - repository.save_key(b'') + repository.save_key(b"") - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file) with Repository(self.repository_path) as repository: repo_key2 = AESOCBRepoKey(repository) @@ -3108,102 +3359,104 @@ def test_key_export_repokey(self): assert repo_key2.enc_key == repo_key2.enc_key def test_key_export_qr(self): - export_file = self.output_path + '/exported.html' - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + export_file = self.output_path + "/exported.html" + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) repo_id = self._extract_repository_id(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'key', 'export', '--qr-html', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "export", "--qr-html", export_file) - with open(export_file, encoding='utf-8') as fd: + with open(export_file, encoding="utf-8") as fd: export_contents = fd.read() assert bin_to_hex(repo_id) in export_contents - assert export_contents.startswith('') - assert export_contents.endswith('\n') + assert export_contents.startswith("") + assert export_contents.endswith("\n") def test_key_export_directory(self): - export_directory = self.output_path + '/exported' + export_directory = self.output_path + "/exported" os.mkdir(export_directory) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'key', 'export', export_directory, exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "key", "export", export_directory, exit_code=EXIT_ERROR) def test_key_import_errors(self): - export_file = self.output_path + '/exported' - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + export_file = self.output_path + "/exported" + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file, exit_code=EXIT_ERROR) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file, exit_code=EXIT_ERROR) - with open(export_file, 'w') as fd: - fd.write('something not a key\n') + with open(export_file, "w") as fd: + fd.write("something not a key\n") if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file, exit_code=2) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file, exit_code=2) else: with pytest.raises(NotABorgKeyFile): - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file) - with open(export_file, 'w') as fd: - fd.write('BORG_KEY a0a0a0\n') + with open(export_file, "w") as fd: + fd.write("BORG_KEY a0a0a0\n") if self.FORK_DEFAULT: - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file, exit_code=2) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file, exit_code=2) else: with pytest.raises(RepoIdMismatch): - self.cmd(f'--repo={self.repository_location}', 'key', 'import', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "import", export_file) def test_key_export_paperkey(self): - repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' + repo_id = "e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239" - export_file = self.output_path + '/exported' - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + export_file = self.output_path + "/exported" + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) self._set_repository_id(self.repository_path, unhexlify(repo_id)) - key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] + key_file = self.keys_path + "/" + os.listdir(self.keys_path)[0] - with open(key_file, 'w') as fd: - fd.write(CHPOKeyfileKey.FILE_ID + ' ' + repo_id + '\n') - fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode()) + with open(key_file, "w") as fd: + fd.write(CHPOKeyfileKey.FILE_ID + " " + repo_id + "\n") + fd.write(b2a_base64(b"abcdefghijklmnopqrstu").decode()) - self.cmd(f'--repo={self.repository_location}', 'key', 'export', '--paper', export_file) + self.cmd(f"--repo={self.repository_location}", "key", "export", "--paper", export_file) with open(export_file) as fd: export_contents = fd.read() - assert export_contents == """To restore key use borg key import --paper /path/to/repo + assert ( + export_contents + == """To restore key use borg key import --paper /path/to/repo BORG PAPER KEY v1 id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 1: 616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d 2: 737475 - 88 """ + ) def test_key_import_paperkey(self): - repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + repo_id = "e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239" + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) self._set_repository_id(self.repository_path, unhexlify(repo_id)) - key_file = self.keys_path + '/' + os.listdir(self.keys_path)[0] - with open(key_file, 'w') as fd: - fd.write(AESOCBKeyfileKey.FILE_ID + ' ' + repo_id + '\n') - fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode()) + key_file = self.keys_path + "/" + os.listdir(self.keys_path)[0] + with open(key_file, "w") as fd: + fd.write(AESOCBKeyfileKey.FILE_ID + " " + repo_id + "\n") + fd.write(b2a_base64(b"abcdefghijklmnopqrstu").decode()) typed_input = ( - b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 02\n' # Forgot to type "-" - b'2 / e29442 3506da 4e1ea7 25f62a 5a3d41 - 02\n' # Forgot to type second "/" - b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d42 - 02\n' # Typo (..42 not ..41) - b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n' # Correct! Congratulations - b'616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n' - b'\n\n' # Abort [yN] => N - b'737475 88\n' # missing "-" - b'73747i - 88\n' # typo - b'73747 - 88\n' # missing nibble - b'73 74 75 - 89\n' # line checksum mismatch - b'00a1 - 88\n' # line hash collision - overall hash mismatch, have to start over - - b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n' - b'616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n' - b'73 74 75 - 88\n' + b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 02\n" # Forgot to type "-" + b"2 / e29442 3506da 4e1ea7 25f62a 5a3d41 - 02\n" # Forgot to type second "/" + b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d42 - 02\n" # Typo (..42 not ..41) + b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n" # Correct! Congratulations + b"616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n" + b"\n\n" # Abort [yN] => N + b"737475 88\n" # missing "-" + b"73747i - 88\n" # typo + b"73747 - 88\n" # missing nibble + b"73 74 75 - 89\n" # line checksum mismatch + b"00a1 - 88\n" # line hash collision - overall hash mismatch, have to start over + b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n" + b"616263 646566 676869 6a6b6c 6d6e6f 707172 - 6d\n" + b"73 74 75 - 88\n" ) # In case that this has to change, here is a quick way to find a colliding line hash: @@ -3215,648 +3468,718 @@ def test_key_import_paperkey(self): # print(i.to_bytes(2, 'big')) # break - self.cmd(f'--repo={self.repository_location}', 'key', 'import', '--paper', input=typed_input) + self.cmd(f"--repo={self.repository_location}", "key", "import", "--paper", input=typed_input) # Test abort paths - typed_input = b'\ny\n' - self.cmd(f'--repo={self.repository_location}', 'key', 'import', '--paper', input=typed_input) - typed_input = b'2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n\ny\n' - self.cmd(f'--repo={self.repository_location}', 'key', 'import', '--paper', input=typed_input) + typed_input = b"\ny\n" + self.cmd(f"--repo={self.repository_location}", "key", "import", "--paper", input=typed_input) + typed_input = b"2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02\n\ny\n" + self.cmd(f"--repo={self.repository_location}", "key", "import", "--paper", input=typed_input) def test_debug_dump_manifest(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - dump_file = self.output_path + '/dump' - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'dump-manifest', dump_file) + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + dump_file = self.output_path + "/dump" + output = self.cmd(f"--repo={self.repository_location}", "debug", "dump-manifest", dump_file) assert output == "" with open(dump_file) as f: result = json.load(f) - assert 'archives' in result - assert 'config' in result - assert 'item_keys' in result - assert 'timestamp' in result - assert 'version' in result + assert "archives" in result + assert "config" in result + assert "item_keys" in result + assert "timestamp" in result + assert "version" in result def test_debug_dump_archive(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - dump_file = self.output_path + '/dump' - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'dump-archive', 'test', dump_file) + self.create_regular_file("file1", size=1024 * 80) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + dump_file = self.output_path + "/dump" + output = self.cmd(f"--repo={self.repository_location}", "debug", "dump-archive", "test", dump_file) assert output == "" with open(dump_file) as f: result = json.load(f) - assert '_name' in result - assert '_manifest_entry' in result - assert '_meta' in result - assert '_items' in result + assert "_name" in result + assert "_manifest_entry" in result + assert "_meta" in result + assert "_items" in result def test_debug_refcount_obj(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'refcount-obj', '0' * 64).strip() - assert output == 'object 0000000000000000000000000000000000000000000000000000000000000000 not found [info from chunks cache].' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "debug", "refcount-obj", "0" * 64).strip() + assert ( + output + == "object 0000000000000000000000000000000000000000000000000000000000000000 not found [info from chunks cache]." + ) - create_json = json.loads(self.cmd(f'--repo={self.repository_location}', 'create', '--json', 'test', 'input')) - archive_id = create_json['archive']['id'] - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'refcount-obj', archive_id).strip() - assert output == 'object ' + archive_id + ' has 1 referrers [info from chunks cache].' + create_json = json.loads(self.cmd(f"--repo={self.repository_location}", "create", "--json", "test", "input")) + archive_id = create_json["archive"]["id"] + output = self.cmd(f"--repo={self.repository_location}", "debug", "refcount-obj", archive_id).strip() + assert output == "object " + archive_id + " has 1 referrers [info from chunks cache]." # Invalid IDs do not abort or return an error - output = self.cmd(f'--repo={self.repository_location}', 'debug', 'refcount-obj', '124', 'xyza').strip() - assert output == 'object id 124 is invalid.\nobject id xyza is invalid.' + output = self.cmd(f"--repo={self.repository_location}", "debug", "refcount-obj", "124", "xyza").strip() + assert output == "object id 124 is invalid.\nobject id xyza is invalid." def test_debug_info(self): - output = self.cmd('debug', 'info') - assert 'CRC implementation' in output - assert 'Python' in output + output = self.cmd("debug", "info") + assert "CRC implementation" in output + assert "Python" in output def test_benchmark_crud(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - with environment_variable(_BORG_BENCHMARK_CRUD_TEST='YES'): - self.cmd(f'--repo={self.repository_location}', 'benchmark', 'crud', self.input_path) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + with environment_variable(_BORG_BENCHMARK_CRUD_TEST="YES"): + self.cmd(f"--repo={self.repository_location}", "benchmark", "crud", self.input_path) def test_config(self): self.create_test_files() - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - output = self.cmd(f'--repo={self.repository_location}', 'config', '--list') - self.assert_in('[repository]', output) - self.assert_in('version', output) - self.assert_in('segments_per_dir', output) - self.assert_in('storage_quota', output) - self.assert_in('append_only', output) - self.assert_in('additional_free_space', output) - self.assert_in('id', output) - self.assert_not_in('last_segment_checked', output) + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + output = self.cmd(f"--repo={self.repository_location}", "config", "--list") + self.assert_in("[repository]", output) + self.assert_in("version", output) + self.assert_in("segments_per_dir", output) + self.assert_in("storage_quota", output) + self.assert_in("append_only", output) + self.assert_in("additional_free_space", output) + self.assert_in("id", output) + self.assert_not_in("last_segment_checked", output) - output = self.cmd(f'--repo={self.repository_location}', 'config', 'last_segment_checked', exit_code=1) - self.assert_in('No option ', output) - self.cmd(f'--repo={self.repository_location}', 'config', 'last_segment_checked', '123') - output = self.cmd(f'--repo={self.repository_location}', 'config', 'last_segment_checked') - assert output == '123' + '\n' - output = self.cmd(f'--repo={self.repository_location}', 'config', '--list') - self.assert_in('last_segment_checked', output) - self.cmd(f'--repo={self.repository_location}', 'config', '--delete', 'last_segment_checked') + output = self.cmd(f"--repo={self.repository_location}", "config", "last_segment_checked", exit_code=1) + self.assert_in("No option ", output) + self.cmd(f"--repo={self.repository_location}", "config", "last_segment_checked", "123") + output = self.cmd(f"--repo={self.repository_location}", "config", "last_segment_checked") + assert output == "123" + "\n" + output = self.cmd(f"--repo={self.repository_location}", "config", "--list") + self.assert_in("last_segment_checked", output) + self.cmd(f"--repo={self.repository_location}", "config", "--delete", "last_segment_checked") - for cfg_key, cfg_value in [ - ('additional_free_space', '2G'), - ('repository.append_only', '1'), - ]: - output = self.cmd(f'--repo={self.repository_location}', 'config', cfg_key) - assert output == '0' + '\n' - self.cmd(f'--repo={self.repository_location}', 'config', cfg_key, cfg_value) - output = self.cmd(f'--repo={self.repository_location}', 'config', cfg_key) - assert output == cfg_value + '\n' - self.cmd(f'--repo={self.repository_location}', 'config', '--delete', cfg_key) - self.cmd(f'--repo={self.repository_location}', 'config', cfg_key, exit_code=1) + for cfg_key, cfg_value in [("additional_free_space", "2G"), ("repository.append_only", "1")]: + output = self.cmd(f"--repo={self.repository_location}", "config", cfg_key) + assert output == "0" + "\n" + self.cmd(f"--repo={self.repository_location}", "config", cfg_key, cfg_value) + output = self.cmd(f"--repo={self.repository_location}", "config", cfg_key) + assert output == cfg_value + "\n" + self.cmd(f"--repo={self.repository_location}", "config", "--delete", cfg_key) + self.cmd(f"--repo={self.repository_location}", "config", cfg_key, exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'config', '--list', '--delete', exit_code=2) - self.cmd(f'--repo={self.repository_location}', 'config', exit_code=2) - self.cmd(f'--repo={self.repository_location}', 'config', 'invalid-option', exit_code=1) + self.cmd(f"--repo={self.repository_location}", "config", "--list", "--delete", exit_code=2) + self.cmd(f"--repo={self.repository_location}", "config", exit_code=2) + self.cmd(f"--repo={self.repository_location}", "config", "invalid-option", exit_code=1) - requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') - requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') + requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason="GNU tar must be installed for this test.") + requires_gzip = pytest.mark.skipif(not shutil.which("gzip"), reason="gzip must be installed for this test.") @requires_gnutar def test_export_tar(self): self.create_test_files() - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'simple.tar', '--progress', '--tar-format=GNU') - with changedir('output'): + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.cmd( + f"--repo={self.repository_location}", "export-tar", "test", "simple.tar", "--progress", "--tar-format=GNU" + ) + with changedir("output"): # This probably assumes GNU tar. Note -p switch to extract permissions regardless of umask. - subprocess.check_call(['tar', 'xpf', '../simple.tar', '--warning=no-timestamp']) - self.assert_dirs_equal('input', 'output/input', ignore_flags=True, ignore_xattrs=True, ignore_ns=True) + subprocess.check_call(["tar", "xpf", "../simple.tar", "--warning=no-timestamp"]) + self.assert_dirs_equal("input", "output/input", ignore_flags=True, ignore_xattrs=True, ignore_ns=True) @requires_gnutar @requires_gzip def test_export_tar_gz(self): - if not shutil.which('gzip'): - pytest.skip('gzip is not installed') + if not shutil.which("gzip"): + pytest.skip("gzip is not installed") self.create_test_files() - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - list = self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'simple.tar.gz', - '--list', '--tar-format=GNU') - assert 'input/file1\n' in list - assert 'input/dir2\n' in list - with changedir('output'): - subprocess.check_call(['tar', 'xpf', '../simple.tar.gz', '--warning=no-timestamp']) - self.assert_dirs_equal('input', 'output/input', ignore_flags=True, ignore_xattrs=True, ignore_ns=True) + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + list = self.cmd( + f"--repo={self.repository_location}", "export-tar", "test", "simple.tar.gz", "--list", "--tar-format=GNU" + ) + assert "input/file1\n" in list + assert "input/dir2\n" in list + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../simple.tar.gz", "--warning=no-timestamp"]) + self.assert_dirs_equal("input", "output/input", ignore_flags=True, ignore_xattrs=True, ignore_ns=True) @requires_gnutar def test_export_tar_strip_components(self): - if not shutil.which('gzip'): - pytest.skip('gzip is not installed') + if not shutil.which("gzip"): + pytest.skip("gzip is not installed") self.create_test_files() - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - list = self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'simple.tar', - '--strip-components=1', '--list', '--tar-format=GNU') + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + list = self.cmd( + f"--repo={self.repository_location}", + "export-tar", + "test", + "simple.tar", + "--strip-components=1", + "--list", + "--tar-format=GNU", + ) # --list's path are those before processing with --strip-components - assert 'input/file1\n' in list - assert 'input/dir2\n' in list - with changedir('output'): - subprocess.check_call(['tar', 'xpf', '../simple.tar', '--warning=no-timestamp']) - self.assert_dirs_equal('input', 'output/', ignore_flags=True, ignore_xattrs=True, ignore_ns=True) + assert "input/file1\n" in list + assert "input/dir2\n" in list + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../simple.tar", "--warning=no-timestamp"]) + self.assert_dirs_equal("input", "output/", ignore_flags=True, ignore_xattrs=True, ignore_ns=True) @requires_hardlinks @requires_gnutar def test_export_tar_strip_components_links(self): self._extract_hardlinks_setup() - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'output.tar', - '--strip-components=2', '--tar-format=GNU') - with changedir('output'): - subprocess.check_call(['tar', 'xpf', '../output.tar', '--warning=no-timestamp']) - assert os.stat('hardlink').st_nlink == 2 - assert os.stat('subdir/hardlink').st_nlink == 2 - assert os.stat('aaaa').st_nlink == 2 - assert os.stat('source2').st_nlink == 2 + self.cmd( + f"--repo={self.repository_location}", + "export-tar", + "test", + "output.tar", + "--strip-components=2", + "--tar-format=GNU", + ) + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../output.tar", "--warning=no-timestamp"]) + assert os.stat("hardlink").st_nlink == 2 + assert os.stat("subdir/hardlink").st_nlink == 2 + assert os.stat("aaaa").st_nlink == 2 + assert os.stat("source2").st_nlink == 2 @requires_hardlinks @requires_gnutar def test_extract_hardlinks_tar(self): self._extract_hardlinks_setup() - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'test', 'output.tar', 'input/dir1', '--tar-format=GNU') - with changedir('output'): - subprocess.check_call(['tar', 'xpf', '../output.tar', '--warning=no-timestamp']) - assert os.stat('input/dir1/hardlink').st_nlink == 2 - assert os.stat('input/dir1/subdir/hardlink').st_nlink == 2 - assert os.stat('input/dir1/aaaa').st_nlink == 2 - assert os.stat('input/dir1/source2').st_nlink == 2 + self.cmd( + f"--repo={self.repository_location}", "export-tar", "test", "output.tar", "input/dir1", "--tar-format=GNU" + ) + with changedir("output"): + subprocess.check_call(["tar", "xpf", "../output.tar", "--warning=no-timestamp"]) + assert os.stat("input/dir1/hardlink").st_nlink == 2 + assert os.stat("input/dir1/subdir/hardlink").st_nlink == 2 + assert os.stat("input/dir1/aaaa").st_nlink == 2 + assert os.stat("input/dir1/source2").st_nlink == 2 - def test_import_tar(self, tar_format='PAX'): + def test_import_tar(self, tar_format="PAX"): self.create_test_files(create_hardlinks=False) # hardlinks become separate files - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.cmd(f'--repo={self.repository_location}', 'create', 'src', 'input') - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'src', 'simple.tar', f'--tar-format={tar_format}') - self.cmd(f'--repo={self.repository_location}', 'import-tar', 'dst', 'simple.tar') + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.cmd(f"--repo={self.repository_location}", "create", "src", "input") + self.cmd(f"--repo={self.repository_location}", "export-tar", "src", "simple.tar", f"--tar-format={tar_format}") + self.cmd(f"--repo={self.repository_location}", "import-tar", "dst", "simple.tar") with changedir(self.output_path): - self.cmd(f'--repo={self.repository_location}', 'extract', 'dst') - self.assert_dirs_equal('input', 'output/input', ignore_ns=True, ignore_xattrs=True) + self.cmd(f"--repo={self.repository_location}", "extract", "dst") + self.assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True) @requires_gzip - def test_import_tar_gz(self, tar_format='GNU'): - if not shutil.which('gzip'): - pytest.skip('gzip is not installed') + def test_import_tar_gz(self, tar_format="GNU"): + if not shutil.which("gzip"): + pytest.skip("gzip is not installed") self.create_test_files(create_hardlinks=False) # hardlinks become separate files - os.unlink('input/flagfile') - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.cmd(f'--repo={self.repository_location}', 'create', 'src', 'input') - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'src', 'simple.tgz', f'--tar-format={tar_format}') - self.cmd(f'--repo={self.repository_location}', 'import-tar', 'dst', 'simple.tgz') + os.unlink("input/flagfile") + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.cmd(f"--repo={self.repository_location}", "create", "src", "input") + self.cmd(f"--repo={self.repository_location}", "export-tar", "src", "simple.tgz", f"--tar-format={tar_format}") + self.cmd(f"--repo={self.repository_location}", "import-tar", "dst", "simple.tgz") with changedir(self.output_path): - self.cmd(f'--repo={self.repository_location}', 'extract', 'dst') - self.assert_dirs_equal('input', 'output/input', ignore_ns=True, ignore_xattrs=True) + self.cmd(f"--repo={self.repository_location}", "extract", "dst") + self.assert_dirs_equal("input", "output/input", ignore_ns=True, ignore_xattrs=True) def test_roundtrip_pax_borg(self): self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', '--encryption=none') - self.cmd(f'--repo={self.repository_location}', 'create', 'src', 'input') - self.cmd(f'--repo={self.repository_location}', 'export-tar', 'src', 'simple.tar', '--tar-format=BORG') - self.cmd(f'--repo={self.repository_location}', 'import-tar', 'dst', 'simple.tar') + self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none") + self.cmd(f"--repo={self.repository_location}", "create", "src", "input") + self.cmd(f"--repo={self.repository_location}", "export-tar", "src", "simple.tar", "--tar-format=BORG") + self.cmd(f"--repo={self.repository_location}", "import-tar", "dst", "simple.tar") with changedir(self.output_path): - self.cmd(f'--repo={self.repository_location}', 'extract', 'dst') - self.assert_dirs_equal('input', 'output/input') + self.cmd(f"--repo={self.repository_location}", "extract", "dst") + self.assert_dirs_equal("input", "output/input") # derived from test_extract_xattrs_errors() - @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='xattr not supported on this system or on this version of' - 'fakeroot') + @pytest.mark.skipif( + not xattr.XATTR_FAKEROOT, reason="xattr not supported on this system or on this version of" "fakeroot" + ) def test_do_not_fail_when_percent_is_in_xattr_name(self): """https://github.com/borgbackup/borg/issues/6063""" - def patched_setxattr_EACCES(*args, **kwargs): - raise OSError(errno.EACCES, 'EACCES') - self.create_regular_file('file') - xattr.setxattr(b'input/file', b'user.attribute%p', b'value') - self.cmd(f'--repo={self.repository_location}', 'rcreate', '-e' 'none') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - with patch.object(xattr, 'setxattr', patched_setxattr_EACCES): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_WARNING) + def patched_setxattr_EACCES(*args, **kwargs): + raise OSError(errno.EACCES, "EACCES") + + self.create_regular_file("file") + xattr.setxattr(b"input/file", b"user.attribute%p", b"value") + self.cmd(f"--repo={self.repository_location}", "rcreate", "-e" "none") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + with patch.object(xattr, "setxattr", patched_setxattr_EACCES): + self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_WARNING) # derived from test_extract_xattrs_errors() - @pytest.mark.skipif(not xattr.XATTR_FAKEROOT, reason='xattr not supported on this system or on this version of' - 'fakeroot') + @pytest.mark.skipif( + not xattr.XATTR_FAKEROOT, reason="xattr not supported on this system or on this version of" "fakeroot" + ) def test_do_not_fail_when_percent_is_in_file_name(self): """https://github.com/borgbackup/borg/issues/6063""" - def patched_setxattr_EACCES(*args, **kwargs): - raise OSError(errno.EACCES, 'EACCES') - os.makedirs(os.path.join(self.input_path, 'dir%p')) - xattr.setxattr(b'input/dir%p', b'user.attribute', b'value') - self.cmd(f'--repo={self.repository_location}', 'rcreate', '-e' 'none') - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - with changedir('output'): - with patch.object(xattr, 'setxattr', patched_setxattr_EACCES): - self.cmd(f'--repo={self.repository_location}', 'extract', 'test', exit_code=EXIT_WARNING) + def patched_setxattr_EACCES(*args, **kwargs): + raise OSError(errno.EACCES, "EACCES") + + os.makedirs(os.path.join(self.input_path, "dir%p")) + xattr.setxattr(b"input/dir%p", b"user.attribute", b"value") + self.cmd(f"--repo={self.repository_location}", "rcreate", "-e" "none") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + with changedir("output"): + with patch.object(xattr, "setxattr", patched_setxattr_EACCES): + self.cmd(f"--repo={self.repository_location}", "extract", "test", exit_code=EXIT_WARNING) def test_do_not_mention_archive_if_you_can_not_find_repo(self): """https://github.com/borgbackup/borg/issues/6014""" - output = self.cmd(f'--repo={self.repository_location}-this-repository-does-not-exist', 'info', '-a', 'test', - exit_code=2, fork=True) - self.assert_in('this-repository-does-not-exist', output) - self.assert_not_in('this-repository-does-not-exist::test', output) + output = self.cmd( + f"--repo={self.repository_location}-this-repository-does-not-exist", + "info", + "-a", + "test", + exit_code=2, + fork=True, + ) + self.assert_in("this-repository-does-not-exist", output) + self.assert_not_in("this-repository-does-not-exist::test", output) def test_init_defaults_to_argon2(self): """https://github.com/borgbackup/borg/issues/747#issuecomment-1076160401""" - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) with Repository(self.repository_path) as repository: key = msgpack.unpackb(a2b_base64(repository.load_key())) - assert key['algorithm'] == 'argon2 chacha20-poly1305' + assert key["algorithm"] == "argon2 chacha20-poly1305" def test_change_passphrase_does_not_change_algorithm_argon2(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase' + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + os.environ["BORG_NEW_PASSPHRASE"] = "newpassphrase" - self.cmd(f'--repo={self.repository_location}', 'key', 'change-passphrase') + self.cmd(f"--repo={self.repository_location}", "key", "change-passphrase") with Repository(self.repository_path) as repository: key = msgpack.unpackb(a2b_base64(repository.load_key())) - assert key['algorithm'] == 'argon2 chacha20-poly1305' + assert key["algorithm"] == "argon2 chacha20-poly1305" def test_change_location_does_not_change_algorithm_argon2(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', KF_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION) - self.cmd(f'--repo={self.repository_location}', 'key', 'change-location', 'repokey') + self.cmd(f"--repo={self.repository_location}", "key", "change-location", "repokey") with Repository(self.repository_path) as repository: key = msgpack.unpackb(a2b_base64(repository.load_key())) - assert key['algorithm'] == 'argon2 chacha20-poly1305' + assert key["algorithm"] == "argon2 chacha20-poly1305" -@unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') +@unittest.skipUnless("binary" in BORG_EXES, "no borg.exe available") class ArchiverTestCaseBinary(ArchiverTestCase): - EXE = 'borg.exe' + EXE = "borg.exe" FORK_DEFAULT = True - @unittest.skip('does not raise Exception, but sets rc==2') + @unittest.skip("does not raise Exception, but sets rc==2") def test_init_parent_dirs(self): pass - @unittest.skip('patches objects') + @unittest.skip("patches objects") def test_init_interrupt(self): pass - @unittest.skip('patches objects') + @unittest.skip("patches objects") def test_extract_capabilities(self): pass - @unittest.skip('patches objects') + @unittest.skip("patches objects") def test_extract_xattrs_errors(self): pass - @unittest.skip('test_basic_functionality seems incompatible with fakeroot and/or the binary.') + @unittest.skip("test_basic_functionality seems incompatible with fakeroot and/or the binary.") def test_basic_functionality(self): pass - @unittest.skip('test_overwrite seems incompatible with fakeroot and/or the binary.') + @unittest.skip("test_overwrite seems incompatible with fakeroot and/or the binary.") def test_overwrite(self): pass def test_fuse(self): if fakeroot_detected(): - unittest.skip('test_fuse with the binary is not compatible with fakeroot') + unittest.skip("test_fuse with the binary is not compatible with fakeroot") else: super().test_fuse() - @unittest.skip('patches objects') + @unittest.skip("patches objects") def test_do_not_fail_when_percent_is_in_xattr_name(self): pass - @unittest.skip('patches objects') + @unittest.skip("patches objects") def test_do_not_fail_when_percent_is_in_file_name(self): pass class ArchiverCheckTestCase(ArchiverTestCaseBase): - def setUp(self): super().setUp() - with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('archive1') - self.create_src_archive('archive2') + with patch.object(ChunkBuffer, "BUFFER_SIZE", 10): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("archive1") + self.create_src_archive("archive2") def test_check_usage(self): - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--progress', exit_code=0) - self.assert_in('Starting repository check', output) - self.assert_in('Starting archive consistency check', output) - self.assert_in('Checking segments', output) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--progress", exit_code=0) + self.assert_in("Starting repository check", output) + self.assert_in("Starting archive consistency check", output) + self.assert_in("Checking segments", output) # reset logging to new process default to avoid need for fork=True on next check - logging.getLogger('borg.output.progress').setLevel(logging.NOTSET) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--repository-only', exit_code=0) - self.assert_in('Starting repository check', output) - self.assert_not_in('Starting archive consistency check', output) - self.assert_not_in('Checking segments', output) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--archives-only', exit_code=0) - self.assert_not_in('Starting repository check', output) - self.assert_in('Starting archive consistency check', output) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--archives-only', '--glob-archives=archive2', exit_code=0) - self.assert_not_in('archive1', output) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--archives-only', '--first=1', exit_code=0) - self.assert_in('archive1', output) - self.assert_not_in('archive2', output) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--archives-only', '--last=1', exit_code=0) - self.assert_not_in('archive1', output) - self.assert_in('archive2', output) + logging.getLogger("borg.output.progress").setLevel(logging.NOTSET) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--repository-only", exit_code=0) + self.assert_in("Starting repository check", output) + self.assert_not_in("Starting archive consistency check", output) + self.assert_not_in("Checking segments", output) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--archives-only", exit_code=0) + self.assert_not_in("Starting repository check", output) + self.assert_in("Starting archive consistency check", output) + output = self.cmd( + f"--repo={self.repository_location}", + "check", + "-v", + "--archives-only", + "--glob-archives=archive2", + exit_code=0, + ) + self.assert_not_in("archive1", output) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--first=1", exit_code=0 + ) + self.assert_in("archive1", output) + self.assert_not_in("archive2", output) + output = self.cmd( + f"--repo={self.repository_location}", "check", "-v", "--archives-only", "--last=1", exit_code=0 + ) + self.assert_not_in("archive1", output) + self.assert_in("archive2", output) def test_missing_file_chunk(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): valid_chunks = item.chunks killed_chunk = valid_chunks[-1] repository.delete(killed_chunk.id) break else: - self.fail('should not happen') + self.fail("should not happen") repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - output = self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) - self.assert_in('New missing file chunk detected', output) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) - output = self.cmd(f'--repo={self.repository_location}', 'list', 'archive1', '--format={health}#{path}{LF}', exit_code=0) - self.assert_in('broken#', output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + output = self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) + self.assert_in("New missing file chunk detected", output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) + output = self.cmd( + f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{LF}", exit_code=0 + ) + self.assert_in("broken#", output) # check that the file in the old archives has now a different chunk list without the killed chunk - for archive_name in ('archive1', 'archive2'): + for archive_name in ("archive1", "archive2"): archive, repository = self.open_archive(archive_name) with repository: for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): self.assert_not_equal(valid_chunks, item.chunks) self.assert_not_in(killed_chunk, item.chunks) break else: - self.fail('should not happen') + self.fail("should not happen") # do a fresh backup (that will include the killed chunk) - with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10): - self.create_src_archive('archive3') + with patch.object(ChunkBuffer, "BUFFER_SIZE", 10): + self.create_src_archive("archive3") # check should be able to heal the file now: - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--repair', exit_code=0) - self.assert_in('Healed previously missing file chunk', output) - self.assert_in('testsuite/archiver.py: Completely healed previously damaged file!', output) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--repair", exit_code=0) + self.assert_in("Healed previously missing file chunk", output) + self.assert_in("testsuite/archiver.py: Completely healed previously damaged file!", output) # check that the file in the old archives has the correct chunks again - for archive_name in ('archive1', 'archive2'): + for archive_name in ("archive1", "archive2"): archive, repository = self.open_archive(archive_name) with repository: for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): self.assert_equal(valid_chunks, item.chunks) break else: - self.fail('should not happen') + self.fail("should not happen") # list is also all-healthy again - output = self.cmd(f'--repo={self.repository_location}', 'list', 'archive1', '--format={health}#{path}{LF}', exit_code=0) - self.assert_not_in('broken#', output) + output = self.cmd( + f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{LF}", exit_code=0 + ) + self.assert_not_in("broken#", output) def test_missing_archive_item_chunk(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: repository.delete(archive.metadata.items[0]) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) def test_missing_archive_metadata(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: repository.delete(archive.id) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) def test_missing_manifest(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: repository.delete(Manifest.MANIFEST_ID) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--repair', exit_code=0) - self.assert_in('archive1', output) - self.assert_in('archive2', output) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--repair", exit_code=0) + self.assert_in("archive1", output) + self.assert_in("archive2", output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) def test_corrupted_manifest(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: manifest = repository.get(Manifest.MANIFEST_ID) - corrupted_manifest = manifest + b'corrupted!' + corrupted_manifest = manifest + b"corrupted!" repository.put(Manifest.MANIFEST_ID, corrupted_manifest) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--repair', exit_code=0) - self.assert_in('archive1', output) - self.assert_in('archive2', output) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--repair", exit_code=0) + self.assert_in("archive1", output) + self.assert_in("archive2", output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) def test_manifest_rebuild_corrupted_chunk(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") with repository: manifest = repository.get(Manifest.MANIFEST_ID) - corrupted_manifest = manifest + b'corrupted!' + corrupted_manifest = manifest + b"corrupted!" repository.put(Manifest.MANIFEST_ID, corrupted_manifest) chunk = repository.get(archive.id) - corrupted_chunk = chunk + b'corrupted!' + corrupted_chunk = chunk + b"corrupted!" repository.put(archive.id, corrupted_chunk) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - output = self.cmd(f'--repo={self.repository_location}', 'check', '-v', '--repair', exit_code=0) - self.assert_in('archive2', output) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + output = self.cmd(f"--repo={self.repository_location}", "check", "-v", "--repair", exit_code=0) + self.assert_in("archive2", output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) def test_manifest_rebuild_duplicate_archive(self): - archive, repository = self.open_archive('archive1') + archive, repository = self.open_archive("archive1") key = archive.key with repository: manifest = repository.get(Manifest.MANIFEST_ID) - corrupted_manifest = manifest + b'corrupted!' + corrupted_manifest = manifest + b"corrupted!" repository.put(Manifest.MANIFEST_ID, corrupted_manifest) - archive = msgpack.packb({ - 'cmdline': [], - 'items': [], - 'hostname': 'foo', - 'username': 'bar', - 'name': 'archive1', - 'time': '2016-12-15T18:49:51.849711', - 'version': 2, - }) + archive = msgpack.packb( + { + "cmdline": [], + "items": [], + "hostname": "foo", + "username": "bar", + "name": "archive1", + "time": "2016-12-15T18:49:51.849711", + "version": 2, + } + ) archive_id = key.id_hash(archive) repository.put(archive_id, key.encrypt(archive_id, archive)) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) - output = self.cmd(f'--repo={self.repository_location}', 'rlist') - self.assert_in('archive1', output) - self.assert_in('archive1.1', output) - self.assert_in('archive2', output) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) + output = self.cmd(f"--repo={self.repository_location}", "rlist") + self.assert_in("archive1", output) + self.assert_in("archive1.1", output) + self.assert_in("archive2", output) def test_extra_chunks(self): - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) with Repository(self.repository_location, exclusive=True) as repository: - repository.put(b'01234567890123456789012345678901', b'xxxx') + repository.put(b"01234567890123456789012345678901", b"xxxx") repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) - self.cmd(f'--repo={self.repository_location}', 'check', '--repair', exit_code=0) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) - self.cmd(f'--repo={self.repository_location}', 'extract', 'archive1', '--dry-run', exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", "--repair", exit_code=0) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) + self.cmd(f"--repo={self.repository_location}", "extract", "archive1", "--dry-run", exit_code=0) def _test_verify_data(self, *init_args): shutil.rmtree(self.repository_path) - self.cmd(f'--repo={self.repository_location}', 'rcreate', *init_args) - self.create_src_archive('archive1') - archive, repository = self.open_archive('archive1') + self.cmd(f"--repo={self.repository_location}", "rcreate", *init_args) + self.create_src_archive("archive1") + archive, repository = self.open_archive("archive1") with repository: for item in archive.iter_items(): - if item.path.endswith('testsuite/archiver.py'): + if item.path.endswith("testsuite/archiver.py"): chunk = item.chunks[-1] - data = repository.get(chunk.id) + b'1234' + data = repository.get(chunk.id) + b"1234" repository.put(chunk.id, data) break repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=0) - output = self.cmd(f'--repo={self.repository_location}', 'check', '--verify-data', exit_code=1) - assert bin_to_hex(chunk.id) + ', integrity error' in output + self.cmd(f"--repo={self.repository_location}", "check", exit_code=0) + output = self.cmd(f"--repo={self.repository_location}", "check", "--verify-data", exit_code=1) + assert bin_to_hex(chunk.id) + ", integrity error" in output # repair (heal is tested in another test) - output = self.cmd(f'--repo={self.repository_location}', 'check', '--repair', '--verify-data', exit_code=0) - assert bin_to_hex(chunk.id) + ', integrity error' in output - assert 'testsuite/archiver.py: New missing file chunk detected' in output + output = self.cmd(f"--repo={self.repository_location}", "check", "--repair", "--verify-data", exit_code=0) + assert bin_to_hex(chunk.id) + ", integrity error" in output + assert "testsuite/archiver.py: New missing file chunk detected" in output def test_verify_data(self): self._test_verify_data(RK_ENCRYPTION) def test_verify_data_unencrypted(self): - self._test_verify_data('--encryption', 'none') + self._test_verify_data("--encryption", "none") def test_empty_repository(self): with Repository(self.repository_location, exclusive=True) as repository: for id_ in repository.list(): repository.delete(id_) repository.commit(compact=False) - self.cmd(f'--repo={self.repository_location}', 'check', exit_code=1) + self.cmd(f"--repo={self.repository_location}", "check", exit_code=1) class ManifestAuthenticationTest(ArchiverTestCaseBase): def spoof_manifest(self, repository): with repository: _, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - repository.put(Manifest.MANIFEST_ID, key.encrypt(Manifest.MANIFEST_ID, msgpack.packb({ - 'version': 1, - 'archives': {}, - 'config': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), - }))) + repository.put( + Manifest.MANIFEST_ID, + key.encrypt( + Manifest.MANIFEST_ID, + msgpack.packb( + { + "version": 1, + "archives": {}, + "config": {}, + "timestamp": (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), + } + ), + ), + ) repository.commit(compact=False) def test_fresh_init_tam_required(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) repository = Repository(self.repository_path, exclusive=True) with repository: manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) - repository.put(Manifest.MANIFEST_ID, key.encrypt(Manifest.MANIFEST_ID, msgpack.packb({ - 'version': 1, - 'archives': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), - }))) + repository.put( + Manifest.MANIFEST_ID, + key.encrypt( + Manifest.MANIFEST_ID, + msgpack.packb( + { + "version": 1, + "archives": {}, + "timestamp": (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), + } + ), + ), + ) repository.commit(compact=False) with pytest.raises(TAMRequiredError): - self.cmd(f'--repo={self.repository_location}', 'rlist') + self.cmd(f"--repo={self.repository_location}", "rlist") def test_not_required(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_src_archive('archive1234') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("archive1234") repository = Repository(self.repository_path, exclusive=True) # Manifest must be authenticated now - output = self.cmd(f'--repo={self.repository_location}', 'rlist', '--debug') - assert 'archive1234' in output - assert 'TAM-verified manifest' in output + output = self.cmd(f"--repo={self.repository_location}", "rlist", "--debug") + assert "archive1234" in output + assert "TAM-verified manifest" in output # Try to spoof / modify pre-1.0.9 self.spoof_manifest(repository) # Fails with pytest.raises(TAMRequiredError): - self.cmd(f'--repo={self.repository_location}', 'rlist') + self.cmd(f"--repo={self.repository_location}", "rlist") class RemoteArchiverTestCase(ArchiverTestCase): - prefix = 'ssh://__testsuite__' + prefix = "ssh://__testsuite__" def open_repository(self): return RemoteRepository(Location(self.repository_location)) def test_remote_repo_restrict_to_path(self): # restricted to repo directory itself: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', self.repository_path]): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", self.repository_path]): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # restricted to repo directory itself, fail for other directories with same prefix: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', self.repository_path]): + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", self.repository_path]): with pytest.raises(PathNotAllowed): - self.cmd(f'--repo={self.repository_location}_0', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}_0", "rcreate", RK_ENCRYPTION) # restricted to a completely different path: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo']): + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", "/foo"]): with pytest.raises(PathNotAllowed): - self.cmd(f'--repo={self.repository_location}_1', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}_1", "rcreate", RK_ENCRYPTION) path_prefix = os.path.dirname(self.repository_path) # restrict to repo directory's parent directory: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', path_prefix]): - self.cmd(f'--repo={self.repository_location}_2', 'rcreate', RK_ENCRYPTION) + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", path_prefix]): + self.cmd(f"--repo={self.repository_location}_2", "rcreate", RK_ENCRYPTION) # restrict to repo directory's parent directory and another directory: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo', '--restrict-to-path', path_prefix]): - self.cmd(f'--repo={self.repository_location}_3', 'rcreate', RK_ENCRYPTION) + with patch.object( + RemoteRepository, "extra_test_args", ["--restrict-to-path", "/foo", "--restrict-to-path", path_prefix] + ): + self.cmd(f"--repo={self.repository_location}_3", "rcreate", RK_ENCRYPTION) def test_remote_repo_restrict_to_repository(self): # restricted to repo directory itself: - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-repository', self.repository_path]): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - parent_path = os.path.join(self.repository_path, '..') - with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-repository', parent_path]): + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", self.repository_path]): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + parent_path = os.path.join(self.repository_path, "..") + with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", parent_path]): with pytest.raises(PathNotAllowed): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - @unittest.skip('only works locally') + @unittest.skip("only works locally") def test_debug_put_get_delete_obj(self): pass - @unittest.skip('only works locally') + @unittest.skip("only works locally") def test_config(self): pass - @unittest.skip('only works locally') + @unittest.skip("only works locally") def test_migrate_lock_alive(self): pass def test_remote_repo_strip_components_doesnt_leak(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.create_regular_file('dir/file', contents=b"test file contents 1") - self.create_regular_file('dir/file2', contents=b"test file contents 2") - self.create_regular_file('skipped-file1', contents=b"test file contents 3") - self.create_regular_file('skipped-file2', contents=b"test file contents 4") - self.create_regular_file('skipped-file3', contents=b"test file contents 5") - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - marker = 'cached responses left in RemoteRepository' - with changedir('output'): - res = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', "--debug", '--strip-components', '3') + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_regular_file("dir/file", contents=b"test file contents 1") + self.create_regular_file("dir/file2", contents=b"test file contents 2") + self.create_regular_file("skipped-file1", contents=b"test file contents 3") + self.create_regular_file("skipped-file2", contents=b"test file contents 4") + self.create_regular_file("skipped-file3", contents=b"test file contents 5") + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + marker = "cached responses left in RemoteRepository" + with changedir("output"): + res = self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--debug", "--strip-components", "3" + ) assert marker not in res - with self.assert_creates_file('file'): - res = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', "--debug", '--strip-components', '2') + with self.assert_creates_file("file"): + res = self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--debug", "--strip-components", "2" + ) assert marker not in res - with self.assert_creates_file('dir/file'): - res = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', "--debug", '--strip-components', '1') + with self.assert_creates_file("dir/file"): + res = self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--debug", "--strip-components", "1" + ) assert marker not in res - with self.assert_creates_file('input/dir/file'): - res = self.cmd(f'--repo={self.repository_location}', 'extract', 'test', "--debug", '--strip-components', '0') + with self.assert_creates_file("input/dir/file"): + res = self.cmd( + f"--repo={self.repository_location}", "extract", "test", "--debug", "--strip-components", "0" + ) assert marker not in res @@ -3864,313 +4187,317 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase): def setUp(self): super().setUp() self.create_test_files() - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) - self.cache_path = json.loads(self.cmd(f'--repo={self.repository_location}', 'rinfo', '--json'))['cache']['path'] + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cache_path = json.loads(self.cmd(f"--repo={self.repository_location}", "rinfo", "--json"))["cache"]["path"] def corrupt(self, file, amount=1): - with open(file, 'r+b') as fd: + with open(file, "r+b") as fd: fd.seek(-amount, io.SEEK_END) - corrupted = bytes(255-c for c in fd.read(amount)) + corrupted = bytes(255 - c for c in fd.read(amount)) fd.seek(-amount, io.SEEK_END) fd.write(corrupted) def test_cache_chunks(self): - self.corrupt(os.path.join(self.cache_path, 'chunks')) + self.corrupt(os.path.join(self.cache_path, "chunks")) if self.FORK_DEFAULT: - out = self.cmd(f'--repo={self.repository_location}', 'rinfo', exit_code=2) - assert 'failed integrity check' in out + out = self.cmd(f"--repo={self.repository_location}", "rinfo", exit_code=2) + assert "failed integrity check" in out else: with pytest.raises(FileIntegrityError): - self.cmd(f'--repo={self.repository_location}', 'rinfo') + self.cmd(f"--repo={self.repository_location}", "rinfo") def test_cache_files(self): - self.cmd(f'--repo={self.repository_location}', 'create', 'test', 'input') - self.corrupt(os.path.join(self.cache_path, 'files')) - out = self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.corrupt(os.path.join(self.cache_path, "files")) + out = self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") # borg warns about the corrupt files cache, but then continues without files cache. - assert 'files cache is corrupted' in out + assert "files cache is corrupted" in out def test_chunks_archive(self): - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") # Find ID of test1 so we can corrupt it later :) - target_id = self.cmd(f'--repo={self.repository_location}', 'rlist', '--format={id}{LF}').strip() - self.cmd(f'--repo={self.repository_location}', 'create', 'test2', 'input') + target_id = self.cmd(f"--repo={self.repository_location}", "rlist", "--format={id}{LF}").strip() + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input") # Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d - self.cmd(f'--repo={self.repository_location}', 'rdelete', '--cache-only') - self.cmd(f'--repo={self.repository_location}', 'rinfo', '--json') + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") + self.cmd(f"--repo={self.repository_location}", "rinfo", "--json") - chunks_archive = os.path.join(self.cache_path, 'chunks.archive.d') + chunks_archive = os.path.join(self.cache_path, "chunks.archive.d") assert len(os.listdir(chunks_archive)) == 4 # two archives, one chunks cache and one .integrity file each - self.corrupt(os.path.join(chunks_archive, target_id + '.compact')) + self.corrupt(os.path.join(chunks_archive, target_id + ".compact")) # Trigger cache sync by changing the manifest ID in the cache config - config_path = os.path.join(self.cache_path, 'config') + config_path = os.path.join(self.cache_path, "config") config = ConfigParser(interpolation=None) config.read(config_path) - config.set('cache', 'manifest', bin_to_hex(bytes(32))) - with open(config_path, 'w') as fd: + config.set("cache", "manifest", bin_to_hex(bytes(32))) + with open(config_path, "w") as fd: config.write(fd) # Cache sync notices corrupted archive chunks, but automatically recovers. - out = self.cmd(f'--repo={self.repository_location}', 'create', '-v', 'test3', 'input', exit_code=1) - assert 'Reading cached archive chunk index for test1' in out - assert 'Cached archive chunk index of test1 is corrupted' in out - assert 'Fetching and building archive index for test1' in out + out = self.cmd(f"--repo={self.repository_location}", "create", "-v", "test3", "input", exit_code=1) + assert "Reading cached archive chunk index for test1" in out + assert "Cached archive chunk index of test1 is corrupted" in out + assert "Fetching and building archive index for test1" in out def test_old_version_interfered(self): # Modify the main manifest ID without touching the manifest ID in the integrity section. # This happens if a version without integrity checking modifies the cache. - config_path = os.path.join(self.cache_path, 'config') + config_path = os.path.join(self.cache_path, "config") config = ConfigParser(interpolation=None) config.read(config_path) - config.set('cache', 'manifest', bin_to_hex(bytes(32))) - with open(config_path, 'w') as fd: + config.set("cache", "manifest", bin_to_hex(bytes(32))) + with open(config_path, "w") as fd: config.write(fd) - out = self.cmd(f'--repo={self.repository_location}', 'rinfo') - assert 'Cache integrity data not available: old Borg version modified the cache.' in out + out = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "Cache integrity data not available: old Borg version modified the cache." in out class DiffArchiverTestCase(ArchiverTestCaseBase): def test_basic_functionality(self): # Setup files for the first snapshot - self.create_regular_file('empty', size=0) - self.create_regular_file('file_unchanged', size=128) - self.create_regular_file('file_removed', size=256) - self.create_regular_file('file_removed2', size=512) - self.create_regular_file('file_replaced', size=1024) - os.mkdir('input/dir_replaced_with_file') - os.chmod('input/dir_replaced_with_file', stat.S_IFDIR | 0o755) - os.mkdir('input/dir_removed') + self.create_regular_file("empty", size=0) + self.create_regular_file("file_unchanged", size=128) + self.create_regular_file("file_removed", size=256) + self.create_regular_file("file_removed2", size=512) + self.create_regular_file("file_replaced", size=1024) + os.mkdir("input/dir_replaced_with_file") + os.chmod("input/dir_replaced_with_file", stat.S_IFDIR | 0o755) + os.mkdir("input/dir_removed") if are_symlinks_supported(): - os.mkdir('input/dir_replaced_with_link') - os.symlink('input/dir_replaced_with_file', 'input/link_changed') - os.symlink('input/file_unchanged', 'input/link_removed') - os.symlink('input/file_removed2', 'input/link_target_removed') - os.symlink('input/empty', 'input/link_target_contents_changed') - os.symlink('input/empty', 'input/link_replaced_by_file') + os.mkdir("input/dir_replaced_with_link") + os.symlink("input/dir_replaced_with_file", "input/link_changed") + os.symlink("input/file_unchanged", "input/link_removed") + os.symlink("input/file_removed2", "input/link_target_removed") + os.symlink("input/empty", "input/link_target_contents_changed") + os.symlink("input/empty", "input/link_replaced_by_file") if are_hardlinks_supported(): - os.link('input/file_replaced', 'input/hardlink_target_replaced') - os.link('input/empty', 'input/hardlink_contents_changed') - os.link('input/file_removed', 'input/hardlink_removed') - os.link('input/file_removed2', 'input/hardlink_target_removed') + os.link("input/file_replaced", "input/hardlink_target_replaced") + os.link("input/empty", "input/hardlink_contents_changed") + os.link("input/file_removed", "input/hardlink_removed") + os.link("input/file_removed2", "input/hardlink_target_removed") - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) # Create the first snapshot - self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") # Setup files for the second snapshot - self.create_regular_file('file_added', size=2048) - self.create_regular_file('file_empty_added', size=0) - os.unlink('input/file_replaced') - self.create_regular_file('file_replaced', contents=b'0' * 4096) - os.unlink('input/file_removed') - os.unlink('input/file_removed2') - os.rmdir('input/dir_replaced_with_file') - self.create_regular_file('dir_replaced_with_file', size=8192) - os.chmod('input/dir_replaced_with_file', stat.S_IFREG | 0o755) - os.mkdir('input/dir_added') - os.rmdir('input/dir_removed') + self.create_regular_file("file_added", size=2048) + self.create_regular_file("file_empty_added", size=0) + os.unlink("input/file_replaced") + self.create_regular_file("file_replaced", contents=b"0" * 4096) + os.unlink("input/file_removed") + os.unlink("input/file_removed2") + os.rmdir("input/dir_replaced_with_file") + self.create_regular_file("dir_replaced_with_file", size=8192) + os.chmod("input/dir_replaced_with_file", stat.S_IFREG | 0o755) + os.mkdir("input/dir_added") + os.rmdir("input/dir_removed") if are_symlinks_supported(): - os.rmdir('input/dir_replaced_with_link') - os.symlink('input/dir_added', 'input/dir_replaced_with_link') - os.unlink('input/link_changed') - os.symlink('input/dir_added', 'input/link_changed') - os.symlink('input/dir_added', 'input/link_added') - os.unlink('input/link_replaced_by_file') - self.create_regular_file('link_replaced_by_file', size=16384) - os.unlink('input/link_removed') + os.rmdir("input/dir_replaced_with_link") + os.symlink("input/dir_added", "input/dir_replaced_with_link") + os.unlink("input/link_changed") + os.symlink("input/dir_added", "input/link_changed") + os.symlink("input/dir_added", "input/link_added") + os.unlink("input/link_replaced_by_file") + self.create_regular_file("link_replaced_by_file", size=16384) + os.unlink("input/link_removed") if are_hardlinks_supported(): - os.unlink('input/hardlink_removed') - os.link('input/file_added', 'input/hardlink_added') + os.unlink("input/hardlink_removed") + os.link("input/file_added", "input/hardlink_added") - with open('input/empty', 'ab') as fd: - fd.write(b'appended_data') + with open("input/empty", "ab") as fd: + fd.write(b"appended_data") # Create the second snapshot - self.cmd(f'--repo={self.repository_location}', 'create', 'test1a', 'input') - self.cmd(f'--repo={self.repository_location}', 'create', 'test1b', 'input', '--chunker-params', '16,18,17,4095') + self.cmd(f"--repo={self.repository_location}", "create", "test1a", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test1b", "input", "--chunker-params", "16,18,17,4095") def do_asserts(output, can_compare_ids): # File contents changed (deleted and replaced with a new file) - change = 'B' if can_compare_ids else '{:<19}'.format('modified') - assert 'file_replaced' in output # added to debug #3494 - assert f'{change} input/file_replaced' in output + change = "B" if can_compare_ids else "{:<19}".format("modified") + assert "file_replaced" in output # added to debug #3494 + assert f"{change} input/file_replaced" in output # File unchanged - assert 'input/file_unchanged' not in output + assert "input/file_unchanged" not in output # Directory replaced with a regular file - if 'BORG_TESTS_IGNORE_MODES' not in os.environ: - assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output + if "BORG_TESTS_IGNORE_MODES" not in os.environ: + assert "[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file" in output # Basic directory cases - assert 'added directory input/dir_added' in output - assert 'removed directory input/dir_removed' in output + assert "added directory input/dir_added" in output + assert "removed directory input/dir_removed" in output if are_symlinks_supported(): # Basic symlink cases - assert 'changed link input/link_changed' in output - assert 'added link input/link_added' in output - assert 'removed link input/link_removed' in output + assert "changed link input/link_changed" in output + assert "added link input/link_added" in output + assert "removed link input/link_removed" in output # Symlink replacing or being replaced - assert '] input/dir_replaced_with_link' in output - assert '] input/link_replaced_by_file' in output + assert "] input/dir_replaced_with_link" in output + assert "] input/link_replaced_by_file" in output # Symlink target removed. Should not affect the symlink at all. - assert 'input/link_target_removed' not in output + assert "input/link_target_removed" not in output # The inode has two links and the file contents changed. Borg # should notice the changes in both links. However, the symlink # pointing to the file is not changed. - change = '0 B' if can_compare_ids else '{:<19}'.format('modified') - assert f'{change} input/empty' in output + change = "0 B" if can_compare_ids else "{:<19}".format("modified") + assert f"{change} input/empty" in output if are_hardlinks_supported(): - assert f'{change} input/hardlink_contents_changed' in output + assert f"{change} input/hardlink_contents_changed" in output if are_symlinks_supported(): - assert 'input/link_target_contents_changed' not in output + assert "input/link_target_contents_changed" not in output # Added a new file and a hard link to it. Both links to the same # inode should appear as separate files. - assert 'added 2.05 kB input/file_added' in output + assert "added 2.05 kB input/file_added" in output if are_hardlinks_supported(): - assert 'added 2.05 kB input/hardlink_added' in output + assert "added 2.05 kB input/hardlink_added" in output # check if a diff between non-existent and empty new file is found - assert 'added 0 B input/file_empty_added' in output + assert "added 0 B input/file_empty_added" in output # The inode has two links and both of them are deleted. They should # appear as two deleted files. - assert 'removed 256 B input/file_removed' in output + assert "removed 256 B input/file_removed" in output if are_hardlinks_supported(): - assert 'removed 256 B input/hardlink_removed' in output + assert "removed 256 B input/hardlink_removed" in output # Another link (marked previously as the source in borg) to the # same inode was removed. This should not change this link at all. if are_hardlinks_supported(): - assert 'input/hardlink_target_removed' not in output + assert "input/hardlink_target_removed" not in output # Another link (marked previously as the source in borg) to the # same inode was replaced with a new regular file. This should not # change this link at all. if are_hardlinks_supported(): - assert 'input/hardlink_target_replaced' not in output + assert "input/hardlink_target_replaced" not in output def do_json_asserts(output, can_compare_ids): def get_changes(filename, data): - chgsets = [j['changes'] for j in data if j['path'] == filename] + chgsets = [j["changes"] for j in data if j["path"] == filename] assert len(chgsets) < 2 # return a flattened list of changes for given filename return [chg for chgset in chgsets for chg in chgset] # convert output to list of dicts - joutput = [json.loads(line) for line in output.split('\n') if line] + joutput = [json.loads(line) for line in output.split("\n") if line] # File contents changed (deleted and replaced with a new file) - expected = {'type': 'modified', 'added': 4096, 'removed': 1024} if can_compare_ids else {'type': 'modified'} - assert expected in get_changes('input/file_replaced', joutput) + expected = {"type": "modified", "added": 4096, "removed": 1024} if can_compare_ids else {"type": "modified"} + assert expected in get_changes("input/file_replaced", joutput) # File unchanged - assert not any(get_changes('input/file_unchanged', joutput)) + assert not any(get_changes("input/file_unchanged", joutput)) # Directory replaced with a regular file - if 'BORG_TESTS_IGNORE_MODES' not in os.environ: - assert {'type': 'mode', 'old_mode': 'drwxr-xr-x', 'new_mode': '-rwxr-xr-x'} in \ - get_changes('input/dir_replaced_with_file', joutput) + if "BORG_TESTS_IGNORE_MODES" not in os.environ: + assert {"type": "mode", "old_mode": "drwxr-xr-x", "new_mode": "-rwxr-xr-x"} in get_changes( + "input/dir_replaced_with_file", joutput + ) # Basic directory cases - assert {'type': 'added directory'} in get_changes('input/dir_added', joutput) - assert {'type': 'removed directory'} in get_changes('input/dir_removed', joutput) + assert {"type": "added directory"} in get_changes("input/dir_added", joutput) + assert {"type": "removed directory"} in get_changes("input/dir_removed", joutput) if are_symlinks_supported(): # Basic symlink cases - assert {'type': 'changed link'} in get_changes('input/link_changed', joutput) - assert {'type': 'added link'} in get_changes('input/link_added', joutput) - assert {'type': 'removed link'} in get_changes('input/link_removed', joutput) + assert {"type": "changed link"} in get_changes("input/link_changed", joutput) + assert {"type": "added link"} in get_changes("input/link_added", joutput) + assert {"type": "removed link"} in get_changes("input/link_removed", joutput) # Symlink replacing or being replaced - assert any(chg['type'] == 'mode' and chg['new_mode'].startswith('l') for chg in - get_changes('input/dir_replaced_with_link', joutput)) - assert any(chg['type'] == 'mode' and chg['old_mode'].startswith('l') for chg in - get_changes('input/link_replaced_by_file', joutput)) + assert any( + chg["type"] == "mode" and chg["new_mode"].startswith("l") + for chg in get_changes("input/dir_replaced_with_link", joutput) + ) + assert any( + chg["type"] == "mode" and chg["old_mode"].startswith("l") + for chg in get_changes("input/link_replaced_by_file", joutput) + ) # Symlink target removed. Should not affect the symlink at all. - assert not any(get_changes('input/link_target_removed', joutput)) + assert not any(get_changes("input/link_target_removed", joutput)) # The inode has two links and the file contents changed. Borg # should notice the changes in both links. However, the symlink # pointing to the file is not changed. - expected = {'type': 'modified', 'added': 13, 'removed': 0} if can_compare_ids else {'type': 'modified'} - assert expected in get_changes('input/empty', joutput) + expected = {"type": "modified", "added": 13, "removed": 0} if can_compare_ids else {"type": "modified"} + assert expected in get_changes("input/empty", joutput) if are_hardlinks_supported(): - assert expected in get_changes('input/hardlink_contents_changed', joutput) + assert expected in get_changes("input/hardlink_contents_changed", joutput) if are_symlinks_supported(): - assert not any(get_changes('input/link_target_contents_changed', joutput)) + assert not any(get_changes("input/link_target_contents_changed", joutput)) # Added a new file and a hard link to it. Both links to the same # inode should appear as separate files. - assert {'type': 'added', 'size': 2048} in get_changes('input/file_added', joutput) + assert {"type": "added", "size": 2048} in get_changes("input/file_added", joutput) if are_hardlinks_supported(): - assert {'type': 'added', 'size': 2048} in get_changes('input/hardlink_added', joutput) + assert {"type": "added", "size": 2048} in get_changes("input/hardlink_added", joutput) # check if a diff between non-existent and empty new file is found - assert {'type': 'added', 'size': 0} in get_changes('input/file_empty_added', joutput) + assert {"type": "added", "size": 0} in get_changes("input/file_empty_added", joutput) # The inode has two links and both of them are deleted. They should # appear as two deleted files. - assert {'type': 'removed', 'size': 256} in get_changes('input/file_removed', joutput) + assert {"type": "removed", "size": 256} in get_changes("input/file_removed", joutput) if are_hardlinks_supported(): - assert {'type': 'removed', 'size': 256} in get_changes('input/hardlink_removed', joutput) + assert {"type": "removed", "size": 256} in get_changes("input/hardlink_removed", joutput) # Another link (marked previously as the source in borg) to the # same inode was removed. This should not change this link at all. if are_hardlinks_supported(): - assert not any(get_changes('input/hardlink_target_removed', joutput)) + assert not any(get_changes("input/hardlink_target_removed", joutput)) # Another link (marked previously as the source in borg) to the # same inode was replaced with a new regular file. This should not # change this link at all. if are_hardlinks_supported(): - assert not any(get_changes('input/hardlink_target_replaced', joutput)) + assert not any(get_changes("input/hardlink_target_replaced", joutput)) - do_asserts(self.cmd(f'--repo={self.repository_location}', 'diff', 'test0', 'test1a'), True) + do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a"), True) # We expect exit_code=1 due to the chunker params warning - do_asserts(self.cmd(f'--repo={self.repository_location}', 'diff', 'test0', 'test1b', exit_code=1), False) - do_json_asserts(self.cmd(f'--repo={self.repository_location}', 'diff', 'test0', 'test1a', '--json-lines'), True) + do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1b", exit_code=1), False) + do_json_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a", "--json-lines"), True) def test_sort_option(self): - self.cmd(f'--repo={self.repository_location}', 'rcreate', RK_ENCRYPTION) + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - self.create_regular_file('a_file_removed', size=8) - self.create_regular_file('f_file_removed', size=16) - self.create_regular_file('c_file_changed', size=32) - self.create_regular_file('e_file_changed', size=64) - self.cmd(f'--repo={self.repository_location}', 'create', 'test0', 'input') + self.create_regular_file("a_file_removed", size=8) + self.create_regular_file("f_file_removed", size=16) + self.create_regular_file("c_file_changed", size=32) + self.create_regular_file("e_file_changed", size=64) + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") - os.unlink('input/a_file_removed') - os.unlink('input/f_file_removed') - os.unlink('input/c_file_changed') - os.unlink('input/e_file_changed') - self.create_regular_file('c_file_changed', size=512) - self.create_regular_file('e_file_changed', size=1024) - self.create_regular_file('b_file_added', size=128) - self.create_regular_file('d_file_added', size=256) - self.cmd(f'--repo={self.repository_location}', 'create', 'test1', 'input') + os.unlink("input/a_file_removed") + os.unlink("input/f_file_removed") + os.unlink("input/c_file_changed") + os.unlink("input/e_file_changed") + self.create_regular_file("c_file_changed", size=512) + self.create_regular_file("e_file_changed", size=1024) + self.create_regular_file("b_file_added", size=128) + self.create_regular_file("d_file_added", size=256) + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") - output = self.cmd(f'--repo={self.repository_location}', 'diff', 'test0', 'test1', - '--sort') + output = self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1", "--sort") expected = [ - 'a_file_removed', - 'b_file_added', - 'c_file_changed', - 'd_file_added', - 'e_file_changed', - 'f_file_removed', + "a_file_removed", + "b_file_added", + "c_file_changed", + "d_file_added", + "e_file_changed", + "f_file_removed", ] assert all(x in line for x, line in zip(expected, output.splitlines())) @@ -4181,38 +4508,46 @@ def test_get_args(): # everything normal: # first param is argv as produced by ssh forced command, # second param is like from SSH_ORIGINAL_COMMAND env variable - args = archiver.get_args(['borg', 'serve', '--umask=0027', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], - 'borg serve --info') + args = archiver.get_args( + ["borg", "serve", "--umask=0027", "--restrict-to-path=/p1", "--restrict-to-path=/p2"], "borg serve --info" + ) assert args.func == archiver.do_serve - assert args.restrict_to_paths == ['/p1', '/p2'] + assert args.restrict_to_paths == ["/p1", "/p2"] assert args.umask == 0o027 - assert args.log_level == 'info' + assert args.log_level == "info" # similar, but with --restrict-to-repository - args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], - 'borg serve --info --umask=0027') - assert args.restrict_to_repositories == ['/r1', '/r2'] + args = archiver.get_args( + ["borg", "serve", "--restrict-to-repository=/r1", "--restrict-to-repository=/r2"], + "borg serve --info --umask=0027", + ) + assert args.restrict_to_repositories == ["/r1", "/r2"] # trying to cheat - break out of path restriction - args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], - 'borg serve --restrict-to-path=/') - assert args.restrict_to_paths == ['/p1', '/p2'] + args = archiver.get_args( + ["borg", "serve", "--restrict-to-path=/p1", "--restrict-to-path=/p2"], "borg serve --restrict-to-path=/" + ) + assert args.restrict_to_paths == ["/p1", "/p2"] # trying to cheat - break out of repository restriction - args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], - 'borg serve --restrict-to-repository=/') - assert args.restrict_to_repositories == ['/r1', '/r2'] + args = archiver.get_args( + ["borg", "serve", "--restrict-to-repository=/r1", "--restrict-to-repository=/r2"], + "borg serve --restrict-to-repository=/", + ) + assert args.restrict_to_repositories == ["/r1", "/r2"] # trying to cheat - break below repository restriction - args = archiver.get_args(['borg', 'serve', '--restrict-to-repository=/r1', '--restrict-to-repository=/r2', ], - 'borg serve --restrict-to-repository=/r1/below') - assert args.restrict_to_repositories == ['/r1', '/r2'] + args = archiver.get_args( + ["borg", "serve", "--restrict-to-repository=/r1", "--restrict-to-repository=/r2"], + "borg serve --restrict-to-repository=/r1/below", + ) + assert args.restrict_to_repositories == ["/r1", "/r2"] # trying to cheat - try to execute different subcommand - args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], - f'borg --repo=/ rcreate {RK_ENCRYPTION}') + args = archiver.get_args( + ["borg", "serve", "--restrict-to-path=/p1", "--restrict-to-path=/p2"], f"borg --repo=/ rcreate {RK_ENCRYPTION}" + ) assert args.func == archiver.do_serve # Check that environment variables in the forced command don't cause issues. If the command # were not forced, environment variables would be interpreted by the shell, but this does not # happen for forced commands - we get the verbatim command line and need to deal with env vars. - args = archiver.get_args(['borg', 'serve', ], - 'BORG_FOO=bar borg serve --info') + args = archiver.get_args(["borg", "serve"], "BORG_FOO=bar borg serve --info") assert args.func == archiver.do_serve @@ -4224,98 +4559,91 @@ def ccc(a, b): compare2 = chunks_contents_equal(iter(chunks_b), iter(chunks_a)) assert compare1 == compare2 return compare1 - assert ccc([ - b'1234', b'567A', b'bC' - ], [ - b'1', b'23', b'4567A', b'b', b'C' - ]) + + assert ccc([b"1234", b"567A", b"bC"], [b"1", b"23", b"4567A", b"b", b"C"]) # one iterator exhausted before the other - assert not ccc([ - b'12345', - ], [ - b'1234', b'56' - ]) + assert not ccc([b"12345"], [b"1234", b"56"]) # content mismatch - assert not ccc([ - b'1234', b'65' - ], [ - b'1234', b'56' - ]) + assert not ccc([b"1234", b"65"], [b"1234", b"56"]) # first is the prefix of second - assert not ccc([ - b'1234', b'56' - ], [ - b'1234', b'565' - ]) + assert not ccc([b"1234", b"56"], [b"1234", b"565"]) class TestBuildFilter: - def test_basic(self): matcher = PatternMatcher() - matcher.add([parse_pattern('included')], IECommand.Include) + matcher.add([parse_pattern("included")], IECommand.Include) filter = Archiver.build_filter(matcher, 0) - assert filter(Item(path='included')) - assert filter(Item(path='included/file')) - assert not filter(Item(path='something else')) + assert filter(Item(path="included")) + assert filter(Item(path="included/file")) + assert not filter(Item(path="something else")) def test_empty(self): matcher = PatternMatcher(fallback=True) filter = Archiver.build_filter(matcher, 0) - assert filter(Item(path='anything')) + assert filter(Item(path="anything")) def test_strip_components(self): matcher = PatternMatcher(fallback=True) filter = Archiver.build_filter(matcher, strip_components=1) - assert not filter(Item(path='shallow')) - assert not filter(Item(path='shallow/')) # can this even happen? paths are normalized... - assert filter(Item(path='deep enough/file')) - assert filter(Item(path='something/dir/file')) + assert not filter(Item(path="shallow")) + assert not filter(Item(path="shallow/")) # can this even happen? paths are normalized... + assert filter(Item(path="deep enough/file")) + assert filter(Item(path="something/dir/file")) class TestCommonOptions: @staticmethod def define_common_options(add_common_option): - add_common_option('-h', '--help', action='help', help='show this help message and exit') - add_common_option('--critical', dest='log_level', help='foo', - action='store_const', const='critical', default='warning') - add_common_option('--error', dest='log_level', help='foo', - action='store_const', const='error', default='warning') - add_common_option('--append', dest='append', help='foo', - action='append', metavar='TOPIC', default=[]) - add_common_option('-p', '--progress', dest='progress', action='store_true', help='foo') - add_common_option('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, - help='(default: %(default)d).') + add_common_option("-h", "--help", action="help", help="show this help message and exit") + add_common_option( + "--critical", dest="log_level", help="foo", action="store_const", const="critical", default="warning" + ) + add_common_option( + "--error", dest="log_level", help="foo", action="store_const", const="error", default="warning" + ) + add_common_option("--append", dest="append", help="foo", action="append", metavar="TOPIC", default=[]) + add_common_option("-p", "--progress", dest="progress", action="store_true", help="foo") + add_common_option( + "--lock-wait", dest="lock_wait", type=int, metavar="N", default=1, help="(default: %(default)d)." + ) @pytest.fixture def basic_parser(self): - parser = argparse.ArgumentParser(prog='test', description='test parser', add_help=False) - parser.common_options = Archiver.CommonOptions(self.define_common_options, - suffix_precedence=('_level0', '_level1')) + parser = argparse.ArgumentParser(prog="test", description="test parser", add_help=False) + parser.common_options = Archiver.CommonOptions( + self.define_common_options, suffix_precedence=("_level0", "_level1") + ) return parser @pytest.fixture def subparsers(self, basic_parser): - return basic_parser.add_subparsers(title='required arguments', metavar='') + return basic_parser.add_subparsers(title="required arguments", metavar="") @pytest.fixture def parser(self, basic_parser): - basic_parser.common_options.add_common_group(basic_parser, '_level0', provide_defaults=True) + basic_parser.common_options.add_common_group(basic_parser, "_level0", provide_defaults=True) return basic_parser @pytest.fixture def common_parser(self, parser): - common_parser = argparse.ArgumentParser(add_help=False, prog='test') - parser.common_options.add_common_group(common_parser, '_level1') + common_parser = argparse.ArgumentParser(add_help=False, prog="test") + parser.common_options.add_common_group(common_parser, "_level1") return common_parser @pytest.fixture def parse_vars_from_line(self, parser, subparsers, common_parser): - subparser = subparsers.add_parser('subcommand', parents=[common_parser], add_help=False, - description='foo', epilog='bar', help='baz', - formatter_class=argparse.RawDescriptionHelpFormatter) + subparser = subparsers.add_parser( + "subcommand", + parents=[common_parser], + add_help=False, + description="foo", + epilog="bar", + help="baz", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) subparser.set_defaults(func=1234) - subparser.add_argument('--append-only', dest='append_only', action='store_true') + subparser.add_argument("--append-only", dest="append_only", action="store_true") def parse_vars_from_line(*line): print(line) @@ -4326,54 +4654,51 @@ def parse_vars_from_line(*line): return parse_vars_from_line def test_simple(self, parse_vars_from_line): - assert parse_vars_from_line('--error') == { - 'append': [], - 'lock_wait': 1, - 'log_level': 'error', - 'progress': False + assert parse_vars_from_line("--error") == { + "append": [], + "lock_wait": 1, + "log_level": "error", + "progress": False, } - assert parse_vars_from_line('--error', 'subcommand', '--critical') == { - 'append': [], - 'lock_wait': 1, - 'log_level': 'critical', - 'progress': False, - 'append_only': False, - 'func': 1234, + assert parse_vars_from_line("--error", "subcommand", "--critical") == { + "append": [], + "lock_wait": 1, + "log_level": "critical", + "progress": False, + "append_only": False, + "func": 1234, } with pytest.raises(SystemExit): - parse_vars_from_line('--append-only', 'subcommand') + parse_vars_from_line("--append-only", "subcommand") - assert parse_vars_from_line('--append=foo', '--append', 'bar', 'subcommand', '--append', 'baz') == { - 'append': ['foo', 'bar', 'baz'], - 'lock_wait': 1, - 'log_level': 'warning', - 'progress': False, - 'append_only': False, - 'func': 1234, + assert parse_vars_from_line("--append=foo", "--append", "bar", "subcommand", "--append", "baz") == { + "append": ["foo", "bar", "baz"], + "lock_wait": 1, + "log_level": "warning", + "progress": False, + "append_only": False, + "func": 1234, } - @pytest.mark.parametrize('position', ('before', 'after', 'both')) - @pytest.mark.parametrize('flag,args_key,args_value', ( - ('-p', 'progress', True), - ('--lock-wait=3', 'lock_wait', 3), - )) + @pytest.mark.parametrize("position", ("before", "after", "both")) + @pytest.mark.parametrize("flag,args_key,args_value", (("-p", "progress", True), ("--lock-wait=3", "lock_wait", 3))) def test_flag_position_independence(self, parse_vars_from_line, position, flag, args_key, args_value): line = [] - if position in ('before', 'both'): + if position in ("before", "both"): line.append(flag) - line.append('subcommand') - if position in ('after', 'both'): + line.append("subcommand") + if position in ("after", "both"): line.append(flag) result = { - 'append': [], - 'lock_wait': 1, - 'log_level': 'warning', - 'progress': False, - 'append_only': False, - 'func': 1234, + "append": [], + "lock_wait": 1, + "log_level": "warning", + "progress": False, + "append_only": False, + "func": 1234, } result[args_key] = args_value @@ -4381,23 +4706,23 @@ def test_flag_position_independence(self, parse_vars_from_line, position, flag, def test_parse_storage_quota(): - assert parse_storage_quota('50M') == 50 * 1000**2 + assert parse_storage_quota("50M") == 50 * 1000**2 with pytest.raises(argparse.ArgumentTypeError): - parse_storage_quota('5M') + parse_storage_quota("5M") def get_all_parsers(): """ Return dict mapping command to parser. """ - parser = Archiver(prog='borg').build_parser() - borgfs_parser = Archiver(prog='borgfs').build_parser() + parser = Archiver(prog="borg").build_parser() + borgfs_parser = Archiver(prog="borgfs").build_parser() parsers = {} def discover_level(prefix, parser, Archiver, extra_choices=None): choices = {} for action in parser._actions: - if action.choices is not None and 'SubParsersAction' in str(action.__class__): + if action.choices is not None and "SubParsersAction" in str(action.__class__): for cmd, parser in action.choices.items(): choices[prefix + cmd] = parser if extra_choices is not None: @@ -4409,16 +4734,16 @@ def discover_level(prefix, parser, Archiver, extra_choices=None): discover_level(command + " ", parser, Archiver) parsers[command] = parser - discover_level("", parser, Archiver, {'borgfs': borgfs_parser}) + discover_level("", parser, Archiver, {"borgfs": borgfs_parser}) return parsers -@pytest.mark.parametrize('command, parser', list(get_all_parsers().items())) +@pytest.mark.parametrize("command, parser", list(get_all_parsers().items())) def test_help_formatting(command, parser): if isinstance(parser.epilog, RstToTextLazy): assert parser.epilog.rst -@pytest.mark.parametrize('topic, helptext', list(Archiver.helptext.items())) +@pytest.mark.parametrize("topic, helptext", list(Archiver.helptext.items())) def test_help_formatting_helptexts(topic, helptext): assert str(rst_to_terminal(helptext)) diff --git a/src/borg/testsuite/benchmark.py b/src/borg/testsuite/benchmark.py index cc21f19f6..563397ad5 100644 --- a/src/borg/testsuite/benchmark.py +++ b/src/borg/testsuite/benchmark.py @@ -16,35 +16,38 @@ @pytest.fixture def repo_url(request, tmpdir, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', '123456') - monkeypatch.setenv('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', 'YES') - monkeypatch.setenv('BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', 'YES') - monkeypatch.setenv('BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', 'yes') - monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir.join('keys'))) - monkeypatch.setenv('BORG_CACHE_DIR', str(tmpdir.join('cache'))) - yield str(tmpdir.join('repository')) + monkeypatch.setenv("BORG_PASSPHRASE", "123456") + monkeypatch.setenv("BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", "YES") + monkeypatch.setenv("BORG_DELETE_I_KNOW_WHAT_I_AM_DOING", "YES") + monkeypatch.setenv("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK", "yes") + monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir.join("keys"))) + monkeypatch.setenv("BORG_CACHE_DIR", str(tmpdir.join("cache"))) + yield str(tmpdir.join("repository")) tmpdir.remove(rec=1) @pytest.fixture(params=["none", "repokey-aes-ocb"]) def repo(request, cmd, repo_url): - cmd(f'--repo={repo_url}', 'rcreate', '--encryption', request.param) + cmd(f"--repo={repo_url}", "rcreate", "--encryption", request.param) return repo_url -@pytest.fixture(scope='session', params=["zeros", "random"]) +@pytest.fixture(scope="session", params=["zeros", "random"]) def testdata(request, tmpdir_factory): - count, size = 10, 1000*1000 + count, size = 10, 1000 * 1000 assert size <= len(zeros) - p = tmpdir_factory.mktemp('data') + p = tmpdir_factory.mktemp("data") data_type = request.param - if data_type == 'zeros': + if data_type == "zeros": # do not use a binary zero (\0) to avoid sparse detection def data(size): return memoryview(zeros)[:size] - elif data_type == 'random': + + elif data_type == "random": + def data(size): return os.urandom(size) + else: raise ValueError("data_type must be 'random' or 'zeros'.") for i in range(count): @@ -54,56 +57,54 @@ def data(size): p.remove(rec=1) -@pytest.fixture(params=['none', 'lz4']) +@pytest.fixture(params=["none", "lz4"]) def repo_archive(request, cmd, repo, testdata): - archive = 'test' - cmd(f'--repo={repo}', 'create', '--compression', request.param, archive, testdata) + archive = "test" + cmd(f"--repo={repo}", "create", "--compression", request.param, archive, testdata) return repo, archive def test_create_none(benchmark, cmd, repo, testdata): - result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'none', - 'test', testdata)) + result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "none", "test", testdata)) assert result == 0 def test_create_lz4(benchmark, cmd, repo, testdata): - result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'lz4', - 'test', testdata)) + result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "lz4", "test", testdata)) assert result == 0 def test_extract(benchmark, cmd, repo_archive, tmpdir): repo, archive = repo_archive with changedir(str(tmpdir)): - result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'extract', archive)) + result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "extract", archive)) assert result == 0 def test_delete(benchmark, cmd, repo_archive): repo, archive = repo_archive - result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'delete', '-a', archive)) + result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "delete", "-a", archive)) assert result == 0 def test_list(benchmark, cmd, repo_archive): repo, archive = repo_archive - result, out = benchmark(cmd, f'--repo={repo}', 'list', archive) + result, out = benchmark(cmd, f"--repo={repo}", "list", archive) assert result == 0 def test_info(benchmark, cmd, repo_archive): repo, archive = repo_archive - result, out = benchmark(cmd, f'--repo={repo}', 'info', '-a', archive) + result, out = benchmark(cmd, f"--repo={repo}", "info", "-a", archive) assert result == 0 def test_check(benchmark, cmd, repo_archive): repo, archive = repo_archive - result, out = benchmark(cmd, f'--repo={repo}', 'check') + result, out = benchmark(cmd, f"--repo={repo}", "check") assert result == 0 def test_help(benchmark, cmd): - result, out = benchmark(cmd, 'help') + result, out = benchmark(cmd, "help") assert result == 0 diff --git a/src/borg/testsuite/cache.py b/src/borg/testsuite/cache.py index f642a588e..a1c8be16a 100644 --- a/src/borg/testsuite/cache.py +++ b/src/borg/testsuite/cache.py @@ -26,75 +26,29 @@ def sync(self, index): return CacheSynchronizer(index) def test_no_chunks(self, index, sync): - data = packb({ - 'foo': 'bar', - 'baz': 1234, - 'bar': 5678, - 'user': 'chunks', - 'chunks': [] - }) + data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": []}) sync.feed(data) assert not len(index) def test_simple(self, index, sync): - data = packb({ - 'foo': 'bar', - 'baz': 1234, - 'bar': 5678, - 'user': 'chunks', - 'chunks': [ - (H(1), 1), - (H(2), 2), - ] - }) + data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]}) sync.feed(data) assert len(index) == 2 assert index[H(1)] == (1, 1) assert index[H(2)] == (1, 2) def test_multiple(self, index, sync): - data = packb({ - 'foo': 'bar', - 'baz': 1234, - 'bar': 5678, - 'user': 'chunks', - 'chunks': [ - (H(1), 1), - (H(2), 2), - ] - }) - data += packb({ - 'xattrs': { - 'security.foo': 'bar', - 'chunks': '123456', - }, - 'stuff': [ - (1, 2, 3), - ] - }) - data += packb({ - 'xattrs': { - 'security.foo': 'bar', - 'chunks': '123456', - }, - 'chunks': [ - (H(1), 1), - (H(2), 2), - ], - 'stuff': [ - (1, 2, 3), - ] - }) - data += packb({ - 'chunks': [ - (H(3), 1), - ], - }) - data += packb({ - 'chunks': [ - (H(1), 1), - ], - }) + data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]}) + data += packb({"xattrs": {"security.foo": "bar", "chunks": "123456"}, "stuff": [(1, 2, 3)]}) + data += packb( + { + "xattrs": {"security.foo": "bar", "chunks": "123456"}, + "chunks": [(H(1), 1), (H(2), 2)], + "stuff": [(1, 2, 3)], + } + ) + data += packb({"chunks": [(H(3), 1)]}) + data += packb({"chunks": [(H(1), 1)]}) part1 = data[:70] part2 = data[70:120] @@ -107,62 +61,68 @@ def test_multiple(self, index, sync): assert index[H(2)] == (2, 2) assert index[H(3)] == (1, 1) - @pytest.mark.parametrize('elem,error', ( - ({1: 2}, 'Unexpected object: map'), - (bytes(213), [ - 'Unexpected bytes in chunks structure', # structure 2/3 - 'Incorrect key length']), # structure 3/3 - (1, 'Unexpected object: integer'), - (1.0, 'Unexpected object: double'), - (True, 'Unexpected object: true'), - (False, 'Unexpected object: false'), - (None, 'Unexpected object: nil'), - )) - @pytest.mark.parametrize('structure', ( - lambda elem: {'chunks': elem}, - lambda elem: {'chunks': [elem]}, - lambda elem: {'chunks': [(elem, 1)]}, - )) + @pytest.mark.parametrize( + "elem,error", + ( + ({1: 2}, "Unexpected object: map"), + ( + bytes(213), + ["Unexpected bytes in chunks structure", "Incorrect key length"], # structure 2/3 + ), # structure 3/3 + (1, "Unexpected object: integer"), + (1.0, "Unexpected object: double"), + (True, "Unexpected object: true"), + (False, "Unexpected object: false"), + (None, "Unexpected object: nil"), + ), + ) + @pytest.mark.parametrize( + "structure", + (lambda elem: {"chunks": elem}, lambda elem: {"chunks": [elem]}, lambda elem: {"chunks": [(elem, 1)]}), + ) def test_corrupted(self, sync, structure, elem, error): packed = packb(structure(elem)) with pytest.raises(ValueError) as excinfo: sync.feed(packed) if isinstance(error, str): error = [error] - possible_errors = ['cache_sync_feed failed: ' + error for error in error] + possible_errors = ["cache_sync_feed failed: " + error for error in error] assert str(excinfo.value) in possible_errors - @pytest.mark.parametrize('data,error', ( - # Incorrect tuple length - ({'chunks': [(bytes(32), 2, 3, 4)]}, 'Invalid chunk list entry length'), - ({'chunks': [(bytes(32), )]}, 'Invalid chunk list entry length'), - # Incorrect types - ({'chunks': [(1, 2)]}, 'Unexpected object: integer'), - ({'chunks': [(1, bytes(32))]}, 'Unexpected object: integer'), - ({'chunks': [(bytes(32), 1.0)]}, 'Unexpected object: double'), - )) + @pytest.mark.parametrize( + "data,error", + ( + # Incorrect tuple length + ({"chunks": [(bytes(32), 2, 3, 4)]}, "Invalid chunk list entry length"), + ({"chunks": [(bytes(32),)]}, "Invalid chunk list entry length"), + # Incorrect types + ({"chunks": [(1, 2)]}, "Unexpected object: integer"), + ({"chunks": [(1, bytes(32))]}, "Unexpected object: integer"), + ({"chunks": [(bytes(32), 1.0)]}, "Unexpected object: double"), + ), + ) def test_corrupted_ancillary(self, index, sync, data, error): packed = packb(data) with pytest.raises(ValueError) as excinfo: sync.feed(packed) - assert str(excinfo.value) == 'cache_sync_feed failed: ' + error + assert str(excinfo.value) == "cache_sync_feed failed: " + error def make_index_with_refcount(self, refcount): index_data = io.BytesIO() - index_data.write(b'BORG_IDX') + index_data.write(b"BORG_IDX") # num_entries - index_data.write((1).to_bytes(4, 'little')) + index_data.write((1).to_bytes(4, "little")) # num_buckets - index_data.write((1).to_bytes(4, 'little')) + index_data.write((1).to_bytes(4, "little")) # key_size - index_data.write((32).to_bytes(1, 'little')) + index_data.write((32).to_bytes(1, "little")) # value_size - index_data.write((3 * 4).to_bytes(1, 'little')) + index_data.write((3 * 4).to_bytes(1, "little")) index_data.write(H(0)) - index_data.write(refcount.to_bytes(4, 'little')) - index_data.write((1234).to_bytes(4, 'little')) - index_data.write((5678).to_bytes(4, 'little')) + index_data.write(refcount.to_bytes(4, "little")) + index_data.write((1234).to_bytes(4, "little")) + index_data.write((5678).to_bytes(4, "little")) index_data.seek(0) index = ChunkIndex.read(index_data) @@ -171,34 +131,22 @@ def make_index_with_refcount(self, refcount): def test_corrupted_refcount(self): index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE + 1) sync = CacheSynchronizer(index) - data = packb({ - 'chunks': [ - (H(0), 1), - ] - }) + data = packb({"chunks": [(H(0), 1)]}) with pytest.raises(ValueError) as excinfo: sync.feed(data) - assert str(excinfo.value) == 'cache_sync_feed failed: invalid reference count' + assert str(excinfo.value) == "cache_sync_feed failed: invalid reference count" def test_refcount_max_value(self): index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE) sync = CacheSynchronizer(index) - data = packb({ - 'chunks': [ - (H(0), 1), - ] - }) + data = packb({"chunks": [(H(0), 1)]}) sync.feed(data) assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234) def test_refcount_one_below_max_value(self): index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE - 1) sync = CacheSynchronizer(index) - data = packb({ - 'chunks': [ - (H(0), 1), - ] - }) + data = packb({"chunks": [(H(0), 1)]}) sync.feed(data) # Incremented to maximum assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234) @@ -209,17 +157,17 @@ def test_refcount_one_below_max_value(self): class TestAdHocCache: @pytest.fixture def repository(self, tmpdir): - self.repository_location = os.path.join(str(tmpdir), 'repository') + self.repository_location = os.path.join(str(tmpdir), "repository") with Repository(self.repository_location, exclusive=True, create=True) as repository: - repository.put(H(1), b'1234') - repository.put(Manifest.MANIFEST_ID, b'5678') + repository.put(H(1), b"1234") + repository.put(Manifest.MANIFEST_ID, b"5678") yield repository @pytest.fixture def key(self, repository, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") key = AESOCBRepoKey.create(repository, TestKey.MockArgs()) - key.compressor = CompressionSpec('none').compressor + key.compressor = CompressionSpec("none").compressor return key @pytest.fixture @@ -237,18 +185,18 @@ def test_does_not_contain_manifest(self, cache): def test_does_not_delete_existing_chunks(self, repository, cache): assert cache.seen_chunk(H(1)) == ChunkIndex.MAX_VALUE cache.chunk_decref(H(1), Statistics()) - assert repository.get(H(1)) == b'1234' + assert repository.get(H(1)) == b"1234" def test_does_not_overwrite(self, cache): with pytest.raises(AssertionError): - cache.add_chunk(H(1), b'5678', Statistics(), overwrite=True) + cache.add_chunk(H(1), b"5678", Statistics(), overwrite=True) def test_seen_chunk_add_chunk_size(self, cache): - assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4) + assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4) def test_deletes_chunks_during_lifetime(self, cache, repository): """E.g. checkpoint archives""" - cache.add_chunk(H(5), b'1010', Statistics()) + cache.add_chunk(H(5), b"1010", Statistics()) assert cache.seen_chunk(H(5)) == 1 cache.chunk_decref(H(5), Statistics()) assert not cache.seen_chunk(H(5)) @@ -256,8 +204,8 @@ def test_deletes_chunks_during_lifetime(self, cache, repository): repository.get(H(5)) def test_files_cache(self, cache): - assert cache.file_known_and_unchanged(b'foo', bytes(32), None) == (False, None) - assert cache.cache_mode == 'd' + assert cache.file_known_and_unchanged(b"foo", bytes(32), None) == (False, None) + assert cache.cache_mode == "d" assert cache.files is None def test_txn(self, cache): @@ -267,13 +215,13 @@ def test_txn(self, cache): assert cache.chunks cache.rollback() assert not cache._txn_active - assert not hasattr(cache, 'chunks') + assert not hasattr(cache, "chunks") def test_incref_after_add_chunk(self, cache): - assert cache.add_chunk(H(3), b'5678', Statistics()) == (H(3), 4) + assert cache.add_chunk(H(3), b"5678", Statistics()) == (H(3), 4) assert cache.chunk_incref(H(3), Statistics()) == (H(3), 4) def test_existing_incref_after_add_chunk(self, cache): """This case occurs with part files, see Archive.chunk_file.""" - assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4) + assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4) assert cache.chunk_incref(H(1), Statistics()) == (H(1), 4) diff --git a/src/borg/testsuite/checksums.py b/src/borg/testsuite/checksums.py index 2567fda8a..9a4f63753 100644 --- a/src/borg/testsuite/checksums.py +++ b/src/borg/testsuite/checksums.py @@ -5,16 +5,24 @@ def test_xxh64(): - assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf' - assert bin_to_hex(checksums.xxh64(b'test')) == '4fdcca5ddb678139' - assert bin_to_hex(checksums.xxh64(unhexlify( - '6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724' - 'fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d' - 'cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331'))) == '35d5d2f545d9511a' + assert bin_to_hex(checksums.xxh64(b"test", 123)) == "2b81b9401bef86cf" + assert bin_to_hex(checksums.xxh64(b"test")) == "4fdcca5ddb678139" + assert ( + bin_to_hex( + checksums.xxh64( + unhexlify( + "6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724" + "fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d" + "cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331" + ) + ) + ) + == "35d5d2f545d9511a" + ) def test_streaming_xxh64(): hasher = checksums.StreamingXXH64(123) - hasher.update(b'te') - hasher.update(b'st') - assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == '2b81b9401bef86cf' + hasher.update(b"te") + hasher.update(b"st") + assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == "2b81b9401bef86cf" diff --git a/src/borg/testsuite/chunker.py b/src/borg/testsuite/chunker.py index ce45b7c96..54aade3ea 100644 --- a/src/borg/testsuite/chunker.py +++ b/src/borg/testsuite/chunker.py @@ -12,113 +12,129 @@ def cf(chunks): """chunk filter""" # this is to simplify testing: either return the data piece (bytes) or the hole length (int). def _cf(chunk): - if chunk.meta['allocation'] == CH_DATA: - assert len(chunk.data) == chunk.meta['size'] + if chunk.meta["allocation"] == CH_DATA: + assert len(chunk.data) == chunk.meta["size"] return bytes(chunk.data) # make sure we have bytes, not memoryview - if chunk.meta['allocation'] in (CH_HOLE, CH_ALLOC): + if chunk.meta["allocation"] in (CH_HOLE, CH_ALLOC): assert chunk.data is None - return chunk.meta['size'] + return chunk.meta["size"] assert False, "unexpected allocation value" + return [_cf(chunk) for chunk in chunks] class ChunkerFixedTestCase(BaseTestCase): - def test_chunkify_just_blocks(self): - data = b'foobar' * 1500 + data = b"foobar" * 1500 chunker = ChunkerFixed(4096) parts = cf(chunker.chunkify(BytesIO(data))) self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]]) def test_chunkify_header_and_blocks(self): - data = b'foobar' * 1500 + data = b"foobar" * 1500 chunker = ChunkerFixed(4096, 123) parts = cf(chunker.chunkify(BytesIO(data))) - self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]]) + self.assert_equal( + parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]] + ) def test_chunkify_just_blocks_fmap_complete(self): - data = b'foobar' * 1500 + data = b"foobar" * 1500 chunker = ChunkerFixed(4096) - fmap = [ - (0, 4096, True), - (4096, 8192, True), - (8192, 99999999, True), - ] + fmap = [(0, 4096, True), (4096, 8192, True), (8192, 99999999, True)] parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap)) self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]]) def test_chunkify_header_and_blocks_fmap_complete(self): - data = b'foobar' * 1500 + data = b"foobar" * 1500 chunker = ChunkerFixed(4096, 123) - fmap = [ - (0, 123, True), - (123, 4096, True), - (123+4096, 4096, True), - (123+8192, 4096, True), - ] + fmap = [(0, 123, True), (123, 4096, True), (123 + 4096, 4096, True), (123 + 8192, 4096, True)] parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap)) - self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]]) + self.assert_equal( + parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]] + ) def test_chunkify_header_and_blocks_fmap_zeros(self): - data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096 + data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096 chunker = ChunkerFixed(4096, 123) - fmap = [ - (0, 123, True), - (123, 4096, False), - (123+4096, 4096, True), - (123+8192, 4096, False), - ] + fmap = [(0, 123, True), (123, 4096, False), (123 + 4096, 4096, True), (123 + 8192, 4096, False)] parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap)) # because we marked the '_' ranges as holes, we will get hole ranges instead! - self.assert_equal(parts, [data[0:123], 4096, data[123+4096:123+8192], 4096]) + self.assert_equal(parts, [data[0:123], 4096, data[123 + 4096 : 123 + 8192], 4096]) def test_chunkify_header_and_blocks_fmap_partial(self): - data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096 + data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096 chunker = ChunkerFixed(4096, 123) fmap = [ (0, 123, True), # (123, 4096, False), - (123+4096, 4096, True), + (123 + 4096, 4096, True), # (123+8192, 4096, False), ] parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap)) # because we left out the '_' ranges from the fmap, we will not get them at all! - self.assert_equal(parts, [data[0:123], data[123+4096:123+8192]]) + self.assert_equal(parts, [data[0:123], data[123 + 4096 : 123 + 8192]]) class ChunkerTestCase(BaseTestCase): - def test_chunkify(self): - data = b'0' * int(1.5 * (1 << CHUNK_MAX_EXP)) + b'Y' + data = b"0" * int(1.5 * (1 << CHUNK_MAX_EXP)) + b"Y" parts = cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(data))) self.assert_equal(len(parts), 2) - self.assert_equal(b''.join(parts), data) - self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b''))), []) - self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz']) - self.assert_equal(cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz']) - self.assert_equal(cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz']) - self.assert_equal(cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3]) - self.assert_equal(cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz']) - self.assert_equal(cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz']) - self.assert_equal(cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3]) - self.assert_equal(cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarbo', b'obazfoobar', b'boobazfo', b'obarboobaz']) - self.assert_equal(cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz', b'foobarboobaz', b'foobarboobaz']) + self.assert_equal(b"".join(parts), data) + self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b""))), []) + self.assert_equal( + cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"fooba", b"rboobaz", b"fooba", b"rboobaz", b"fooba", b"rboobaz"], + ) + self.assert_equal( + cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"fo", b"obarb", b"oob", b"azf", b"oobarb", b"oob", b"azf", b"oobarb", b"oobaz"], + ) + self.assert_equal( + cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"foob", b"ar", b"boobazfoob", b"ar", b"boobazfoob", b"ar", b"boobaz"], + ) + self.assert_equal( + cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3] + ) + self.assert_equal( + cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"foobar", b"boobazfo", b"obar", b"boobazfo", b"obar", b"boobaz"], + ) + self.assert_equal( + cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"foob", b"arboobaz", b"foob", b"arboobaz", b"foob", b"arboobaz"], + ) + self.assert_equal( + cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3] + ) + self.assert_equal( + cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"foobarbo", b"obazfoobar", b"boobazfo", b"obarboobaz"], + ) + self.assert_equal( + cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), + [b"foobarboobaz", b"foobarboobaz", b"foobarboobaz"], + ) def test_buzhash(self): - self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769) - self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502) - self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1)) + self.assert_equal(buzhash(b"abcdefghijklmnop", 0), 3795437769) + self.assert_equal(buzhash(b"abcdefghijklmnop", 1), 3795400502) + self.assert_equal( + buzhash(b"abcdefghijklmnop", 1), buzhash_update(buzhash(b"Xabcdefghijklmno", 1), ord("X"), ord("p"), 16, 1) + ) # Test with more than 31 bytes to make sure our barrel_shift macro works correctly - self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248) + self.assert_equal(buzhash(b"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", 0), 566521248) def test_small_reads(self): class SmallReadFile: - input = b'a' * (20 + 1) + input = b"a" * (20 + 1) def read(self, nbytes): self.input = self.input[:-1] return self.input[:1] chunker = get_chunker(*CHUNKER_PARAMS, seed=0) - reconstructed = b''.join(cf(chunker.chunkify(SmallReadFile()))) - assert reconstructed == b'a' * 20 + reconstructed = b"".join(cf(chunker.chunkify(SmallReadFile()))) + assert reconstructed == b"a" * 20 diff --git a/src/borg/testsuite/chunker_pytest.py b/src/borg/testsuite/chunker_pytest.py index 59c7a4515..b4161268d 100644 --- a/src/borg/testsuite/chunker_pytest.py +++ b/src/borg/testsuite/chunker_pytest.py @@ -12,37 +12,27 @@ # some sparse files. X = content blocks, _ = sparse blocks. # X__XXX____ -map_sparse1 = [ - (0 * BS, 1 * BS, True), - (1 * BS, 2 * BS, False), - (3 * BS, 3 * BS, True), - (6 * BS, 4 * BS, False), -] +map_sparse1 = [(0 * BS, 1 * BS, True), (1 * BS, 2 * BS, False), (3 * BS, 3 * BS, True), (6 * BS, 4 * BS, False)] # _XX___XXXX -map_sparse2 = [ - (0 * BS, 1 * BS, False), - (1 * BS, 2 * BS, True), - (3 * BS, 3 * BS, False), - (6 * BS, 4 * BS, True), -] +map_sparse2 = [(0 * BS, 1 * BS, False), (1 * BS, 2 * BS, True), (3 * BS, 3 * BS, False), (6 * BS, 4 * BS, True)] # XXX -map_notsparse = [(0 * BS, 3 * BS, True), ] +map_notsparse = [(0 * BS, 3 * BS, True)] # ___ -map_onlysparse = [(0 * BS, 3 * BS, False), ] +map_onlysparse = [(0 * BS, 3 * BS, False)] def make_sparsefile(fname, sparsemap, header_size=0): - with open(fname, 'wb') as fd: + with open(fname, "wb") as fd: total = 0 if header_size: - fd.write(b'H' * header_size) + fd.write(b"H" * header_size) total += header_size for offset, size, is_data in sparsemap: if is_data: - fd.write(b'X' * size) + fd.write(b"X" * size) else: fd.seek(size, os.SEEK_CUR) total += size @@ -54,11 +44,11 @@ def make_content(sparsemap, header_size=0): result = [] total = 0 if header_size: - result.append(b'H' * header_size) + result.append(b"H" * header_size) total += header_size for offset, size, is_data in sparsemap: if is_data: - result.append(b'X' * size) # bytes! + result.append(b"X" * size) # bytes! else: result.append(size) # int! total += size @@ -69,9 +59,9 @@ def fs_supports_sparse(): if not has_seek_hole: return False with tempfile.TemporaryDirectory() as tmpdir: - fn = os.path.join(tmpdir, 'test_sparse') + fn = os.path.join(tmpdir, "test_sparse") make_sparsefile(fn, [(0, BS, False), (BS, BS, True)]) - with open(fn, 'rb') as f: + with open(fn, "rb") as f: try: offset_hole = f.seek(0, os.SEEK_HOLE) offset_data = f.seek(0, os.SEEK_DATA) @@ -81,15 +71,12 @@ def fs_supports_sparse(): return offset_hole == 0 and offset_data == BS -@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files') -@pytest.mark.parametrize("fname, sparse_map", [ - ('sparse1', map_sparse1), - ('sparse2', map_sparse2), - ('onlysparse', map_onlysparse), - ('notsparse', map_notsparse), -]) +@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files") +@pytest.mark.parametrize( + "fname, sparse_map", + [("sparse1", map_sparse1), ("sparse2", map_sparse2), ("onlysparse", map_onlysparse), ("notsparse", map_notsparse)], +) def test_sparsemap(tmpdir, fname, sparse_map): - def get_sparsemap_fh(fname): fh = os.open(fname, flags=os.O_RDONLY) try: @@ -98,7 +85,7 @@ def get_sparsemap_fh(fname): os.close(fh) def get_sparsemap_fd(fname): - with open(fname, 'rb') as fd: + with open(fname, "rb") as fd: return list(sparsemap(fd=fd)) fn = str(tmpdir / fname) @@ -107,30 +94,32 @@ def get_sparsemap_fd(fname): assert get_sparsemap_fd(fn) == sparse_map -@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files') -@pytest.mark.parametrize("fname, sparse_map, header_size, sparse", [ - ('sparse1', map_sparse1, 0, False), - ('sparse1', map_sparse1, 0, True), - ('sparse1', map_sparse1, BS, False), - ('sparse1', map_sparse1, BS, True), - ('sparse2', map_sparse2, 0, False), - ('sparse2', map_sparse2, 0, True), - ('sparse2', map_sparse2, BS, False), - ('sparse2', map_sparse2, BS, True), - ('onlysparse', map_onlysparse, 0, False), - ('onlysparse', map_onlysparse, 0, True), - ('onlysparse', map_onlysparse, BS, False), - ('onlysparse', map_onlysparse, BS, True), - ('notsparse', map_notsparse, 0, False), - ('notsparse', map_notsparse, 0, True), - ('notsparse', map_notsparse, BS, False), - ('notsparse', map_notsparse, BS, True), -]) +@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files") +@pytest.mark.parametrize( + "fname, sparse_map, header_size, sparse", + [ + ("sparse1", map_sparse1, 0, False), + ("sparse1", map_sparse1, 0, True), + ("sparse1", map_sparse1, BS, False), + ("sparse1", map_sparse1, BS, True), + ("sparse2", map_sparse2, 0, False), + ("sparse2", map_sparse2, 0, True), + ("sparse2", map_sparse2, BS, False), + ("sparse2", map_sparse2, BS, True), + ("onlysparse", map_onlysparse, 0, False), + ("onlysparse", map_onlysparse, 0, True), + ("onlysparse", map_onlysparse, BS, False), + ("onlysparse", map_onlysparse, BS, True), + ("notsparse", map_notsparse, 0, False), + ("notsparse", map_notsparse, 0, True), + ("notsparse", map_notsparse, BS, False), + ("notsparse", map_notsparse, BS, True), + ], +) def test_chunkify_sparse(tmpdir, fname, sparse_map, header_size, sparse): - def get_chunks(fname, sparse, header_size): chunker = ChunkerFixed(4096, header_size=header_size, sparse=sparse) - with open(fname, 'rb') as fd: + with open(fname, "rb") as fd: return cf(chunker.chunkify(fd)) fn = str(tmpdir / fname) diff --git a/src/borg/testsuite/chunker_slow.py b/src/borg/testsuite/chunker_slow.py index 4247e2730..9b5e27613 100644 --- a/src/borg/testsuite/chunker_slow.py +++ b/src/borg/testsuite/chunker_slow.py @@ -9,7 +9,6 @@ class ChunkerRegressionTestCase(BaseTestCase): - def test_chunkpoints_unchanged(self): def twist(size): x = 1 @@ -31,10 +30,10 @@ def twist(size): for seed in (1849058162, 1234567653): fh = BytesIO(data) chunker = Chunker(seed, minexp, maxexp, maskbits, winsize) - chunks = [blake2b_256(b'', c) for c in cf(chunker.chunkify(fh, -1))] - runs.append(blake2b_256(b'', b''.join(chunks))) + chunks = [blake2b_256(b"", c) for c in cf(chunker.chunkify(fh, -1))] + runs.append(blake2b_256(b"", b"".join(chunks))) # The "correct" hash below matches the existing chunker behavior. # Future chunker optimisations must not change this, or existing repos will bloat. - overall_hash = blake2b_256(b'', b''.join(runs)) + overall_hash = blake2b_256(b"", b"".join(runs)) self.assert_equal(overall_hash, unhexlify("b559b0ac8df8daaa221201d018815114241ea5c6609d98913cd2246a702af4e3")) diff --git a/src/borg/testsuite/compress.py b/src/borg/testsuite/compress.py index 4d6cdafc8..ad383ab0f 100644 --- a/src/borg/testsuite/compress.py +++ b/src/borg/testsuite/compress.py @@ -1,5 +1,6 @@ import os import zlib + try: import lzma except ImportError: @@ -11,23 +12,23 @@ buffer = bytes(2**16) -data = b'fooooooooobaaaaaaaar' * 10 -params = dict(name='zlib', level=6) +data = b"fooooooooobaaaaaaaar" * 10 +params = dict(name="zlib", level=6) def test_get_compressor(): - c = get_compressor(name='none') + c = get_compressor(name="none") assert isinstance(c, CNONE) - c = get_compressor(name='lz4') + c = get_compressor(name="lz4") assert isinstance(c, LZ4) - c = get_compressor(name='zlib') + c = get_compressor(name="zlib") assert isinstance(c, ZLIB) with pytest.raises(KeyError): - get_compressor(name='foobar') + get_compressor(name="foobar") def test_cnull(): - c = get_compressor(name='none') + c = get_compressor(name="none") cdata = c.compress(data) assert len(cdata) > len(data) assert data in cdata # it's not compressed and just in there 1:1 @@ -36,7 +37,7 @@ def test_cnull(): def test_lz4(): - c = get_compressor(name='lz4') + c = get_compressor(name="lz4") cdata = c.compress(data) assert len(cdata) < len(data) assert data == c.decompress(cdata) @@ -45,18 +46,18 @@ def test_lz4(): def test_lz4_buffer_allocation(monkeypatch): # disable fallback to no compression on incompressible data - monkeypatch.setattr(LZ4, 'decide', lambda always_compress: LZ4) + monkeypatch.setattr(LZ4, "decide", lambda always_compress: LZ4) # test with a rather huge data object to see if buffer allocation / resizing works data = os.urandom(5 * 2**20) * 10 # 50MiB badly compressible data assert len(data) == 50 * 2**20 - c = Compressor('lz4') + c = Compressor("lz4") cdata = c.compress(data) assert len(cdata) > len(data) assert data == c.decompress(cdata) def test_zlib(): - c = get_compressor(name='zlib') + c = get_compressor(name="zlib") cdata = c.compress(data) assert len(cdata) < len(data) assert data == c.decompress(cdata) @@ -66,7 +67,7 @@ def test_zlib(): def test_lzma(): if lzma is None: pytest.skip("No lzma support found.") - c = get_compressor(name='lzma') + c = get_compressor(name="lzma") cdata = c.compress(data) assert len(cdata) < len(data) assert data == c.decompress(cdata) @@ -74,7 +75,7 @@ def test_lzma(): def test_zstd(): - c = get_compressor(name='zstd') + c = get_compressor(name="zstd") cdata = c.compress(data) assert len(cdata) < len(data) assert data == c.decompress(cdata) @@ -83,16 +84,16 @@ def test_zstd(): def test_autodetect_invalid(): with pytest.raises(ValueError): - Compressor(**params).decompress(b'\xff\xfftotalcrap') + Compressor(**params).decompress(b"\xff\xfftotalcrap") with pytest.raises(ValueError): - Compressor(**params).decompress(b'\x08\x00notreallyzlib') + Compressor(**params).decompress(b"\x08\x00notreallyzlib") def test_zlib_legacy_compat(): # for compatibility reasons, we do not add an extra header for zlib, # nor do we expect one when decompressing / autodetecting for level in range(10): - c = get_compressor(name='zlib_legacy', level=level) + c = get_compressor(name="zlib_legacy", level=level) cdata1 = c.compress(data) cdata2 = zlib.compress(data, level) assert cdata1 == cdata2 @@ -104,19 +105,19 @@ def test_zlib_legacy_compat(): def test_compressor(): params_list = [ - dict(name='none'), - dict(name='lz4'), - dict(name='zstd', level=1), - dict(name='zstd', level=3), + dict(name="none"), + dict(name="lz4"), + dict(name="zstd", level=1), + dict(name="zstd", level=3), # avoiding high zstd levels, memory needs unclear - dict(name='zlib', level=0), - dict(name='zlib', level=6), - dict(name='zlib', level=9), + dict(name="zlib", level=0), + dict(name="zlib", level=6), + dict(name="zlib", level=9), ] if lzma: params_list += [ - dict(name='lzma', level=0), - dict(name='lzma', level=6), + dict(name="lzma", level=0), + dict(name="lzma", level=6), # we do not test lzma on level 9 because of the huge memory needs ] for params in params_list: @@ -125,9 +126,9 @@ def test_compressor(): def test_auto(): - compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor - compressor_lz4 = CompressionSpec('lz4').compressor - compressor_zlib = CompressionSpec('zlib,9').compressor + compressor_auto_zlib = CompressionSpec("auto,zlib,9").compressor + compressor_lz4 = CompressionSpec("lz4").compressor + compressor_zlib = CompressionSpec("zlib,9").compressor data = bytes(500) compressed_auto_zlib = compressor_auto_zlib.compress(data) compressed_lz4 = compressor_lz4.compress(data) @@ -135,13 +136,13 @@ def test_auto(): ratio = len(compressed_zlib) / len(compressed_lz4) assert Compressor.detect(compressed_auto_zlib)[0] == ZLIB if ratio < 0.99 else LZ4 - data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~' + data = b"\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~" compressed = compressor_auto_zlib.compress(data) assert Compressor.detect(compressed)[0] == CNONE def test_obfuscate(): - compressor = CompressionSpec('obfuscate,1,none').compressor + compressor = CompressionSpec("obfuscate,1,none").compressor data = bytes(10000) compressed = compressor.compress(data) # 2 id bytes compression, 2 id bytes obfuscator. 4 length bytes @@ -149,7 +150,7 @@ def test_obfuscate(): # compressing 100 times the same data should give at least 50 different result sizes assert len({len(compressor.compress(data)) for i in range(100)}) > 50 - cs = CompressionSpec('obfuscate,2,lz4') + cs = CompressionSpec("obfuscate,2,lz4") assert isinstance(cs.inner.compressor, LZ4) compressor = cs.compressor data = bytes(10000) @@ -160,7 +161,7 @@ def test_obfuscate(): # compressing 100 times the same data should give multiple different result sizes assert len({len(compressor.compress(data)) for i in range(100)}) > 10 - cs = CompressionSpec('obfuscate,6,zstd,3') + cs = CompressionSpec("obfuscate,6,zstd,3") assert isinstance(cs.inner.compressor, ZSTD) compressor = cs.compressor data = bytes(10000) @@ -171,7 +172,7 @@ def test_obfuscate(): # compressing 100 times the same data should give multiple different result sizes assert len({len(compressor.compress(data)) for i in range(100)}) > 90 - cs = CompressionSpec('obfuscate,2,auto,zstd,10') + cs = CompressionSpec("obfuscate,2,auto,zstd,10") assert isinstance(cs.inner.compressor, Auto) compressor = cs.compressor data = bytes(10000) @@ -182,7 +183,7 @@ def test_obfuscate(): # compressing 100 times the same data should give multiple different result sizes assert len({len(compressor.compress(data)) for i in range(100)}) > 10 - cs = CompressionSpec('obfuscate,110,none') + cs = CompressionSpec("obfuscate,110,none") assert isinstance(cs.inner.compressor, CNONE) compressor = cs.compressor data = bytes(1000) @@ -199,44 +200,44 @@ def test_obfuscate(): def test_compression_specs(): with pytest.raises(ValueError): - CompressionSpec('') + CompressionSpec("") - assert isinstance(CompressionSpec('none').compressor, CNONE) - assert isinstance(CompressionSpec('lz4').compressor, LZ4) + assert isinstance(CompressionSpec("none").compressor, CNONE) + assert isinstance(CompressionSpec("lz4").compressor, LZ4) - zlib = CompressionSpec('zlib').compressor + zlib = CompressionSpec("zlib").compressor assert isinstance(zlib, ZLIB) assert zlib.level == 6 - zlib = CompressionSpec('zlib,0').compressor + zlib = CompressionSpec("zlib,0").compressor assert isinstance(zlib, ZLIB) assert zlib.level == 0 - zlib = CompressionSpec('zlib,9').compressor + zlib = CompressionSpec("zlib,9").compressor assert isinstance(zlib, ZLIB) assert zlib.level == 9 with pytest.raises(ValueError): - CompressionSpec('zlib,9,invalid') + CompressionSpec("zlib,9,invalid") - lzma = CompressionSpec('lzma').compressor + lzma = CompressionSpec("lzma").compressor assert isinstance(lzma, LZMA) assert lzma.level == 6 - lzma = CompressionSpec('lzma,0').compressor + lzma = CompressionSpec("lzma,0").compressor assert isinstance(lzma, LZMA) assert lzma.level == 0 - lzma = CompressionSpec('lzma,9').compressor + lzma = CompressionSpec("lzma,9").compressor assert isinstance(lzma, LZMA) assert lzma.level == 9 - zstd = CompressionSpec('zstd').compressor + zstd = CompressionSpec("zstd").compressor assert isinstance(zstd, ZSTD) assert zstd.level == 3 - zstd = CompressionSpec('zstd,1').compressor + zstd = CompressionSpec("zstd,1").compressor assert isinstance(zstd, ZSTD) assert zstd.level == 1 - zstd = CompressionSpec('zstd,22').compressor + zstd = CompressionSpec("zstd,22").compressor assert isinstance(zstd, ZSTD) assert zstd.level == 22 with pytest.raises(ValueError): - CompressionSpec('lzma,9,invalid') + CompressionSpec("lzma,9,invalid") with pytest.raises(ValueError): - CompressionSpec('invalid') + CompressionSpec("invalid") diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index e131f63ac..78a6c8949 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -16,18 +16,17 @@ class CryptoTestCase(BaseTestCase): - def test_bytes_to_int(self): - self.assert_equal(bytes_to_int(b'\0\0\0\1'), 1) + self.assert_equal(bytes_to_int(b"\0\0\0\1"), 1) def test_bytes_to_long(self): - self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) - self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') + self.assert_equal(bytes_to_long(b"\0\0\0\0\0\0\0\1"), 1) + self.assert_equal(long_to_bytes(1), b"\0\0\0\0\0\0\0\1") def test_UNENCRYPTED(self): - iv = b'' # any IV is ok, it just must be set and not None - data = b'data' - header = b'header' + iv = b"" # any IV is ok, it just must be set and not None + data = b"data" + header = b"header" cs = UNENCRYPTED(None, None, iv, header_len=6) envelope = cs.encrypt(data, header=header) self.assert_equal(envelope, header + data) @@ -36,11 +35,11 @@ def test_UNENCRYPTED(self): def test_AES256_CTR_HMAC_SHA256(self): # this tests the layout as in attic / borg < 1.2 (1 type byte, no aad) - mac_key = b'Y' * 32 - enc_key = b'X' * 32 + mac_key = b"Y" * 32 + enc_key = b"X" * 32 iv = 0 - data = b'foo' * 10 - header = b'\x42' + data = b"foo" * 10 + header = b"\x42" # encrypt-then-mac cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=1, aad_offset=1) hdr_mac_iv_cdata = cs.encrypt(data, header=header) @@ -48,10 +47,10 @@ def test_AES256_CTR_HMAC_SHA256(self): mac = hdr_mac_iv_cdata[1:33] iv = hdr_mac_iv_cdata[33:41] cdata = hdr_mac_iv_cdata[41:] - self.assert_equal(hexlify(hdr), b'42') - self.assert_equal(hexlify(mac), b'af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8') - self.assert_equal(hexlify(iv), b'0000000000000000') - self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466') + self.assert_equal(hexlify(hdr), b"42") + self.assert_equal(hexlify(mac), b"af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8") + self.assert_equal(hexlify(iv), b"0000000000000000") + self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466") self.assert_equal(cs.next_iv(), 2) # auth-then-decrypt cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1) @@ -60,16 +59,15 @@ def test_AES256_CTR_HMAC_SHA256(self): self.assert_equal(cs.next_iv(), 2) # auth-failure due to corruption (corrupted data) cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b'\0' + hdr_mac_iv_cdata[42:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b"\0" + hdr_mac_iv_cdata[42:] + self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) def test_AES256_CTR_HMAC_SHA256_aad(self): - mac_key = b'Y' * 32 - enc_key = b'X' * 32 + mac_key = b"Y" * 32 + enc_key = b"X" * 32 iv = 0 - data = b'foo' * 10 - header = b'\x12\x34\x56' + data = b"foo" * 10 + header = b"\x12\x34\x56" # encrypt-then-mac cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=3, aad_offset=1) hdr_mac_iv_cdata = cs.encrypt(data, header=header) @@ -77,10 +75,10 @@ def test_AES256_CTR_HMAC_SHA256_aad(self): mac = hdr_mac_iv_cdata[3:35] iv = hdr_mac_iv_cdata[35:43] cdata = hdr_mac_iv_cdata[43:] - self.assert_equal(hexlify(hdr), b'123456') - self.assert_equal(hexlify(mac), b'7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138') - self.assert_equal(hexlify(iv), b'0000000000000000') - self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466') + self.assert_equal(hexlify(hdr), b"123456") + self.assert_equal(hexlify(mac), b"7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138") + self.assert_equal(hexlify(iv), b"0000000000000000") + self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466") self.assert_equal(cs.next_iv(), 2) # auth-then-decrypt cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1) @@ -89,24 +87,27 @@ def test_AES256_CTR_HMAC_SHA256_aad(self): self.assert_equal(cs.next_iv(), 2) # auth-failure due to corruption (corrupted aad) cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:] + self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) def test_AE(self): # used in legacy-like layout (1 type byte, no aad) - key = b'X' * 32 + key = b"X" * 32 iv_int = 0 - data = b'foo' * 10 - header = b'\x23' + iv_int.to_bytes(12, 'big') + data = b"foo" * 10 + header = b"\x23" + iv_int.to_bytes(12, "big") tests = [ # (ciphersuite class, exp_mac, exp_cdata) - (AES256_OCB, - b'b6909c23c9aaebd9abbe1ff42097652d', - b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ), - (CHACHA20_POLY1305, - b'fd08594796e0706cde1e8b461e3e0555', - b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', ) + ( + AES256_OCB, + b"b6909c23c9aaebd9abbe1ff42097652d", + b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493", + ), + ( + CHACHA20_POLY1305, + b"fd08594796e0706cde1e8b461e3e0555", + b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775", + ), ] for cs_cls, exp_mac, exp_cdata in tests: # print(repr(cs_cls)) @@ -117,9 +118,9 @@ def test_AE(self): iv = hdr_mac_iv_cdata[1:13] mac = hdr_mac_iv_cdata[13:29] cdata = hdr_mac_iv_cdata[29:] - self.assert_equal(hexlify(hdr), b'23') + self.assert_equal(hexlify(hdr), b"23") self.assert_equal(hexlify(mac), exp_mac) - self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(iv), b"000000000000000000000000") self.assert_equal(hexlify(cdata), exp_cdata) self.assert_equal(cs.next_iv(), 1) # auth/decrypt @@ -129,24 +130,27 @@ def test_AE(self): self.assert_equal(cs.next_iv(), 1) # auth-failure due to corruption (corrupted data) cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b"\0" + hdr_mac_iv_cdata[30:] + self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) def test_AEAD(self): # test with aad - key = b'X' * 32 + key = b"X" * 32 iv_int = 0 - data = b'foo' * 10 - header = b'\x12\x34\x56' + iv_int.to_bytes(12, 'big') + data = b"foo" * 10 + header = b"\x12\x34\x56" + iv_int.to_bytes(12, "big") tests = [ # (ciphersuite class, exp_mac, exp_cdata) - (AES256_OCB, - b'f2748c412af1c7ead81863a18c2c1893', - b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ), - (CHACHA20_POLY1305, - b'b7e7c9a79f2404e14f9aad156bf091dd', - b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', ) + ( + AES256_OCB, + b"f2748c412af1c7ead81863a18c2c1893", + b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493", + ), + ( + CHACHA20_POLY1305, + b"b7e7c9a79f2404e14f9aad156bf091dd", + b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775", + ), ] for cs_cls, exp_mac, exp_cdata in tests: # print(repr(cs_cls)) @@ -157,9 +161,9 @@ def test_AEAD(self): iv = hdr_mac_iv_cdata[3:15] mac = hdr_mac_iv_cdata[15:31] cdata = hdr_mac_iv_cdata[31:] - self.assert_equal(hexlify(hdr), b'123456') + self.assert_equal(hexlify(hdr), b"123456") self.assert_equal(hexlify(mac), exp_mac) - self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(iv), b"000000000000000000000000") self.assert_equal(hexlify(cdata), exp_cdata) self.assert_equal(cs.next_iv(), 1) # auth/decrypt @@ -169,101 +173,117 @@ def test_AEAD(self): self.assert_equal(cs.next_iv(), 1) # auth-failure due to corruption (corrupted aad) cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:] + self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) def test_AEAD_with_more_AAD(self): # test giving extra aad to the .encrypt() and .decrypt() calls - key = b'X' * 32 + key = b"X" * 32 iv_int = 0 - data = b'foo' * 10 - header = b'\x12\x34' + data = b"foo" * 10 + header = b"\x12\x34" tests = [AES256_OCB, CHACHA20_POLY1305] for cs_cls in tests: # encrypt/mac cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b'correct_chunkid') + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b"correct_chunkid") # successful auth/decrypt (correct aad) cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0) - pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b'correct_chunkid') + pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b"correct_chunkid") self.assert_equal(data, pdata) # unsuccessful auth (incorrect aad) cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0) - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b'incorrect_chunkid')) + self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b"incorrect_chunkid")) # These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/ # who claims to have verified these against independent Python and C++ implementations. def test_hkdf_hmac_sha512(self): - ikm = b'\x0b' * 22 - salt = bytes.fromhex('000102030405060708090a0b0c') - info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9') + ikm = b"\x0b" * 22 + salt = bytes.fromhex("000102030405060708090a0b0c") + info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9") l = 42 okm = hkdf_hmac_sha512(ikm, salt, info, l) - assert okm == bytes.fromhex('832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb') + assert okm == bytes.fromhex( + "832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb" + ) def test_hkdf_hmac_sha512_2(self): - ikm = bytes.fromhex('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627' - '28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f') - salt = bytes.fromhex('606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868' - '788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf') - info = bytes.fromhex('b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7' - 'd8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff') + ikm = bytes.fromhex( + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627" + "28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f" + ) + salt = bytes.fromhex( + "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868" + "788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + ) + info = bytes.fromhex( + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7" + "d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff" + ) l = 82 okm = hkdf_hmac_sha512(ikm, salt, info, l) - assert okm == bytes.fromhex('ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844' - '1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93') + assert okm == bytes.fromhex( + "ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844" + "1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93" + ) def test_hkdf_hmac_sha512_3(self): - ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b') + ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b") salt = None - info = b'' + info = b"" l = 42 okm = hkdf_hmac_sha512(ikm, salt, info, l) - assert okm == bytes.fromhex('f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac') + assert okm == bytes.fromhex( + "f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac" + ) def test_hkdf_hmac_sha512_4(self): - ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b') - salt = bytes.fromhex('000102030405060708090a0b0c') - info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9') + ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b") + salt = bytes.fromhex("000102030405060708090a0b0c") + info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9") l = 42 okm = hkdf_hmac_sha512(ikm, salt, info, l) - assert okm == bytes.fromhex('7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068') + assert okm == bytes.fromhex( + "7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068" + ) def test_hkdf_hmac_sha512_5(self): - ikm = bytes.fromhex('0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c') + ikm = bytes.fromhex("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c") salt = None - info = b'' + info = b"" l = 42 okm = hkdf_hmac_sha512(ikm, salt, info, l) - assert okm == bytes.fromhex('1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb') + assert okm == bytes.fromhex( + "1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb" + ) def test_decrypt_key_file_argon2_chacha20_poly1305(): - plain = b'hello' + plain = b"hello" # echo -n "hello, pass phrase" | argon2 saltsaltsaltsalt -id -t 1 -k 8 -p 1 -l 32 -r - key = bytes.fromhex('a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec') + key = bytes.fromhex("a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec") ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0) envelope = ae_cipher.encrypt(plain) - encrypted = msgpack.packb({ - 'version': 1, - 'salt': b'salt'*4, - 'argon2_time_cost': 1, - 'argon2_memory_cost': 8, - 'argon2_parallelism': 1, - 'argon2_type': b'id', - 'algorithm': 'argon2 chacha20-poly1305', - 'data': envelope, - }) + encrypted = msgpack.packb( + { + "version": 1, + "salt": b"salt" * 4, + "argon2_time_cost": 1, + "argon2_memory_cost": 8, + "argon2_parallelism": 1, + "argon2_type": b"id", + "algorithm": "argon2 chacha20-poly1305", + "data": envelope, + } + ) key = CHPOKeyfileKey(None) decrypted = key.decrypt_key_file(encrypted, "hello, pass phrase") @@ -272,20 +292,15 @@ def test_decrypt_key_file_argon2_chacha20_poly1305(): def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256(): - plain = b'hello' - salt = b'salt'*4 + plain = b"hello" + salt = b"salt" * 4 passphrase = "hello, pass phrase" key = FlexiKey.pbkdf2(passphrase, salt, 1, 32) hash = hmac_sha256(key, plain) - data = AES(key, b'\0'*16).encrypt(plain) - encrypted = msgpack.packb({ - 'version': 1, - 'algorithm': 'sha256', - 'iterations': 1, - 'salt': salt, - 'data': data, - 'hash': hash, - }) + data = AES(key, b"\0" * 16).encrypt(plain) + encrypted = msgpack.packb( + {"version": 1, "algorithm": "sha256", "iterations": 1, "salt": salt, "data": data, "hash": hash} + ) key = CHPOKeyfileKey(None) decrypted = key.decrypt_key_file(encrypted, passphrase) @@ -293,7 +308,7 @@ def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256(): assert decrypted == plain -@unittest.mock.patch('getpass.getpass') +@unittest.mock.patch("getpass.getpass") def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch): """https://github.com/borgbackup/borg/pull/6469#discussion_r832670411 @@ -322,10 +337,10 @@ def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch): 2. FlexiKey.detect() relies on that interface - it tries an empty passphrase before prompting the user 3. my initial implementation of decrypt_key_file_argon2() was simply passing through the IntegrityError() from AES256_CTR_BASE.decrypt() """ - repository = MagicMock(id=b'repository_id') + repository = MagicMock(id=b"repository_id") getpass.return_value = "hello, pass phrase" - monkeypatch.setenv('BORG_DISPLAY_PASSPHRASE', 'no') - AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2')) + monkeypatch.setenv("BORG_DISPLAY_PASSPHRASE", "no") + AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2")) repository.load_key.return_value = repository.save_key.call_args.args[0] AESOCBRepoKey.detect(repository, manifest_data=None) diff --git a/src/borg/testsuite/efficient_collection_queue.py b/src/borg/testsuite/efficient_collection_queue.py index e23673f5e..aab2cf9b2 100644 --- a/src/borg/testsuite/efficient_collection_queue.py +++ b/src/borg/testsuite/efficient_collection_queue.py @@ -6,13 +6,13 @@ class TestEfficientQueue: def test_base_usage(self): queue = EfficientCollectionQueue(100, bytes) - assert queue.peek_front() == b'' - queue.push_back(b'1234') - assert queue.peek_front() == b'1234' + assert queue.peek_front() == b"" + queue.push_back(b"1234") + assert queue.peek_front() == b"1234" assert len(queue) == 4 assert queue queue.pop_front(4) - assert queue.peek_front() == b'' + assert queue.peek_front() == b"" assert len(queue) == 0 assert not queue @@ -30,22 +30,22 @@ def test_usage_with_arrays(self): def test_chunking(self): queue = EfficientCollectionQueue(2, bytes) - queue.push_back(b'1') - queue.push_back(b'23') - queue.push_back(b'4567') + queue.push_back(b"1") + queue.push_back(b"23") + queue.push_back(b"4567") assert len(queue) == 7 - assert queue.peek_front() == b'12' + assert queue.peek_front() == b"12" queue.pop_front(3) - assert queue.peek_front() == b'4' + assert queue.peek_front() == b"4" queue.pop_front(1) - assert queue.peek_front() == b'56' + assert queue.peek_front() == b"56" queue.pop_front(2) assert len(queue) == 1 assert queue with pytest.raises(EfficientCollectionQueue.SizeUnderflow): queue.pop_front(2) - assert queue.peek_front() == b'7' + assert queue.peek_front() == b"7" queue.pop_front(1) - assert queue.peek_front() == b'' + assert queue.peek_front() == b"" assert len(queue) == 0 assert not queue diff --git a/src/borg/testsuite/file_integrity.py b/src/borg/testsuite/file_integrity.py index af50f15c0..8249d8e6b 100644 --- a/src/borg/testsuite/file_integrity.py +++ b/src/borg/testsuite/file_integrity.py @@ -5,34 +5,30 @@ class TestReadIntegrityFile: def test_no_integrity(self, tmpdir): - protected_file = tmpdir.join('file') - protected_file.write('1234') + protected_file = tmpdir.join("file") + protected_file.write("1234") assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None def test_truncated_integrity(self, tmpdir): - protected_file = tmpdir.join('file') - protected_file.write('1234') - tmpdir.join('file.integrity').write('') + protected_file = tmpdir.join("file") + protected_file.write("1234") + tmpdir.join("file.integrity").write("") with pytest.raises(FileIntegrityError): DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) def test_unknown_algorithm(self, tmpdir): - protected_file = tmpdir.join('file') - protected_file.write('1234') - tmpdir.join('file.integrity').write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}') + protected_file = tmpdir.join("file") + protected_file.write("1234") + tmpdir.join("file.integrity").write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}') assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None - @pytest.mark.parametrize('json', ( - '{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}', - '[]', - '1234.5', - '"A string"', - 'Invalid JSON', - )) + @pytest.mark.parametrize( + "json", ('{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}', "[]", "1234.5", '"A string"', "Invalid JSON") + ) def test_malformed(self, tmpdir, json): - protected_file = tmpdir.join('file') - protected_file.write('1234') - tmpdir.join('file.integrity').write(json) + protected_file = tmpdir.join("file") + protected_file.write("1234") + tmpdir.join("file.integrity").write(json) with pytest.raises(FileIntegrityError): DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) @@ -40,74 +36,71 @@ def test_malformed(self, tmpdir, json): class TestDetachedIntegrityCheckedFile: @pytest.fixture def integrity_protected_file(self, tmpdir): - path = str(tmpdir.join('file')) + path = str(tmpdir.join("file")) with DetachedIntegrityCheckedFile(path, write=True) as fd: - fd.write(b'foo and bar') + fd.write(b"foo and bar") return path def test_simple(self, tmpdir, integrity_protected_file): - assert tmpdir.join('file').check(file=True) - assert tmpdir.join('file.integrity').check(file=True) + assert tmpdir.join("file").check(file=True) + assert tmpdir.join("file.integrity").check(file=True) with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - assert fd.read() == b'foo and bar' + assert fd.read() == b"foo and bar" def test_corrupted_file(self, integrity_protected_file): - with open(integrity_protected_file, 'ab') as fd: - fd.write(b' extra data') + with open(integrity_protected_file, "ab") as fd: + fd.write(b" extra data") with pytest.raises(FileIntegrityError): with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - assert fd.read() == b'foo and bar extra data' + assert fd.read() == b"foo and bar extra data" def test_corrupted_file_partial_read(self, integrity_protected_file): - with open(integrity_protected_file, 'ab') as fd: - fd.write(b' extra data') + with open(integrity_protected_file, "ab") as fd: + fd.write(b" extra data") with pytest.raises(FileIntegrityError): with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - data = b'foo and bar' + data = b"foo and bar" assert fd.read(len(data)) == data - @pytest.mark.parametrize('new_name', ( - 'different_file', - 'different_file.different_ext', - )) + @pytest.mark.parametrize("new_name", ("different_file", "different_file.different_ext")) def test_renamed_file(self, tmpdir, integrity_protected_file, new_name): new_path = tmpdir.join(new_name) - tmpdir.join('file').move(new_path) - tmpdir.join('file.integrity').move(new_path + '.integrity') + tmpdir.join("file").move(new_path) + tmpdir.join("file.integrity").move(new_path + ".integrity") with pytest.raises(FileIntegrityError): with DetachedIntegrityCheckedFile(str(new_path), write=False) as fd: - assert fd.read() == b'foo and bar' + assert fd.read() == b"foo and bar" def test_moved_file(self, tmpdir, integrity_protected_file): - new_dir = tmpdir.mkdir('another_directory') - tmpdir.join('file').move(new_dir.join('file')) - tmpdir.join('file.integrity').move(new_dir.join('file.integrity')) - new_path = str(new_dir.join('file')) + new_dir = tmpdir.mkdir("another_directory") + tmpdir.join("file").move(new_dir.join("file")) + tmpdir.join("file.integrity").move(new_dir.join("file.integrity")) + new_path = str(new_dir.join("file")) with DetachedIntegrityCheckedFile(new_path, write=False) as fd: - assert fd.read() == b'foo and bar' + assert fd.read() == b"foo and bar" def test_no_integrity(self, tmpdir, integrity_protected_file): - tmpdir.join('file.integrity').remove() + tmpdir.join("file.integrity").remove() with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - assert fd.read() == b'foo and bar' + assert fd.read() == b"foo and bar" class TestDetachedIntegrityCheckedFileParts: @pytest.fixture def integrity_protected_file(self, tmpdir): - path = str(tmpdir.join('file')) + path = str(tmpdir.join("file")) with DetachedIntegrityCheckedFile(path, write=True) as fd: - fd.write(b'foo and bar') - fd.hash_part('foopart') - fd.write(b' other data') + fd.write(b"foo and bar") + fd.hash_part("foopart") + fd.write(b" other data") return path def test_simple(self, integrity_protected_file): with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - data1 = b'foo and bar' + data1 = b"foo and bar" assert fd.read(len(data1)) == data1 - fd.hash_part('foopart') - assert fd.read() == b' other data' + fd.hash_part("foopart") + assert fd.read() == b" other data" def test_wrong_part_name(self, integrity_protected_file): with pytest.raises(FileIntegrityError): @@ -115,25 +108,25 @@ def test_wrong_part_name(self, integrity_protected_file): # the failing hash_part. This is intentional: (1) it makes the code simpler (2) it's a good fail-safe # against overly broad exception handling. with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - data1 = b'foo and bar' + data1 = b"foo and bar" assert fd.read(len(data1)) == data1 with pytest.raises(FileIntegrityError): # This specific bit raises it directly - fd.hash_part('barpart') + fd.hash_part("barpart") # Still explodes in the end. - @pytest.mark.parametrize('partial_read', (False, True)) + @pytest.mark.parametrize("partial_read", (False, True)) def test_part_independence(self, integrity_protected_file, partial_read): - with open(integrity_protected_file, 'ab') as fd: - fd.write(b'some extra stuff that does not belong') + with open(integrity_protected_file, "ab") as fd: + fd.write(b"some extra stuff that does not belong") with pytest.raises(FileIntegrityError): with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd: - data1 = b'foo and bar' + data1 = b"foo and bar" try: assert fd.read(len(data1)) == data1 - fd.hash_part('foopart') + fd.hash_part("foopart") except FileIntegrityError: - assert False, 'This part must not raise, since this part is still valid.' + assert False, "This part must not raise, since this part is still valid." if not partial_read: fd.read() # But overall it explodes with the final digest. Neat, eh? diff --git a/src/borg/testsuite/hashindex.py b/src/borg/testsuite/hashindex.py index b98444c3a..ce2616dc6 100644 --- a/src/borg/testsuite/hashindex.py +++ b/src/borg/testsuite/hashindex.py @@ -15,7 +15,7 @@ def H(x): # make some 32byte long thing that depends on x - return bytes('%-0.32d' % x, 'ascii') + return bytes("%-0.32d" % x, "ascii") def H2(x): @@ -24,7 +24,6 @@ def H2(x): class HashIndexTestCase(BaseTestCase): - def _generic_test(self, cls, make_value, sha): idx = cls() self.assert_equal(len(idx), 0) @@ -57,7 +56,7 @@ def _generic_test(self, cls, make_value, sha): idx.write(filepath) del idx # Verify file contents - with open(filepath, 'rb') as fd: + with open(filepath, "rb") as fd: self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha) # Make sure we can open the file idx = cls.read(filepath) @@ -86,12 +85,14 @@ def _generic_test(self, cls, make_value, sha): del idx def test_nsindex(self): - self._generic_test(NSIndex, lambda x: (x, x, x), - '7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256') + self._generic_test( + NSIndex, lambda x: (x, x, x), "7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256" + ) def test_chunkindex(self): - self._generic_test(ChunkIndex, lambda x: (x, x), - '85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef') + self._generic_test( + ChunkIndex, lambda x: (x, x), "85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef" + ) def test_resize(self): n = 2000 # Must be >= MIN_BUCKETS @@ -218,8 +219,8 @@ def test_flags_iteritems(self): class HashIndexExtraTestCase(BaseTestCase): - """These tests are separate because they should not become part of the selftest. - """ + """These tests are separate because they should not become part of the selftest.""" + def test_chunk_indexer(self): # see _hashindex.c hash_sizes, we want to be close to the max. load # because interesting errors happen there. @@ -227,7 +228,7 @@ def test_chunk_indexer(self): index = ChunkIndex(key_count) all_keys = [hashlib.sha256(H(k)).digest() for k in range(key_count)] # we're gonna delete 1/3 of all_keys, so let's split them 2/3 and 1/3: - keys, to_delete_keys = all_keys[0:(2*key_count//3)], all_keys[(2*key_count//3):] + keys, to_delete_keys = all_keys[0 : (2 * key_count // 3)], all_keys[(2 * key_count // 3) :] for i, key in enumerate(keys): index[key] = (i, i) @@ -286,6 +287,7 @@ def merge(refcount1, refcount2): idx1.merge(idx2) refcount, *_ = idx1[H(1)] return refcount + result = merge(refcounta, refcountb) # check for commutativity assert result == merge(refcountb, refcounta) @@ -367,22 +369,24 @@ def test_keyerror(self): class HashIndexDataTestCase(BaseTestCase): # This bytestring was created with borg2-pre 2022-06-10 - HASHINDEX = b'eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4' \ - b'juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4' \ - b'juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO' \ - b'4ziO4ziO4ziO4ziO4ziO437LHbSVHGw=' + HASHINDEX = ( + b"eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4" + b"juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4" + b"juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO" + b"4ziO4ziO4ziO4ziO4ziO437LHbSVHGw=" + ) def _serialize_hashindex(self, idx): with tempfile.TemporaryDirectory() as tempdir: - file = os.path.join(tempdir, 'idx') + file = os.path.join(tempdir, "idx") idx.write(file) - with open(file, 'rb') as f: + with open(file, "rb") as f: return self._pack(f.read()) def _deserialize_hashindex(self, bytestring): with tempfile.TemporaryDirectory() as tempdir: - file = os.path.join(tempdir, 'idx') - with open(file, 'wb') as f: + file = os.path.join(tempdir, "idx") + with open(file, "wb") as f: f.write(self._unpack(bytestring)) return ChunkIndex.read(file) @@ -416,19 +420,19 @@ def test_read_known_good(self): class HashIndexIntegrityTestCase(HashIndexDataTestCase): def write_integrity_checked_index(self, tempdir): idx = self._deserialize_hashindex(self.HASHINDEX) - file = os.path.join(tempdir, 'idx') + file = os.path.join(tempdir, "idx") with IntegrityCheckedFile(path=file, write=True) as fd: idx.write(fd) integrity_data = fd.integrity_data - assert 'final' in integrity_data - assert 'HashHeader' in integrity_data + assert "final" in integrity_data + assert "HashHeader" in integrity_data return file, integrity_data def test_integrity_checked_file(self): with tempfile.TemporaryDirectory() as tempdir: file, integrity_data = self.write_integrity_checked_index(tempdir) - with open(file, 'r+b') as fd: - fd.write(b'Foo') + with open(file, "r+b") as fd: + fd.write(b"Foo") with self.assert_raises(FileIntegrityError): with IntegrityCheckedFile(path=file, write=False, integrity_data=integrity_data) as fd: ChunkIndex.read(fd) @@ -437,15 +441,15 @@ def test_integrity_checked_file(self): class HashIndexCompactTestCase(HashIndexDataTestCase): def index(self, num_entries, num_buckets): index_data = io.BytesIO() - index_data.write(b'BORG_IDX') + index_data.write(b"BORG_IDX") # num_entries - index_data.write(num_entries.to_bytes(4, 'little')) + index_data.write(num_entries.to_bytes(4, "little")) # num_buckets - index_data.write(num_buckets.to_bytes(4, 'little')) + index_data.write(num_buckets.to_bytes(4, "little")) # key_size - index_data.write((32).to_bytes(1, 'little')) + index_data.write((32).to_bytes(1, "little")) # value_size - index_data.write((3 * 4).to_bytes(1, 'little')) + index_data.write((3 * 4).to_bytes(1, "little")) self.index_data = index_data @@ -468,13 +472,13 @@ def index_from_data_compact_to_data(self): def write_entry(self, key, *values): self.index_data.write(key) for value in values: - self.index_data.write(value.to_bytes(4, 'little')) + self.index_data.write(value.to_bytes(4, "little")) def write_empty(self, key): - self.write_entry(key, 0xffffffff, 0, 0) + self.write_entry(key, 0xFFFFFFFF, 0, 0) def write_deleted(self, key): - self.write_entry(key, 0xfffffffe, 0, 0) + self.write_entry(key, 0xFFFFFFFE, 0, 0) def test_simple(self): self.index(num_entries=3, num_buckets=6) @@ -600,7 +604,7 @@ def HH(x, y, z): # first 4 bytes. giving a specific x targets bucket index x. # y is to create different keys and does not go into the bucket index calculation. # so, same x + different y --> collision - return pack(' datetime(2038, 1, 1) assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) else: # ns fit into int64 - assert safe_ns(2 ** 64) <= 2 ** 63 - 1 + assert safe_ns(2**64) <= 2**63 - 1 assert safe_ns(-1) == 0 # s are so that their ns conversion fits into int64 - assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1 + assert safe_s(2**64) * 1000000000 <= 2**63 - 1 assert safe_s(-1) == 0 # datetime won't fall over its y10k problem - beyond_y10k = 2 ** 100 + beyond_y10k = 2**100 with pytest.raises(OverflowError): datetime.utcfromtimestamp(beyond_y10k) assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) @@ -926,66 +983,64 @@ def test_safe_timestamps(): class TestPopenWithErrorHandling: - @pytest.mark.skipif(not shutil.which('test'), reason='"test" binary is needed') + @pytest.mark.skipif(not shutil.which("test"), reason='"test" binary is needed') def test_simple(self): - proc = popen_with_error_handling('test 1') + proc = popen_with_error_handling("test 1") assert proc.wait() == 0 - @pytest.mark.skipif(shutil.which('borg-foobar-test-notexist'), reason='"borg-foobar-test-notexist" binary exists (somehow?)') + @pytest.mark.skipif( + shutil.which("borg-foobar-test-notexist"), reason='"borg-foobar-test-notexist" binary exists (somehow?)' + ) def test_not_found(self): - proc = popen_with_error_handling('borg-foobar-test-notexist 1234') + proc = popen_with_error_handling("borg-foobar-test-notexist 1234") assert proc is None - @pytest.mark.parametrize('cmd', ( - 'mismatched "quote', - 'foo --bar="baz', - '' - )) + @pytest.mark.parametrize("cmd", ('mismatched "quote', 'foo --bar="baz', "")) def test_bad_syntax(self, cmd): proc = popen_with_error_handling(cmd) assert proc is None def test_shell(self): with pytest.raises(AssertionError): - popen_with_error_handling('', shell=True) + popen_with_error_handling("", shell=True) def test_dash_open(): - assert dash_open('-', 'r') is sys.stdin - assert dash_open('-', 'w') is sys.stdout - assert dash_open('-', 'rb') is sys.stdin.buffer - assert dash_open('-', 'wb') is sys.stdout.buffer + assert dash_open("-", "r") is sys.stdin + assert dash_open("-", "w") is sys.stdout + assert dash_open("-", "rb") is sys.stdin.buffer + assert dash_open("-", "wb") is sys.stdout.buffer def test_iter_separated(): # newline and utf-8 - sep, items = '\n', ['foo', 'bar/baz', 'αáčő'] + sep, items = "\n", ["foo", "bar/baz", "αáčő"] fd = StringIO(sep.join(items)) assert list(iter_separated(fd)) == items # null and bogus ending - sep, items = '\0', ['foo/bar', 'baz', 'spam'] - fd = StringIO(sep.join(items) + '\0') - assert list(iter_separated(fd, sep=sep)) == ['foo/bar', 'baz', 'spam'] + sep, items = "\0", ["foo/bar", "baz", "spam"] + fd = StringIO(sep.join(items) + "\0") + assert list(iter_separated(fd, sep=sep)) == ["foo/bar", "baz", "spam"] # multichar - sep, items = 'SEP', ['foo/bar', 'baz', 'spam'] + sep, items = "SEP", ["foo/bar", "baz", "spam"] fd = StringIO(sep.join(items)) assert list(iter_separated(fd, sep=sep)) == items # bytes - sep, items = b'\n', [b'foo', b'blop\t', b'gr\xe4ezi'] + sep, items = b"\n", [b"foo", b"blop\t", b"gr\xe4ezi"] fd = BytesIO(sep.join(items)) assert list(iter_separated(fd)) == items def test_eval_escapes(): - assert eval_escapes('\\n\\0\\x23') == '\n\0#' - assert eval_escapes('äç\\n') == 'äç\n' + assert eval_escapes("\\n\\0\\x23") == "\n\0#" + assert eval_escapes("äç\\n") == "äç\n" def test_safe_unlink_is_safe(tmpdir): contents = b"Hello, world\n" - victim = tmpdir / 'victim' + victim = tmpdir / "victim" victim.write_binary(contents) - hard_link = tmpdir / 'hardlink' + hard_link = tmpdir / "hardlink" hard_link.mklinkto(victim) safe_unlink(hard_link) @@ -995,13 +1050,14 @@ def test_safe_unlink_is_safe(tmpdir): def test_safe_unlink_is_safe_ENOSPC(tmpdir, monkeypatch): contents = b"Hello, world\n" - victim = tmpdir / 'victim' + victim = tmpdir / "victim" victim.write_binary(contents) - hard_link = tmpdir / 'hardlink' + hard_link = tmpdir / "hardlink" hard_link.mklinkto(victim) def os_unlink(_): raise OSError(errno.ENOSPC, "Pretend that we ran out of space") + monkeypatch.setattr(os, "unlink", os_unlink) with pytest.raises(OSError): @@ -1012,38 +1068,38 @@ def os_unlink(_): class TestPassphrase: def test_passphrase_new_verification(self, capsys, monkeypatch): - monkeypatch.setattr(getpass, 'getpass', lambda prompt: "12aöäü") - monkeypatch.setenv('BORG_DISPLAY_PASSPHRASE', 'no') + monkeypatch.setattr(getpass, "getpass", lambda prompt: "12aöäü") + monkeypatch.setenv("BORG_DISPLAY_PASSPHRASE", "no") Passphrase.new() out, err = capsys.readouterr() assert "12" not in out assert "12" not in err - monkeypatch.setenv('BORG_DISPLAY_PASSPHRASE', 'yes') + monkeypatch.setenv("BORG_DISPLAY_PASSPHRASE", "yes") passphrase = Passphrase.new() out, err = capsys.readouterr() assert "313261c3b6c3a4c3bc" not in out assert "313261c3b6c3a4c3bc" in err assert passphrase == "12aöäü" - monkeypatch.setattr(getpass, 'getpass', lambda prompt: "1234/@=") + monkeypatch.setattr(getpass, "getpass", lambda prompt: "1234/@=") Passphrase.new() out, err = capsys.readouterr() assert "1234/@=" not in out assert "1234/@=" in err def test_passphrase_new_empty(self, capsys, monkeypatch): - monkeypatch.delenv('BORG_PASSPHRASE', False) - monkeypatch.setattr(getpass, 'getpass', lambda prompt: "") + monkeypatch.delenv("BORG_PASSPHRASE", False) + monkeypatch.setattr(getpass, "getpass", lambda prompt: "") with pytest.raises(PasswordRetriesExceeded): Passphrase.new(allow_empty=False) out, err = capsys.readouterr() assert "must not be blank" in err def test_passphrase_new_retries(self, monkeypatch): - monkeypatch.delenv('BORG_PASSPHRASE', False) + monkeypatch.delenv("BORG_PASSPHRASE", False) ascending_numbers = iter(range(20)) - monkeypatch.setattr(getpass, 'getpass', lambda prompt: str(next(ascending_numbers))) + monkeypatch.setattr(getpass, "getpass", lambda prompt: str(next(ascending_numbers))) with pytest.raises(PasswordRetriesExceeded): Passphrase.new() diff --git a/src/borg/testsuite/item.py b/src/borg/testsuite/item.py index 47f8e1021..e7cb9b2ac 100644 --- a/src/borg/testsuite/item.py +++ b/src/borg/testsuite/item.py @@ -11,20 +11,20 @@ def test_item_empty(): assert item.as_dict() == {} - assert 'path' not in item + assert "path" not in item with pytest.raises(ValueError): - 'invalid-key' in item + "invalid-key" in item with pytest.raises(TypeError): - b'path' in item + b"path" in item with pytest.raises(TypeError): 42 in item - assert item.get('mode') is None - assert item.get('mode', 0o666) == 0o666 + assert item.get("mode") is None + assert item.get("mode", 0o666) == 0o666 with pytest.raises(ValueError): - item.get('invalid-key') + item.get("invalid-key") with pytest.raises(TypeError): - item.get(b'mode') + item.get(b"mode") with pytest.raises(TypeError): item.get(42) @@ -37,16 +37,16 @@ def test_item_empty(): def test_item_from_dict(): # does not matter whether we get str or bytes keys - item = Item({b'path': '/a/b/c', b'mode': 0o666}) - assert item.path == '/a/b/c' + item = Item({b"path": "/a/b/c", b"mode": 0o666}) + assert item.path == "/a/b/c" assert item.mode == 0o666 - assert 'path' in item + assert "path" in item # does not matter whether we get str or bytes keys - item = Item({'path': '/a/b/c', 'mode': 0o666}) - assert item.path == '/a/b/c' + item = Item({"path": "/a/b/c", "mode": 0o666}) + assert item.path == "/a/b/c" assert item.mode == 0o666 - assert 'mode' in item + assert "mode" in item # invalid - no dict with pytest.raises(TypeError): @@ -58,12 +58,12 @@ def test_item_from_dict(): # invalid - unknown key with pytest.raises(ValueError): - Item({'foobar': 'baz'}) + Item({"foobar": "baz"}) def test_item_from_kw(): - item = Item(path='/a/b/c', mode=0o666) - assert item.path == '/a/b/c' + item = Item(path="/a/b/c", mode=0o666) + assert item.path == "/a/b/c" assert item.mode == 0o666 @@ -71,7 +71,7 @@ def test_item_int_property(): item = Item() item.mode = 0o666 assert item.mode == 0o666 - assert item.as_dict() == {'mode': 0o666} + assert item.as_dict() == {"mode": 0o666} del item.mode assert item.as_dict() == {} with pytest.raises(TypeError): @@ -80,34 +80,34 @@ def test_item_int_property(): def test_item_mptimestamp_property(): item = Item() - small, big = 42, 2 ** 65 + small, big = 42, 2**65 item.atime = small assert item.atime == small - assert item.as_dict() == {'atime': Timestamp.from_unix_nano(small)} + assert item.as_dict() == {"atime": Timestamp.from_unix_nano(small)} item.atime = big assert item.atime == big - assert item.as_dict() == {'atime': Timestamp.from_unix_nano(big)} + assert item.as_dict() == {"atime": Timestamp.from_unix_nano(big)} def test_item_se_str_property(): # start simple item = Item() - item.path = '/a/b/c' - assert item.path == '/a/b/c' - assert item.as_dict() == {'path': '/a/b/c'} + item.path = "/a/b/c" + assert item.path == "/a/b/c" + assert item.as_dict() == {"path": "/a/b/c"} del item.path assert item.as_dict() == {} with pytest.raises(TypeError): item.path = 42 # non-utf-8 path, needing surrogate-escaping for latin-1 u-umlaut - item = Item(internal_dict={'path': b'/a/\xfc/c'}) - assert item.path == '/a/\udcfc/c' # getting a surrogate-escaped representation - assert item.as_dict() == {'path': '/a/\udcfc/c'} + item = Item(internal_dict={"path": b"/a/\xfc/c"}) + assert item.path == "/a/\udcfc/c" # getting a surrogate-escaped representation + assert item.as_dict() == {"path": "/a/\udcfc/c"} del item.path - assert 'path' not in item - item.path = '/a/\udcfc/c' # setting using a surrogate-escaped representation - assert item.as_dict() == {'path': '/a/\udcfc/c'} + assert "path" not in item + item.path = "/a/\udcfc/c" # setting using a surrogate-escaped representation + assert item.as_dict() == {"path": "/a/\udcfc/c"} def test_item_list_property(): @@ -118,18 +118,18 @@ def test_item_list_property(): assert item.chunks == [0] item.chunks.append(1) assert item.chunks == [0, 1] - assert item.as_dict() == {'chunks': [0, 1]} + assert item.as_dict() == {"chunks": [0, 1]} def test_item_dict_property(): item = Item() item.xattrs = StableDict() assert item.xattrs == StableDict() - item.xattrs['foo'] = 'bar' - assert item.xattrs['foo'] == 'bar' - item.xattrs['bar'] = 'baz' - assert item.xattrs == StableDict({'foo': 'bar', 'bar': 'baz'}) - assert item.as_dict() == {'xattrs': {'foo': 'bar', 'bar': 'baz'}} + item.xattrs["foo"] = "bar" + assert item.xattrs["foo"] == "bar" + item.xattrs["bar"] = "baz" + assert item.xattrs == StableDict({"foo": "bar", "bar": "baz"}) + assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}} def test_unknown_property(): @@ -142,10 +142,7 @@ def test_unknown_property(): def test_item_file_size(): - item = Item(mode=0o100666, chunks=[ - ChunkListEntry(size=1000, id=None), - ChunkListEntry(size=2000, id=None), - ]) + item = Item(mode=0o100666, chunks=[ChunkListEntry(size=1000, id=None), ChunkListEntry(size=2000, id=None)]) assert item.get_size() == 3000 item.get_size(memorize=True) assert item.size == 3000 diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 2b69e73b0..512649232 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -11,7 +11,13 @@ from ..crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey from ..crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256 -from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError, UnsupportedKeyFormatError +from ..crypto.key import ( + TAMRequiredError, + TAMInvalid, + TAMUnsupportedSuiteError, + UnsupportedManifestError, + UnsupportedKeyFormatError, +) from ..crypto.key import identify_key from ..crypto.low_level import IntegrityError as IntegrityErrorBase from ..helpers import IntegrityError @@ -36,11 +42,17 @@ class MockArgs: F84MsMMiqpbz4KVICeBZhfAaTPs4W7BC63qml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgLENQ 2uVCoR7EnAoiRzn8J+orbojKtJlNCnQ31SSC8rendmVyc2lvbgE=""".strip() - keyfile2_cdata = unhexlify(re.sub(r'\W', '', """ + keyfile2_cdata = unhexlify( + re.sub( + r"\W", + "", + """ 0055f161493fcfc16276e8c31493c4641e1eb19a79d0326fad0291e5a9c98e5933 00000000000003e8d21eaf9b86c297a8cd56432e1915bb - """)) - keyfile2_id = unhexlify('c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314') + """, + ) + ) + keyfile2_id = unhexlify("c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314") keyfile_blake2_key_file = """ BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000 @@ -56,8 +68,9 @@ class MockArgs: UTHFJg343jqml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgz3YaUZZ/s+UWywj97EY5b4KhtJYi qkPqtDDxs2j/T7+ndmVyc2lvbgE=""".strip() - keyfile_blake2_cdata = bytes.fromhex('04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448' - '0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947') + keyfile_blake2_cdata = bytes.fromhex( + "04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448" "0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947" + ) # Verified against b2sum. Entire string passed to BLAKE2, including the padded 64 byte key contained in # keyfile_blake2_key_file above is # 19280471de95185ec27ecb6fc9edbb4f4db26974c315ede1cd505fab4250ce7cd0d081ea66946c @@ -65,33 +78,42 @@ class MockArgs: # 000000000000000000000000000000000000000000000000000000000000000000000000000000 # 00000000000000000000007061796c6f6164 # p a y l o a d - keyfile_blake2_id = bytes.fromhex('d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb') + keyfile_blake2_id = bytes.fromhex("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb") @pytest.fixture def keys_dir(self, request, monkeypatch, tmpdir): - monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir)) + monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir)) return tmpdir - @pytest.fixture(params=( - # not encrypted - PlaintextKey, - AuthenticatedKey, Blake2AuthenticatedKey, - # legacy crypto - KeyfileKey, Blake2KeyfileKey, - RepoKey, Blake2RepoKey, - # new crypto - AESOCBKeyfileKey, AESOCBRepoKey, - Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey, - CHPOKeyfileKey, CHPORepoKey, - Blake2CHPOKeyfileKey, Blake2CHPORepoKey, - )) + @pytest.fixture( + params=( + # not encrypted + PlaintextKey, + AuthenticatedKey, + Blake2AuthenticatedKey, + # legacy crypto + KeyfileKey, + Blake2KeyfileKey, + RepoKey, + Blake2RepoKey, + # new crypto + AESOCBKeyfileKey, + AESOCBRepoKey, + Blake2AESOCBKeyfileKey, + Blake2AESOCBRepoKey, + CHPOKeyfileKey, + CHPORepoKey, + Blake2CHPOKeyfileKey, + Blake2CHPORepoKey, + ) + ) def key(self, request, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") return request.param.create(self.MockRepository(), self.MockArgs()) class MockRepository: class _Location: - raw = processed = '/some/place' + raw = processed = "/some/place" def canonical_path(self): return self.processed @@ -114,16 +136,16 @@ def load_key(self): def test_plaintext(self): key = PlaintextKey.create(None, None) - chunk = b'foo' + chunk = b"foo" id = key.id_hash(chunk) - assert hexlify(id) == b'2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae' + assert hexlify(id) == b"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" assert chunk == key.decrypt(id, key.encrypt(id, chunk)) def test_keyfile(self, monkeypatch, keys_dir): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") key = KeyfileKey.create(self.MockRepository(), self.MockArgs()) assert key.cipher.next_iv() == 0 - chunk = b'ABC' + chunk = b"ABC" id = key.id_hash(chunk) manifest = key.encrypt(id, chunk) assert key.cipher.extract_iv(manifest) == 0 @@ -137,18 +159,18 @@ def test_keyfile(self, monkeypatch, keys_dir): # Key data sanity check assert len({key2.id_key, key2.enc_key, key2.enc_hmac_key}) == 3 assert key2.chunk_seed != 0 - chunk = b'foo' + chunk = b"foo" id = key.id_hash(chunk) assert chunk == key2.decrypt(id, key.encrypt(id, chunk)) def test_keyfile_kfenv(self, tmpdir, monkeypatch): - keyfile = tmpdir.join('keyfile') - monkeypatch.setenv('BORG_KEY_FILE', str(keyfile)) - monkeypatch.setenv('BORG_PASSPHRASE', 'testkf') + keyfile = tmpdir.join("keyfile") + monkeypatch.setenv("BORG_KEY_FILE", str(keyfile)) + monkeypatch.setenv("BORG_PASSPHRASE", "testkf") assert not keyfile.exists() key = CHPOKeyfileKey.create(self.MockRepository(), self.MockArgs()) assert keyfile.exists() - chunk = b'ABC' + chunk = b"ABC" chunk_id = key.id_hash(chunk) chunk_cdata = key.encrypt(chunk_id, chunk) key = CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata) @@ -158,27 +180,27 @@ def test_keyfile_kfenv(self, tmpdir, monkeypatch): CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata) def test_keyfile2(self, monkeypatch, keys_dir): - with keys_dir.join('keyfile').open('w') as fd: + with keys_dir.join("keyfile").open("w") as fd: fd.write(self.keyfile2_key_file) - monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase') + monkeypatch.setenv("BORG_PASSPHRASE", "passphrase") key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) - assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload' + assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload" def test_keyfile2_kfenv(self, tmpdir, monkeypatch): - keyfile = tmpdir.join('keyfile') - with keyfile.open('w') as fd: + keyfile = tmpdir.join("keyfile") + with keyfile.open("w") as fd: fd.write(self.keyfile2_key_file) - monkeypatch.setenv('BORG_KEY_FILE', str(keyfile)) - monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase') + monkeypatch.setenv("BORG_KEY_FILE", str(keyfile)) + monkeypatch.setenv("BORG_PASSPHRASE", "passphrase") key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) - assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload' + assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload" def test_keyfile_blake2(self, monkeypatch, keys_dir): - with keys_dir.join('keyfile').open('w') as fd: + with keys_dir.join("keyfile").open("w") as fd: fd.write(self.keyfile_blake2_key_file) - monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase') + monkeypatch.setenv("BORG_PASSPHRASE", "passphrase") key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata) - assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b'payload' + assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b"payload" def _corrupt_byte(self, key, data, offset): data = bytearray(data) @@ -186,12 +208,12 @@ def _corrupt_byte(self, key, data, offset): # will trigger an IntegrityError (does not happen while we stay within TYPES_ACCEPTABLE). data[offset] ^= 64 with pytest.raises(IntegrityErrorBase): - key.decrypt(b'', data) + key.decrypt(b"", data) def test_decrypt_integrity(self, monkeypatch, keys_dir): - with keys_dir.join('keyfile').open('w') as fd: + with keys_dir.join("keyfile").open("w") as fd: fd.write(self.keyfile2_key_file) - monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase') + monkeypatch.setenv("BORG_PASSPHRASE", "passphrase") key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) data = self.keyfile2_cdata @@ -206,7 +228,7 @@ def test_decrypt_integrity(self, monkeypatch, keys_dir): def test_roundtrip(self, key): repository = key.repository - plaintext = b'foo' + plaintext = b"foo" id = key.id_hash(plaintext) encrypted = key.encrypt(id, plaintext) identified_key_class = identify_key(encrypted) @@ -216,59 +238,59 @@ def test_roundtrip(self, key): assert decrypted == plaintext def test_decrypt_decompress(self, key): - plaintext = b'123456789' + plaintext = b"123456789" id = key.id_hash(plaintext) encrypted = key.encrypt(id, plaintext) assert key.decrypt(id, encrypted, decompress=False) != plaintext assert key.decrypt(id, encrypted) == plaintext def test_assert_id(self, key): - plaintext = b'123456789' + plaintext = b"123456789" id = key.id_hash(plaintext) key.assert_id(id, plaintext) id_changed = bytearray(id) id_changed[0] ^= 1 with pytest.raises(IntegrityError): key.assert_id(id_changed, plaintext) - plaintext_changed = plaintext + b'1' + plaintext_changed = plaintext + b"1" with pytest.raises(IntegrityError): key.assert_id(id, plaintext_changed) def test_authenticated_encrypt(self, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") key = AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) assert AuthenticatedKey.id_hash is ID_HMAC_SHA_256.id_hash assert len(key.id_key) == 32 - plaintext = b'123456789' + plaintext = b"123456789" id = key.id_hash(plaintext) authenticated = key.encrypt(id, plaintext) # 0x07 is the key TYPE, \x00ff identifies no compression / unknown level. - assert authenticated == b'\x07\x00\xff' + plaintext + assert authenticated == b"\x07\x00\xff" + plaintext def test_blake2_authenticated_encrypt(self, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash assert len(key.id_key) == 128 - plaintext = b'123456789' + plaintext = b"123456789" id = key.id_hash(plaintext) authenticated = key.encrypt(id, plaintext) # 0x06 is the key TYPE, 0x00ff identifies no compression / unknown level. - assert authenticated == b'\x06\x00\xff' + plaintext + assert authenticated == b"\x06\x00\xff" + plaintext class TestTAM: @pytest.fixture def key(self, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") return CHPOKeyfileKey.create(TestKey.MockRepository(), TestKey.MockArgs()) def test_unpack_future(self, key): - blob = b'\xc1\xc1\xc1\xc1foobar' + blob = b"\xc1\xc1\xc1\xc1foobar" with pytest.raises(UnsupportedManifestError): key.unpack_and_verify_manifest(blob) - blob = b'\xc1\xc1\xc1' + blob = b"\xc1\xc1\xc1" with pytest.raises(msgpack.UnpackException): key.unpack_and_verify_manifest(blob) @@ -285,84 +307,66 @@ def test_missing(self, key): assert not verified def test_unknown_type_when_required(self, key): - blob = msgpack.packb({ - 'tam': { - 'type': 'HMAC_VOLLBIT', - }, - }) + blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}}) with pytest.raises(TAMUnsupportedSuiteError): key.unpack_and_verify_manifest(blob) def test_unknown_type(self, key): - blob = msgpack.packb({ - 'tam': { - 'type': 'HMAC_VOLLBIT', - }, - }) + blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}}) key.tam_required = False unpacked, verified = key.unpack_and_verify_manifest(blob) assert unpacked == {} assert not verified - @pytest.mark.parametrize('tam, exc', ( - ({}, TAMUnsupportedSuiteError), - ({'type': b'\xff'}, TAMUnsupportedSuiteError), - (None, TAMInvalid), - (1234, TAMInvalid), - )) + @pytest.mark.parametrize( + "tam, exc", + ( + ({}, TAMUnsupportedSuiteError), + ({"type": b"\xff"}, TAMUnsupportedSuiteError), + (None, TAMInvalid), + (1234, TAMInvalid), + ), + ) def test_invalid(self, key, tam, exc): - blob = msgpack.packb({ - 'tam': tam, - }) + blob = msgpack.packb({"tam": tam}) with pytest.raises(exc): key.unpack_and_verify_manifest(blob) - @pytest.mark.parametrize('hmac, salt', ( - ({}, bytes(64)), - (bytes(64), {}), - (None, bytes(64)), - (bytes(64), None), - )) + @pytest.mark.parametrize("hmac, salt", (({}, bytes(64)), (bytes(64), {}), (None, bytes(64)), (bytes(64), None))) def test_wrong_types(self, key, hmac, salt): - data = { - 'tam': { - 'type': 'HKDF_HMAC_SHA512', - 'hmac': hmac, - 'salt': salt - }, - } - tam = data['tam'] + data = {"tam": {"type": "HKDF_HMAC_SHA512", "hmac": hmac, "salt": salt}} + tam = data["tam"] if hmac is None: - del tam['hmac'] + del tam["hmac"] if salt is None: - del tam['salt'] + del tam["salt"] blob = msgpack.packb(data) with pytest.raises(TAMInvalid): key.unpack_and_verify_manifest(blob) def test_round_trip(self, key): - data = {'foo': 'bar'} + data = {"foo": "bar"} blob = key.pack_and_authenticate_metadata(data) - assert blob.startswith(b'\x82') + assert blob.startswith(b"\x82") unpacked = msgpack.unpackb(blob) - assert unpacked['tam']['type'] == 'HKDF_HMAC_SHA512' + assert unpacked["tam"]["type"] == "HKDF_HMAC_SHA512" unpacked, verified = key.unpack_and_verify_manifest(blob) assert verified - assert unpacked['foo'] == 'bar' - assert 'tam' not in unpacked + assert unpacked["foo"] == "bar" + assert "tam" not in unpacked - @pytest.mark.parametrize('which', ('hmac', 'salt')) + @pytest.mark.parametrize("which", ("hmac", "salt")) def test_tampered(self, key, which): - data = {'foo': 'bar'} + data = {"foo": "bar"} blob = key.pack_and_authenticate_metadata(data) - assert blob.startswith(b'\x82') + assert blob.startswith(b"\x82") unpacked = msgpack.unpackb(blob, object_hook=StableDict) - assert len(unpacked['tam'][which]) == 64 - unpacked['tam'][which] = unpacked['tam'][which][0:32] + bytes(32) - assert len(unpacked['tam'][which]) == 64 + assert len(unpacked["tam"][which]) == 64 + unpacked["tam"][which] = unpacked["tam"][which][0:32] + bytes(32) + assert len(unpacked["tam"][which]) == 64 blob = msgpack.packb(unpacked) with pytest.raises(TAMInvalid): @@ -372,10 +376,7 @@ def test_tampered(self, key, which): def test_decrypt_key_file_unsupported_algorithm(): """We will add more algorithms in the future. We should raise a helpful error.""" key = CHPOKeyfileKey(None) - encrypted = msgpack.packb({ - 'algorithm': 'THIS ALGORITHM IS NOT SUPPORTED', - 'version': 1, - }) + encrypted = msgpack.packb({"algorithm": "THIS ALGORITHM IS NOT SUPPORTED", "version": 1}) with pytest.raises(UnsupportedKeyFormatError): key.decrypt_key_file(encrypted, "hello, pass phrase") @@ -384,9 +385,7 @@ def test_decrypt_key_file_unsupported_algorithm(): def test_decrypt_key_file_v2_is_unsupported(): """There may eventually be a version 2 of the format. For now we should raise a helpful error.""" key = CHPOKeyfileKey(None) - encrypted = msgpack.packb({ - 'version': 2, - }) + encrypted = msgpack.packb({"version": 2}) with pytest.raises(UnsupportedKeyFormatError): key.decrypt_key_file(encrypted, "hello, pass phrase") @@ -394,16 +393,16 @@ def test_decrypt_key_file_v2_is_unsupported(): def test_key_file_roundtrip(monkeypatch): def to_dict(key): - extract = 'repository_id', 'enc_key', 'enc_hmac_key', 'id_key', 'chunk_seed' + extract = "repository_id", "enc_key", "enc_hmac_key", "id_key", "chunk_seed" return {a: getattr(key, a) for a in extract} - repository = MagicMock(id=b'repository_id') - monkeypatch.setenv('BORG_PASSPHRASE', "hello, pass phrase") + repository = MagicMock(id=b"repository_id") + monkeypatch.setenv("BORG_PASSPHRASE", "hello, pass phrase") - save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2')) + save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2")) saved = repository.save_key.call_args.args[0] repository.load_key.return_value = saved load_me = AESOCBRepoKey.detect(repository, manifest_data=None) assert to_dict(load_me) == to_dict(save_me) - assert msgpack.unpackb(a2b_base64(saved))['algorithm'] == KEY_ALGORITHMS['argon2'] + assert msgpack.unpackb(a2b_base64(saved))["algorithm"] == KEY_ALGORITHMS["argon2"] diff --git a/src/borg/testsuite/locking.py b/src/borg/testsuite/locking.py index 656909ac5..69a47bb58 100644 --- a/src/borg/testsuite/locking.py +++ b/src/borg/testsuite/locking.py @@ -6,8 +6,19 @@ import pytest from ..platform import get_process_id, process_alive -from ..locking import TimeoutTimer, ExclusiveLock, Lock, LockRoster, \ - ADD, REMOVE, SHARED, EXCLUSIVE, LockTimeout, NotLocked, NotMyLock +from ..locking import ( + TimeoutTimer, + ExclusiveLock, + Lock, + LockRoster, + ADD, + REMOVE, + SHARED, + EXCLUSIVE, + LockTimeout, + NotLocked, + NotMyLock, +) ID1 = "foo", 1, 1 ID2 = "bar", 2, 2 @@ -45,7 +56,7 @@ def test_notimeout_sleep(self): @pytest.fixture() def lockpath(tmpdir): - return str(tmpdir.join('lock')) + return str(tmpdir.join("lock")) class TestExclusiveLock: @@ -67,7 +78,7 @@ def test_timeout(self, lockpath): def test_kill_stale(self, lockpath, free_pid): host, pid, tid = our_id = get_process_id() dead_id = host, free_pid, tid - cant_know_if_dead_id = 'foo.bar.example.net', 1, 2 + cant_know_if_dead_id = "foo.bar.example.net", 1, 2 dead_lock = ExclusiveLock(lockpath, id=dead_id).acquire() with ExclusiveLock(lockpath, id=our_id): @@ -94,9 +105,7 @@ def test_migrate_lock(self, lockpath): assert old_unique_name != new_unique_name # locking filename is different now def test_race_condition(self, lockpath): - class SynchronizedCounter: - def __init__(self, count=0): self.lock = ThreadingLock() self.count = count @@ -126,44 +135,77 @@ def print_locked(msg): with print_lock: print(msg) - def acquire_release_loop(id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None): - print_locked("Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s" % (thread_id, id, timeout, lockpath)) + def acquire_release_loop( + id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None + ): + print_locked( + "Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s" + % (thread_id, id, timeout, lockpath) + ) timer = TimeoutTimer(timeout, -1).start() cycle = 0 while not timer.timed_out(): cycle += 1 try: - with ExclusiveLock(lockpath, id=id, timeout=timeout/20, sleep=-1): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway. + with ExclusiveLock( + lockpath, id=id, timeout=timeout / 20, sleep=-1 + ): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway. lock_owner_count = lock_owner_counter.incr() - print_locked("Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently." % (thread_id, cycle, lock_owner_count)) + print_locked( + "Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently." + % (thread_id, cycle, lock_owner_count) + ) time.sleep(0.005) lock_owner_count = lock_owner_counter.decr() - print_locked("Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock." % (thread_id, cycle, lock_owner_count)) + print_locked( + "Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock." + % (thread_id, cycle, lock_owner_count) + ) except LockTimeout: print_locked("Thread %2d: Got LockTimeout, finishing my %d. loop cycle." % (thread_id, cycle)) except: exception_count = exception_counter.incr() e = format_exc() - print_locked("Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s" % (thread_id, cycle, exception_count, e)) + print_locked( + "Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s" + % (thread_id, cycle, exception_count, e) + ) print_locked("Thread %2d: Loop timed out--terminating after %d loop cycles." % (thread_id, cycle)) if last_thread is not None: # joining its predecessor, if any last_thread.join() - print('') + print("") lock_owner_counter = SynchronizedCounter() exception_counter = SynchronizedCounter() print_lock = ThreadingLock() thread = None for thread_id in range(RACE_TEST_NUM_THREADS): - thread = Thread(target=acquire_release_loop, args=(('foo', thread_id, 0), RACE_TEST_DURATION, thread_id, lock_owner_counter, exception_counter, print_lock, thread)) + thread = Thread( + target=acquire_release_loop, + args=( + ("foo", thread_id, 0), + RACE_TEST_DURATION, + thread_id, + lock_owner_counter, + exception_counter, + print_lock, + thread, + ), + ) thread.start() thread.join() # joining the last thread - assert lock_owner_counter.maxvalue() > 0, 'Never gained the lock? Something went wrong here...' - assert lock_owner_counter.maxvalue() <= 1, "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % (lock_owner_counter.maxvalue()) - assert exception_counter.value() == 0, "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value()) + assert lock_owner_counter.maxvalue() > 0, "Never gained the lock? Something went wrong here..." + assert ( + lock_owner_counter.maxvalue() <= 1 + ), "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % ( + lock_owner_counter.maxvalue() + ) + assert ( + exception_counter.value() == 0 + ), "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value()) class TestLock: @@ -228,7 +270,7 @@ def test_timeout(self, lockpath): def test_kill_stale(self, lockpath, free_pid): host, pid, tid = our_id = get_process_id() dead_id = host, free_pid, tid - cant_know_if_dead_id = 'foo.bar.example.net', 1, 2 + cant_know_if_dead_id = "foo.bar.example.net", 1, 2 dead_lock = Lock(lockpath, id=dead_id, exclusive=True).acquire() roster = dead_lock._roster @@ -263,7 +305,7 @@ def test_migrate_lock(self, lockpath): @pytest.fixture() def rosterpath(tmpdir): - return str(tmpdir.join('roster')) + return str(tmpdir.join("roster")) class TestLockRoster: @@ -277,13 +319,13 @@ def test_modify_get(self, rosterpath): roster1 = LockRoster(rosterpath, id=ID1) assert roster1.get(SHARED) == set() roster1.modify(SHARED, ADD) - assert roster1.get(SHARED) == {ID1, } + assert roster1.get(SHARED) == {ID1} roster2 = LockRoster(rosterpath, id=ID2) roster2.modify(SHARED, ADD) - assert roster2.get(SHARED) == {ID1, ID2, } + assert roster2.get(SHARED) == {ID1, ID2} roster1 = LockRoster(rosterpath, id=ID1) roster1.modify(SHARED, REMOVE) - assert roster1.get(SHARED) == {ID2, } + assert roster1.get(SHARED) == {ID2} roster2 = LockRoster(rosterpath, id=ID2) roster2.modify(SHARED, REMOVE) assert roster2.get(SHARED) == set() @@ -300,7 +342,7 @@ def test_kill_stale(self, rosterpath, free_pid): assert roster1.get(SHARED) == {dead_id} # put a unknown-state remote process lock into roster - cant_know_if_dead_id = 'foo.bar.example.net', 1, 2 + cant_know_if_dead_id = "foo.bar.example.net", 1, 2 roster1 = LockRoster(rosterpath, id=cant_know_if_dead_id) roster1.kill_stale_locks = False assert roster1.get(SHARED) == {dead_id} diff --git a/src/borg/testsuite/logger.py b/src/borg/testsuite/logger.py index b6dc29653..d24ec0fd1 100644 --- a/src/borg/testsuite/logger.py +++ b/src/borg/testsuite/logger.py @@ -4,6 +4,7 @@ import pytest from ..logger import find_parent_module, create_logger, setup_logging + logger = create_logger() @@ -11,27 +12,27 @@ def io_logger(): io = StringIO() handler = setup_logging(stream=io, env_var=None) - handler.setFormatter(logging.Formatter('%(name)s: %(message)s')) + handler.setFormatter(logging.Formatter("%(name)s: %(message)s")) logger.setLevel(logging.DEBUG) return io def test_setup_logging(io_logger): - logger.info('hello world') + logger.info("hello world") assert io_logger.getvalue() == "borg.testsuite.logger: hello world\n" def test_multiple_loggers(io_logger): logger = logging.getLogger(__name__) - logger.info('hello world 1') + logger.info("hello world 1") assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\n" - logger = logging.getLogger('borg.testsuite.logger') - logger.info('hello world 2') + logger = logging.getLogger("borg.testsuite.logger") + logger.info("hello world 2") assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\nborg.testsuite.logger: hello world 2\n" io_logger.truncate(0) io_logger.seek(0) - logger = logging.getLogger('borg.testsuite.logger') - logger.info('hello world 2') + logger = logging.getLogger("borg.testsuite.logger") + logger.info("hello world 2") assert io_logger.getvalue() == "borg.testsuite.logger: hello world 2\n" diff --git a/src/borg/testsuite/lrucache.py b/src/borg/testsuite/lrucache.py index 6d1ab4adf..79bb734c4 100644 --- a/src/borg/testsuite/lrucache.py +++ b/src/borg/testsuite/lrucache.py @@ -6,33 +6,32 @@ class TestLRUCache: - def test_lrucache(self): c = LRUCache(2, dispose=lambda _: None) assert len(c) == 0 assert c.items() == set() - for i, x in enumerate('abc'): + for i, x in enumerate("abc"): c[x] = i assert len(c) == 2 - assert c.items() == {('b', 1), ('c', 2)} - assert 'a' not in c - assert 'b' in c + assert c.items() == {("b", 1), ("c", 2)} + assert "a" not in c + assert "b" in c with pytest.raises(KeyError): - c['a'] - assert c.get('a') is None - assert c.get('a', 'foo') == 'foo' - assert c['b'] == 1 - assert c.get('b') == 1 - assert c['c'] == 2 - c['d'] = 3 + c["a"] + assert c.get("a") is None + assert c.get("a", "foo") == "foo" + assert c["b"] == 1 + assert c.get("b") == 1 + assert c["c"] == 2 + c["d"] = 3 assert len(c) == 2 - assert c['c'] == 2 - assert c['d'] == 3 - del c['c'] + assert c["c"] == 2 + assert c["d"] == 3 + del c["c"] assert len(c) == 1 with pytest.raises(KeyError): - c['c'] - assert c['d'] == 3 + c["c"] + assert c["d"] == 3 c.clear() assert c.items() == set() diff --git a/src/borg/testsuite/nanorst.py b/src/borg/testsuite/nanorst.py index b25336b62..fad21d842 100644 --- a/src/borg/testsuite/nanorst.py +++ b/src/borg/testsuite/nanorst.py @@ -4,15 +4,15 @@ def test_inline(): - assert rst_to_text('*foo* and ``bar``.') == 'foo and bar.' + assert rst_to_text("*foo* and ``bar``.") == "foo and bar." def test_inline_spread(): - assert rst_to_text('*foo and bar, thusly\nfoobar*.') == 'foo and bar, thusly\nfoobar.' + assert rst_to_text("*foo and bar, thusly\nfoobar*.") == "foo and bar, thusly\nfoobar." def test_comment_inline(): - assert rst_to_text('Foo and Bar\n.. foo\nbar') == 'Foo and Bar\n.. foo\nbar' + assert rst_to_text("Foo and Bar\n.. foo\nbar") == "Foo and Bar\n.. foo\nbar" def test_inline_escape(): @@ -20,21 +20,19 @@ def test_inline_escape(): def test_comment(): - assert rst_to_text('Foo and Bar\n\n.. foo\nbar') == 'Foo and Bar\n\nbar' + assert rst_to_text("Foo and Bar\n\n.. foo\nbar") == "Foo and Bar\n\nbar" def test_directive_note(): - assert rst_to_text('.. note::\n Note this and that') == 'Note:\n Note this and that' + assert rst_to_text(".. note::\n Note this and that") == "Note:\n Note this and that" def test_ref(): - references = { - 'foo': 'baz' - } - assert rst_to_text('See :ref:`fo\no`.', references=references) == 'See baz.' + references = {"foo": "baz"} + assert rst_to_text("See :ref:`fo\no`.", references=references) == "See baz." def test_undefined_ref(): with pytest.raises(ValueError) as exc_info: - rst_to_text('See :ref:`foo`.') - assert 'Undefined reference' in str(exc_info.value) + rst_to_text("See :ref:`foo`.") + assert "Undefined reference" in str(exc_info.value) diff --git a/src/borg/testsuite/nonces.py b/src/borg/testsuite/nonces.py index 6d6701631..d9548fa91 100644 --- a/src/borg/testsuite/nonces.py +++ b/src/borg/testsuite/nonces.py @@ -10,10 +10,9 @@ class TestNonceManager: - class MockRepository: class _Location: - orig = '/some/place' + orig = "/some/place" _location = _Location() id = bytes(32) @@ -37,15 +36,15 @@ def setUp(self): self.repository = None def cache_nonce(self): - with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce')) as fd: + with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd: return fd.read() def set_cache_nonce(self, nonce): - with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce'), "w") as fd: + with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd: assert fd.write(nonce) def test_empty_cache_and_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockOldRepository() manager = NonceManager(self.repository, 0x2000) @@ -55,7 +54,7 @@ def test_empty_cache_and_old_server(self, monkeypatch): assert self.cache_nonce() == "0000000000002033" def test_empty_cache(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = 0x2000 @@ -66,7 +65,7 @@ def test_empty_cache(self, monkeypatch): assert self.cache_nonce() == "0000000000002033" def test_empty_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = None @@ -99,10 +98,10 @@ def test_empty_nonce(self, monkeypatch): next_nonce = manager.ensure_reservation(0x2043, 64) assert next_nonce == 0x2063 assert self.cache_nonce() == "00000000000020c3" - assert self.repository.next_free == 0x20c3 + assert self.repository.next_free == 0x20C3 def test_sync_nonce(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = 0x2000 @@ -116,7 +115,7 @@ def test_sync_nonce(self, monkeypatch): assert self.repository.next_free == 0x2033 def test_server_just_upgraded(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = None @@ -130,7 +129,7 @@ def test_server_just_upgraded(self, monkeypatch): assert self.repository.next_free == 0x2033 def test_transaction_abort_no_cache(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = 0x2000 @@ -143,7 +142,7 @@ def test_transaction_abort_no_cache(self, monkeypatch): assert self.repository.next_free == 0x2033 def test_transaction_abort_old_server(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockOldRepository() self.set_cache_nonce("0000000000002000") @@ -155,7 +154,7 @@ def test_transaction_abort_old_server(self, monkeypatch): assert self.cache_nonce() == "0000000000002033" def test_transaction_abort_on_other_client(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = 0x2000 @@ -169,7 +168,7 @@ def test_transaction_abort_on_other_client(self, monkeypatch): assert self.repository.next_free == 0x2033 def test_interleaved(self, monkeypatch): - monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20) + monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20) self.repository = self.MockRepository() self.repository.next_free = 0x2000 @@ -192,7 +191,7 @@ def test_interleaved(self, monkeypatch): assert self.repository.next_free == 0x4000 # spans reservation boundary - next_nonce = manager.ensure_reservation(0x201f, 21) + next_nonce = manager.ensure_reservation(0x201F, 21) assert next_nonce == 0x4000 assert self.cache_nonce() == "0000000000004035" assert self.repository.next_free == 0x4035 diff --git a/src/borg/testsuite/patterns.py b/src/borg/testsuite/patterns.py index e26ead1e8..5c8a2b12b 100644 --- a/src/borg/testsuite/patterns.py +++ b/src/borg/testsuite/patterns.py @@ -11,8 +11,7 @@ def check_patterns(files, pattern, expected): - """Utility for testing patterns. - """ + """Utility for testing patterns.""" assert all([f == os.path.normpath(f) for f in files]), "Pattern matchers expect normalized input paths" matched = [f for f in files if pattern.match(f)] @@ -20,167 +19,291 @@ def check_patterns(files, pattern, expected): assert matched == (files if expected is None else expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("/", []), - ("/home", ["home"]), - ("/home///", ["home"]), - ("/./home", ["home"]), - ("/home/user", ["home/user"]), - ("/home/user2", ["home/user2"]), - ("/home/user/.bashrc", ["home/user/.bashrc"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("/", []), + ("/home", ["home"]), + ("/home///", ["home"]), + ("/./home", ["home"]), + ("/home/user", ["home/user"]), + ("/home/user2", ["home/user2"]), + ("/home/user/.bashrc", ["home/user/.bashrc"]), + ], +) def test_patterns_full(pattern, expected): - files = ["home", "home/user", "home/user2", "home/user/.bashrc", ] + files = ["home", "home/user", "home/user2", "home/user/.bashrc"] check_patterns(files, PathFullPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("", []), - ("relative", []), - ("relative/path/", ["relative/path"]), - ("relative/path", ["relative/path"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("", []), + ("relative", []), + ("relative/path/", ["relative/path"]), + ("relative/path", ["relative/path"]), + ], +) def test_patterns_full_relative(pattern, expected): - files = ["relative/path", "relative/path2", ] + files = ["relative/path", "relative/path2"] check_patterns(files, PathFullPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("/", None), - ("/./", None), - ("", []), - ("/home/u", []), - ("/home/user", ["home/user/.profile", "home/user/.bashrc"]), - ("/etc", ["etc/server/config", "etc/server/hosts"]), - ("///etc//////", ["etc/server/config", "etc/server/hosts"]), - ("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]), - ("/srv", ["srv/messages", "srv/dmesg"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("/", None), + ("/./", None), + ("", []), + ("/home/u", []), + ("/home/user", ["home/user/.profile", "home/user/.bashrc"]), + ("/etc", ["etc/server/config", "etc/server/hosts"]), + ("///etc//////", ["etc/server/config", "etc/server/hosts"]), + ("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]), + ("/srv", ["srv/messages", "srv/dmesg"]), + ], +) def test_patterns_prefix(pattern, expected): files = [ - "etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", + "etc/server/config", + "etc/server/hosts", + "home", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", ] check_patterns(files, PathPrefixPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("", []), - ("foo", []), - ("relative", ["relative/path1", "relative/two"]), - ("more", ["more/relative"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("", []), + ("foo", []), + ("relative", ["relative/path1", "relative/two"]), + ("more", ["more/relative"]), + ], +) def test_patterns_prefix_relative(pattern, expected): files = ["relative/path1", "relative/two", "more/relative"] check_patterns(files, PathPrefixPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("/*", None), - ("/./*", None), - ("*", None), - ("*/*", - ["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", - "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("*///*", - ["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", - "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("/home/u", []), - ("/home/*", - ["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html", - "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]), - ("/etc/*", ["etc/server/config", "etc/server/hosts"]), - ("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]), - ("///etc//////*", ["etc/server/config", "etc/server/hosts"]), - ("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]), - ("/srv*", ["srv/messages", "srv/dmesg"]), - ("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("/*", None), + ("/./*", None), + ("*", None), + ( + "*/*", + [ + "etc/server/config", + "etc/server/hosts", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ( + "*///*", + [ + "etc/server/config", + "etc/server/hosts", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ("/home/u", []), + ( + "/home/*", + [ + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]), + ("/etc/*", ["etc/server/config", "etc/server/hosts"]), + ("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]), + ("///etc//////*", ["etc/server/config", "etc/server/hosts"]), + ("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]), + ("/srv*", ["srv/messages", "srv/dmesg"]), + ("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), + ], +) def test_patterns_fnmatch(pattern, expected): files = [ - "etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", - "home/foo/.thumbnails", "home/foo/bar/.thumbnails", + "etc/server/config", + "etc/server/hosts", + "home", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", ] check_patterns(files, FnmatchPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("*", None), - ("**/*", None), - ("/**/*", None), - ("/./*", None), - ("*/*", - ["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", - "srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("*///*", - ["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg", - "srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("/home/u", []), - ("/home/*", - ["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html", - "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]), - ("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]), - ("/etc/*/*", ["etc/server/config", "etc/server/hosts"]), - ("/etc/**/*", ["etc/server/config", "etc/server/hosts"]), - ("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]), - ("*/.pr????e", []), - ("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]), - ("///etc//////*", ["etc/server/config", "etc/server/hosts"]), - ("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]), - ("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]), - ("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]), - ("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]), - ("/srv/*", ["srv/messages", "srv/dmesg"]), - ("/srv2/**", ["srv2", "srv2/blafasel"]), - ("/srv2/**/", ["srv2/blafasel"]), - ("/home/*/.thumbnails", ["home/foo/.thumbnails"]), - ("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("*", None), + ("**/*", None), + ("/**/*", None), + ("/./*", None), + ( + "*/*", + [ + "etc/server/config", + "etc/server/hosts", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", + "srv2/blafasel", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ( + "*///*", + [ + "etc/server/config", + "etc/server/hosts", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv/messages", + "srv/dmesg", + "srv2/blafasel", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ("/home/u", []), + ( + "/home/*", + [ + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", + ], + ), + ("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]), + ("/etc/*/*", ["etc/server/config", "etc/server/hosts"]), + ("/etc/**/*", ["etc/server/config", "etc/server/hosts"]), + ("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]), + ("*/.pr????e", []), + ("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]), + ("///etc//////*", ["etc/server/config", "etc/server/hosts"]), + ("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]), + ("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]), + ("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]), + ("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]), + ("/srv/*", ["srv/messages", "srv/dmesg"]), + ("/srv2/**", ["srv2", "srv2/blafasel"]), + ("/srv2/**/", ["srv2/blafasel"]), + ("/home/*/.thumbnails", ["home/foo/.thumbnails"]), + ("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]), + ], +) def test_patterns_shell(pattern, expected): files = [ - "etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc", - "home/user2/.profile", "home/user2/public_html/index.html", "srv", "srv/messages", "srv/dmesg", - "srv2", "srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails", + "etc/server/config", + "etc/server/hosts", + "home", + "home/user/.profile", + "home/user/.bashrc", + "home/user2/.profile", + "home/user2/public_html/index.html", + "srv", + "srv/messages", + "srv/dmesg", + "srv2", + "srv2/blafasel", + "home/foo/.thumbnails", + "home/foo/bar/.thumbnails", ] check_patterns(files, ShellPattern(pattern), expected) -@pytest.mark.parametrize("pattern, expected", [ - # "None" means all files, i.e. all match the given pattern - ("", None), - (".*", None), - ("^/", None), - ("^abc$", []), - ("^[^/]", []), - ("^(?!/srv|/foo|/opt)", - ["/home", "/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile", - "/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", ]), - ]) +@pytest.mark.parametrize( + "pattern, expected", + [ + # "None" means all files, i.e. all match the given pattern + ("", None), + (".*", None), + ("^/", None), + ("^abc$", []), + ("^[^/]", []), + ( + "^(?!/srv|/foo|/opt)", + [ + "/home", + "/home/user/.profile", + "/home/user/.bashrc", + "/home/user2/.profile", + "/home/user2/public_html/index.html", + "/home/foo/.thumbnails", + "/home/foo/bar/.thumbnails", + ], + ), + ], +) def test_patterns_regex(pattern, expected): files = [ - '/srv/data', '/foo/bar', '/home', - '/home/user/.profile', '/home/user/.bashrc', - '/home/user2/.profile', '/home/user2/public_html/index.html', - '/opt/log/messages.txt', '/opt/log/dmesg.txt', - "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", + "/srv/data", + "/foo/bar", + "/home", + "/home/user/.profile", + "/home/user/.bashrc", + "/home/user2/.profile", + "/home/user2/public_html/index.html", + "/opt/log/messages.txt", + "/opt/log/dmesg.txt", + "/home/foo/.thumbnails", + "/home/foo/bar/.thumbnails", ] obj = RegexPattern(pattern) @@ -202,11 +325,12 @@ def use_normalized_unicode(): def _make_test_patterns(pattern): - return [PathPrefixPattern(pattern), - FnmatchPattern(pattern), - RegexPattern(f"^{pattern}/foo$"), - ShellPattern(pattern), - ] + return [ + PathPrefixPattern(pattern), + FnmatchPattern(pattern), + RegexPattern(f"^{pattern}/foo$"), + ShellPattern(pattern), + ] @pytest.mark.parametrize("pattern", _make_test_patterns("b\N{LATIN SMALL LETTER A WITH ACUTE}")) @@ -227,51 +351,63 @@ def test_invalid_unicode_pattern(pattern): assert pattern.match(str(b"ba\x80/foo", "latin1")) -@pytest.mark.parametrize("lines, expected", [ - # "None" means all files, i.e. none excluded - ([], None), - (["# Comment only"], None), - (["*"], []), - (["# Comment", - "*/something00.txt", - " *whitespace* ", - # Whitespace before comment - " #/ws*", - # Empty line - "", - "# EOF"], - ["more/data", "home", " #/wsfoobar"]), - ([r"re:.*"], []), - ([r"re:\s"], ["data/something00.txt", "more/data", "home"]), - ([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]), - (["", "", "", - "# This is a test with mixed pattern styles", - # Case-insensitive pattern - r"re:(?i)BAR|ME$", - "", - "*whitespace*", - "fm:*/something00*"], - ["more/data"]), - ([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]), - ([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]), - (["pp:./"], None), - # leading slash is removed - (["pp:/"], []), - (["pp:aaabbb"], None), - (["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]), - (["/nomatch", "/more/*"], - ['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']), - # the order of exclude patterns shouldn't matter - (["/more/*", "/nomatch"], - ['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']), - ]) +@pytest.mark.parametrize( + "lines, expected", + [ + # "None" means all files, i.e. none excluded + ([], None), + (["# Comment only"], None), + (["*"], []), + ( + [ + "# Comment", + "*/something00.txt", + " *whitespace* ", + # Whitespace before comment + " #/ws*", + # Empty line + "", + "# EOF", + ], + ["more/data", "home", " #/wsfoobar"], + ), + ([r"re:.*"], []), + ([r"re:\s"], ["data/something00.txt", "more/data", "home"]), + ([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]), + ( + [ + "", + "", + "", + "# This is a test with mixed pattern styles", + # Case-insensitive pattern + r"re:(?i)BAR|ME$", + "", + "*whitespace*", + "fm:*/something00*", + ], + ["more/data"], + ), + ([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]), + ([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]), + (["pp:./"], None), + # leading slash is removed + (["pp:/"], []), + (["pp:aaabbb"], None), + (["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]), + ( + ["/nomatch", "/more/*"], + ["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"], + ), + # the order of exclude patterns shouldn't matter + ( + ["/more/*", "/nomatch"], + ["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"], + ), + ], +) def test_exclude_patterns_from_file(tmpdir, lines, expected): - files = [ - 'data/something00.txt', 'more/data', 'home', - ' #/wsfoobar', - '\tstart/whitespace', - 'whitespace/end\t', - ] + files = ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"] def evaluate(filename): patterns = [] @@ -288,26 +424,26 @@ def evaluate(filename): assert evaluate(str(exclfile)) == (files if expected is None else expected) -@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [ - # "None" means all files, i.e. none excluded - ([], [], 0), - (["# Comment only"], [], 0), - (["- *"], [], 1), - (["+fm:*/something00.txt", - "-/data"], [], 2), - (["R /"], ["/"], 0), - (["R /", - "# comment"], ["/"], 0), - (["# comment", - "- /data", - "R /home"], ["/home"], 1), -]) +@pytest.mark.parametrize( + "lines, expected_roots, expected_numpatterns", + [ + # "None" means all files, i.e. none excluded + ([], [], 0), + (["# Comment only"], [], 0), + (["- *"], [], 1), + (["+fm:*/something00.txt", "-/data"], [], 2), + (["R /"], ["/"], 0), + (["R /", "# comment"], ["/"], 0), + (["# comment", "- /data", "R /home"], ["/home"], 1), + ], +) def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns): def evaluate(filename): roots = [] inclexclpatterns = [] load_pattern_file(open(filename), roots, inclexclpatterns) return roots, len(inclexclpatterns) + patternfile = tmpdir.join("patterns.txt") with patternfile.open("wt") as fh: @@ -344,10 +480,9 @@ def test_switch_patterns_style(): assert isinstance(patterns[5].val, ShellPattern) -@pytest.mark.parametrize("lines", [ - (["X /data"]), # illegal pattern type prefix - (["/data"]), # need a pattern type prefix -]) +@pytest.mark.parametrize( + "lines", [(["X /data"]), (["/data"])] # illegal pattern type prefix # need a pattern type prefix +) def test_load_invalid_patterns_from_file(tmpdir, lines): patternfile = tmpdir.join("patterns.txt") with patternfile.open("wt") as fh: @@ -359,41 +494,47 @@ def test_load_invalid_patterns_from_file(tmpdir, lines): load_pattern_file(open(filename), roots, inclexclpatterns) -@pytest.mark.parametrize("lines, expected", [ - # "None" means all files, i.e. none excluded - ([], None), - (["# Comment only"], None), - (["- *"], []), - # default match type is sh: for patterns -> * doesn't match a / - (["-*/something0?.txt"], - ['data', 'data/subdir/something01.txt', - 'home', 'home/leo', 'home/leo/t', 'home/other']), - (["-fm:*/something00.txt"], - ['data', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t', 'home/other']), - (["-fm:*/something0?.txt"], - ["data", 'home', 'home/leo', 'home/leo/t', 'home/other']), - (["+/*/something0?.txt", - "-/data"], - ["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']), - (["+fm:*/something00.txt", - "-/data"], - ["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']), - # include /home/leo and exclude the rest of /home: - (["+/home/leo", - "-/home/*"], - ['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']), - # wrong order, /home/leo is already excluded by -/home/*: - (["-/home/*", - "+/home/leo"], - ['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home']), - (["+fm:/home/leo", - "-/home/"], - ['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']), -]) +@pytest.mark.parametrize( + "lines, expected", + [ + # "None" means all files, i.e. none excluded + ([], None), + (["# Comment only"], None), + (["- *"], []), + # default match type is sh: for patterns -> * doesn't match a / + ( + ["-*/something0?.txt"], + ["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"], + ), + ( + ["-fm:*/something00.txt"], + ["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"], + ), + (["-fm:*/something0?.txt"], ["data", "home", "home/leo", "home/leo/t", "home/other"]), + (["+/*/something0?.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]), + (["+fm:*/something00.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]), + # include /home/leo and exclude the rest of /home: + ( + ["+/home/leo", "-/home/*"], + ["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"], + ), + # wrong order, /home/leo is already excluded by -/home/*: + (["-/home/*", "+/home/leo"], ["data", "data/something00.txt", "data/subdir/something01.txt", "home"]), + ( + ["+fm:/home/leo", "-/home/"], + ["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"], + ), + ], +) def test_inclexcl_patterns_from_file(tmpdir, lines, expected): files = [ - 'data', 'data/something00.txt', 'data/subdir/something01.txt', - 'home', 'home/leo', 'home/leo/t', 'home/other' + "data", + "data/something00.txt", + "data/subdir/something01.txt", + "home", + "home/leo", + "home/leo/t", + "home/other", ] def evaluate(filename): @@ -412,37 +553,35 @@ def evaluate(filename): assert evaluate(str(patternfile)) == (files if expected is None else expected) -@pytest.mark.parametrize("pattern, cls", [ - ("", FnmatchPattern), - - # Default style - ("*", FnmatchPattern), - ("/data/*", FnmatchPattern), - - # fnmatch style - ("fm:", FnmatchPattern), - ("fm:*", FnmatchPattern), - ("fm:/data/*", FnmatchPattern), - ("fm:fm:/data/*", FnmatchPattern), - - # Regular expression - ("re:", RegexPattern), - ("re:.*", RegexPattern), - ("re:^/something/", RegexPattern), - ("re:re:^/something/", RegexPattern), - - # Path prefix - ("pp:", PathPrefixPattern), - ("pp:/", PathPrefixPattern), - ("pp:/data/", PathPrefixPattern), - ("pp:pp:/data/", PathPrefixPattern), - - # Shell-pattern style - ("sh:", ShellPattern), - ("sh:*", ShellPattern), - ("sh:/data/*", ShellPattern), - ("sh:sh:/data/*", ShellPattern), - ]) +@pytest.mark.parametrize( + "pattern, cls", + [ + ("", FnmatchPattern), + # Default style + ("*", FnmatchPattern), + ("/data/*", FnmatchPattern), + # fnmatch style + ("fm:", FnmatchPattern), + ("fm:*", FnmatchPattern), + ("fm:/data/*", FnmatchPattern), + ("fm:fm:/data/*", FnmatchPattern), + # Regular expression + ("re:", RegexPattern), + ("re:.*", RegexPattern), + ("re:^/something/", RegexPattern), + ("re:re:^/something/", RegexPattern), + # Path prefix + ("pp:", PathPrefixPattern), + ("pp:/", PathPrefixPattern), + ("pp:/data/", PathPrefixPattern), + ("pp:pp:/data/", PathPrefixPattern), + # Shell-pattern style + ("sh:", ShellPattern), + ("sh:*", ShellPattern), + ("sh:/data/*", ShellPattern), + ("sh:sh:/data/*", ShellPattern), + ], +) def test_parse_pattern(pattern, cls): assert isinstance(parse_pattern(pattern), cls) diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 35d444845..0b0f1d047 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -21,7 +21,9 @@ group:9999:r--:9999 mask::rw- other::r-- -""".strip().encode('ascii') +""".strip().encode( + "ascii" +) DEFAULT_ACL = """ user::rw- @@ -32,18 +34,21 @@ group:8888:r--:8888 mask::rw- other::r-- -""".strip().encode('ascii') +""".strip().encode( + "ascii" +) _acls_working = None def fakeroot_detected(): - return 'FAKEROOTKEY' in os.environ + return "FAKEROOTKEY" in os.environ def user_exists(username): if not is_win32: import pwd + try: pwd.getpwnam(username) return True @@ -55,25 +60,24 @@ def user_exists(username): @functools.lru_cache def are_acls_working(): with unopened_tempfile() as filepath: - open(filepath, 'w').close() + open(filepath, "w").close() try: - access = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n' - acl = {'acl_access': access} + access = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n" + acl = {"acl_access": access} acl_set(filepath, acl) read_acl = {} acl_get(filepath, read_acl, os.stat(filepath)) - read_acl_access = read_acl.get('acl_access', None) - if read_acl_access and b'user::rw-' in read_acl_access: + read_acl_access = read_acl.get("acl_access", None) + if read_acl_access and b"user::rw-" in read_acl_access: return True except PermissionError: pass return False -@unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test') -@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot') +@unittest.skipUnless(sys.platform.startswith("linux"), "linux only test") +@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot") class PlatformLinuxTestCase(BaseTestCase): - def setUp(self): self.tmpdir = tempfile.mkdtemp() @@ -86,72 +90,80 @@ def get_acl(self, path, numeric_ids=False): return item def set_acl(self, path, access=None, default=None, numeric_ids=False): - item = {'acl_access': access, 'acl_default': default} + item = {"acl_access": access, "acl_default": default} acl_set(path, item, numeric_ids=numeric_ids) - @unittest.skipIf(not are_acls_working(), 'ACLs do not work') + @unittest.skipIf(not are_acls_working(), "ACLs do not work") def test_access_acl(self): file = tempfile.NamedTemporaryFile() self.assert_equal(self.get_acl(file.name), {}) - self.set_acl(file.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=False) - self.assert_in(b'user:root:rw-:0', self.get_acl(file.name)['acl_access']) - self.assert_in(b'group:root:rw-:0', self.get_acl(file.name)['acl_access']) - self.assert_in(b'user:0:rw-:0', self.get_acl(file.name, numeric_ids=True)['acl_access']) + self.set_acl( + file.name, + access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n", + numeric_ids=False, + ) + self.assert_in(b"user:root:rw-:0", self.get_acl(file.name)["acl_access"]) + self.assert_in(b"group:root:rw-:0", self.get_acl(file.name)["acl_access"]) + self.assert_in(b"user:0:rw-:0", self.get_acl(file.name, numeric_ids=True)["acl_access"]) file2 = tempfile.NamedTemporaryFile() - self.set_acl(file2.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=True) - self.assert_in(b'user:9999:rw-:9999', self.get_acl(file2.name)['acl_access']) - self.assert_in(b'group:9999:rw-:9999', self.get_acl(file2.name)['acl_access']) + self.set_acl( + file2.name, + access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n", + numeric_ids=True, + ) + self.assert_in(b"user:9999:rw-:9999", self.get_acl(file2.name)["acl_access"]) + self.assert_in(b"group:9999:rw-:9999", self.get_acl(file2.name)["acl_access"]) - @unittest.skipIf(not are_acls_working(), 'ACLs do not work') + @unittest.skipIf(not are_acls_working(), "ACLs do not work") def test_default_acl(self): self.assert_equal(self.get_acl(self.tmpdir), {}) self.set_acl(self.tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL) - self.assert_equal(self.get_acl(self.tmpdir)['acl_access'], ACCESS_ACL) - self.assert_equal(self.get_acl(self.tmpdir)['acl_default'], DEFAULT_ACL) + self.assert_equal(self.get_acl(self.tmpdir)["acl_access"], ACCESS_ACL) + self.assert_equal(self.get_acl(self.tmpdir)["acl_default"], DEFAULT_ACL) - @unittest.skipIf(not user_exists('übel'), 'requires übel user') - @unittest.skipIf(not are_acls_working(), 'ACLs do not work') + @unittest.skipIf(not user_exists("übel"), "requires übel user") + @unittest.skipIf(not are_acls_working(), "ACLs do not work") def test_non_ascii_acl(self): # Testing non-ascii ACL processing to see whether our code is robust. # I have no idea whether non-ascii ACLs are allowed by the standard, # but in practice they seem to be out there and must not make our code explode. file = tempfile.NamedTemporaryFile() self.assert_equal(self.get_acl(file.name), {}) - nothing_special = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\n' + nothing_special = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\n" # TODO: can this be tested without having an existing system user übel with uid 666 gid 666? - user_entry = 'user:übel:rw-:666'.encode() - user_entry_numeric = b'user:666:rw-:666' - group_entry = 'group:übel:rw-:666'.encode() - group_entry_numeric = b'group:666:rw-:666' - acl = b'\n'.join([nothing_special, user_entry, group_entry]) + user_entry = "user:übel:rw-:666".encode() + user_entry_numeric = b"user:666:rw-:666" + group_entry = "group:übel:rw-:666".encode() + group_entry_numeric = b"group:666:rw-:666" + acl = b"\n".join([nothing_special, user_entry, group_entry]) self.set_acl(file.name, access=acl, numeric_ids=False) - acl_access = self.get_acl(file.name, numeric_ids=False)['acl_access'] + acl_access = self.get_acl(file.name, numeric_ids=False)["acl_access"] self.assert_in(user_entry, acl_access) self.assert_in(group_entry, acl_access) - acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access'] + acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"] self.assert_in(user_entry_numeric, acl_access_numeric) self.assert_in(group_entry_numeric, acl_access_numeric) file2 = tempfile.NamedTemporaryFile() self.set_acl(file2.name, access=acl, numeric_ids=True) - acl_access = self.get_acl(file2.name, numeric_ids=False)['acl_access'] + acl_access = self.get_acl(file2.name, numeric_ids=False)["acl_access"] self.assert_in(user_entry, acl_access) self.assert_in(group_entry, acl_access) - acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access'] + acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"] self.assert_in(user_entry_numeric, acl_access_numeric) self.assert_in(group_entry_numeric, acl_access_numeric) def test_utils(self): from ..platform.linux import acl_use_local_uid_gid - self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-') - self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-') - self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-') - self.assert_equal(acl_use_local_uid_gid(b'group:root:rw-:0'), b'group:0:rw-') + + self.assert_equal(acl_use_local_uid_gid(b"user:nonexistent1234:rw-:1234"), b"user:1234:rw-") + self.assert_equal(acl_use_local_uid_gid(b"group:nonexistent1234:rw-:1234"), b"group:1234:rw-") + self.assert_equal(acl_use_local_uid_gid(b"user:root:rw-:0"), b"user:0:rw-") + self.assert_equal(acl_use_local_uid_gid(b"group:root:rw-:0"), b"group:0:rw-") -@unittest.skipUnless(sys.platform.startswith('darwin'), 'OS X only test') -@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot') +@unittest.skipUnless(sys.platform.startswith("darwin"), "OS X only test") +@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot") class PlatformDarwinTestCase(BaseTestCase): - def setUp(self): self.tmpdir = tempfile.mkdtemp() @@ -164,25 +176,41 @@ def get_acl(self, path, numeric_ids=False): return item def set_acl(self, path, acl, numeric_ids=False): - item = {'acl_extended': acl} + item = {"acl_extended": acl} acl_set(path, item, numeric_ids=numeric_ids) - @unittest.skipIf(not are_acls_working(), 'ACLs do not work') + @unittest.skipIf(not are_acls_working(), "ACLs do not work") def test_access_acl(self): file = tempfile.NamedTemporaryFile() file2 = tempfile.NamedTemporaryFile() self.assert_equal(self.get_acl(file.name), {}) - self.set_acl(file.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=False) - self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read', self.get_acl(file.name)['acl_extended']) - self.assert_in(b'user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read', self.get_acl(file.name)['acl_extended']) - self.set_acl(file2.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=True) - self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read', self.get_acl(file2.name)['acl_extended']) - self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read', self.get_acl(file2.name, numeric_ids=True)['acl_extended']) + self.set_acl( + file.name, + b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n", + numeric_ids=False, + ) + self.assert_in( + b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read", self.get_acl(file.name)["acl_extended"] + ) + self.assert_in( + b"user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read", self.get_acl(file.name)["acl_extended"] + ) + self.set_acl( + file2.name, + b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n", + numeric_ids=True, + ) + self.assert_in( + b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read", self.get_acl(file2.name)["acl_extended"] + ) + self.assert_in( + b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read", + self.get_acl(file2.name, numeric_ids=True)["acl_extended"], + ) -@unittest.skipUnless(sys.platform.startswith(('linux', 'freebsd', 'darwin')), 'POSIX only tests') +@unittest.skipUnless(sys.platform.startswith(("linux", "freebsd", "darwin")), "POSIX only tests") class PlatformPosixTestCase(BaseTestCase): - def test_swidth_ascii(self): self.assert_equal(swidth("borg"), 4) @@ -197,7 +225,7 @@ def test_process_alive(free_pid): id = get_process_id() assert process_alive(*id) host, pid, tid = id - assert process_alive(host + 'abc', pid, tid) + assert process_alive(host + "abc", pid, tid) assert process_alive(host, pid, tid + 1) assert not process_alive(host, free_pid, tid) diff --git a/src/borg/testsuite/remote.py b/src/borg/testsuite/remote.py index fe94dc414..125cd9260 100644 --- a/src/borg/testsuite/remote.py +++ b/src/borg/testsuite/remote.py @@ -72,10 +72,10 @@ def test_write(self, monkeypatch): class TestRepositoryCache: @pytest.fixture def repository(self, tmpdir): - self.repository_location = os.path.join(str(tmpdir), 'repository') + self.repository_location = os.path.join(str(tmpdir), "repository") with Repository(self.repository_location, exclusive=True, create=True) as repository: - repository.put(H(1), b'1234') - repository.put(H(2), b'5678') + repository.put(H(1), b"1234") + repository.put(H(2), b"5678") repository.put(H(3), bytes(100)) yield repository @@ -85,19 +85,19 @@ def cache(self, repository): def test_simple(self, cache: RepositoryCache): # Single get()s are not cached, since they are used for unique objects like archives. - assert cache.get(H(1)) == b'1234' + assert cache.get(H(1)) == b"1234" assert cache.misses == 1 assert cache.hits == 0 - assert list(cache.get_many([H(1)])) == [b'1234'] + assert list(cache.get_many([H(1)])) == [b"1234"] assert cache.misses == 2 assert cache.hits == 0 - assert list(cache.get_many([H(1)])) == [b'1234'] + assert list(cache.get_many([H(1)])) == [b"1234"] assert cache.misses == 2 assert cache.hits == 1 - assert cache.get(H(1)) == b'1234' + assert cache.get(H(1)) == b"1234" assert cache.misses == 2 assert cache.hits == 2 @@ -105,11 +105,11 @@ def test_backoff(self, cache: RepositoryCache): def query_size_limit(): cache.size_limit = 0 - assert list(cache.get_many([H(1), H(2)])) == [b'1234', b'5678'] + assert list(cache.get_many([H(1), H(2)])) == [b"1234", b"5678"] assert cache.misses == 2 assert cache.evictions == 0 iterator = cache.get_many([H(1), H(3), H(2)]) - assert next(iterator) == b'1234' + assert next(iterator) == b"1234" # Force cache to back off qsl = cache.query_size_limit @@ -124,7 +124,7 @@ def query_size_limit(): assert cache.slow_misses == 0 # Since H(2) was in the cache when we called get_many(), but has # been evicted during iterating the generator, it will be a slow miss. - assert next(iterator) == b'5678' + assert next(iterator) == b"5678" assert cache.slow_misses == 1 def test_enospc(self, cache: RepositoryCache): @@ -139,16 +139,16 @@ def __exit__(self, exc_type, exc_val, exc_tb): pass def write(self, data): - raise OSError(errno.ENOSPC, 'foo') + raise OSError(errno.ENOSPC, "foo") def truncate(self, n=None): pass iterator = cache.get_many([H(1), H(2), H(3)]) - assert next(iterator) == b'1234' + assert next(iterator) == b"1234" - with patch('builtins.open', enospc_open): - assert next(iterator) == b'5678' + with patch("builtins.open", enospc_open): + assert next(iterator) == b"5678" assert cache.enospc == 1 # We didn't patch query_size_limit which would set size_limit to some low # value, so nothing was actually evicted. @@ -158,9 +158,9 @@ def truncate(self, n=None): @pytest.fixture def key(self, repository, monkeypatch): - monkeypatch.setenv('BORG_PASSPHRASE', 'test') + monkeypatch.setenv("BORG_PASSPHRASE", "test") key = PlaintextKey.create(repository, TestKey.MockArgs()) - key.compressor = CompressionSpec('none').compressor + key.compressor = CompressionSpec("none").compressor return key def _put_encrypted_object(self, key, repository, data): @@ -170,11 +170,11 @@ def _put_encrypted_object(self, key, repository, data): @pytest.fixture def H1(self, key, repository): - return self._put_encrypted_object(key, repository, b'1234') + return self._put_encrypted_object(key, repository, b"1234") @pytest.fixture def H2(self, key, repository): - return self._put_encrypted_object(key, repository, b'5678') + return self._put_encrypted_object(key, repository, b"5678") @pytest.fixture def H3(self, key, repository): @@ -188,14 +188,14 @@ def test_cache_corruption(self, decrypted_cache: RepositoryCache, H1, H2, H3): list(decrypted_cache.get_many([H1, H2, H3])) iterator = decrypted_cache.get_many([H1, H2, H3]) - assert next(iterator) == (7, b'1234') + assert next(iterator) == (7, b"1234") - with open(decrypted_cache.key_filename(H2), 'a+b') as fd: + with open(decrypted_cache.key_filename(H2), "a+b") as fd: fd.seek(-1, io.SEEK_END) - corrupted = (int.from_bytes(fd.read(), 'little') ^ 2).to_bytes(1, 'little') + corrupted = (int.from_bytes(fd.read(), "little") ^ 2).to_bytes(1, "little") fd.seek(-1, io.SEEK_END) fd.write(corrupted) fd.truncate() with pytest.raises(IntegrityError): - assert next(iterator) == (7, b'5678') + assert next(iterator) == (7, b"5678") diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 9248e5d8f..9c2ffefe4 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -29,7 +29,7 @@ class RepositoryTestCaseBase(BaseTestCase): def open(self, create=False, exclusive=UNSPECIFIED): if exclusive is UNSPECIFIED: exclusive = self.exclusive - return Repository(os.path.join(self.tmppath, 'repository'), exclusive=exclusive, create=create) + return Repository(os.path.join(self.tmppath, "repository"), exclusive=exclusive, create=create) def setUp(self): self.tmppath = tempfile.mkdtemp() @@ -46,19 +46,19 @@ def reopen(self, exclusive=UNSPECIFIED): self.repository = self.open(exclusive=exclusive) def add_keys(self): - self.repository.put(H(0), b'foo') - self.repository.put(H(1), b'bar') - self.repository.put(H(3), b'bar') + self.repository.put(H(0), b"foo") + self.repository.put(H(1), b"bar") + self.repository.put(H(3), b"bar") self.repository.commit(compact=False) - self.repository.put(H(1), b'bar2') - self.repository.put(H(2), b'boo') + self.repository.put(H(1), b"bar2") + self.repository.put(H(2), b"boo") self.repository.delete(H(3)) def repo_dump(self, label=None): - label = label + ': ' if label is not None else '' + label = label + ": " if label is not None else "" H_trans = {H(i): i for i in range(10)} H_trans[None] = -1 # key == None appears in commits - tag_trans = {TAG_PUT2: 'put2', TAG_PUT: 'put', TAG_DELETE: 'del', TAG_COMMIT: 'comm'} + tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"} for segment, fn in self.repository.io.segment_iterator(): for tag, key, offset, size in self.repository.io.iter_objects(segment): print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size)) @@ -66,12 +66,11 @@ def repo_dump(self, label=None): class RepositoryTestCase(RepositoryTestCaseBase): - def test1(self): for x in range(100): - self.repository.put(H(x), b'SOMEDATA') + self.repository.put(H(x), b"SOMEDATA") key50 = H(50) - self.assert_equal(self.repository.get(key50), b'SOMEDATA') + self.assert_equal(self.repository.get(key50), b"SOMEDATA") self.repository.delete(key50) self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50)) self.repository.commit(compact=False) @@ -81,59 +80,55 @@ def test1(self): for x in range(100): if x == 50: continue - self.assert_equal(repository2.get(H(x)), b'SOMEDATA') + self.assert_equal(repository2.get(H(x)), b"SOMEDATA") def test2(self): - """Test multiple sequential transactions - """ - self.repository.put(H(0), b'foo') - self.repository.put(H(1), b'foo') + """Test multiple sequential transactions""" + self.repository.put(H(0), b"foo") + self.repository.put(H(1), b"foo") self.repository.commit(compact=False) self.repository.delete(H(0)) - self.repository.put(H(1), b'bar') + self.repository.put(H(1), b"bar") self.repository.commit(compact=False) - self.assert_equal(self.repository.get(H(1)), b'bar') + self.assert_equal(self.repository.get(H(1)), b"bar") def test_consistency(self): - """Test cache consistency - """ - self.repository.put(H(0), b'foo') - self.assert_equal(self.repository.get(H(0)), b'foo') - self.repository.put(H(0), b'foo2') - self.assert_equal(self.repository.get(H(0)), b'foo2') - self.repository.put(H(0), b'bar') - self.assert_equal(self.repository.get(H(0)), b'bar') + """Test cache consistency""" + self.repository.put(H(0), b"foo") + self.assert_equal(self.repository.get(H(0)), b"foo") + self.repository.put(H(0), b"foo2") + self.assert_equal(self.repository.get(H(0)), b"foo2") + self.repository.put(H(0), b"bar") + self.assert_equal(self.repository.get(H(0)), b"bar") self.repository.delete(H(0)) self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0))) def test_consistency2(self): - """Test cache consistency2 - """ - self.repository.put(H(0), b'foo') - self.assert_equal(self.repository.get(H(0)), b'foo') + """Test cache consistency2""" + self.repository.put(H(0), b"foo") + self.assert_equal(self.repository.get(H(0)), b"foo") self.repository.commit(compact=False) - self.repository.put(H(0), b'foo2') - self.assert_equal(self.repository.get(H(0)), b'foo2') + self.repository.put(H(0), b"foo2") + self.assert_equal(self.repository.get(H(0)), b"foo2") self.repository.rollback() - self.assert_equal(self.repository.get(H(0)), b'foo') + self.assert_equal(self.repository.get(H(0)), b"foo") def test_overwrite_in_same_transaction(self): - """Test cache consistency2 - """ - self.repository.put(H(0), b'foo') - self.repository.put(H(0), b'foo2') + """Test cache consistency2""" + self.repository.put(H(0), b"foo") + self.repository.put(H(0), b"foo2") self.repository.commit(compact=False) - self.assert_equal(self.repository.get(H(0)), b'foo2') + self.assert_equal(self.repository.get(H(0)), b"foo2") def test_single_kind_transactions(self): # put - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) self.repository.close() # replace self.repository = self.open() with self.repository: - self.repository.put(H(0), b'bar') + self.repository.put(H(0), b"bar") self.repository.commit(compact=False) # delete self.repository = self.open() @@ -143,7 +138,7 @@ def test_single_kind_transactions(self): def test_list(self): for x in range(100): - self.repository.put(H(x), b'SOMEDATA') + self.repository.put(H(x), b"SOMEDATA") self.repository.commit(compact=False) all = self.repository.list() self.assert_equal(len(all), 100) @@ -157,7 +152,7 @@ def test_list(self): def test_scan(self): for x in range(100): - self.repository.put(H(x), b'SOMEDATA') + self.repository.put(H(x), b"SOMEDATA") self.repository.commit(compact=False) all = self.repository.scan() assert len(all) == 100 @@ -173,11 +168,10 @@ def test_scan(self): assert all[x] == H(x) def test_max_data_size(self): - max_data = b'x' * MAX_DATA_SIZE + max_data = b"x" * MAX_DATA_SIZE self.repository.put(H(0), max_data) self.assert_equal(self.repository.get(H(0)), max_data) - self.assert_raises(IntegrityError, - lambda: self.repository.put(H(1), max_data + b'x')) + self.assert_raises(IntegrityError, lambda: self.repository.put(H(1), max_data + b"x")) class LocalRepositoryTestCase(RepositoryTestCaseBase): @@ -194,21 +188,21 @@ def _assert_sparse(self): assert self.repository.compact[0] == 41 + 8 + 9 def test_sparse1(self): - self.repository.put(H(0), b'foo') - self.repository.put(H(1), b'123456789') + self.repository.put(H(0), b"foo") + self.repository.put(H(1), b"123456789") self.repository.commit(compact=False) - self.repository.put(H(1), b'bar') + self.repository.put(H(1), b"bar") self._assert_sparse() def test_sparse2(self): - self.repository.put(H(0), b'foo') - self.repository.put(H(1), b'123456789') + self.repository.put(H(0), b"foo") + self.repository.put(H(1), b"123456789") self.repository.commit(compact=False) self.repository.delete(H(1)) self._assert_sparse() def test_sparse_delete(self): - self.repository.put(H(0), b'1245') + self.repository.put(H(0), b"1245") self.repository.delete(H(0)) self.repository.io._write_fd.sync() @@ -224,27 +218,26 @@ def test_sparse_delete(self): def test_uncommitted_garbage(self): # uncommitted garbage should be no problem, it is cleaned up automatically. # we just have to be careful with invalidation of cached FDs in LoggedIO. - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) # write some crap to a uncommitted segment file last_segment = self.repository.io.get_latest_segment() - with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f: - f.write(MAGIC + b'crapcrapcrap') + with open(self.repository.io.segment_filename(last_segment + 1), "wb") as f: + f.write(MAGIC + b"crapcrapcrap") self.repository.close() # usually, opening the repo and starting a transaction should trigger a cleanup. self.repository = self.open() with self.repository: - self.repository.put(H(0), b'bar') # this may trigger compact_segments() + self.repository.put(H(0), b"bar") # this may trigger compact_segments() self.repository.commit(compact=True) # the point here is that nothing blows up with an exception. class RepositoryCommitTestCase(RepositoryTestCaseBase): - def test_replay_of_missing_index(self): self.add_keys() for name in os.listdir(self.repository.path): - if name.startswith('index.'): + if name.startswith("index."): os.unlink(os.path.join(self.repository.path, name)) self.reopen() with self.repository: @@ -278,9 +271,9 @@ def test_crash_before_write_index(self): def test_replay_lock_upgrade_old(self): self.add_keys() for name in os.listdir(self.repository.path): - if name.startswith('index.'): + if name.startswith("index."): os.unlink(os.path.join(self.repository.path, name)) - with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade: + with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade: self.reopen(exclusive=None) # simulate old client that always does lock upgrades with self.repository: # the repo is only locked by a shared read lock, but to replay segments, @@ -291,9 +284,9 @@ def test_replay_lock_upgrade_old(self): def test_replay_lock_upgrade(self): self.add_keys() for name in os.listdir(self.repository.path): - if name.startswith('index.'): + if name.startswith("index."): os.unlink(os.path.join(self.repository.path, name)) - with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade: + with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade: self.reopen(exclusive=False) # current client usually does not do lock upgrade, except for replay with self.repository: # the repo is only locked by a shared read lock, but to replay segments, @@ -322,13 +315,13 @@ def test_ignores_commit_tag_in_data(self): assert not io.is_committed_segment(io.get_latest_segment()) def test_moved_deletes_are_tracked(self): - self.repository.put(H(1), b'1') - self.repository.put(H(2), b'2') + self.repository.put(H(1), b"1") + self.repository.put(H(2), b"2") self.repository.commit(compact=False) - self.repo_dump('p1 p2 c') + self.repo_dump("p1 p2 c") self.repository.delete(H(1)) self.repository.commit(compact=True) - self.repo_dump('d1 cc') + self.repo_dump("d1 cc") last_segment = self.repository.io.get_latest_segment() - 1 num_deletes = 0 for tag, key, offset, size in self.repository.io.iter_objects(last_segment): @@ -337,9 +330,9 @@ def test_moved_deletes_are_tracked(self): num_deletes += 1 assert num_deletes == 1 assert last_segment in self.repository.compact - self.repository.put(H(3), b'3') + self.repository.put(H(3), b"3") self.repository.commit(compact=True) - self.repo_dump('p3 cc') + self.repo_dump("p3 cc") assert last_segment not in self.repository.compact assert not self.repository.io.segment_exists(last_segment) for segment, _ in self.repository.io.segment_iterator(): @@ -352,7 +345,7 @@ def test_moved_deletes_are_tracked(self): def test_shadowed_entries_are_preserved(self): get_latest_segment = self.repository.io.get_latest_segment - self.repository.put(H(1), b'1') + self.repository.put(H(1), b"1") # This is the segment with our original PUT of interest put_segment = get_latest_segment() self.repository.commit(compact=False) @@ -360,7 +353,7 @@ def test_shadowed_entries_are_preserved(self): # We now delete H(1), and force this segment to not be compacted, which can happen # if it's not sparse enough (symbolized by H(2) here). self.repository.delete(H(1)) - self.repository.put(H(2), b'1') + self.repository.put(H(2), b"1") delete_segment = get_latest_segment() # We pretend these are mostly dense (not sparse) and won't be compacted @@ -380,33 +373,33 @@ def test_shadowed_entries_are_preserved(self): # Basic case, since the index survived this must be ok assert H(1) not in self.repository # Nuke index, force replay - os.unlink(os.path.join(self.repository.path, 'index.%d' % get_latest_segment())) + os.unlink(os.path.join(self.repository.path, "index.%d" % get_latest_segment())) # Must not reappear assert H(1) not in self.repository def test_shadow_index_rollback(self): - self.repository.put(H(1), b'1') + self.repository.put(H(1), b"1") self.repository.delete(H(1)) assert self.repository.shadow_index[H(1)] == [0] self.repository.commit(compact=True) - self.repo_dump('p1 d1 cc') + self.repo_dump("p1 d1 cc") # note how an empty list means that nothing is shadowed for sure assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable - self.repository.put(H(1), b'1') + self.repository.put(H(1), b"1") self.repository.delete(H(1)) - self.repo_dump('p1 d1') + self.repo_dump("p1 d1") # 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete assert self.repository.shadow_index[H(1)] == [4] self.repository.rollback() - self.repo_dump('r') - self.repository.put(H(2), b'1') + self.repo_dump("r") + self.repository.put(H(2), b"1") # After the rollback segment 4 shouldn't be considered anymore assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase): def open(self, create=False): - return Repository(os.path.join(self.tmppath, 'repository'), exclusive=True, create=create, append_only=True) + return Repository(os.path.join(self.tmppath, "repository"), exclusive=True, create=create, append_only=True) def test_destroy_append_only(self): # Can't destroy append only repo (via the API) @@ -417,19 +410,20 @@ def test_destroy_append_only(self): def test_append_only(self): def segments_in_repository(): return len(list(self.repository.io.segment_iterator())) - self.repository.put(H(0), b'foo') + + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) self.repository.append_only = False assert segments_in_repository() == 2 - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=True) # normal: compact squashes the data together, only one segment assert segments_in_repository() == 2 self.repository.append_only = True assert segments_in_repository() == 2 - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) # append only: does not compact, only new segments written assert segments_in_repository() == 4 @@ -438,12 +432,12 @@ def segments_in_repository(): class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase): def test_additional_free_space(self): self.add_keys() - self.repository.config.set('repository', 'additional_free_space', '1000T') - self.repository.save_key(b'shortcut to save_config') + self.repository.config.set("repository", "additional_free_space", "1000T") + self.repository.save_key(b"shortcut to save_config") self.reopen() with self.repository: - self.repository.put(H(0), b'foobar') + self.repository.put(H(0), b"foobar") with pytest.raises(Repository.InsufficientFreeSpaceError): self.repository.commit(compact=False) assert os.path.exists(self.repository.path) @@ -469,7 +463,7 @@ def test_tracking(self): self.reopen() with self.repository: # Open new transaction; hints and thus quota data is not loaded unless needed. - self.repository.put(H(3), b'') + self.repository.put(H(3), b"") self.repository.delete(H(3)) assert self.repository.storage_quota_use == 1234 + 5678 + 3 * (41 + 8) # we have not compacted yet self.repository.commit(compact=True) @@ -478,11 +472,11 @@ def test_tracking(self): def test_exceed_quota(self): assert self.repository.storage_quota_use == 0 self.repository.storage_quota = 80 - self.repository.put(H(1), b'') + self.repository.put(H(1), b"") assert self.repository.storage_quota_use == 41 + 8 self.repository.commit(compact=False) with pytest.raises(Repository.StorageQuotaExceeded): - self.repository.put(H(2), b'') + self.repository.put(H(2), b"") assert self.repository.storage_quota_use == (41 + 8) * 2 with pytest.raises(Repository.StorageQuotaExceeded): self.repository.commit(compact=False) @@ -491,8 +485,10 @@ def test_exceed_quota(self): with self.repository: self.repository.storage_quota = 150 # Open new transaction; hints and thus quota data is not loaded unless needed. - self.repository.put(H(1), b'') - assert self.repository.storage_quota_use == (41 + 8) * 2 # we have 2 puts for H(1) here and not yet compacted. + self.repository.put(H(1), b"") + assert ( + self.repository.storage_quota_use == (41 + 8) * 2 + ) # we have 2 puts for H(1) here and not yet compacted. self.repository.commit(compact=True) assert self.repository.storage_quota_use == 41 + 8 # now we have compacted. @@ -542,54 +538,54 @@ def test_commit_nonce_reservation(self): class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase): def setUp(self): super().setUp() - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) self.repository.close() def do_commit(self): with self.repository: - self.repository.put(H(0), b'fox') + self.repository.put(H(0), b"fox") self.repository.commit(compact=False) def test_corrupted_hints(self): - with open(os.path.join(self.repository.path, 'hints.1'), 'ab') as fd: - fd.write(b'123456789') + with open(os.path.join(self.repository.path, "hints.1"), "ab") as fd: + fd.write(b"123456789") self.do_commit() def test_deleted_hints(self): - os.unlink(os.path.join(self.repository.path, 'hints.1')) + os.unlink(os.path.join(self.repository.path, "hints.1")) self.do_commit() def test_deleted_index(self): - os.unlink(os.path.join(self.repository.path, 'index.1')) + os.unlink(os.path.join(self.repository.path, "index.1")) self.do_commit() def test_unreadable_hints(self): - hints = os.path.join(self.repository.path, 'hints.1') + hints = os.path.join(self.repository.path, "hints.1") os.unlink(hints) os.mkdir(hints) with self.assert_raises(OSError): self.do_commit() def test_index(self): - with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd: - fd.write(b'123456789') + with open(os.path.join(self.repository.path, "index.1"), "wb") as fd: + fd.write(b"123456789") self.do_commit() def test_index_outside_transaction(self): - with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd: - fd.write(b'123456789') + with open(os.path.join(self.repository.path, "index.1"), "wb") as fd: + fd.write(b"123456789") with self.repository: assert len(self.repository) == 1 def _corrupt_index(self): # HashIndex is able to detect incorrect headers and file lengths, # but on its own it can't tell if the data is correct. - index_path = os.path.join(self.repository.path, 'index.1') - with open(index_path, 'r+b') as fd: + index_path = os.path.join(self.repository.path, "index.1") + with open(index_path, "r+b") as fd: index_data = fd.read() # Flip one bit in a key stored in the index - corrupted_key = (int.from_bytes(H(0), 'little') ^ 1).to_bytes(32, 'little') + corrupted_key = (int.from_bytes(H(0), "little") ^ 1).to_bytes(32, "little") corrupted_index_data = index_data.replace(H(0), corrupted_key) assert corrupted_index_data != index_data assert len(corrupted_index_data) == len(index_data) @@ -604,11 +600,11 @@ def test_index_corrupted(self): # Data corruption is detected due to mismatching checksums # and fixed by rebuilding the index. assert len(self.repository) == 1 - assert self.repository.get(H(0)) == b'foo' + assert self.repository.get(H(0)) == b"foo" def test_index_corrupted_without_integrity(self): self._corrupt_index() - integrity_path = os.path.join(self.repository.path, 'integrity.1') + integrity_path = os.path.join(self.repository.path, "integrity.1") os.unlink(integrity_path) with self.repository: # Since the corrupted key is not noticed, the repository still thinks @@ -619,7 +615,7 @@ def test_index_corrupted_without_integrity(self): self.repository.get(H(0)) def test_unreadable_index(self): - index = os.path.join(self.repository.path, 'index.1') + index = os.path.join(self.repository.path, "index.1") os.unlink(index) os.mkdir(index) with self.assert_raises(OSError): @@ -627,36 +623,39 @@ def test_unreadable_index(self): def test_unknown_integrity_version(self): # For now an unknown integrity data version is ignored and not an error. - integrity_path = os.path.join(self.repository.path, 'integrity.1') - with open(integrity_path, 'r+b') as fd: - msgpack.pack({ - # Borg only understands version 2 - b'version': 4.7, - }, fd) + integrity_path = os.path.join(self.repository.path, "integrity.1") + with open(integrity_path, "r+b") as fd: + msgpack.pack( + { + # Borg only understands version 2 + b"version": 4.7 + }, + fd, + ) fd.truncate() with self.repository: # No issues accessing the repository assert len(self.repository) == 1 - assert self.repository.get(H(0)) == b'foo' + assert self.repository.get(H(0)) == b"foo" def _subtly_corrupted_hints_setup(self): with self.repository: self.repository.append_only = True assert len(self.repository) == 1 - assert self.repository.get(H(0)) == b'foo' - self.repository.put(H(1), b'bar') - self.repository.put(H(2), b'baz') + assert self.repository.get(H(0)) == b"foo" + self.repository.put(H(1), b"bar") + self.repository.put(H(2), b"baz") self.repository.commit(compact=False) - self.repository.put(H(2), b'bazz') + self.repository.put(H(2), b"bazz") self.repository.commit(compact=False) - hints_path = os.path.join(self.repository.path, 'hints.5') - with open(hints_path, 'r+b') as fd: + hints_path = os.path.join(self.repository.path, "hints.5") + with open(hints_path, "r+b") as fd: hints = msgpack.unpack(fd) fd.seek(0) # Corrupt segment refcount - assert hints['segments'][2] == 1 - hints['segments'][2] = 0 + assert hints["segments"][2] == 1 + hints["segments"][2] = 0 msgpack.pack(hints, fd) fd.truncate() @@ -664,37 +663,40 @@ def test_subtly_corrupted_hints(self): self._subtly_corrupted_hints_setup() with self.repository: self.repository.append_only = False - self.repository.put(H(3), b'1234') + self.repository.put(H(3), b"1234") # Do a compaction run. Succeeds, since the failed checksum prompted a rebuild of the index+hints. self.repository.commit(compact=True) assert len(self.repository) == 4 - assert self.repository.get(H(0)) == b'foo' - assert self.repository.get(H(1)) == b'bar' - assert self.repository.get(H(2)) == b'bazz' + assert self.repository.get(H(0)) == b"foo" + assert self.repository.get(H(1)) == b"bar" + assert self.repository.get(H(2)) == b"bazz" def test_subtly_corrupted_hints_without_integrity(self): self._subtly_corrupted_hints_setup() - integrity_path = os.path.join(self.repository.path, 'integrity.5') + integrity_path = os.path.join(self.repository.path, "integrity.5") os.unlink(integrity_path) with self.repository: self.repository.append_only = False - self.repository.put(H(3), b'1234') + self.repository.put(H(3), b"1234") # Do a compaction run. Fails, since the corrupted refcount was not detected and leads to an assertion failure. with pytest.raises(AssertionError) as exc_info: self.repository.commit(compact=True) - assert 'Corrupted segment reference count' in str(exc_info.value) + assert "Corrupted segment reference count" in str(exc_info.value) class RepositoryCheckTestCase(RepositoryTestCaseBase): - def list_indices(self): - return [name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if name.startswith('index.')] + return [name for name in os.listdir(os.path.join(self.tmppath, "repository")) if name.startswith("index.")] def check(self, repair=False, status=True): self.assert_equal(self.repository.check(repair=repair), status) # Make sure no tmp files are left behind - self.assert_equal([name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if 'tmp' in name], [], 'Found tmp files') + self.assert_equal( + [name for name in os.listdir(os.path.join(self.tmppath, "repository")) if "tmp" in name], + [], + "Found tmp files", + ) def get_objects(self, *ids): for id_ in ids: @@ -703,31 +705,35 @@ def get_objects(self, *ids): def add_objects(self, segments): for ids in segments: for id_ in ids: - self.repository.put(H(id_), b'data') + self.repository.put(H(id_), b"data") self.repository.commit(compact=False) def get_head(self): - return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1] + return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, "repository", "data", "0")) if n.isdigit())[ + -1 + ] def open_index(self): - return NSIndex.read(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}')) + return NSIndex.read(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}")) def corrupt_object(self, id_): idx = self.open_index() segment, offset, _ = idx[H(id_)] - with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd: + with open(os.path.join(self.tmppath, "repository", "data", "0", str(segment)), "r+b") as fd: fd.seek(offset) - fd.write(b'BOOM') + fd.write(b"BOOM") def delete_segment(self, segment): - os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment))) + os.unlink(os.path.join(self.tmppath, "repository", "data", "0", str(segment))) def delete_index(self): - os.unlink(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}')) + os.unlink(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}")) def rename_index(self, new_name): - os.rename(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'), - os.path.join(self.tmppath, 'repository', new_name)) + os.rename( + os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"), + os.path.join(self.tmppath, "repository", new_name), + ) def list_objects(self): return {int(key) for key in self.repository.list()} @@ -765,9 +771,9 @@ def test_repair_missing_commit_segment(self): def test_repair_corrupted_commit_segment(self): self.add_objects([[1, 2, 3], [4, 5, 6]]) - with open(os.path.join(self.tmppath, 'repository', 'data', '0', '3'), 'r+b') as fd: + with open(os.path.join(self.tmppath, "repository", "data", "0", "3"), "r+b") as fd: fd.seek(-1, os.SEEK_END) - fd.write(b'X') + fd.write(b"X") self.assert_raises(Repository.ObjectNotFound, lambda: self.get_objects(4)) self.check(status=True) self.get_objects(3) @@ -775,15 +781,15 @@ def test_repair_corrupted_commit_segment(self): def test_repair_no_commits(self): self.add_objects([[1, 2, 3]]) - with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd: + with open(os.path.join(self.tmppath, "repository", "data", "0", "1"), "r+b") as fd: fd.seek(-1, os.SEEK_END) - fd.write(b'X') + fd.write(b"X") self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4)) self.check(status=False) self.check(status=False) - self.assert_equal(self.list_indices(), ['index.1']) + self.assert_equal(self.list_indices(), ["index.1"]) self.check(repair=True, status=True) - self.assert_equal(self.list_indices(), ['index.2']) + self.assert_equal(self.list_indices(), ["index.2"]) self.check(status=True) self.get_objects(3) self.assert_equal({1, 2, 3}, self.list_objects()) @@ -797,30 +803,29 @@ def test_repair_missing_index(self): def test_repair_index_too_new(self): self.add_objects([[1, 2, 3], [4, 5, 6]]) - self.assert_equal(self.list_indices(), ['index.3']) - self.rename_index('index.100') + self.assert_equal(self.list_indices(), ["index.3"]) + self.rename_index("index.100") self.check(status=True) - self.assert_equal(self.list_indices(), ['index.3']) + self.assert_equal(self.list_indices(), ["index.3"]) self.get_objects(4) self.assert_equal({1, 2, 3, 4, 5, 6}, self.list_objects()) def test_crash_before_compact(self): - self.repository.put(H(0), b'data') - self.repository.put(H(0), b'data2') + self.repository.put(H(0), b"data") + self.repository.put(H(0), b"data2") # Simulate a crash before compact - with patch.object(Repository, 'compact_segments') as compact: + with patch.object(Repository, "compact_segments") as compact: self.repository.commit(compact=True) compact.assert_called_once_with(0.1) self.reopen() with self.repository: self.check(repair=True) - self.assert_equal(self.repository.get(H(0)), b'data2') + self.assert_equal(self.repository.get(H(0)), b"data2") class RepositoryHintsTestCase(RepositoryTestCaseBase): - def test_hints_persistence(self): - self.repository.put(H(0), b'data') + self.repository.put(H(0), b"data") self.repository.delete(H(0)) self.repository.commit(compact=False) shadow_index_expected = self.repository.shadow_index @@ -831,7 +836,7 @@ def test_hints_persistence(self): self.reopen() with self.repository: # see also do_compact() - self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data + self.repository.put(H(42), b"foobar") # this will call prepare_txn() and load the hints data # check if hints persistence worked: self.assert_equal(shadow_index_expected, self.repository.shadow_index) self.assert_equal(compact_expected, self.repository.compact) @@ -839,7 +844,7 @@ def test_hints_persistence(self): self.assert_equal(segments_expected, self.repository.segments) def test_hints_behaviour(self): - self.repository.put(H(0), b'data') + self.repository.put(H(0), b"data") self.assert_equal(self.repository.shadow_index, {}) assert len(self.repository.compact) == 0 self.repository.delete(H(0)) @@ -848,7 +853,7 @@ def test_hints_behaviour(self): self.assert_in(H(0), self.repository.shadow_index) self.assert_equal(len(self.repository.shadow_index[H(0)]), 1) self.assert_in(0, self.repository.compact) # segment 0 can be compacted - self.repository.put(H(42), b'foobar') # see also do_compact() + self.repository.put(H(42), b"foobar") # see also do_compact() self.repository.commit(compact=True, threshold=0.0) # compact completely! # nothing to compact any more! no info left about stuff that does not exist any more: self.assert_not_in(H(0), self.repository.shadow_index) @@ -861,12 +866,13 @@ class RemoteRepositoryTestCase(RepositoryTestCase): repository = None # type: RemoteRepository def open(self, create=False): - return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')), - exclusive=True, create=create) + return RemoteRepository( + Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create + ) def _get_mock_args(self): class MockArgs: - remote_path = 'borg' + remote_path = "borg" umask = 0o077 debug_topics = [] rsh = None @@ -878,111 +884,122 @@ def __contains__(self, item): return MockArgs() def test_invalid_rpc(self): - self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', {})) + self.assert_raises(InvalidRPCMethod, lambda: self.repository.call("__init__", {})) def test_rpc_exception_transport(self): - s1 = 'test string' + s1 = "test string" try: - self.repository.call('inject_exception', {'kind': 'DoesNotExist'}) + self.repository.call("inject_exception", {"kind": "DoesNotExist"}) except Repository.DoesNotExist as e: assert len(e.args) == 1 assert e.args[0] == self.repository.location.processed try: - self.repository.call('inject_exception', {'kind': 'AlreadyExists'}) + self.repository.call("inject_exception", {"kind": "AlreadyExists"}) except Repository.AlreadyExists as e: assert len(e.args) == 1 assert e.args[0] == self.repository.location.processed try: - self.repository.call('inject_exception', {'kind': 'CheckNeeded'}) + self.repository.call("inject_exception", {"kind": "CheckNeeded"}) except Repository.CheckNeeded as e: assert len(e.args) == 1 assert e.args[0] == self.repository.location.processed try: - self.repository.call('inject_exception', {'kind': 'IntegrityError'}) + self.repository.call("inject_exception", {"kind": "IntegrityError"}) except IntegrityError as e: assert len(e.args) == 1 assert e.args[0] == s1 try: - self.repository.call('inject_exception', {'kind': 'PathNotAllowed'}) + self.repository.call("inject_exception", {"kind": "PathNotAllowed"}) except PathNotAllowed as e: assert len(e.args) == 1 - assert e.args[0] == 'foo' + assert e.args[0] == "foo" try: - self.repository.call('inject_exception', {'kind': 'ObjectNotFound'}) + self.repository.call("inject_exception", {"kind": "ObjectNotFound"}) except Repository.ObjectNotFound as e: assert len(e.args) == 2 assert e.args[0] == s1 assert e.args[1] == self.repository.location.processed try: - self.repository.call('inject_exception', {'kind': 'InvalidRPCMethod'}) + self.repository.call("inject_exception", {"kind": "InvalidRPCMethod"}) except InvalidRPCMethod as e: assert len(e.args) == 1 assert e.args[0] == s1 try: - self.repository.call('inject_exception', {'kind': 'divide'}) + self.repository.call("inject_exception", {"kind": "divide"}) except RemoteRepository.RPCError as e: assert e.unpacked - assert e.get_message() == 'ZeroDivisionError: integer division or modulo by zero\n' - assert e.exception_class == 'ZeroDivisionError' + assert e.get_message() == "ZeroDivisionError: integer division or modulo by zero\n" + assert e.exception_class == "ZeroDivisionError" assert len(e.exception_full) > 0 def test_ssh_cmd(self): args = self._get_mock_args() self.repository._args = args - assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com'] - assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com'] - assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com'] - os.environ['BORG_RSH'] = 'ssh --foo' - assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '--foo', 'example.com'] + assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "example.com"] + assert self.repository.ssh_cmd(Location("ssh://user@example.com/foo")) == ["ssh", "user@example.com"] + assert self.repository.ssh_cmd(Location("ssh://user@example.com:1234/foo")) == [ + "ssh", + "-p", + "1234", + "user@example.com", + ] + os.environ["BORG_RSH"] = "ssh --foo" + assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "--foo", "example.com"] def test_borg_cmd(self): - assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve'] + assert self.repository.borg_cmd(None, testing=True) == [sys.executable, "-m", "borg.archiver", "serve"] args = self._get_mock_args() # XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown: logging.getLogger().setLevel(logging.INFO) # note: test logger is on info log level, so --info gets added automagically - assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info'] - args.remote_path = 'borg-0.28.2' - assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info'] - args.debug_topics = ['something_client_side', 'repository_compaction'] - assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info', - '--debug-topic=borg.debug.repository_compaction'] + assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"] + args.remote_path = "borg-0.28.2" + assert self.repository.borg_cmd(args, testing=False) == ["borg-0.28.2", "serve", "--info"] + args.debug_topics = ["something_client_side", "repository_compaction"] + assert self.repository.borg_cmd(args, testing=False) == [ + "borg-0.28.2", + "serve", + "--info", + "--debug-topic=borg.debug.repository_compaction", + ] args = self._get_mock_args() args.storage_quota = 0 - assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info'] + assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"] args.storage_quota = 314159265 - assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info', - '--storage-quota=314159265'] - args.rsh = 'ssh -i foo' + assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info", "--storage-quota=314159265"] + args.rsh = "ssh -i foo" self.repository._args = args - assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '-i', 'foo', 'example.com'] + assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "-i", "foo", "example.com"] class RemoteLegacyFree(RepositoryTestCaseBase): # Keep testing this so we can someday safely remove the legacy tuple format. def open(self, create=False): - with patch.object(RemoteRepository, 'dictFormat', True): - return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')), - exclusive=True, create=create) + with patch.object(RemoteRepository, "dictFormat", True): + return RemoteRepository( + Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), + exclusive=True, + create=create, + ) def test_legacy_free(self): # put - self.repository.put(H(0), b'foo') + self.repository.put(H(0), b"foo") self.repository.commit(compact=False) self.repository.close() # replace self.repository = self.open() with self.repository: - self.repository.put(H(0), b'bar') + self.repository.put(H(0), b"bar") self.repository.commit(compact=False) # delete self.repository = self.open() @@ -992,10 +1009,10 @@ def test_legacy_free(self): class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase): - def open(self, create=False): - return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')), - exclusive=True, create=create) + return RemoteRepository( + Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create + ) def test_crash_before_compact(self): # skip this test, we can't mock-patch a Repository class in another process! @@ -1007,8 +1024,8 @@ def setUp(self): self.stream = io.StringIO() self.handler = logging.StreamHandler(self.stream) logging.getLogger().handlers[:] = [self.handler] - logging.getLogger('borg.repository').handlers[:] = [] - logging.getLogger('borg.repository.foo').handlers[:] = [] + logging.getLogger("borg.repository").handlers[:] = [] + logging.getLogger("borg.repository.foo").handlers[:] = [] # capture stderr sys.stderr.flush() self.old_stderr = sys.stderr @@ -1019,31 +1036,31 @@ def tearDown(self): def test_stderr_messages(self): handle_remote_line("unstructured stderr message\n") - self.assert_equal(self.stream.getvalue(), '') + self.assert_equal(self.stream.getvalue(), "") # stderr messages don't get an implicit newline - self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message\n') + self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr message\n") def test_stderr_progress_messages(self): handle_remote_line("unstructured stderr progress message\r") - self.assert_equal(self.stream.getvalue(), '') + self.assert_equal(self.stream.getvalue(), "") # stderr messages don't get an implicit newline - self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr progress message\r') + self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr progress message\r") def test_pre11_format_messages(self): self.handler.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) handle_remote_line("$LOG INFO Remote: borg < 1.1 format message\n") - self.assert_equal(self.stream.getvalue(), 'Remote: borg < 1.1 format message\n') - self.assert_equal(self.stderr.getvalue(), '') + self.assert_equal(self.stream.getvalue(), "Remote: borg < 1.1 format message\n") + self.assert_equal(self.stderr.getvalue(), "") def test_post11_format_messages(self): self.handler.setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message\n") - self.assert_equal(self.stream.getvalue(), 'Remote: borg >= 1.1 format message\n') - self.assert_equal(self.stderr.getvalue(), '') + self.assert_equal(self.stream.getvalue(), "Remote: borg >= 1.1 format message\n") + self.assert_equal(self.stderr.getvalue(), "") def test_remote_messages_screened(self): # default borg config for root logger @@ -1051,12 +1068,12 @@ def test_remote_messages_screened(self): logging.getLogger().setLevel(logging.WARNING) handle_remote_line("$LOG INFO borg.repository Remote: new format info message\n") - self.assert_equal(self.stream.getvalue(), '') - self.assert_equal(self.stderr.getvalue(), '') + self.assert_equal(self.stream.getvalue(), "") + self.assert_equal(self.stderr.getvalue(), "") def test_info_to_correct_local_child(self): - logging.getLogger('borg.repository').setLevel(logging.INFO) - logging.getLogger('borg.repository.foo').setLevel(logging.INFO) + logging.getLogger("borg.repository").setLevel(logging.INFO) + logging.getLogger("borg.repository.foo").setLevel(logging.INFO) # default borg config for root logger self.handler.setLevel(logging.WARNING) logging.getLogger().setLevel(logging.WARNING) @@ -1064,14 +1081,14 @@ def test_info_to_correct_local_child(self): child_stream = io.StringIO() child_handler = logging.StreamHandler(child_stream) child_handler.setLevel(logging.INFO) - logging.getLogger('borg.repository').handlers[:] = [child_handler] + logging.getLogger("borg.repository").handlers[:] = [child_handler] foo_stream = io.StringIO() foo_handler = logging.StreamHandler(foo_stream) foo_handler.setLevel(logging.INFO) - logging.getLogger('borg.repository.foo').handlers[:] = [foo_handler] + logging.getLogger("borg.repository.foo").handlers[:] = [foo_handler] handle_remote_line("$LOG INFO borg.repository Remote: new format child message\n") - self.assert_equal(foo_stream.getvalue(), '') - self.assert_equal(child_stream.getvalue(), 'Remote: new format child message\n') - self.assert_equal(self.stream.getvalue(), '') - self.assert_equal(self.stderr.getvalue(), '') + self.assert_equal(foo_stream.getvalue(), "") + self.assert_equal(child_stream.getvalue(), "Remote: new format child message\n") + self.assert_equal(self.stream.getvalue(), "") + self.assert_equal(self.stderr.getvalue(), "") diff --git a/src/borg/testsuite/shellpattern.py b/src/borg/testsuite/shellpattern.py index 5ca5af400..d0428ccda 100644 --- a/src/borg/testsuite/shellpattern.py +++ b/src/borg/testsuite/shellpattern.py @@ -11,103 +11,96 @@ def check(path, pattern): return bool(compiled.match(path)) -@pytest.mark.parametrize("path, patterns", [ - # Literal string - ("foo/bar", ["foo/bar"]), - ("foo\\bar", ["foo\\bar"]), - - # Non-ASCII - ("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]), - ("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]), - - # Question mark - ("foo", ["fo?"]), - ("foo", ["f?o"]), - ("foo", ["f??"]), - ("foo", ["?oo"]), - ("foo", ["?o?"]), - ("foo", ["??o"]), - ("foo", ["???"]), - - # Single asterisk - ("", ["*"]), - ("foo", ["*", "**", "***"]), - ("foo", ["foo*"]), - ("foobar", ["foo*"]), - ("foobar", ["foo*bar"]), - ("foobarbaz", ["foo*baz"]), - ("bar", ["*bar"]), - ("foobar", ["*bar"]), - ("foo/bar", ["foo/*bar"]), - ("foo/bar", ["foo/*ar"]), - ("foo/bar", ["foo/*r"]), - ("foo/bar", ["foo/*"]), - ("foo/bar", ["foo*/bar"]), - ("foo/bar", ["fo*/bar"]), - ("foo/bar", ["f*/bar"]), - ("foo/bar", ["*/bar"]), - - # Double asterisk (matches 0..n directory layers) - ("foo/bar", ["foo/**/bar"]), - ("foo/1/bar", ["foo/**/bar"]), - ("foo/1/22/333/bar", ["foo/**/bar"]), - ("foo/", ["foo/**/"]), - ("foo/1/", ["foo/**/"]), - ("foo/1/22/333/", ["foo/**/"]), - ("bar", ["**/bar"]), - ("1/bar", ["**/bar"]), - ("1/22/333/bar", ["**/bar"]), - ("foo/bar/baz", ["foo/**/*"]), - - # Set - ("foo1", ["foo[12]"]), - ("foo2", ["foo[12]"]), - ("foo2/bar", ["foo[12]/*"]), - ("f??f", ["f??f", "f[?][?]f"]), - ("foo]", ["foo[]]"]), - - # Inverted set - ("foo3", ["foo[!12]"]), - ("foo^", ["foo[^!]"]), - ("foo!", ["foo[^!]"]), - ]) +@pytest.mark.parametrize( + "path, patterns", + [ + # Literal string + ("foo/bar", ["foo/bar"]), + ("foo\\bar", ["foo\\bar"]), + # Non-ASCII + ("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]), + ("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]), + # Question mark + ("foo", ["fo?"]), + ("foo", ["f?o"]), + ("foo", ["f??"]), + ("foo", ["?oo"]), + ("foo", ["?o?"]), + ("foo", ["??o"]), + ("foo", ["???"]), + # Single asterisk + ("", ["*"]), + ("foo", ["*", "**", "***"]), + ("foo", ["foo*"]), + ("foobar", ["foo*"]), + ("foobar", ["foo*bar"]), + ("foobarbaz", ["foo*baz"]), + ("bar", ["*bar"]), + ("foobar", ["*bar"]), + ("foo/bar", ["foo/*bar"]), + ("foo/bar", ["foo/*ar"]), + ("foo/bar", ["foo/*r"]), + ("foo/bar", ["foo/*"]), + ("foo/bar", ["foo*/bar"]), + ("foo/bar", ["fo*/bar"]), + ("foo/bar", ["f*/bar"]), + ("foo/bar", ["*/bar"]), + # Double asterisk (matches 0..n directory layers) + ("foo/bar", ["foo/**/bar"]), + ("foo/1/bar", ["foo/**/bar"]), + ("foo/1/22/333/bar", ["foo/**/bar"]), + ("foo/", ["foo/**/"]), + ("foo/1/", ["foo/**/"]), + ("foo/1/22/333/", ["foo/**/"]), + ("bar", ["**/bar"]), + ("1/bar", ["**/bar"]), + ("1/22/333/bar", ["**/bar"]), + ("foo/bar/baz", ["foo/**/*"]), + # Set + ("foo1", ["foo[12]"]), + ("foo2", ["foo[12]"]), + ("foo2/bar", ["foo[12]/*"]), + ("f??f", ["f??f", "f[?][?]f"]), + ("foo]", ["foo[]]"]), + # Inverted set + ("foo3", ["foo[!12]"]), + ("foo^", ["foo[^!]"]), + ("foo!", ["foo[^!]"]), + ], +) def test_match(path, patterns): for p in patterns: assert check(path, p) -@pytest.mark.parametrize("path, patterns", [ - ("", ["?", "[]"]), - ("foo", ["foo?"]), - ("foo", ["?foo"]), - ("foo", ["f?oo"]), - - # do not match path separator - ("foo/ar", ["foo?ar"]), - - # do not match/cross over os.path.sep - ("foo/bar", ["*"]), - ("foo/bar", ["foo*bar"]), - ("foo/bar", ["foo*ar"]), - ("foo/bar", ["fo*bar"]), - ("foo/bar", ["fo*ar"]), - - # Double asterisk - ("foobar", ["foo/**/bar"]), - - # Two asterisks without slash do not match directory separator - ("foo/bar", ["**"]), - - # Double asterisk not matching filename - ("foo/bar", ["**/"]), - - # Set - ("foo3", ["foo[12]"]), - - # Inverted set - ("foo1", ["foo[!12]"]), - ("foo2", ["foo[!12]"]), - ]) +@pytest.mark.parametrize( + "path, patterns", + [ + ("", ["?", "[]"]), + ("foo", ["foo?"]), + ("foo", ["?foo"]), + ("foo", ["f?oo"]), + # do not match path separator + ("foo/ar", ["foo?ar"]), + # do not match/cross over os.path.sep + ("foo/bar", ["*"]), + ("foo/bar", ["foo*bar"]), + ("foo/bar", ["foo*ar"]), + ("foo/bar", ["fo*bar"]), + ("foo/bar", ["fo*ar"]), + # Double asterisk + ("foobar", ["foo/**/bar"]), + # Two asterisks without slash do not match directory separator + ("foo/bar", ["**"]), + # Double asterisk not matching filename + ("foo/bar", ["**/"]), + # Set + ("foo3", ["foo[12]"]), + # Inverted set + ("foo1", ["foo[!12]"]), + ("foo2", ["foo[!12]"]), + ], +) def test_mismatch(path, patterns): for p in patterns: assert not check(path, p) @@ -115,10 +108,10 @@ def test_mismatch(path, patterns): def test_match_end(): regex = shellpattern.translate("*-home") # default is match_end == string end - assert re.match(regex, '2017-07-03-home') - assert not re.match(regex, '2017-07-03-home.checkpoint') + assert re.match(regex, "2017-07-03-home") + assert not re.match(regex, "2017-07-03-home.checkpoint") - match_end = r'(%s)?\Z' % r'\.checkpoint(\.\d+)?' # with/without checkpoint ending + match_end = r"(%s)?\Z" % r"\.checkpoint(\.\d+)?" # with/without checkpoint ending regex = shellpattern.translate("*-home", match_end=match_end) - assert re.match(regex, '2017-07-03-home') - assert re.match(regex, '2017-07-03-home.checkpoint') + assert re.match(regex, "2017-07-03-home") + assert re.match(regex, "2017-07-03-home.checkpoint") diff --git a/src/borg/testsuite/version.py b/src/borg/testsuite/version.py index d17dee0e7..d7f32b7c1 100644 --- a/src/borg/testsuite/version.py +++ b/src/borg/testsuite/version.py @@ -3,51 +3,57 @@ from ..version import parse_version, format_version -@pytest.mark.parametrize("version_str, version_tuple", [ - # setuptools < 8.0 uses "-" - ('1.0.0a1.dev204-g8866961.d20170606', (1, 0, 0, -4, 1)), - ('1.0.0a1.dev204-g8866961', (1, 0, 0, -4, 1)), - ('1.0.0-d20170606', (1, 0, 0, -1)), - # setuptools >= 8.0 uses "+" - ('1.0.0a1.dev204+g8866961.d20170606', (1, 0, 0, -4, 1)), - ('1.0.0a1.dev204+g8866961', (1, 0, 0, -4, 1)), - ('1.0.0+d20170606', (1, 0, 0, -1)), - # pre-release versions: - ('1.0.0a1', (1, 0, 0, -4, 1)), - ('1.0.0a2', (1, 0, 0, -4, 2)), - ('1.0.0b3', (1, 0, 0, -3, 3)), - ('1.0.0rc4', (1, 0, 0, -2, 4)), - # release versions: - ('0.0.0', (0, 0, 0, -1)), - ('0.0.11', (0, 0, 11, -1)), - ('0.11.0', (0, 11, 0, -1)), - ('11.0.0', (11, 0, 0, -1)), -]) +@pytest.mark.parametrize( + "version_str, version_tuple", + [ + # setuptools < 8.0 uses "-" + ("1.0.0a1.dev204-g8866961.d20170606", (1, 0, 0, -4, 1)), + ("1.0.0a1.dev204-g8866961", (1, 0, 0, -4, 1)), + ("1.0.0-d20170606", (1, 0, 0, -1)), + # setuptools >= 8.0 uses "+" + ("1.0.0a1.dev204+g8866961.d20170606", (1, 0, 0, -4, 1)), + ("1.0.0a1.dev204+g8866961", (1, 0, 0, -4, 1)), + ("1.0.0+d20170606", (1, 0, 0, -1)), + # pre-release versions: + ("1.0.0a1", (1, 0, 0, -4, 1)), + ("1.0.0a2", (1, 0, 0, -4, 2)), + ("1.0.0b3", (1, 0, 0, -3, 3)), + ("1.0.0rc4", (1, 0, 0, -2, 4)), + # release versions: + ("0.0.0", (0, 0, 0, -1)), + ("0.0.11", (0, 0, 11, -1)), + ("0.11.0", (0, 11, 0, -1)), + ("11.0.0", (11, 0, 0, -1)), + ], +) def test_parse_version(version_str, version_tuple): assert parse_version(version_str) == version_tuple def test_parse_version_invalid(): with pytest.raises(ValueError): - assert parse_version('') # we require x.y.z versions + assert parse_version("") # we require x.y.z versions with pytest.raises(ValueError): - assert parse_version('1') # we require x.y.z versions + assert parse_version("1") # we require x.y.z versions with pytest.raises(ValueError): - assert parse_version('1.2') # we require x.y.z versions + assert parse_version("1.2") # we require x.y.z versions with pytest.raises(ValueError): - assert parse_version('crap') + assert parse_version("crap") -@pytest.mark.parametrize("version_str, version_tuple", [ - ('1.0.0a1', (1, 0, 0, -4, 1)), - ('1.0.0', (1, 0, 0, -1)), - ('1.0.0a2', (1, 0, 0, -4, 2)), - ('1.0.0b3', (1, 0, 0, -3, 3)), - ('1.0.0rc4', (1, 0, 0, -2, 4)), - ('0.0.0', (0, 0, 0, -1)), - ('0.0.11', (0, 0, 11, -1)), - ('0.11.0', (0, 11, 0, -1)), - ('11.0.0', (11, 0, 0, -1)), -]) +@pytest.mark.parametrize( + "version_str, version_tuple", + [ + ("1.0.0a1", (1, 0, 0, -4, 1)), + ("1.0.0", (1, 0, 0, -1)), + ("1.0.0a2", (1, 0, 0, -4, 2)), + ("1.0.0b3", (1, 0, 0, -3, 3)), + ("1.0.0rc4", (1, 0, 0, -2, 4)), + ("0.0.0", (0, 0, 0, -1)), + ("0.0.11", (0, 0, 11, -1)), + ("0.11.0", (0, 11, 0, -1)), + ("11.0.0", (11, 0, 0, -1)), + ], +) def test_format_version(version_str, version_tuple): assert format_version(version_tuple) == version_str diff --git a/src/borg/testsuite/xattr.py b/src/borg/testsuite/xattr.py index 944975bfd..947f287e8 100644 --- a/src/borg/testsuite/xattr.py +++ b/src/borg/testsuite/xattr.py @@ -10,12 +10,11 @@ from . import BaseTestCase -@unittest.skipUnless(is_enabled(), 'xattr not enabled on filesystem') +@unittest.skipUnless(is_enabled(), "xattr not enabled on filesystem") class XattrTestCase(BaseTestCase): - def setUp(self): self.tmpfile = tempfile.NamedTemporaryFile() - self.symlink = self.tmpfile.name + '.symlink' + self.symlink = self.tmpfile.name + ".symlink" os.symlink(self.tmpfile.name, self.symlink) def tearDown(self): @@ -23,7 +22,7 @@ def tearDown(self): def assert_equal_se(self, is_x, want_x): # check 2 xattr lists for equality, but ignore security.selinux attr - is_x = set(is_x) - {b'security.selinux'} + is_x = set(is_x) - {b"security.selinux"} want_x = set(want_x) self.assert_equal(is_x, want_x) @@ -34,32 +33,32 @@ def test(self): self.assert_equal_se(listxattr(tmp_fn), []) self.assert_equal_se(listxattr(tmp_fd), []) self.assert_equal_se(listxattr(tmp_lfn), []) - setxattr(tmp_fn, b'user.foo', b'bar') - setxattr(tmp_fd, b'user.bar', b'foo') - setxattr(tmp_fn, b'user.empty', b'') + setxattr(tmp_fn, b"user.foo", b"bar") + setxattr(tmp_fd, b"user.bar", b"foo") + setxattr(tmp_fn, b"user.empty", b"") if not is_linux: # linux does not allow setting user.* xattrs on symlinks - setxattr(tmp_lfn, b'user.linkxattr', b'baz') - self.assert_equal_se(listxattr(tmp_fn), [b'user.foo', b'user.bar', b'user.empty']) - self.assert_equal_se(listxattr(tmp_fd), [b'user.foo', b'user.bar', b'user.empty']) - self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b'user.foo', b'user.bar', b'user.empty']) + setxattr(tmp_lfn, b"user.linkxattr", b"baz") + self.assert_equal_se(listxattr(tmp_fn), [b"user.foo", b"user.bar", b"user.empty"]) + self.assert_equal_se(listxattr(tmp_fd), [b"user.foo", b"user.bar", b"user.empty"]) + self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b"user.foo", b"user.bar", b"user.empty"]) if not is_linux: - self.assert_equal_se(listxattr(tmp_lfn), [b'user.linkxattr']) - self.assert_equal(getxattr(tmp_fn, b'user.foo'), b'bar') - self.assert_equal(getxattr(tmp_fd, b'user.foo'), b'bar') - self.assert_equal(getxattr(tmp_lfn, b'user.foo', follow_symlinks=True), b'bar') + self.assert_equal_se(listxattr(tmp_lfn), [b"user.linkxattr"]) + self.assert_equal(getxattr(tmp_fn, b"user.foo"), b"bar") + self.assert_equal(getxattr(tmp_fd, b"user.foo"), b"bar") + self.assert_equal(getxattr(tmp_lfn, b"user.foo", follow_symlinks=True), b"bar") if not is_linux: - self.assert_equal(getxattr(tmp_lfn, b'user.linkxattr'), b'baz') - self.assert_equal(getxattr(tmp_fn, b'user.empty'), b'') + self.assert_equal(getxattr(tmp_lfn, b"user.linkxattr"), b"baz") + self.assert_equal(getxattr(tmp_fn, b"user.empty"), b"") def test_listxattr_buffer_growth(self): tmp_fn = os.fsencode(self.tmpfile.name) # make it work even with ext4, which imposes rather low limits buffer.resize(size=64, init=True) # xattr raw key list will be > 64 - keys = [b'user.attr%d' % i for i in range(20)] + keys = [b"user.attr%d" % i for i in range(20)] for key in keys: - setxattr(tmp_fn, key, b'x') + setxattr(tmp_fn, key, b"x") got_keys = listxattr(tmp_fn) self.assert_equal_se(got_keys, keys) assert len(buffer) > 64 @@ -68,18 +67,15 @@ def test_getxattr_buffer_growth(self): tmp_fn = os.fsencode(self.tmpfile.name) # make it work even with ext4, which imposes rather low limits buffer.resize(size=64, init=True) - value = b'x' * 126 - setxattr(tmp_fn, b'user.big', value) - got_value = getxattr(tmp_fn, b'user.big') + value = b"x" * 126 + setxattr(tmp_fn, b"user.big", value) + got_value = getxattr(tmp_fn, b"user.big") self.assert_equal(value, got_value) self.assert_equal(len(buffer), 128) -@pytest.mark.parametrize('lstring, splitted', ( - (b'', []), - (b'\x00', [b'']), - (b'\x01a', [b'a']), - (b'\x01a\x02cd', [b'a', b'cd']), -)) +@pytest.mark.parametrize( + "lstring, splitted", ((b"", []), (b"\x00", [b""]), (b"\x01a", [b"a"]), (b"\x01a\x02cd", [b"a", b"cd"])) +) def test_split_lstring(lstring, splitted): assert split_lstring(lstring) == splitted diff --git a/src/borg/upgrade.py b/src/borg/upgrade.py index 1dfef64ec..329fa7983 100644 --- a/src/borg/upgrade.py +++ b/src/borg/upgrade.py @@ -25,8 +25,16 @@ def upgrade_compressed_chunk(self, *, chunk): def upgrade_archive_metadata(self, *, metadata): new_metadata = {} # keep all metadata except archive version and stats. - for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment', - 'chunker_params', 'recreate_cmdline'): + for attr in ( + "cmdline", + "hostname", + "username", + "time", + "time_end", + "comment", + "chunker_params", + "recreate_cmdline", + ): if hasattr(metadata, attr): new_metadata[attr] = getattr(metadata, attr) return new_metadata @@ -42,24 +50,45 @@ def new_archive(self, *, archive): def upgrade_item(self, *, item): """upgrade item as needed, get rid of legacy crap""" - ITEM_KEY_WHITELIST = {'path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hlid', - 'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size', - 'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended', - 'part'} + ITEM_KEY_WHITELIST = { + "path", + "source", + "rdev", + "chunks", + "chunks_healthy", + "hlid", + "mode", + "user", + "group", + "uid", + "gid", + "mtime", + "atime", + "ctime", + "birthtime", + "size", + "xattrs", + "bsdflags", + "acl_nfs4", + "acl_access", + "acl_default", + "acl_extended", + "part", + } if self.hlm.borg1_hardlink_master(item): - item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['path']) - self.hlm.remember(id=hlid, info=(item._dict.get('chunks'), item._dict.get('chunks_healthy'))) + item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["path"]) + self.hlm.remember(id=hlid, info=(item._dict.get("chunks"), item._dict.get("chunks_healthy"))) elif self.hlm.borg1_hardlink_slave(item): - item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['source']) + item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["source"]) chunks, chunks_healthy = self.hlm.retrieve(id=hlid, default=(None, None)) if chunks is not None: - item._dict['chunks'] = chunks + item._dict["chunks"] = chunks for chunk_id, _ in chunks: self.cache.chunk_incref(chunk_id, self.archive.stats) if chunks_healthy is not None: - item._dict['chunks_healthy'] = chunks - item._dict.pop('source') # not used for hardlinks any more, replaced by hlid + item._dict["chunks_healthy"] = chunks + item._dict.pop("source") # not used for hardlinks any more, replaced by hlid # make sure we only have desired stuff in the new item. specifically, make sure to get rid of: # - 'acl' remnants of bug in attic <= 0.13 # - 'hardlink_master' (superseded by hlid) @@ -80,17 +109,17 @@ def upgrade_zlib_and_level(chunk): return chunk ctype = chunk[0:1] - level = b'\xFF' # FF means unknown compression level + level = b"\xFF" # FF means unknown compression level if ctype == ObfuscateSize.ID: # in older borg, we used unusual byte order - old_header_fmt = Struct('>I') + old_header_fmt = Struct(">I") new_header_fmt = ObfuscateSize.header_fmt length = ObfuscateSize.header_len - size_bytes = chunk[2:2+length] + size_bytes = chunk[2 : 2 + length] size = old_header_fmt.unpack(size_bytes) size_bytes = new_header_fmt.pack(size) - compressed = chunk[2+length:] + compressed = chunk[2 + length :] compressed = upgrade_zlib_and_level(compressed) chunk = ctype + level + size_bytes + compressed else: @@ -101,8 +130,16 @@ def upgrade_archive_metadata(self, *, metadata): new_metadata = {} # keep all metadata except archive version and stats. also do not keep # recreate_source_id, recreate_args, recreate_partial_chunks which were used only in 1.1.0b1 .. b2. - for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment', - 'chunker_params', 'recreate_cmdline'): + for attr in ( + "cmdline", + "hostname", + "username", + "time", + "time_end", + "comment", + "chunker_params", + "recreate_cmdline", + ): if hasattr(metadata, attr): new_metadata[attr] = getattr(metadata, attr) return new_metadata diff --git a/src/borg/version.py b/src/borg/version.py index a7a997f7b..278054f2e 100644 --- a/src/borg/version.py +++ b/src/borg/version.py @@ -21,12 +21,12 @@ def parse_version(version): """ m = re.match(version_re, version, re.VERBOSE) if m is None: - raise ValueError('Invalid version string %s' % version) + raise ValueError("Invalid version string %s" % version) gd = m.groupdict() - version = [int(gd['major']), int(gd['minor']), int(gd['patch'])] - if m.lastgroup == 'prerelease': - p_type = {'a': -4, 'b': -3, 'rc': -2}[gd['ptype']] - p_num = int(gd['pnum']) + version = [int(gd["major"]), int(gd["minor"]), int(gd["patch"])] + if m.lastgroup == "prerelease": + p_type = {"a": -4, "b": -3, "rc": -2}[gd["ptype"]] + p_num = int(gd["pnum"]) version += [p_type, p_num] else: version += [-1] @@ -44,6 +44,6 @@ def format_version(version): elif part == -1: break else: - f[-1] = f[-1] + {-2: 'rc', -3: 'b', -4: 'a'}[part] + str(next(it)) + f[-1] = f[-1] + {-2: "rc", -3: "b", -4: "a"}[part] + str(next(it)) break - return '.'.join(f) + return ".".join(f) diff --git a/src/borg/xattr.py b/src/borg/xattr.py index 175f6372c..7fd2680cd 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -22,14 +22,14 @@ # TODO: Check whether fakeroot supports xattrs on all platforms supported below. # TODO: If that's the case then we can make Borg fakeroot-xattr-compatible on these as well. XATTR_FAKEROOT = False -if sys.platform.startswith('linux'): - LD_PRELOAD = os.environ.get('LD_PRELOAD', '') +if sys.platform.startswith("linux"): + LD_PRELOAD = os.environ.get("LD_PRELOAD", "") preloads = re.split("[ :]", LD_PRELOAD) for preload in preloads: if preload.startswith("libfakeroot"): env = prepare_subprocess_env(system=True) - fakeroot_output = subprocess.check_output(['fakeroot', '-v'], env=env) - fakeroot_version = parse_version(fakeroot_output.decode('ascii').split()[-1]) + fakeroot_output = subprocess.check_output(["fakeroot", "-v"], env=env) + fakeroot_version = parse_version(fakeroot_output.decode("ascii").split()[-1]) if fakeroot_version >= parse_version("1.20.2"): # 1.20.2 has been confirmed to have xattr support # 1.18.2 has been confirmed not to have xattr support @@ -39,11 +39,10 @@ def is_enabled(path=None): - """Determine if xattr is enabled on the filesystem - """ - with tempfile.NamedTemporaryFile(dir=path, prefix='borg-tmp') as f: + """Determine if xattr is enabled on the filesystem""" + with tempfile.NamedTemporaryFile(dir=path, prefix="borg-tmp") as f: fd = f.fileno() - name, value = b'user.name', b'value' + name, value = b"user.name", b"value" try: setxattr(fd, name, value) except OSError: @@ -80,7 +79,7 @@ def get_all(path, follow_symlinks=False): except OSError as e: name_str = name.decode() if isinstance(path, int): - path_str = '' % path + path_str = "" % path else: path_str = os.fsdecode(path) if e.errno == ENOATTR: @@ -89,8 +88,9 @@ def get_all(path, follow_symlinks=False): pass elif e.errno == errno.EPERM: # we were not permitted to read this attribute, still can continue trying to read others - logger.warning('{}: Operation not permitted when reading extended attribute {}'.format( - path_str, name_str)) + logger.warning( + "{}: Operation not permitted when reading extended attribute {}".format(path_str, name_str) + ) else: raise except OSError as e: @@ -125,21 +125,21 @@ def set_all(path, xattrs, follow_symlinks=False): warning = True k_str = k.decode() if isinstance(path, int): - path_str = '' % path + path_str = "" % path else: path_str = os.fsdecode(path) if e.errno == errno.E2BIG: - err_str = 'too big for this filesystem' + err_str = "too big for this filesystem" elif e.errno == errno.ENOTSUP: - err_str = 'xattrs not supported on this filesystem' + err_str = "xattrs not supported on this filesystem" elif e.errno == errno.ENOSPC: # ext4 reports ENOSPC when trying to set an xattr with >4kiB while ext4 can only support 4kiB xattrs # (in this case, this is NOT a "disk full" error, just a ext4 limitation). - err_str = 'no space left on device [xattr len = %d]' % (len(v),) + err_str = "no space left on device [xattr len = %d]" % (len(v),) else: # generic handler # EACCES: permission denied to set this specific xattr (this may happen related to security.* keys) # EPERM: operation not permitted err_str = os.strerror(e.errno) - logger.warning('%s: when setting extended attribute %s: %s', path_str, k_str, err_str) + logger.warning("%s: when setting extended attribute %s: %s", path_str, k_str, err_str) return warning