borgbackup/borg
1
1
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-10-26 12:41:29 +03:00
Code Issues Projects Releases Wiki Activity

blacken all the code

Browse Source 
https://black.readthedocs.io/
This commit is contained in:
Thomas Waldmann 2022-07-06 15:37:27 +02:00
parent 7f8f671102
commit 7957af562d
78 changed files with 10568 additions and 8337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
conftest.py
View File

@ -3,8 +3,8 @@
import pytest
# needed to get pretty assertion failures in unit tests:
if hasattr(pytest, 'register_assert_rewrite'):
pytest.register_assert_rewrite('borg.testsuite')
if hasattr(pytest, "register_assert_rewrite"):
pytest.register_assert_rewrite("borg.testsuite")
import borg.cache # noqa: E402
@ -21,11 +21,10 @@
@pytest.fixture(autouse=True)
def clean_env(tmpdir_factory, monkeypatch):
# avoid that we access / modify the user's normal .config / .cache directory:
monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home')))
monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home')))
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir_factory.mktemp("xdg-config-home")))
monkeypatch.setenv("XDG_CACHE_HOME", str(tmpdir_factory.mktemp("xdg-cache-home")))
# also avoid to use anything from the outside environment:
keys = [key for key in os.environ
if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )]
keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)]
for key in keys:
monkeypatch.delenv(key, raising=False)
# Speed up tests
@ -41,7 +40,7 @@ def pytest_report_header(config, startdir):
"symlinks": are_symlinks_supported(),
"hardlinks": are_hardlinks_supported(),
"atime/mtime": is_utime_fully_supported(),
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ,
}
enabled = []
disabled = []
@ -60,9 +59,11 @@ def __init__(self, request):
self.org_cache_wipe_cache = borg.cache.LocalCache.wipe_cache
def wipe_should_not_be_called(*a, **kw):
raise AssertionError("Cache wipe was triggered, if this is part of the test add "
"@pytest.mark.allow_cache_wipe")
if 'allow_cache_wipe' not in request.keywords:
raise AssertionError(
"Cache wipe was triggered, if this is part of the test add " "@pytest.mark.allow_cache_wipe"
)
if "allow_cache_wipe" not in request.keywords:
borg.cache.LocalCache.wipe_cache = wipe_should_not_be_called
request.addfinalizer(self.undo)

148
docs/conf.py
View File

@ -13,84 +13,85 @@
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
import sys, os
sys.path.insert(0, os.path.abspath('../src'))
sys.path.insert(0, os.path.abspath("../src"))
from borg import __version__ as sw_version
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = []
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
templates_path = ["_templates"]
# The suffix of source filenames.
source_suffix = '.rst'
source_suffix = ".rst"
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
master_doc = "index"
# General information about the project.
project = 'Borg - Deduplicating Archiver'
copyright = '2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)'
project = "Borg - Deduplicating Archiver"
copyright = "2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)"
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
split_char = '+' if '+' in sw_version else '-'
split_char = "+" if "+" in sw_version else "-"
version = sw_version.split(split_char)[0]
# The full version, including alpha/beta/rc tags.
release = version
suppress_warnings = ['image.nonlocal_uri']
suppress_warnings = ["image.nonlocal_uri"]
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%Y-%m-%d'
today_fmt = "%Y-%m-%d"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
exclude_patterns = ["_build"]
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# default_role = None
# The Borg docs contain no or very little Python docs.
# Thus, the primary domain is rst.
primary_domain = 'rst'
primary_domain = "rst"
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
pygments_style = "sphinx"
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# modindex_common_prefix = []
# -- Options for HTML output ---------------------------------------------------
@ -100,79 +101,73 @@
import guzzle_sphinx_theme
html_theme_path = guzzle_sphinx_theme.html_theme_path()
html_theme = 'guzzle_sphinx_theme'
html_theme = "guzzle_sphinx_theme"
def set_rst_settings(app):
app.env.settings.update({
'field_name_limit': 0,
'option_limit': 0,
})
app.env.settings.update({"field_name_limit": 0, "option_limit": 0})
def setup(app):
app.add_css_file('css/borg.css')
app.connect('builder-inited', set_rst_settings)
app.add_css_file("css/borg.css")
app.connect("builder-inited", set_rst_settings)
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
'project_nav_name': 'Borg %s' % version,
}
html_theme_options = {"project_nav_name": "Borg %s" % version}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = ['_themes']
# html_theme_path = ['_themes']
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '_static/logo.svg'
html_logo = "_static/logo.svg"
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
html_favicon = '_static/favicon.ico'
html_favicon = "_static/favicon.ico"
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['borg_theme']
html_static_path = ["borg_theme"]
html_extra_path = ['../src/borg/paperkey.html']
html_extra_path = ["../src/borg/paperkey.html"]
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%Y-%m-%d'
html_last_updated_fmt = "%Y-%m-%d"
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
html_use_smartypants = True
smartquotes_action = 'qe' # no D in there means "do not transform -- and ---"
smartquotes_action = "qe" # no D in there means "do not transform -- and ---"
# Custom sidebar templates, maps document names to template names.
html_sidebars = {
'**': ['logo-text.html', 'searchbox.html', 'globaltoc.html'],
}
html_sidebars = {"**": ["logo-text.html", "searchbox.html", "globaltoc.html"]}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# html_domain_indices = True
# If false, no index is generated.
html_use_index = False
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# html_split_index = False
# If true, links to the reST sources are added to the pages.
html_show_sourcelink = False
@ -186,57 +181,45 @@ def setup(app):
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'borgdoc'
htmlhelp_basename = "borgdoc"
# -- Options for LaTeX output --------------------------------------------------
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('book', 'Borg.tex', 'Borg Documentation',
'The Borg Collective', 'manual'),
]
latex_documents = [("book", "Borg.tex", "Borg Documentation", "The Borg Collective", "manual")]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
latex_logo = '_static/logo.pdf'
latex_logo = "_static/logo.pdf"
latex_elements = {
'papersize': 'a4paper',
'pointsize': '10pt',
'figure_align': 'H',
}
latex_elements = {"papersize": "a4paper", "pointsize": "10pt", "figure_align": "H"}
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# latex_show_pagerefs = False
# If true, show URL addresses after external links.
latex_show_urls = 'footnote'
latex_show_urls = "footnote"
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# latex_preamble = ''
# Documents to append as an appendix to all manuals.
latex_appendices = [
'support',
'resources',
'changes',
'authors',
]
latex_appendices = ["support", "resources", "changes", "authors"]
# If false, no module index is generated.
#latex_domain_indices = True
# latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
@ -244,21 +227,24 @@ def setup(app):
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('usage', 'borg',
'BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.',
['The Borg Collective (see AUTHORS file)'],
1),
(
"usage",
"borg",
"BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.",
["The Borg Collective (see AUTHORS file)"],
1,
)
]
extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.autodoc',
'sphinx.ext.todo',
'sphinx.ext.coverage',
'sphinx.ext.viewcode',
"sphinx.ext.extlinks",
"sphinx.ext.autodoc",
"sphinx.ext.todo",
"sphinx.ext.coverage",
"sphinx.ext.viewcode",
]
extlinks = {
'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),
'targz_url': ('https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz' % version, None),
"issue": ("https://github.com/borgbackup/borg/issues/%s", "#"),
"targz_url": ("https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz" % version, None),
}

4
scripts/errorlist.py
View File

@ -10,5 +10,5 @@
for cls in sorted(classes, key=lambda cls: (cls.__module__, cls.__qualname__)):
if cls is ErrorWithTraceback:
continue
print(' ', cls.__qualname__)
print(indent(cls.__doc__, ' ' * 8))
print(" ", cls.__qualname__)
print(indent(cls.__doc__, " " * 8))

21
scripts/glibc_check.py
View File

@ -13,11 +13,11 @@
verbose = True
objdump = "objdump -T %s"
glibc_re = re.compile(r'GLIBC_([0-9]\.[0-9]+)')
glibc_re = re.compile(r"GLIBC_([0-9]\.[0-9]+)")
def parse_version(v):
major, minor = v.split('.')
major, minor = v.split(".")
return int(major), int(minor)
@ -32,11 +32,9 @@ def main():
overall_versions = set()
for filename in filenames:
try:
output = subprocess.check_output(objdump % filename, shell=True,
stderr=subprocess.STDOUT)
output = subprocess.check_output(objdump % filename, shell=True, stderr=subprocess.STDOUT)
output = output.decode()
versions = {parse_version(match.group(1))
for match in glibc_re.finditer(output)}
versions = {parse_version(match.group(1)) for match in glibc_re.finditer(output)}
requires_glibc = max(versions)
overall_versions.add(requires_glibc)
if verbose:
@ -50,14 +48,15 @@ def main():
if verbose:
if ok:
print("The binaries work with the given glibc %s." %
format_version(given))
print("The binaries work with the given glibc %s." % format_version(given))
else:
print("The binaries do not work with the given glibc %s. "
"Minimum is: %s" % (format_version(given), format_version(wanted)))
print(
"The binaries do not work with the given glibc %s. "
"Minimum is: %s" % (format_version(given), format_version(wanted))
)
return ok
if __name__ == '__main__':
if __name__ == "__main__":
ok = main()
sys.exit(0 if ok else 1)

19
scripts/hash_sizes.py
View File

@ -23,11 +23,11 @@
# which growth factor to use when growing a hashtable of size < upto
# grow fast (*2.0) at the start so we do not have to resize too often (expensive).
# grow slow (*1.1) for huge hash tables (do not jump too much in memory usage)
Policy(256*K, 2.0),
Policy(2*M, 1.7),
Policy(16*M, 1.4),
Policy(128*M, 1.2),
Policy(2*G-1, 1.1),
Policy(256 * K, 2.0),
Policy(2 * M, 1.7),
Policy(16 * M, 1.4),
Policy(128 * M, 1.2),
Policy(2 * G - 1, 1.1),
]
@ -92,12 +92,15 @@ def main():
sizes.append(p)
i = int(i * grow_factor)
print("""\
print(
"""\
static int hash_sizes[] = {
%s
};
""" % ', '.join(str(size) for size in sizes))
"""
% ", ".join(str(size) for size in sizes)
)
if __name__ == '__main__':
if __name__ == "__main__":
main()

142
setup.py
View File

@ -22,11 +22,11 @@
sys.path += [os.path.dirname(__file__)]
import setup_docs
is_win32 = sys.platform.startswith('win32')
is_openbsd = sys.platform.startswith('openbsd')
is_win32 = sys.platform.startswith("win32")
is_openbsd = sys.platform.startswith("openbsd")
# Number of threads to use for cythonize, not used on windows
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != 'spawn' else None
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None
# How the build process finds the system libs:
#
@ -38,27 +38,23 @@
# 3. otherwise raise a fatal error.
# Are we building on ReadTheDocs?
on_rtd = os.environ.get('READTHEDOCS')
on_rtd = os.environ.get("READTHEDOCS")
# Extra cflags for all extensions, usually just warnings we want to explicitly enable
cflags = [
'-Wall',
'-Wextra',
'-Wpointer-arith',
]
cflags = ["-Wall", "-Wextra", "-Wpointer-arith"]
compress_source = 'src/borg/compress.pyx'
crypto_ll_source = 'src/borg/crypto/low_level.pyx'
chunker_source = 'src/borg/chunker.pyx'
hashindex_source = 'src/borg/hashindex.pyx'
item_source = 'src/borg/item.pyx'
checksums_source = 'src/borg/checksums.pyx'
platform_posix_source = 'src/borg/platform/posix.pyx'
platform_linux_source = 'src/borg/platform/linux.pyx'
platform_syncfilerange_source = 'src/borg/platform/syncfilerange.pyx'
platform_darwin_source = 'src/borg/platform/darwin.pyx'
platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
platform_windows_source = 'src/borg/platform/windows.pyx'
compress_source = "src/borg/compress.pyx"
crypto_ll_source = "src/borg/crypto/low_level.pyx"
chunker_source = "src/borg/chunker.pyx"
hashindex_source = "src/borg/hashindex.pyx"
item_source = "src/borg/item.pyx"
checksums_source = "src/borg/checksums.pyx"
platform_posix_source = "src/borg/platform/posix.pyx"
platform_linux_source = "src/borg/platform/linux.pyx"
platform_syncfilerange_source = "src/borg/platform/syncfilerange.pyx"
platform_darwin_source = "src/borg/platform/darwin.pyx"
platform_freebsd_source = "src/borg/platform/freebsd.pyx"
platform_windows_source = "src/borg/platform/windows.pyx"
cython_sources = [
compress_source,
@ -67,7 +63,6 @@
hashindex_source,
item_source,
checksums_source,
platform_posix_source,
platform_linux_source,
platform_syncfilerange_source,
@ -79,19 +74,20 @@
if cythonize:
Sdist = sdist
else:
class Sdist(sdist):
def __init__(self, *args, **kwargs):
raise Exception('Cython is required to run sdist')
raise Exception("Cython is required to run sdist")
cython_c_files = [fn.replace('.pyx', '.c') for fn in cython_sources]
cython_c_files = [fn.replace(".pyx", ".c") for fn in cython_sources]
if not on_rtd and not all(os.path.exists(path) for path in cython_c_files):
raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
raise ImportError("The GIT version of Borg needs Cython. Install Cython or use a released version.")
def rm(file):
try:
os.unlink(file)
print('rm', file)
print("rm", file)
except FileNotFoundError:
pass
@ -107,19 +103,19 @@ def finalize_options(self):
def run(self):
for source in cython_sources:
genc = source.replace('.pyx', '.c')
genc = source.replace(".pyx", ".c")
rm(genc)
compiled_glob = source.replace('.pyx', '.cpython*')
compiled_glob = source.replace(".pyx", ".cpython*")
for compiled in sorted(glob(compiled_glob)):
rm(compiled)
cmdclass = {
'build_ext': build_ext,
'build_usage': setup_docs.build_usage,
'build_man': setup_docs.build_man,
'sdist': Sdist,
'clean2': Clean,
"build_ext": build_ext,
"build_usage": setup_docs.build_usage,
"build_man": setup_docs.build_man,
"sdist": Sdist,
"clean2": Clean,
}
@ -137,16 +133,18 @@ def members_appended(*ds):
try:
import pkgconfig as pc
except ImportError:
print('Warning: can not import pkgconfig python package.')
print("Warning: can not import pkgconfig python package.")
pc = None
def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir='lib'):
def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir="lib"):
system_prefix = os.environ.get(prefix_env_var)
if system_prefix:
print(f"Detected and preferring {lib_pkg_name} [via {prefix_env_var}]")
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
library_dirs=[os.path.join(system_prefix, lib_subdir)],
libraries=[lib_name])
return dict(
include_dirs=[os.path.join(system_prefix, "include")],
library_dirs=[os.path.join(system_prefix, lib_subdir)],
libraries=[lib_name],
)
if pc and pc.installed(lib_pkg_name, pc_version):
print(f"Detected and preferring {lib_pkg_name} [via pkg-config]")
@ -158,16 +156,13 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
crypto_ldflags = []
if is_win32:
crypto_ext_lib = lib_ext_kwargs(
pc, 'BORG_OPENSSL_PREFIX', 'libcrypto', 'libcrypto', '>=1.1.1', lib_subdir='')
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="")
elif is_openbsd:
# use openssl (not libressl) because we need AES-OCB and CHACHA20-POLY1305 via EVP api
crypto_ext_lib = lib_ext_kwargs(
pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libecrypto11', '>=1.1.1')
crypto_ldflags += ['-Wl,-rpath=/usr/local/lib/eopenssl11']
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libecrypto11", ">=1.1.1")
crypto_ldflags += ["-Wl,-rpath=/usr/local/lib/eopenssl11"]
else:
crypto_ext_lib = lib_ext_kwargs(
pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libcrypto', '>=1.1.1')
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1")
crypto_ext_kwargs = members_appended(
dict(sources=[crypto_ll_source]),
@ -178,57 +173,60 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
compress_ext_kwargs = members_appended(
dict(sources=[compress_source]),
lib_ext_kwargs(pc, 'BORG_LIBLZ4_PREFIX', 'lz4', 'liblz4', '>= 1.7.0'),
lib_ext_kwargs(pc, 'BORG_LIBZSTD_PREFIX', 'zstd', 'libzstd', '>= 1.3.0'),
lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"),
dict(extra_compile_args=cflags),
)
checksums_ext_kwargs = members_appended(
dict(sources=[checksums_source]),
lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'),
lib_ext_kwargs(pc, "BORG_LIBXXHASH_PREFIX", "xxhash", "libxxhash", ">= 0.7.3"),
dict(extra_compile_args=cflags),
)
ext_modules += [
Extension('borg.crypto.low_level', **crypto_ext_kwargs),
Extension('borg.compress', **compress_ext_kwargs),
Extension('borg.hashindex', [hashindex_source], extra_compile_args=cflags),
Extension('borg.item', [item_source], extra_compile_args=cflags),
Extension('borg.chunker', [chunker_source], extra_compile_args=cflags),
Extension('borg.checksums', **checksums_ext_kwargs),
Extension("borg.crypto.low_level", **crypto_ext_kwargs),
Extension("borg.compress", **compress_ext_kwargs),
Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags),
Extension("borg.item", [item_source], extra_compile_args=cflags),
Extension("borg.chunker", [chunker_source], extra_compile_args=cflags),
Extension("borg.checksums", **checksums_ext_kwargs),
]
posix_ext = Extension('borg.platform.posix', [platform_posix_source], extra_compile_args=cflags)
linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'], extra_compile_args=cflags)
syncfilerange_ext = Extension('borg.platform.syncfilerange', [platform_syncfilerange_source], extra_compile_args=cflags)
freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source], extra_compile_args=cflags)
darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source], extra_compile_args=cflags)
windows_ext = Extension('borg.platform.windows', [platform_windows_source], extra_compile_args=cflags)
posix_ext = Extension("borg.platform.posix", [platform_posix_source], extra_compile_args=cflags)
linux_ext = Extension("borg.platform.linux", [platform_linux_source], libraries=["acl"], extra_compile_args=cflags)
syncfilerange_ext = Extension(
"borg.platform.syncfilerange", [platform_syncfilerange_source], extra_compile_args=cflags
)
freebsd_ext = Extension("borg.platform.freebsd", [platform_freebsd_source], extra_compile_args=cflags)
darwin_ext = Extension("borg.platform.darwin", [platform_darwin_source], extra_compile_args=cflags)
windows_ext = Extension("borg.platform.windows", [platform_windows_source], extra_compile_args=cflags)
if not is_win32:
ext_modules.append(posix_ext)
else:
ext_modules.append(windows_ext)
if sys.platform == 'linux':
if sys.platform == "linux":
ext_modules.append(linux_ext)
ext_modules.append(syncfilerange_ext)
elif sys.platform.startswith('freebsd'):
elif sys.platform.startswith("freebsd"):
ext_modules.append(freebsd_ext)
elif sys.platform == 'darwin':
elif sys.platform == "darwin":
ext_modules.append(darwin_ext)
# sometimes there's no need to cythonize
# this breaks chained commands like 'clean sdist'
cythonizing = len(sys.argv) > 1 and sys.argv[1] not in (
('clean', 'clean2', 'egg_info', '--help-commands', '--version')) and '--help' not in sys.argv[1:]
cythonizing = (
len(sys.argv) > 1
and sys.argv[1] not in (("clean", "clean2", "egg_info", "--help-commands", "--version"))
and "--help" not in sys.argv[1:]
)
if cythonize and cythonizing:
cython_opts = dict(
compiler_directives={'language_level': '3str'},
)
cython_opts = dict(compiler_directives={"language_level": "3str"})
if not is_win32:
# compile .pyx extensions to .c in parallel, does not work on windows
cython_opts['nthreads'] = cpu_threads
cython_opts["nthreads"] = cpu_threads
# generate C code from Cython for ALL supported platforms, so we have them in the sdist.
# the sdist does not require Cython at install time, so we need all as C.
@ -237,8 +235,4 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
ext_modules = cythonize(ext_modules, **cython_opts)
setup(
cmdclass=cmdclass,
ext_modules=ext_modules,
long_description=setup_docs.long_desc_from_readme()
)
setup(cmdclass=cmdclass, ext_modules=ext_modules, long_description=setup_docs.long_desc_from_readme())

380
setup_docs.py
View File

@ -12,36 +12,34 @@
def long_desc_from_readme():
with open('README.rst') as fd:
with open("README.rst") as fd:
long_description = fd.read()
# remove header, but have one \n before first headline
start = long_description.find('What is BorgBackup?')
start = long_description.find("What is BorgBackup?")
assert start >= 0
long_description = '\n' + long_description[start:]
long_description = "\n" + long_description[start:]
# remove badges
long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
long_description = re.compile(r"^\.\. start-badges.*^\.\. end-badges", re.M | re.S).sub("", long_description)
# remove unknown directives
long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
long_description = re.compile(r"^\.\. highlight:: \w+$", re.M).sub("", long_description)
return long_description
def format_metavar(option):
if option.nargs in ('*', '...'):
return '[%s...]' % option.metavar
elif option.nargs == '?':
return '[%s]' % option.metavar
if option.nargs in ("*", "..."):
return "[%s...]" % option.metavar
elif option.nargs == "?":
return "[%s]" % option.metavar
elif option.nargs is None:
return option.metavar
else:
raise ValueError(f'Can\'t format metavar {option.metavar}, unknown nargs {option.nargs}!')
raise ValueError(f"Can't format metavar {option.metavar}, unknown nargs {option.nargs}!")
class build_usage(Command):
description = "generate usage for each command"
user_options = [
('output=', 'O', 'output directory'),
]
user_options = [("output=", "O", "output directory")]
def initialize_options(self):
pass
@ -50,17 +48,19 @@ def finalize_options(self):
pass
def run(self):
print('generating usage docs')
print("generating usage docs")
import borg
borg.doc_mode = 'build_man'
if not os.path.exists('docs/usage'):
os.mkdir('docs/usage')
borg.doc_mode = "build_man"
if not os.path.exists("docs/usage"):
os.mkdir("docs/usage")
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog='borg').build_parser()
parser = Archiver(prog="borg").build_parser()
# borgfs has a separate man page to satisfy debian's "every program from a package
# must have a man page" requirement, but it doesn't need a separate HTML docs page
#borgfs_parser = Archiver(prog='borgfs').build_parser()
# borgfs_parser = Archiver(prog='borgfs').build_parser()
self.generate_level("", parser, Archiver)
@ -68,7 +68,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
if action.choices is not None and "SubParsersAction" in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
@ -76,32 +76,37 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
choices.update(extra_choices)
if prefix and not choices:
return
print('found commands: %s' % list(choices.keys()))
print("found commands: %s" % list(choices.keys()))
for command, parser in sorted(choices.items()):
if command.startswith('debug'):
print('skipping', command)
if command.startswith("debug"):
print("skipping", command)
continue
print('generating help for %s' % command)
print("generating help for %s" % command)
if self.generate_level(command + " ", parser, Archiver):
continue
with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
with open("docs/usage/%s.rst.inc" % command.replace(" ", "_"), "w") as doc:
doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
if command == 'help':
if command == "help":
for topic in Archiver.helptext:
params = {"topic": topic,
"underline": '~' * len('borg help ' + topic)}
params = {"topic": topic, "underline": "~" * len("borg help " + topic)}
doc.write(".. _borg_{topic}:\n\n".format(**params))
doc.write("borg help {topic}\n{underline}\n\n".format(**params))
doc.write(Archiver.helptext[topic])
else:
params = {"command": command,
"command_": command.replace(' ', '_'),
"underline": '-' * len('borg ' + command)}
params = {
"command": command,
"command_": command.replace(" ", "_"),
"underline": "-" * len("borg " + command),
}
doc.write(".. _borg_{command_}:\n\n".format(**params))
doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params))
doc.write(
"borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(
**params
)
)
self.write_usage(parser, doc)
epilog = parser.epilog
parser.epilog = None
@ -109,21 +114,21 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
doc.write("\n\nDescription\n~~~~~~~~~~~\n")
doc.write(epilog)
if 'create' in choices:
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
with open('docs/usage/common-options.rst.inc', 'w') as doc:
if "create" in choices:
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
with open("docs/usage/common-options.rst.inc", "w") as doc:
self.write_options_group(common_options, doc, False, base_indent=0)
return is_subcommand
def write_usage(self, parser, fp):
if any(len(o.option_strings) for o in parser._actions):
fp.write(' [options]')
fp.write(" [options]")
for option in parser._actions:
if option.option_strings:
continue
fp.write(' ' + format_metavar(option))
fp.write('\n\n')
fp.write(" " + format_metavar(option))
fp.write("\n\n")
def write_options(self, parser, fp):
def is_positional_group(group):
@ -134,58 +139,58 @@ def is_positional_group(group):
def html_write(s):
for line in s.splitlines():
fp.write(' ' + line + '\n')
fp.write(" " + line + "\n")
rows = []
for group in parser._action_groups:
if group.title == 'Common options':
if group.title == "Common options":
# (no of columns used, columns, ...)
rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`'))
rows.append((1, ".. class:: borg-common-opt-ref\n\n:ref:`common_options`"))
else:
if not group._group_actions:
continue
group_header = '**%s**' % group.title
group_header = "**%s**" % group.title
if group.description:
group_header += ' — ' + group.description
group_header += " — " + group.description
rows.append((1, group_header))
if is_positional_group(group):
for option in group._group_actions:
rows.append((3, '', '``%s``' % option.metavar, option.help or ''))
rows.append((3, "", "``%s``" % option.metavar, option.help or ""))
else:
for option in group._group_actions:
if option.metavar:
option_fmt = '``%s ' + option.metavar + '``'
option_fmt = "``%s " + option.metavar + "``"
else:
option_fmt = '``%s``'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
rows.append((3, '', option_str, option_desc))
option_fmt = "``%s``"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
rows.append((3, "", option_str, option_desc))
fp.write('.. only:: html\n\n')
fp.write(".. only:: html\n\n")
table = io.StringIO()
table.write('.. class:: borg-options-table\n\n')
table.write(".. class:: borg-options-table\n\n")
self.rows_to_table(rows, table.write)
fp.write(textwrap.indent(table.getvalue(), ' ' * 4))
fp.write(textwrap.indent(table.getvalue(), " " * 4))
# LaTeX output:
# Regular rST option lists (irregular column widths)
latex_options = io.StringIO()
for group in parser._action_groups:
if group.title == 'Common options':
latex_options.write('\n\n:ref:`common_options`\n')
latex_options.write(' |')
if group.title == "Common options":
latex_options.write("\n\n:ref:`common_options`\n")
latex_options.write(" |")
else:
self.write_options_group(group, latex_options)
fp.write('\n.. only:: latex\n\n')
fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4))
fp.write("\n.. only:: latex\n\n")
fp.write(textwrap.indent(latex_options.getvalue(), " " * 4))
def rows_to_table(self, rows, write):
def write_row_separator():
write('+')
write("+")
for column_width in column_widths:
write('-' * (column_width + 1))
write('+')
write('\n')
write("-" * (column_width + 1))
write("+")
write("\n")
# Find column count and width
column_count = max(columns for columns, *_ in rows)
@ -201,22 +206,22 @@ def write_row_separator():
# where each cell contains no newline.
rowspanning_cells = []
original_cells = list(original_cells)
while any('\n' in cell for cell in original_cells):
while any("\n" in cell for cell in original_cells):
cell_bloc = []
for i, cell in enumerate(original_cells):
pre, _, original_cells[i] = cell.partition('\n')
pre, _, original_cells[i] = cell.partition("\n")
cell_bloc.append(pre)
rowspanning_cells.append(cell_bloc)
rowspanning_cells.append(original_cells)
for cells in rowspanning_cells:
for i, column_width in enumerate(column_widths):
if i < columns:
write('| ')
write("| ")
write(cells[i].ljust(column_width))
else:
write(' ')
write(''.ljust(column_width))
write('|\n')
write(" ")
write("".ljust(column_width))
write("|\n")
write_row_separator()
# This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
@ -224,7 +229,9 @@ def write_row_separator():
# with CSS alone.
# Since this is HTML-only output, it would be possible to just generate a <table> directly,
# but then we'd lose rST formatting.
write(textwrap.dedent("""
write(
textwrap.dedent(
"""
.. raw:: html
<script type='text/javascript'>
@ -232,88 +239,88 @@ def write_row_separator():
$('.borg-options-table colgroup').remove();
})
</script>
"""))
"""
)
)
def write_options_group(self, group, fp, with_title=True, base_indent=4):
def is_positional_group(group):
return any(not o.option_strings for o in group._group_actions)
indent = ' ' * base_indent
indent = " " * base_indent
if is_positional_group(group):
for option in group._group_actions:
fp.write(option.metavar + '\n')
fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n')
fp.write(option.metavar + "\n")
fp.write(textwrap.indent(option.help or "", " " * base_indent) + "\n")
return
if not group._group_actions:
return
if with_title:
fp.write('\n\n')
fp.write(group.title + '\n')
fp.write("\n\n")
fp.write(group.title + "\n")
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = '%s ' + option.metavar
option_fmt = "%s " + option.metavar
else:
option_fmt = '%s'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
option_fmt = "%s"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, " " * 4)
padding = len(max(opts)) + 1
for option, desc in opts.items():
fp.write(indent + option.ljust(padding) + desc + '\n')
fp.write(indent + option.ljust(padding) + desc + "\n")
class build_man(Command):
description = 'build man pages'
description = "build man pages"
user_options = []
see_also = {
'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'),
'recreate': ('patterns', 'placeholders', 'compression'),
'list': ('info', 'diff', 'prune', 'patterns'),
'info': ('list', 'diff'),
'rcreate': ('rdelete', 'rlist', 'check', 'key-import', 'key-export', 'key-change-passphrase'),
'key-import': ('key-export', ),
'key-export': ('key-import', ),
'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page
'umount': ('mount', ),
'extract': ('mount', ),
'delete': ('compact', ),
'prune': ('compact', ),
"create": ("delete", "prune", "check", "patterns", "placeholders", "compression"),
"recreate": ("patterns", "placeholders", "compression"),
"list": ("info", "diff", "prune", "patterns"),
"info": ("list", "diff"),
"rcreate": ("rdelete", "rlist", "check", "key-import", "key-export", "key-change-passphrase"),
"key-import": ("key-export",),
"key-export": ("key-import",),
"mount": ("umount", "extract"), # Would be cooler if these two were on the same page
"umount": ("mount",),
"extract": ("mount",),
"delete": ("compact",),
"prune": ("compact",),
}
rst_prelude = textwrap.dedent("""
rst_prelude = textwrap.dedent(
"""
.. role:: ref(title)
.. |project_name| replace:: Borg
""")
"""
)
usage_group = {
'break-lock': 'lock',
'with-lock': 'lock',
'key_change-passphrase': 'key',
'key_change-location': 'key',
'key_export': 'key',
'key_import': 'key',
'key_migrate-to-repokey': 'key',
'export-tar': 'tar',
'import-tar': 'tar',
'benchmark_crud': 'benchmark',
'benchmark_cpu': 'benchmark',
'umount': 'mount',
"break-lock": "lock",
"with-lock": "lock",
"key_change-passphrase": "key",
"key_change-location": "key",
"key_export": "key",
"key_import": "key",
"key_migrate-to-repokey": "key",
"export-tar": "tar",
"import-tar": "tar",
"benchmark_crud": "benchmark",
"benchmark_cpu": "benchmark",
"umount": "mount",
}
def initialize_options(self):
@ -323,16 +330,18 @@ def finalize_options(self):
pass
def run(self):
print('building man pages (in docs/man)', file=sys.stderr)
print("building man pages (in docs/man)", file=sys.stderr)
import borg
borg.doc_mode = 'build_man'
os.makedirs('docs/man', exist_ok=True)
borg.doc_mode = "build_man"
os.makedirs("docs/man", exist_ok=True)
# allows us to build docs without the C modules fully loaded during help generation
from borg.archiver import Archiver
parser = Archiver(prog='borg').build_parser()
borgfs_parser = Archiver(prog='borgfs').build_parser()
self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser})
parser = Archiver(prog="borg").build_parser()
borgfs_parser = Archiver(prog="borgfs").build_parser()
self.generate_level("", parser, Archiver, {"borgfs": borgfs_parser})
self.build_topic_pages(Archiver)
self.build_intro_page()
@ -340,7 +349,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
is_subcommand = False
choices = {}
for action in parser._actions:
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
if action.choices is not None and "SubParsersAction" in str(action.__class__):
is_subcommand = True
for cmd, parser in action.choices.items():
choices[prefix + cmd] = parser
@ -350,50 +359,50 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
return
for command, parser in sorted(choices.items()):
if command.startswith('debug') or command == 'help':
if command.startswith("debug") or command == "help":
continue
if command == "borgfs":
man_title = command
else:
man_title = 'borg-' + command.replace(' ', '-')
print('building man page', man_title + '(1)', file=sys.stderr)
man_title = "borg-" + command.replace(" ", "-")
print("building man page", man_title + "(1)", file=sys.stderr)
is_intermediary = self.generate_level(command + ' ', parser, Archiver)
is_intermediary = self.generate_level(command + " ", parser, Archiver)
doc, write = self.new_doc()
self.write_man_header(write, man_title, parser.description)
self.write_heading(write, 'SYNOPSIS')
self.write_heading(write, "SYNOPSIS")
if is_intermediary:
subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
subparsers = [action for action in parser._actions if "SubParsersAction" in str(action.__class__)][0]
for subcommand in subparsers.choices:
write('| borg', '[common options]', command, subcommand, '...')
self.see_also.setdefault(command, []).append(f'{command}-{subcommand}')
write("| borg", "[common options]", command, subcommand, "...")
self.see_also.setdefault(command, []).append(f"{command}-{subcommand}")
else:
if command == "borgfs":
write(command, end='')
write(command, end="")
else:
write('borg', '[common options]', command, end='')
write("borg", "[common options]", command, end="")
self.write_usage(write, parser)
write('\n')
write("\n")
description, _, notes = parser.epilog.partition('\n.. man NOTES')
description, _, notes = parser.epilog.partition("\n.. man NOTES")
if description:
self.write_heading(write, 'DESCRIPTION')
self.write_heading(write, "DESCRIPTION")
write(description)
if not is_intermediary:
self.write_heading(write, 'OPTIONS')
write('See `borg-common(1)` for common options of Borg commands.')
self.write_heading(write, "OPTIONS")
write("See `borg-common(1)` for common options of Borg commands.")
write()
self.write_options(write, parser)
self.write_examples(write, command)
if notes:
self.write_heading(write, 'NOTES')
self.write_heading(write, "NOTES")
write(notes)
self.write_see_also(write, man_title)
@ -401,14 +410,14 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
self.gen_man_page(man_title, doc.getvalue())
# Generate the borg-common(1) man page with the common options.
if 'create' in choices:
if "create" in choices:
doc, write = self.new_doc()
man_title = 'borg-common'
self.write_man_header(write, man_title, 'Common options of Borg commands')
man_title = "borg-common"
self.write_man_header(write, man_title, "Common options of Borg commands")
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
self.write_heading(write, 'SYNOPSIS')
self.write_heading(write, "SYNOPSIS")
self.write_options_group(write, common_options)
self.write_see_also(write, man_title)
self.gen_man_page(man_title, doc.getvalue())
@ -418,20 +427,20 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
def build_topic_pages(self, Archiver):
for topic, text in Archiver.helptext.items():
doc, write = self.new_doc()
man_title = 'borg-' + topic
print('building man page', man_title + '(1)', file=sys.stderr)
man_title = "borg-" + topic
print("building man page", man_title + "(1)", file=sys.stderr)
self.write_man_header(write, man_title, 'Details regarding ' + topic)
self.write_heading(write, 'DESCRIPTION')
self.write_man_header(write, man_title, "Details regarding " + topic)
self.write_heading(write, "DESCRIPTION")
write(text)
self.gen_man_page(man_title, doc.getvalue())
def build_intro_page(self):
doc, write = self.new_doc()
man_title = 'borg'
print('building man page borg(1)', file=sys.stderr)
man_title = "borg"
print("building man page borg(1)", file=sys.stderr)
with open('docs/man_intro.rst') as fd:
with open("docs/man_intro.rst") as fd:
man_intro = fd.read()
self.write_man_header(write, man_title, "deduplicating and encrypting backup tool")
@ -446,9 +455,10 @@ def new_doc(self):
def printer(self, fd):
def write(*args, **kwargs):
print(*args, file=fd, **kwargs)
return write
def write_heading(self, write, header, char='-', double_sided=False):
def write_heading(self, write, header, char="-", double_sided=False):
write()
if double_sided:
write(char * len(header))
@ -457,43 +467,43 @@ def write_heading(self, write, header, char='-', double_sided=False):
write()
def write_man_header(self, write, title, description):
self.write_heading(write, title, '=', double_sided=True)
self.write_heading(write, title, "=", double_sided=True)
self.write_heading(write, description, double_sided=True)
# man page metadata
write(':Author: The Borg Collective')
write(':Date:', datetime.utcnow().date().isoformat())
write(':Manual section: 1')
write(':Manual group: borg backup tool')
write(":Author: The Borg Collective")
write(":Date:", datetime.utcnow().date().isoformat())
write(":Manual section: 1")
write(":Manual group: borg backup tool")
write()
def write_examples(self, write, command):
command = command.replace(' ', '_')
with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd:
command = command.replace(" ", "_")
with open("docs/usage/%s.rst" % self.usage_group.get(command, command)) as fd:
usage = fd.read()
usage_include = '.. include:: %s.rst.inc' % command
usage_include = ".. include:: %s.rst.inc" % command
begin = usage.find(usage_include)
end = usage.find('.. include', begin + 1)
end = usage.find(".. include", begin + 1)
# If a command has a dedicated anchor, it will occur before the command's include.
if 0 < usage.find('.. _', begin + 1) < end:
end = usage.find('.. _', begin + 1)
if 0 < usage.find(".. _", begin + 1) < end:
end = usage.find(".. _", begin + 1)
examples = usage[begin:end]
examples = examples.replace(usage_include, '')
examples = examples.replace('Examples\n~~~~~~~~', '')
examples = examples.replace('Miscellaneous Help\n------------------', '')
examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.')
examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight
examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE)
examples = examples.replace(usage_include, "")
examples = examples.replace("Examples\n~~~~~~~~", "")
examples = examples.replace("Miscellaneous Help\n------------------", "")
examples = examples.replace("``docs/misc/prune-example.txt``:", "``docs/misc/prune-example.txt``.")
examples = examples.replace(".. highlight:: none\n", "") # we don't support highlight
examples = re.sub("^(~+)$", lambda matches: "+" * len(matches.group(0)), examples, flags=re.MULTILINE)
examples = examples.strip()
if examples:
self.write_heading(write, 'EXAMPLES', '-')
self.write_heading(write, "EXAMPLES", "-")
write(examples)
def write_see_also(self, write, man_title):
see_also = self.see_also.get(man_title.replace('borg-', ''), ())
see_also = ['`borg-%s(1)`' % s for s in see_also]
see_also.insert(0, '`borg-common(1)`')
self.write_heading(write, 'SEE ALSO')
write(', '.join(see_also))
see_also = self.see_also.get(man_title.replace("borg-", ""), ())
see_also = ["`borg-%s(1)`" % s for s in see_also]
see_also.insert(0, "`borg-common(1)`")
self.write_heading(write, "SEE ALSO")
write(", ".join(see_also))
def gen_man_page(self, name, rst):
from docutils.writers import manpage
@ -502,29 +512,29 @@ def gen_man_page(self, name, rst):
from docutils.parsers.rst import roles
def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
return [inline(rawtext, '#' + text)], []
return [inline(rawtext, "#" + text)], []
roles.register_local_role('issue', issue)
roles.register_local_role("issue", issue)
# We give the source_path so that docutils can find relative includes
# as-if the document where located in the docs/ directory.
man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer())
with open('docs/man/%s.1' % name, 'wb') as fd:
man_page = publish_string(source=rst, source_path="docs/%s.rst" % name, writer=manpage.Writer())
with open("docs/man/%s.1" % name, "wb") as fd:
fd.write(man_page)
def write_usage(self, write, parser):
if any(len(o.option_strings) for o in parser._actions):
write(' [options] ', end='')
write(" [options] ", end="")
for option in parser._actions:
if option.option_strings:
continue
write(format_metavar(option), end=' ')
write(format_metavar(option), end=" ")
def write_options(self, write, parser):
for group in parser._action_groups:
if group.title == 'Common options' or not group._group_actions:
if group.title == "Common options" or not group._group_actions:
continue
title = 'arguments' if group.title == 'positional arguments' else group.title
self.write_heading(write, title, '+')
title = "arguments" if group.title == "positional arguments" else group.title
self.write_heading(write, title, "+")
self.write_options_group(write, group)
def write_options_group(self, write, group):
@ -534,19 +544,19 @@ def is_positional_group(group):
if is_positional_group(group):
for option in group._group_actions:
write(option.metavar)
write(textwrap.indent(option.help or '', ' ' * 4))
write(textwrap.indent(option.help or "", " " * 4))
return
opts = OrderedDict()
for option in group._group_actions:
if option.metavar:
option_fmt = '%s ' + option.metavar
option_fmt = "%s " + option.metavar
else:
option_fmt = '%s'
option_str = ', '.join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
option_fmt = "%s"
option_str = ", ".join(option_fmt % s for s in option.option_strings)
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
opts[option_str] = textwrap.indent(option_desc, " " * 4)
padding = len(max(opts)) + 1

6
src/borg/__init__.py
View File

@ -9,11 +9,13 @@
# assert that all semver components are integers
# this is mainly to show errors when people repackage poorly
# and setuptools_scm determines a 0.1.dev... version
assert all(isinstance(v, int) for v in __version_tuple__), \
assert all(isinstance(v, int) for v in __version_tuple__), (
"""\
broken borgbackup version metadata: %r
version metadata is obtained dynamically on installation via setuptools_scm,
please ensure your git repo has the correct tags or you provide the version
using SETUPTOOLS_SCM_PRETEND_VERSION in your build script.
""" % __version__
"""
% __version__
)

7
src/borg/__main__.py
View File

@ -5,11 +5,12 @@
# containing the dll is not in the search path. The dll is shipped
# with python in the "DLLs" folder, so let's add this folder
# to the path. The folder is always in sys.path, get it from there.
if sys.platform.startswith('win32'):
if sys.platform.startswith("win32"):
# Keep it an iterable to support multiple folder which contain "DLLs".
dll_path = (p for p in sys.path if 'DLLs' in os.path.normpath(p).split(os.path.sep))
os.environ['PATH'] = os.pathsep.join(dll_path) + os.pathsep + os.environ['PATH']
dll_path = (p for p in sys.path if "DLLs" in os.path.normpath(p).split(os.path.sep))
os.environ["PATH"] = os.pathsep.join(dll_path) + os.pathsep + os.environ["PATH"]
from borg.archiver import main
main()

816
src/borg/archive.py
View File

File diff suppressed because it is too large Load Diff

4193
src/borg/archiver.py
View File

File diff suppressed because it is too large Load Diff

533
src/borg/cache.py
View File

@ -10,7 +10,7 @@
logger = create_logger()
files_cache_logger = create_logger('borg.debug.files_cache')
files_cache_logger = create_logger("borg.debug.files_cache")
from .constants import CACHE_README, FILES_CACHE_MODE_DISABLED
from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
@ -37,7 +37,7 @@
from .repository import LIST_SCAN_LIMIT
# note: cmtime might me either a ctime or a mtime timestamp
FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size cmtime chunk_ids')
FileCacheEntry = namedtuple("FileCacheEntry", "age inode size cmtime chunk_ids")
class SecurityManager:
@ -64,9 +64,9 @@ def __init__(self, repository):
self.repository = repository
self.dir = get_security_dir(repository.id_str)
self.cache_dir = cache_dir(repository)
self.key_type_file = os.path.join(self.dir, 'key-type')
self.location_file = os.path.join(self.dir, 'location')
self.manifest_ts_file = os.path.join(self.dir, 'manifest-timestamp')
self.key_type_file = os.path.join(self.dir, "key-type")
self.location_file = os.path.join(self.dir, "location")
self.manifest_ts_file = os.path.join(self.dir, "manifest-timestamp")
@staticmethod
def destroy(repository, path=None):
@ -76,8 +76,7 @@ def destroy(repository, path=None):
shutil.rmtree(path)
def known(self):
return all(os.path.exists(f)
for f in (self.key_type_file, self.location_file, self.manifest_ts_file))
return all(os.path.exists(f) for f in (self.key_type_file, self.location_file, self.manifest_ts_file))
def key_matches(self, key):
if not self.known():
@ -87,14 +86,14 @@ def key_matches(self, key):
type = fd.read()
return type == str(key.TYPE)
except OSError as exc:
logger.warning('Could not read/parse key type file: %s', exc)
logger.warning("Could not read/parse key type file: %s", exc)
def save(self, manifest, key):
logger.debug('security: saving state for %s to %s', self.repository.id_str, self.dir)
logger.debug("security: saving state for %s to %s", self.repository.id_str, self.dir)
current_location = self.repository._location.canonical_path()
logger.debug('security: current location %s', current_location)
logger.debug('security: key type %s', str(key.TYPE))
logger.debug('security: manifest timestamp %s', manifest.timestamp)
logger.debug("security: current location %s", current_location)
logger.debug("security: key type %s", str(key.TYPE))
logger.debug("security: manifest timestamp %s", manifest.timestamp)
with SaveFile(self.location_file) as fd:
fd.write(current_location)
with SaveFile(self.key_type_file) as fd:
@ -107,28 +106,36 @@ def assert_location_matches(self, cache_config=None):
try:
with open(self.location_file) as fd:
previous_location = fd.read()
logger.debug('security: read previous location %r', previous_location)
logger.debug("security: read previous location %r", previous_location)
except FileNotFoundError:
logger.debug('security: previous location file %s not found', self.location_file)
logger.debug("security: previous location file %s not found", self.location_file)
previous_location = None
except OSError as exc:
logger.warning('Could not read previous location file: %s', exc)
logger.warning("Could not read previous location file: %s", exc)
previous_location = None
if cache_config and cache_config.previous_location and previous_location != cache_config.previous_location:
# Reconcile cache and security dir; we take the cache location.
previous_location = cache_config.previous_location
logger.debug('security: using previous_location of cache: %r', previous_location)
logger.debug("security: using previous_location of cache: %r", previous_location)
repository_location = self.repository._location.canonical_path()
if previous_location and previous_location != repository_location:
msg = ("Warning: The repository at location {} was previously located at {}\n".format(
repository_location, previous_location) +
"Do you want to continue? [yN] ")
if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
msg = (
"Warning: The repository at location {} was previously located at {}\n".format(
repository_location, previous_location
)
+ "Do you want to continue? [yN] "
)
if not yes(
msg,
false_msg="Aborting.",
invalid_msg="Invalid answer, aborting.",
retry=False,
env_var_override="BORG_RELOCATED_REPO_ACCESS_IS_OK",
):
raise Cache.RepositoryAccessAborted()
# adapt on-disk config immediately if the new location was accepted
logger.debug('security: updating location stored in cache and security dir')
logger.debug("security: updating location stored in cache and security dir")
with SaveFile(self.location_file) as fd:
fd.write(repository_location)
if cache_config:
@ -138,16 +145,16 @@ def assert_no_manifest_replay(self, manifest, key, cache_config=None):
try:
with open(self.manifest_ts_file) as fd:
timestamp = fd.read()
logger.debug('security: read manifest timestamp %r', timestamp)
logger.debug("security: read manifest timestamp %r", timestamp)
except FileNotFoundError:
logger.debug('security: manifest timestamp file %s not found', self.manifest_ts_file)
timestamp = ''
logger.debug("security: manifest timestamp file %s not found", self.manifest_ts_file)
timestamp = ""
except OSError as exc:
logger.warning('Could not read previous location file: %s', exc)
timestamp = ''
logger.warning("Could not read previous location file: %s", exc)
timestamp = ""
if cache_config:
timestamp = max(timestamp, cache_config.timestamp or '')
logger.debug('security: determined newest manifest timestamp as %s', timestamp)
timestamp = max(timestamp, cache_config.timestamp or "")
logger.debug("security: determined newest manifest timestamp as %s", timestamp)
# If repository is older than the cache or security dir something fishy is going on
if timestamp and timestamp > manifest.timestamp:
if isinstance(key, PlaintextKey):
@ -175,30 +182,36 @@ def assert_secure(self, manifest, key, *, cache_config=None, warn_if_unencrypted
self._assert_secure(manifest, key, cache_config)
else:
self._assert_secure(manifest, key)
logger.debug('security: repository checks ok, allowing access')
logger.debug("security: repository checks ok, allowing access")
def _assert_secure(self, manifest, key, cache_config=None):
self.assert_location_matches(cache_config)
self.assert_key_type(key, cache_config)
self.assert_no_manifest_replay(manifest, key, cache_config)
if not self.known():
logger.debug('security: remembering previously unknown repository')
logger.debug("security: remembering previously unknown repository")
self.save(manifest, key)
def assert_access_unknown(self, warn_if_unencrypted, manifest, key):
# warn_if_unencrypted=False is only used for initializing a new repository.
# Thus, avoiding asking about a repository that's currently initializing.
if not key.logically_encrypted and not self.known():
msg = ("Warning: Attempting to access a previously unknown unencrypted repository!\n" +
"Do you want to continue? [yN] ")
allow_access = not warn_if_unencrypted or yes(msg, false_msg="Aborting.",
msg = (
"Warning: Attempting to access a previously unknown unencrypted repository!\n"
+ "Do you want to continue? [yN] "
)
allow_access = not warn_if_unencrypted or yes(
msg,
false_msg="Aborting.",
invalid_msg="Invalid answer, aborting.",
retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK')
retry=False,
env_var_override="BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK",
)
if allow_access:
if warn_if_unencrypted:
logger.debug('security: remembering unknown unencrypted repository (explicitly allowed)')
logger.debug("security: remembering unknown unencrypted repository (explicitly allowed)")
else:
logger.debug('security: initializing unencrypted repository')
logger.debug("security: initializing unencrypted repository")
self.save(manifest, key)
else:
raise Cache.CacheInitAbortedError()
@ -214,10 +227,17 @@ def recanonicalize_relative_location(cache_location, repository):
repo_location = repository._location.canonical_path()
rl = Location(repo_location)
cl = Location(cache_location)
if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
and \
cl.path and rl.path and \
cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
if (
cl.proto == rl.proto
and cl.user == rl.user
and cl.host == rl.host
and cl.port == rl.port
and cl.path
and rl.path
and cl.path.startswith("/~/")
and rl.path.startswith("/./")
and cl.path[3:] == rl.path[3:]
):
# everything is same except the expected change in relative path canonicalization,
# update previous_location to avoid warning / user query about changed location:
return repo_location
@ -230,19 +250,19 @@ def cache_dir(repository, path=None):
def files_cache_name():
suffix = os.environ.get('BORG_FILES_CACHE_SUFFIX', '')
return 'files.' + suffix if suffix else 'files'
suffix = os.environ.get("BORG_FILES_CACHE_SUFFIX", "")
return "files." + suffix if suffix else "files"
def discover_files_cache_name(path):
return [fn for fn in os.listdir(path) if fn == 'files' or fn.startswith('files.')][0]
return [fn for fn in os.listdir(path) if fn == "files" or fn.startswith("files.")][0]
class CacheConfig:
def __init__(self, repository, path=None, lock_wait=None):
self.repository = repository
self.path = cache_dir(repository, path)
self.config_path = os.path.join(self.path, 'config')
self.config_path = os.path.join(self.path, "config")
self.lock = None
self.lock_wait = lock_wait
@ -259,17 +279,17 @@ def exists(self):
def create(self):
assert not self.exists()
config = configparser.ConfigParser(interpolation=None)
config.add_section('cache')
config.set('cache', 'version', '1')
config.set('cache', 'repository', self.repository.id_str)
config.set('cache', 'manifest', '')
config.add_section('integrity')
config.set('integrity', 'manifest', '')
config.add_section("cache")
config.set("cache", "version", "1")
config.set("cache", "repository", self.repository.id_str)
config.set("cache", "manifest", "")
config.add_section("integrity")
config.set("integrity", "manifest", "")
with SaveFile(self.config_path) as fd:
config.write(fd)
def open(self):
self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=self.lock_wait).acquire()
self.lock = Lock(os.path.join(self.path, "lock"), exclusive=True, timeout=self.lock_wait).acquire()
self.load()
def load(self):
@ -277,15 +297,17 @@ def load(self):
with open(self.config_path) as fd:
self._config.read_file(fd)
self._check_upgrade(self.config_path)
self.id = self._config.get('cache', 'repository')
self.manifest_id = unhexlify(self._config.get('cache', 'manifest'))
self.timestamp = self._config.get('cache', 'timestamp', fallback=None)
self.key_type = self._config.get('cache', 'key_type', fallback=None)
self.ignored_features = set(parse_stringified_list(self._config.get('cache', 'ignored_features', fallback='')))
self.mandatory_features = set(parse_stringified_list(self._config.get('cache', 'mandatory_features', fallback='')))
self.id = self._config.get("cache", "repository")
self.manifest_id = unhexlify(self._config.get("cache", "manifest"))
self.timestamp = self._config.get("cache", "timestamp", fallback=None)
self.key_type = self._config.get("cache", "key_type", fallback=None)
self.ignored_features = set(parse_stringified_list(self._config.get("cache", "ignored_features", fallback="")))
self.mandatory_features = set(
parse_stringified_list(self._config.get("cache", "mandatory_features", fallback=""))
)
try:
self.integrity = dict(self._config.items('integrity'))
if self._config.get('cache', 'manifest') != self.integrity.pop('manifest'):
self.integrity = dict(self._config.items("integrity"))
if self._config.get("cache", "manifest") != self.integrity.pop("manifest"):
# The cache config file is updated (parsed with ConfigParser, the state of the ConfigParser
# is modified and then written out.), not re-created.
# Thus, older versions will leave our [integrity] section alone, making the section's data invalid.
@ -293,30 +315,30 @@ def load(self):
# can discern whether an older version interfered by comparing the manifest IDs of this section
# and the main [cache] section.
self.integrity = {}
logger.warning('Cache integrity data not available: old Borg version modified the cache.')
logger.warning("Cache integrity data not available: old Borg version modified the cache.")
except configparser.NoSectionError:
logger.debug('Cache integrity: No integrity data found (files, chunks). Cache is from old version.')
logger.debug("Cache integrity: No integrity data found (files, chunks). Cache is from old version.")
self.integrity = {}
previous_location = self._config.get('cache', 'previous_location', fallback=None)
previous_location = self._config.get("cache", "previous_location", fallback=None)
if previous_location:
self.previous_location = recanonicalize_relative_location(previous_location, self.repository)
else:
self.previous_location = None
self._config.set('cache', 'previous_location', self.repository._location.canonical_path())
self._config.set("cache", "previous_location", self.repository._location.canonical_path())
def save(self, manifest=None, key=None):
if manifest:
self._config.set('cache', 'manifest', manifest.id_str)
self._config.set('cache', 'timestamp', manifest.timestamp)
self._config.set('cache', 'ignored_features', ','.join(self.ignored_features))
self._config.set('cache', 'mandatory_features', ','.join(self.mandatory_features))
if not self._config.has_section('integrity'):
self._config.add_section('integrity')
self._config.set("cache", "manifest", manifest.id_str)
self._config.set("cache", "timestamp", manifest.timestamp)
self._config.set("cache", "ignored_features", ",".join(self.ignored_features))
self._config.set("cache", "mandatory_features", ",".join(self.mandatory_features))
if not self._config.has_section("integrity"):
self._config.add_section("integrity")
for file, integrity_data in self.integrity.items():
self._config.set('integrity', file, integrity_data)
self._config.set('integrity', 'manifest', manifest.id_str)
self._config.set("integrity", file, integrity_data)
self._config.set("integrity", "manifest", manifest.id_str)
if key:
self._config.set('cache', 'key_type', str(key.TYPE))
self._config.set("cache", "key_type", str(key.TYPE))
with SaveFile(self.config_path) as fd:
self._config.write(fd)
@ -327,20 +349,21 @@ def close(self):
def _check_upgrade(self, config_path):
try:
cache_version = self._config.getint('cache', 'version')
cache_version = self._config.getint("cache", "version")
wanted_version = 1
if cache_version != wanted_version:
self.close()
raise Exception('%s has unexpected cache version %d (wanted: %d).' %
(config_path, cache_version, wanted_version))
raise Exception(
"%s has unexpected cache version %d (wanted: %d)." % (config_path, cache_version, wanted_version)
)
except configparser.NoSectionError:
self.close()
raise Exception('%s does not look like a Borg cache.' % config_path) from None
raise Exception("%s does not look like a Borg cache." % config_path) from None
class Cache:
"""Client Side cache
"""
"""Client Side cache"""
class RepositoryIDNotUnique(Error):
"""Cache is newer than repository - do you have multiple, independently updated repos with same ID?"""
@ -359,29 +382,56 @@ class EncryptionMethodMismatch(Error):
@staticmethod
def break_lock(repository, path=None):
path = cache_dir(repository, path)
Lock(os.path.join(path, 'lock'), exclusive=True).break_lock()
Lock(os.path.join(path, "lock"), exclusive=True).break_lock()
@staticmethod
def destroy(repository, path=None):
"""destroy the cache for ``repository`` or at ``path``"""
path = path or os.path.join(get_cache_dir(), repository.id_str)
config = os.path.join(path, 'config')
config = os.path.join(path, "config")
if os.path.exists(config):
os.remove(config) # kill config first
shutil.rmtree(path)
def __new__(cls, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=FILES_CACHE_MODE_DISABLED,
consider_part_files=False, iec=False):
def __new__(
cls,
repository,
key,
manifest,
path=None,
sync=True,
warn_if_unencrypted=True,
progress=False,
lock_wait=None,
permit_adhoc_cache=False,
cache_mode=FILES_CACHE_MODE_DISABLED,
consider_part_files=False,
iec=False,
):
def local():
return LocalCache(repository=repository, key=key, manifest=manifest, path=path, sync=sync,
warn_if_unencrypted=warn_if_unencrypted, progress=progress, iec=iec,
lock_wait=lock_wait, cache_mode=cache_mode, consider_part_files=consider_part_files)
return LocalCache(
repository=repository,
key=key,
manifest=manifest,
path=path,
sync=sync,
warn_if_unencrypted=warn_if_unencrypted,
progress=progress,
iec=iec,
lock_wait=lock_wait,
cache_mode=cache_mode,
consider_part_files=consider_part_files,
)
def adhoc():
return AdHocCache(repository=repository, key=key, manifest=manifest, lock_wait=lock_wait, iec=iec,
consider_part_files=consider_part_files)
return AdHocCache(
repository=repository,
key=key,
manifest=manifest,
lock_wait=lock_wait,
iec=iec,
consider_part_files=consider_part_files,
)
if not permit_adhoc_cache:
return local()
@ -397,9 +447,9 @@ def adhoc():
# Don't nest cache locks
if cache_in_sync:
# Local cache is in sync, use it
logger.debug('Cache: choosing local cache (in sync)')
logger.debug("Cache: choosing local cache (in sync)")
return local()
logger.debug('Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)')
logger.debug("Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)")
return adhoc()
@ -417,10 +467,11 @@ def __init__(self, iec=False):
def __str__(self):
return self.str_format.format(self.format_tuple())
Summary = namedtuple('Summary', ['total_size', 'unique_size', 'total_unique_chunks', 'total_chunks'])
Summary = namedtuple("Summary", ["total_size", "unique_size", "total_unique_chunks", "total_chunks"])
def stats(self):
from .archive import Archive
# XXX: this should really be moved down to `hashindex.pyx`
total_size, unique_size, total_unique_chunks, total_chunks = self.chunks.summarize()
# the above values have the problem that they do not consider part files,
@ -430,8 +481,9 @@ def stats(self):
# so we can just sum up all archives to get the "all archives" stats:
total_size = 0
for archive_name in self.manifest.archives:
archive = Archive(self.repository, self.key, self.manifest, archive_name,
consider_part_files=self.consider_part_files)
archive = Archive(
self.repository, self.key, self.manifest, archive_name, consider_part_files=self.consider_part_files
)
stats = archive.calc_stats(self, want_unique=False)
total_size += stats.osize
stats = self.Summary(total_size, unique_size, total_unique_chunks, total_chunks)._asdict()
@ -439,7 +491,7 @@ def stats(self):
def format_tuple(self):
stats = self.stats()
for field in ['total_size', 'unique_size']:
for field in ["total_size", "unique_size"]:
stats[field] = format_file_size(stats[field], iec=self.iec)
return self.Summary(**stats)
@ -449,9 +501,20 @@ class LocalCache(CacheStatsMixin):
Persistent, local (client-side) cache.
"""
def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
progress=False, lock_wait=None, cache_mode=FILES_CACHE_MODE_DISABLED, consider_part_files=False,
iec=False):
def __init__(
self,
repository,
key,
manifest,
path=None,
sync=True,
warn_if_unencrypted=True,
progress=False,
lock_wait=None,
cache_mode=FILES_CACHE_MODE_DISABLED,
consider_part_files=False,
iec=False,
):
"""
:param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
:param lock_wait: timeout for lock acquisition (int [s] or None [wait forever])
@ -500,30 +563,32 @@ def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def create(self):
"""Create a new empty cache at `self.path`
"""
"""Create a new empty cache at `self.path`"""
os.makedirs(self.path)
with open(os.path.join(self.path, 'README'), 'w') as fd:
with open(os.path.join(self.path, "README"), "w") as fd:
fd.write(CACHE_README)
self.cache_config.create()
ChunkIndex().write(os.path.join(self.path, 'chunks'))
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
ChunkIndex().write(os.path.join(self.path, "chunks"))
os.makedirs(os.path.join(self.path, "chunks.archive.d"))
with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
pass # empty file
def _do_open(self):
self.cache_config.load()
with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=False,
integrity_data=self.cache_config.integrity.get('chunks')) as fd:
with IntegrityCheckedFile(
path=os.path.join(self.path, "chunks"),
write=False,
integrity_data=self.cache_config.integrity.get("chunks"),
) as fd:
self.chunks = ChunkIndex.read(fd)
if 'd' in self.cache_mode: # d(isabled)
if "d" in self.cache_mode: # d(isabled)
self.files = None
else:
self._read_files()
def open(self):
if not os.path.isdir(self.path):
raise Exception('%s Does not look like a Borg cache' % self.path)
raise Exception("%s Does not look like a Borg cache" % self.path)
self.cache_config.open()
self.rollback()
@ -535,12 +600,15 @@ def close(self):
def _read_files(self):
self.files = {}
self._newest_cmtime = None
logger.debug('Reading files cache ...')
logger.debug("Reading files cache ...")
files_cache_logger.debug("FILES-CACHE-LOAD: starting...")
msg = None
try:
with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=False,
integrity_data=self.cache_config.integrity.get(files_cache_name())) as fd:
with IntegrityCheckedFile(
path=os.path.join(self.path, files_cache_name()),
write=False,
integrity_data=self.cache_config.integrity.get(files_cache_name()),
) as fd:
u = msgpack.Unpacker(use_list=True)
while True:
data = fd.read(64 * 1024)
@ -561,43 +629,41 @@ def _read_files(self):
msg = "The files cache is corrupted. [%s]" % str(fie)
if msg is not None:
logger.warning(msg)
logger.warning('Continuing without files cache - expect lower performance.')
logger.warning("Continuing without files cache - expect lower performance.")
self.files = {}
files_cache_logger.debug("FILES-CACHE-LOAD: finished, %d entries loaded.", len(self.files))
def begin_txn(self):
# Initialize transaction snapshot
pi = ProgressIndicatorMessage(msgid='cache.begin_transaction')
txn_dir = os.path.join(self.path, 'txn.tmp')
pi = ProgressIndicatorMessage(msgid="cache.begin_transaction")
txn_dir = os.path.join(self.path, "txn.tmp")
os.mkdir(txn_dir)
pi.output('Initializing cache transaction: Reading config')
shutil.copy(os.path.join(self.path, 'config'), txn_dir)
pi.output('Initializing cache transaction: Reading chunks')
shutil.copy(os.path.join(self.path, 'chunks'), txn_dir)
pi.output('Initializing cache transaction: Reading files')
pi.output("Initializing cache transaction: Reading config")
shutil.copy(os.path.join(self.path, "config"), txn_dir)
pi.output("Initializing cache transaction: Reading chunks")
shutil.copy(os.path.join(self.path, "chunks"), txn_dir)
pi.output("Initializing cache transaction: Reading files")
try:
shutil.copy(os.path.join(self.path, files_cache_name()), txn_dir)
except FileNotFoundError:
with SaveFile(os.path.join(txn_dir, files_cache_name()), binary=True):
pass # empty file
os.rename(os.path.join(self.path, 'txn.tmp'),
os.path.join(self.path, 'txn.active'))
os.rename(os.path.join(self.path, "txn.tmp"), os.path.join(self.path, "txn.active"))
self.txn_active = True
pi.finish()
def commit(self):
"""Commit transaction
"""
"""Commit transaction"""
if not self.txn_active:
return
self.security_manager.save(self.manifest, self.key)
pi = ProgressIndicatorMessage(msgid='cache.commit')
pi = ProgressIndicatorMessage(msgid="cache.commit")
if self.files is not None:
if self._newest_cmtime is None:
# was never set because no files were modified/added
self._newest_cmtime = 2 ** 63 - 1 # nanoseconds, good until y2262
ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
pi.output('Saving files cache')
self._newest_cmtime = 2**63 - 1 # nanoseconds, good until y2262
ttl = int(os.environ.get("BORG_FILES_CACHE_TTL", 20))
pi.output("Saving files cache")
files_cache_logger.debug("FILES-CACHE-SAVE: starting...")
with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=True) as fd:
entry_count = 0
@ -606,41 +672,45 @@ def commit(self):
# this is to avoid issues with filesystem snapshots and cmtime granularity.
# Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
entry = FileCacheEntry(*msgpack.unpackb(item))
if entry.age == 0 and timestamp_to_int(entry.cmtime) < self._newest_cmtime or \
entry.age > 0 and entry.age < ttl:
if (
entry.age == 0
and timestamp_to_int(entry.cmtime) < self._newest_cmtime
or entry.age > 0
and entry.age < ttl
):
msgpack.pack((path_hash, entry), fd)
entry_count += 1
files_cache_logger.debug("FILES-CACHE-KILL: removed all old entries with age >= TTL [%d]", ttl)
files_cache_logger.debug("FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime)
files_cache_logger.debug(
"FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime
)
files_cache_logger.debug("FILES-CACHE-SAVE: finished, %d remaining entries saved.", entry_count)
self.cache_config.integrity[files_cache_name()] = fd.integrity_data
pi.output('Saving chunks cache')
with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=True) as fd:
pi.output("Saving chunks cache")
with IntegrityCheckedFile(path=os.path.join(self.path, "chunks"), write=True) as fd:
self.chunks.write(fd)
self.cache_config.integrity['chunks'] = fd.integrity_data
pi.output('Saving cache config')
self.cache_config.integrity["chunks"] = fd.integrity_data
pi.output("Saving cache config")
self.cache_config.save(self.manifest, self.key)
os.rename(os.path.join(self.path, 'txn.active'),
os.path.join(self.path, 'txn.tmp'))
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
os.rename(os.path.join(self.path, "txn.active"), os.path.join(self.path, "txn.tmp"))
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
self.txn_active = False
pi.finish()
def rollback(self):
"""Roll back partial and aborted transactions
"""
"""Roll back partial and aborted transactions"""
# Remove partial transaction
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
if os.path.exists(os.path.join(self.path, "txn.tmp")):
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
# Roll back active transaction
txn_dir = os.path.join(self.path, 'txn.active')
txn_dir = os.path.join(self.path, "txn.active")
if os.path.exists(txn_dir):
shutil.copy(os.path.join(txn_dir, 'config'), self.path)
shutil.copy(os.path.join(txn_dir, 'chunks'), self.path)
shutil.copy(os.path.join(txn_dir, "config"), self.path)
shutil.copy(os.path.join(txn_dir, "chunks"), self.path)
shutil.copy(os.path.join(txn_dir, discover_files_cache_name(txn_dir)), self.path)
os.rename(txn_dir, os.path.join(self.path, 'txn.tmp'))
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
os.rename(txn_dir, os.path.join(self.path, "txn.tmp"))
if os.path.exists(os.path.join(self.path, "txn.tmp")):
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
self.txn_active = False
self._do_open()
@ -654,13 +724,13 @@ def sync(self):
get removed and a new master chunks index is built by merging all
archive indexes.
"""
archive_path = os.path.join(self.path, 'chunks.archive.d')
archive_path = os.path.join(self.path, "chunks.archive.d")
# Instrumentation
processed_item_metadata_bytes = 0
processed_item_metadata_chunks = 0
compact_chunks_archive_saved_space = 0
def mkpath(id, suffix=''):
def mkpath(id, suffix=""):
id_hex = bin_to_hex(id)
path = os.path.join(archive_path, id_hex + suffix)
return path
@ -670,8 +740,9 @@ def cached_archives():
fns = os.listdir(archive_path)
# filenames with 64 hex digits == 256bit,
# or compact indices which are 64 hex digits + ".compact"
return {unhexlify(fn) for fn in fns if len(fn) == 64} | \
{unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith('.compact')}
return {unhexlify(fn) for fn in fns if len(fn) == 64} | {
unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith(".compact")
}
else:
return set()
@ -685,14 +756,14 @@ def cleanup_outdated(ids):
def cleanup_cached_archive(id, cleanup_compact=True):
try:
os.unlink(mkpath(id))
os.unlink(mkpath(id) + '.integrity')
os.unlink(mkpath(id) + ".integrity")
except FileNotFoundError:
pass
if not cleanup_compact:
return
try:
os.unlink(mkpath(id, suffix='.compact'))
os.unlink(mkpath(id, suffix='.compact') + '.integrity')
os.unlink(mkpath(id, suffix=".compact"))
os.unlink(mkpath(id, suffix=".compact") + ".integrity")
except FileNotFoundError:
pass
@ -703,7 +774,7 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx):
chunk_idx.add(archive_id, 1, len(data))
archive = ArchiveItem(internal_dict=msgpack.unpackb(data))
if archive.version not in (1, 2): # legacy
raise Exception('Unknown archive metadata version')
raise Exception("Unknown archive metadata version")
sync = CacheSynchronizer(chunk_idx)
for item_id, (csize, data) in zip(archive.items, decrypted_repository.get_many(archive.items)):
chunk_idx.add(item_id, 1, len(data))
@ -716,11 +787,12 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx):
def write_archive_index(archive_id, chunk_idx):
nonlocal compact_chunks_archive_saved_space
compact_chunks_archive_saved_space += chunk_idx.compact()
fn = mkpath(archive_id, suffix='.compact')
fn_tmp = mkpath(archive_id, suffix='.tmp')
fn = mkpath(archive_id, suffix=".compact")
fn_tmp = mkpath(archive_id, suffix=".tmp")
try:
with DetachedIntegrityCheckedFile(path=fn_tmp, write=True,
filename=bin_to_hex(archive_id) + '.compact') as fd:
with DetachedIntegrityCheckedFile(
path=fn_tmp, write=True, filename=bin_to_hex(archive_id) + ".compact"
) as fd:
chunk_idx.write(fd)
except Exception:
safe_unlink(fn_tmp)
@ -733,7 +805,7 @@ def read_archive_index(archive_id, archive_name):
try:
try:
# Attempt to load compact index first
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + '.compact', write=False) as fd:
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + ".compact", write=False) as fd:
archive_chunk_idx = ChunkIndex.read(fd, permit_compact=True)
# In case a non-compact index exists, delete it.
cleanup_cached_archive(archive_id, cleanup_compact=False)
@ -744,14 +816,14 @@ def read_archive_index(archive_id, archive_name):
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
archive_chunk_idx = ChunkIndex.read(fd)
except FileIntegrityError as fie:
logger.error('Cached archive chunk index of %s is corrupted: %s', archive_name, fie)
logger.error("Cached archive chunk index of %s is corrupted: %s", archive_name, fie)
# Delete corrupted index, set warning. A new index must be build.
cleanup_cached_archive(archive_id)
set_ec(EXIT_WARNING)
return None
# Convert to compact index. Delete the existing index first.
logger.debug('Found non-compact index for %s, converting to compact.', archive_name)
logger.debug("Found non-compact index for %s, converting to compact.", archive_name)
cleanup_cached_archive(archive_id)
write_archive_index(archive_id, archive_chunk_idx)
return archive_chunk_idx
@ -769,12 +841,16 @@ def get_archive_ids_to_names(archive_ids):
return archive_names
def create_master_idx(chunk_idx):
logger.info('Synchronizing chunks cache...')
logger.info("Synchronizing chunks cache...")
cached_ids = cached_archives()
archive_ids = repo_archives()
logger.info('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.',
len(archive_ids), len(cached_ids),
len(cached_ids - archive_ids), len(archive_ids - cached_ids))
logger.info(
"Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.",
len(archive_ids),
len(cached_ids),
len(cached_ids - archive_ids),
len(archive_ids - cached_ids),
)
# deallocates old hashindex, creates empty hashindex:
chunk_idx.clear()
cleanup_outdated(cached_ids - archive_ids)
@ -783,9 +859,12 @@ def create_master_idx(chunk_idx):
master_index_capacity = len(self.repository)
if archive_ids:
chunk_idx = None if not self.do_cache else ChunkIndex(usable=master_index_capacity)
pi = ProgressIndicatorPercent(total=len(archive_ids), step=0.1,
msg='%3.0f%% Syncing chunks cache. Processing archive %s',
msgid='cache.sync')
pi = ProgressIndicatorPercent(
total=len(archive_ids),
step=0.1,
msg="%3.0f%% Syncing chunks cache. Processing archive %s",
msgid="cache.sync",
)
archive_ids_to_names = get_archive_ids_to_names(archive_ids)
for archive_id, archive_name in archive_ids_to_names.items():
pi.show(info=[remove_surrogates(archive_name)])
@ -797,31 +876,36 @@ def create_master_idx(chunk_idx):
if archive_id not in cached_ids:
# Do not make this an else branch; the FileIntegrityError exception handler
# above can remove *archive_id* from *cached_ids*.
logger.info('Fetching and building archive index for %s ...', archive_name)
logger.info("Fetching and building archive index for %s ...", archive_name)
archive_chunk_idx = ChunkIndex()
fetch_and_build_idx(archive_id, decrypted_repository, archive_chunk_idx)
logger.info("Merging into master chunks index ...")
chunk_idx.merge(archive_chunk_idx)
else:
chunk_idx = chunk_idx or ChunkIndex(usable=master_index_capacity)
logger.info('Fetching archive index for %s ...', archive_name)
logger.info("Fetching archive index for %s ...", archive_name)
fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx)
pi.finish()
logger.debug('Cache sync: processed %s (%d chunks) of metadata',
format_file_size(processed_item_metadata_bytes), processed_item_metadata_chunks)
logger.debug('Cache sync: compact chunks.archive.d storage saved %s bytes',
format_file_size(compact_chunks_archive_saved_space))
logger.info('Done.')
logger.debug(
"Cache sync: processed %s (%d chunks) of metadata",
format_file_size(processed_item_metadata_bytes),
processed_item_metadata_chunks,
)
logger.debug(
"Cache sync: compact chunks.archive.d storage saved %s bytes",
format_file_size(compact_chunks_archive_saved_space),
)
logger.info("Done.")
return chunk_idx
def legacy_cleanup():
"""bring old cache dirs into the desired state (cleanup and adapt)"""
try:
os.unlink(os.path.join(self.path, 'chunks.archive'))
os.unlink(os.path.join(self.path, "chunks.archive"))
except:
pass
try:
os.unlink(os.path.join(self.path, 'chunks.archive.tmp'))
os.unlink(os.path.join(self.path, "chunks.archive.tmp"))
except:
pass
try:
@ -832,7 +916,7 @@ def legacy_cleanup():
# The cache can be used by a command that e.g. only checks against Manifest.Operation.WRITE,
# which does not have to include all flags from Manifest.Operation.READ.
# Since the sync will attempt to read archives, check compatibility with Manifest.Operation.READ.
self.manifest.check_repository_compatibility((Manifest.Operation.READ, ))
self.manifest.check_repository_compatibility((Manifest.Operation.READ,))
self.begin_txn()
with cache_if_remote(self.repository, decrypted_cache=self.key) as decrypted_repository:
@ -856,15 +940,15 @@ def check_cache_compatibility(self):
def wipe_cache(self):
logger.warning("Discarding incompatible cache and forcing a cache rebuild")
archive_path = os.path.join(self.path, 'chunks.archive.d')
archive_path = os.path.join(self.path, "chunks.archive.d")
if os.path.isdir(archive_path):
shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
shutil.rmtree(os.path.join(self.path, "chunks.archive.d"))
os.makedirs(os.path.join(self.path, "chunks.archive.d"))
self.chunks = ChunkIndex()
with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
pass # empty file
self.cache_config.manifest_id = ''
self.cache_config._config.set('cache', 'manifest', '')
self.cache_config.manifest_id = ""
self.cache_config._config.set("cache", "manifest", "")
self.cache_config.ignored_features = set()
self.cache_config.mandatory_features = set()
@ -900,8 +984,9 @@ def seen_chunk(self, id, size=None):
if size is not None and stored_size is not None and size != stored_size:
# we already have a chunk with that id, but different size.
# this is either a hash collision (unlikely) or corruption or a bug.
raise Exception("chunk has same id [%r], but different size (stored: %d new: %d)!" % (
id, stored_size, size))
raise Exception(
"chunk has same id [%r], but different size (stored: %d new: %d)!" % (id, stored_size, size)
)
return refcount
def chunk_incref(self, id, stats, size=None, part=False):
@ -936,32 +1021,32 @@ def file_known_and_unchanged(self, hashed_path, path_hash, st):
if not stat.S_ISREG(st.st_mode):
return False, None
cache_mode = self.cache_mode
if 'd' in cache_mode: # d(isabled)
files_cache_logger.debug('UNKNOWN: files cache disabled')
if "d" in cache_mode: # d(isabled)
files_cache_logger.debug("UNKNOWN: files cache disabled")
return False, None
# note: r(echunk) does not need the files cache in this method, but the files cache will
# be updated and saved to disk to memorize the files. To preserve previous generations in
# the cache, this means that it also needs to get loaded from disk first.
if 'r' in cache_mode: # r(echunk)
files_cache_logger.debug('UNKNOWN: rechunking enforced')
if "r" in cache_mode: # r(echunk)
files_cache_logger.debug("UNKNOWN: rechunking enforced")
return False, None
entry = self.files.get(path_hash)
if not entry:
files_cache_logger.debug('UNKNOWN: no file metadata in cache for: %r', hashed_path)
files_cache_logger.debug("UNKNOWN: no file metadata in cache for: %r", hashed_path)
return False, None
# we know the file!
entry = FileCacheEntry(*msgpack.unpackb(entry))
if 's' in cache_mode and entry.size != st.st_size:
files_cache_logger.debug('KNOWN-CHANGED: file size has changed: %r', hashed_path)
if "s" in cache_mode and entry.size != st.st_size:
files_cache_logger.debug("KNOWN-CHANGED: file size has changed: %r", hashed_path)
return True, None
if 'i' in cache_mode and entry.inode != st.st_ino:
files_cache_logger.debug('KNOWN-CHANGED: file inode number has changed: %r', hashed_path)
if "i" in cache_mode and entry.inode != st.st_ino:
files_cache_logger.debug("KNOWN-CHANGED: file inode number has changed: %r", hashed_path)
return True, None
if 'c' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns:
files_cache_logger.debug('KNOWN-CHANGED: file ctime has changed: %r', hashed_path)
if "c" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns:
files_cache_logger.debug("KNOWN-CHANGED: file ctime has changed: %r", hashed_path)
return True, None
elif 'm' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns:
files_cache_logger.debug('KNOWN-CHANGED: file mtime has changed: %r', hashed_path)
elif "m" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns:
files_cache_logger.debug("KNOWN-CHANGED: file mtime has changed: %r", hashed_path)
return True, None
# we ignored the inode number in the comparison above or it is still same.
# if it is still the same, replacing it in the tuple doesn't change it.
@ -979,21 +1064,26 @@ def memorize_file(self, hashed_path, path_hash, st, ids):
return
cache_mode = self.cache_mode
# note: r(echunk) modes will update the files cache, d(isabled) mode won't
if 'd' in cache_mode:
files_cache_logger.debug('FILES-CACHE-NOUPDATE: files cache disabled')
if "d" in cache_mode:
files_cache_logger.debug("FILES-CACHE-NOUPDATE: files cache disabled")
return
if 'c' in cache_mode:
cmtime_type = 'ctime'
if "c" in cache_mode:
cmtime_type = "ctime"
cmtime_ns = safe_ns(st.st_ctime_ns)
elif 'm' in cache_mode:
cmtime_type = 'mtime'
elif "m" in cache_mode:
cmtime_type = "mtime"
cmtime_ns = safe_ns(st.st_mtime_ns)
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids)
entry = FileCacheEntry(
age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids
)
self.files[path_hash] = msgpack.packb(entry)
self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns)
files_cache_logger.debug('FILES-CACHE-UPDATE: put %r [has %s] <- %r',
entry._replace(chunk_ids='[%d entries]' % len(entry.chunk_ids)),
cmtime_type, hashed_path)
files_cache_logger.debug(
"FILES-CACHE-UPDATE: put %r [has %s] <- %r",
entry._replace(chunk_ids="[%d entries]" % len(entry.chunk_ids)),
cmtime_type,
hashed_path,
)
class AdHocCache(CacheStatsMixin):
@ -1012,8 +1102,9 @@ class AdHocCache(CacheStatsMixin):
Unique chunks Total chunks
Chunk index: {0.total_unique_chunks:20d} unknown"""
def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False,
iec=False):
def __init__(
self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False, iec=False
):
CacheStatsMixin.__init__(self, iec=iec)
self.repository = repository
self.key = key
@ -1024,7 +1115,7 @@ def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wai
self.security_manager = SecurityManager(repository)
self.security_manager.assert_secure(manifest, key, lock_wait=lock_wait)
logger.warning('Note: --no-cache-sync is an experimental feature.')
logger.warning("Note: --no-cache-sync is an experimental feature.")
# Public API
@ -1035,7 +1126,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
pass
files = None
cache_mode = 'd'
cache_mode = "d"
def file_known_and_unchanged(self, hashed_path, path_hash, st):
files_cache_logger.debug("UNKNOWN: files cache not implemented")
@ -1045,7 +1136,7 @@ def memorize_file(self, hashed_path, path_hash, st, ids):
pass
def add_chunk(self, id, chunk, stats, *, overwrite=False, wait=True, compress=True, size=None):
assert not overwrite, 'AdHocCache does not permit overwrites — trying to use it for recreate?'
assert not overwrite, "AdHocCache does not permit overwrites — trying to use it for recreate?"
if not self._txn_active:
self.begin_txn()
if size is None and compress:
@ -1111,8 +1202,9 @@ def begin_txn(self):
# Since we're creating an archive, add 10 % from the start.
num_chunks = len(self.repository)
self.chunks = ChunkIndex(usable=num_chunks * 1.1)
pi = ProgressIndicatorPercent(total=num_chunks, msg='Downloading chunk list... %3.0f%%',
msgid='cache.download_chunks')
pi = ProgressIndicatorPercent(
total=num_chunks, msg="Downloading chunk list... %3.0f%%", msgid="cache.download_chunks"
)
t0 = perf_counter()
num_requests = 0
marker = None
@ -1134,7 +1226,12 @@ def begin_txn(self):
del self.chunks[self.manifest.MANIFEST_ID]
duration = perf_counter() - t0 or 0.01
pi.finish()
logger.debug('AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s',
num_chunks, duration, num_requests, format_file_size(num_chunks * 34 / duration))
logger.debug(
"AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s",
num_chunks,
duration,
num_requests,
format_file_size(num_chunks * 34 / duration),
)
# Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes.
# Protocol overhead is neglected in this calculation.

38
src/borg/constants.py
View File

@ -7,7 +7,7 @@
# fmt: on
# this is the set of keys that are always present in items:
REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ])
REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"])
# this set must be kept complete, otherwise rebuild_manifest might malfunction:
# fmt: off
@ -19,7 +19,7 @@
# fmt: on
# this is the set of keys that are always present in archives:
REQUIRED_ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'time', ])
REQUIRED_ARCHIVE_KEYS = frozenset(["version", "name", "items", "cmdline", "time"])
# default umask, overridden by --umask, defaults to read/write only for owner
UMASK_DEFAULT = 0o077
@ -28,8 +28,8 @@
# forcing to 0o100XXX later
STDIN_MODE_DEFAULT = 0o660
CACHE_TAG_NAME = 'CACHEDIR.TAG'
CACHE_TAG_CONTENTS = b'Signature: 8a477f597d28d172789f06886806bc55'
CACHE_TAG_NAME = "CACHEDIR.TAG"
CACHE_TAG_CONTENTS = b"Signature: 8a477f597d28d172789f06886806bc55"
# A large, but not unreasonably large segment size. Always less than 2 GiB (for legacy file systems). We choose
# 500 MiB which means that no indirection from the inode is needed for typical Linux file systems.
@ -48,7 +48,7 @@
MAX_OBJECT_SIZE = MAX_DATA_SIZE + 41 + 8 # see assertion at end of repository module
# repo config max_segment_size value must be below this limit to stay within uint32 offsets:
MAX_SEGMENT_SIZE_LIMIT = 2 ** 32 - MAX_OBJECT_SIZE
MAX_SEGMENT_SIZE_LIMIT = 2**32 - MAX_OBJECT_SIZE
# have one all-zero bytes object
# we use it at all places where we need to detect or create all-zero buffers
@ -71,12 +71,12 @@
CHUNK_MIN_EXP = 19 # 2**19 == 512kiB
CHUNK_MAX_EXP = 23 # 2**23 == 8MiB
HASH_WINDOW_SIZE = 0xfff # 4095B
HASH_WINDOW_SIZE = 0xFFF # 4095B
HASH_MASK_BITS = 21 # results in ~2MiB chunks statistically
# chunker algorithms
CH_BUZHASH = 'buzhash'
CH_FIXED = 'fixed'
CH_BUZHASH = "buzhash"
CH_FIXED = "fixed"
# defaults, use --chunker-params to override
CHUNKER_PARAMS = (CH_BUZHASH, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)
@ -88,8 +88,8 @@
CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2
# operating mode of the files cache (for fast skipping of unchanged files)
FILES_CACHE_MODE_UI_DEFAULT = 'ctime,size,inode' # default for "borg create" command (CLI UI)
FILES_CACHE_MODE_DISABLED = 'd' # most borg commands do not use the files cache at all (disable)
FILES_CACHE_MODE_UI_DEFAULT = "ctime,size,inode" # default for "borg create" command (CLI UI)
FILES_CACHE_MODE_DISABLED = "d" # most borg commands do not use the files cache at all (disable)
# return codes returned by borg command
# when borg is killed by signal N, rc = 128 + N
@ -101,30 +101,30 @@
# never use datetime.isoformat(), it is evil. always use one of these:
# datetime.strftime(ISO_FORMAT) # output always includes .microseconds
# datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds
ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S'
ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f'
ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S"
ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f"
DASHES = '-' * 78
DASHES = "-" * 78
PBKDF2_ITERATIONS = 100000
# https://www.rfc-editor.org/rfc/rfc9106.html#section-4-6.2
ARGON2_ARGS = {'time_cost': 3, 'memory_cost': 2**16, 'parallelism': 4, 'type': 'id'}
ARGON2_ARGS = {"time_cost": 3, "memory_cost": 2**16, "parallelism": 4, "type": "id"}
ARGON2_SALT_BYTES = 16
# Maps the CLI argument to our internal identifier for the format
KEY_ALGORITHMS = {
# encrypt-and-MAC, kdf: PBKDF2(HMAC−SHA256), encryption: AES256-CTR, authentication: HMAC-SHA256
'pbkdf2': 'sha256',
"pbkdf2": "sha256",
# encrypt-then-MAC, kdf: argon2, encryption: chacha20, authentication: poly1305
'argon2': 'argon2 chacha20-poly1305',
"argon2": "argon2 chacha20-poly1305",
}
class KeyBlobStorage:
NO_STORAGE = 'no_storage'
KEYFILE = 'keyfile'
REPO = 'repository'
NO_STORAGE = "no_storage"
KEYFILE = "keyfile"
REPO = "repository"
class KeyType:

41
src/borg/crypto/file_integrity.py
View File

@ -102,12 +102,12 @@ def hash_length(self, seek_to_end=False):
class SHA512FileHashingWrapper(FileHashingWrapper):
ALGORITHM = 'SHA512'
ALGORITHM = "SHA512"
FACTORY = hashlib.sha512
class XXH64FileHashingWrapper(FileHashingWrapper):
ALGORITHM = 'XXH64'
ALGORITHM = "XXH64"
FACTORY = StreamingXXH64
@ -125,7 +125,7 @@ class IntegrityCheckedFile(FileLikeWrapper):
def __init__(self, path, write, filename=None, override_fd=None, integrity_data=None):
self.path = path
self.writing = write
mode = 'wb' if write else 'rb'
mode = "wb" if write else "rb"
self.file_fd = override_fd or open(path, mode)
self.digests = {}
@ -155,7 +155,7 @@ def hash_filename(self, filename=None):
# While Borg does not use anything except ASCII in these file names, it's important to use
# the same encoding everywhere for portability. Using os.fsencode() would be wrong.
filename = os.path.basename(filename or self.path)
self.hasher.update(('%10d' % len(filename)).encode())
self.hasher.update(("%10d" % len(filename)).encode())
self.hasher.update(filename.encode())
@classmethod
@ -163,44 +163,41 @@ def parse_integrity_data(cls, path: str, data: str):
try:
integrity_data = json.loads(data)
# Provisions for agility now, implementation later, but make sure the on-disk joint is oiled.
algorithm = integrity_data['algorithm']
algorithm = integrity_data["algorithm"]
if algorithm not in SUPPORTED_ALGORITHMS:
logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm)
logger.warning("Cannot verify integrity of %s: Unknown algorithm %r", path, algorithm)
return
digests = integrity_data['digests']
digests = integrity_data["digests"]
# Require at least presence of the final digest
digests['final']
digests["final"]
return algorithm, digests
except (ValueError, TypeError, KeyError) as e:
logger.warning('Could not parse integrity data for %s: %s', path, e)
logger.warning("Could not parse integrity data for %s: %s", path, e)
raise FileIntegrityError(path)
def hash_part(self, partname, is_final=False):
if not self.writing and not self.digests:
return
self.hasher.update(('%10d' % len(partname)).encode())
self.hasher.update(("%10d" % len(partname)).encode())
self.hasher.update(partname.encode())
self.hasher.hash_length(seek_to_end=is_final)
digest = self.hasher.hexdigest()
if self.writing:
self.digests[partname] = digest
elif self.digests and not compare_digest(self.digests.get(partname, ''), digest):
elif self.digests and not compare_digest(self.digests.get(partname, ""), digest):
raise FileIntegrityError(self.path)
def __exit__(self, exc_type, exc_val, exc_tb):
exception = exc_type is not None
if not exception:
self.hash_part('final', is_final=True)
self.hash_part("final", is_final=True)
self.hasher.__exit__(exc_type, exc_val, exc_tb)
if exception:
return
if self.writing:
self.store_integrity_data(json.dumps({
'algorithm': self.hasher.ALGORITHM,
'digests': self.digests,
}))
self.store_integrity_data(json.dumps({"algorithm": self.hasher.ALGORITHM, "digests": self.digests}))
elif self.digests:
logger.debug('Verified integrity of %s', self.path)
logger.debug("Verified integrity of %s", self.path)
def store_integrity_data(self, data: str):
self.integrity_data = data
@ -214,12 +211,12 @@ def __init__(self, path, write, filename=None, override_fd=None):
self.output_integrity_file = self.integrity_file_path(os.path.join(output_dir, filename))
def load_integrity_data(self, path, integrity_data):
assert not integrity_data, 'Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile'
assert not integrity_data, "Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile"
return self.read_integrity_file(self.path)
@staticmethod
def integrity_file_path(path):
return path + '.integrity'
return path + ".integrity"
@classmethod
def read_integrity_file(cls, path):
@ -227,11 +224,11 @@ def read_integrity_file(cls, path):
with open(cls.integrity_file_path(path)) as fd:
return cls.parse_integrity_data(path, fd.read())
except FileNotFoundError:
logger.info('No integrity file found for %s', path)
logger.info("No integrity file found for %s", path)
except OSError as e:
logger.warning('Could not read integrity file for %s: %s', path, e)
logger.warning("Could not read integrity file for %s: %s", path, e)
raise FileIntegrityError(path)
def store_integrity_data(self, data: str):
with open(self.output_integrity_file, 'w') as fd:
with open(self.output_integrity_file, "w") as fd:
fd.write(data)

294
src/borg/crypto/key.py
View File

@ -59,9 +59,11 @@ class UnsupportedKeyFormatError(Error):
class TAMRequiredError(IntegrityError):
__doc__ = textwrap.dedent("""
__doc__ = textwrap.dedent(
"""
Manifest is unauthenticated, but it is required for this repository. Is somebody attacking you?
""").strip()
"""
).strip()
traceback = False
@ -71,11 +73,12 @@ class TAMInvalid(IntegrityError):
def __init__(self):
# Error message becomes: "Data integrity error: Manifest authentication did not verify"
super().__init__('Manifest authentication did not verify')
super().__init__("Manifest authentication did not verify")
class TAMUnsupportedSuiteError(IntegrityError):
"""Could not verify manifest: Unsupported suite {!r}; a newer version is needed."""
traceback = False
@ -110,7 +113,7 @@ def key_factory(repository, manifest_data):
def tam_required_file(repository):
security_dir = get_security_dir(bin_to_hex(repository.id))
return os.path.join(security_dir, 'tam_required')
return os.path.join(security_dir, "tam_required")
def tam_required(repository):
@ -126,9 +129,10 @@ def uses_same_id_hash(other_key, key):
old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey)
new_blake2_ids = (Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey)
same_ids = (
isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids) and isinstance(key, new_hmac_sha256_ids)
or
isinstance(other_key, old_blake2_ids + new_blake2_ids) and isinstance(key, new_blake2_ids)
isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids)
and isinstance(key, new_hmac_sha256_ids)
or isinstance(other_key, old_blake2_ids + new_blake2_ids)
and isinstance(key, new_blake2_ids)
)
return same_ids
@ -140,10 +144,10 @@ class KeyBase:
TYPES_ACCEPTABLE = None # override in subclasses
# Human-readable name
NAME = 'UNDEFINED'
NAME = "UNDEFINED"
# Name used in command line / API (e.g. borg init --encryption=...)
ARG_NAME = 'UNDEFINED'
ARG_NAME = "UNDEFINED"
# Storage type (no key blob storage / keyfile / repo)
STORAGE = KeyBlobStorage.NO_STORAGE
@ -167,13 +171,12 @@ def __init__(self, repository):
self.target = None # key location file path / repo obj
# Some commands write new chunks (e.g. rename) but don't take a --compression argument. This duplicates
# the default used by those commands who do take a --compression argument.
self.compressor = Compressor('lz4')
self.compressor = Compressor("lz4")
self.decompress = self.compressor.decompress
self.tam_required = True
def id_hash(self, data):
"""Return HMAC hash using the "id" HMAC key
"""
"""Return HMAC hash using the "id" HMAC key"""
raise NotImplementedError
def encrypt(self, id, data, compress=True):
@ -186,83 +189,79 @@ def assert_id(self, id, data):
if id and id != Manifest.MANIFEST_ID:
id_computed = self.id_hash(data)
if not hmac.compare_digest(id_computed, id):
raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
raise IntegrityError("Chunk %s: id verification failed" % bin_to_hex(id))
def assert_type(self, type_byte, id=None):
if type_byte not in self.TYPES_ACCEPTABLE:
id_str = bin_to_hex(id) if id is not None else '(unknown)'
raise IntegrityError(f'Chunk {id_str}: Invalid encryption envelope')
id_str = bin_to_hex(id) if id is not None else "(unknown)"
raise IntegrityError(f"Chunk {id_str}: Invalid encryption envelope")
def _tam_key(self, salt, context):
return hkdf_hmac_sha512(
ikm=self.id_key + self.enc_key + self.enc_hmac_key,
salt=salt,
info=b'borg-metadata-authentication-' + context,
output_length=64
info=b"borg-metadata-authentication-" + context,
output_length=64,
)
def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'):
def pack_and_authenticate_metadata(self, metadata_dict, context=b"manifest"):
metadata_dict = StableDict(metadata_dict)
tam = metadata_dict['tam'] = StableDict({
'type': 'HKDF_HMAC_SHA512',
'hmac': bytes(64),
'salt': os.urandom(64),
})
tam = metadata_dict["tam"] = StableDict({"type": "HKDF_HMAC_SHA512", "hmac": bytes(64), "salt": os.urandom(64)})
packed = msgpack.packb(metadata_dict)
tam_key = self._tam_key(tam['salt'], context)
tam['hmac'] = hmac.digest(tam_key, packed, 'sha512')
tam_key = self._tam_key(tam["salt"], context)
tam["hmac"] = hmac.digest(tam_key, packed, "sha512")
return msgpack.packb(metadata_dict)
def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
"""Unpack msgpacked *data* and return (object, did_verify)."""
if data.startswith(b'\xc1' * 4):
if data.startswith(b"\xc1" * 4):
# This is a manifest from the future, we can't read it.
raise UnsupportedManifestError()
tam_required = self.tam_required
if force_tam_not_required and tam_required:
logger.warning('Manifest authentication DISABLED.')
logger.warning("Manifest authentication DISABLED.")
tam_required = False
data = bytearray(data)
unpacker = get_limited_unpacker('manifest')
unpacker = get_limited_unpacker("manifest")
unpacker.feed(data)
unpacked = unpacker.unpack()
if 'tam' not in unpacked:
if "tam" not in unpacked:
if tam_required:
raise TAMRequiredError(self.repository._location.canonical_path())
else:
logger.debug('TAM not found and not required')
logger.debug("TAM not found and not required")
return unpacked, False
tam = unpacked.pop('tam', None)
tam = unpacked.pop("tam", None)
if not isinstance(tam, dict):
raise TAMInvalid()
tam_type = tam.get('type', '<none>')
if tam_type != 'HKDF_HMAC_SHA512':
tam_type = tam.get("type", "<none>")
if tam_type != "HKDF_HMAC_SHA512":
if tam_required:
raise TAMUnsupportedSuiteError(repr(tam_type))
else:
logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type)
logger.debug("Ignoring TAM made with unsupported suite, since TAM is not required: %r", tam_type)
return unpacked, False
tam_hmac = tam.get('hmac')
tam_salt = tam.get('salt')
tam_hmac = tam.get("hmac")
tam_salt = tam.get("salt")
if not isinstance(tam_salt, (bytes, str)) or not isinstance(tam_hmac, (bytes, str)):
raise TAMInvalid()
tam_hmac = want_bytes(tam_hmac) # legacy
tam_salt = want_bytes(tam_salt) # legacy
offset = data.index(tam_hmac)
data[offset:offset + 64] = bytes(64)
tam_key = self._tam_key(tam_salt, context=b'manifest')
calculated_hmac = hmac.digest(tam_key, data, 'sha512')
data[offset : offset + 64] = bytes(64)
tam_key = self._tam_key(tam_salt, context=b"manifest")
calculated_hmac = hmac.digest(tam_key, data, "sha512")
if not hmac.compare_digest(calculated_hmac, tam_hmac):
raise TAMInvalid()
logger.debug('TAM-verified manifest')
logger.debug("TAM-verified manifest")
return unpacked, True
class PlaintextKey(KeyBase):
TYPE = KeyType.PLAINTEXT
TYPES_ACCEPTABLE = {TYPE}
NAME = 'plaintext'
ARG_NAME = 'none'
NAME = "plaintext"
ARG_NAME = "none"
STORAGE = KeyBlobStorage.NO_STORAGE
chunk_seed = 0
@ -287,7 +286,7 @@ def id_hash(self, data):
def encrypt(self, id, data, compress=True):
if compress:
data = self.compressor.compress(data)
return b''.join([self.TYPE_STR, data])
return b"".join([self.TYPE_STR, data])
def decrypt(self, id, data, decompress=True):
self.assert_type(data[0], id)
@ -364,8 +363,7 @@ class AESKeyBase(KeyBase):
def encrypt(self, id, data, compress=True):
if compress:
data = self.compressor.compress(data)
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(),
self.cipher.block_count(len(data)))
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data)))
return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)
def decrypt(self, id, data, decompress=True):
@ -395,12 +393,10 @@ def init_from_random_data(self):
chunk_seed = bytes_to_int(data[96:100])
# Convert to signed int32
if chunk_seed & 0x80000000:
chunk_seed = chunk_seed - 0xffffffff - 1
chunk_seed = chunk_seed - 0xFFFFFFFF - 1
self.init_from_given_data(
enc_key=data[0:32],
enc_hmac_key=data[32:64],
id_key=data[64:96],
chunk_seed=chunk_seed)
enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed
)
def init_ciphers(self, manifest_data=None):
self.cipher = self.CIPHERSUITE(mac_key=self.enc_hmac_key, enc_key=self.enc_key, header_len=1, aad_offset=1)
@ -418,13 +414,13 @@ def init_ciphers(self, manifest_data=None):
class FlexiKey:
FILE_ID = 'BORG_KEY'
FILE_ID = "BORG_KEY"
@classmethod
def detect(cls, repository, manifest_data):
key = cls(repository)
target = key.find_key()
prompt = 'Enter passphrase for key %s: ' % target
prompt = "Enter passphrase for key %s: " % target
passphrase = Passphrase.env_passphrase()
if passphrase is None:
passphrase = Passphrase()
@ -449,18 +445,18 @@ def _load(self, key_data, passphrase):
data = msgpack.unpackb(data)
key = Key(internal_dict=data)
if key.version != 1:
raise IntegrityError('Invalid key file header')
raise IntegrityError("Invalid key file header")
self.repository_id = key.repository_id
self.enc_key = key.enc_key
self.enc_hmac_key = key.enc_hmac_key
self.id_key = key.id_key
self.chunk_seed = key.chunk_seed
self.tam_required = key.get('tam_required', tam_required(self.repository))
self.tam_required = key.get("tam_required", tam_required(self.repository))
return True
return False
def decrypt_key_file(self, data, passphrase):
unpacker = get_limited_unpacker('key')
unpacker = get_limited_unpacker("key")
unpacker.feed(data)
data = unpacker.unpack()
encrypted_key = EncryptedKey(internal_dict=data)
@ -468,9 +464,9 @@ def decrypt_key_file(self, data, passphrase):
raise UnsupportedKeyFormatError()
else:
self._encrypted_key_algorithm = encrypted_key.algorithm
if encrypted_key.algorithm == 'sha256':
if encrypted_key.algorithm == "sha256":
return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
elif encrypted_key.algorithm == 'argon2 chacha20-poly1305':
elif encrypted_key.algorithm == "argon2 chacha20-poly1305":
return self.decrypt_key_file_argon2(encrypted_key, passphrase)
else:
raise UnsupportedKeyFormatError()
@ -479,7 +475,7 @@ def decrypt_key_file(self, data, passphrase):
def pbkdf2(passphrase, salt, iterations, output_len_in_bytes):
if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
iterations = 1
return pbkdf2_hmac('sha256', passphrase.encode('utf-8'), salt, iterations, output_len_in_bytes)
return pbkdf2_hmac("sha256", passphrase.encode("utf-8"), salt, iterations, output_len_in_bytes)
@staticmethod
def argon2(
@ -489,18 +485,14 @@ def argon2(
time_cost: int,
memory_cost: int,
parallelism: int,
type: Literal['i', 'd', 'id']
type: Literal["i", "d", "id"],
) -> bytes:
if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
time_cost = 1
parallelism = 1
# 8 is the smallest value that avoids the "Memory cost is too small" exception
memory_cost = 8
type_map = {
'i': argon2.low_level.Type.I,
'd': argon2.low_level.Type.D,
'id': argon2.low_level.Type.ID,
}
type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID}
key = argon2.low_level.hash_secret_raw(
secret=passphrase.encode("utf-8"),
hash_len=output_len_in_bytes,
@ -514,7 +506,7 @@ def argon2(
def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)
data = AES(key, b'\0'*16).decrypt(encrypted_key.data)
data = AES(key, b"\0" * 16).decrypt(encrypted_key.data)
if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
return data
return None
@ -536,44 +528,32 @@ def decrypt_key_file_argon2(self, encrypted_key, passphrase):
return None
def encrypt_key_file(self, data, passphrase, algorithm):
if algorithm == 'sha256':
if algorithm == "sha256":
return self.encrypt_key_file_pbkdf2(data, passphrase)
elif algorithm == 'argon2 chacha20-poly1305':
elif algorithm == "argon2 chacha20-poly1305":
return self.encrypt_key_file_argon2(data, passphrase)
else:
raise ValueError(f'Unexpected algorithm: {algorithm}')
raise ValueError(f"Unexpected algorithm: {algorithm}")
def encrypt_key_file_pbkdf2(self, data, passphrase):
salt = os.urandom(32)
iterations = PBKDF2_ITERATIONS
key = self.pbkdf2(passphrase, salt, iterations, 32)
hash = hmac_sha256(key, data)
cdata = AES(key, b'\0'*16).encrypt(data)
enc_key = EncryptedKey(
version=1,
salt=salt,
iterations=iterations,
algorithm='sha256',
hash=hash,
data=cdata,
)
cdata = AES(key, b"\0" * 16).encrypt(data)
enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata)
return msgpack.packb(enc_key.as_dict())
def encrypt_key_file_argon2(self, data, passphrase):
salt = os.urandom(ARGON2_SALT_BYTES)
key = self.argon2(
passphrase,
output_len_in_bytes=32,
salt=salt,
**ARGON2_ARGS,
)
key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS)
ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0)
encrypted_key = EncryptedKey(
version=1,
algorithm='argon2 chacha20-poly1305',
algorithm="argon2 chacha20-poly1305",
salt=salt,
data=ae_cipher.encrypt(data),
**{'argon2_' + k: v for k, v in ARGON2_ARGS.items()},
**{"argon2_" + k: v for k, v in ARGON2_ARGS.items()},
)
return msgpack.packb(encrypted_key.as_dict())
@ -588,7 +568,7 @@ def _save(self, passphrase, algorithm):
tam_required=self.tam_required,
)
data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm)
key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
key_data = "\n".join(textwrap.wrap(b2a_base64(data).decode("ascii")))
return key_data
def change_passphrase(self, passphrase=None):
@ -612,22 +592,23 @@ def create(cls, repository, args, *, other_key=None):
enc_key=other_key.enc_key,
enc_hmac_key=other_key.enc_hmac_key,
id_key=other_key.id_key,
chunk_seed=other_key.chunk_seed)
chunk_seed=other_key.chunk_seed,
)
passphrase = other_key._passphrase
else:
key.init_from_random_data()
passphrase = Passphrase.new(allow_empty=True)
key.init_ciphers()
target = key.get_new_target(args)
key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS['argon2'])
key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS["argon2"])
logger.info('Key in "%s" created.' % target)
logger.info('Keep this key safe. Your data will be inaccessible without it.')
logger.info("Keep this key safe. Your data will be inaccessible without it.")
return key
def sanity_check(self, filename, id):
file_id = self.FILE_ID.encode() + b' '
file_id = self.FILE_ID.encode() + b" "
repo_id = hexlify(id)
with open(filename, 'rb') as fd:
with open(filename, "rb") as fd:
# we do the magic / id check in binary mode to avoid stumbling over
# decoding errors if somebody has binary files in the keys dir for some reason.
if fd.read(len(file_id)) != file_id:
@ -653,7 +634,7 @@ def find_key(self):
raise RepoKeyNotFoundError(loc) from None
return loc
else:
raise TypeError('Unsupported borg key storage type')
raise TypeError("Unsupported borg key storage type")
def get_existing_or_new_target(self, args):
keyfile = self._find_key_file_from_environment()
@ -683,10 +664,10 @@ def get_new_target(self, args):
elif self.STORAGE == KeyBlobStorage.REPO:
return self.repository
else:
raise TypeError('Unsupported borg key storage type')
raise TypeError("Unsupported borg key storage type")
def _find_key_file_from_environment(self):
keyfile = os.environ.get('BORG_KEY_FILE')
keyfile = os.environ.get("BORG_KEY_FILE")
if keyfile:
return os.path.abspath(keyfile)
@ -696,17 +677,17 @@ def _get_new_target_in_keys_dir(self, args):
i = 1
while os.path.exists(path):
i += 1
path = filename + '.%d' % i
path = filename + ".%d" % i
return path
def load(self, target, passphrase):
if self.STORAGE == KeyBlobStorage.KEYFILE:
with open(target) as fd:
key_data = ''.join(fd.readlines()[1:])
key_data = "".join(fd.readlines()[1:])
elif self.STORAGE == KeyBlobStorage.REPO:
# While the repository is encrypted, we consider a repokey repository with a blank
# passphrase an unencrypted repository.
self.logically_encrypted = passphrase != ''
self.logically_encrypted = passphrase != ""
# what we get in target is just a repo location, but we already have the repo obj:
target = self.repository
@ -715,9 +696,9 @@ def load(self, target, passphrase):
# if we got an empty key, it means there is no key.
loc = target._location.canonical_path()
raise RepoKeyNotFoundError(loc) from None
key_data = key_data.decode('utf-8') # remote repo: msgpack issue #99, getting bytes
key_data = key_data.decode("utf-8") # remote repo: msgpack issue #99, getting bytes
else:
raise TypeError('Unsupported borg key storage type')
raise TypeError("Unsupported borg key storage type")
success = self._load(key_data, passphrase)
if success:
self.target = target
@ -732,31 +713,31 @@ def save(self, target, passphrase, algorithm, create=False):
# see issue #6036
raise Error('Aborting because key in "%s" already exists.' % target)
with SaveFile(target) as fd:
fd.write(f'{self.FILE_ID} {bin_to_hex(self.repository_id)}\n')
fd.write(f"{self.FILE_ID} {bin_to_hex(self.repository_id)}\n")
fd.write(key_data)
fd.write('\n')
fd.write("\n")
elif self.STORAGE == KeyBlobStorage.REPO:
self.logically_encrypted = passphrase != ''
key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes
self.logically_encrypted = passphrase != ""
key_data = key_data.encode("utf-8") # remote repo: msgpack issue #99, giving bytes
target.save_key(key_data)
else:
raise TypeError('Unsupported borg key storage type')
raise TypeError("Unsupported borg key storage type")
self.target = target
def remove(self, target):
if self.STORAGE == KeyBlobStorage.KEYFILE:
os.remove(target)
elif self.STORAGE == KeyBlobStorage.REPO:
target.save_key(b'') # save empty key (no new api at remote repo necessary)
target.save_key(b"") # save empty key (no new api at remote repo necessary)
else:
raise TypeError('Unsupported borg key storage type')
raise TypeError("Unsupported borg key storage type")
class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
TYPE = KeyType.KEYFILE
NAME = 'key file'
ARG_NAME = 'keyfile'
NAME = "key file"
ARG_NAME = "keyfile"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = AES256_CTR_HMAC_SHA256
@ -764,8 +745,8 @@ class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
TYPE = KeyType.REPO
NAME = 'repokey'
ARG_NAME = 'repokey'
NAME = "repokey"
ARG_NAME = "repokey"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = AES256_CTR_HMAC_SHA256
@ -773,8 +754,8 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
TYPE = KeyType.BLAKE2KEYFILE
NAME = 'key file BLAKE2b'
ARG_NAME = 'keyfile-blake2'
NAME = "key file BLAKE2b"
ARG_NAME = "keyfile-blake2"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = AES256_CTR_BLAKE2b
@ -782,8 +763,8 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
TYPE = KeyType.BLAKE2REPO
NAME = 'repokey BLAKE2b'
ARG_NAME = 'repokey-blake2'
NAME = "repokey BLAKE2b"
ARG_NAME = "repokey-blake2"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = AES256_CTR_BLAKE2b
@ -810,7 +791,7 @@ def init_ciphers(self, manifest_data=None):
def encrypt(self, id, data, compress=True):
if compress:
data = self.compressor.compress(data)
return b''.join([self.TYPE_STR, data])
return b"".join([self.TYPE_STR, data])
def decrypt(self, id, data, decompress=True):
self.assert_type(data[0], id)
@ -825,15 +806,15 @@ def decrypt(self, id, data, decompress=True):
class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
TYPE = KeyType.AUTHENTICATED
TYPES_ACCEPTABLE = {TYPE}
NAME = 'authenticated'
ARG_NAME = 'authenticated'
NAME = "authenticated"
ARG_NAME = "authenticated"
class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase):
TYPE = KeyType.BLAKE2AUTHENTICATED
TYPES_ACCEPTABLE = {TYPE}
NAME = 'authenticated BLAKE2b'
ARG_NAME = 'authenticated-blake2'
NAME = "authenticated BLAKE2b"
ARG_NAME = "authenticated-blake2"
# ------------ new crypto ------------
@ -862,17 +843,17 @@ class AEADKeyBase(KeyBase):
logically_encrypted = True
MAX_IV = 2 ** 48 - 1
MAX_IV = 2**48 - 1
def encrypt(self, id, data, compress=True):
# to encrypt new data in this session we use always self.cipher and self.sessionid
if compress:
data = self.compressor.compress(data)
reserved = b'\0'
reserved = b"\0"
iv = self.cipher.next_iv()
if iv > self.MAX_IV: # see the data-structures docs about why the IV range is enough
raise IntegrityError("IV overflow, should never happen.")
iv_48bit = iv.to_bytes(6, 'big')
iv_48bit = iv.to_bytes(6, "big")
header = self.TYPE_STR + reserved + iv_48bit + self.sessionid
return self.cipher.encrypt(data, header=header, iv=iv, aad=id)
@ -881,7 +862,7 @@ def decrypt(self, id, data, decompress=True):
self.assert_type(data[0], id)
iv_48bit = data[2:8]
sessionid = data[8:32]
iv = int.from_bytes(iv_48bit, 'big')
iv = int.from_bytes(iv_48bit, "big")
cipher = self._get_cipher(sessionid, iv)
try:
payload = cipher.decrypt(data, aad=id)
@ -908,27 +889,25 @@ def init_from_random_data(self):
chunk_seed = bytes_to_int(data[96:100])
# Convert to signed int32
if chunk_seed & 0x80000000:
chunk_seed = chunk_seed - 0xffffffff - 1
chunk_seed = chunk_seed - 0xFFFFFFFF - 1
self.init_from_given_data(
enc_key=data[0:32],
enc_hmac_key=data[32:64],
id_key=data[64:96],
chunk_seed=chunk_seed)
enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed
)
def _get_session_key(self, sessionid):
assert len(sessionid) == 24 # 192bit
key = hkdf_hmac_sha512(
ikm=self.enc_key + self.enc_hmac_key,
salt=sessionid,
info=b'borg-session-key-' + self.CIPHERSUITE.__name__.encode(),
output_length=32
info=b"borg-session-key-" + self.CIPHERSUITE.__name__.encode(),
output_length=32,
)
return key
def _get_cipher(self, sessionid, iv):
assert isinstance(iv, int)
key = self._get_session_key(sessionid)
cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1+1+6+24, aad_offset=0)
cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1 + 1 + 6 + 24, aad_offset=0)
return cipher
def init_ciphers(self, manifest_data=None, iv=0):
@ -940,8 +919,8 @@ def init_ciphers(self, manifest_data=None, iv=0):
class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO}
TYPE = KeyType.AESOCBKEYFILE
NAME = 'key file AES-OCB'
ARG_NAME = 'keyfile-aes-ocb'
NAME = "key file AES-OCB"
ARG_NAME = "keyfile-aes-ocb"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = AES256_OCB
@ -949,8 +928,8 @@ class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO}
TYPE = KeyType.AESOCBREPO
NAME = 'repokey AES-OCB'
ARG_NAME = 'repokey-aes-ocb'
NAME = "repokey AES-OCB"
ARG_NAME = "repokey-aes-ocb"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = AES256_OCB
@ -958,8 +937,8 @@ class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO}
TYPE = KeyType.CHPOKEYFILE
NAME = 'key file ChaCha20-Poly1305'
ARG_NAME = 'keyfile-chacha20-poly1305'
NAME = "key file ChaCha20-Poly1305"
ARG_NAME = "keyfile-chacha20-poly1305"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = CHACHA20_POLY1305
@ -967,8 +946,8 @@ class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO}
TYPE = KeyType.CHPOREPO
NAME = 'repokey ChaCha20-Poly1305'
ARG_NAME = 'repokey-chacha20-poly1305'
NAME = "repokey ChaCha20-Poly1305"
ARG_NAME = "repokey-chacha20-poly1305"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = CHACHA20_POLY1305
@ -976,8 +955,8 @@ class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
TYPE = KeyType.BLAKE2AESOCBKEYFILE
NAME = 'key file BLAKE2b AES-OCB'
ARG_NAME = 'keyfile-blake2-aes-ocb'
NAME = "key file BLAKE2b AES-OCB"
ARG_NAME = "keyfile-blake2-aes-ocb"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = AES256_OCB
@ -985,8 +964,8 @@ class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
TYPE = KeyType.BLAKE2AESOCBREPO
NAME = 'repokey BLAKE2b AES-OCB'
ARG_NAME = 'repokey-blake2-aes-ocb'
NAME = "repokey BLAKE2b AES-OCB"
ARG_NAME = "repokey-blake2-aes-ocb"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = AES256_OCB
@ -994,8 +973,8 @@ class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
TYPE = KeyType.BLAKE2CHPOKEYFILE
NAME = 'key file BLAKE2b ChaCha20-Poly1305'
ARG_NAME = 'keyfile-blake2-chacha20-poly1305'
NAME = "key file BLAKE2b ChaCha20-Poly1305"
ARG_NAME = "keyfile-blake2-chacha20-poly1305"
STORAGE = KeyBlobStorage.KEYFILE
CIPHERSUITE = CHACHA20_POLY1305
@ -1003,26 +982,33 @@ class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
TYPE = KeyType.BLAKE2CHPOREPO
NAME = 'repokey BLAKE2b ChaCha20-Poly1305'
ARG_NAME = 'repokey-blake2-chacha20-poly1305'
NAME = "repokey BLAKE2b ChaCha20-Poly1305"
ARG_NAME = "repokey-blake2-chacha20-poly1305"
STORAGE = KeyBlobStorage.REPO
CIPHERSUITE = CHACHA20_POLY1305
LEGACY_KEY_TYPES = (
# legacy (AES-CTR based) crypto
KeyfileKey, RepoKey,
Blake2KeyfileKey, Blake2RepoKey,
KeyfileKey,
RepoKey,
Blake2KeyfileKey,
Blake2RepoKey,
)
AVAILABLE_KEY_TYPES = (
# these are available encryption modes for new repositories
# not encrypted modes
PlaintextKey,
AuthenticatedKey, Blake2AuthenticatedKey,
AuthenticatedKey,
Blake2AuthenticatedKey,
# new crypto
AESOCBKeyfileKey, AESOCBRepoKey,
CHPOKeyfileKey, CHPORepoKey,
Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey,
Blake2CHPOKeyfileKey, Blake2CHPORepoKey,
AESOCBKeyfileKey,
AESOCBRepoKey,
CHPOKeyfileKey,
CHPORepoKey,
Blake2AESOCBKeyfileKey,
Blake2AESOCBRepoKey,
Blake2CHPOKeyfileKey,
Blake2CHPORepoKey,
)

92
src/borg/crypto/keymanager.py
View File

@ -53,7 +53,7 @@ def load_keyblob(self):
k = CHPOKeyfileKey(self.repository)
target = k.find_key()
with open(target) as fd:
self.keyblob = ''.join(fd.readlines()[1:])
self.keyblob = "".join(fd.readlines()[1:])
elif self.keyblob_storage == KeyBlobStorage.REPO:
key_data = self.repository.load_key().decode()
@ -70,75 +70,77 @@ def store_keyblob(self, args):
self.store_keyfile(target)
elif self.keyblob_storage == KeyBlobStorage.REPO:
self.repository.save_key(self.keyblob.encode('utf-8'))
self.repository.save_key(self.keyblob.encode("utf-8"))
def get_keyfile_data(self):
data = f'{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n'
data = f"{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n"
data += self.keyblob
if not self.keyblob.endswith('\n'):
data += '\n'
if not self.keyblob.endswith("\n"):
data += "\n"
return data
def store_keyfile(self, target):
with dash_open(target, 'w') as fd:
with dash_open(target, "w") as fd:
fd.write(self.get_keyfile_data())
def export(self, path):
if path is None:
path = '-'
path = "-"
self.store_keyfile(path)
def export_qr(self, path):
if path is None:
path = '-'
path = "-"
with dash_open(path, 'wb') as fd:
with dash_open(path, "wb") as fd:
key_data = self.get_keyfile_data()
html = pkgutil.get_data('borg', 'paperkey.html')
html = html.replace(b'</textarea>', key_data.encode() + b'</textarea>')
html = pkgutil.get_data("borg", "paperkey.html")
html = html.replace(b"</textarea>", key_data.encode() + b"</textarea>")
fd.write(html)
def export_paperkey(self, path):
if path is None:
path = '-'
path = "-"
def grouped(s):
ret = ''
ret = ""
i = 0
for ch in s:
if i and i % 6 == 0:
ret += ' '
ret += " "
ret += ch
i += 1
return ret
export = 'To restore key use borg key import --paper /path/to/repo\n\n'
export = "To restore key use borg key import --paper /path/to/repo\n\n"
binary = a2b_base64(self.keyblob)
export += 'BORG PAPER KEY v1\n'
export += "BORG PAPER KEY v1\n"
lines = (len(binary) + 17) // 18
repoid = bin_to_hex(self.repository.id)[:18]
complete_checksum = sha256_truncated(binary, 12)
export += 'id: {:d} / {} / {} - {}\n'.format(lines,
grouped(repoid),
grouped(complete_checksum),
sha256_truncated((str(lines) + '/' + repoid + '/' + complete_checksum).encode('ascii'), 2))
export += "id: {:d} / {} / {} - {}\n".format(
lines,
grouped(repoid),
grouped(complete_checksum),
sha256_truncated((str(lines) + "/" + repoid + "/" + complete_checksum).encode("ascii"), 2),
)
idx = 0
while len(binary):
idx += 1
binline = binary[:18]
checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2)
export += f'{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n'
checksum = sha256_truncated(idx.to_bytes(2, byteorder="big") + binline, 2)
export += f"{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n"
binary = binary[18:]
with dash_open(path, 'w') as fd:
with dash_open(path, "w") as fd:
fd.write(export)
def import_keyfile(self, args):
file_id = CHPOKeyfileKey.FILE_ID
first_line = file_id + ' ' + bin_to_hex(self.repository.id) + '\n'
with dash_open(args.path, 'r') as fd:
first_line = file_id + " " + bin_to_hex(self.repository.id) + "\n"
with dash_open(args.path, "r") as fd:
file_first_line = fd.read(len(first_line))
if file_first_line != first_line:
if not file_first_line.startswith(file_id):
@ -154,52 +156,52 @@ def import_paperkey(self, args):
# imported here because it has global side effects
import readline
except ImportError:
print('Note: No line editing available due to missing readline support')
print("Note: No line editing available due to missing readline support")
repoid = bin_to_hex(self.repository.id)[:18]
try:
while True: # used for repeating on overall checksum mismatch
# id line input
while True:
idline = input('id: ').replace(' ', '')
if idline == '':
if yes('Abort import? [yN]:'):
idline = input("id: ").replace(" ", "")
if idline == "":
if yes("Abort import? [yN]:"):
raise EOFError()
try:
(data, checksum) = idline.split('-')
(data, checksum) = idline.split("-")
except ValueError:
print("each line must contain exactly one '-', try again")
continue
try:
(id_lines, id_repoid, id_complete_checksum) = data.split('/')
(id_lines, id_repoid, id_complete_checksum) = data.split("/")
except ValueError:
print("the id line must contain exactly three '/', try again")
continue
if sha256_truncated(data.lower().encode('ascii'), 2) != checksum:
print('line checksum did not match, try same line again')
if sha256_truncated(data.lower().encode("ascii"), 2) != checksum:
print("line checksum did not match, try same line again")
continue
try:
lines = int(id_lines)
except ValueError:
print('internal error while parsing length')
print("internal error while parsing length")
break
if repoid != id_repoid:
raise RepoIdMismatch()
result = b''
result = b""
idx = 1
# body line input
while True:
inline = input(f'{idx:2d}: ')
inline = inline.replace(' ', '')
if inline == '':
if yes('Abort import? [yN]:'):
inline = input(f"{idx:2d}: ")
inline = inline.replace(" ", "")
if inline == "":
if yes("Abort import? [yN]:"):
raise EOFError()
try:
(data, checksum) = inline.split('-')
(data, checksum) = inline.split("-")
except ValueError:
print("each line must contain exactly one '-', try again")
continue
@ -208,8 +210,8 @@ def import_paperkey(self, args):
except binascii.Error:
print("only characters 0-9 and a-f and '-' are valid, try again")
continue
if sha256_truncated(idx.to_bytes(2, byteorder='big') + part, 2) != checksum:
print(f'line checksum did not match, try line {idx} again')
if sha256_truncated(idx.to_bytes(2, byteorder="big") + part, 2) != checksum:
print(f"line checksum did not match, try line {idx} again")
continue
result += part
if idx == lines:
@ -217,13 +219,13 @@ def import_paperkey(self, args):
idx += 1
if sha256_truncated(result, 12) != id_complete_checksum:
print('The overall checksum did not match, retry or enter a blank line to abort.')
print("The overall checksum did not match, retry or enter a blank line to abort.")
continue
self.keyblob = '\n'.join(textwrap.wrap(b2a_base64(result).decode('ascii'))) + '\n'
self.keyblob = "\n".join(textwrap.wrap(b2a_base64(result).decode("ascii"))) + "\n"
self.store_keyblob(args)
break
except EOFError:
print('\n - aborted')
print("\n - aborted")
return

8
src/borg/crypto/nonces.py
View File

@ -18,7 +18,7 @@ def __init__(self, repository, manifest_nonce):
self.repository = repository
self.end_of_nonce_reservation = None
self.manifest_nonce = manifest_nonce
self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), 'nonce')
self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce")
def get_local_free_nonce(self):
try:
@ -78,7 +78,11 @@ def ensure_reservation(self, nonce, nonce_space_needed):
repo_free_nonce = self.get_repo_free_nonce()
local_free_nonce = self.get_local_free_nonce()
free_nonce_space = max(x for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) if x is not None)
free_nonce_space = max(
x
for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation)
if x is not None
)
reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION
assert reservation_end < MAX_REPRESENTABLE_NONCE
self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce)

213
src/borg/fuse.py
View File

@ -20,7 +20,9 @@ def async_wrapper(fn):
@functools.wraps(fn)
async def wrapper(*args, **kwargs):
return fn(*args, **kwargs)
return wrapper
else:
trio = None
@ -29,6 +31,7 @@ def async_wrapper(fn):
from .logger import create_logger
logger = create_logger()
from .crypto.low_level import blake2b_128
@ -79,7 +82,7 @@ class ItemCache:
# to resize it in the first place; that's free).
GROW_META_BY = 2 * 1024 * 1024
indirect_entry_struct = struct.Struct('=cII')
indirect_entry_struct = struct.Struct("=cII")
assert indirect_entry_struct.size == 9
def __init__(self, decrypted_repository):
@ -105,7 +108,7 @@ def __init__(self, decrypted_repository):
# These are items that span more than one chunk and thus cannot be efficiently cached
# by the object cache (self.decrypted_repository), which would require variable-length structures;
# possible but not worth the effort, see iter_archive_items.
self.fd = tempfile.TemporaryFile(prefix='borg-tmp')
self.fd = tempfile.TemporaryFile(prefix="borg-tmp")
# A small LRU cache for chunks requested by ItemCache.get() from the object cache,
# this significantly speeds up directory traversal and similar operations which
@ -123,12 +126,12 @@ def __init__(self, decrypted_repository):
def get(self, inode):
offset = inode - self.offset
if offset < 0:
raise ValueError('ItemCache.get() called with an invalid inode number')
if self.meta[offset] == ord(b'I'):
raise ValueError("ItemCache.get() called with an invalid inode number")
if self.meta[offset] == ord(b"I"):
_, chunk_id_relative_offset, chunk_offset = self.indirect_entry_struct.unpack_from(self.meta, offset)
chunk_id_offset = offset - chunk_id_relative_offset
# bytearray slices are bytearrays as well, explicitly convert to bytes()
chunk_id = bytes(self.meta[chunk_id_offset:chunk_id_offset + 32])
chunk_id = bytes(self.meta[chunk_id_offset : chunk_id_offset + 32])
chunk = self.chunks.get(chunk_id)
if not chunk:
csize, chunk = next(self.decrypted_repository.get_many([chunk_id]))
@ -137,12 +140,12 @@ def get(self, inode):
unpacker = msgpack.Unpacker()
unpacker.feed(data)
return Item(internal_dict=next(unpacker))
elif self.meta[offset] == ord(b'S'):
fd_offset = int.from_bytes(self.meta[offset + 1:offset + 9], 'little')
elif self.meta[offset] == ord(b"S"):
fd_offset = int.from_bytes(self.meta[offset + 1 : offset + 9], "little")
self.fd.seek(fd_offset, io.SEEK_SET)
return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024)))
else:
raise ValueError('Invalid entry type in self.meta')
raise ValueError("Invalid entry type in self.meta")
def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=False):
unpacker = msgpack.Unpacker()
@ -153,7 +156,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
chunk_begin = 0
# Length of the chunk preceding the current chunk
last_chunk_length = 0
msgpacked_bytes = b''
msgpacked_bytes = b""
write_offset = self.write_offset
meta = self.meta
@ -163,7 +166,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
# Store the chunk ID in the meta-array
if write_offset + 32 >= len(meta):
self.meta = meta = meta + bytes(self.GROW_META_BY)
meta[write_offset:write_offset + 32] = key
meta[write_offset : write_offset + 32] = key
current_id_offset = write_offset
write_offset += 32
@ -182,7 +185,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
# tell() is not helpful for the need_more_data case, but we know it is the remainder
# of the data in that case. in the other case, tell() works as expected.
length = (len(data) - start) if need_more_data else (unpacker.tell() - stream_offset)
msgpacked_bytes += data[start:start+length]
msgpacked_bytes += data[start : start + length]
stream_offset += length
if need_more_data:
@ -190,14 +193,14 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
break
item = Item(internal_dict=item)
if filter and not filter(item) or not consider_part_files and 'part' in item:
msgpacked_bytes = b''
if filter and not filter(item) or not consider_part_files and "part" in item:
msgpacked_bytes = b""
continue
current_item = msgpacked_bytes
current_item_length = len(current_item)
current_spans_chunks = stream_offset - current_item_length < chunk_begin
msgpacked_bytes = b''
msgpacked_bytes = b""
if write_offset + 9 >= len(meta):
self.meta = meta = meta + bytes(self.GROW_META_BY)
@ -221,11 +224,11 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
if current_spans_chunks:
pos = self.fd.seek(0, io.SEEK_END)
self.fd.write(current_item)
meta[write_offset:write_offset + 9] = b'S' + pos.to_bytes(8, 'little')
meta[write_offset : write_offset + 9] = b"S" + pos.to_bytes(8, "little")
self.direct_items += 1
else:
item_offset = stream_offset - current_item_length - chunk_begin
pack_indirect_into(meta, write_offset, b'I', write_offset - current_id_offset, item_offset)
pack_indirect_into(meta, write_offset, b"I", write_offset - current_id_offset, item_offset)
self.indirect_items += 1
inode = write_offset + self.offset
write_offset += 9
@ -236,8 +239,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
class FuseBackend:
"""Virtual filesystem based on archive(s) to provide information to fuse
"""
"""Virtual filesystem based on archive(s) to provide information to fuse"""
def __init__(self, key, manifest, repository, args, decrypted_repository):
self.repository_uncached = repository
@ -307,8 +309,7 @@ def _allocate_inode(self):
return self.inode_count
def _create_dir(self, parent, mtime=None):
"""Create directory
"""
"""Create directory"""
ino = self._allocate_inode()
if mtime is not None:
self._items[ino] = Item(internal_dict=self.default_dir.as_dict())
@ -319,26 +320,31 @@ def _create_dir(self, parent, mtime=None):
return ino
def find_inode(self, path, prefix=[]):
segments = prefix + path.split(b'/')
segments = prefix + path.split(b"/")
inode = 1
for segment in segments:
inode = self.contents[inode][segment]
return inode
def _process_archive(self, archive_name, prefix=[]):
"""Build FUSE inode hierarchy from archive metadata
"""
"""Build FUSE inode hierarchy from archive metadata"""
self.file_versions = {} # for versions mode: original path -> version
t0 = time.perf_counter()
archive = Archive(self.repository_uncached, self.key, self._manifest, archive_name,
consider_part_files=self._args.consider_part_files)
archive = Archive(
self.repository_uncached,
self.key,
self._manifest,
archive_name,
consider_part_files=self._args.consider_part_files,
)
strip_components = self._args.strip_components
matcher = Archiver.build_matcher(self._args.patterns, self._args.paths)
hlm = HardLinkManager(id_type=bytes, info_type=str) # hlid -> path
filter = Archiver.build_filter(matcher, strip_components)
for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter,
consider_part_files=self._args.consider_part_files):
for item_inode, item in self.cache.iter_archive_items(
archive.metadata.items, filter=filter, consider_part_files=self._args.consider_part_files
):
if strip_components:
item.path = os.sep.join(item.path.split(os.sep)[strip_components:])
path = os.fsencode(item.path)
@ -354,24 +360,24 @@ def _process_archive(self, archive_name, prefix=[]):
else:
self._items[inode] = item
continue
segments = prefix + path.split(b'/')
segments = prefix + path.split(b"/")
parent = 1
for segment in segments[:-1]:
parent = self._process_inner(segment, parent)
self._process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode, hlm)
duration = time.perf_counter() - t0
logger.debug('fuse: _process_archive completed in %.1f s for archive %s', duration, archive.name)
logger.debug("fuse: _process_archive completed in %.1f s for archive %s", duration, archive.name)
def _process_leaf(self, name, item, parent, prefix, is_dir, item_inode, hlm):
path = item.path
del item.path # save some space
def file_version(item, path):
if 'chunks' in item:
if "chunks" in item:
file_id = blake2b_128(path)
current_version, previous_id = self.versions_index.get(file_id, (0, None))
contents_id = blake2b_128(b''.join(chunk_id for chunk_id, _ in item.chunks))
contents_id = blake2b_128(b"".join(chunk_id for chunk_id, _ in item.chunks))
if contents_id != previous_id:
current_version += 1
@ -382,14 +388,14 @@ def file_version(item, path):
def make_versioned_name(name, version, add_dir=False):
if add_dir:
# add intermediate directory with same name as filename
path_fname = name.rsplit(b'/', 1)
name += b'/' + path_fname[-1]
path_fname = name.rsplit(b"/", 1)
name += b"/" + path_fname[-1]
# keep original extension at end to avoid confusing tools
name, ext = os.path.splitext(name)
version_enc = os.fsencode('.%05d' % version)
version_enc = os.fsencode(".%05d" % version)
return name + version_enc + ext
if 'hlid' in item:
if "hlid" in item:
link_target = hlm.retrieve(id=item.hlid, default=None)
if link_target is not None:
# Hard link was extracted previously, just link
@ -401,10 +407,10 @@ def make_versioned_name(name, version, add_dir=False):
try:
inode = self.find_inode(link_target, prefix)
except KeyError:
logger.warning('Skipping broken hard link: %s -> %s', path, link_target)
logger.warning("Skipping broken hard link: %s -> %s", path, link_target)
return
item = self.get_item(inode)
item.nlink = item.get('nlink', 1) + 1
item.nlink = item.get("nlink", 1) + 1
self._items[inode] = item
else:
inode = item_inode
@ -439,31 +445,41 @@ def _process_inner(self, name, parent_inode):
class FuseOperations(llfuse.Operations, FuseBackend):
"""Export archive as a FUSE filesystem
"""
"""Export archive as a FUSE filesystem"""
def __init__(self, key, repository, manifest, args, decrypted_repository):
llfuse.Operations.__init__(self)
FuseBackend.__init__(self, key, manifest, repository, args, decrypted_repository)
self.decrypted_repository = decrypted_repository
data_cache_capacity = int(os.environ.get('BORG_MOUNT_DATA_CACHE_ENTRIES', os.cpu_count() or 1))
logger.debug('mount data cache capacity: %d chunks', data_cache_capacity)
data_cache_capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
logger.debug("mount data cache capacity: %d chunks", data_cache_capacity)
self.data_cache = LRUCache(capacity=data_cache_capacity, dispose=lambda _: None)
self._last_pos = LRUCache(capacity=FILES, dispose=lambda _: None)
def sig_info_handler(self, sig_no, stack):
logger.debug('fuse: %d synth inodes, %d edges (%s)',
self.inode_count, len(self.parent),
# getsizeof is the size of the dict itself; key and value are two small-ish integers,
# which are shared due to code structure (this has been verified).
format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)))
logger.debug('fuse: %d pending archives', len(self.pending_archives))
logger.debug('fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s',
self.cache.direct_items + self.cache.indirect_items, self.cache.direct_items, self.cache.indirect_items,
format_file_size(sys.getsizeof(self.cache.meta)),
format_file_size(os.stat(self.cache.fd.fileno()).st_size))
logger.debug('fuse: data cache: %d/%d entries, %s', len(self.data_cache.items()), self.data_cache._capacity,
format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())))
logger.debug(
"fuse: %d synth inodes, %d edges (%s)",
self.inode_count,
len(self.parent),
# getsizeof is the size of the dict itself; key and value are two small-ish integers,
# which are shared due to code structure (this has been verified).
format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)),
)
logger.debug("fuse: %d pending archives", len(self.pending_archives))
logger.debug(
"fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s",
self.cache.direct_items + self.cache.indirect_items,
self.cache.direct_items,
self.cache.indirect_items,
format_file_size(sys.getsizeof(self.cache.meta)),
format_file_size(os.stat(self.cache.fd.fileno()).st_size),
)
logger.debug(
"fuse: data cache: %d/%d entries, %s",
len(self.data_cache.items()),
self.data_cache._capacity,
format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())),
)
self.decrypted_repository.log_instrumentation()
def mount(self, mountpoint, mount_options, foreground=False):
@ -475,25 +491,25 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
if option == key:
options.pop(idx)
return present
if option.startswith(key + '='):
if option.startswith(key + "="):
options.pop(idx)
value = option.split('=', 1)[1]
value = option.split("=", 1)[1]
if wanted_type is bool:
v = value.lower()
if v in ('y', 'yes', 'true', '1'):
if v in ("y", "yes", "true", "1"):
return True
if v in ('n', 'no', 'false', '0'):
if v in ("n", "no", "false", "0"):
return False
raise ValueError('unsupported value in option: %s' % option)
raise ValueError("unsupported value in option: %s" % option)
if wanted_type is int:
try:
return int(value, base=int_base)
except ValueError:
raise ValueError('unsupported value in option: %s' % option) from None
raise ValueError("unsupported value in option: %s" % option) from None
try:
return wanted_type(value)
except ValueError:
raise ValueError('unsupported value in option: %s' % option) from None
raise ValueError("unsupported value in option: %s" % option) from None
else:
return not_present
@ -502,20 +518,20 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
# cause security issues if used with allow_other mount option.
# When not using allow_other or allow_root, access is limited to the
# mounting user anyway.
options = ['fsname=borgfs', 'ro', 'default_permissions']
options = ["fsname=borgfs", "ro", "default_permissions"]
if mount_options:
options.extend(mount_options.split(','))
ignore_permissions = pop_option(options, 'ignore_permissions', True, False, bool)
options.extend(mount_options.split(","))
ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool)
if ignore_permissions:
# in case users have a use-case that requires NOT giving "default_permissions",
# this is enabled by the custom "ignore_permissions" mount option which just
# removes "default_permissions" again:
pop_option(options, 'default_permissions', True, False, bool)
self.allow_damaged_files = pop_option(options, 'allow_damaged_files', True, False, bool)
self.versions = pop_option(options, 'versions', True, False, bool)
self.uid_forced = pop_option(options, 'uid', None, None, int)
self.gid_forced = pop_option(options, 'gid', None, None, int)
self.umask = pop_option(options, 'umask', 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222
pop_option(options, "default_permissions", True, False, bool)
self.allow_damaged_files = pop_option(options, "allow_damaged_files", True, False, bool)
self.versions = pop_option(options, "versions", True, False, bool)
self.uid_forced = pop_option(options, "uid", None, None, int)
self.gid_forced = pop_option(options, "gid", None, None, int)
self.umask = pop_option(options, "umask", 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222
dir_uid = self.uid_forced if self.uid_forced is not None else self.default_uid
dir_gid = self.gid_forced if self.gid_forced is not None else self.default_gid
dir_user = uid2user(dir_uid)
@ -523,8 +539,9 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
assert isinstance(dir_user, str)
assert isinstance(dir_group, str)
dir_mode = 0o40755 & ~self.umask
self.default_dir = Item(mode=dir_mode, mtime=int(time.time() * 1e9),
user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid)
self.default_dir = Item(
mode=dir_mode, mtime=int(time.time() * 1e9), user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid
)
self._create_filesystem()
llfuse.init(self, mountpoint, options)
if not foreground:
@ -533,7 +550,7 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
else:
with daemonizing() as (old_id, new_id):
# local repo: the locking process' PID is changing, migrate it:
logger.debug('fuse: mount local repo, going to background: migrating lock.')
logger.debug("fuse: mount local repo, going to background: migrating lock.")
self.repository_uncached.migrate_lock(old_id, new_id)
# If the file system crashes, we do not want to umount because in that
@ -543,11 +560,10 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
# mirror.
umount = False
try:
with signal_handler('SIGUSR1', self.sig_info_handler), \
signal_handler('SIGINFO', self.sig_info_handler):
with signal_handler("SIGUSR1", self.sig_info_handler), signal_handler("SIGINFO", self.sig_info_handler):
signal = fuse_main()
# no crash and no signal (or it's ^C and we're in the foreground) -> umount request
umount = (signal is None or (signal == SIGINT and foreground))
umount = signal is None or (signal == SIGINT and foreground)
finally:
llfuse.close(umount)
@ -573,19 +589,24 @@ def _getattr(self, inode, ctx=None):
entry.entry_timeout = 300
entry.attr_timeout = 300
entry.st_mode = item.mode & ~self.umask
entry.st_nlink = item.get('nlink', 1)
entry.st_uid, entry.st_gid = get_item_uid_gid(item, numeric=self.numeric_ids,
uid_default=self.default_uid, gid_default=self.default_gid,
uid_forced=self.uid_forced, gid_forced=self.gid_forced)
entry.st_rdev = item.get('rdev', 0)
entry.st_nlink = item.get("nlink", 1)
entry.st_uid, entry.st_gid = get_item_uid_gid(
item,
numeric=self.numeric_ids,
uid_default=self.default_uid,
gid_default=self.default_gid,
uid_forced=self.uid_forced,
gid_forced=self.gid_forced,
)
entry.st_rdev = item.get("rdev", 0)
entry.st_size = item.get_size()
entry.st_blksize = 512
entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize
# note: older archives only have mtime (not atime nor ctime)
entry.st_mtime_ns = mtime_ns = item.mtime
entry.st_atime_ns = item.get('atime', mtime_ns)
entry.st_ctime_ns = item.get('ctime', mtime_ns)
entry.st_birthtime_ns = item.get('birthtime', mtime_ns)
entry.st_atime_ns = item.get("atime", mtime_ns)
entry.st_ctime_ns = item.get("ctime", mtime_ns)
entry.st_birthtime_ns = item.get("birthtime", mtime_ns)
return entry
@async_wrapper
@ -595,22 +616,22 @@ def getattr(self, inode, ctx=None):
@async_wrapper
def listxattr(self, inode, ctx=None):
item = self.get_item(inode)
return item.get('xattrs', {}).keys()
return item.get("xattrs", {}).keys()
@async_wrapper
def getxattr(self, inode, name, ctx=None):
item = self.get_item(inode)
try:
return item.get('xattrs', {})[name] or b''
return item.get("xattrs", {})[name] or b""
except KeyError:
raise llfuse.FUSEError(llfuse.ENOATTR) from None
@async_wrapper
def lookup(self, parent_inode, name, ctx=None):
self.check_pending_archive(parent_inode)
if name == b'.':
if name == b".":
inode = parent_inode
elif name == b'..':
elif name == b"..":
inode = self.parent[parent_inode]
else:
inode = self.contents[parent_inode].get(name)
@ -622,12 +643,14 @@ def lookup(self, parent_inode, name, ctx=None):
def open(self, inode, flags, ctx=None):
if not self.allow_damaged_files:
item = self.get_item(inode)
if 'chunks_healthy' in item:
if "chunks_healthy" in item:
# Processed archive items don't carry the path anymore; for converting the inode
# to the path we'd either have to store the inverse of the current structure,
# or search the entire archive. So we just don't print it. It's easy to correlate anyway.
logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. '
'Mount with allow_damaged_files to read damaged files.')
logger.warning(
"File has damaged (all-zero) chunks. Try running borg check --repair. "
"Mount with allow_damaged_files to read damaged files."
)
raise llfuse.FUSEError(errno.EIO)
return llfuse.FileInfo(fh=inode) if has_pyfuse3 else inode
@ -669,7 +692,7 @@ def read(self, fh, offset, size):
if offset + n < len(data):
# chunk was only partially read, cache it
self.data_cache[id] = data
parts.append(data[offset:offset + n])
parts.append(data[offset : offset + n])
offset = 0
size -= n
if not size:
@ -678,12 +701,13 @@ def read(self, fh, offset, size):
else:
self._last_pos[fh] = (chunk_no, chunk_offset)
break
return b''.join(parts)
return b"".join(parts)
# note: we can't have a generator (with yield) and not a generator (async) in the same method
if has_pyfuse3:
async def readdir(self, fh, off, token):
entries = [(b'.', fh), (b'..', self.parent[fh])]
entries = [(b".", fh), (b"..", self.parent[fh])]
entries.extend(self.contents[fh].items())
for i, (name, inode) in enumerate(entries[off:], off):
attrs = self._getattr(inode)
@ -691,8 +715,9 @@ async def readdir(self, fh, off, token):
break
else:
def readdir(self, fh, off):
entries = [(b'.', fh), (b'..', self.parent[fh])]
entries = [(b".", fh), (b"..", self.parent[fh])]
entries.extend(self.contents[fh].items())
for i, (name, inode) in enumerate(entries[off:], off):
attrs = self._getattr(inode)

10
src/borg/fuse_impl.py
View File

@ -4,11 +4,11 @@
import os
BORG_FUSE_IMPL = os.environ.get('BORG_FUSE_IMPL', 'pyfuse3,llfuse')
BORG_FUSE_IMPL = os.environ.get("BORG_FUSE_IMPL", "pyfuse3,llfuse")
for FUSE_IMPL in BORG_FUSE_IMPL.split(','):
for FUSE_IMPL in BORG_FUSE_IMPL.split(","):
FUSE_IMPL = FUSE_IMPL.strip()
if FUSE_IMPL == 'pyfuse3':
if FUSE_IMPL == "pyfuse3":
try:
import pyfuse3 as llfuse
except ImportError:
@ -17,7 +17,7 @@
has_llfuse = False
has_pyfuse3 = True
break
elif FUSE_IMPL == 'llfuse':
elif FUSE_IMPL == "llfuse":
try:
import llfuse
except ImportError:
@ -26,7 +26,7 @@
has_llfuse = True
has_pyfuse3 = False
break
elif FUSE_IMPL == 'none':
elif FUSE_IMPL == "none":
pass
else:
raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL)

2
src/borg/helpers/__init__.py
View File

@ -24,7 +24,7 @@
# generic mechanism to enable users to invoke workarounds by setting the
# BORG_WORKAROUNDS environment variable to a list of comma-separated strings.
# see the docs for a list of known workaround strings.
workarounds = tuple(os.environ.get('BORG_WORKAROUNDS', '').split(','))
workarounds = tuple(os.environ.get("BORG_WORKAROUNDS", "").split(","))
"""
The global exit_code variable is used so that modules other than archiver can increase the program exit code if a

13
src/borg/helpers/checks.py
View File

@ -23,15 +23,16 @@ class ExtensionModuleError(Error):
def check_extension_modules():
from .. import platform, compress, crypto, item, chunker, hashindex
if hashindex.API_VERSION != '1.2_01':
if hashindex.API_VERSION != "1.2_01":
raise ExtensionModuleError
if chunker.API_VERSION != '1.2_01':
if chunker.API_VERSION != "1.2_01":
raise ExtensionModuleError
if compress.API_VERSION != '1.2_02':
if compress.API_VERSION != "1.2_02":
raise ExtensionModuleError
if crypto.low_level.API_VERSION != '1.3_01':
if crypto.low_level.API_VERSION != "1.3_01":
raise ExtensionModuleError
if item.API_VERSION != '1.2_01':
if item.API_VERSION != "1.2_01":
raise ExtensionModuleError
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.2_05':
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != "1.2_05":
raise ExtensionModuleError

5
src/borg/helpers/datastruct.py
View File

@ -3,6 +3,7 @@
class StableDict(dict):
"""A dict subclass with stable items() ordering"""
def items(self):
return sorted(super().items())
@ -20,8 +21,8 @@ def __init__(self, allocator, size=4096, limit=None):
Initialize the buffer: use allocator(size) call to allocate a buffer.
Optionally, set the upper <limit> for the buffer size.
"""
assert callable(allocator), 'must give alloc(size) function as first param'
assert limit is None or size <= limit, 'initial size must be <= limit'
assert callable(allocator), "must give alloc(size) function as first param"
assert limit is None or size <= limit, "initial size must be <= limit"
self.allocator = allocator
self.limit = limit
self.resize(size, init=True)

2
src/borg/helpers/errors.py
View File

@ -5,6 +5,7 @@
class Error(Exception):
"""Error: {}"""
# Error base class
# if we raise such an Error and it is only caught by the uppermost
@ -26,6 +27,7 @@ def get_message(self):
class ErrorWithTraceback(Error):
"""Error: {}"""
# like Error, but show a traceback also
traceback = True

94
src/borg/helpers/fs.py
View File

@ -16,6 +16,7 @@
from ..constants import * # NOQA
from ..logger import create_logger
logger = create_logger()
@ -50,32 +51,32 @@ def get_base_dir():
- ~$USER, if USER is set
- ~
"""
base_dir = os.environ.get('BORG_BASE_DIR') or os.environ.get('HOME')
base_dir = os.environ.get("BORG_BASE_DIR") or os.environ.get("HOME")
# os.path.expanduser() behaves differently for '~' and '~someuser' as
# parameters: when called with an explicit username, the possibly set
# environment variable HOME is no longer respected. So we have to check if
# it is set and only expand the user's home directory if HOME is unset.
if not base_dir:
base_dir = os.path.expanduser('~%s' % os.environ.get('USER', ''))
base_dir = os.path.expanduser("~%s" % os.environ.get("USER", ""))
return base_dir
def get_keys_dir():
"""Determine where to repository keys and cache"""
keys_dir = os.environ.get('BORG_KEYS_DIR')
keys_dir = os.environ.get("BORG_KEYS_DIR")
if keys_dir is None:
# note: do not just give this as default to the environment.get(), see issue #5979.
keys_dir = os.path.join(get_config_dir(), 'keys')
keys_dir = os.path.join(get_config_dir(), "keys")
ensure_dir(keys_dir)
return keys_dir
def get_security_dir(repository_id=None):
"""Determine where to store local security information."""
security_dir = os.environ.get('BORG_SECURITY_DIR')
security_dir = os.environ.get("BORG_SECURITY_DIR")
if security_dir is None:
# note: do not just give this as default to the environment.get(), see issue #5979.
security_dir = os.path.join(get_config_dir(), 'security')
security_dir = os.path.join(get_config_dir(), "security")
if repository_id:
security_dir = os.path.join(security_dir, repository_id)
ensure_dir(security_dir)
@ -85,22 +86,28 @@ def get_security_dir(repository_id=None):
def get_cache_dir():
"""Determine where to repository keys and cache"""
# Get cache home path
cache_home = os.path.join(get_base_dir(), '.cache')
cache_home = os.path.join(get_base_dir(), ".cache")
# Try to use XDG_CACHE_HOME instead if BORG_BASE_DIR isn't explicitly set
if not os.environ.get('BORG_BASE_DIR'):
cache_home = os.environ.get('XDG_CACHE_HOME', cache_home)
if not os.environ.get("BORG_BASE_DIR"):
cache_home = os.environ.get("XDG_CACHE_HOME", cache_home)
# Use BORG_CACHE_DIR if set, otherwise assemble final path from cache home path
cache_dir = os.environ.get('BORG_CACHE_DIR', os.path.join(cache_home, 'borg'))
cache_dir = os.environ.get("BORG_CACHE_DIR", os.path.join(cache_home, "borg"))
# Create path if it doesn't exist yet
ensure_dir(cache_dir)
cache_tag_fn = os.path.join(cache_dir, CACHE_TAG_NAME)
if not os.path.exists(cache_tag_fn):
cache_tag_contents = CACHE_TAG_CONTENTS + textwrap.dedent("""
cache_tag_contents = (
CACHE_TAG_CONTENTS
+ textwrap.dedent(
"""
# This file is a cache directory tag created by Borg.
# For information about cache directory tags, see:
# http://www.bford.info/cachedir/spec.html
""").encode('ascii')
"""
).encode("ascii")
)
from ..platform import SaveFile
with SaveFile(cache_tag_fn, binary=True) as fd:
fd.write(cache_tag_contents)
return cache_dir
@ -109,12 +116,12 @@ def get_cache_dir():
def get_config_dir():
"""Determine where to store whole config"""
# Get config home path
config_home = os.path.join(get_base_dir(), '.config')
config_home = os.path.join(get_base_dir(), ".config")
# Try to use XDG_CONFIG_HOME instead if BORG_BASE_DIR isn't explicitly set
if not os.environ.get('BORG_BASE_DIR'):
config_home = os.environ.get('XDG_CONFIG_HOME', config_home)
if not os.environ.get("BORG_BASE_DIR"):
config_home = os.environ.get("XDG_CONFIG_HOME", config_home)
# Use BORG_CONFIG_DIR if set, otherwise assemble final path from config home path
config_dir = os.environ.get('BORG_CONFIG_DIR', os.path.join(config_home, 'borg'))
config_dir = os.environ.get("BORG_CONFIG_DIR", os.path.join(config_home, "borg"))
# Create path if it doesn't exist yet
ensure_dir(config_dir)
return config_dir
@ -130,7 +137,7 @@ def dir_is_cachedir(path):
tag_path = os.path.join(path, CACHE_TAG_NAME)
try:
if os.path.exists(tag_path):
with open(tag_path, 'rb') as tag_file:
with open(tag_path, "rb") as tag_file:
tag_data = tag_file.read(len(CACHE_TAG_CONTENTS))
if tag_data == CACHE_TAG_CONTENTS:
return True
@ -157,13 +164,12 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
return tag_names
_safe_re = re.compile(r'^((\.\.)?/+)+')
_safe_re = re.compile(r"^((\.\.)?/+)+")
def make_path_safe(path):
"""Make path safe by making it relative and local
"""
return _safe_re.sub('', path) or '.'
"""Make path safe by making it relative and local"""
return _safe_re.sub("", path) or "."
class HardLinkManager:
@ -189,6 +195,7 @@ class HardLinkManager:
For better hardlink support (including the very first hardlink item for each group of same-target hardlinks),
we would need a 2-pass processing, which is not yet implemented.
"""
def __init__(self, *, id_type, info_type):
self._map = {}
self.id_type = id_type
@ -198,21 +205,21 @@ def borg1_hardlinkable(self, mode): # legacy
return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
def borg1_hardlink_master(self, item): # legacy
return item.get('hardlink_master', True) and 'source' not in item and self.borg1_hardlinkable(item.mode)
return item.get("hardlink_master", True) and "source" not in item and self.borg1_hardlinkable(item.mode)
def borg1_hardlink_slave(self, item): # legacy
return 'source' in item and self.borg1_hardlinkable(item.mode)
return "source" in item and self.borg1_hardlinkable(item.mode)
def hardlink_id_from_path(self, path):
"""compute a hardlink id from a path"""
assert isinstance(path, str)
return hashlib.sha256(path.encode('utf-8', errors='surrogateescape')).digest()
return hashlib.sha256(path.encode("utf-8", errors="surrogateescape")).digest()
def hardlink_id_from_inode(self, *, ino, dev):
"""compute a hardlink id from an inode"""
assert isinstance(ino, int)
assert isinstance(dev, int)
return hashlib.sha256(f'{ino}/{dev}'.encode()).digest()
return hashlib.sha256(f"{ino}/{dev}".encode()).digest()
def remember(self, *, id, info):
"""
@ -243,7 +250,7 @@ def scandir_keyfunc(dirent):
return (0, dirent.inode())
except OSError as e:
# maybe a permission denied error while doing a stat() on the dirent
logger.debug('scandir_inorder: Unable to stat %s: %s', dirent.path, e)
logger.debug("scandir_inorder: Unable to stat %s: %s", dirent.path, e)
# order this dirent after all the others lexically by file name
# we may not break the whole scandir just because of an exception in one dirent
# ignore the exception for now, since another stat will be done later anyways
@ -268,7 +275,7 @@ def secure_erase(path, *, avoid_collateral_damage):
If avoid_collateral_damage is False, we always secure erase.
If there are hardlinks pointing to the same inode as <path>, they will contain random garbage afterwards.
"""
with open(path, 'r+b') as fd:
with open(path, "r+b") as fd:
st = os.stat(fd.fileno())
if not (st.st_nlink > 1 and avoid_collateral_damage):
fd.write(os.urandom(st.st_size))
@ -303,7 +310,7 @@ def safe_unlink(path):
# no other hardlink! try to recover free space by truncating this file.
try:
# Do not create *path* if it does not exist, open for truncation in r+b mode (=O_RDWR|O_BINARY).
with open(path, 'r+b') as fd:
with open(path, "r+b") as fd:
fd.truncate()
except OSError:
# truncate didn't work, so we still have the original unlink issue - give up:
@ -314,10 +321,10 @@ def safe_unlink(path):
def dash_open(path, mode):
assert '+' not in mode # the streams are either r or w, but never both
if path == '-':
stream = sys.stdin if 'r' in mode else sys.stdout
return stream.buffer if 'b' in mode else stream
assert "+" not in mode # the streams are either r or w, but never both
if path == "-":
stream = sys.stdin if "r" in mode else sys.stdout
return stream.buffer if "b" in mode else stream
else:
return open(path, mode)
@ -325,17 +332,17 @@ def dash_open(path, mode):
def O_(*flags):
result = 0
for flag in flags:
result |= getattr(os, 'O_' + flag, 0)
result |= getattr(os, "O_" + flag, 0)
return result
flags_base = O_('BINARY', 'NOCTTY', 'RDONLY')
flags_special = flags_base | O_('NOFOLLOW') # BLOCK == wait when reading devices or fifos
flags_base = O_("BINARY", "NOCTTY", "RDONLY")
flags_special = flags_base | O_("NOFOLLOW") # BLOCK == wait when reading devices or fifos
flags_special_follow = flags_base # BLOCK == wait when reading symlinked devices or fifos
flags_normal = flags_base | O_('NONBLOCK', 'NOFOLLOW')
flags_noatime = flags_normal | O_('NOATIME')
flags_root = O_('RDONLY')
flags_dir = O_('DIRECTORY', 'RDONLY', 'NOFOLLOW')
flags_normal = flags_base | O_("NONBLOCK", "NOFOLLOW")
flags_noatime = flags_normal | O_("NOATIME")
flags_root = O_("RDONLY")
flags_dir = O_("DIRECTORY", "RDONLY", "NOFOLLOW")
def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
@ -362,7 +369,7 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
return None
_flags_normal = flags
if noatime:
_flags_noatime = _flags_normal | O_('NOATIME')
_flags_noatime = _flags_normal | O_("NOATIME")
try:
# if we have O_NOATIME, this likely will succeed if we are root or owner of file:
fd = os.open(fname, _flags_noatime, dir_fd=parent_fd)
@ -375,7 +382,8 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
except OSError as exc:
# O_NOATIME causes EROFS when accessing a volume shadow copy in WSL1
from . import workarounds
if 'retry_erofs' in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal:
if "retry_erofs" in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal:
fd = os.open(fname, _flags_normal, dir_fd=parent_fd)
else:
raise
@ -407,6 +415,6 @@ def os_stat(*, path=None, parent_fd=None, name=None, follow_symlinks=False):
def umount(mountpoint):
env = prepare_subprocess_env(system=True)
try:
return subprocess.call(['fusermount', '-u', mountpoint], env=env)
return subprocess.call(["fusermount", "-u", mountpoint], env=env)
except FileNotFoundError:
return subprocess.call(['umount', mountpoint], env=env)
return subprocess.call(["umount", mountpoint], env=env)

91
src/borg/helpers/manifest.py
View File

@ -9,6 +9,7 @@
from .errors import Error
from ..logger import create_logger
logger = create_logger()
from .datastruct import StableDict
@ -26,10 +27,10 @@ class MandatoryFeatureUnsupported(Error):
"""Unsupported repository feature(s) {}. A newer version of borg is required to access this repository."""
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')
ArchiveInfo = namedtuple("ArchiveInfo", "name id ts")
AI_HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields)
AI_HUMAN_SORT_KEYS.remove('ts')
AI_HUMAN_SORT_KEYS = ["timestamp"] + list(ArchiveInfo._fields)
AI_HUMAN_SORT_KEYS.remove("ts")
class Archives(abc.MutableMapping):
@ -38,6 +39,7 @@ class Archives(abc.MutableMapping):
and we can deal with str keys (and it internally encodes to byte keys) and either
str timestamps or datetime timestamps.
"""
def __init__(self):
# key: str archive name, value: dict('id': bytes_id, 'time': str_iso_ts)
self._archives = {}
@ -53,8 +55,8 @@ def __getitem__(self, name):
values = self._archives.get(name)
if values is None:
raise KeyError
ts = parse_timestamp(values['time'])
return ArchiveInfo(name=name, id=values['id'], ts=ts)
ts = parse_timestamp(values["time"])
return ArchiveInfo(name=name, id=values["id"], ts=ts)
def __setitem__(self, name, info):
assert isinstance(name, str)
@ -64,13 +66,15 @@ def __setitem__(self, name, info):
if isinstance(ts, datetime):
ts = ts.replace(tzinfo=None).strftime(ISO_FORMAT)
assert isinstance(ts, str)
self._archives[name] = {'id': id, 'time': ts}
self._archives[name] = {"id": id, "time": ts}
def __delitem__(self, name):
assert isinstance(name, str)
del self._archives[name]
def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False):
def list(
self, *, glob=None, match_end=r"\Z", sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False
):
"""
Return list of ArchiveInfo instances according to the parameters.
@ -84,17 +88,17 @@ def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=T
some callers EXPECT to iterate over all archives in a repo for correct operation.
"""
if isinstance(sort_by, (str, bytes)):
raise TypeError('sort_by must be a sequence of str')
regex = re.compile(shellpattern.translate(glob or '*', match_end=match_end))
raise TypeError("sort_by must be a sequence of str")
regex = re.compile(shellpattern.translate(glob or "*", match_end=match_end))
archives = [x for x in self.values() if regex.match(x.name) is not None]
if not consider_checkpoints:
archives = [x for x in archives if '.checkpoint' not in x.name]
archives = [x for x in archives if ".checkpoint" not in x.name]
for sortkey in reversed(sort_by):
archives.sort(key=attrgetter(sortkey))
if first:
archives = archives[:first]
elif last:
archives = archives[max(len(archives) - last, 0):]
archives = archives[max(len(archives) - last, 0) :]
if reverse:
archives.reverse()
return archives
@ -103,17 +107,25 @@ def list_considering(self, args):
"""
get a list of archives, considering --first/last/prefix/glob-archives/sort/consider-checkpoints cmdline args
"""
name = getattr(args, 'name', None)
consider_checkpoints = getattr(args, 'consider_checkpoints', None)
name = getattr(args, "name", None)
consider_checkpoints = getattr(args, "consider_checkpoints", None)
if name is not None:
raise Error('Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints.')
return self.list(sort_by=args.sort_by.split(','), consider_checkpoints=consider_checkpoints, glob=args.glob_archives, first=args.first, last=args.last)
raise Error(
"Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints."
)
return self.list(
sort_by=args.sort_by.split(","),
consider_checkpoints=consider_checkpoints,
glob=args.glob_archives,
first=args.first,
last=args.last,
)
def set_raw_dict(self, d):
"""set the dict we get from the msgpack unpacker"""
for k, v in d.items():
assert isinstance(k, str)
assert isinstance(v, dict) and 'id' in v and 'time' in v
assert isinstance(v, dict) and "id" in v and "time" in v
self._archives[k] = v
def get_raw_dict(self):
@ -122,7 +134,6 @@ def get_raw_dict(self):
class Manifest:
@enum.unique
class Operation(enum.Enum):
# The comments here only roughly describe the scope of each feature. In the end, additions need to be
@ -133,25 +144,25 @@ class Operation(enum.Enum):
# The READ operation describes which features are needed to safely list and extract the archives in the
# repository.
READ = 'read'
READ = "read"
# The CHECK operation is for all operations that need either to understand every detail
# of the repository (for consistency checks and repairs) or are seldom used functions that just
# should use the most restrictive feature set because more fine grained compatibility tracking is
# not needed.
CHECK = 'check'
CHECK = "check"
# The WRITE operation is for adding archives. Features here ensure that older clients don't add archives
# in an old format, or is used to lock out clients that for other reasons can no longer safely add new
# archives.
WRITE = 'write'
WRITE = "write"
# The DELETE operation is for all operations (like archive deletion) that need a 100% correct reference
# count and the need to be able to find all (directly and indirectly) referenced chunks of a given archive.
DELETE = 'delete'
DELETE = "delete"
NO_OPERATION_CHECK = tuple()
SUPPORTED_REPO_FEATURES = frozenset([])
MANIFEST_ID = b'\0' * 32
MANIFEST_ID = b"\0" * 32
def __init__(self, key, repository, item_keys=None):
self.archives = Archives()
@ -175,6 +186,7 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
from ..item import ManifestItem
from ..crypto.key import key_factory, tam_required_file, tam_required
from ..repository import Repository
try:
cdata = repository.get(cls.MANIFEST_ID)
except Repository.ObjectNotFound:
@ -183,26 +195,28 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
key = key_factory(repository, cdata)
manifest = cls(key, repository)
data = key.decrypt(cls.MANIFEST_ID, cdata)
manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(data, force_tam_not_required=force_tam_not_required)
manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(
data, force_tam_not_required=force_tam_not_required
)
m = ManifestItem(internal_dict=manifest_dict)
manifest.id = key.id_hash(data)
if m.get('version') not in (1, 2):
raise ValueError('Invalid manifest version')
if m.get("version") not in (1, 2):
raise ValueError("Invalid manifest version")
manifest.archives.set_raw_dict(m.archives)
manifest.timestamp = m.get('timestamp')
manifest.timestamp = m.get("timestamp")
manifest.config = m.config
# valid item keys are whatever is known in the repo or every key we know
manifest.item_keys = ITEM_KEYS | frozenset(m.get('item_keys', []))
manifest.item_keys = ITEM_KEYS | frozenset(m.get("item_keys", []))
if manifest.tam_verified:
manifest_required = manifest.config.get('tam_required', False)
manifest_required = manifest.config.get("tam_required", False)
security_required = tam_required(repository)
if manifest_required and not security_required:
logger.debug('Manifest is TAM verified and says TAM is required, updating security database...')
logger.debug("Manifest is TAM verified and says TAM is required, updating security database...")
file = tam_required_file(repository)
open(file, 'w').close()
open(file, "w").close()
if not manifest_required and security_required:
logger.debug('Manifest is TAM verified and says TAM is *not* required, updating security database...')
logger.debug("Manifest is TAM verified and says TAM is *not* required, updating security database...")
os.unlink(tam_required_file(repository))
manifest.check_repository_compatibility(operations)
return manifest, key
@ -210,32 +224,33 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
def check_repository_compatibility(self, operations):
for operation in operations:
assert isinstance(operation, self.Operation)
feature_flags = self.config.get('feature_flags', None)
feature_flags = self.config.get("feature_flags", None)
if feature_flags is None:
return
if operation.value not in feature_flags:
continue
requirements = feature_flags[operation.value]
if 'mandatory' in requirements:
unsupported = set(requirements['mandatory']) - self.SUPPORTED_REPO_FEATURES
if "mandatory" in requirements:
unsupported = set(requirements["mandatory"]) - self.SUPPORTED_REPO_FEATURES
if unsupported:
raise MandatoryFeatureUnsupported(list(unsupported))
def get_all_mandatory_features(self):
result = {}
feature_flags = self.config.get('feature_flags', None)
feature_flags = self.config.get("feature_flags", None)
if feature_flags is None:
return result
for operation, requirements in feature_flags.items():
if 'mandatory' in requirements:
result[operation] = set(requirements['mandatory'])
if "mandatory" in requirements:
result[operation] = set(requirements["mandatory"])
return result
def write(self):
from ..item import ManifestItem
if self.key.tam_required:
self.config['tam_required'] = True
self.config["tam_required"] = True
# self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
if self.timestamp is None:
self.timestamp = datetime.utcnow().strftime(ISO_FORMAT)

103
src/borg/helpers/misc.py
View File

@ -10,6 +10,7 @@
from operator import attrgetter
from ..logger import create_logger
logger = create_logger()
from .time import to_localtime
@ -30,15 +31,17 @@ def prune_within(archives, hours, kept_because):
return result
PRUNING_PATTERNS = OrderedDict([
("secondly", '%Y-%m-%d %H:%M:%S'),
("minutely", '%Y-%m-%d %H:%M'),
("hourly", '%Y-%m-%d %H'),
("daily", '%Y-%m-%d'),
("weekly", '%G-%V'),
("monthly", '%Y-%m'),
("yearly", '%Y'),
])
PRUNING_PATTERNS = OrderedDict(
[
("secondly", "%Y-%m-%d %H:%M:%S"),
("minutely", "%Y-%m-%d %H:%M"),
("hourly", "%Y-%m-%d %H"),
("daily", "%Y-%m-%d"),
("weekly", "%G-%V"),
("monthly", "%Y-%m"),
("yearly", "%Y"),
]
)
def prune_split(archives, rule, n, kept_because=None):
@ -51,7 +54,7 @@ def prune_split(archives, rule, n, kept_because=None):
return keep
a = None
for a in sorted(archives, key=attrgetter('ts'), reverse=True):
for a in sorted(archives, key=attrgetter("ts"), reverse=True):
period = to_localtime(a.ts).strftime(pattern)
if period != last:
last = period
@ -63,14 +66,14 @@ def prune_split(archives, rule, n, kept_because=None):
# Keep oldest archive if we didn't reach the target retention count
if a is not None and len(keep) < n and a.id not in kept_because:
keep.append(a)
kept_because[a.id] = (rule+"[oldest]", len(keep))
kept_because[a.id] = (rule + "[oldest]", len(keep))
return keep
def sysinfo():
show_sysinfo = os.environ.get('BORG_SHOW_SYSINFO', 'yes').lower()
if show_sysinfo == 'no':
return ''
show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
if show_sysinfo == "no":
return ""
python_implementation = platform.python_implementation()
python_version = platform.python_version()
@ -80,30 +83,34 @@ def sysinfo():
uname = os.uname()
except AttributeError:
uname = None
if sys.platform.startswith('linux'):
linux_distribution = ('Unknown Linux', '', '')
if sys.platform.startswith("linux"):
linux_distribution = ("Unknown Linux", "", "")
else:
linux_distribution = None
try:
msgpack_version = '.'.join(str(v) for v in msgpack.version)
msgpack_version = ".".join(str(v) for v in msgpack.version)
except:
msgpack_version = 'unknown'
msgpack_version = "unknown"
from ..fuse_impl import llfuse, BORG_FUSE_IMPL
llfuse_name = llfuse.__name__ if llfuse else 'None'
llfuse_version = (' %s' % llfuse.__version__) if llfuse else ''
llfuse_info = f'{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]'
llfuse_name = llfuse.__name__ if llfuse else "None"
llfuse_version = (" %s" % llfuse.__version__) if llfuse else ""
llfuse_info = f"{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]"
info = []
if uname is not None:
info.append('Platform: {}'.format(' '.join(uname)))
info.append("Platform: {}".format(" ".join(uname)))
if linux_distribution is not None:
info.append('Linux: %s %s %s' % linux_distribution)
info.append('Borg: {} Python: {} {} msgpack: {} fuse: {}'.format(
borg_version, python_implementation, python_version, msgpack_version, llfuse_info))
info.append('PID: %d CWD: %s' % (os.getpid(), os.getcwd()))
info.append('sys.argv: %r' % sys.argv)
info.append('SSH_ORIGINAL_COMMAND: %r' % os.environ.get('SSH_ORIGINAL_COMMAND'))
info.append('')
return '\n'.join(info)
info.append("Linux: %s %s %s" % linux_distribution)
info.append(
"Borg: {} Python: {} {} msgpack: {} fuse: {}".format(
borg_version, python_implementation, python_version, msgpack_version, llfuse_info
)
)
info.append("PID: %d CWD: %s" % (os.getpid(), os.getcwd()))
info.append("sys.argv: %r" % sys.argv)
info.append("SSH_ORIGINAL_COMMAND: %r" % os.environ.get("SSH_ORIGINAL_COMMAND"))
info.append("")
return "\n".join(info)
def log_multi(*msgs, level=logging.INFO, logger=logger):
@ -133,7 +140,7 @@ def __init__(self, chunk_iterator, read_callback=None):
"""
self.chunk_iterator = chunk_iterator
self.chunk_offset = 0
self.chunk = b''
self.chunk = b""
self.exhausted = False
self.read_callback = read_callback
@ -152,11 +159,11 @@ def _refill(self):
def _read(self, nbytes):
if not nbytes:
return b''
return b""
remaining = self._refill()
will_read = min(remaining, nbytes)
self.chunk_offset += will_read
return self.chunk[self.chunk_offset - will_read:self.chunk_offset]
return self.chunk[self.chunk_offset - will_read : self.chunk_offset]
def read(self, nbytes):
parts = []
@ -166,7 +173,7 @@ def read(self, nbytes):
parts.append(read_data)
if self.read_callback:
self.read_callback(read_data)
return b''.join(parts)
return b"".join(parts)
def open_item(archive, item):
@ -207,7 +214,7 @@ def read(self, n):
super().close()
except OSError:
pass
return ''
return ""
def write(self, s):
if not self.closed:
@ -225,8 +232,8 @@ def iter_separated(fd, sep=None, read_size=4096):
"""Iter over chunks of open file ``fd`` delimited by ``sep``. Doesn't trim."""
buf = fd.read(read_size)
is_str = isinstance(buf, str)
part = '' if is_str else b''
sep = sep or ('\n' if is_str else b'\n')
part = "" if is_str else b""
sep = sep or ("\n" if is_str else b"\n")
while len(buf) > 0:
part2, *items = buf.split(sep)
*full, part = (part + part2, *items)
@ -240,17 +247,17 @@ def iter_separated(fd, sep=None, read_size=4096):
def get_tar_filter(fname, decompress):
# Note that filter is None if fname is '-'.
if fname.endswith(('.tar.gz', '.tgz')):
filter = 'gzip -d' if decompress else 'gzip'
elif fname.endswith(('.tar.bz2', '.tbz')):
filter = 'bzip2 -d' if decompress else 'bzip2'
elif fname.endswith(('.tar.xz', '.txz')):
filter = 'xz -d' if decompress else 'xz'
elif fname.endswith(('.tar.lz4', )):
filter = 'lz4 -d' if decompress else 'lz4'
elif fname.endswith(('.tar.zstd', )):
filter = 'zstd -d' if decompress else 'zstd'
if fname.endswith((".tar.gz", ".tgz")):
filter = "gzip -d" if decompress else "gzip"
elif fname.endswith((".tar.bz2", ".tbz")):
filter = "bzip2 -d" if decompress else "bzip2"
elif fname.endswith((".tar.xz", ".txz")):
filter = "xz -d" if decompress else "xz"
elif fname.endswith((".tar.lz4",)):
filter = "lz4 -d" if decompress else "lz4"
elif fname.endswith((".tar.zstd",)):
filter = "zstd -d" if decompress else "zstd"
else:
filter = None
logger.debug('Automatically determined tar filter: %s', filter)
logger.debug("Automatically determined tar filter: %s", filter)
return filter

100
src/borg/helpers/msgpack.py
View File

@ -64,7 +64,7 @@
USE_BIN_TYPE = True
RAW = False
UNICODE_ERRORS = 'surrogateescape'
UNICODE_ERRORS = "surrogateescape"
class PackException(Exception):
@ -76,13 +76,25 @@ class UnpackException(Exception):
class Packer(mp_Packer):
def __init__(self, *, default=None, unicode_errors=UNICODE_ERRORS,
use_single_float=False, autoreset=True, use_bin_type=USE_BIN_TYPE,
strict_types=False):
def __init__(
self,
*,
default=None,
unicode_errors=UNICODE_ERRORS,
use_single_float=False,
autoreset=True,
use_bin_type=USE_BIN_TYPE,
strict_types=False
):
assert unicode_errors == UNICODE_ERRORS
super().__init__(default=default, unicode_errors=unicode_errors,
use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type,
strict_types=strict_types)
super().__init__(
default=default,
unicode_errors=unicode_errors,
use_single_float=use_single_float,
autoreset=autoreset,
use_bin_type=use_bin_type,
strict_types=strict_types,
)
def pack(self, obj):
try:
@ -108,18 +120,36 @@ def pack(o, stream, *, use_bin_type=USE_BIN_TYPE, unicode_errors=UNICODE_ERRORS,
class Unpacker(mp_Unpacker):
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=RAW,
object_hook=None, object_pairs_hook=None, list_hook=None,
unicode_errors=UNICODE_ERRORS, max_buffer_size=0,
ext_hook=ExtType,
strict_map_key=False):
def __init__(
self,
file_like=None,
*,
read_size=0,
use_list=True,
raw=RAW,
object_hook=None,
object_pairs_hook=None,
list_hook=None,
unicode_errors=UNICODE_ERRORS,
max_buffer_size=0,
ext_hook=ExtType,
strict_map_key=False
):
assert raw == RAW
assert unicode_errors == UNICODE_ERRORS
kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
ext_hook=ext_hook,
strict_map_key=strict_map_key)
kw = dict(
file_like=file_like,
read_size=read_size,
use_list=use_list,
raw=raw,
object_hook=object_hook,
object_pairs_hook=object_pairs_hook,
list_hook=list_hook,
unicode_errors=unicode_errors,
max_buffer_size=max_buffer_size,
ext_hook=ext_hook,
strict_map_key=strict_map_key,
)
super().__init__(**kw)
def unpack(self):
@ -141,28 +171,22 @@ def __next__(self):
next = __next__
def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
strict_map_key=False,
**kwargs):
def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs):
assert raw == RAW
assert unicode_errors == UNICODE_ERRORS
try:
kw = dict(raw=raw, unicode_errors=unicode_errors,
strict_map_key=strict_map_key)
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
kw.update(kwargs)
return mp_unpackb(packed, **kw)
except Exception as e:
raise UnpackException(e)
def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
strict_map_key=False,
**kwargs):
def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs):
assert raw == RAW
assert unicode_errors == UNICODE_ERRORS
try:
kw = dict(raw=raw, unicode_errors=unicode_errors,
strict_map_key=strict_map_key)
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
kw.update(kwargs)
return mp_unpack(stream, **kw)
except Exception as e:
@ -171,32 +195,34 @@ def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
# msgpacking related utilities -----------------------------------------------
def is_slow_msgpack():
import msgpack
import msgpack.fallback
return msgpack.Packer is msgpack.fallback.Packer
def is_supported_msgpack():
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
import msgpack
return (1, 0, 3) <= msgpack.version <= (1, 0, 4) and \
msgpack.version not in [] # < add bad releases here to deny list
return (1, 0, 3) <= msgpack.version <= (
1,
0,
4,
) and msgpack.version not in [] # < add bad releases here to deny list
def get_limited_unpacker(kind):
"""return a limited Unpacker because we should not trust msgpack data received from remote"""
# Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for
# unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N).
args = dict(use_list=False, # return tuples, not lists
max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
)
if kind in ('server', 'client'):
args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE)) # return tuples, not lists
if kind in ("server", "client"):
pass # nothing special
elif kind in ('manifest', 'key'):
args.update(dict(use_list=True, # default value
object_hook=StableDict,
))
elif kind in ("manifest", "key"):
args.update(dict(use_list=True, object_hook=StableDict)) # default value
else:
raise ValueError('kind must be "server", "client", "manifest" or "key"')
return Unpacker(**args)

548
src/borg/helpers/parseformat.py
View File

@ -15,6 +15,7 @@
from string import Formatter
from ..logger import create_logger
logger = create_logger()
from .errors import Error
@ -28,34 +29,34 @@
def bin_to_hex(binary):
return hexlify(binary).decode('ascii')
return hexlify(binary).decode("ascii")
def safe_decode(s, coding='utf-8', errors='surrogateescape'):
def safe_decode(s, coding="utf-8", errors="surrogateescape"):
"""decode bytes to str, with round-tripping "invalid" bytes"""
if s is None:
return None
return s.decode(coding, errors)
def safe_encode(s, coding='utf-8', errors='surrogateescape'):
def safe_encode(s, coding="utf-8", errors="surrogateescape"):
"""encode str to bytes, with round-tripping "invalid" bytes"""
if s is None:
return None
return s.encode(coding, errors)
def remove_surrogates(s, errors='replace'):
def remove_surrogates(s, errors="replace"):
"""Replace surrogates generated by fsdecode with '?'"""
return s.encode('utf-8', errors).decode('utf-8')
return s.encode("utf-8", errors).decode("utf-8")
def eval_escapes(s):
"""Evaluate literal escape sequences in a string (eg `\\n` -> `\n`)."""
return s.encode('ascii', 'backslashreplace').decode('unicode-escape')
return s.encode("ascii", "backslashreplace").decode("unicode-escape")
def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'):
def decode_dict(d, keys, encoding="utf-8", errors="surrogateescape"):
for key in keys:
if isinstance(d.get(key), bytes):
d[key] = d[key].decode(encoding, errors)
@ -66,13 +67,13 @@ def positive_int_validator(value):
"""argparse type for positive integers"""
int_value = int(value)
if int_value <= 0:
raise argparse.ArgumentTypeError('A positive integer is required: %s' % value)
raise argparse.ArgumentTypeError("A positive integer is required: %s" % value)
return int_value
def interval(s):
"""Convert a string representing a valid interval to a number of hours."""
multiplier = {'H': 1, 'd': 24, 'w': 24 * 7, 'm': 24 * 31, 'y': 24 * 365}
multiplier = {"H": 1, "d": 24, "w": 24 * 7, "m": 24 * 31, "y": 24 * 365}
if s.endswith(tuple(multiplier.keys())):
number = s[:-1]
@ -80,8 +81,7 @@ def interval(s):
else:
# range suffixes in ascending multiplier order
ranges = [k for k, v in sorted(multiplier.items(), key=lambda t: t[1])]
raise argparse.ArgumentTypeError(
f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}')
raise argparse.ArgumentTypeError(f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}')
try:
hours = int(number) * multiplier[suffix]
@ -89,17 +89,16 @@ def interval(s):
hours = -1
if hours <= 0:
raise argparse.ArgumentTypeError(
'Unexpected interval number "%s": expected an integer greater than 0' % number)
raise argparse.ArgumentTypeError('Unexpected interval number "%s": expected an integer greater than 0' % number)
return hours
def ChunkerParams(s):
params = s.strip().split(',')
params = s.strip().split(",")
count = len(params)
if count == 0:
raise ValueError('no chunker params given')
raise ValueError("no chunker params given")
algo = params[0].lower()
if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size]
block_size = int(params[1])
@ -110,36 +109,36 @@ def ChunkerParams(s):
# or in-memory chunk management.
# choose the block (chunk) size wisely: if you have a lot of data and you cut
# it into very small chunks, you are asking for trouble!
raise ValueError('block_size must not be less than 64 Bytes')
raise ValueError("block_size must not be less than 64 Bytes")
if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
raise ValueError('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE)
raise ValueError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE)
return algo, block_size, header_size
if algo == 'default' and count == 1: # default
if algo == "default" and count == 1: # default
return CHUNKER_PARAMS
# this must stay last as it deals with old-style compat mode (no algorithm, 4 params, buzhash):
if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:])
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4 :])
if not (chunk_min <= chunk_mask <= chunk_max):
raise ValueError('required: chunk_min <= chunk_mask <= chunk_max')
raise ValueError("required: chunk_min <= chunk_mask <= chunk_max")
if chunk_min < 6:
# see comment in 'fixed' algo check
raise ValueError('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)')
raise ValueError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
if chunk_max > 23:
raise ValueError('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)')
raise ValueError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
raise ValueError('invalid chunker params')
raise ValueError("invalid chunker params")
def FilesCacheMode(s):
ENTRIES_MAP = dict(ctime='c', mtime='m', size='s', inode='i', rechunk='r', disabled='d')
VALID_MODES = ('cis', 'ims', 'cs', 'ms', 'cr', 'mr', 'd', 's') # letters in alpha order
entries = set(s.strip().split(','))
ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d")
VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s") # letters in alpha order
entries = set(s.strip().split(","))
if not entries <= set(ENTRIES_MAP):
raise ValueError('cache mode must be a comma-separated list of: %s' % ','.join(sorted(ENTRIES_MAP)))
raise ValueError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP)))
short_entries = {ENTRIES_MAP[entry] for entry in entries}
mode = ''.join(sorted(short_entries))
mode = "".join(sorted(short_entries))
if mode not in VALID_MODES:
raise ValueError('cache mode short must be one of: %s' % ','.join(VALID_MODES))
raise ValueError("cache mode short must be one of: %s" % ",".join(VALID_MODES))
return mode
@ -151,9 +150,9 @@ def partial_format(format, mapping):
"""
for key, value in mapping.items():
key = re.escape(key)
format = re.sub(fr'(?<!\{{)((\{{{key}\}})|(\{{{key}:[^\}}]*\}}))',
lambda match: match.group(1).format_map(mapping),
format)
format = re.sub(
rf"(?<!\{{)((\{{{key}\}})|(\{{{key}:[^\}}]*\}}))", lambda match: match.group(1).format_map(mapping), format
)
return format
@ -162,7 +161,7 @@ def __init__(self, dt):
self.dt = dt
def __format__(self, format_spec):
if format_spec == '':
if format_spec == "":
format_spec = ISO_FORMAT_NO_USECS
return self.dt.__format__(format_spec)
@ -190,20 +189,21 @@ def format_line(format, data):
def replace_placeholders(text, overrides={}):
"""Replace placeholders in text with their values."""
from ..platform import fqdn, hostname, getosusername
current_time = datetime.now(timezone.utc)
data = {
'pid': os.getpid(),
'fqdn': fqdn,
'reverse-fqdn': '.'.join(reversed(fqdn.split('.'))),
'hostname': hostname,
'now': DatetimeWrapper(current_time.astimezone(None)),
'utcnow': DatetimeWrapper(current_time),
'user': getosusername(),
'uuid4': str(uuid.uuid4()),
'borgversion': borg_version,
'borgmajor': '%d' % borg_version_tuple[:1],
'borgminor': '%d.%d' % borg_version_tuple[:2],
'borgpatch': '%d.%d.%d' % borg_version_tuple[:3],
"pid": os.getpid(),
"fqdn": fqdn,
"reverse-fqdn": ".".join(reversed(fqdn.split("."))),
"hostname": hostname,
"now": DatetimeWrapper(current_time.astimezone(None)),
"utcnow": DatetimeWrapper(current_time),
"user": getosusername(),
"uuid4": str(uuid.uuid4()),
"borgversion": borg_version,
"borgmajor": "%d" % borg_version_tuple[:1],
"borgminor": "%d.%d" % borg_version_tuple[:2],
"borgpatch": "%d.%d.%d" % borg_version_tuple[:3],
**overrides,
}
return format_line(text, data)
@ -220,17 +220,17 @@ def replace_placeholders(text, overrides={}):
def SortBySpec(text):
from .manifest import AI_HUMAN_SORT_KEYS
for token in text.split(','):
for token in text.split(","):
if token not in AI_HUMAN_SORT_KEYS:
raise ValueError('Invalid sort key: %s' % token)
return text.replace('timestamp', 'ts')
raise ValueError("Invalid sort key: %s" % token)
return text.replace("timestamp", "ts")
def format_file_size(v, precision=2, sign=False, iec=False):
"""Format file size into a human friendly format
"""
"""Format file size into a human friendly format"""
fn = sizeof_fmt_iec if iec else sizeof_fmt_decimal
return fn(v, suffix='B', sep=' ', precision=precision, sign=sign)
return fn(v, suffix="B", sep=" ", precision=precision, sign=sign)
class FileSize(int):
@ -250,22 +250,16 @@ def parse_file_size(s):
suffix = s[-1]
power = 1000
try:
factor = {
'K': power,
'M': power**2,
'G': power**3,
'T': power**4,
'P': power**5,
}[suffix]
factor = {"K": power, "M": power**2, "G": power**3, "T": power**4, "P": power**5}[suffix]
s = s[:-1]
except KeyError:
factor = 1
return int(float(s) * factor)
def sizeof_fmt(num, suffix='B', units=None, power=None, sep='', precision=2, sign=False):
sign = '+' if sign and num > 0 else ''
fmt = '{0:{1}.{2}f}{3}{4}{5}'
def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False):
sign = "+" if sign and num > 0 else ""
fmt = "{0:{1}.{2}f}{3}{4}{5}"
prec = 0
for unit in units[:-1]:
if abs(round(num, precision)) < power:
@ -277,27 +271,37 @@ def sizeof_fmt(num, suffix='B', units=None, power=None, sep='', precision=2, sig
return fmt.format(num, sign, prec, sep, unit, suffix)
def sizeof_fmt_iec(num, suffix='B', sep='', precision=2, sign=False):
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign,
units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024)
def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False):
return sizeof_fmt(
num,
suffix=suffix,
sep=sep,
precision=precision,
sign=sign,
units=["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"],
power=1024,
)
def sizeof_fmt_decimal(num, suffix='B', sep='', precision=2, sign=False):
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign,
units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000)
def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False):
return sizeof_fmt(
num,
suffix=suffix,
sep=sep,
precision=precision,
sign=sign,
units=["", "k", "M", "G", "T", "P", "E", "Z", "Y"],
power=1000,
)
def format_archive(archive):
return '%-36s %s [%s]' % (
archive.name,
format_time(to_localtime(archive.ts)),
bin_to_hex(archive.id),
)
return "%-36s %s [%s]" % (archive.name, format_time(to_localtime(archive.ts)), bin_to_hex(archive.id))
def parse_stringified_list(s):
l = re.split(" *, *", s)
return [item for item in l if item != '']
return [item for item in l if item != ""]
class Location:
@ -343,28 +347,42 @@ class Location:
"""
# regexes for misc. kinds of supported location specifiers:
ssh_re = re.compile(r"""
ssh_re = re.compile(
r"""
(?P<proto>ssh):// # ssh://
""" + optional_user_re + host_re + r""" # user@ (optional), host name or address
"""
+ optional_user_re
+ host_re
+ r""" # user@ (optional), host name or address
(?::(?P<port>\d+))? # :port (optional)
""" + abs_path_re, re.VERBOSE) # path
"""
+ abs_path_re,
re.VERBOSE,
) # path
file_re = re.compile(r"""
file_re = re.compile(
r"""
(?P<proto>file):// # file://
""" + file_path_re, re.VERBOSE) # servername/path or path
"""
+ file_path_re,
re.VERBOSE,
) # servername/path or path
local_re = re.compile(local_path_re, re.VERBOSE) # local path
local_re = re.compile(local_path_re, re.VERBOSE) # local path
win_file_re = re.compile(r"""
win_file_re = re.compile(
r"""
(?:file://)? # optional file protocol
(?P<path>
(?:[a-zA-Z]:)? # Drive letter followed by a colon (optional)
(?:[^:]+) # Anything which does not contain a :, at least one character
)
""", re.VERBOSE)
""",
re.VERBOSE,
)
def __init__(self, text='', overrides={}, other=False):
self.repo_env_var = 'BORG_OTHER_REPO' if other else 'BORG_REPO'
def __init__(self, text="", overrides={}, other=False):
self.repo_env_var = "BORG_OTHER_REPO" if other else "BORG_REPO"
self.valid = False
self.proto = None
self.user = None
@ -393,15 +411,15 @@ def parse(self, text, overrides={}):
def _parse(self, text):
def normpath_special(p):
# avoid that normpath strips away our relative path hack and even makes p absolute
relative = p.startswith('/./')
relative = p.startswith("/./")
p = os.path.normpath(p)
return ('/.' + p) if relative else p
return ("/." + p) if relative else p
if is_win32:
m = self.win_file_re.match(text)
if m:
self.proto = 'file'
self.path = m.group('path')
self.proto = "file"
self.path = m.group("path")
return True
# On windows we currently only support windows paths.
@ -409,38 +427,38 @@ def normpath_special(p):
m = self.ssh_re.match(text)
if m:
self.proto = m.group('proto')
self.user = m.group('user')
self._host = m.group('host')
self.port = m.group('port') and int(m.group('port')) or None
self.path = normpath_special(m.group('path'))
self.proto = m.group("proto")
self.user = m.group("user")
self._host = m.group("host")
self.port = m.group("port") and int(m.group("port")) or None
self.path = normpath_special(m.group("path"))
return True
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = normpath_special(m.group('path'))
self.proto = m.group("proto")
self.path = normpath_special(m.group("path"))
return True
m = self.local_re.match(text)
if m:
self.proto = 'file'
self.path = normpath_special(m.group('path'))
self.proto = "file"
self.path = normpath_special(m.group("path"))
return True
return False
def __str__(self):
items = [
'proto=%r' % self.proto,
'user=%r' % self.user,
'host=%r' % self.host,
'port=%r' % self.port,
'path=%r' % self.path,
"proto=%r" % self.proto,
"user=%r" % self.user,
"host=%r" % self.host,
"port=%r" % self.port,
"path=%r" % self.path,
]
return ', '.join(items)
return ", ".join(items)
def to_key_filename(self):
name = re.sub(r'[^\w]', '_', self.path).strip('_')
if self.proto != 'file':
name = re.sub(r'[^\w]', '_', self.host) + '__' + name
name = re.sub(r"[^\w]", "_", self.path).strip("_")
if self.proto != "file":
name = re.sub(r"[^\w]", "_", self.host) + "__" + name
if len(name) > 100:
# Limit file names to some reasonable length. Most file systems
# limit them to 255 [unit of choice]; due to variations in unicode
@ -455,28 +473,30 @@ def __repr__(self):
def host(self):
# strip square brackets used for IPv6 addrs
if self._host is not None:
return self._host.lstrip('[').rstrip(']')
return self._host.lstrip("[").rstrip("]")
def canonical_path(self):
if self.proto == 'file':
if self.proto == "file":
return self.path
else:
if self.path and self.path.startswith('~'):
path = '/' + self.path # /~/x = path x relative to home dir
elif self.path and not self.path.startswith('/'):
path = '/./' + self.path # /./x = path x relative to cwd
if self.path and self.path.startswith("~"):
path = "/" + self.path # /~/x = path x relative to home dir
elif self.path and not self.path.startswith("/"):
path = "/./" + self.path # /./x = path x relative to cwd
else:
path = self.path
return 'ssh://{}{}{}{}'.format(f'{self.user}@' if self.user else '',
self._host, # needed for ipv6 addrs
f':{self.port}' if self.port else '',
path)
return "ssh://{}{}{}{}".format(
f"{self.user}@" if self.user else "",
self._host, # needed for ipv6 addrs
f":{self.port}" if self.port else "",
path,
)
def with_timestamp(self, timestamp):
return Location(self.raw, overrides={
'now': DatetimeWrapper(timestamp.astimezone(None)),
'utcnow': DatetimeWrapper(timestamp),
})
return Location(
self.raw,
overrides={"now": DatetimeWrapper(timestamp.astimezone(None)), "utcnow": DatetimeWrapper(timestamp)},
)
def location_validator(proto=None, other=False):
@ -486,33 +506,35 @@ def validator(text):
except ValueError as err:
raise argparse.ArgumentTypeError(str(err)) from None
if proto is not None and loc.proto != proto:
if proto == 'file':
if proto == "file":
raise argparse.ArgumentTypeError('"%s": Repository must be local' % text)
else:
raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text)
return loc
return validator
def archivename_validator():
def validator(text):
text = replace_placeholders(text)
if '/' in text or '::' in text or not text:
if "/" in text or "::" in text or not text:
raise argparse.ArgumentTypeError('Invalid archive name: "%s"' % text)
return text
return validator
class BaseFormatter:
FIXED_KEYS = {
# Formatting aids
'LF': '\n',
'SPACE': ' ',
'TAB': '\t',
'CR': '\r',
'NUL': '\0',
'NEWLINE': os.linesep,
'NL': os.linesep,
"LF": "\n",
"SPACE": " ",
"TAB": "\t",
"CR": "\r",
"NUL": "\0",
"NEWLINE": os.linesep,
"NL": os.linesep,
}
def get_item_data(self, item):
@ -523,42 +545,45 @@ def format_item(self, item):
@staticmethod
def keys_help():
return "- NEWLINE: OS dependent line separator\n" \
"- NL: alias of NEWLINE\n" \
"- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n" \
"- SPACE\n" \
"- TAB\n" \
"- CR\n" \
"- LF"
return (
"- NEWLINE: OS dependent line separator\n"
"- NL: alias of NEWLINE\n"
"- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n"
"- SPACE\n"
"- TAB\n"
"- CR\n"
"- LF"
)
class ArchiveFormatter(BaseFormatter):
KEY_DESCRIPTIONS = {
'archive': 'archive name interpreted as text (might be missing non-text characters, see barchive)',
'name': 'alias of "archive"',
'barchive': 'verbatim archive name, can contain any character except NUL',
'comment': 'archive comment interpreted as text (might be missing non-text characters, see bcomment)',
'bcomment': 'verbatim archive comment, can contain any character except NUL',
"archive": "archive name interpreted as text (might be missing non-text characters, see barchive)",
"name": 'alias of "archive"',
"barchive": "verbatim archive name, can contain any character except NUL",
"comment": "archive comment interpreted as text (might be missing non-text characters, see bcomment)",
"bcomment": "verbatim archive comment, can contain any character except NUL",
# *start* is the key used by borg-info for this timestamp, this makes the formats more compatible
'start': 'time (start) of creation of the archive',
'time': 'alias of "start"',
'end': 'time (end) of creation of the archive',
'command_line': 'command line which was used to create the archive',
'id': 'internal ID of the archive',
'hostname': 'hostname of host on which this archive was created',
'username': 'username of user who created this archive',
"start": "time (start) of creation of the archive",
"time": 'alias of "start"',
"end": "time (end) of creation of the archive",
"command_line": "command line which was used to create the archive",
"id": "internal ID of the archive",
"hostname": "hostname of host on which this archive was created",
"username": "username of user who created this archive",
}
KEY_GROUPS = (
('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'),
('start', 'time', 'end', 'command_line'),
('hostname', 'username'),
("archive", "name", "barchive", "comment", "bcomment", "id"),
("start", "time", "end", "command_line"),
("hostname", "username"),
)
@classmethod
def available_keys(cls):
from .manifest import ArchiveInfo
fake_archive_info = ArchiveInfo('archivename', b'\1'*32, datetime(1970, 1, 1, tzinfo=timezone.utc))
formatter = cls('', None, None, None)
fake_archive_info = ArchiveInfo("archivename", b"\1" * 32, datetime(1970, 1, 1, tzinfo=timezone.utc))
formatter = cls("", None, None, None)
keys = []
keys.extend(formatter.call_keys.keys())
keys.extend(formatter.get_item_data(fake_archive_info).keys())
@ -596,12 +621,12 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False):
self.format = partial_format(format, static_keys)
self.format_keys = {f[1] for f in Formatter().parse(format)}
self.call_keys = {
'hostname': partial(self.get_meta, 'hostname', rs=True),
'username': partial(self.get_meta, 'username', rs=True),
'comment': partial(self.get_meta, 'comment', rs=True),
'bcomment': partial(self.get_meta, 'comment', rs=False),
'end': self.get_ts_end,
'command_line': self.get_cmdline,
"hostname": partial(self.get_meta, "hostname", rs=True),
"username": partial(self.get_meta, "username", rs=True),
"comment": partial(self.get_meta, "comment", rs=True),
"bcomment": partial(self.get_meta, "comment", rs=False),
"end": self.get_ts_end,
"command_line": self.get_cmdline,
}
self.used_call_keys = set(self.call_keys) & self.format_keys
if self.json:
@ -611,21 +636,23 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False):
self.item_data = static_keys
def format_item_json(self, item):
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n"
def get_item_data(self, archive_info):
self.name = archive_info.name
self.id = archive_info.id
item_data = {}
item_data.update(self.item_data)
item_data.update({
'name': remove_surrogates(archive_info.name),
'archive': remove_surrogates(archive_info.name),
'barchive': archive_info.name,
'id': bin_to_hex(archive_info.id),
'time': self.format_time(archive_info.ts),
'start': self.format_time(archive_info.ts),
})
item_data.update(
{
"name": remove_surrogates(archive_info.name),
"archive": remove_surrogates(archive_info.name),
"barchive": archive_info.name,
"id": bin_to_hex(archive_info.id),
"time": self.format_time(archive_info.ts),
"start": self.format_time(archive_info.ts),
}
)
for key in self.used_call_keys:
item_data[key] = self.call_keys[key]()
return item_data
@ -635,19 +662,20 @@ def archive(self):
"""lazy load / update loaded archive"""
if self._archive is None or self._archive.id != self.id:
from ..archive import Archive
self._archive = Archive(self.repository, self.key, self.manifest, self.name, iec=self.iec)
return self._archive
def get_meta(self, key, rs):
value = self.archive.metadata.get(key, '')
value = self.archive.metadata.get(key, "")
return remove_surrogates(value) if rs else value
def get_cmdline(self):
cmdline = map(remove_surrogates, self.archive.metadata.get('cmdline', []))
cmdline = map(remove_surrogates, self.archive.metadata.get("cmdline", []))
if self.json:
return list(cmdline)
else:
return ' '.join(map(shlex.quote, cmdline))
return " ".join(map(shlex.quote, cmdline))
def get_ts_end(self):
return self.format_time(self.archive.ts_end)
@ -659,31 +687,29 @@ def format_time(self, ts):
class ItemFormatter(BaseFormatter):
# we provide the hash algos from python stdlib (except shake_*) and additionally xxh64.
# shake_* is not provided because it uses an incompatible .digest() method to support variable length.
hash_algorithms = hashlib.algorithms_guaranteed.union({'xxh64'}).difference({'shake_128', 'shake_256'})
hash_algorithms = hashlib.algorithms_guaranteed.union({"xxh64"}).difference({"shake_128", "shake_256"})
KEY_DESCRIPTIONS = {
'bpath': 'verbatim POSIX path, can contain any character except NUL',
'path': 'path interpreted as text (might be missing non-text characters, see bpath)',
'source': 'link target for symlinks (identical to linktarget)',
'hlid': 'hard link identity (same if hardlinking same fs object)',
'extra': 'prepends {source} with " -> " for soft links and " link to " for hard links',
'dsize': 'deduplicated size',
'num_chunks': 'number of chunks in this file',
'unique_chunks': 'number of unique chunks in this file',
'xxh64': 'XXH64 checksum of this file (note: this is NOT a cryptographic hash!)',
'health': 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)',
"bpath": "verbatim POSIX path, can contain any character except NUL",
"path": "path interpreted as text (might be missing non-text characters, see bpath)",
"source": "link target for symlinks (identical to linktarget)",
"hlid": "hard link identity (same if hardlinking same fs object)",
"extra": 'prepends {source} with " -> " for soft links and " link to " for hard links',
"dsize": "deduplicated size",
"num_chunks": "number of chunks in this file",
"unique_chunks": "number of unique chunks in this file",
"xxh64": "XXH64 checksum of this file (note: this is NOT a cryptographic hash!)",
"health": 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)',
}
KEY_GROUPS = (
('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'hlid', 'flags'),
('size', 'dsize', 'num_chunks', 'unique_chunks'),
('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'),
("type", "mode", "uid", "gid", "user", "group", "path", "bpath", "source", "linktarget", "hlid", "flags"),
("size", "dsize", "num_chunks", "unique_chunks"),
("mtime", "ctime", "atime", "isomtime", "isoctime", "isoatime"),
tuple(sorted(hash_algorithms)),
('archiveid', 'archivename', 'extra'),
('health', )
("archiveid", "archivename", "extra"),
("health",),
)
KEYS_REQUIRING_CACHE = (
'dsize', 'unique_chunks',
)
KEYS_REQUIRING_CACHE = ("dsize", "unique_chunks")
@classmethod
def available_keys(cls):
@ -691,7 +717,8 @@ class FakeArchive:
fpr = name = ""
from ..item import Item
fake_item = Item(mode=0, path='', user='', group='', mtime=0, uid=0, gid=0)
fake_item = Item(mode=0, path="", user="", group="", mtime=0, uid=0, gid=0)
formatter = cls(FakeArchive, "")
keys = []
keys.extend(formatter.call_keys.keys())
@ -723,13 +750,11 @@ def format_needs_cache(cls, format):
def __init__(self, archive, format, *, json_lines=False):
from ..checksums import StreamingXXH64
self.xxh64 = StreamingXXH64
self.archive = archive
self.json_lines = json_lines
static_keys = {
'archivename': archive.name,
'archiveid': archive.fpr,
}
static_keys = {"archivename": archive.name, "archiveid": archive.fpr}
static_keys.update(self.FIXED_KEYS)
if self.json_lines:
self.item_data = {}
@ -739,23 +764,23 @@ def __init__(self, archive, format, *, json_lines=False):
self.format = partial_format(format, static_keys)
self.format_keys = {f[1] for f in Formatter().parse(format)}
self.call_keys = {
'size': self.calculate_size,
'dsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size),
'num_chunks': self.calculate_num_chunks,
'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
'isomtime': partial(self.format_iso_time, 'mtime'),
'isoctime': partial(self.format_iso_time, 'ctime'),
'isoatime': partial(self.format_iso_time, 'atime'),
'mtime': partial(self.format_time, 'mtime'),
'ctime': partial(self.format_time, 'ctime'),
'atime': partial(self.format_time, 'atime'),
"size": self.calculate_size,
"dsize": partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size),
"num_chunks": self.calculate_num_chunks,
"unique_chunks": partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
"isomtime": partial(self.format_iso_time, "mtime"),
"isoctime": partial(self.format_iso_time, "ctime"),
"isoatime": partial(self.format_iso_time, "atime"),
"mtime": partial(self.format_time, "mtime"),
"ctime": partial(self.format_time, "ctime"),
"atime": partial(self.format_time, "atime"),
}
for hash_function in self.hash_algorithms:
self.call_keys[hash_function] = partial(self.hash_item, hash_function)
self.used_call_keys = set(self.call_keys) & self.format_keys
def format_item_json(self, item):
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n"
def get_item_data(self, item):
item_data = {}
@ -763,30 +788,30 @@ def get_item_data(self, item):
mode = stat.filemode(item.mode)
item_type = mode[0]
source = item.get('source', '')
extra = ''
source = item.get("source", "")
extra = ""
if source:
source = remove_surrogates(source)
extra = ' -> %s' % source
hlid = item.get('hlid')
hlid = bin_to_hex(hlid) if hlid else ''
item_data['type'] = item_type
item_data['mode'] = mode
item_data['user'] = item.get('user', str(item.uid))
item_data['group'] = item.get('group', str(item.gid))
item_data['uid'] = item.uid
item_data['gid'] = item.gid
item_data['path'] = remove_surrogates(item.path)
extra = " -> %s" % source
hlid = item.get("hlid")
hlid = bin_to_hex(hlid) if hlid else ""
item_data["type"] = item_type
item_data["mode"] = mode
item_data["user"] = item.get("user", str(item.uid))
item_data["group"] = item.get("group", str(item.gid))
item_data["uid"] = item.uid
item_data["gid"] = item.gid
item_data["path"] = remove_surrogates(item.path)
if self.json_lines:
item_data['healthy'] = 'chunks_healthy' not in item
item_data["healthy"] = "chunks_healthy" not in item
else:
item_data['bpath'] = item.path
item_data['extra'] = extra
item_data['health'] = 'broken' if 'chunks_healthy' in item else 'healthy'
item_data['source'] = source
item_data['linktarget'] = source
item_data['hlid'] = hlid
item_data['flags'] = item.get('bsdflags')
item_data["bpath"] = item.path
item_data["extra"] = extra
item_data["health"] = "broken" if "chunks_healthy" in item else "healthy"
item_data["source"] = source
item_data["linktarget"] = source
item_data["hlid"] = hlid
item_data["flags"] = item.get("bsdflags")
for key in self.used_call_keys:
item_data[key] = self.call_keys[key](item)
return item_data
@ -801,21 +826,21 @@ def sum_unique_chunks_metadata(self, metadata_func, item):
the metadata needed from the chunk
"""
chunk_index = self.archive.cache.chunks
chunks = item.get('chunks', [])
chunks = item.get("chunks", [])
chunks_counter = Counter(c.id for c in chunks)
return sum(metadata_func(c) for c in chunks if chunk_index[c.id].refcount == chunks_counter[c.id])
def calculate_num_chunks(self, item):
return len(item.get('chunks', []))
return len(item.get("chunks", []))
def calculate_size(self, item):
# note: does not support hardlink slaves, they will be size 0
return item.get_size()
def hash_item(self, hash_function, item):
if 'chunks' not in item:
if "chunks" not in item:
return ""
if hash_function == 'xxh64':
if hash_function == "xxh64":
hash = self.xxh64()
elif hash_function in self.hash_algorithms:
hash = hashlib.new(hash_function)
@ -832,18 +857,18 @@ def format_iso_time(self, key, item):
def file_status(mode):
if stat.S_ISREG(mode):
return 'A'
return "A"
elif stat.S_ISDIR(mode):
return 'd'
return "d"
elif stat.S_ISBLK(mode):
return 'b'
return "b"
elif stat.S_ISCHR(mode):
return 'c'
return "c"
elif stat.S_ISLNK(mode):
return 's'
return "s"
elif stat.S_ISFIFO(mode):
return 'f'
return '?'
return "f"
return "?"
def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True):
@ -868,7 +893,7 @@ def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comme
line = line.rstrip(rstrip)
if remove_empty and not line:
continue
if remove_comments and line.startswith('#'):
if remove_comments and line.startswith("#"):
continue
yield line
@ -883,6 +908,7 @@ def swidth_slice(string, max_width):
Latin characters are usually one cell wide, many CJK characters are two cells wide.
"""
from ..platform import swidth
reverse = max_width < 0
max_width = abs(max_width)
if reverse:
@ -896,7 +922,7 @@ def swidth_slice(string, max_width):
result.append(character)
if reverse:
result.reverse()
return ''.join(result)
return "".join(result)
def ellipsis_truncate(msg, space):
@ -905,15 +931,15 @@ def ellipsis_truncate(msg, space):
this_is_a_very_long_string -------> this_is..._string
"""
from ..platform import swidth
ellipsis_width = swidth('...')
ellipsis_width = swidth("...")
msg_width = swidth(msg)
if space < 8:
# if there is very little space, just show ...
return '...' + ' ' * (space - ellipsis_width)
return "..." + " " * (space - ellipsis_width)
if space < ellipsis_width + msg_width:
return '{}...{}'.format(swidth_slice(msg, space // 2 - ellipsis_width),
swidth_slice(msg, -space // 2))
return msg + ' ' * (space - msg_width)
return "{}...{}".format(swidth_slice(msg, space // 2 - ellipsis_width), swidth_slice(msg, -space // 2))
return msg + " " * (space - msg_width)
class BorgJsonEncoder(json.JSONEncoder):
@ -922,23 +948,16 @@ def default(self, o):
from ..remote import RemoteRepository
from ..archive import Archive
from ..cache import LocalCache, AdHocCache
if isinstance(o, Repository) or isinstance(o, RemoteRepository):
return {
'id': bin_to_hex(o.id),
'location': o._location.canonical_path(),
}
return {"id": bin_to_hex(o.id), "location": o._location.canonical_path()}
if isinstance(o, Archive):
return o.info()
if isinstance(o, LocalCache):
return {
'path': o.path,
'stats': o.stats(),
}
return {"path": o.path, "stats": o.stats()}
if isinstance(o, AdHocCache):
return {
'stats': o.stats(),
}
if callable(getattr(o, 'to_json', None)):
return {"stats": o.stats()}
if callable(getattr(o, "to_json", None)):
return o.to_json()
return super().default(o)
@ -946,17 +965,12 @@ def default(self, o):
def basic_json_data(manifest, *, cache=None, extra=None):
key = manifest.key
data = extra or {}
data.update({
'repository': BorgJsonEncoder().default(manifest.repository),
'encryption': {
'mode': key.ARG_NAME,
},
})
data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
if key.NAME.startswith('key file'):
data['encryption']['keyfile'] = key.find_key()
data.update({"repository": BorgJsonEncoder().default(manifest.repository), "encryption": {"mode": key.ARG_NAME}})
data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
if key.NAME.startswith("key file"):
data["encryption"]["keyfile"] = key.find_key()
if cache:
data['cache'] = cache
data["cache"] = cache
return data
@ -975,13 +989,13 @@ def decode_bytes(value):
# look nice and chunk ids should mostly show in hex. Use a special
# inband signaling character (ASCII DEL) to distinguish between
# decoded and hex mode.
if not value.startswith(b'\x7f'):
if not value.startswith(b"\x7f"):
try:
value = value.decode()
return value
except UnicodeDecodeError:
pass
return '\u007f' + bin_to_hex(value)
return "\u007f" + bin_to_hex(value)
def decode_tuple(t):
res = []

66
src/borg/helpers/passphrase.py
View File

@ -39,7 +39,7 @@ def _env_passphrase(cls, env_var, default=None):
@classmethod
def env_passphrase(cls, default=None):
passphrase = cls._env_passphrase('BORG_PASSPHRASE', default)
passphrase = cls._env_passphrase("BORG_PASSPHRASE", default)
if passphrase is not None:
return passphrase
passphrase = cls.env_passcommand()
@ -51,7 +51,7 @@ def env_passphrase(cls, default=None):
@classmethod
def env_passcommand(cls, default=None):
passcommand = os.environ.get('BORG_PASSCOMMAND', None)
passcommand = os.environ.get("BORG_PASSCOMMAND", None)
if passcommand is not None:
# passcommand is a system command (not inside pyinstaller env)
env = prepare_subprocess_env(system=True)
@ -59,21 +59,21 @@ def env_passcommand(cls, default=None):
passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True, env=env)
except (subprocess.CalledProcessError, FileNotFoundError) as e:
raise PasscommandFailure(e)
return cls(passphrase.rstrip('\n'))
return cls(passphrase.rstrip("\n"))
@classmethod
def fd_passphrase(cls):
try:
fd = int(os.environ.get('BORG_PASSPHRASE_FD'))
fd = int(os.environ.get("BORG_PASSPHRASE_FD"))
except (ValueError, TypeError):
return None
with os.fdopen(fd, mode='r') as f:
with os.fdopen(fd, mode="r") as f:
passphrase = f.read()
return cls(passphrase.rstrip('\n'))
return cls(passphrase.rstrip("\n"))
@classmethod
def env_new_passphrase(cls, default=None):
return cls._env_passphrase('BORG_NEW_PASSPHRASE', default)
return cls._env_passphrase("BORG_NEW_PASSPHRASE", default)
@classmethod
def getpass(cls, prompt):
@ -83,32 +83,38 @@ def getpass(cls, prompt):
if prompt:
print() # avoid err msg appearing right of prompt
msg = []
for env_var in 'BORG_PASSPHRASE', 'BORG_PASSCOMMAND':
for env_var in "BORG_PASSPHRASE", "BORG_PASSCOMMAND":
env_var_set = os.environ.get(env_var) is not None
msg.append('{} is {}.'.format(env_var, 'set' if env_var_set else 'not set'))
msg.append('Interactive password query failed.')
raise NoPassphraseFailure(' '.join(msg)) from None
msg.append("{} is {}.".format(env_var, "set" if env_var_set else "not set"))
msg.append("Interactive password query failed.")
raise NoPassphraseFailure(" ".join(msg)) from None
else:
return cls(pw)
@classmethod
def verification(cls, passphrase):
msg = 'Do you want your passphrase to be displayed for verification? [yN]: '
if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.',
retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'):
print('Your passphrase (between double-quotes): "%s"' % passphrase,
file=sys.stderr)
print('Make sure the passphrase displayed above is exactly what you wanted.',
file=sys.stderr)
msg = "Do you want your passphrase to be displayed for verification? [yN]: "
if yes(
msg,
retry_msg=msg,
invalid_msg="Invalid answer, try again.",
retry=True,
env_var_override="BORG_DISPLAY_PASSPHRASE",
):
print('Your passphrase (between double-quotes): "%s"' % passphrase, file=sys.stderr)
print("Make sure the passphrase displayed above is exactly what you wanted.", file=sys.stderr)
try:
passphrase.encode('ascii')
passphrase.encode("ascii")
except UnicodeEncodeError:
print('Your passphrase (UTF-8 encoding in hex): %s' %
bin_to_hex(passphrase.encode('utf-8')),
file=sys.stderr)
print('As you have a non-ASCII passphrase, it is recommended to keep the '
'UTF-8 encoding in hex together with the passphrase at a safe place.',
file=sys.stderr)
print(
"Your passphrase (UTF-8 encoding in hex): %s" % bin_to_hex(passphrase.encode("utf-8")),
file=sys.stderr,
)
print(
"As you have a non-ASCII passphrase, it is recommended to keep the "
"UTF-8 encoding in hex together with the passphrase at a safe place.",
file=sys.stderr,
)
@classmethod
def new(cls, allow_empty=False):
@ -119,17 +125,17 @@ def new(cls, allow_empty=False):
if passphrase is not None:
return passphrase
for retry in range(1, 11):
passphrase = cls.getpass('Enter new passphrase: ')
passphrase = cls.getpass("Enter new passphrase: ")
if allow_empty or passphrase:
passphrase2 = cls.getpass('Enter same passphrase again: ')
passphrase2 = cls.getpass("Enter same passphrase again: ")
if passphrase == passphrase2:
cls.verification(passphrase)
logger.info('Remember your passphrase. Your data will be inaccessible without it.')
logger.info("Remember your passphrase. Your data will be inaccessible without it.")
return passphrase
else:
print('Passphrases do not match', file=sys.stderr)
print("Passphrases do not match", file=sys.stderr)
else:
print('Passphrase must not be blank', file=sys.stderr)
print("Passphrase must not be blank", file=sys.stderr)
else:
raise PasswordRetriesExceeded

88
src/borg/helpers/process.py
View File

@ -13,6 +13,7 @@
from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
from ..logger import create_logger
logger = create_logger()
from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error
@ -21,6 +22,7 @@
@contextlib.contextmanager
def _daemonize():
from ..platform import get_process_id
old_id = get_process_id()
pid = os.fork()
if pid:
@ -30,13 +32,13 @@ def _daemonize():
except _ExitCodeException as e:
exit_code = e.exit_code
finally:
logger.debug('Daemonizing: Foreground process (%s, %s, %s) is now dying.' % old_id)
logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id)
os._exit(exit_code)
os.setsid()
pid = os.fork()
if pid:
os._exit(0)
os.chdir('/')
os.chdir("/")
os.close(0)
os.close(1)
fd = os.open(os.devnull, os.O_RDWR)
@ -78,12 +80,12 @@ def daemonizing(*, timeout=5):
with _daemonize() as (old_id, new_id):
if new_id is None:
# The original / parent process, waiting for a signal to die.
logger.debug('Daemonizing: Foreground process (%s, %s, %s) is waiting for background process...' % old_id)
logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id)
exit_code = EXIT_SUCCESS
# Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case...
with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \
signal_handler('SIGHUP', raising_signal_handler(SigHup)), \
signal_handler('SIGTERM', raising_signal_handler(SigTerm)):
with signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), signal_handler(
"SIGHUP", raising_signal_handler(SigHup)
), signal_handler("SIGTERM", raising_signal_handler(SigTerm)):
try:
if timeout > 0:
time.sleep(timeout)
@ -96,15 +98,17 @@ def daemonizing(*, timeout=5):
exit_code = EXIT_WARNING
except KeyboardInterrupt:
# Manual termination.
logger.debug('Daemonizing: Foreground process (%s, %s, %s) received SIGINT.' % old_id)
logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id)
exit_code = EXIT_SIGNAL_BASE + 2
except BaseException as e:
# Just in case...
logger.warning('Daemonizing: Foreground process received an exception while waiting:\n' +
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
logger.warning(
"Daemonizing: Foreground process received an exception while waiting:\n"
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
)
exit_code = EXIT_WARNING
else:
logger.warning('Daemonizing: Background process did not respond (timeout). Is it alive?')
logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?")
exit_code = EXIT_WARNING
finally:
# Don't call with-body, but die immediately!
@ -113,22 +117,26 @@ def daemonizing(*, timeout=5):
# The background / grandchild process.
sig_to_foreground = signal.SIGTERM
logger.debug('Daemonizing: Background process (%s, %s, %s) is starting...' % new_id)
logger.debug("Daemonizing: Background process (%s, %s, %s) is starting..." % new_id)
try:
yield old_id, new_id
except BaseException as e:
sig_to_foreground = signal.SIGHUP
logger.warning('Daemonizing: Background process raised an exception while starting:\n' +
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
logger.warning(
"Daemonizing: Background process raised an exception while starting:\n"
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
)
raise e
else:
logger.debug('Daemonizing: Background process (%s, %s, %s) has started.' % new_id)
logger.debug("Daemonizing: Background process (%s, %s, %s) has started." % new_id)
finally:
try:
os.kill(old_id[1], sig_to_foreground)
except BaseException as e:
logger.error('Daemonizing: Trying to kill the foreground process raised an exception:\n' +
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
logger.error(
"Daemonizing: Trying to kill the foreground process raised an exception:\n"
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
)
class _ExitCodeException(BaseException):
@ -187,7 +195,7 @@ def __init__(self):
self._sig_int_triggered = False
self._action_triggered = False
self._action_done = False
self.ctx = signal_handler('SIGINT', self.handler)
self.ctx = signal_handler("SIGINT", self.handler)
def __bool__(self):
# this will be True (and stay True) after the first Ctrl-C/SIGINT
@ -228,7 +236,7 @@ def __exit__(self, exception_type, exception_value, traceback):
sig_int = SigIntManager()
def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
def popen_with_error_handling(cmd_line: str, log_prefix="", **kwargs):
"""
Handle typical errors raised by subprocess.Popen. Return None if an error occurred,
otherwise return the Popen object.
@ -240,27 +248,27 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
Does not change the exit code.
"""
assert not kwargs.get('shell'), 'Sorry pal, shell mode is a no-no'
assert not kwargs.get("shell"), "Sorry pal, shell mode is a no-no"
try:
command = shlex.split(cmd_line)
if not command:
raise ValueError('an empty command line is not permitted')
raise ValueError("an empty command line is not permitted")
except ValueError as ve:
logger.error('%s%s', log_prefix, ve)
logger.error("%s%s", log_prefix, ve)
return
logger.debug('%scommand line: %s', log_prefix, command)
logger.debug("%scommand line: %s", log_prefix, command)
try:
return subprocess.Popen(command, **kwargs)
except FileNotFoundError:
logger.error('%sexecutable not found: %s', log_prefix, command[0])
logger.error("%sexecutable not found: %s", log_prefix, command[0])
return
except PermissionError:
logger.error('%spermission denied: %s', log_prefix, command[0])
logger.error("%spermission denied: %s", log_prefix, command[0])
return
def is_terminal(fd=sys.stdout):
return hasattr(fd, 'isatty') and fd.isatty() and (not is_win32 or 'ANSICON' in os.environ)
return hasattr(fd, "isatty") and fd.isatty() and (not is_win32 or "ANSICON" in os.environ)
def prepare_subprocess_env(system, env=None):
@ -278,8 +286,8 @@ def prepare_subprocess_env(system, env=None):
# but we do not want that system binaries (like ssh or other) pick up
# (non-matching) libraries from there.
# thus we install the original LDLP, before pyinstaller has modified it:
lp_key = 'LD_LIBRARY_PATH'
lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 / v3.2.1 has this
lp_key = "LD_LIBRARY_PATH"
lp_orig = env.get(lp_key + "_ORIG") # pyinstaller >= 20160820 / v3.2.1 has this
if lp_orig is not None:
env[lp_key] = lp_orig
else:
@ -292,12 +300,12 @@ def prepare_subprocess_env(system, env=None):
# in this case, we must kill LDLP.
# We can recognize this via sys.frozen and sys._MEIPASS being set.
lp = env.get(lp_key)
if lp is not None and getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
if lp is not None and getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
env.pop(lp_key)
# security: do not give secrets to subprocess
env.pop('BORG_PASSPHRASE', None)
env.pop("BORG_PASSPHRASE", None)
# for information, give borg version to the subprocess
env['BORG_VERSION'] = __version__
env["BORG_VERSION"] = __version__
return env
@ -314,13 +322,15 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
# communication with the process is a one-way road, i.e. the process can never block
# for us to do something while we block on the process for something different.
if inbound:
proc = popen_with_error_handling(cmd, stdout=subprocess.PIPE, stdin=filter_stream,
log_prefix='filter-process: ', env=env)
proc = popen_with_error_handling(
cmd, stdout=subprocess.PIPE, stdin=filter_stream, log_prefix="filter-process: ", env=env
)
else:
proc = popen_with_error_handling(cmd, stdin=subprocess.PIPE, stdout=filter_stream,
log_prefix='filter-process: ', env=env)
proc = popen_with_error_handling(
cmd, stdin=subprocess.PIPE, stdout=filter_stream, log_prefix="filter-process: ", env=env
)
if not proc:
raise Error(f'filter {cmd}: process creation failed')
raise Error(f"filter {cmd}: process creation failed")
stream = proc.stdout if inbound else proc.stdin
# inbound: do not close the pipe (this is the task of the filter process [== writer])
# outbound: close the pipe, otherwise the filter process would not notice when we are done.
@ -331,7 +341,7 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
except Exception:
# something went wrong with processing the stream by borg
logger.debug('Exception, killing the filter...')
logger.debug("Exception, killing the filter...")
if cmd:
proc.kill()
borg_succeeded = False
@ -343,11 +353,11 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
stream.close()
if cmd:
logger.debug('Done, waiting for filter to die...')
logger.debug("Done, waiting for filter to die...")
rc = proc.wait()
logger.debug('filter cmd exited with code %d', rc)
logger.debug("filter cmd exited with code %d", rc)
if filter_stream_close:
filter_stream.close()
if borg_succeeded and rc:
# if borg did not succeed, we know that we killed the filter process
raise Error('filter %s failed, rc=%d' % (cmd, rc))
raise Error("filter %s failed, rc=%d" % (cmd, rc))

29
src/borg/helpers/progress.py
View File

@ -5,6 +5,7 @@
from shutil import get_terminal_size
from ..logger import create_logger
logger = create_logger()
from .parseformat import ellipsis_truncate
@ -19,7 +20,7 @@ def justify_to_terminal_size(message):
class ProgressIndicatorBase:
LOGGER = 'borg.output.progress'
LOGGER = "borg.output.progress"
JSON_TYPE = None
json = False
@ -43,15 +44,15 @@ def __init__(self, msgid=None):
if not self.logger.handlers:
self.handler = logging.StreamHandler(stream=sys.stderr)
self.handler.setLevel(logging.INFO)
logger = logging.getLogger('borg')
logger = logging.getLogger("borg")
# Some special attributes on the borg logger, created by setup_logging
# But also be able to work without that
try:
formatter = logger.formatter
terminator = '\n' if logger.json else '\r'
terminator = "\n" if logger.json else "\r"
self.json = logger.json
except AttributeError:
terminator = '\r'
terminator = "\r"
else:
self.handler.setFormatter(formatter)
self.handler.terminator = terminator
@ -79,24 +80,20 @@ def output_json(self, *, finished=False, **kwargs):
assert self.json
if not self.emit:
return
kwargs.update(dict(
operation=self.id,
msgid=self.msgid,
type=self.JSON_TYPE,
finished=finished,
time=time.time(),
))
kwargs.update(
dict(operation=self.id, msgid=self.msgid, type=self.JSON_TYPE, finished=finished, time=time.time())
)
print(json.dumps(kwargs), file=sys.stderr, flush=True)
def finish(self):
if self.json:
self.output_json(finished=True)
else:
self.output('')
self.output("")
class ProgressIndicatorMessage(ProgressIndicatorBase):
JSON_TYPE = 'progress_message'
JSON_TYPE = "progress_message"
def output(self, msg):
if self.json:
@ -106,7 +103,7 @@ def output(self, msg):
class ProgressIndicatorPercent(ProgressIndicatorBase):
JSON_TYPE = 'progress_percent'
JSON_TYPE = "progress_percent"
def __init__(self, total=0, step=5, start=0, msg="%3.0f%%", msgid=None):
"""
@ -150,7 +147,7 @@ def show(self, current=None, increase=1, info=None):
# no need to truncate if we're not outputting to a terminal
terminal_space = get_terminal_size(fallback=(-1, -1))[0]
if terminal_space != -1:
space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + ['']))
space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [""]))
info[-1] = ellipsis_truncate(info[-1], space)
return self.output(self.msg % tuple([pct] + info), justify=False, info=info)
@ -193,7 +190,7 @@ def show(self):
return self.output(self.triggered)
def output(self, triggered):
print('.', end='', file=self.file, flush=True)
print(".", end="", file=self.file, flush=True)
def finish(self):
print(file=self.file)

38
src/borg/helpers/time.py
View File

@ -12,7 +12,7 @@ def to_localtime(ts):
def parse_timestamp(timestamp, tzinfo=timezone.utc):
"""Parse a ISO 8601 timestamp string"""
fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS
fmt = ISO_FORMAT if "." in timestamp else ISO_FORMAT_NO_USECS
dt = datetime.strptime(timestamp, fmt)
if tzinfo is not None:
dt = dt.replace(tzinfo=tzinfo)
@ -27,11 +27,16 @@ def timestamp(s):
return datetime.fromtimestamp(ts, tz=timezone.utc)
except OSError:
# didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support.
for format in ('%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S+00:00',
'%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S',
'%Y-%m-%dT%H:%M', '%Y-%m-%d %H:%M',
'%Y-%m-%d', '%Y-%j',
):
for format in (
"%Y-%m-%dT%H:%M:%SZ",
"%Y-%m-%dT%H:%M:%S+00:00",
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%dT%H:%M",
"%Y-%m-%d %H:%M",
"%Y-%m-%d",
"%Y-%j",
):
try:
return datetime.strptime(s, format).replace(tzinfo=timezone.utc)
except ValueError:
@ -54,7 +59,7 @@ def timestamp(s):
# subtract last 48h to avoid any issues that could be caused by tz calculations.
# this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit).
# msgpack can pack up to uint64.
MAX_S = 2**31-1 - 48*3600
MAX_S = 2**31 - 1 - 48 * 3600
MAX_NS = MAX_S * 1000000000
else:
# nanosecond timestamps will fit into a signed int64.
@ -62,7 +67,7 @@ def timestamp(s):
# this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit).
# round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S.
# msgpack can pack up to uint64.
MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000
MAX_NS = (2**63 - 1 - 48 * 3600 * 1000000000) // 1000000000 * 1000000000
MAX_S = MAX_NS // 1000000000
@ -89,11 +94,11 @@ def safe_timestamp(item_timestamp_ns):
return datetime.fromtimestamp(t_ns / 1e9)
def format_time(ts: datetime, format_spec=''):
def format_time(ts: datetime, format_spec=""):
"""
Convert *ts* to a human-friendly format with textual weekday.
"""
return ts.strftime('%a, %Y-%m-%d %H:%M:%S' if format_spec == '' else format_spec)
return ts.strftime("%a, %Y-%m-%d %H:%M:%S" if format_spec == "" else format_spec)
def isoformat_time(ts: datetime):
@ -105,19 +110,18 @@ def isoformat_time(ts: datetime):
def format_timedelta(td):
"""Format timedelta in a human friendly format
"""
"""Format timedelta in a human friendly format"""
ts = td.total_seconds()
s = ts % 60
m = int(ts / 60) % 60
h = int(ts / 3600) % 24
txt = '%.2f seconds' % s
txt = "%.2f seconds" % s
if m:
txt = '%d minutes %s' % (m, txt)
txt = "%d minutes %s" % (m, txt)
if h:
txt = '%d hours %s' % (h, txt)
txt = "%d hours %s" % (h, txt)
if td.days:
txt = '%d days %s' % (td.days, txt)
txt = "%d days %s" % (td.days, txt)
return txt
@ -131,7 +135,7 @@ def __format__(self, format_spec):
return format_time(self.ts, format_spec=format_spec)
def __str__(self):
return f'{self}'
return f"{self}"
def isoformat(self):
return isoformat_time(self.ts)

55
src/borg/helpers/yes.py
View File

@ -4,16 +4,30 @@
import os.path
import sys
FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', )
TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', )
DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', )
FALSISH = ("No", "NO", "no", "N", "n", "0")
TRUISH = ("Yes", "YES", "yes", "Y", "y", "1")
DEFAULTISH = ("Default", "DEFAULT", "default", "D", "d", "")
def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
retry_msg=None, invalid_msg=None, env_msg='{} (from {})',
falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH,
default=False, retry=True, env_var_override=None, ofile=None, input=input, prompt=True,
msgid=None):
def yes(
msg=None,
false_msg=None,
true_msg=None,
default_msg=None,
retry_msg=None,
invalid_msg=None,
env_msg="{} (from {})",
falsish=FALSISH,
truish=TRUISH,
defaultish=DEFAULTISH,
default=False,
retry=True,
env_var_override=None,
ofile=None,
input=input,
prompt=True,
msgid=None,
):
"""Output <msg> (usually a question) and let user input an answer.
Qualifies the answer according to falsish, truish and defaultish as True, False or <default>.
If it didn't qualify and retry is False (no retries wanted), return the default [which
@ -43,18 +57,15 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
:param input: input function [input from builtins]
:return: boolean answer value, True or False
"""
def output(msg, msg_type, is_prompt=False, **kwargs):
json_output = getattr(logging.getLogger('borg'), 'json', False)
json_output = getattr(logging.getLogger("borg"), "json", False)
if json_output:
kwargs.update(dict(
type='question_%s' % msg_type,
msgid=msgid,
message=msg,
))
kwargs.update(dict(type="question_%s" % msg_type, msgid=msgid, message=msg))
print(json.dumps(kwargs), file=sys.stderr)
else:
if is_prompt:
print(msg, file=ofile, end='', flush=True)
print(msg, file=ofile, end="", flush=True)
else:
print(msg, file=ofile)
@ -66,13 +77,13 @@ def output(msg, msg_type, is_prompt=False, **kwargs):
if default not in (True, False):
raise ValueError("invalid default value, must be True or False")
if msg:
output(msg, 'prompt', is_prompt=True)
output(msg, "prompt", is_prompt=True)
while True:
answer = None
if env_var_override:
answer = os.environ.get(env_var_override)
if answer is not None and env_msg:
output(env_msg.format(answer, env_var_override), 'env_answer', env_var=env_var_override)
output(env_msg.format(answer, env_var_override), "env_answer", env_var=env_var_override)
if answer is None:
if not prompt:
return default
@ -83,22 +94,22 @@ def output(msg, msg_type, is_prompt=False, **kwargs):
answer = truish[0] if default else falsish[0]
if answer in defaultish:
if default_msg:
output(default_msg, 'accepted_default')
output(default_msg, "accepted_default")
return default
if answer in truish:
if true_msg:
output(true_msg, 'accepted_true')
output(true_msg, "accepted_true")
return True
if answer in falsish:
if false_msg:
output(false_msg, 'accepted_false')
output(false_msg, "accepted_false")
return False
# if we get here, the answer was invalid
if invalid_msg:
output(invalid_msg, 'invalid_answer')
output(invalid_msg, "invalid_answer")
if not retry:
return default
if retry_msg:
output(retry_msg, 'prompt_retry', is_prompt=True)
output(retry_msg, "prompt_retry", is_prompt=True)
# in case we used an environment variable and it gave an invalid answer, do not use it again:
env_var_override = None

39
src/borg/locking.py
View File

@ -8,8 +8,8 @@
from .helpers import Error, ErrorWithTraceback
from .logger import create_logger
ADD, REMOVE = 'add', 'remove'
SHARED, EXCLUSIVE = 'shared', 'exclusive'
ADD, REMOVE = "add", "remove"
SHARED, EXCLUSIVE = "shared", "exclusive"
logger = create_logger(__name__)
@ -20,6 +20,7 @@ class TimeoutTimer:
It can also compute and optionally execute a reasonable sleep time (e.g. to avoid
polling too often or to support thread/process rescheduling).
"""
def __init__(self, timeout=None, sleep=None):
"""
Initialize a timer.
@ -43,8 +44,8 @@ def __init__(self, timeout=None, sleep=None):
def __repr__(self):
return "<{}: start={!r} end={!r} timeout={!r} sleep={!r}>".format(
self.__class__.__name__, self.start_time, self.end_time,
self.timeout_interval, self.sleep_interval)
self.__class__.__name__, self.start_time, self.end_time, self.timeout_interval, self.sleep_interval
)
def start(self):
self.start_time = time.time()
@ -102,6 +103,7 @@ class ExclusiveLock:
This makes sure the lock is released again if the block is left, no
matter how (e.g. if an exception occurred).
"""
def __init__(self, path, timeout=None, sleep=None, id=None):
self.timeout = timeout
self.sleep = sleep
@ -129,7 +131,7 @@ def acquire(self, timeout=None, sleep=None):
unique_base_name = os.path.basename(self.unique_name)
temp_path = None
try:
temp_path = tempfile.mkdtemp(".tmp", base_name + '.', parent_path)
temp_path = tempfile.mkdtemp(".tmp", base_name + ".", parent_path)
temp_unique_name = os.path.join(temp_path, unique_base_name)
with open(temp_unique_name, "wb"):
pass
@ -192,8 +194,8 @@ def kill_stale_lock(self):
else:
for name in names:
try:
host_pid, thread_str = name.rsplit('-', 1)
host, pid_str = host_pid.rsplit('.', 1)
host_pid, thread_str = name.rsplit("-", 1)
host, pid_str = host_pid.rsplit(".", 1)
pid = int(pid_str)
thread = int(thread_str)
except ValueError:
@ -207,17 +209,19 @@ def kill_stale_lock(self):
if not self.kill_stale_locks:
if not self.stale_warning_printed:
# Log this at warning level to hint the user at the ability
logger.warning("Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name)
logger.warning(
"Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name
)
self.stale_warning_printed = True
return False
try:
os.unlink(os.path.join(self.path, name))
logger.warning('Killed stale lock %s.', name)
logger.warning("Killed stale lock %s.", name)
except OSError as err:
if not self.stale_warning_printed:
# This error will bubble up and likely result in locking failure
logger.error('Found stale lock %s, but cannot delete due to %s', name, str(err))
logger.error("Found stale lock %s, but cannot delete due to %s", name, str(err))
self.stale_warning_printed = True
return False
@ -228,7 +232,7 @@ def kill_stale_lock(self):
# Directory is not empty or doesn't exist any more = we lost the race to somebody else--which is ok.
return False
# EACCES or EIO or ... = we cannot operate anyway
logger.error('Failed to remove lock dir: %s', str(err))
logger.error("Failed to remove lock dir: %s", str(err))
return False
return True
@ -257,6 +261,7 @@ class LockRoster:
Note: you usually should call the methods with an exclusive lock held,
to avoid conflicting access by multiple threads/processes/machines.
"""
def __init__(self, path, id=None):
self.path = path
self.id = id or platform.get_process_id()
@ -279,8 +284,9 @@ def load(self):
if platform.process_alive(host, pid, thread):
elements.add((host, pid, thread))
else:
logger.warning('Removed stale %s roster lock for host %s pid %d thread %d.',
key, host, pid, thread)
logger.warning(
"Removed stale %s roster lock for host %s pid %d thread %d.", key, host, pid, thread
)
data[key] = list(elements)
except (FileNotFoundError, ValueError):
# no or corrupt/empty roster file?
@ -315,7 +321,7 @@ def modify(self, key, op):
elif op == REMOVE:
elements.remove(self.id)
else:
raise ValueError('Unknown LockRoster op %r' % op)
raise ValueError("Unknown LockRoster op %r" % op)
roster[key] = list(list(e) for e in elements)
self.save(roster)
@ -354,6 +360,7 @@ class Lock:
This makes sure the lock is released again if the block is left, no
matter how (e.g. if an exception occurred).
"""
def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None):
self.path = path
self.is_exclusive = exclusive
@ -361,11 +368,11 @@ def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None):
self.timeout = timeout
self.id = id or platform.get_process_id()
# globally keeping track of shared and exclusive lockers:
self._roster = LockRoster(path + '.roster', id=id)
self._roster = LockRoster(path + ".roster", id=id)
# an exclusive lock, used for:
# - holding while doing roster queries / updates
# - holding while the Lock itself is exclusive
self._lock = ExclusiveLock(path + '.exclusive', id=id, timeout=timeout)
self._lock = ExclusiveLock(path + ".exclusive", id=id, timeout=timeout)
def __enter__(self):
return self.acquire()

64
src/borg/logger.py
View File

@ -53,7 +53,7 @@ def _log_warning(message, category, filename, lineno, file=None, line=None):
logger.warning(msg)
def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info', is_serve=False, json=False):
def setup_logging(stream=None, conf_fname=None, env_var="BORG_LOGGING_CONF", level="info", is_serve=False, json=False):
"""setup logging module according to the arguments provided
if conf_fname is given (or the config file name can be determined via
@ -80,7 +80,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
logging.config.fileConfig(f)
configured = True
logger = logging.getLogger(__name__)
borg_logger = logging.getLogger('borg')
borg_logger = logging.getLogger("borg")
borg_logger.json = json
logger.debug(f'using logging configuration read from "{conf_fname}"')
warnings.showwarning = _log_warning
@ -88,15 +88,15 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
except Exception as err: # XXX be more precise
err_msg = str(err)
# if we did not / not successfully load a logging configuration, fallback to this:
logger = logging.getLogger('')
logger = logging.getLogger("")
handler = logging.StreamHandler(stream)
if is_serve and not json:
fmt = '$LOG %(levelname)s %(name)s Remote: %(message)s'
fmt = "$LOG %(levelname)s %(name)s Remote: %(message)s"
else:
fmt = '%(message)s'
fmt = "%(message)s"
formatter = JsonFormatter(fmt) if json else logging.Formatter(fmt)
handler.setFormatter(formatter)
borg_logger = logging.getLogger('borg')
borg_logger = logging.getLogger("borg")
borg_logger.formatter = formatter
borg_logger.json = json
if configured and logger.handlers:
@ -111,7 +111,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
logger = logging.getLogger(__name__)
if err_msg:
logger.warning(f'setup_logging for "{conf_fname}" failed with "{err_msg}".')
logger.debug('using builtin fallback logging configuration')
logger.debug("using builtin fallback logging configuration")
warnings.showwarning = _log_warning
return handler
@ -151,6 +151,7 @@ def create_logger(name=None):
be careful not to call any logger methods before the setup_logging() call.
If you try, you'll get an exception.
"""
class LazyLogger:
def __init__(self, name=None):
self.__name = name or find_parent_module()
@ -162,49 +163,49 @@ def __logger(self):
if not configured:
raise Exception("tried to call a logger before setup_logging() was called")
self.__real_logger = logging.getLogger(self.__name)
if self.__name.startswith('borg.debug.') and self.__real_logger.level == logging.NOTSET:
self.__real_logger.setLevel('WARNING')
if self.__name.startswith("borg.debug.") and self.__real_logger.level == logging.NOTSET:
self.__real_logger.setLevel("WARNING")
return self.__real_logger
def getChild(self, suffix):
return LazyLogger(self.__name + '.' + suffix)
return LazyLogger(self.__name + "." + suffix)
def setLevel(self, *args, **kw):
return self.__logger.setLevel(*args, **kw)
def log(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.log(*args, **kw)
def exception(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.exception(*args, **kw)
def debug(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.debug(*args, **kw)
def info(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.info(*args, **kw)
def warning(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.warning(*args, **kw)
def error(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.error(*args, **kw)
def critical(self, *args, **kw):
if 'msgid' in kw:
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
if "msgid" in kw:
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
return self.__logger.critical(*args, **kw)
return LazyLogger(name)
@ -212,11 +213,11 @@ def critical(self, *args, **kw):
class JsonFormatter(logging.Formatter):
RECORD_ATTRIBUTES = (
'levelname',
'name',
'message',
"levelname",
"name",
"message",
# msgid is an attribute we made up in Borg to expose a non-changing handle for log messages
'msgid',
"msgid",
)
# Other attributes that are not very useful but do exist:
@ -229,12 +230,7 @@ class JsonFormatter(logging.Formatter):
def format(self, record):
super().format(record)
data = {
'type': 'log_message',
'time': record.created,
'message': '',
'levelname': 'CRITICAL',
}
data = {"type": "log_message", "time": record.created, "message": "", "levelname": "CRITICAL"}
for attr in self.RECORD_ATTRIBUTES:
value = getattr(record, attr, None)
if value:

4
src/borg/lrucache.py
View File

@ -10,8 +10,8 @@ def __init__(self, capacity, dispose):
def __setitem__(self, key, value):
assert key not in self._cache, (
"Unexpected attempt to replace a cached item,"
" without first deleting the old item.")
"Unexpected attempt to replace a cached item," " without first deleting the old item."
)
self._lru.append(key)
while len(self._lru) > self._capacity:
del self[self._lru[0]]

128
src/borg/nanorst.py
View File

@ -11,18 +11,18 @@ def __init__(self, s):
def read(self, n):
self.i += n
return self.str[self.i - n:self.i]
return self.str[self.i - n : self.i]
def peek(self, n):
if n >= 0:
return self.str[self.i:self.i + n]
return self.str[self.i : self.i + n]
else:
return self.str[self.i + n - 1:self.i - 1]
return self.str[self.i + n - 1 : self.i - 1]
def peekline(self):
out = ''
out = ""
i = self.i
while i < len(self.str) and self.str[i] != '\n':
while i < len(self.str) and self.str[i] != "\n":
out += self.str[i]
i += 1
return out
@ -34,18 +34,18 @@ def readline(self):
def process_directive(directive, arguments, out, state_hook):
if directive == 'container' and arguments == 'experimental':
state_hook('text', '**', out)
out.write('++ Experimental ++')
state_hook('**', 'text', out)
if directive == "container" and arguments == "experimental":
state_hook("text", "**", out)
out.write("++ Experimental ++")
state_hook("**", "text", out)
else:
state_hook('text', '**', out)
state_hook("text", "**", out)
out.write(directive.title())
out.write(':\n')
state_hook('**', 'text', out)
out.write(":\n")
state_hook("**", "text", out)
if arguments:
out.write(arguments)
out.write('\n')
out.write("\n")
def rst_to_text(text, state_hook=None, references=None):
@ -58,12 +58,12 @@ def rst_to_text(text, state_hook=None, references=None):
"""
state_hook = state_hook or (lambda old_state, new_state, out: None)
references = references or {}
state = 'text'
inline_mode = 'replace'
state = "text"
inline_mode = "replace"
text = TextPecker(text)
out = io.StringIO()
inline_single = ('*', '`')
inline_single = ("*", "`")
while True:
char = text.read(1)
@ -71,81 +71,83 @@ def rst_to_text(text, state_hook=None, references=None):
break
next = text.peek(1) # type: str
if state == 'text':
if char == '\\' and text.peek(1) in inline_single:
if state == "text":
if char == "\\" and text.peek(1) in inline_single:
continue
if text.peek(-1) != '\\':
if text.peek(-1) != "\\":
if char in inline_single and next != char:
state_hook(state, char, out)
state = char
continue
if char == next == '*':
state_hook(state, '**', out)
state = '**'
if char == next == "*":
state_hook(state, "**", out)
state = "**"
text.read(1)
continue
if char == next == '`':
state_hook(state, '``', out)
state = '``'
if char == next == "`":
state_hook(state, "``", out)
state = "``"
text.read(1)
continue
if text.peek(-1).isspace() and char == ':' and text.peek(5) == 'ref:`':
if text.peek(-1).isspace() and char == ":" and text.peek(5) == "ref:`":
# translate reference
text.read(5)
ref = ''
ref = ""
while True:
char = text.peek(1)
if char == '`':
if char == "`":
text.read(1)
break
if char == '\n':
if char == "\n":
text.read(1)
continue # merge line breaks in :ref:`...\n...`
ref += text.read(1)
try:
out.write(references[ref])
except KeyError:
raise ValueError("Undefined reference in Archiver help: %r — please add reference "
"substitution to 'rst_plain_text_references'" % ref)
raise ValueError(
"Undefined reference in Archiver help: %r — please add reference "
"substitution to 'rst_plain_text_references'" % ref
)
continue
if char == ':' and text.peek(2) == ':\n': # End of line code block
if char == ":" and text.peek(2) == ":\n": # End of line code block
text.read(2)
state_hook(state, 'code-block', out)
state = 'code-block'
out.write(':\n')
state_hook(state, "code-block", out)
state = "code-block"
out.write(":\n")
continue
if text.peek(-2) in ('\n\n', '') and char == next == '.':
if text.peek(-2) in ("\n\n", "") and char == next == ".":
text.read(2)
directive, is_directive, arguments = text.readline().partition('::')
directive, is_directive, arguments = text.readline().partition("::")
text.read(1)
if not is_directive:
# partition: if the separator is not in the text, the leftmost output is the entire input
if directive == 'nanorst: inline-fill':
inline_mode = 'fill'
elif directive == 'nanorst: inline-replace':
inline_mode = 'replace'
if directive == "nanorst: inline-fill":
inline_mode = "fill"
elif directive == "nanorst: inline-replace":
inline_mode = "replace"
continue
process_directive(directive, arguments.strip(), out, state_hook)
continue
if state in inline_single and char == state:
state_hook(state, 'text', out)
state = 'text'
if inline_mode == 'fill':
out.write(2 * ' ')
state_hook(state, "text", out)
state = "text"
if inline_mode == "fill":
out.write(2 * " ")
continue
if state == '``' and char == next == '`':
state_hook(state, 'text', out)
state = 'text'
if state == "``" and char == next == "`":
state_hook(state, "text", out)
state = "text"
text.read(1)
if inline_mode == 'fill':
out.write(4 * ' ')
if inline_mode == "fill":
out.write(4 * " ")
continue
if state == '**' and char == next == '*':
state_hook(state, 'text', out)
state = 'text'
if state == "**" and char == next == "*":
state_hook(state, "text", out)
state = "text"
text.read(1)
continue
if state == 'code-block' and char == next == '\n' and text.peek(5)[1:] != ' ':
if state == "code-block" and char == next == "\n" and text.peek(5)[1:] != " ":
# Foo::
#
# *stuff* *code* *ignore .. all markup*
@ -153,11 +155,11 @@ def rst_to_text(text, state_hook=None, references=None):
# More arcane stuff
#
# Regular text...
state_hook(state, 'text', out)
state = 'text'
state_hook(state, "text", out)
state = "text"
out.write(char)
assert state == 'text', 'Invalid final state %r (This usually indicates unmatched */**)' % state
assert state == "text", "Invalid final state %r (This usually indicates unmatched */**)" % state
return out.getvalue()
@ -191,12 +193,12 @@ def __contains__(self, item):
def ansi_escapes(old_state, new_state, out):
if old_state == 'text' and new_state in ('*', '`', '``'):
out.write('\033[4m')
if old_state == 'text' and new_state == '**':
out.write('\033[1m')
if old_state in ('*', '`', '``', '**') and new_state == 'text':
out.write('\033[0m')
if old_state == "text" and new_state in ("*", "`", "``"):
out.write("\033[4m")
if old_state == "text" and new_state == "**":
out.write("\033[1m")
if old_state in ("*", "`", "``", "**") and new_state == "text":
out.write("\033[0m")
def rst_to_terminal(rst, references=None, destination=sys.stdout):

84
src/borg/patterns.py
View File

@ -75,6 +75,7 @@ class PatternMatcher:
*fallback* is a boolean value that *match()* returns if no matching patterns are found.
"""
def __init__(self, fallback=None):
self._items = []
@ -96,18 +97,13 @@ def __init__(self, fallback=None):
self.include_patterns = []
# TODO: move this info to parse_inclexcl_command and store in PatternBase subclass?
self.is_include_cmd = {
IECommand.Exclude: False,
IECommand.ExcludeNoRecurse: False,
IECommand.Include: True
}
self.is_include_cmd = {IECommand.Exclude: False, IECommand.ExcludeNoRecurse: False, IECommand.Include: True}
def empty(self):
return not len(self._items) and not len(self._path_full_patterns)
def _add(self, pattern, cmd):
"""*cmd* is an IECommand value.
"""
"""*cmd* is an IECommand value."""
if isinstance(pattern, PathFullPattern):
key = pattern.pattern # full, normalized path
self._path_full_patterns[key] = cmd
@ -123,8 +119,7 @@ def add(self, patterns, cmd):
self._add(pattern, cmd)
def add_includepaths(self, include_paths):
"""Used to add inclusion-paths from args.paths (from commandline).
"""
"""Used to add inclusion-paths from args.paths (from commandline)."""
include_patterns = [parse_pattern(p, PathPrefixPattern) for p in include_paths]
self.add(include_patterns, IECommand.Include)
self.fallback = not include_patterns
@ -135,8 +130,7 @@ def get_unmatched_include_patterns(self):
return [p for p in self.include_patterns if p.match_count == 0]
def add_inclexcl(self, patterns):
"""Add list of patterns (of type CmdTuple) to internal list.
"""
"""Add list of patterns (of type CmdTuple) to internal list."""
for pattern, cmd in patterns:
self._add(pattern, cmd)
@ -172,12 +166,12 @@ def normalize_path(path):
"""normalize paths for MacOS (but do nothing on other platforms)"""
# HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match.
# Windows and Unix filesystems allow different forms, so users always have to enter an exact match.
return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path
return unicodedata.normalize("NFD", path) if sys.platform == "darwin" else path
class PatternBase:
"""Shared logic for inclusion/exclusion patterns.
"""
"""Shared logic for inclusion/exclusion patterns."""
PREFIX = NotImplemented
def __init__(self, pattern, recurse_dir=False):
@ -201,7 +195,7 @@ def match(self, path, normalize=True):
return matches
def __repr__(self):
return f'{type(self)}({self.pattern})'
return f"{type(self)}({self.pattern})"
def __str__(self):
return self.pattern_orig
@ -216,6 +210,7 @@ def _match(self, path):
class PathFullPattern(PatternBase):
"""Full match of a path."""
PREFIX = "pf"
def _prepare(self, pattern):
@ -236,6 +231,7 @@ class PathPrefixPattern(PatternBase):
If a directory is specified, all paths that start with that
path match as well. A trailing slash makes no difference.
"""
PREFIX = "pp"
def _prepare(self, pattern):
@ -251,13 +247,14 @@ class FnmatchPattern(PatternBase):
"""Shell glob patterns to exclude. A trailing slash means to
exclude the contents of a directory, but not the directory itself.
"""
PREFIX = "fm"
def _prepare(self, pattern):
if pattern.endswith(os.path.sep):
pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep
pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + "*" + os.path.sep
else:
pattern = os.path.normpath(pattern) + os.path.sep + '*'
pattern = os.path.normpath(pattern) + os.path.sep + "*"
self.pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed
@ -266,13 +263,14 @@ def _prepare(self, pattern):
self.regex = re.compile(fnmatch.translate(self.pattern))
def _match(self, path):
return (self.regex.match(path + os.path.sep) is not None)
return self.regex.match(path + os.path.sep) is not None
class ShellPattern(PatternBase):
"""Shell glob patterns to exclude. A trailing slash means to
exclude the contents of a directory, but not the directory itself.
"""
PREFIX = "sh"
def _prepare(self, pattern):
@ -287,12 +285,12 @@ def _prepare(self, pattern):
self.regex = re.compile(shellpattern.translate(self.pattern))
def _match(self, path):
return (self.regex.match(path + os.path.sep) is not None)
return self.regex.match(path + os.path.sep) is not None
class RegexPattern(PatternBase):
"""Regular expression to exclude.
"""
"""Regular expression to exclude."""
PREFIX = "re"
def _prepare(self, pattern):
@ -301,28 +299,22 @@ def _prepare(self, pattern):
def _match(self, path):
# Normalize path separators
if os.path.sep != '/':
path = path.replace(os.path.sep, '/')
if os.path.sep != "/":
path = path.replace(os.path.sep, "/")
return (self.regex.search(path) is not None)
return self.regex.search(path) is not None
_PATTERN_CLASSES = {
FnmatchPattern,
PathFullPattern,
PathPrefixPattern,
RegexPattern,
ShellPattern,
}
_PATTERN_CLASSES = {FnmatchPattern, PathFullPattern, PathPrefixPattern, RegexPattern, ShellPattern}
_PATTERN_CLASS_BY_PREFIX = {i.PREFIX: i for i in _PATTERN_CLASSES}
CmdTuple = namedtuple('CmdTuple', 'val cmd')
CmdTuple = namedtuple("CmdTuple", "val cmd")
class IECommand(Enum):
"""A command that an InclExcl file line can represent.
"""
"""A command that an InclExcl file line can represent."""
RootPath = 1
PatternStyle = 2
Include = 3
@ -343,9 +335,7 @@ def get_pattern_class(prefix):
def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True):
"""Read pattern from string and return an instance of the appropriate implementation class.
"""
"""Read pattern from string and return an instance of the appropriate implementation class."""
if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum():
(style, pattern) = (pattern[:2], pattern[3:])
cls = get_pattern_class(style)
@ -355,8 +345,7 @@ def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True):
def parse_exclude_pattern(pattern_str, fallback=FnmatchPattern):
"""Read pattern from string and return an instance of the appropriate implementation class.
"""
"""Read pattern from string and return an instance of the appropriate implementation class."""
epattern_obj = parse_pattern(pattern_str, fallback, recurse_dir=False)
return CmdTuple(epattern_obj, IECommand.ExcludeNoRecurse)
@ -365,21 +354,20 @@ def parse_inclexcl_command(cmd_line_str, fallback=ShellPattern):
"""Read a --patterns-from command from string and return a CmdTuple object."""
cmd_prefix_map = {
'-': IECommand.Exclude,
'!': IECommand.ExcludeNoRecurse,
'+': IECommand.Include,
'R': IECommand.RootPath,
'r': IECommand.RootPath,
'P': IECommand.PatternStyle,
'p': IECommand.PatternStyle,
"-": IECommand.Exclude,
"!": IECommand.ExcludeNoRecurse,
"+": IECommand.Include,
"R": IECommand.RootPath,
"r": IECommand.RootPath,
"P": IECommand.PatternStyle,
"p": IECommand.PatternStyle,
}
if not cmd_line_str:
raise argparse.ArgumentTypeError("A pattern/command must not be empty.")
cmd = cmd_prefix_map.get(cmd_line_str[0])
if cmd is None:
raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" %
', '.join(cmd_prefix_map))
raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" % ", ".join(cmd_prefix_map))
# remaining text on command-line following the command character
remainder_str = cmd_line_str[1:].lstrip()

1
src/borg/platform/__init__.py
View File

@ -17,6 +17,7 @@
if not is_win32:
from .posix import process_alive, local_pid_alive
# posix swidth implementation works for: linux, freebsd, darwin, openindiana, cygwin
from .posix import swidth
from .posix import get_errno

37
src/borg/platform/base.py
View File

@ -20,9 +20,9 @@
are correctly composed into the base functionality.
"""
API_VERSION = '1.2_05'
API_VERSION = "1.2_05"
fdatasync = getattr(os, 'fdatasync', os.fsync)
fdatasync = getattr(os, "fdatasync", os.fsync)
from .xattr import ENOATTR
@ -86,14 +86,16 @@ def acl_set(path, item, numeric_ids=False, fd=None):
def set_flags(path, bsd_flags, fd=None):
lchflags(path, bsd_flags)
except ImportError:
def set_flags(path, bsd_flags, fd=None):
pass
def get_flags(path, st, fd=None):
"""Return BSD-style file flags for path or stat without following symlinks."""
return getattr(st, 'st_flags', 0)
return getattr(st, "st_flags", 0)
def sync_dir(path):
@ -114,8 +116,8 @@ def sync_dir(path):
def safe_fadvise(fd, offset, len, advice):
if hasattr(os, 'posix_fadvise'):
advice = getattr(os, 'POSIX_FADV_' + advice)
if hasattr(os, "posix_fadvise"):
advice = getattr(os, "POSIX_FADV_" + advice)
try:
os.posix_fadvise(fd, offset, len, advice)
except OSError:
@ -158,7 +160,7 @@ def __init__(self, path, *, fd=None, binary=False):
that corresponds to path (like from os.open(path, ...) or os.mkstemp(...))
:param binary: whether to open in binary mode, default is False.
"""
mode = 'xb' if binary else 'x' # x -> raise FileExists exception in open() if file exists already
mode = "xb" if binary else "x" # x -> raise FileExists exception in open() if file exists already
self.path = path
if fd is None:
self.f = open(path, mode=mode) # python file object
@ -181,15 +183,17 @@ def sync(self):
after sync().
"""
from .. import platform
self.f.flush()
platform.fdatasync(self.fd)
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
safe_fadvise(self.fd, 0, 0, 'DONTNEED')
safe_fadvise(self.fd, 0, 0, "DONTNEED")
def close(self):
"""sync() and close."""
from .. import platform
dirname = None
try:
dirname = os.path.dirname(self.path)
@ -216,23 +220,26 @@ class SaveFile:
Internally used temporary files are created in the target directory and are
named <BASENAME>-<RANDOMCHARS>.tmp and cleaned up in normal and error conditions.
"""
def __init__(self, path, binary=False):
self.binary = binary
self.path = path
self.dir = os.path.dirname(path)
self.tmp_prefix = os.path.basename(path) + '-'
self.tmp_prefix = os.path.basename(path) + "-"
self.tmp_fd = None # OS-level fd
self.tmp_fname = None # full path/filename corresponding to self.tmp_fd
self.f = None # python-file-like SyncFile
def __enter__(self):
from .. import platform
self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix='.tmp', dir=self.dir)
self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix=".tmp", dir=self.dir)
self.f = platform.SyncFile(self.tmp_fname, fd=self.tmp_fd, binary=self.binary)
return self.f
def __exit__(self, exc_type, exc_val, exc_tb):
from .. import platform
self.f.close() # this indirectly also closes self.tmp_fd
self.tmp_fd = None
if exc_type is not None:
@ -246,7 +253,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
# thanks to the crappy os.umask api, we can't query the umask without setting it. :-(
umask = os.umask(UMASK_DEFAULT)
os.umask(umask)
os.chmod(self.tmp_fname, mode=0o666 & ~ umask)
os.chmod(self.tmp_fname, mode=0o666 & ~umask)
except OSError:
# chmod might fail if the fs does not support it.
# this is not harmful, the file will still have permissions for the owner.
@ -270,13 +277,13 @@ def swidth(s):
# patched socket.getfqdn() - see https://bugs.python.org/issue5004
def getfqdn(name=''):
def getfqdn(name=""):
"""Get fully qualified domain name from name.
An empty argument is interpreted as meaning the local host.
"""
name = name.strip()
if not name or name == '0.0.0.0':
if not name or name == "0.0.0.0":
name = socket.gethostname()
try:
addrs = socket.getaddrinfo(name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME)
@ -296,14 +303,14 @@ def getfqdn(name=''):
fqdn = getfqdn(hostname)
# some people put the fqdn into /etc/hostname (which is wrong, should be the short hostname)
# fix this (do the same as "hostname --short" cli command does internally):
hostname = hostname.split('.')[0]
hostname = hostname.split(".")[0]
# uuid.getnode() is problematic in some environments (e.g. OpenVZ, see #3968) where the virtual MAC address
# is all-zero. uuid.getnode falls back to returning a random value in that case, which is not what we want.
# thus, we offer BORG_HOST_ID where a user can set an own, unique id for each of his hosts.
hostid = os.environ.get('BORG_HOST_ID')
hostid = os.environ.get("BORG_HOST_ID")
if not hostid:
hostid = f'{fqdn}@{uuid.getnode()}'
hostid = f"{fqdn}@{uuid.getnode()}"
def get_process_id():

11
src/borg/platform/xattr.py
View File

@ -18,7 +18,7 @@ def split_string0(buf):
"""split a list of zero-terminated strings into python not-zero-terminated bytes"""
if isinstance(buf, bytearray):
buf = bytes(buf) # use a bytes object, so we return a list of bytes objects
return buf.split(b'\0')[:-1]
return buf.split(b"\0")[:-1]
def split_lstring(buf):
@ -27,8 +27,8 @@ def split_lstring(buf):
mv = memoryview(buf)
while mv:
length = mv[0]
result.append(bytes(mv[1:1 + length]))
mv = mv[1 + length:]
result.append(bytes(mv[1 : 1 + length]))
mv = mv[1 + length :]
return result
@ -38,6 +38,7 @@ class BufferTooSmallError(Exception):
def _check(rv, path=None, detect_buffer_too_small=False):
from . import get_errno
if rv < 0:
e = get_errno()
if detect_buffer_too_small and e == errno.ERANGE:
@ -48,9 +49,9 @@ def _check(rv, path=None, detect_buffer_too_small=False):
try:
msg = os.strerror(e)
except ValueError:
msg = ''
msg = ""
if isinstance(path, int):
path = '<FD %d>' % path
path = "<FD %d>" % path
raise OSError(e, msg, path)
if detect_buffer_too_small and rv >= len(buffer):
# freebsd does not error with ERANGE if the buffer is too small,

8
src/borg/platformflags.py
View File

@ -6,7 +6,7 @@
import sys
is_win32 = sys.platform.startswith('win32')
is_linux = sys.platform.startswith('linux')
is_freebsd = sys.platform.startswith('freebsd')
is_darwin = sys.platform.startswith('darwin')
is_win32 = sys.platform.startswith("win32")
is_linux = sys.platform.startswith("linux")
is_freebsd = sys.platform.startswith("freebsd")
is_darwin = sys.platform.startswith("darwin")

590
src/borg/remote.py
View File

File diff suppressed because it is too large Load Diff

530
src/borg/repository.py
View File

File diff suppressed because it is too large Load Diff

23
src/borg/selftest.py
View File

@ -50,16 +50,16 @@ def test_name(self, test):
def log_results(self, logger):
for test, failure in self.errors + self.failures + self.unexpectedSuccesses:
logger.error('self test %s FAILED:\n%s', self.test_name(test), failure)
logger.error("self test %s FAILED:\n%s", self.test_name(test), failure)
for test, reason in self.skipped:
logger.warning('self test %s skipped: %s', self.test_name(test), reason)
logger.warning("self test %s skipped: %s", self.test_name(test), reason)
def successful_test_count(self):
return len(self.successes)
def selftest(logger):
if os.environ.get('BORG_SELFTEST') == 'disabled':
if os.environ.get("BORG_SELFTEST") == "disabled":
logger.debug("borg selftest disabled via BORG_SELFTEST env variable")
return
selftest_started = time.perf_counter()
@ -69,7 +69,7 @@ def selftest(logger):
module = sys.modules[test_case.__module__]
# a normal borg user does not have pytest installed, we must not require it in the test modules used here.
# note: this only detects the usual toplevel import
assert 'pytest' not in dir(module), "pytest must not be imported in %s" % module.__name__
assert "pytest" not in dir(module), "pytest must not be imported in %s" % module.__name__
test_suite.addTest(defaultTestLoader.loadTestsFromTestCase(test_case))
test_suite.run(result)
result.log_results(logger)
@ -77,12 +77,17 @@ def selftest(logger):
count_mismatch = successful_tests != SELFTEST_COUNT
if result.wasSuccessful() and count_mismatch:
# only print this if all tests succeeded
logger.error("self test count (%d != %d) mismatch, either test discovery is broken or a test was added "
"without updating borg.selftest",
successful_tests, SELFTEST_COUNT)
logger.error(
"self test count (%d != %d) mismatch, either test discovery is broken or a test was added "
"without updating borg.selftest",
successful_tests,
SELFTEST_COUNT,
)
if not result.wasSuccessful() or count_mismatch:
logger.error("self test failed\n"
"Could be a bug either in Borg, the package / distribution you use, your OS or your hardware.")
logger.error(
"self test failed\n"
"Could be a bug either in Borg, the package / distribution you use, your OS or your hardware."
)
sys.exit(2)
assert False, "sanity assertion failed: ran beyond sys.exit()"
selftest_elapsed = time.perf_counter() - selftest_started

2
src/borg/shellpattern.py
View File

@ -33,7 +33,7 @@ def translate(pat, match_end=r"\Z"):
if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep:
# **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands
# for the platform-specific path separator
res += fr"(?:[^\{sep}]*\{sep})*"
res += rf"(?:[^\{sep}]*\{sep})*"
i += 2
else:
# * == wildcard for name parts (does not cross path separator)

67
src/borg/testsuite/__init__.py
View File

@ -2,6 +2,7 @@
import filecmp
import functools
import os
try:
import posix
except ImportError:
@ -25,14 +26,14 @@
from ..fuse_impl import llfuse, has_pyfuse3, has_llfuse
# Does this version of llfuse support ns precision?
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, "st_mtime_ns") if llfuse else False
try:
from pytest import raises
except: # noqa
raises = None
has_lchflags = hasattr(os, 'lchflags') or sys.platform.startswith('linux')
has_lchflags = hasattr(os, "lchflags") or sys.platform.startswith("linux")
try:
with tempfile.NamedTemporaryFile() as file:
platform.set_flags(file.name, stat.UF_NODUMP)
@ -40,14 +41,14 @@
has_lchflags = False
# The mtime get/set precision varies on different OS and Python versions
if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
if posix and "HAVE_FUTIMENS" in getattr(posix, "_have_functions", []):
st_mtime_ns_round = 0
elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
elif "HAVE_UTIMES" in sysconfig.get_config_vars():
st_mtime_ns_round = -6
else:
st_mtime_ns_round = -9
if sys.platform.startswith('netbsd'):
if sys.platform.startswith("netbsd"):
st_mtime_ns_round = -4 # only >1 microsecond resolution here?
@ -61,8 +62,8 @@ def unopened_tempfile():
def are_symlinks_supported():
with unopened_tempfile() as filepath:
try:
os.symlink('somewhere', filepath)
if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == 'somewhere':
os.symlink("somewhere", filepath)
if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == "somewhere":
return True
except OSError:
pass
@ -71,12 +72,12 @@ def are_symlinks_supported():
@functools.lru_cache
def are_hardlinks_supported():
if not hasattr(os, 'link'):
if not hasattr(os, "link"):
# some pythons do not have os.link
return False
with unopened_tempfile() as file1path, unopened_tempfile() as file2path:
open(file1path, 'w').close()
open(file1path, "w").close()
try:
os.link(file1path, file2path)
stat1 = os.stat(file1path)
@ -108,9 +109,9 @@ def is_utime_fully_supported():
with unopened_tempfile() as filepath:
# Some filesystems (such as SSHFS) don't support utime on symlinks
if are_symlinks_supported():
os.symlink('something', filepath)
os.symlink("something", filepath)
else:
open(filepath, 'w').close()
open(filepath, "w").close()
try:
os.utime(filepath, (1000, 2000), follow_symlinks=False)
new_stats = os.stat(filepath, follow_symlinks=False)
@ -125,14 +126,14 @@ def is_utime_fully_supported():
@functools.lru_cache
def is_birthtime_fully_supported():
if not hasattr(os.stat_result, 'st_birthtime'):
if not hasattr(os.stat_result, "st_birthtime"):
return False
with unopened_tempfile() as filepath:
# Some filesystems (such as SSHFS) don't support utime on symlinks
if are_symlinks_supported():
os.symlink('something', filepath)
os.symlink("something", filepath)
else:
open(filepath, 'w').close()
open(filepath, "w").close()
try:
birthtime, mtime, atime = 946598400, 946684800, 946771200
os.utime(filepath, (atime, birthtime), follow_symlinks=False)
@ -149,7 +150,7 @@ def is_birthtime_fully_supported():
def no_selinux(x):
# selinux fails our FUSE tests, thus ignore selinux xattrs
SELINUX_KEY = b'security.selinux'
SELINUX_KEY = b"security.selinux"
if isinstance(x, dict):
return {k: v for k, v in x.items() if k != SELINUX_KEY}
if isinstance(x, list):
@ -157,8 +158,8 @@ def no_selinux(x):
class BaseTestCase(unittest.TestCase):
"""
"""
""" """
assert_in = unittest.TestCase.assertIn
assert_not_in = unittest.TestCase.assertNotIn
assert_equal = unittest.TestCase.assertEqual
@ -171,9 +172,9 @@ class BaseTestCase(unittest.TestCase):
@contextmanager
def assert_creates_file(self, path):
assert not os.path.exists(path), f'{path} should not exist'
assert not os.path.exists(path), f"{path} should not exist"
yield
assert os.path.exists(path), f'{path} should exist'
assert os.path.exists(path), f"{path} should exist"
def assert_dirs_equal(self, dir1, dir2, **kwargs):
diff = filecmp.dircmp(dir1, dir2)
@ -191,10 +192,10 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False,
s2 = os.stat(path2, follow_symlinks=False)
# Assume path2 is on FUSE if st_dev is different
fuse = s1.st_dev != s2.st_dev
attrs = ['st_uid', 'st_gid', 'st_rdev']
attrs = ["st_uid", "st_gid", "st_rdev"]
if not fuse or not os.path.isdir(path1):
# dir nlink is always 1 on our FUSE filesystem
attrs.append('st_nlink')
attrs.append("st_nlink")
d1 = [filename] + [getattr(s1, a) for a in attrs]
d2 = [filename] + [getattr(s2, a) for a in attrs]
d1.insert(1, oct(s1.st_mode))
@ -225,7 +226,9 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False,
d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
self.assert_equal(d1, d2)
for sub_diff in diff.subdirs.values():
self._assert_dirs_equal_cmp(sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns)
self._assert_dirs_equal_cmp(
sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns
)
@contextmanager
def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=False, **kwargs):
@ -247,7 +250,7 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal
mountpoint = tempfile.mkdtemp()
else:
os.mkdir(mountpoint)
args = [f'--repo={location}', 'mount', mountpoint] + list(options)
args = [f"--repo={location}", "mount", mountpoint] + list(options)
if os_fork:
# Do not spawn, but actually (OS) fork.
if os.fork() == 0:
@ -264,12 +267,11 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal
# This should never be reached, since it daemonizes,
# and the grandchild process exits before cmd() returns.
# However, just in case...
print('Fatal: borg mount did not daemonize properly. Force exiting.',
file=sys.stderr, flush=True)
print("Fatal: borg mount did not daemonize properly. Force exiting.", file=sys.stderr, flush=True)
os._exit(0)
else:
self.cmd(*args, fork=fork, **kwargs)
if kwargs.get('exit_code', EXIT_SUCCESS) == EXIT_ERROR:
if kwargs.get("exit_code", EXIT_SUCCESS) == EXIT_ERROR:
# If argument `exit_code = EXIT_ERROR`, then this call
# is testing the behavior of an unsuccessful mount and
# we must not continue, as there is no mount to work
@ -292,7 +294,7 @@ def wait_for_mountstate(self, mountpoint, *, mounted, timeout=5):
if os.path.ismount(mountpoint) == mounted:
return
time.sleep(0.1)
message = 'Waiting for {} of {}'.format('mount' if mounted else 'umount', mountpoint)
message = "Waiting for {} of {}".format("mount" if mounted else "umount", mountpoint)
raise TimeoutError(message)
@contextmanager
@ -308,17 +310,17 @@ def read_only(self, path):
tests are running with root privileges. Instead, the folder is
rendered immutable with chattr or chflags, respectively.
"""
if sys.platform.startswith('linux'):
if sys.platform.startswith("linux"):
cmd_immutable = 'chattr +i "%s"' % path
cmd_mutable = 'chattr -i "%s"' % path
elif sys.platform.startswith(('darwin', 'freebsd', 'netbsd', 'openbsd')):
elif sys.platform.startswith(("darwin", "freebsd", "netbsd", "openbsd")):
cmd_immutable = 'chflags uchg "%s"' % path
cmd_mutable = 'chflags nouchg "%s"' % path
elif sys.platform.startswith('sunos'): # openindiana
elif sys.platform.startswith("sunos"): # openindiana
cmd_immutable = 'chmod S+vimmutable "%s"' % path
cmd_mutable = 'chmod S-vimmutable "%s"' % path
else:
message = 'Testing read-only repos is not supported on platform %s' % sys.platform
message = "Testing read-only repos is not supported on platform %s" % sys.platform
self.skipTest(message)
try:
os.system('LD_PRELOAD= chmod -R ugo-w "%s"' % path)
@ -365,12 +367,13 @@ def __exit__(self, *args, **kw):
class FakeInputs:
"""Simulate multiple user inputs, can be used as input() replacement"""
def __init__(self, inputs):
self.inputs = inputs
def __call__(self, prompt=None):
if prompt is not None:
print(prompt, end='')
print(prompt, end="")
try:
return self.inputs.pop(0)
except IndexError:

153
src/borg/testsuite/archive.py
View File

@ -33,64 +33,66 @@ def test_stats_basic(stats):
def tests_stats_progress(stats, monkeypatch, columns=80):
monkeypatch.setenv('COLUMNS', str(columns))
monkeypatch.setenv("COLUMNS", str(columns))
out = StringIO()
stats.show_progress(stream=out)
s = '20 B O 20 B U 1 N '
buf = ' ' * (columns - len(s))
s = "20 B O 20 B U 1 N "
buf = " " * (columns - len(s))
assert out.getvalue() == s + buf + "\r"
out = StringIO()
stats.update(10 ** 3, unique=False)
stats.show_progress(item=Item(path='foo'), final=False, stream=out)
s = '1.02 kB O 20 B U 1 N foo'
buf = ' ' * (columns - len(s))
stats.update(10**3, unique=False)
stats.show_progress(item=Item(path="foo"), final=False, stream=out)
s = "1.02 kB O 20 B U 1 N foo"
buf = " " * (columns - len(s))
assert out.getvalue() == s + buf + "\r"
out = StringIO()
stats.show_progress(item=Item(path='foo'*40), final=False, stream=out)
s = '1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo'
buf = ' ' * (columns - len(s))
stats.show_progress(item=Item(path="foo" * 40), final=False, stream=out)
s = "1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo"
buf = " " * (columns - len(s))
assert out.getvalue() == s + buf + "\r"
def test_stats_format(stats):
assert str(stats) == """\
assert (
str(stats)
== """\
Number of files: 1
Original size: 20 B
Deduplicated size: 20 B
"""
)
s = f"{stats.osize_fmt}"
assert s == "20 B"
# kind of redundant, but id is variable so we can't match reliably
assert repr(stats) == f'<Statistics object at {id(stats):#x} (20, 20)>'
assert repr(stats) == f"<Statistics object at {id(stats):#x} (20, 20)>"
def test_stats_progress_json(stats):
stats.output_json = True
out = StringIO()
stats.show_progress(item=Item(path='foo'), stream=out)
stats.show_progress(item=Item(path="foo"), stream=out)
result = json.loads(out.getvalue())
assert result['type'] == 'archive_progress'
assert isinstance(result['time'], float)
assert result['finished'] is False
assert result['path'] == 'foo'
assert result['original_size'] == 20
assert result['nfiles'] == 1
assert result["type"] == "archive_progress"
assert isinstance(result["time"], float)
assert result["finished"] is False
assert result["path"] == "foo"
assert result["original_size"] == 20
assert result["nfiles"] == 1
out = StringIO()
stats.show_progress(stream=out, final=True)
result = json.loads(out.getvalue())
assert result['type'] == 'archive_progress'
assert isinstance(result['time'], float)
assert result['finished'] is True # see #6570
assert 'path' not in result
assert 'original_size' not in result
assert 'nfiles' not in result
assert result["type"] == "archive_progress"
assert isinstance(result["time"], float)
assert result["finished"] is True # see #6570
assert "path" not in result
assert "original_size" not in result
assert "nfiles" not in result
class MockCache:
class MockRepo:
def async_response(self, wait=True):
pass
@ -105,30 +107,24 @@ def add_chunk(self, id, chunk, stats=None, wait=True):
class ArchiveTimestampTestCase(BaseTestCase):
def _test_timestamp_parsing(self, isoformat, expected):
repository = Mock()
key = PlaintextKey(repository)
manifest = Manifest(repository, key)
a = Archive(repository, key, manifest, 'test', create=True)
a = Archive(repository, key, manifest, "test", create=True)
a.metadata = ArchiveItem(time=isoformat)
self.assert_equal(a.ts, expected)
def test_with_microseconds(self):
self._test_timestamp_parsing(
'1970-01-01T00:00:01.000001',
datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc))
self._test_timestamp_parsing("1970-01-01T00:00:01.000001", datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc))
def test_without_microseconds(self):
self._test_timestamp_parsing(
'1970-01-01T00:00:01',
datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc))
self._test_timestamp_parsing("1970-01-01T00:00:01", datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc))
class ChunkBufferTestCase(BaseTestCase):
def test(self):
data = [Item(path='p1'), Item(path='p2')]
data = [Item(path="p1"), Item(path="p2")]
cache = MockCache()
key = PlaintextKey(None)
chunks = CacheChunkBuffer(cache, key, None)
@ -144,7 +140,7 @@ def test(self):
def test_partial(self):
big = "0123456789abcdefghijklmnopqrstuvwxyz" * 25000
data = [Item(path='full', source=big), Item(path='partial', source=big)]
data = [Item(path="full", source=big), Item(path="partial", source=big)]
cache = MockCache()
key = PlaintextKey(None)
chunks = CacheChunkBuffer(cache, key, None)
@ -165,12 +161,11 @@ def test_partial(self):
class RobustUnpackerTestCase(BaseTestCase):
def make_chunks(self, items):
return b''.join(msgpack.packb({'path': item}) for item in items)
return b"".join(msgpack.packb({"path": item}) for item in items)
def _validator(self, value):
return isinstance(value, dict) and value.get('path') in ('foo', 'bar', 'boo', 'baz')
return isinstance(value, dict) and value.get("path") in ("foo", "bar", "boo", "baz")
def process(self, input):
unpacker = RobustUnpacker(validator=self._validator, item_keys=ITEM_KEYS)
@ -185,14 +180,14 @@ def process(self, input):
return result
def test_extra_garbage_no_sync(self):
chunks = [(False, [self.make_chunks(['foo', 'bar'])]),
(False, [b'garbage'] + [self.make_chunks(['boo', 'baz'])])]
chunks = [
(False, [self.make_chunks(["foo", "bar"])]),
(False, [b"garbage"] + [self.make_chunks(["boo", "baz"])]),
]
result = self.process(chunks)
self.assert_equal(result, [
{'path': 'foo'}, {'path': 'bar'},
103, 97, 114, 98, 97, 103, 101,
{'path': 'boo'},
{'path': 'baz'}])
self.assert_equal(
result, [{"path": "foo"}, {"path": "bar"}, 103, 97, 114, 98, 97, 103, 101, {"path": "boo"}, {"path": "baz"}]
)
def split(self, left, length):
parts = []
@ -202,22 +197,22 @@ def split(self, left, length):
return parts
def test_correct_stream(self):
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 2)
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 2)
input = [(False, chunks)]
result = self.process(input)
self.assert_equal(result, [{'path': 'foo'}, {'path': 'bar'}, {'path': 'boo'}, {'path': 'baz'}])
self.assert_equal(result, [{"path": "foo"}, {"path": "bar"}, {"path": "boo"}, {"path": "baz"}])
def test_missing_chunk(self):
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4)
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4)
input = [(False, chunks[:3]), (True, chunks[4:])]
result = self.process(input)
self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}])
self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}])
def test_corrupt_chunk(self):
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4)
input = [(False, chunks[:3]), (True, [b'gar', b'bage'] + chunks[3:])]
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4)
input = [(False, chunks[:3]), (True, [b"gar", b"bage"] + chunks[3:])]
result = self.process(input)
self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}])
self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}])
@pytest.fixture
@ -225,12 +220,17 @@ def item_keys_serialized():
return [msgpack.packb(name) for name in ITEM_KEYS]
@pytest.mark.parametrize('packed',
[b'', b'x', b'foobar', ] +
[msgpack.packb(o) for o in (
[None, 0, 0.0, False, '', {}, [], ()] +
[42, 23.42, True, b'foobar', {b'foo': b'bar'}, [b'foo', b'bar'], (b'foo', b'bar')]
)])
@pytest.mark.parametrize(
"packed",
[b"", b"x", b"foobar"]
+ [
msgpack.packb(o)
for o in (
[None, 0, 0.0, False, "", {}, [], ()]
+ [42, 23.42, True, b"foobar", {b"foo": b"bar"}, [b"foo", b"bar"], (b"foo", b"bar")]
)
],
)
def test_invalid_msgpacked_item(packed, item_keys_serialized):
assert not valid_msgpacked_dict(packed, item_keys_serialized)
@ -239,20 +239,25 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized):
IK = sorted(list(ITEM_KEYS))
@pytest.mark.parametrize('packed',
[msgpack.packb(o) for o in [
{'path': b'/a/b/c'}, # small (different msgpack mapping type!)
OrderedDict((k, b'') for k in IK), # as big (key count) as it gets
OrderedDict((k, b'x' * 1000) for k in IK), # as big (key count and volume) as it gets
]])
@pytest.mark.parametrize(
"packed",
[
msgpack.packb(o)
for o in [
{"path": b"/a/b/c"}, # small (different msgpack mapping type!)
OrderedDict((k, b"") for k in IK), # as big (key count) as it gets
OrderedDict((k, b"x" * 1000) for k in IK), # as big (key count and volume) as it gets
]
],
)
def test_valid_msgpacked_items(packed, item_keys_serialized):
assert valid_msgpacked_dict(packed, item_keys_serialized)
def test_key_length_msgpacked_items():
key = 'x' * 32 # 31 bytes is the limit for fixstr msgpack type
data = {key: b''}
item_keys_serialized = [msgpack.packb(key), ]
key = "x" * 32 # 31 bytes is the limit for fixstr msgpack type
data = {key: b""}
item_keys_serialized = [msgpack.packb(key)]
assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
@ -277,7 +282,7 @@ def __next__(self):
normal_iterator = Iterator(StopIteration)
for _ in backup_io_iter(normal_iterator):
assert False, 'StopIteration handled incorrectly'
assert False, "StopIteration handled incorrectly"
def test_get_item_uid_gid():
@ -288,7 +293,7 @@ def test_get_item_uid_gid():
user0, group0 = uid2user(0), gid2group(0)
# this is intentionally a "strange" item, with not matching ids/names.
item = Item(path='filename', uid=1, gid=2, user=user0, group=group0)
item = Item(path="filename", uid=1, gid=2, user=user0, group=group0)
uid, gid = get_item_uid_gid(item, numeric=False)
# these are found via a name-to-id lookup
@ -306,7 +311,7 @@ def test_get_item_uid_gid():
assert gid == 4
# item metadata broken, has negative ids.
item = Item(path='filename', uid=-1, gid=-2, user=user0, group=group0)
item = Item(path="filename", uid=-1, gid=-2, user=user0, group=group0)
uid, gid = get_item_uid_gid(item, numeric=True)
# use the uid/gid defaults (which both default to 0).
@ -319,7 +324,7 @@ def test_get_item_uid_gid():
assert gid == 6
# item metadata broken, has negative ids and non-existing user/group names.
item = Item(path='filename', uid=-3, gid=-4, user='udoesnotexist', group='gdoesnotexist')
item = Item(path="filename", uid=-3, gid=-4, user="udoesnotexist", group="gdoesnotexist")
uid, gid = get_item_uid_gid(item, numeric=False)
# use the uid/gid defaults (which both default to 0).
@ -332,7 +337,7 @@ def test_get_item_uid_gid():
assert gid == 8
# item metadata has valid uid/gid, but non-existing user/group names.
item = Item(path='filename', uid=9, gid=10, user='udoesnotexist', group='gdoesnotexist')
item = Item(path="filename", uid=9, gid=10, user="udoesnotexist", group="gdoesnotexist")
uid, gid = get_item_uid_gid(item, numeric=False)
# because user/group name does not exist here, use valid numeric ids from item metadata.

4829
src/borg/testsuite/archiver.py
View File

File diff suppressed because it is too large Load Diff

53
src/borg/testsuite/benchmark.py
View File

@ -16,35 +16,38 @@
@pytest.fixture
def repo_url(request, tmpdir, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', '123456')
monkeypatch.setenv('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', 'YES')
monkeypatch.setenv('BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', 'YES')
monkeypatch.setenv('BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', 'yes')
monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir.join('keys')))
monkeypatch.setenv('BORG_CACHE_DIR', str(tmpdir.join('cache')))
yield str(tmpdir.join('repository'))
monkeypatch.setenv("BORG_PASSPHRASE", "123456")
monkeypatch.setenv("BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", "YES")
monkeypatch.setenv("BORG_DELETE_I_KNOW_WHAT_I_AM_DOING", "YES")
monkeypatch.setenv("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK", "yes")
monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir.join("keys")))
monkeypatch.setenv("BORG_CACHE_DIR", str(tmpdir.join("cache")))
yield str(tmpdir.join("repository"))
tmpdir.remove(rec=1)
@pytest.fixture(params=["none", "repokey-aes-ocb"])
def repo(request, cmd, repo_url):
cmd(f'--repo={repo_url}', 'rcreate', '--encryption', request.param)
cmd(f"--repo={repo_url}", "rcreate", "--encryption", request.param)
return repo_url
@pytest.fixture(scope='session', params=["zeros", "random"])
@pytest.fixture(scope="session", params=["zeros", "random"])
def testdata(request, tmpdir_factory):
count, size = 10, 1000*1000
count, size = 10, 1000 * 1000
assert size <= len(zeros)
p = tmpdir_factory.mktemp('data')
p = tmpdir_factory.mktemp("data")
data_type = request.param
if data_type == 'zeros':
if data_type == "zeros":
# do not use a binary zero (\0) to avoid sparse detection
def data(size):
return memoryview(zeros)[:size]
elif data_type == 'random':
elif data_type == "random":
def data(size):
return os.urandom(size)
else:
raise ValueError("data_type must be 'random' or 'zeros'.")
for i in range(count):
@ -54,56 +57,54 @@ def data(size):
p.remove(rec=1)
@pytest.fixture(params=['none', 'lz4'])
@pytest.fixture(params=["none", "lz4"])
def repo_archive(request, cmd, repo, testdata):
archive = 'test'
cmd(f'--repo={repo}', 'create', '--compression', request.param, archive, testdata)
archive = "test"
cmd(f"--repo={repo}", "create", "--compression", request.param, archive, testdata)
return repo, archive
def test_create_none(benchmark, cmd, repo, testdata):
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'none',
'test', testdata))
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "none", "test", testdata))
assert result == 0
def test_create_lz4(benchmark, cmd, repo, testdata):
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'lz4',
'test', testdata))
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "lz4", "test", testdata))
assert result == 0
def test_extract(benchmark, cmd, repo_archive, tmpdir):
repo, archive = repo_archive
with changedir(str(tmpdir)):
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'extract', archive))
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "extract", archive))
assert result == 0
def test_delete(benchmark, cmd, repo_archive):
repo, archive = repo_archive
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'delete', '-a', archive))
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "delete", "-a", archive))
assert result == 0
def test_list(benchmark, cmd, repo_archive):
repo, archive = repo_archive
result, out = benchmark(cmd, f'--repo={repo}', 'list', archive)
result, out = benchmark(cmd, f"--repo={repo}", "list", archive)
assert result == 0
def test_info(benchmark, cmd, repo_archive):
repo, archive = repo_archive
result, out = benchmark(cmd, f'--repo={repo}', 'info', '-a', archive)
result, out = benchmark(cmd, f"--repo={repo}", "info", "-a", archive)
assert result == 0
def test_check(benchmark, cmd, repo_archive):
repo, archive = repo_archive
result, out = benchmark(cmd, f'--repo={repo}', 'check')
result, out = benchmark(cmd, f"--repo={repo}", "check")
assert result == 0
def test_help(benchmark, cmd):
result, out = benchmark(cmd, 'help')
result, out = benchmark(cmd, "help")
assert result == 0

196
src/borg/testsuite/cache.py
View File

@ -26,75 +26,29 @@ def sync(self, index):
return CacheSynchronizer(index)
def test_no_chunks(self, index, sync):
data = packb({
'foo': 'bar',
'baz': 1234,
'bar': 5678,
'user': 'chunks',
'chunks': []
})
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": []})
sync.feed(data)
assert not len(index)
def test_simple(self, index, sync):
data = packb({
'foo': 'bar',
'baz': 1234,
'bar': 5678,
'user': 'chunks',
'chunks': [
(H(1), 1),
(H(2), 2),
]
})
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
sync.feed(data)
assert len(index) == 2
assert index[H(1)] == (1, 1)
assert index[H(2)] == (1, 2)
def test_multiple(self, index, sync):
data = packb({
'foo': 'bar',
'baz': 1234,
'bar': 5678,
'user': 'chunks',
'chunks': [
(H(1), 1),
(H(2), 2),
]
})
data += packb({
'xattrs': {
'security.foo': 'bar',
'chunks': '123456',
},
'stuff': [
(1, 2, 3),
]
})
data += packb({
'xattrs': {
'security.foo': 'bar',
'chunks': '123456',
},
'chunks': [
(H(1), 1),
(H(2), 2),
],
'stuff': [
(1, 2, 3),
]
})
data += packb({
'chunks': [
(H(3), 1),
],
})
data += packb({
'chunks': [
(H(1), 1),
],
})
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
data += packb({"xattrs": {"security.foo": "bar", "chunks": "123456"}, "stuff": [(1, 2, 3)]})
data += packb(
{
"xattrs": {"security.foo": "bar", "chunks": "123456"},
"chunks": [(H(1), 1), (H(2), 2)],
"stuff": [(1, 2, 3)],
}
)
data += packb({"chunks": [(H(3), 1)]})
data += packb({"chunks": [(H(1), 1)]})
part1 = data[:70]
part2 = data[70:120]
@ -107,62 +61,68 @@ def test_multiple(self, index, sync):
assert index[H(2)] == (2, 2)
assert index[H(3)] == (1, 1)
@pytest.mark.parametrize('elem,error', (
({1: 2}, 'Unexpected object: map'),
(bytes(213), [
'Unexpected bytes in chunks structure', # structure 2/3
'Incorrect key length']), # structure 3/3
(1, 'Unexpected object: integer'),
(1.0, 'Unexpected object: double'),
(True, 'Unexpected object: true'),
(False, 'Unexpected object: false'),
(None, 'Unexpected object: nil'),
))
@pytest.mark.parametrize('structure', (
lambda elem: {'chunks': elem},
lambda elem: {'chunks': [elem]},
lambda elem: {'chunks': [(elem, 1)]},
))
@pytest.mark.parametrize(
"elem,error",
(
({1: 2}, "Unexpected object: map"),
(
bytes(213),
["Unexpected bytes in chunks structure", "Incorrect key length"], # structure 2/3
), # structure 3/3
(1, "Unexpected object: integer"),
(1.0, "Unexpected object: double"),
(True, "Unexpected object: true"),
(False, "Unexpected object: false"),
(None, "Unexpected object: nil"),
),
)
@pytest.mark.parametrize(
"structure",
(lambda elem: {"chunks": elem}, lambda elem: {"chunks": [elem]}, lambda elem: {"chunks": [(elem, 1)]}),
)
def test_corrupted(self, sync, structure, elem, error):
packed = packb(structure(elem))
with pytest.raises(ValueError) as excinfo:
sync.feed(packed)
if isinstance(error, str):
error = [error]
possible_errors = ['cache_sync_feed failed: ' + error for error in error]
possible_errors = ["cache_sync_feed failed: " + error for error in error]
assert str(excinfo.value) in possible_errors
@pytest.mark.parametrize('data,error', (
# Incorrect tuple length
({'chunks': [(bytes(32), 2, 3, 4)]}, 'Invalid chunk list entry length'),
({'chunks': [(bytes(32), )]}, 'Invalid chunk list entry length'),
# Incorrect types
({'chunks': [(1, 2)]}, 'Unexpected object: integer'),
({'chunks': [(1, bytes(32))]}, 'Unexpected object: integer'),
({'chunks': [(bytes(32), 1.0)]}, 'Unexpected object: double'),
))
@pytest.mark.parametrize(
"data,error",
(
# Incorrect tuple length
({"chunks": [(bytes(32), 2, 3, 4)]}, "Invalid chunk list entry length"),
({"chunks": [(bytes(32),)]}, "Invalid chunk list entry length"),
# Incorrect types
({"chunks": [(1, 2)]}, "Unexpected object: integer"),
({"chunks": [(1, bytes(32))]}, "Unexpected object: integer"),
({"chunks": [(bytes(32), 1.0)]}, "Unexpected object: double"),
),
)
def test_corrupted_ancillary(self, index, sync, data, error):
packed = packb(data)
with pytest.raises(ValueError) as excinfo:
sync.feed(packed)
assert str(excinfo.value) == 'cache_sync_feed failed: ' + error
assert str(excinfo.value) == "cache_sync_feed failed: " + error
def make_index_with_refcount(self, refcount):
index_data = io.BytesIO()
index_data.write(b'BORG_IDX')
index_data.write(b"BORG_IDX")
# num_entries
index_data.write((1).to_bytes(4, 'little'))
index_data.write((1).to_bytes(4, "little"))
# num_buckets
index_data.write((1).to_bytes(4, 'little'))
index_data.write((1).to_bytes(4, "little"))
# key_size
index_data.write((32).to_bytes(1, 'little'))
index_data.write((32).to_bytes(1, "little"))
# value_size
index_data.write((3 * 4).to_bytes(1, 'little'))
index_data.write((3 * 4).to_bytes(1, "little"))
index_data.write(H(0))
index_data.write(refcount.to_bytes(4, 'little'))
index_data.write((1234).to_bytes(4, 'little'))
index_data.write((5678).to_bytes(4, 'little'))
index_data.write(refcount.to_bytes(4, "little"))
index_data.write((1234).to_bytes(4, "little"))
index_data.write((5678).to_bytes(4, "little"))
index_data.seek(0)
index = ChunkIndex.read(index_data)
@ -171,34 +131,22 @@ def make_index_with_refcount(self, refcount):
def test_corrupted_refcount(self):
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE + 1)
sync = CacheSynchronizer(index)
data = packb({
'chunks': [
(H(0), 1),
]
})
data = packb({"chunks": [(H(0), 1)]})
with pytest.raises(ValueError) as excinfo:
sync.feed(data)
assert str(excinfo.value) == 'cache_sync_feed failed: invalid reference count'
assert str(excinfo.value) == "cache_sync_feed failed: invalid reference count"
def test_refcount_max_value(self):
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE)
sync = CacheSynchronizer(index)
data = packb({
'chunks': [
(H(0), 1),
]
})
data = packb({"chunks": [(H(0), 1)]})
sync.feed(data)
assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
def test_refcount_one_below_max_value(self):
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE - 1)
sync = CacheSynchronizer(index)
data = packb({
'chunks': [
(H(0), 1),
]
})
data = packb({"chunks": [(H(0), 1)]})
sync.feed(data)
# Incremented to maximum
assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
@ -209,17 +157,17 @@ def test_refcount_one_below_max_value(self):
class TestAdHocCache:
@pytest.fixture
def repository(self, tmpdir):
self.repository_location = os.path.join(str(tmpdir), 'repository')
self.repository_location = os.path.join(str(tmpdir), "repository")
with Repository(self.repository_location, exclusive=True, create=True) as repository:
repository.put(H(1), b'1234')
repository.put(Manifest.MANIFEST_ID, b'5678')
repository.put(H(1), b"1234")
repository.put(Manifest.MANIFEST_ID, b"5678")
yield repository
@pytest.fixture
def key(self, repository, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
key = AESOCBRepoKey.create(repository, TestKey.MockArgs())
key.compressor = CompressionSpec('none').compressor
key.compressor = CompressionSpec("none").compressor
return key
@pytest.fixture
@ -237,18 +185,18 @@ def test_does_not_contain_manifest(self, cache):
def test_does_not_delete_existing_chunks(self, repository, cache):
assert cache.seen_chunk(H(1)) == ChunkIndex.MAX_VALUE
cache.chunk_decref(H(1), Statistics())
assert repository.get(H(1)) == b'1234'
assert repository.get(H(1)) == b"1234"
def test_does_not_overwrite(self, cache):
with pytest.raises(AssertionError):
cache.add_chunk(H(1), b'5678', Statistics(), overwrite=True)
cache.add_chunk(H(1), b"5678", Statistics(), overwrite=True)
def test_seen_chunk_add_chunk_size(self, cache):
assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4)
assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4)
def test_deletes_chunks_during_lifetime(self, cache, repository):
"""E.g. checkpoint archives"""
cache.add_chunk(H(5), b'1010', Statistics())
cache.add_chunk(H(5), b"1010", Statistics())
assert cache.seen_chunk(H(5)) == 1
cache.chunk_decref(H(5), Statistics())
assert not cache.seen_chunk(H(5))
@ -256,8 +204,8 @@ def test_deletes_chunks_during_lifetime(self, cache, repository):
repository.get(H(5))
def test_files_cache(self, cache):
assert cache.file_known_and_unchanged(b'foo', bytes(32), None) == (False, None)
assert cache.cache_mode == 'd'
assert cache.file_known_and_unchanged(b"foo", bytes(32), None) == (False, None)
assert cache.cache_mode == "d"
assert cache.files is None
def test_txn(self, cache):
@ -267,13 +215,13 @@ def test_txn(self, cache):
assert cache.chunks
cache.rollback()
assert not cache._txn_active
assert not hasattr(cache, 'chunks')
assert not hasattr(cache, "chunks")
def test_incref_after_add_chunk(self, cache):
assert cache.add_chunk(H(3), b'5678', Statistics()) == (H(3), 4)
assert cache.add_chunk(H(3), b"5678", Statistics()) == (H(3), 4)
assert cache.chunk_incref(H(3), Statistics()) == (H(3), 4)
def test_existing_incref_after_add_chunk(self, cache):
"""This case occurs with part files, see Archive.chunk_file."""
assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4)
assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4)
assert cache.chunk_incref(H(1), Statistics()) == (H(1), 4)

26
src/borg/testsuite/checksums.py
View File

@ -5,16 +5,24 @@
def test_xxh64():
assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'
assert bin_to_hex(checksums.xxh64(b'test')) == '4fdcca5ddb678139'
assert bin_to_hex(checksums.xxh64(unhexlify(
'6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724'
'fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d'
'cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331'))) == '35d5d2f545d9511a'
assert bin_to_hex(checksums.xxh64(b"test", 123)) == "2b81b9401bef86cf"
assert bin_to_hex(checksums.xxh64(b"test")) == "4fdcca5ddb678139"
assert (
bin_to_hex(
checksums.xxh64(
unhexlify(
"6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724"
"fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d"
"cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331"
)
)
)
== "35d5d2f545d9511a"
)
def test_streaming_xxh64():
hasher = checksums.StreamingXXH64(123)
hasher.update(b'te')
hasher.update(b'st')
assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == '2b81b9401bef86cf'
hasher.update(b"te")
hasher.update(b"st")
assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == "2b81b9401bef86cf"

122
src/borg/testsuite/chunker.py
View File

@ -12,113 +12,129 @@ def cf(chunks):
"""chunk filter"""
# this is to simplify testing: either return the data piece (bytes) or the hole length (int).
def _cf(chunk):
if chunk.meta['allocation'] == CH_DATA:
assert len(chunk.data) == chunk.meta['size']
if chunk.meta["allocation"] == CH_DATA:
assert len(chunk.data) == chunk.meta["size"]
return bytes(chunk.data) # make sure we have bytes, not memoryview
if chunk.meta['allocation'] in (CH_HOLE, CH_ALLOC):
if chunk.meta["allocation"] in (CH_HOLE, CH_ALLOC):
assert chunk.data is None
return chunk.meta['size']
return chunk.meta["size"]
assert False, "unexpected allocation value"
return [_cf(chunk) for chunk in chunks]
class ChunkerFixedTestCase(BaseTestCase):
def test_chunkify_just_blocks(self):
data = b'foobar' * 1500
data = b"foobar" * 1500
chunker = ChunkerFixed(4096)
parts = cf(chunker.chunkify(BytesIO(data)))
self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]])
def test_chunkify_header_and_blocks(self):
data = b'foobar' * 1500
data = b"foobar" * 1500
chunker = ChunkerFixed(4096, 123)
parts = cf(chunker.chunkify(BytesIO(data)))
self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]])
self.assert_equal(
parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]]
)
def test_chunkify_just_blocks_fmap_complete(self):
data = b'foobar' * 1500
data = b"foobar" * 1500
chunker = ChunkerFixed(4096)
fmap = [
(0, 4096, True),
(4096, 8192, True),
(8192, 99999999, True),
]
fmap = [(0, 4096, True), (4096, 8192, True), (8192, 99999999, True)]
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]])
def test_chunkify_header_and_blocks_fmap_complete(self):
data = b'foobar' * 1500
data = b"foobar" * 1500
chunker = ChunkerFixed(4096, 123)
fmap = [
(0, 123, True),
(123, 4096, True),
(123+4096, 4096, True),
(123+8192, 4096, True),
]
fmap = [(0, 123, True), (123, 4096, True), (123 + 4096, 4096, True), (123 + 8192, 4096, True)]
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]])
self.assert_equal(
parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]]
)
def test_chunkify_header_and_blocks_fmap_zeros(self):
data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096
data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096
chunker = ChunkerFixed(4096, 123)
fmap = [
(0, 123, True),
(123, 4096, False),
(123+4096, 4096, True),
(123+8192, 4096, False),
]
fmap = [(0, 123, True), (123, 4096, False), (123 + 4096, 4096, True), (123 + 8192, 4096, False)]
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
# because we marked the '_' ranges as holes, we will get hole ranges instead!
self.assert_equal(parts, [data[0:123], 4096, data[123+4096:123+8192], 4096])
self.assert_equal(parts, [data[0:123], 4096, data[123 + 4096 : 123 + 8192], 4096])
def test_chunkify_header_and_blocks_fmap_partial(self):
data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096
data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096
chunker = ChunkerFixed(4096, 123)
fmap = [
(0, 123, True),
# (123, 4096, False),
(123+4096, 4096, True),
(123 + 4096, 4096, True),
# (123+8192, 4096, False),
]
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
# because we left out the '_' ranges from the fmap, we will not get them at all!
self.assert_equal(parts, [data[0:123], data[123+4096:123+8192]])
self.assert_equal(parts, [data[0:123], data[123 + 4096 : 123 + 8192]])
class ChunkerTestCase(BaseTestCase):
def test_chunkify(self):
data = b'0' * int(1.5 * (1 << CHUNK_MAX_EXP)) + b'Y'
data = b"0" * int(1.5 * (1 << CHUNK_MAX_EXP)) + b"Y"
parts = cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(data)))
self.assert_equal(len(parts), 2)
self.assert_equal(b''.join(parts), data)
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b''))), [])
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz'])
self.assert_equal(cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz'])
self.assert_equal(cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz'])
self.assert_equal(cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3])
self.assert_equal(cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz'])
self.assert_equal(cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz'])
self.assert_equal(cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3])
self.assert_equal(cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarbo', b'obazfoobar', b'boobazfo', b'obarboobaz'])
self.assert_equal(cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz', b'foobarboobaz', b'foobarboobaz'])
self.assert_equal(b"".join(parts), data)
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b""))), [])
self.assert_equal(
cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"fooba", b"rboobaz", b"fooba", b"rboobaz", b"fooba", b"rboobaz"],
)
self.assert_equal(
cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"fo", b"obarb", b"oob", b"azf", b"oobarb", b"oob", b"azf", b"oobarb", b"oobaz"],
)
self.assert_equal(
cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"foob", b"ar", b"boobazfoob", b"ar", b"boobazfoob", b"ar", b"boobaz"],
)
self.assert_equal(
cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3]
)
self.assert_equal(
cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"foobar", b"boobazfo", b"obar", b"boobazfo", b"obar", b"boobaz"],
)
self.assert_equal(
cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"foob", b"arboobaz", b"foob", b"arboobaz", b"foob", b"arboobaz"],
)
self.assert_equal(
cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3]
)
self.assert_equal(
cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"foobarbo", b"obazfoobar", b"boobazfo", b"obarboobaz"],
)
self.assert_equal(
cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
[b"foobarboobaz", b"foobarboobaz", b"foobarboobaz"],
)
def test_buzhash(self):
self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1))
self.assert_equal(buzhash(b"abcdefghijklmnop", 0), 3795437769)
self.assert_equal(buzhash(b"abcdefghijklmnop", 1), 3795400502)
self.assert_equal(
buzhash(b"abcdefghijklmnop", 1), buzhash_update(buzhash(b"Xabcdefghijklmno", 1), ord("X"), ord("p"), 16, 1)
)
# Test with more than 31 bytes to make sure our barrel_shift macro works correctly
self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248)
self.assert_equal(buzhash(b"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", 0), 566521248)
def test_small_reads(self):
class SmallReadFile:
input = b'a' * (20 + 1)
input = b"a" * (20 + 1)
def read(self, nbytes):
self.input = self.input[:-1]
return self.input[:1]
chunker = get_chunker(*CHUNKER_PARAMS, seed=0)
reconstructed = b''.join(cf(chunker.chunkify(SmallReadFile())))
assert reconstructed == b'a' * 20
reconstructed = b"".join(cf(chunker.chunkify(SmallReadFile())))
assert reconstructed == b"a" * 20

91
src/borg/testsuite/chunker_pytest.py
View File

@ -12,37 +12,27 @@
# some sparse files. X = content blocks, _ = sparse blocks.
# X__XXX____
map_sparse1 = [
(0 * BS, 1 * BS, True),
(1 * BS, 2 * BS, False),
(3 * BS, 3 * BS, True),
(6 * BS, 4 * BS, False),
]
map_sparse1 = [(0 * BS, 1 * BS, True), (1 * BS, 2 * BS, False), (3 * BS, 3 * BS, True), (6 * BS, 4 * BS, False)]
# _XX___XXXX
map_sparse2 = [
(0 * BS, 1 * BS, False),
(1 * BS, 2 * BS, True),
(3 * BS, 3 * BS, False),
(6 * BS, 4 * BS, True),
]
map_sparse2 = [(0 * BS, 1 * BS, False), (1 * BS, 2 * BS, True), (3 * BS, 3 * BS, False), (6 * BS, 4 * BS, True)]
# XXX
map_notsparse = [(0 * BS, 3 * BS, True), ]
map_notsparse = [(0 * BS, 3 * BS, True)]
# ___
map_onlysparse = [(0 * BS, 3 * BS, False), ]
map_onlysparse = [(0 * BS, 3 * BS, False)]
def make_sparsefile(fname, sparsemap, header_size=0):
with open(fname, 'wb') as fd:
with open(fname, "wb") as fd:
total = 0
if header_size:
fd.write(b'H' * header_size)
fd.write(b"H" * header_size)
total += header_size
for offset, size, is_data in sparsemap:
if is_data:
fd.write(b'X' * size)
fd.write(b"X" * size)
else:
fd.seek(size, os.SEEK_CUR)
total += size
@ -54,11 +44,11 @@ def make_content(sparsemap, header_size=0):
result = []
total = 0
if header_size:
result.append(b'H' * header_size)
result.append(b"H" * header_size)
total += header_size
for offset, size, is_data in sparsemap:
if is_data:
result.append(b'X' * size) # bytes!
result.append(b"X" * size) # bytes!
else:
result.append(size) # int!
total += size
@ -69,9 +59,9 @@ def fs_supports_sparse():
if not has_seek_hole:
return False
with tempfile.TemporaryDirectory() as tmpdir:
fn = os.path.join(tmpdir, 'test_sparse')
fn = os.path.join(tmpdir, "test_sparse")
make_sparsefile(fn, [(0, BS, False), (BS, BS, True)])
with open(fn, 'rb') as f:
with open(fn, "rb") as f:
try:
offset_hole = f.seek(0, os.SEEK_HOLE)
offset_data = f.seek(0, os.SEEK_DATA)
@ -81,15 +71,12 @@ def fs_supports_sparse():
return offset_hole == 0 and offset_data == BS
@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files')
@pytest.mark.parametrize("fname, sparse_map", [
('sparse1', map_sparse1),
('sparse2', map_sparse2),
('onlysparse', map_onlysparse),
('notsparse', map_notsparse),
])
@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files")
@pytest.mark.parametrize(
"fname, sparse_map",
[("sparse1", map_sparse1), ("sparse2", map_sparse2), ("onlysparse", map_onlysparse), ("notsparse", map_notsparse)],
)
def test_sparsemap(tmpdir, fname, sparse_map):
def get_sparsemap_fh(fname):
fh = os.open(fname, flags=os.O_RDONLY)
try:
@ -98,7 +85,7 @@ def get_sparsemap_fh(fname):
os.close(fh)
def get_sparsemap_fd(fname):
with open(fname, 'rb') as fd:
with open(fname, "rb") as fd:
return list(sparsemap(fd=fd))
fn = str(tmpdir / fname)
@ -107,30 +94,32 @@ def get_sparsemap_fd(fname):
assert get_sparsemap_fd(fn) == sparse_map
@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files')
@pytest.mark.parametrize("fname, sparse_map, header_size, sparse", [
('sparse1', map_sparse1, 0, False),
('sparse1', map_sparse1, 0, True),
('sparse1', map_sparse1, BS, False),
('sparse1', map_sparse1, BS, True),
('sparse2', map_sparse2, 0, False),
('sparse2', map_sparse2, 0, True),
('sparse2', map_sparse2, BS, False),
('sparse2', map_sparse2, BS, True),
('onlysparse', map_onlysparse, 0, False),
('onlysparse', map_onlysparse, 0, True),
('onlysparse', map_onlysparse, BS, False),
('onlysparse', map_onlysparse, BS, True),
('notsparse', map_notsparse, 0, False),
('notsparse', map_notsparse, 0, True),
('notsparse', map_notsparse, BS, False),
('notsparse', map_notsparse, BS, True),
])
@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files")
@pytest.mark.parametrize(
"fname, sparse_map, header_size, sparse",
[
("sparse1", map_sparse1, 0, False),
("sparse1", map_sparse1, 0, True),
("sparse1", map_sparse1, BS, False),
("sparse1", map_sparse1, BS, True),
("sparse2", map_sparse2, 0, False),
("sparse2", map_sparse2, 0, True),
("sparse2", map_sparse2, BS, False),
("sparse2", map_sparse2, BS, True),
("onlysparse", map_onlysparse, 0, False),
("onlysparse", map_onlysparse, 0, True),
("onlysparse", map_onlysparse, BS, False),
("onlysparse", map_onlysparse, BS, True),
("notsparse", map_notsparse, 0, False),
("notsparse", map_notsparse, 0, True),
("notsparse", map_notsparse, BS, False),
("notsparse", map_notsparse, BS, True),
],
)
def test_chunkify_sparse(tmpdir, fname, sparse_map, header_size, sparse):
def get_chunks(fname, sparse, header_size):
chunker = ChunkerFixed(4096, header_size=header_size, sparse=sparse)
with open(fname, 'rb') as fd:
with open(fname, "rb") as fd:
return cf(chunker.chunkify(fd))
fn = str(tmpdir / fname)

7
src/borg/testsuite/chunker_slow.py
View File

@ -9,7 +9,6 @@
class ChunkerRegressionTestCase(BaseTestCase):
def test_chunkpoints_unchanged(self):
def twist(size):
x = 1
@ -31,10 +30,10 @@ def twist(size):
for seed in (1849058162, 1234567653):
fh = BytesIO(data)
chunker = Chunker(seed, minexp, maxexp, maskbits, winsize)
chunks = [blake2b_256(b'', c) for c in cf(chunker.chunkify(fh, -1))]
runs.append(blake2b_256(b'', b''.join(chunks)))
chunks = [blake2b_256(b"", c) for c in cf(chunker.chunkify(fh, -1))]
runs.append(blake2b_256(b"", b"".join(chunks)))
# The "correct" hash below matches the existing chunker behavior.
# Future chunker optimisations must not change this, or existing repos will bloat.
overall_hash = blake2b_256(b'', b''.join(runs))
overall_hash = blake2b_256(b"", b"".join(runs))
self.assert_equal(overall_hash, unhexlify("b559b0ac8df8daaa221201d018815114241ea5c6609d98913cd2246a702af4e3"))

99
src/borg/testsuite/compress.py
View File

@ -1,5 +1,6 @@
import os
import zlib
try:
import lzma
except ImportError:
@ -11,23 +12,23 @@
buffer = bytes(2**16)
data = b'fooooooooobaaaaaaaar' * 10
params = dict(name='zlib', level=6)
data = b"fooooooooobaaaaaaaar" * 10
params = dict(name="zlib", level=6)
def test_get_compressor():
c = get_compressor(name='none')
c = get_compressor(name="none")
assert isinstance(c, CNONE)
c = get_compressor(name='lz4')
c = get_compressor(name="lz4")
assert isinstance(c, LZ4)
c = get_compressor(name='zlib')
c = get_compressor(name="zlib")
assert isinstance(c, ZLIB)
with pytest.raises(KeyError):
get_compressor(name='foobar')
get_compressor(name="foobar")
def test_cnull():
c = get_compressor(name='none')
c = get_compressor(name="none")
cdata = c.compress(data)
assert len(cdata) > len(data)
assert data in cdata # it's not compressed and just in there 1:1
@ -36,7 +37,7 @@ def test_cnull():
def test_lz4():
c = get_compressor(name='lz4')
c = get_compressor(name="lz4")
cdata = c.compress(data)
assert len(cdata) < len(data)
assert data == c.decompress(cdata)
@ -45,18 +46,18 @@ def test_lz4():
def test_lz4_buffer_allocation(monkeypatch):
# disable fallback to no compression on incompressible data
monkeypatch.setattr(LZ4, 'decide', lambda always_compress: LZ4)
monkeypatch.setattr(LZ4, "decide", lambda always_compress: LZ4)
# test with a rather huge data object to see if buffer allocation / resizing works
data = os.urandom(5 * 2**20) * 10 # 50MiB badly compressible data
assert len(data) == 50 * 2**20
c = Compressor('lz4')
c = Compressor("lz4")
cdata = c.compress(data)
assert len(cdata) > len(data)
assert data == c.decompress(cdata)
def test_zlib():
c = get_compressor(name='zlib')
c = get_compressor(name="zlib")
cdata = c.compress(data)
assert len(cdata) < len(data)
assert data == c.decompress(cdata)
@ -66,7 +67,7 @@ def test_zlib():
def test_lzma():
if lzma is None:
pytest.skip("No lzma support found.")
c = get_compressor(name='lzma')
c = get_compressor(name="lzma")
cdata = c.compress(data)
assert len(cdata) < len(data)
assert data == c.decompress(cdata)
@ -74,7 +75,7 @@ def test_lzma():
def test_zstd():
c = get_compressor(name='zstd')
c = get_compressor(name="zstd")
cdata = c.compress(data)
assert len(cdata) < len(data)
assert data == c.decompress(cdata)
@ -83,16 +84,16 @@ def test_zstd():
def test_autodetect_invalid():
with pytest.raises(ValueError):
Compressor(**params).decompress(b'\xff\xfftotalcrap')
Compressor(**params).decompress(b"\xff\xfftotalcrap")
with pytest.raises(ValueError):
Compressor(**params).decompress(b'\x08\x00notreallyzlib')
Compressor(**params).decompress(b"\x08\x00notreallyzlib")
def test_zlib_legacy_compat():
# for compatibility reasons, we do not add an extra header for zlib,
# nor do we expect one when decompressing / autodetecting
for level in range(10):
c = get_compressor(name='zlib_legacy', level=level)
c = get_compressor(name="zlib_legacy", level=level)
cdata1 = c.compress(data)
cdata2 = zlib.compress(data, level)
assert cdata1 == cdata2
@ -104,19 +105,19 @@ def test_zlib_legacy_compat():
def test_compressor():
params_list = [
dict(name='none'),
dict(name='lz4'),
dict(name='zstd', level=1),
dict(name='zstd', level=3),
dict(name="none"),
dict(name="lz4"),
dict(name="zstd", level=1),
dict(name="zstd", level=3),
# avoiding high zstd levels, memory needs unclear
dict(name='zlib', level=0),
dict(name='zlib', level=6),
dict(name='zlib', level=9),
dict(name="zlib", level=0),
dict(name="zlib", level=6),
dict(name="zlib", level=9),
]
if lzma:
params_list += [
dict(name='lzma', level=0),
dict(name='lzma', level=6),
dict(name="lzma", level=0),
dict(name="lzma", level=6),
# we do not test lzma on level 9 because of the huge memory needs
]
for params in params_list:
@ -125,9 +126,9 @@ def test_compressor():
def test_auto():
compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor
compressor_lz4 = CompressionSpec('lz4').compressor
compressor_zlib = CompressionSpec('zlib,9').compressor
compressor_auto_zlib = CompressionSpec("auto,zlib,9").compressor
compressor_lz4 = CompressionSpec("lz4").compressor
compressor_zlib = CompressionSpec("zlib,9").compressor
data = bytes(500)
compressed_auto_zlib = compressor_auto_zlib.compress(data)
compressed_lz4 = compressor_lz4.compress(data)
@ -135,13 +136,13 @@ def test_auto():
ratio = len(compressed_zlib) / len(compressed_lz4)
assert Compressor.detect(compressed_auto_zlib)[0] == ZLIB if ratio < 0.99 else LZ4
data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~'
data = b"\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~"
compressed = compressor_auto_zlib.compress(data)
assert Compressor.detect(compressed)[0] == CNONE
def test_obfuscate():
compressor = CompressionSpec('obfuscate,1,none').compressor
compressor = CompressionSpec("obfuscate,1,none").compressor
data = bytes(10000)
compressed = compressor.compress(data)
# 2 id bytes compression, 2 id bytes obfuscator. 4 length bytes
@ -149,7 +150,7 @@ def test_obfuscate():
# compressing 100 times the same data should give at least 50 different result sizes
assert len({len(compressor.compress(data)) for i in range(100)}) > 50
cs = CompressionSpec('obfuscate,2,lz4')
cs = CompressionSpec("obfuscate,2,lz4")
assert isinstance(cs.inner.compressor, LZ4)
compressor = cs.compressor
data = bytes(10000)
@ -160,7 +161,7 @@ def test_obfuscate():
# compressing 100 times the same data should give multiple different result sizes
assert len({len(compressor.compress(data)) for i in range(100)}) > 10
cs = CompressionSpec('obfuscate,6,zstd,3')
cs = CompressionSpec("obfuscate,6,zstd,3")
assert isinstance(cs.inner.compressor, ZSTD)
compressor = cs.compressor
data = bytes(10000)
@ -171,7 +172,7 @@ def test_obfuscate():
# compressing 100 times the same data should give multiple different result sizes
assert len({len(compressor.compress(data)) for i in range(100)}) > 90
cs = CompressionSpec('obfuscate,2,auto,zstd,10')
cs = CompressionSpec("obfuscate,2,auto,zstd,10")
assert isinstance(cs.inner.compressor, Auto)
compressor = cs.compressor
data = bytes(10000)
@ -182,7 +183,7 @@ def test_obfuscate():
# compressing 100 times the same data should give multiple different result sizes
assert len({len(compressor.compress(data)) for i in range(100)}) > 10
cs = CompressionSpec('obfuscate,110,none')
cs = CompressionSpec("obfuscate,110,none")
assert isinstance(cs.inner.compressor, CNONE)
compressor = cs.compressor
data = bytes(1000)
@ -199,44 +200,44 @@ def test_obfuscate():
def test_compression_specs():
with pytest.raises(ValueError):
CompressionSpec('')
CompressionSpec("")
assert isinstance(CompressionSpec('none').compressor, CNONE)
assert isinstance(CompressionSpec('lz4').compressor, LZ4)
assert isinstance(CompressionSpec("none").compressor, CNONE)
assert isinstance(CompressionSpec("lz4").compressor, LZ4)
zlib = CompressionSpec('zlib').compressor
zlib = CompressionSpec("zlib").compressor
assert isinstance(zlib, ZLIB)
assert zlib.level == 6
zlib = CompressionSpec('zlib,0').compressor
zlib = CompressionSpec("zlib,0").compressor
assert isinstance(zlib, ZLIB)
assert zlib.level == 0
zlib = CompressionSpec('zlib,9').compressor
zlib = CompressionSpec("zlib,9").compressor
assert isinstance(zlib, ZLIB)
assert zlib.level == 9
with pytest.raises(ValueError):
CompressionSpec('zlib,9,invalid')
CompressionSpec("zlib,9,invalid")
lzma = CompressionSpec('lzma').compressor
lzma = CompressionSpec("lzma").compressor
assert isinstance(lzma, LZMA)
assert lzma.level == 6
lzma = CompressionSpec('lzma,0').compressor
lzma = CompressionSpec("lzma,0").compressor
assert isinstance(lzma, LZMA)
assert lzma.level == 0
lzma = CompressionSpec('lzma,9').compressor
lzma = CompressionSpec("lzma,9").compressor
assert isinstance(lzma, LZMA)
assert lzma.level == 9
zstd = CompressionSpec('zstd').compressor
zstd = CompressionSpec("zstd").compressor
assert isinstance(zstd, ZSTD)
assert zstd.level == 3
zstd = CompressionSpec('zstd,1').compressor
zstd = CompressionSpec("zstd,1").compressor
assert isinstance(zstd, ZSTD)
assert zstd.level == 1
zstd = CompressionSpec('zstd,22').compressor
zstd = CompressionSpec("zstd,22").compressor
assert isinstance(zstd, ZSTD)
assert zstd.level == 22
with pytest.raises(ValueError):
CompressionSpec('lzma,9,invalid')
CompressionSpec("lzma,9,invalid")
with pytest.raises(ValueError):
CompressionSpec('invalid')
CompressionSpec("invalid")

241
src/borg/testsuite/crypto.py
View File

@ -16,18 +16,17 @@
class CryptoTestCase(BaseTestCase):
def test_bytes_to_int(self):
self.assert_equal(bytes_to_int(b'\0\0\0\1'), 1)
self.assert_equal(bytes_to_int(b"\0\0\0\1"), 1)
def test_bytes_to_long(self):
self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1)
self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1')
self.assert_equal(bytes_to_long(b"\0\0\0\0\0\0\0\1"), 1)
self.assert_equal(long_to_bytes(1), b"\0\0\0\0\0\0\0\1")
def test_UNENCRYPTED(self):
iv = b'' # any IV is ok, it just must be set and not None
data = b'data'
header = b'header'
iv = b"" # any IV is ok, it just must be set and not None
data = b"data"
header = b"header"
cs = UNENCRYPTED(None, None, iv, header_len=6)
envelope = cs.encrypt(data, header=header)
self.assert_equal(envelope, header + data)
@ -36,11 +35,11 @@ def test_UNENCRYPTED(self):
def test_AES256_CTR_HMAC_SHA256(self):
# this tests the layout as in attic / borg < 1.2 (1 type byte, no aad)
mac_key = b'Y' * 32
enc_key = b'X' * 32
mac_key = b"Y" * 32
enc_key = b"X" * 32
iv = 0
data = b'foo' * 10
header = b'\x42'
data = b"foo" * 10
header = b"\x42"
# encrypt-then-mac
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=1, aad_offset=1)
hdr_mac_iv_cdata = cs.encrypt(data, header=header)
@ -48,10 +47,10 @@ def test_AES256_CTR_HMAC_SHA256(self):
mac = hdr_mac_iv_cdata[1:33]
iv = hdr_mac_iv_cdata[33:41]
cdata = hdr_mac_iv_cdata[41:]
self.assert_equal(hexlify(hdr), b'42')
self.assert_equal(hexlify(mac), b'af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8')
self.assert_equal(hexlify(iv), b'0000000000000000')
self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466')
self.assert_equal(hexlify(hdr), b"42")
self.assert_equal(hexlify(mac), b"af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8")
self.assert_equal(hexlify(iv), b"0000000000000000")
self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466")
self.assert_equal(cs.next_iv(), 2)
# auth-then-decrypt
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
@ -60,16 +59,15 @@ def test_AES256_CTR_HMAC_SHA256(self):
self.assert_equal(cs.next_iv(), 2)
# auth-failure due to corruption (corrupted data)
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b'\0' + hdr_mac_iv_cdata[42:]
self.assert_raises(IntegrityError,
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b"\0" + hdr_mac_iv_cdata[42:]
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
def test_AES256_CTR_HMAC_SHA256_aad(self):
mac_key = b'Y' * 32
enc_key = b'X' * 32
mac_key = b"Y" * 32
enc_key = b"X" * 32
iv = 0
data = b'foo' * 10
header = b'\x12\x34\x56'
data = b"foo" * 10
header = b"\x12\x34\x56"
# encrypt-then-mac
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=3, aad_offset=1)
hdr_mac_iv_cdata = cs.encrypt(data, header=header)
@ -77,10 +75,10 @@ def test_AES256_CTR_HMAC_SHA256_aad(self):
mac = hdr_mac_iv_cdata[3:35]
iv = hdr_mac_iv_cdata[35:43]
cdata = hdr_mac_iv_cdata[43:]
self.assert_equal(hexlify(hdr), b'123456')
self.assert_equal(hexlify(mac), b'7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138')
self.assert_equal(hexlify(iv), b'0000000000000000')
self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466')
self.assert_equal(hexlify(hdr), b"123456")
self.assert_equal(hexlify(mac), b"7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138")
self.assert_equal(hexlify(iv), b"0000000000000000")
self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466")
self.assert_equal(cs.next_iv(), 2)
# auth-then-decrypt
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
@ -89,24 +87,27 @@ def test_AES256_CTR_HMAC_SHA256_aad(self):
self.assert_equal(cs.next_iv(), 2)
# auth-failure due to corruption (corrupted aad)
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:]
self.assert_raises(IntegrityError,
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:]
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
def test_AE(self):
# used in legacy-like layout (1 type byte, no aad)
key = b'X' * 32
key = b"X" * 32
iv_int = 0
data = b'foo' * 10
header = b'\x23' + iv_int.to_bytes(12, 'big')
data = b"foo" * 10
header = b"\x23" + iv_int.to_bytes(12, "big")
tests = [
# (ciphersuite class, exp_mac, exp_cdata)
(AES256_OCB,
b'b6909c23c9aaebd9abbe1ff42097652d',
b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ),
(CHACHA20_POLY1305,
b'fd08594796e0706cde1e8b461e3e0555',
b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', )
(
AES256_OCB,
b"b6909c23c9aaebd9abbe1ff42097652d",
b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493",
),
(
CHACHA20_POLY1305,
b"fd08594796e0706cde1e8b461e3e0555",
b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775",
),
]
for cs_cls, exp_mac, exp_cdata in tests:
# print(repr(cs_cls))
@ -117,9 +118,9 @@ def test_AE(self):
iv = hdr_mac_iv_cdata[1:13]
mac = hdr_mac_iv_cdata[13:29]
cdata = hdr_mac_iv_cdata[29:]
self.assert_equal(hexlify(hdr), b'23')
self.assert_equal(hexlify(hdr), b"23")
self.assert_equal(hexlify(mac), exp_mac)
self.assert_equal(hexlify(iv), b'000000000000000000000000')
self.assert_equal(hexlify(iv), b"000000000000000000000000")
self.assert_equal(hexlify(cdata), exp_cdata)
self.assert_equal(cs.next_iv(), 1)
# auth/decrypt
@ -129,24 +130,27 @@ def test_AE(self):
self.assert_equal(cs.next_iv(), 1)
# auth-failure due to corruption (corrupted data)
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1)
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:]
self.assert_raises(IntegrityError,
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b"\0" + hdr_mac_iv_cdata[30:]
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
def test_AEAD(self):
# test with aad
key = b'X' * 32
key = b"X" * 32
iv_int = 0
data = b'foo' * 10
header = b'\x12\x34\x56' + iv_int.to_bytes(12, 'big')
data = b"foo" * 10
header = b"\x12\x34\x56" + iv_int.to_bytes(12, "big")
tests = [
# (ciphersuite class, exp_mac, exp_cdata)
(AES256_OCB,
b'f2748c412af1c7ead81863a18c2c1893',
b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ),
(CHACHA20_POLY1305,
b'b7e7c9a79f2404e14f9aad156bf091dd',
b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', )
(
AES256_OCB,
b"f2748c412af1c7ead81863a18c2c1893",
b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493",
),
(
CHACHA20_POLY1305,
b"b7e7c9a79f2404e14f9aad156bf091dd",
b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775",
),
]
for cs_cls, exp_mac, exp_cdata in tests:
# print(repr(cs_cls))
@ -157,9 +161,9 @@ def test_AEAD(self):
iv = hdr_mac_iv_cdata[3:15]
mac = hdr_mac_iv_cdata[15:31]
cdata = hdr_mac_iv_cdata[31:]
self.assert_equal(hexlify(hdr), b'123456')
self.assert_equal(hexlify(hdr), b"123456")
self.assert_equal(hexlify(mac), exp_mac)
self.assert_equal(hexlify(iv), b'000000000000000000000000')
self.assert_equal(hexlify(iv), b"000000000000000000000000")
self.assert_equal(hexlify(cdata), exp_cdata)
self.assert_equal(cs.next_iv(), 1)
# auth/decrypt
@ -169,101 +173,117 @@ def test_AEAD(self):
self.assert_equal(cs.next_iv(), 1)
# auth-failure due to corruption (corrupted aad)
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1)
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:]
self.assert_raises(IntegrityError,
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:]
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
def test_AEAD_with_more_AAD(self):
# test giving extra aad to the .encrypt() and .decrypt() calls
key = b'X' * 32
key = b"X" * 32
iv_int = 0
data = b'foo' * 10
header = b'\x12\x34'
data = b"foo" * 10
header = b"\x12\x34"
tests = [AES256_OCB, CHACHA20_POLY1305]
for cs_cls in tests:
# encrypt/mac
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b'correct_chunkid')
hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b"correct_chunkid")
# successful auth/decrypt (correct aad)
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b'correct_chunkid')
pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b"correct_chunkid")
self.assert_equal(data, pdata)
# unsuccessful auth (incorrect aad)
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
self.assert_raises(IntegrityError,
lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b'incorrect_chunkid'))
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b"incorrect_chunkid"))
# These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/
# who claims to have verified these against independent Python and C++ implementations.
def test_hkdf_hmac_sha512(self):
ikm = b'\x0b' * 22
salt = bytes.fromhex('000102030405060708090a0b0c')
info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9')
ikm = b"\x0b" * 22
salt = bytes.fromhex("000102030405060708090a0b0c")
info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9")
l = 42
okm = hkdf_hmac_sha512(ikm, salt, info, l)
assert okm == bytes.fromhex('832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb')
assert okm == bytes.fromhex(
"832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb"
)
def test_hkdf_hmac_sha512_2(self):
ikm = bytes.fromhex('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627'
'28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f')
salt = bytes.fromhex('606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868'
'788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf')
info = bytes.fromhex('b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7'
'd8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff')
ikm = bytes.fromhex(
"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627"
"28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f"
)
salt = bytes.fromhex(
"606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868"
"788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
)
info = bytes.fromhex(
"b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7"
"d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
)
l = 82
okm = hkdf_hmac_sha512(ikm, salt, info, l)
assert okm == bytes.fromhex('ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844'
'1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93')
assert okm == bytes.fromhex(
"ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844"
"1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93"
)
def test_hkdf_hmac_sha512_3(self):
ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b')
ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b")
salt = None
info = b''
info = b""
l = 42
okm = hkdf_hmac_sha512(ikm, salt, info, l)
assert okm == bytes.fromhex('f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac')
assert okm == bytes.fromhex(
"f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac"
)
def test_hkdf_hmac_sha512_4(self):
ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b')
salt = bytes.fromhex('000102030405060708090a0b0c')
info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9')
ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b")
salt = bytes.fromhex("000102030405060708090a0b0c")
info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9")
l = 42
okm = hkdf_hmac_sha512(ikm, salt, info, l)
assert okm == bytes.fromhex('7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068')
assert okm == bytes.fromhex(
"7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068"
)
def test_hkdf_hmac_sha512_5(self):
ikm = bytes.fromhex('0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c')
ikm = bytes.fromhex("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c")
salt = None
info = b''
info = b""
l = 42
okm = hkdf_hmac_sha512(ikm, salt, info, l)
assert okm == bytes.fromhex('1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb')
assert okm == bytes.fromhex(
"1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb"
)
def test_decrypt_key_file_argon2_chacha20_poly1305():
plain = b'hello'
plain = b"hello"
# echo -n "hello, pass phrase" | argon2 saltsaltsaltsalt -id -t 1 -k 8 -p 1 -l 32 -r
key = bytes.fromhex('a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec')
key = bytes.fromhex("a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec")
ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0)
envelope = ae_cipher.encrypt(plain)
encrypted = msgpack.packb({
'version': 1,
'salt': b'salt'*4,
'argon2_time_cost': 1,
'argon2_memory_cost': 8,
'argon2_parallelism': 1,
'argon2_type': b'id',
'algorithm': 'argon2 chacha20-poly1305',
'data': envelope,
})
encrypted = msgpack.packb(
{
"version": 1,
"salt": b"salt" * 4,
"argon2_time_cost": 1,
"argon2_memory_cost": 8,
"argon2_parallelism": 1,
"argon2_type": b"id",
"algorithm": "argon2 chacha20-poly1305",
"data": envelope,
}
)
key = CHPOKeyfileKey(None)
decrypted = key.decrypt_key_file(encrypted, "hello, pass phrase")
@ -272,20 +292,15 @@ def test_decrypt_key_file_argon2_chacha20_poly1305():
def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256():
plain = b'hello'
salt = b'salt'*4
plain = b"hello"
salt = b"salt" * 4
passphrase = "hello, pass phrase"
key = FlexiKey.pbkdf2(passphrase, salt, 1, 32)
hash = hmac_sha256(key, plain)
data = AES(key, b'\0'*16).encrypt(plain)
encrypted = msgpack.packb({
'version': 1,
'algorithm': 'sha256',
'iterations': 1,
'salt': salt,
'data': data,
'hash': hash,
})
data = AES(key, b"\0" * 16).encrypt(plain)
encrypted = msgpack.packb(
{"version": 1, "algorithm": "sha256", "iterations": 1, "salt": salt, "data": data, "hash": hash}
)
key = CHPOKeyfileKey(None)
decrypted = key.decrypt_key_file(encrypted, passphrase)
@ -293,7 +308,7 @@ def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256():
assert decrypted == plain
@unittest.mock.patch('getpass.getpass')
@unittest.mock.patch("getpass.getpass")
def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch):
"""https://github.com/borgbackup/borg/pull/6469#discussion_r832670411
@ -322,10 +337,10 @@ def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch):
2. FlexiKey.detect() relies on that interface - it tries an empty passphrase before prompting the user
3. my initial implementation of decrypt_key_file_argon2() was simply passing through the IntegrityError() from AES256_CTR_BASE.decrypt()
"""
repository = MagicMock(id=b'repository_id')
repository = MagicMock(id=b"repository_id")
getpass.return_value = "hello, pass phrase"
monkeypatch.setenv('BORG_DISPLAY_PASSPHRASE', 'no')
AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2'))
monkeypatch.setenv("BORG_DISPLAY_PASSPHRASE", "no")
AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2"))
repository.load_key.return_value = repository.save_key.call_args.args[0]
AESOCBRepoKey.detect(repository, manifest_data=None)

24
src/borg/testsuite/efficient_collection_queue.py
View File

@ -6,13 +6,13 @@
class TestEfficientQueue:
def test_base_usage(self):
queue = EfficientCollectionQueue(100, bytes)
assert queue.peek_front() == b''
queue.push_back(b'1234')
assert queue.peek_front() == b'1234'
assert queue.peek_front() == b""
queue.push_back(b"1234")
assert queue.peek_front() == b"1234"
assert len(queue) == 4
assert queue
queue.pop_front(4)
assert queue.peek_front() == b''
assert queue.peek_front() == b""
assert len(queue) == 0
assert not queue
@ -30,22 +30,22 @@ def test_usage_with_arrays(self):
def test_chunking(self):
queue = EfficientCollectionQueue(2, bytes)
queue.push_back(b'1')
queue.push_back(b'23')
queue.push_back(b'4567')
queue.push_back(b"1")
queue.push_back(b"23")
queue.push_back(b"4567")
assert len(queue) == 7
assert queue.peek_front() == b'12'
assert queue.peek_front() == b"12"
queue.pop_front(3)
assert queue.peek_front() == b'4'
assert queue.peek_front() == b"4"
queue.pop_front(1)
assert queue.peek_front() == b'56'
assert queue.peek_front() == b"56"
queue.pop_front(2)
assert len(queue) == 1
assert queue
with pytest.raises(EfficientCollectionQueue.SizeUnderflow):
queue.pop_front(2)
assert queue.peek_front() == b'7'
assert queue.peek_front() == b"7"
queue.pop_front(1)
assert queue.peek_front() == b''
assert queue.peek_front() == b""
assert len(queue) == 0
assert not queue

109
src/borg/testsuite/file_integrity.py
View File

@ -5,34 +5,30 @@
class TestReadIntegrityFile:
def test_no_integrity(self, tmpdir):
protected_file = tmpdir.join('file')
protected_file.write('1234')
protected_file = tmpdir.join("file")
protected_file.write("1234")
assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None
def test_truncated_integrity(self, tmpdir):
protected_file = tmpdir.join('file')
protected_file.write('1234')
tmpdir.join('file.integrity').write('')
protected_file = tmpdir.join("file")
protected_file.write("1234")
tmpdir.join("file.integrity").write("")
with pytest.raises(FileIntegrityError):
DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file))
def test_unknown_algorithm(self, tmpdir):
protected_file = tmpdir.join('file')
protected_file.write('1234')
tmpdir.join('file.integrity').write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}')
protected_file = tmpdir.join("file")
protected_file.write("1234")
tmpdir.join("file.integrity").write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}')
assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None
@pytest.mark.parametrize('json', (
'{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}',
'[]',
'1234.5',
'"A string"',
'Invalid JSON',
))
@pytest.mark.parametrize(
"json", ('{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}', "[]", "1234.5", '"A string"', "Invalid JSON")
)
def test_malformed(self, tmpdir, json):
protected_file = tmpdir.join('file')
protected_file.write('1234')
tmpdir.join('file.integrity').write(json)
protected_file = tmpdir.join("file")
protected_file.write("1234")
tmpdir.join("file.integrity").write(json)
with pytest.raises(FileIntegrityError):
DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file))
@ -40,74 +36,71 @@ def test_malformed(self, tmpdir, json):
class TestDetachedIntegrityCheckedFile:
@pytest.fixture
def integrity_protected_file(self, tmpdir):
path = str(tmpdir.join('file'))
path = str(tmpdir.join("file"))
with DetachedIntegrityCheckedFile(path, write=True) as fd:
fd.write(b'foo and bar')
fd.write(b"foo and bar")
return path
def test_simple(self, tmpdir, integrity_protected_file):
assert tmpdir.join('file').check(file=True)
assert tmpdir.join('file.integrity').check(file=True)
assert tmpdir.join("file").check(file=True)
assert tmpdir.join("file.integrity").check(file=True)
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
assert fd.read() == b'foo and bar'
assert fd.read() == b"foo and bar"
def test_corrupted_file(self, integrity_protected_file):
with open(integrity_protected_file, 'ab') as fd:
fd.write(b' extra data')
with open(integrity_protected_file, "ab") as fd:
fd.write(b" extra data")
with pytest.raises(FileIntegrityError):
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
assert fd.read() == b'foo and bar extra data'
assert fd.read() == b"foo and bar extra data"
def test_corrupted_file_partial_read(self, integrity_protected_file):
with open(integrity_protected_file, 'ab') as fd:
fd.write(b' extra data')
with open(integrity_protected_file, "ab") as fd:
fd.write(b" extra data")
with pytest.raises(FileIntegrityError):
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
data = b'foo and bar'
data = b"foo and bar"
assert fd.read(len(data)) == data
@pytest.mark.parametrize('new_name', (
'different_file',
'different_file.different_ext',
))
@pytest.mark.parametrize("new_name", ("different_file", "different_file.different_ext"))
def test_renamed_file(self, tmpdir, integrity_protected_file, new_name):
new_path = tmpdir.join(new_name)
tmpdir.join('file').move(new_path)
tmpdir.join('file.integrity').move(new_path + '.integrity')
tmpdir.join("file").move(new_path)
tmpdir.join("file.integrity").move(new_path + ".integrity")
with pytest.raises(FileIntegrityError):
with DetachedIntegrityCheckedFile(str(new_path), write=False) as fd:
assert fd.read() == b'foo and bar'
assert fd.read() == b"foo and bar"
def test_moved_file(self, tmpdir, integrity_protected_file):
new_dir = tmpdir.mkdir('another_directory')
tmpdir.join('file').move(new_dir.join('file'))
tmpdir.join('file.integrity').move(new_dir.join('file.integrity'))
new_path = str(new_dir.join('file'))
new_dir = tmpdir.mkdir("another_directory")
tmpdir.join("file").move(new_dir.join("file"))
tmpdir.join("file.integrity").move(new_dir.join("file.integrity"))
new_path = str(new_dir.join("file"))
with DetachedIntegrityCheckedFile(new_path, write=False) as fd:
assert fd.read() == b'foo and bar'
assert fd.read() == b"foo and bar"
def test_no_integrity(self, tmpdir, integrity_protected_file):
tmpdir.join('file.integrity').remove()
tmpdir.join("file.integrity").remove()
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
assert fd.read() == b'foo and bar'
assert fd.read() == b"foo and bar"
class TestDetachedIntegrityCheckedFileParts:
@pytest.fixture
def integrity_protected_file(self, tmpdir):
path = str(tmpdir.join('file'))
path = str(tmpdir.join("file"))
with DetachedIntegrityCheckedFile(path, write=True) as fd:
fd.write(b'foo and bar')
fd.hash_part('foopart')
fd.write(b' other data')
fd.write(b"foo and bar")
fd.hash_part("foopart")
fd.write(b" other data")
return path
def test_simple(self, integrity_protected_file):
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
data1 = b'foo and bar'
data1 = b"foo and bar"
assert fd.read(len(data1)) == data1
fd.hash_part('foopart')
assert fd.read() == b' other data'
fd.hash_part("foopart")
assert fd.read() == b" other data"
def test_wrong_part_name(self, integrity_protected_file):
with pytest.raises(FileIntegrityError):
@ -115,25 +108,25 @@ def test_wrong_part_name(self, integrity_protected_file):
# the failing hash_part. This is intentional: (1) it makes the code simpler (2) it's a good fail-safe
# against overly broad exception handling.
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
data1 = b'foo and bar'
data1 = b"foo and bar"
assert fd.read(len(data1)) == data1
with pytest.raises(FileIntegrityError):
# This specific bit raises it directly
fd.hash_part('barpart')
fd.hash_part("barpart")
# Still explodes in the end.
@pytest.mark.parametrize('partial_read', (False, True))
@pytest.mark.parametrize("partial_read", (False, True))
def test_part_independence(self, integrity_protected_file, partial_read):
with open(integrity_protected_file, 'ab') as fd:
fd.write(b'some extra stuff that does not belong')
with open(integrity_protected_file, "ab") as fd:
fd.write(b"some extra stuff that does not belong")
with pytest.raises(FileIntegrityError):
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
data1 = b'foo and bar'
data1 = b"foo and bar"
try:
assert fd.read(len(data1)) == data1
fd.hash_part('foopart')
fd.hash_part("foopart")
except FileIntegrityError:
assert False, 'This part must not raise, since this part is still valid.'
assert False, "This part must not raise, since this part is still valid."
if not partial_read:
fd.read()
# But overall it explodes with the final digest. Neat, eh?

68
src/borg/testsuite/hashindex.py
View File

@ -15,7 +15,7 @@
def H(x):
# make some 32byte long thing that depends on x
return bytes('%-0.32d' % x, 'ascii')
return bytes("%-0.32d" % x, "ascii")
def H2(x):
@ -24,7 +24,6 @@ def H2(x):
class HashIndexTestCase(BaseTestCase):
def _generic_test(self, cls, make_value, sha):
idx = cls()
self.assert_equal(len(idx), 0)
@ -57,7 +56,7 @@ def _generic_test(self, cls, make_value, sha):
idx.write(filepath)
del idx
# Verify file contents
with open(filepath, 'rb') as fd:
with open(filepath, "rb") as fd:
self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha)
# Make sure we can open the file
idx = cls.read(filepath)
@ -86,12 +85,14 @@ def _generic_test(self, cls, make_value, sha):
del idx
def test_nsindex(self):
self._generic_test(NSIndex, lambda x: (x, x, x),
'7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256')
self._generic_test(
NSIndex, lambda x: (x, x, x), "7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256"
)
def test_chunkindex(self):
self._generic_test(ChunkIndex, lambda x: (x, x),
'85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef')
self._generic_test(
ChunkIndex, lambda x: (x, x), "85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef"
)
def test_resize(self):
n = 2000 # Must be >= MIN_BUCKETS
@ -218,8 +219,8 @@ def test_flags_iteritems(self):
class HashIndexExtraTestCase(BaseTestCase):
"""These tests are separate because they should not become part of the selftest.
"""
"""These tests are separate because they should not become part of the selftest."""
def test_chunk_indexer(self):
# see _hashindex.c hash_sizes, we want to be close to the max. load
# because interesting errors happen there.
@ -227,7 +228,7 @@ def test_chunk_indexer(self):
index = ChunkIndex(key_count)
all_keys = [hashlib.sha256(H(k)).digest() for k in range(key_count)]
# we're gonna delete 1/3 of all_keys, so let's split them 2/3 and 1/3:
keys, to_delete_keys = all_keys[0:(2*key_count//3)], all_keys[(2*key_count//3):]
keys, to_delete_keys = all_keys[0 : (2 * key_count // 3)], all_keys[(2 * key_count // 3) :]
for i, key in enumerate(keys):
index[key] = (i, i)
@ -286,6 +287,7 @@ def merge(refcount1, refcount2):
idx1.merge(idx2)
refcount, *_ = idx1[H(1)]
return refcount
result = merge(refcounta, refcountb)
# check for commutativity
assert result == merge(refcountb, refcounta)
@ -367,22 +369,24 @@ def test_keyerror(self):
class HashIndexDataTestCase(BaseTestCase):
# This bytestring was created with borg2-pre 2022-06-10
HASHINDEX = b'eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4' \
b'juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4' \
b'juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO' \
b'4ziO4ziO4ziO4ziO4ziO437LHbSVHGw='
HASHINDEX = (
b"eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4"
b"juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4"
b"juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO"
b"4ziO4ziO4ziO4ziO4ziO437LHbSVHGw="
)
def _serialize_hashindex(self, idx):
with tempfile.TemporaryDirectory() as tempdir:
file = os.path.join(tempdir, 'idx')
file = os.path.join(tempdir, "idx")
idx.write(file)
with open(file, 'rb') as f:
with open(file, "rb") as f:
return self._pack(f.read())
def _deserialize_hashindex(self, bytestring):
with tempfile.TemporaryDirectory() as tempdir:
file = os.path.join(tempdir, 'idx')
with open(file, 'wb') as f:
file = os.path.join(tempdir, "idx")
with open(file, "wb") as f:
f.write(self._unpack(bytestring))
return ChunkIndex.read(file)
@ -416,19 +420,19 @@ def test_read_known_good(self):
class HashIndexIntegrityTestCase(HashIndexDataTestCase):
def write_integrity_checked_index(self, tempdir):
idx = self._deserialize_hashindex(self.HASHINDEX)
file = os.path.join(tempdir, 'idx')
file = os.path.join(tempdir, "idx")
with IntegrityCheckedFile(path=file, write=True) as fd:
idx.write(fd)
integrity_data = fd.integrity_data
assert 'final' in integrity_data
assert 'HashHeader' in integrity_data
assert "final" in integrity_data
assert "HashHeader" in integrity_data
return file, integrity_data
def test_integrity_checked_file(self):
with tempfile.TemporaryDirectory() as tempdir:
file, integrity_data = self.write_integrity_checked_index(tempdir)
with open(file, 'r+b') as fd:
fd.write(b'Foo')
with open(file, "r+b") as fd:
fd.write(b"Foo")
with self.assert_raises(FileIntegrityError):
with IntegrityCheckedFile(path=file, write=False, integrity_data=integrity_data) as fd:
ChunkIndex.read(fd)
@ -437,15 +441,15 @@ def test_integrity_checked_file(self):
class HashIndexCompactTestCase(HashIndexDataTestCase):
def index(self, num_entries, num_buckets):
index_data = io.BytesIO()
index_data.write(b'BORG_IDX')
index_data.write(b"BORG_IDX")
# num_entries
index_data.write(num_entries.to_bytes(4, 'little'))
index_data.write(num_entries.to_bytes(4, "little"))
# num_buckets
index_data.write(num_buckets.to_bytes(4, 'little'))
index_data.write(num_buckets.to_bytes(4, "little"))
# key_size
index_data.write((32).to_bytes(1, 'little'))
index_data.write((32).to_bytes(1, "little"))
# value_size
index_data.write((3 * 4).to_bytes(1, 'little'))
index_data.write((3 * 4).to_bytes(1, "little"))
self.index_data = index_data
@ -468,13 +472,13 @@ def index_from_data_compact_to_data(self):
def write_entry(self, key, *values):
self.index_data.write(key)
for value in values:
self.index_data.write(value.to_bytes(4, 'little'))
self.index_data.write(value.to_bytes(4, "little"))
def write_empty(self, key):
self.write_entry(key, 0xffffffff, 0, 0)
self.write_entry(key, 0xFFFFFFFF, 0, 0)
def write_deleted(self, key):
self.write_entry(key, 0xfffffffe, 0, 0)
self.write_entry(key, 0xFFFFFFFE, 0, 0)
def test_simple(self):
self.index(num_entries=3, num_buckets=6)
@ -600,7 +604,7 @@ def HH(x, y, z):
# first 4 bytes. giving a specific x targets bucket index x.
# y is to create different keys and does not go into the bucket index calculation.
# so, same x + different y --> collision
return pack('<IIIIIIII', x, y, z, 0, 0, 0, 0, 0) # 8 * 4 == 32
return pack("<IIIIIIII", x, y, z, 0, 0, 0, 0, 0) # 8 * 4 == 32
idx = NSIndex()

872
src/borg/testsuite/helpers.py
View File

File diff suppressed because it is too large Load Diff

75
src/borg/testsuite/item.py
View File

@ -11,20 +11,20 @@ def test_item_empty():
assert item.as_dict() == {}
assert 'path' not in item
assert "path" not in item
with pytest.raises(ValueError):
'invalid-key' in item
"invalid-key" in item
with pytest.raises(TypeError):
b'path' in item
b"path" in item
with pytest.raises(TypeError):
42 in item
assert item.get('mode') is None
assert item.get('mode', 0o666) == 0o666
assert item.get("mode") is None
assert item.get("mode", 0o666) == 0o666
with pytest.raises(ValueError):
item.get('invalid-key')
item.get("invalid-key")
with pytest.raises(TypeError):
item.get(b'mode')
item.get(b"mode")
with pytest.raises(TypeError):
item.get(42)
@ -37,16 +37,16 @@ def test_item_empty():
def test_item_from_dict():
# does not matter whether we get str or bytes keys
item = Item({b'path': '/a/b/c', b'mode': 0o666})
assert item.path == '/a/b/c'
item = Item({b"path": "/a/b/c", b"mode": 0o666})
assert item.path == "/a/b/c"
assert item.mode == 0o666
assert 'path' in item
assert "path" in item
# does not matter whether we get str or bytes keys
item = Item({'path': '/a/b/c', 'mode': 0o666})
assert item.path == '/a/b/c'
item = Item({"path": "/a/b/c", "mode": 0o666})
assert item.path == "/a/b/c"
assert item.mode == 0o666
assert 'mode' in item
assert "mode" in item
# invalid - no dict
with pytest.raises(TypeError):
@ -58,12 +58,12 @@ def test_item_from_dict():
# invalid - unknown key
with pytest.raises(ValueError):
Item({'foobar': 'baz'})
Item({"foobar": "baz"})
def test_item_from_kw():
item = Item(path='/a/b/c', mode=0o666)
assert item.path == '/a/b/c'
item = Item(path="/a/b/c", mode=0o666)
assert item.path == "/a/b/c"
assert item.mode == 0o666
@ -71,7 +71,7 @@ def test_item_int_property():
item = Item()
item.mode = 0o666
assert item.mode == 0o666
assert item.as_dict() == {'mode': 0o666}
assert item.as_dict() == {"mode": 0o666}
del item.mode
assert item.as_dict() == {}
with pytest.raises(TypeError):
@ -80,34 +80,34 @@ def test_item_int_property():
def test_item_mptimestamp_property():
item = Item()
small, big = 42, 2 ** 65
small, big = 42, 2**65
item.atime = small
assert item.atime == small
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(small)}
assert item.as_dict() == {"atime": Timestamp.from_unix_nano(small)}
item.atime = big
assert item.atime == big
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(big)}
assert item.as_dict() == {"atime": Timestamp.from_unix_nano(big)}
def test_item_se_str_property():
# start simple
item = Item()
item.path = '/a/b/c'
assert item.path == '/a/b/c'
assert item.as_dict() == {'path': '/a/b/c'}
item.path = "/a/b/c"
assert item.path == "/a/b/c"
assert item.as_dict() == {"path": "/a/b/c"}
del item.path
assert item.as_dict() == {}
with pytest.raises(TypeError):
item.path = 42
# non-utf-8 path, needing surrogate-escaping for latin-1 u-umlaut
item = Item(internal_dict={'path': b'/a/\xfc/c'})
assert item.path == '/a/\udcfc/c' # getting a surrogate-escaped representation
assert item.as_dict() == {'path': '/a/\udcfc/c'}
item = Item(internal_dict={"path": b"/a/\xfc/c"})
assert item.path == "/a/\udcfc/c" # getting a surrogate-escaped representation
assert item.as_dict() == {"path": "/a/\udcfc/c"}
del item.path
assert 'path' not in item
item.path = '/a/\udcfc/c' # setting using a surrogate-escaped representation
assert item.as_dict() == {'path': '/a/\udcfc/c'}
assert "path" not in item
item.path = "/a/\udcfc/c" # setting using a surrogate-escaped representation
assert item.as_dict() == {"path": "/a/\udcfc/c"}
def test_item_list_property():
@ -118,18 +118,18 @@ def test_item_list_property():
assert item.chunks == [0]
item.chunks.append(1)
assert item.chunks == [0, 1]
assert item.as_dict() == {'chunks': [0, 1]}
assert item.as_dict() == {"chunks": [0, 1]}
def test_item_dict_property():
item = Item()
item.xattrs = StableDict()
assert item.xattrs == StableDict()
item.xattrs['foo'] = 'bar'
assert item.xattrs['foo'] == 'bar'
item.xattrs['bar'] = 'baz'
assert item.xattrs == StableDict({'foo': 'bar', 'bar': 'baz'})
assert item.as_dict() == {'xattrs': {'foo': 'bar', 'bar': 'baz'}}
item.xattrs["foo"] = "bar"
assert item.xattrs["foo"] == "bar"
item.xattrs["bar"] = "baz"
assert item.xattrs == StableDict({"foo": "bar", "bar": "baz"})
assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}}
def test_unknown_property():
@ -142,10 +142,7 @@ def test_unknown_property():
def test_item_file_size():
item = Item(mode=0o100666, chunks=[
ChunkListEntry(size=1000, id=None),
ChunkListEntry(size=2000, id=None),
])
item = Item(mode=0o100666, chunks=[ChunkListEntry(size=1000, id=None), ChunkListEntry(size=2000, id=None)])
assert item.get_size() == 3000
item.get_size(memorize=True)
assert item.size == 3000

233
src/borg/testsuite/key.py
View File

@ -11,7 +11,13 @@
from ..crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey
from ..crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey
from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256
from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError, UnsupportedKeyFormatError
from ..crypto.key import (
TAMRequiredError,
TAMInvalid,
TAMUnsupportedSuiteError,
UnsupportedManifestError,
UnsupportedKeyFormatError,
)
from ..crypto.key import identify_key
from ..crypto.low_level import IntegrityError as IntegrityErrorBase
from ..helpers import IntegrityError
@ -36,11 +42,17 @@ class MockArgs:
F84MsMMiqpbz4KVICeBZhfAaTPs4W7BC63qml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgLENQ
2uVCoR7EnAoiRzn8J+orbojKtJlNCnQ31SSC8rendmVyc2lvbgE=""".strip()
keyfile2_cdata = unhexlify(re.sub(r'\W', '', """
keyfile2_cdata = unhexlify(
re.sub(
r"\W",
"",
"""
0055f161493fcfc16276e8c31493c4641e1eb19a79d0326fad0291e5a9c98e5933
00000000000003e8d21eaf9b86c297a8cd56432e1915bb
"""))
keyfile2_id = unhexlify('c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314')
""",
)
)
keyfile2_id = unhexlify("c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314")
keyfile_blake2_key_file = """
BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000
@ -56,8 +68,9 @@ class MockArgs:
UTHFJg343jqml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgz3YaUZZ/s+UWywj97EY5b4KhtJYi
qkPqtDDxs2j/T7+ndmVyc2lvbgE=""".strip()
keyfile_blake2_cdata = bytes.fromhex('04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448'
'0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947')
keyfile_blake2_cdata = bytes.fromhex(
"04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448" "0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947"
)
# Verified against b2sum. Entire string passed to BLAKE2, including the padded 64 byte key contained in
# keyfile_blake2_key_file above is
# 19280471de95185ec27ecb6fc9edbb4f4db26974c315ede1cd505fab4250ce7cd0d081ea66946c
@ -65,33 +78,42 @@ class MockArgs:
# 000000000000000000000000000000000000000000000000000000000000000000000000000000
# 00000000000000000000007061796c6f6164
# p a y l o a d
keyfile_blake2_id = bytes.fromhex('d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb')
keyfile_blake2_id = bytes.fromhex("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb")
@pytest.fixture
def keys_dir(self, request, monkeypatch, tmpdir):
monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir))
monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir))
return tmpdir
@pytest.fixture(params=(
# not encrypted
PlaintextKey,
AuthenticatedKey, Blake2AuthenticatedKey,
# legacy crypto
KeyfileKey, Blake2KeyfileKey,
RepoKey, Blake2RepoKey,
# new crypto
AESOCBKeyfileKey, AESOCBRepoKey,
Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey,
CHPOKeyfileKey, CHPORepoKey,
Blake2CHPOKeyfileKey, Blake2CHPORepoKey,
))
@pytest.fixture(
params=(
# not encrypted
PlaintextKey,
AuthenticatedKey,
Blake2AuthenticatedKey,
# legacy crypto
KeyfileKey,
Blake2KeyfileKey,
RepoKey,
Blake2RepoKey,
# new crypto
AESOCBKeyfileKey,
AESOCBRepoKey,
Blake2AESOCBKeyfileKey,
Blake2AESOCBRepoKey,
CHPOKeyfileKey,
CHPORepoKey,
Blake2CHPOKeyfileKey,
Blake2CHPORepoKey,
)
)
def key(self, request, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
return request.param.create(self.MockRepository(), self.MockArgs())
class MockRepository:
class _Location:
raw = processed = '/some/place'
raw = processed = "/some/place"
def canonical_path(self):
return self.processed
@ -114,16 +136,16 @@ def load_key(self):
def test_plaintext(self):
key = PlaintextKey.create(None, None)
chunk = b'foo'
chunk = b"foo"
id = key.id_hash(chunk)
assert hexlify(id) == b'2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'
assert hexlify(id) == b"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
assert chunk == key.decrypt(id, key.encrypt(id, chunk))
def test_keyfile(self, monkeypatch, keys_dir):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
key = KeyfileKey.create(self.MockRepository(), self.MockArgs())
assert key.cipher.next_iv() == 0
chunk = b'ABC'
chunk = b"ABC"
id = key.id_hash(chunk)
manifest = key.encrypt(id, chunk)
assert key.cipher.extract_iv(manifest) == 0
@ -137,18 +159,18 @@ def test_keyfile(self, monkeypatch, keys_dir):
# Key data sanity check
assert len({key2.id_key, key2.enc_key, key2.enc_hmac_key}) == 3
assert key2.chunk_seed != 0
chunk = b'foo'
chunk = b"foo"
id = key.id_hash(chunk)
assert chunk == key2.decrypt(id, key.encrypt(id, chunk))
def test_keyfile_kfenv(self, tmpdir, monkeypatch):
keyfile = tmpdir.join('keyfile')
monkeypatch.setenv('BORG_KEY_FILE', str(keyfile))
monkeypatch.setenv('BORG_PASSPHRASE', 'testkf')
keyfile = tmpdir.join("keyfile")
monkeypatch.setenv("BORG_KEY_FILE", str(keyfile))
monkeypatch.setenv("BORG_PASSPHRASE", "testkf")
assert not keyfile.exists()
key = CHPOKeyfileKey.create(self.MockRepository(), self.MockArgs())
assert keyfile.exists()
chunk = b'ABC'
chunk = b"ABC"
chunk_id = key.id_hash(chunk)
chunk_cdata = key.encrypt(chunk_id, chunk)
key = CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata)
@ -158,27 +180,27 @@ def test_keyfile_kfenv(self, tmpdir, monkeypatch):
CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata)
def test_keyfile2(self, monkeypatch, keys_dir):
with keys_dir.join('keyfile').open('w') as fd:
with keys_dir.join("keyfile").open("w") as fd:
fd.write(self.keyfile2_key_file)
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload'
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload"
def test_keyfile2_kfenv(self, tmpdir, monkeypatch):
keyfile = tmpdir.join('keyfile')
with keyfile.open('w') as fd:
keyfile = tmpdir.join("keyfile")
with keyfile.open("w") as fd:
fd.write(self.keyfile2_key_file)
monkeypatch.setenv('BORG_KEY_FILE', str(keyfile))
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
monkeypatch.setenv("BORG_KEY_FILE", str(keyfile))
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload'
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload"
def test_keyfile_blake2(self, monkeypatch, keys_dir):
with keys_dir.join('keyfile').open('w') as fd:
with keys_dir.join("keyfile").open("w") as fd:
fd.write(self.keyfile_blake2_key_file)
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata)
assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b'payload'
assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b"payload"
def _corrupt_byte(self, key, data, offset):
data = bytearray(data)
@ -186,12 +208,12 @@ def _corrupt_byte(self, key, data, offset):
# will trigger an IntegrityError (does not happen while we stay within TYPES_ACCEPTABLE).
data[offset] ^= 64
with pytest.raises(IntegrityErrorBase):
key.decrypt(b'', data)
key.decrypt(b"", data)
def test_decrypt_integrity(self, monkeypatch, keys_dir):
with keys_dir.join('keyfile').open('w') as fd:
with keys_dir.join("keyfile").open("w") as fd:
fd.write(self.keyfile2_key_file)
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
data = self.keyfile2_cdata
@ -206,7 +228,7 @@ def test_decrypt_integrity(self, monkeypatch, keys_dir):
def test_roundtrip(self, key):
repository = key.repository
plaintext = b'foo'
plaintext = b"foo"
id = key.id_hash(plaintext)
encrypted = key.encrypt(id, plaintext)
identified_key_class = identify_key(encrypted)
@ -216,59 +238,59 @@ def test_roundtrip(self, key):
assert decrypted == plaintext
def test_decrypt_decompress(self, key):
plaintext = b'123456789'
plaintext = b"123456789"
id = key.id_hash(plaintext)
encrypted = key.encrypt(id, plaintext)
assert key.decrypt(id, encrypted, decompress=False) != plaintext
assert key.decrypt(id, encrypted) == plaintext
def test_assert_id(self, key):
plaintext = b'123456789'
plaintext = b"123456789"
id = key.id_hash(plaintext)
key.assert_id(id, plaintext)
id_changed = bytearray(id)
id_changed[0] ^= 1
with pytest.raises(IntegrityError):
key.assert_id(id_changed, plaintext)
plaintext_changed = plaintext + b'1'
plaintext_changed = plaintext + b"1"
with pytest.raises(IntegrityError):
key.assert_id(id, plaintext_changed)
def test_authenticated_encrypt(self, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
key = AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
assert AuthenticatedKey.id_hash is ID_HMAC_SHA_256.id_hash
assert len(key.id_key) == 32
plaintext = b'123456789'
plaintext = b"123456789"
id = key.id_hash(plaintext)
authenticated = key.encrypt(id, plaintext)
# 0x07 is the key TYPE, \x00ff identifies no compression / unknown level.
assert authenticated == b'\x07\x00\xff' + plaintext
assert authenticated == b"\x07\x00\xff" + plaintext
def test_blake2_authenticated_encrypt(self, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash
assert len(key.id_key) == 128
plaintext = b'123456789'
plaintext = b"123456789"
id = key.id_hash(plaintext)
authenticated = key.encrypt(id, plaintext)
# 0x06 is the key TYPE, 0x00ff identifies no compression / unknown level.
assert authenticated == b'\x06\x00\xff' + plaintext
assert authenticated == b"\x06\x00\xff" + plaintext
class TestTAM:
@pytest.fixture
def key(self, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
return CHPOKeyfileKey.create(TestKey.MockRepository(), TestKey.MockArgs())
def test_unpack_future(self, key):
blob = b'\xc1\xc1\xc1\xc1foobar'
blob = b"\xc1\xc1\xc1\xc1foobar"
with pytest.raises(UnsupportedManifestError):
key.unpack_and_verify_manifest(blob)
blob = b'\xc1\xc1\xc1'
blob = b"\xc1\xc1\xc1"
with pytest.raises(msgpack.UnpackException):
key.unpack_and_verify_manifest(blob)
@ -285,84 +307,66 @@ def test_missing(self, key):
assert not verified
def test_unknown_type_when_required(self, key):
blob = msgpack.packb({
'tam': {
'type': 'HMAC_VOLLBIT',
},
})
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
with pytest.raises(TAMUnsupportedSuiteError):
key.unpack_and_verify_manifest(blob)
def test_unknown_type(self, key):
blob = msgpack.packb({
'tam': {
'type': 'HMAC_VOLLBIT',
},
})
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
key.tam_required = False
unpacked, verified = key.unpack_and_verify_manifest(blob)
assert unpacked == {}
assert not verified
@pytest.mark.parametrize('tam, exc', (
({}, TAMUnsupportedSuiteError),
({'type': b'\xff'}, TAMUnsupportedSuiteError),
(None, TAMInvalid),
(1234, TAMInvalid),
))
@pytest.mark.parametrize(
"tam, exc",
(
({}, TAMUnsupportedSuiteError),
({"type": b"\xff"}, TAMUnsupportedSuiteError),
(None, TAMInvalid),
(1234, TAMInvalid),
),
)
def test_invalid(self, key, tam, exc):
blob = msgpack.packb({
'tam': tam,
})
blob = msgpack.packb({"tam": tam})
with pytest.raises(exc):
key.unpack_and_verify_manifest(blob)
@pytest.mark.parametrize('hmac, salt', (
({}, bytes(64)),
(bytes(64), {}),
(None, bytes(64)),
(bytes(64), None),
))
@pytest.mark.parametrize("hmac, salt", (({}, bytes(64)), (bytes(64), {}), (None, bytes(64)), (bytes(64), None)))
def test_wrong_types(self, key, hmac, salt):
data = {
'tam': {
'type': 'HKDF_HMAC_SHA512',
'hmac': hmac,
'salt': salt
},
}
tam = data['tam']
data = {"tam": {"type": "HKDF_HMAC_SHA512", "hmac": hmac, "salt": salt}}
tam = data["tam"]
if hmac is None:
del tam['hmac']
del tam["hmac"]
if salt is None:
del tam['salt']
del tam["salt"]
blob = msgpack.packb(data)
with pytest.raises(TAMInvalid):
key.unpack_and_verify_manifest(blob)
def test_round_trip(self, key):
data = {'foo': 'bar'}
data = {"foo": "bar"}
blob = key.pack_and_authenticate_metadata(data)
assert blob.startswith(b'\x82')
assert blob.startswith(b"\x82")
unpacked = msgpack.unpackb(blob)
assert unpacked['tam']['type'] == 'HKDF_HMAC_SHA512'
assert unpacked["tam"]["type"] == "HKDF_HMAC_SHA512"
unpacked, verified = key.unpack_and_verify_manifest(blob)
assert verified
assert unpacked['foo'] == 'bar'
assert 'tam' not in unpacked
assert unpacked["foo"] == "bar"
assert "tam" not in unpacked
@pytest.mark.parametrize('which', ('hmac', 'salt'))
@pytest.mark.parametrize("which", ("hmac", "salt"))
def test_tampered(self, key, which):
data = {'foo': 'bar'}
data = {"foo": "bar"}
blob = key.pack_and_authenticate_metadata(data)
assert blob.startswith(b'\x82')
assert blob.startswith(b"\x82")
unpacked = msgpack.unpackb(blob, object_hook=StableDict)
assert len(unpacked['tam'][which]) == 64
unpacked['tam'][which] = unpacked['tam'][which][0:32] + bytes(32)
assert len(unpacked['tam'][which]) == 64
assert len(unpacked["tam"][which]) == 64
unpacked["tam"][which] = unpacked["tam"][which][0:32] + bytes(32)
assert len(unpacked["tam"][which]) == 64
blob = msgpack.packb(unpacked)
with pytest.raises(TAMInvalid):
@ -372,10 +376,7 @@ def test_tampered(self, key, which):
def test_decrypt_key_file_unsupported_algorithm():
"""We will add more algorithms in the future. We should raise a helpful error."""
key = CHPOKeyfileKey(None)
encrypted = msgpack.packb({
'algorithm': 'THIS ALGORITHM IS NOT SUPPORTED',
'version': 1,
})
encrypted = msgpack.packb({"algorithm": "THIS ALGORITHM IS NOT SUPPORTED", "version": 1})
with pytest.raises(UnsupportedKeyFormatError):
key.decrypt_key_file(encrypted, "hello, pass phrase")
@ -384,9 +385,7 @@ def test_decrypt_key_file_unsupported_algorithm():
def test_decrypt_key_file_v2_is_unsupported():
"""There may eventually be a version 2 of the format. For now we should raise a helpful error."""
key = CHPOKeyfileKey(None)
encrypted = msgpack.packb({
'version': 2,
})
encrypted = msgpack.packb({"version": 2})
with pytest.raises(UnsupportedKeyFormatError):
key.decrypt_key_file(encrypted, "hello, pass phrase")
@ -394,16 +393,16 @@ def test_decrypt_key_file_v2_is_unsupported():
def test_key_file_roundtrip(monkeypatch):
def to_dict(key):
extract = 'repository_id', 'enc_key', 'enc_hmac_key', 'id_key', 'chunk_seed'
extract = "repository_id", "enc_key", "enc_hmac_key", "id_key", "chunk_seed"
return {a: getattr(key, a) for a in extract}
repository = MagicMock(id=b'repository_id')
monkeypatch.setenv('BORG_PASSPHRASE', "hello, pass phrase")
repository = MagicMock(id=b"repository_id")
monkeypatch.setenv("BORG_PASSPHRASE", "hello, pass phrase")
save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2'))
save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2"))
saved = repository.save_key.call_args.args[0]
repository.load_key.return_value = saved
load_me = AESOCBRepoKey.detect(repository, manifest_data=None)
assert to_dict(load_me) == to_dict(save_me)
assert msgpack.unpackb(a2b_base64(saved))['algorithm'] == KEY_ALGORITHMS['argon2']
assert msgpack.unpackb(a2b_base64(saved))["algorithm"] == KEY_ALGORITHMS["argon2"]

88
src/borg/testsuite/locking.py
View File

@ -6,8 +6,19 @@
import pytest
from ..platform import get_process_id, process_alive
from ..locking import TimeoutTimer, ExclusiveLock, Lock, LockRoster, \
ADD, REMOVE, SHARED, EXCLUSIVE, LockTimeout, NotLocked, NotMyLock
from ..locking import (
TimeoutTimer,
ExclusiveLock,
Lock,
LockRoster,
ADD,
REMOVE,
SHARED,
EXCLUSIVE,
LockTimeout,
NotLocked,
NotMyLock,
)
ID1 = "foo", 1, 1
ID2 = "bar", 2, 2
@ -45,7 +56,7 @@ def test_notimeout_sleep(self):
@pytest.fixture()
def lockpath(tmpdir):
return str(tmpdir.join('lock'))
return str(tmpdir.join("lock"))
class TestExclusiveLock:
@ -67,7 +78,7 @@ def test_timeout(self, lockpath):
def test_kill_stale(self, lockpath, free_pid):
host, pid, tid = our_id = get_process_id()
dead_id = host, free_pid, tid
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
dead_lock = ExclusiveLock(lockpath, id=dead_id).acquire()
with ExclusiveLock(lockpath, id=our_id):
@ -94,9 +105,7 @@ def test_migrate_lock(self, lockpath):
assert old_unique_name != new_unique_name # locking filename is different now
def test_race_condition(self, lockpath):
class SynchronizedCounter:
def __init__(self, count=0):
self.lock = ThreadingLock()
self.count = count
@ -126,44 +135,77 @@ def print_locked(msg):
with print_lock:
print(msg)
def acquire_release_loop(id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None):
print_locked("Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s" % (thread_id, id, timeout, lockpath))
def acquire_release_loop(
id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None
):
print_locked(
"Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s"
% (thread_id, id, timeout, lockpath)
)
timer = TimeoutTimer(timeout, -1).start()
cycle = 0
while not timer.timed_out():
cycle += 1
try:
with ExclusiveLock(lockpath, id=id, timeout=timeout/20, sleep=-1): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway.
with ExclusiveLock(
lockpath, id=id, timeout=timeout / 20, sleep=-1
): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway.
lock_owner_count = lock_owner_counter.incr()
print_locked("Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently." % (thread_id, cycle, lock_owner_count))
print_locked(
"Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently."
% (thread_id, cycle, lock_owner_count)
)
time.sleep(0.005)
lock_owner_count = lock_owner_counter.decr()
print_locked("Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock." % (thread_id, cycle, lock_owner_count))
print_locked(
"Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock."
% (thread_id, cycle, lock_owner_count)
)
except LockTimeout:
print_locked("Thread %2d: Got LockTimeout, finishing my %d. loop cycle." % (thread_id, cycle))
except:
exception_count = exception_counter.incr()
e = format_exc()
print_locked("Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s" % (thread_id, cycle, exception_count, e))
print_locked(
"Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s"
% (thread_id, cycle, exception_count, e)
)
print_locked("Thread %2d: Loop timed out--terminating after %d loop cycles." % (thread_id, cycle))
if last_thread is not None: # joining its predecessor, if any
last_thread.join()
print('')
print("")
lock_owner_counter = SynchronizedCounter()
exception_counter = SynchronizedCounter()
print_lock = ThreadingLock()
thread = None
for thread_id in range(RACE_TEST_NUM_THREADS):
thread = Thread(target=acquire_release_loop, args=(('foo', thread_id, 0), RACE_TEST_DURATION, thread_id, lock_owner_counter, exception_counter, print_lock, thread))
thread = Thread(
target=acquire_release_loop,
args=(
("foo", thread_id, 0),
RACE_TEST_DURATION,
thread_id,
lock_owner_counter,
exception_counter,
print_lock,
thread,
),
)
thread.start()
thread.join() # joining the last thread
assert lock_owner_counter.maxvalue() > 0, 'Never gained the lock? Something went wrong here...'
assert lock_owner_counter.maxvalue() <= 1, "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % (lock_owner_counter.maxvalue())
assert exception_counter.value() == 0, "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value())
assert lock_owner_counter.maxvalue() > 0, "Never gained the lock? Something went wrong here..."
assert (
lock_owner_counter.maxvalue() <= 1
), "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % (
lock_owner_counter.maxvalue()
)
assert (
exception_counter.value() == 0
), "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value())
class TestLock:
@ -228,7 +270,7 @@ def test_timeout(self, lockpath):
def test_kill_stale(self, lockpath, free_pid):
host, pid, tid = our_id = get_process_id()
dead_id = host, free_pid, tid
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
dead_lock = Lock(lockpath, id=dead_id, exclusive=True).acquire()
roster = dead_lock._roster
@ -263,7 +305,7 @@ def test_migrate_lock(self, lockpath):
@pytest.fixture()
def rosterpath(tmpdir):
return str(tmpdir.join('roster'))
return str(tmpdir.join("roster"))
class TestLockRoster:
@ -277,13 +319,13 @@ def test_modify_get(self, rosterpath):
roster1 = LockRoster(rosterpath, id=ID1)
assert roster1.get(SHARED) == set()
roster1.modify(SHARED, ADD)
assert roster1.get(SHARED) == {ID1, }
assert roster1.get(SHARED) == {ID1}
roster2 = LockRoster(rosterpath, id=ID2)
roster2.modify(SHARED, ADD)
assert roster2.get(SHARED) == {ID1, ID2, }
assert roster2.get(SHARED) == {ID1, ID2}
roster1 = LockRoster(rosterpath, id=ID1)
roster1.modify(SHARED, REMOVE)
assert roster1.get(SHARED) == {ID2, }
assert roster1.get(SHARED) == {ID2}
roster2 = LockRoster(rosterpath, id=ID2)
roster2.modify(SHARED, REMOVE)
assert roster2.get(SHARED) == set()
@ -300,7 +342,7 @@ def test_kill_stale(self, rosterpath, free_pid):
assert roster1.get(SHARED) == {dead_id}
# put a unknown-state remote process lock into roster
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
roster1 = LockRoster(rosterpath, id=cant_know_if_dead_id)
roster1.kill_stale_locks = False
assert roster1.get(SHARED) == {dead_id}

15
src/borg/testsuite/logger.py
View File

@ -4,6 +4,7 @@
import pytest
from ..logger import find_parent_module, create_logger, setup_logging
logger = create_logger()
@ -11,27 +12,27 @@
def io_logger():
io = StringIO()
handler = setup_logging(stream=io, env_var=None)
handler.setFormatter(logging.Formatter('%(name)s: %(message)s'))
handler.setFormatter(logging.Formatter("%(name)s: %(message)s"))
logger.setLevel(logging.DEBUG)
return io
def test_setup_logging(io_logger):
logger.info('hello world')
logger.info("hello world")
assert io_logger.getvalue() == "borg.testsuite.logger: hello world\n"
def test_multiple_loggers(io_logger):
logger = logging.getLogger(__name__)
logger.info('hello world 1')
logger.info("hello world 1")
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\n"
logger = logging.getLogger('borg.testsuite.logger')
logger.info('hello world 2')
logger = logging.getLogger("borg.testsuite.logger")
logger.info("hello world 2")
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\nborg.testsuite.logger: hello world 2\n"
io_logger.truncate(0)
io_logger.seek(0)
logger = logging.getLogger('borg.testsuite.logger')
logger.info('hello world 2')
logger = logging.getLogger("borg.testsuite.logger")
logger.info("hello world 2")
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 2\n"

33
src/borg/testsuite/lrucache.py
View File

@ -6,33 +6,32 @@
class TestLRUCache:
def test_lrucache(self):
c = LRUCache(2, dispose=lambda _: None)
assert len(c) == 0
assert c.items() == set()
for i, x in enumerate('abc'):
for i, x in enumerate("abc"):
c[x] = i
assert len(c) == 2
assert c.items() == {('b', 1), ('c', 2)}
assert 'a' not in c
assert 'b' in c
assert c.items() == {("b", 1), ("c", 2)}
assert "a" not in c
assert "b" in c
with pytest.raises(KeyError):
c['a']
assert c.get('a') is None
assert c.get('a', 'foo') == 'foo'
assert c['b'] == 1
assert c.get('b') == 1
assert c['c'] == 2
c['d'] = 3
c["a"]
assert c.get("a") is None
assert c.get("a", "foo") == "foo"
assert c["b"] == 1
assert c.get("b") == 1
assert c["c"] == 2
c["d"] = 3
assert len(c) == 2
assert c['c'] == 2
assert c['d'] == 3
del c['c']
assert c["c"] == 2
assert c["d"] == 3
del c["c"]
assert len(c) == 1
with pytest.raises(KeyError):
c['c']
assert c['d'] == 3
c["c"]
assert c["d"] == 3
c.clear()
assert c.items() == set()

20
src/borg/testsuite/nanorst.py
View File

@ -4,15 +4,15 @@
def test_inline():
assert rst_to_text('*foo* and ``bar``.') == 'foo and bar.'
assert rst_to_text("*foo* and ``bar``.") == "foo and bar."
def test_inline_spread():
assert rst_to_text('*foo and bar, thusly\nfoobar*.') == 'foo and bar, thusly\nfoobar.'
assert rst_to_text("*foo and bar, thusly\nfoobar*.") == "foo and bar, thusly\nfoobar."
def test_comment_inline():
assert rst_to_text('Foo and Bar\n.. foo\nbar') == 'Foo and Bar\n.. foo\nbar'
assert rst_to_text("Foo and Bar\n.. foo\nbar") == "Foo and Bar\n.. foo\nbar"
def test_inline_escape():
@ -20,21 +20,19 @@ def test_inline_escape():
def test_comment():
assert rst_to_text('Foo and Bar\n\n.. foo\nbar') == 'Foo and Bar\n\nbar'
assert rst_to_text("Foo and Bar\n\n.. foo\nbar") == "Foo and Bar\n\nbar"
def test_directive_note():
assert rst_to_text('.. note::\n Note this and that') == 'Note:\n Note this and that'
assert rst_to_text(".. note::\n Note this and that") == "Note:\n Note this and that"
def test_ref():
references = {
'foo': 'baz'
}
assert rst_to_text('See :ref:`fo\no`.', references=references) == 'See baz.'
references = {"foo": "baz"}
assert rst_to_text("See :ref:`fo\no`.", references=references) == "See baz."
def test_undefined_ref():
with pytest.raises(ValueError) as exc_info:
rst_to_text('See :ref:`foo`.')
assert 'Undefined reference' in str(exc_info.value)
rst_to_text("See :ref:`foo`.")
assert "Undefined reference" in str(exc_info.value)

29
src/borg/testsuite/nonces.py
View File

@ -10,10 +10,9 @@
class TestNonceManager:
class MockRepository:
class _Location:
orig = '/some/place'
orig = "/some/place"
_location = _Location()
id = bytes(32)
@ -37,15 +36,15 @@ def setUp(self):
self.repository = None
def cache_nonce(self):
with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce')) as fd:
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd:
return fd.read()
def set_cache_nonce(self, nonce):
with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce'), "w") as fd:
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd:
assert fd.write(nonce)
def test_empty_cache_and_old_server(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockOldRepository()
manager = NonceManager(self.repository, 0x2000)
@ -55,7 +54,7 @@ def test_empty_cache_and_old_server(self, monkeypatch):
assert self.cache_nonce() == "0000000000002033"
def test_empty_cache(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = 0x2000
@ -66,7 +65,7 @@ def test_empty_cache(self, monkeypatch):
assert self.cache_nonce() == "0000000000002033"
def test_empty_nonce(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = None
@ -99,10 +98,10 @@ def test_empty_nonce(self, monkeypatch):
next_nonce = manager.ensure_reservation(0x2043, 64)
assert next_nonce == 0x2063
assert self.cache_nonce() == "00000000000020c3"
assert self.repository.next_free == 0x20c3
assert self.repository.next_free == 0x20C3
def test_sync_nonce(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = 0x2000
@ -116,7 +115,7 @@ def test_sync_nonce(self, monkeypatch):
assert self.repository.next_free == 0x2033
def test_server_just_upgraded(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = None
@ -130,7 +129,7 @@ def test_server_just_upgraded(self, monkeypatch):
assert self.repository.next_free == 0x2033
def test_transaction_abort_no_cache(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = 0x2000
@ -143,7 +142,7 @@ def test_transaction_abort_no_cache(self, monkeypatch):
assert self.repository.next_free == 0x2033
def test_transaction_abort_old_server(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockOldRepository()
self.set_cache_nonce("0000000000002000")
@ -155,7 +154,7 @@ def test_transaction_abort_old_server(self, monkeypatch):
assert self.cache_nonce() == "0000000000002033"
def test_transaction_abort_on_other_client(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = 0x2000
@ -169,7 +168,7 @@ def test_transaction_abort_on_other_client(self, monkeypatch):
assert self.repository.next_free == 0x2033
def test_interleaved(self, monkeypatch):
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
self.repository = self.MockRepository()
self.repository.next_free = 0x2000
@ -192,7 +191,7 @@ def test_interleaved(self, monkeypatch):
assert self.repository.next_free == 0x4000
# spans reservation boundary
next_nonce = manager.ensure_reservation(0x201f, 21)
next_nonce = manager.ensure_reservation(0x201F, 21)
assert next_nonce == 0x4000
assert self.cache_nonce() == "0000000000004035"
assert self.repository.next_free == 0x4035

649
src/borg/testsuite/patterns.py
View File

@ -11,8 +11,7 @@
def check_patterns(files, pattern, expected):
"""Utility for testing patterns.
"""
"""Utility for testing patterns."""
assert all([f == os.path.normpath(f) for f in files]), "Pattern matchers expect normalized input paths"
matched = [f for f in files if pattern.match(f)]
@ -20,167 +19,291 @@ def check_patterns(files, pattern, expected):
assert matched == (files if expected is None else expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("/", []),
("/home", ["home"]),
("/home///", ["home"]),
("/./home", ["home"]),
("/home/user", ["home/user"]),
("/home/user2", ["home/user2"]),
("/home/user/.bashrc", ["home/user/.bashrc"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("/", []),
("/home", ["home"]),
("/home///", ["home"]),
("/./home", ["home"]),
("/home/user", ["home/user"]),
("/home/user2", ["home/user2"]),
("/home/user/.bashrc", ["home/user/.bashrc"]),
],
)
def test_patterns_full(pattern, expected):
files = ["home", "home/user", "home/user2", "home/user/.bashrc", ]
files = ["home", "home/user", "home/user2", "home/user/.bashrc"]
check_patterns(files, PathFullPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("", []),
("relative", []),
("relative/path/", ["relative/path"]),
("relative/path", ["relative/path"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("", []),
("relative", []),
("relative/path/", ["relative/path"]),
("relative/path", ["relative/path"]),
],
)
def test_patterns_full_relative(pattern, expected):
files = ["relative/path", "relative/path2", ]
files = ["relative/path", "relative/path2"]
check_patterns(files, PathFullPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("/", None),
("/./", None),
("", []),
("/home/u", []),
("/home/user", ["home/user/.profile", "home/user/.bashrc"]),
("/etc", ["etc/server/config", "etc/server/hosts"]),
("///etc//////", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv", ["srv/messages", "srv/dmesg"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("/", None),
("/./", None),
("", []),
("/home/u", []),
("/home/user", ["home/user/.profile", "home/user/.bashrc"]),
("/etc", ["etc/server/config", "etc/server/hosts"]),
("///etc//////", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv", ["srv/messages", "srv/dmesg"]),
],
)
def test_patterns_prefix(pattern, expected):
files = [
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"etc/server/config",
"etc/server/hosts",
"home",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
]
check_patterns(files, PathPrefixPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("", []),
("foo", []),
("relative", ["relative/path1", "relative/two"]),
("more", ["more/relative"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("", []),
("foo", []),
("relative", ["relative/path1", "relative/two"]),
("more", ["more/relative"]),
],
)
def test_patterns_prefix_relative(pattern, expected):
files = ["relative/path1", "relative/two", "more/relative"]
check_patterns(files, PathPrefixPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("/*", None),
("/./*", None),
("*", None),
("*/*",
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("*///*",
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("/home/u", []),
("/home/*",
["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html",
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
("/etc/*", ["etc/server/config", "etc/server/hosts"]),
("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv*", ["srv/messages", "srv/dmesg"]),
("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("/*", None),
("/./*", None),
("*", None),
(
"*/*",
[
"etc/server/config",
"etc/server/hosts",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
(
"*///*",
[
"etc/server/config",
"etc/server/hosts",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
("/home/u", []),
(
"/home/*",
[
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
("/etc/*", ["etc/server/config", "etc/server/hosts"]),
("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv*", ["srv/messages", "srv/dmesg"]),
("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
],
)
def test_patterns_fnmatch(pattern, expected):
files = [
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"home/foo/.thumbnails", "home/foo/bar/.thumbnails",
"etc/server/config",
"etc/server/hosts",
"home",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
]
check_patterns(files, FnmatchPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("*", None),
("**/*", None),
("/**/*", None),
("/./*", None),
("*/*",
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("*///*",
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
"srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("/home/u", []),
("/home/*",
["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html",
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
("/etc/*/*", ["etc/server/config", "etc/server/hosts"]),
("/etc/**/*", ["etc/server/config", "etc/server/hosts"]),
("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]),
("*/.pr????e", []),
("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]),
("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]),
("/srv/*", ["srv/messages", "srv/dmesg"]),
("/srv2/**", ["srv2", "srv2/blafasel"]),
("/srv2/**/", ["srv2/blafasel"]),
("/home/*/.thumbnails", ["home/foo/.thumbnails"]),
("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("*", None),
("**/*", None),
("/**/*", None),
("/./*", None),
(
"*/*",
[
"etc/server/config",
"etc/server/hosts",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
"srv2/blafasel",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
(
"*///*",
[
"etc/server/config",
"etc/server/hosts",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv/messages",
"srv/dmesg",
"srv2/blafasel",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
("/home/u", []),
(
"/home/*",
[
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
],
),
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
("/etc/*/*", ["etc/server/config", "etc/server/hosts"]),
("/etc/**/*", ["etc/server/config", "etc/server/hosts"]),
("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]),
("*/.pr????e", []),
("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]),
("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]),
("/srv/*", ["srv/messages", "srv/dmesg"]),
("/srv2/**", ["srv2", "srv2/blafasel"]),
("/srv2/**/", ["srv2/blafasel"]),
("/home/*/.thumbnails", ["home/foo/.thumbnails"]),
("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]),
],
)
def test_patterns_shell(pattern, expected):
files = [
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
"home/user2/.profile", "home/user2/public_html/index.html", "srv", "srv/messages", "srv/dmesg",
"srv2", "srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails",
"etc/server/config",
"etc/server/hosts",
"home",
"home/user/.profile",
"home/user/.bashrc",
"home/user2/.profile",
"home/user2/public_html/index.html",
"srv",
"srv/messages",
"srv/dmesg",
"srv2",
"srv2/blafasel",
"home/foo/.thumbnails",
"home/foo/bar/.thumbnails",
]
check_patterns(files, ShellPattern(pattern), expected)
@pytest.mark.parametrize("pattern, expected", [
# "None" means all files, i.e. all match the given pattern
("", None),
(".*", None),
("^/", None),
("^abc$", []),
("^[^/]", []),
("^(?!/srv|/foo|/opt)",
["/home", "/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile",
"/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", ]),
])
@pytest.mark.parametrize(
"pattern, expected",
[
# "None" means all files, i.e. all match the given pattern
("", None),
(".*", None),
("^/", None),
("^abc$", []),
("^[^/]", []),
(
"^(?!/srv|/foo|/opt)",
[
"/home",
"/home/user/.profile",
"/home/user/.bashrc",
"/home/user2/.profile",
"/home/user2/public_html/index.html",
"/home/foo/.thumbnails",
"/home/foo/bar/.thumbnails",
],
),
],
)
def test_patterns_regex(pattern, expected):
files = [
'/srv/data', '/foo/bar', '/home',
'/home/user/.profile', '/home/user/.bashrc',
'/home/user2/.profile', '/home/user2/public_html/index.html',
'/opt/log/messages.txt', '/opt/log/dmesg.txt',
"/home/foo/.thumbnails", "/home/foo/bar/.thumbnails",
"/srv/data",
"/foo/bar",
"/home",
"/home/user/.profile",
"/home/user/.bashrc",
"/home/user2/.profile",
"/home/user2/public_html/index.html",
"/opt/log/messages.txt",
"/opt/log/dmesg.txt",
"/home/foo/.thumbnails",
"/home/foo/bar/.thumbnails",
]
obj = RegexPattern(pattern)
@ -202,11 +325,12 @@ def use_normalized_unicode():
def _make_test_patterns(pattern):
return [PathPrefixPattern(pattern),
FnmatchPattern(pattern),
RegexPattern(f"^{pattern}/foo$"),
ShellPattern(pattern),
]
return [
PathPrefixPattern(pattern),
FnmatchPattern(pattern),
RegexPattern(f"^{pattern}/foo$"),
ShellPattern(pattern),
]
@pytest.mark.parametrize("pattern", _make_test_patterns("b\N{LATIN SMALL LETTER A WITH ACUTE}"))
@ -227,51 +351,63 @@ def test_invalid_unicode_pattern(pattern):
assert pattern.match(str(b"ba\x80/foo", "latin1"))
@pytest.mark.parametrize("lines, expected", [
# "None" means all files, i.e. none excluded
([], None),
(["# Comment only"], None),
(["*"], []),
(["# Comment",
"*/something00.txt",
" *whitespace* ",
# Whitespace before comment
" #/ws*",
# Empty line
"",
"# EOF"],
["more/data", "home", " #/wsfoobar"]),
([r"re:.*"], []),
([r"re:\s"], ["data/something00.txt", "more/data", "home"]),
([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]),
(["", "", "",
"# This is a test with mixed pattern styles",
# Case-insensitive pattern
r"re:(?i)BAR|ME$",
"",
"*whitespace*",
"fm:*/something00*"],
["more/data"]),
([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]),
([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]),
(["pp:./"], None),
# leading slash is removed
(["pp:/"], []),
(["pp:aaabbb"], None),
(["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]),
(["/nomatch", "/more/*"],
['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']),
# the order of exclude patterns shouldn't matter
(["/more/*", "/nomatch"],
['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']),
])
@pytest.mark.parametrize(
"lines, expected",
[
# "None" means all files, i.e. none excluded
([], None),
(["# Comment only"], None),
(["*"], []),
(
[
"# Comment",
"*/something00.txt",
" *whitespace* ",
# Whitespace before comment
" #/ws*",
# Empty line
"",
"# EOF",
],
["more/data", "home", " #/wsfoobar"],
),
([r"re:.*"], []),
([r"re:\s"], ["data/something00.txt", "more/data", "home"]),
([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]),
(
[
"",
"",
"",
"# This is a test with mixed pattern styles",
# Case-insensitive pattern
r"re:(?i)BAR|ME$",
"",
"*whitespace*",
"fm:*/something00*",
],
["more/data"],
),
([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]),
([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]),
(["pp:./"], None),
# leading slash is removed
(["pp:/"], []),
(["pp:aaabbb"], None),
(["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]),
(
["/nomatch", "/more/*"],
["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"],
),
# the order of exclude patterns shouldn't matter
(
["/more/*", "/nomatch"],
["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"],
),
],
)
def test_exclude_patterns_from_file(tmpdir, lines, expected):
files = [
'data/something00.txt', 'more/data', 'home',
' #/wsfoobar',
'\tstart/whitespace',
'whitespace/end\t',
]
files = ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"]
def evaluate(filename):
patterns = []
@ -288,26 +424,26 @@ def evaluate(filename):
assert evaluate(str(exclfile)) == (files if expected is None else expected)
@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [
# "None" means all files, i.e. none excluded
([], [], 0),
(["# Comment only"], [], 0),
(["- *"], [], 1),
(["+fm:*/something00.txt",
"-/data"], [], 2),
(["R /"], ["/"], 0),
(["R /",
"# comment"], ["/"], 0),
(["# comment",
"- /data",
"R /home"], ["/home"], 1),
])
@pytest.mark.parametrize(
"lines, expected_roots, expected_numpatterns",
[
# "None" means all files, i.e. none excluded
([], [], 0),
(["# Comment only"], [], 0),
(["- *"], [], 1),
(["+fm:*/something00.txt", "-/data"], [], 2),
(["R /"], ["/"], 0),
(["R /", "# comment"], ["/"], 0),
(["# comment", "- /data", "R /home"], ["/home"], 1),
],
)
def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns):
def evaluate(filename):
roots = []
inclexclpatterns = []
load_pattern_file(open(filename), roots, inclexclpatterns)
return roots, len(inclexclpatterns)
patternfile = tmpdir.join("patterns.txt")
with patternfile.open("wt") as fh:
@ -344,10 +480,9 @@ def test_switch_patterns_style():
assert isinstance(patterns[5].val, ShellPattern)
@pytest.mark.parametrize("lines", [
(["X /data"]), # illegal pattern type prefix
(["/data"]), # need a pattern type prefix
])
@pytest.mark.parametrize(
"lines", [(["X /data"]), (["/data"])] # illegal pattern type prefix # need a pattern type prefix
)
def test_load_invalid_patterns_from_file(tmpdir, lines):
patternfile = tmpdir.join("patterns.txt")
with patternfile.open("wt") as fh:
@ -359,41 +494,47 @@ def test_load_invalid_patterns_from_file(tmpdir, lines):
load_pattern_file(open(filename), roots, inclexclpatterns)
@pytest.mark.parametrize("lines, expected", [
# "None" means all files, i.e. none excluded
([], None),
(["# Comment only"], None),
(["- *"], []),
# default match type is sh: for patterns -> * doesn't match a /
(["-*/something0?.txt"],
['data', 'data/subdir/something01.txt',
'home', 'home/leo', 'home/leo/t', 'home/other']),
(["-fm:*/something00.txt"],
['data', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t', 'home/other']),
(["-fm:*/something0?.txt"],
["data", 'home', 'home/leo', 'home/leo/t', 'home/other']),
(["+/*/something0?.txt",
"-/data"],
["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']),
(["+fm:*/something00.txt",
"-/data"],
["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']),
# include /home/leo and exclude the rest of /home:
(["+/home/leo",
"-/home/*"],
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']),
# wrong order, /home/leo is already excluded by -/home/*:
(["-/home/*",
"+/home/leo"],
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home']),
(["+fm:/home/leo",
"-/home/"],
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']),
])
@pytest.mark.parametrize(
"lines, expected",
[
# "None" means all files, i.e. none excluded
([], None),
(["# Comment only"], None),
(["- *"], []),
# default match type is sh: for patterns -> * doesn't match a /
(
["-*/something0?.txt"],
["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"],
),
(
["-fm:*/something00.txt"],
["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"],
),
(["-fm:*/something0?.txt"], ["data", "home", "home/leo", "home/leo/t", "home/other"]),
(["+/*/something0?.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]),
(["+fm:*/something00.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]),
# include /home/leo and exclude the rest of /home:
(
["+/home/leo", "-/home/*"],
["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"],
),
# wrong order, /home/leo is already excluded by -/home/*:
(["-/home/*", "+/home/leo"], ["data", "data/something00.txt", "data/subdir/something01.txt", "home"]),
(
["+fm:/home/leo", "-/home/"],
["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"],
),
],
)
def test_inclexcl_patterns_from_file(tmpdir, lines, expected):
files = [
'data', 'data/something00.txt', 'data/subdir/something01.txt',
'home', 'home/leo', 'home/leo/t', 'home/other'
"data",
"data/something00.txt",
"data/subdir/something01.txt",
"home",
"home/leo",
"home/leo/t",
"home/other",
]
def evaluate(filename):
@ -412,37 +553,35 @@ def evaluate(filename):
assert evaluate(str(patternfile)) == (files if expected is None else expected)
@pytest.mark.parametrize("pattern, cls", [
("", FnmatchPattern),
# Default style
("*", FnmatchPattern),
("/data/*", FnmatchPattern),
# fnmatch style
("fm:", FnmatchPattern),
("fm:*", FnmatchPattern),
("fm:/data/*", FnmatchPattern),
("fm:fm:/data/*", FnmatchPattern),
# Regular expression
("re:", RegexPattern),
("re:.*", RegexPattern),
("re:^/something/", RegexPattern),
("re:re:^/something/", RegexPattern),
# Path prefix
("pp:", PathPrefixPattern),
("pp:/", PathPrefixPattern),
("pp:/data/", PathPrefixPattern),
("pp:pp:/data/", PathPrefixPattern),
# Shell-pattern style
("sh:", ShellPattern),
("sh:*", ShellPattern),
("sh:/data/*", ShellPattern),
("sh:sh:/data/*", ShellPattern),
])
@pytest.mark.parametrize(
"pattern, cls",
[
("", FnmatchPattern),
# Default style
("*", FnmatchPattern),
("/data/*", FnmatchPattern),
# fnmatch style
("fm:", FnmatchPattern),
("fm:*", FnmatchPattern),
("fm:/data/*", FnmatchPattern),
("fm:fm:/data/*", FnmatchPattern),
# Regular expression
("re:", RegexPattern),
("re:.*", RegexPattern),
("re:^/something/", RegexPattern),
("re:re:^/something/", RegexPattern),
# Path prefix
("pp:", PathPrefixPattern),
("pp:/", PathPrefixPattern),
("pp:/data/", PathPrefixPattern),
("pp:pp:/data/", PathPrefixPattern),
# Shell-pattern style
("sh:", ShellPattern),
("sh:*", ShellPattern),
("sh:/data/*", ShellPattern),
("sh:sh:/data/*", ShellPattern),
],
)
def test_parse_pattern(pattern, cls):
assert isinstance(parse_pattern(pattern), cls)

134
src/borg/testsuite/platform.py
View File

@ -21,7 +21,9 @@
group:9999:r--:9999
mask::rw-
other::r--
""".strip().encode('ascii')
""".strip().encode(
"ascii"
)
DEFAULT_ACL = """
user::rw-
@ -32,18 +34,21 @@
group:8888:r--:8888
mask::rw-
other::r--
""".strip().encode('ascii')
""".strip().encode(
"ascii"
)
_acls_working = None
def fakeroot_detected():
return 'FAKEROOTKEY' in os.environ
return "FAKEROOTKEY" in os.environ
def user_exists(username):
if not is_win32:
import pwd
try:
pwd.getpwnam(username)
return True
@ -55,25 +60,24 @@ def user_exists(username):
@functools.lru_cache
def are_acls_working():
with unopened_tempfile() as filepath:
open(filepath, 'w').close()
open(filepath, "w").close()
try:
access = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n'
acl = {'acl_access': access}
access = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n"
acl = {"acl_access": access}
acl_set(filepath, acl)
read_acl = {}
acl_get(filepath, read_acl, os.stat(filepath))
read_acl_access = read_acl.get('acl_access', None)
if read_acl_access and b'user::rw-' in read_acl_access:
read_acl_access = read_acl.get("acl_access", None)
if read_acl_access and b"user::rw-" in read_acl_access:
return True
except PermissionError:
pass
return False
@unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test')
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
@unittest.skipUnless(sys.platform.startswith("linux"), "linux only test")
@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot")
class PlatformLinuxTestCase(BaseTestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
@ -86,72 +90,80 @@ def get_acl(self, path, numeric_ids=False):
return item
def set_acl(self, path, access=None, default=None, numeric_ids=False):
item = {'acl_access': access, 'acl_default': default}
item = {"acl_access": access, "acl_default": default}
acl_set(path, item, numeric_ids=numeric_ids)
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
def test_access_acl(self):
file = tempfile.NamedTemporaryFile()
self.assert_equal(self.get_acl(file.name), {})
self.set_acl(file.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=False)
self.assert_in(b'user:root:rw-:0', self.get_acl(file.name)['acl_access'])
self.assert_in(b'group:root:rw-:0', self.get_acl(file.name)['acl_access'])
self.assert_in(b'user:0:rw-:0', self.get_acl(file.name, numeric_ids=True)['acl_access'])
self.set_acl(
file.name,
access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n",
numeric_ids=False,
)
self.assert_in(b"user:root:rw-:0", self.get_acl(file.name)["acl_access"])
self.assert_in(b"group:root:rw-:0", self.get_acl(file.name)["acl_access"])
self.assert_in(b"user:0:rw-:0", self.get_acl(file.name, numeric_ids=True)["acl_access"])
file2 = tempfile.NamedTemporaryFile()
self.set_acl(file2.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=True)
self.assert_in(b'user:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
self.assert_in(b'group:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
self.set_acl(
file2.name,
access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n",
numeric_ids=True,
)
self.assert_in(b"user:9999:rw-:9999", self.get_acl(file2.name)["acl_access"])
self.assert_in(b"group:9999:rw-:9999", self.get_acl(file2.name)["acl_access"])
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
def test_default_acl(self):
self.assert_equal(self.get_acl(self.tmpdir), {})
self.set_acl(self.tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL)
self.assert_equal(self.get_acl(self.tmpdir)['acl_access'], ACCESS_ACL)
self.assert_equal(self.get_acl(self.tmpdir)['acl_default'], DEFAULT_ACL)
self.assert_equal(self.get_acl(self.tmpdir)["acl_access"], ACCESS_ACL)
self.assert_equal(self.get_acl(self.tmpdir)["acl_default"], DEFAULT_ACL)
@unittest.skipIf(not user_exists('übel'), 'requires übel user')
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
@unittest.skipIf(not user_exists("übel"), "requires übel user")
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
def test_non_ascii_acl(self):
# Testing non-ascii ACL processing to see whether our code is robust.
# I have no idea whether non-ascii ACLs are allowed by the standard,
# but in practice they seem to be out there and must not make our code explode.
file = tempfile.NamedTemporaryFile()
self.assert_equal(self.get_acl(file.name), {})
nothing_special = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\n'
nothing_special = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\n"
# TODO: can this be tested without having an existing system user übel with uid 666 gid 666?
user_entry = 'user:übel:rw-:666'.encode()
user_entry_numeric = b'user:666:rw-:666'
group_entry = 'group:übel:rw-:666'.encode()
group_entry_numeric = b'group:666:rw-:666'
acl = b'\n'.join([nothing_special, user_entry, group_entry])
user_entry = "user:übel:rw-:666".encode()
user_entry_numeric = b"user:666:rw-:666"
group_entry = "group:übel:rw-:666".encode()
group_entry_numeric = b"group:666:rw-:666"
acl = b"\n".join([nothing_special, user_entry, group_entry])
self.set_acl(file.name, access=acl, numeric_ids=False)
acl_access = self.get_acl(file.name, numeric_ids=False)['acl_access']
acl_access = self.get_acl(file.name, numeric_ids=False)["acl_access"]
self.assert_in(user_entry, acl_access)
self.assert_in(group_entry, acl_access)
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access']
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"]
self.assert_in(user_entry_numeric, acl_access_numeric)
self.assert_in(group_entry_numeric, acl_access_numeric)
file2 = tempfile.NamedTemporaryFile()
self.set_acl(file2.name, access=acl, numeric_ids=True)
acl_access = self.get_acl(file2.name, numeric_ids=False)['acl_access']
acl_access = self.get_acl(file2.name, numeric_ids=False)["acl_access"]
self.assert_in(user_entry, acl_access)
self.assert_in(group_entry, acl_access)
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access']
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"]
self.assert_in(user_entry_numeric, acl_access_numeric)
self.assert_in(group_entry_numeric, acl_access_numeric)
def test_utils(self):
from ..platform.linux import acl_use_local_uid_gid
self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-')
self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-')
self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-')
self.assert_equal(acl_use_local_uid_gid(b'group:root:rw-:0'), b'group:0:rw-')
self.assert_equal(acl_use_local_uid_gid(b"user:nonexistent1234:rw-:1234"), b"user:1234:rw-")
self.assert_equal(acl_use_local_uid_gid(b"group:nonexistent1234:rw-:1234"), b"group:1234:rw-")
self.assert_equal(acl_use_local_uid_gid(b"user:root:rw-:0"), b"user:0:rw-")
self.assert_equal(acl_use_local_uid_gid(b"group:root:rw-:0"), b"group:0:rw-")
@unittest.skipUnless(sys.platform.startswith('darwin'), 'OS X only test')
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
@unittest.skipUnless(sys.platform.startswith("darwin"), "OS X only test")
@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot")
class PlatformDarwinTestCase(BaseTestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
@ -164,25 +176,41 @@ def get_acl(self, path, numeric_ids=False):
return item
def set_acl(self, path, acl, numeric_ids=False):
item = {'acl_extended': acl}
item = {"acl_extended": acl}
acl_set(path, item, numeric_ids=numeric_ids)
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
def test_access_acl(self):
file = tempfile.NamedTemporaryFile()
file2 = tempfile.NamedTemporaryFile()
self.assert_equal(self.get_acl(file.name), {})
self.set_acl(file.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=False)
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read', self.get_acl(file.name)['acl_extended'])
self.assert_in(b'user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read', self.get_acl(file.name)['acl_extended'])
self.set_acl(file2.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=True)
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read', self.get_acl(file2.name)['acl_extended'])
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read', self.get_acl(file2.name, numeric_ids=True)['acl_extended'])
self.set_acl(
file.name,
b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n",
numeric_ids=False,
)
self.assert_in(
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read", self.get_acl(file.name)["acl_extended"]
)
self.assert_in(
b"user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read", self.get_acl(file.name)["acl_extended"]
)
self.set_acl(
file2.name,
b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n",
numeric_ids=True,
)
self.assert_in(
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read", self.get_acl(file2.name)["acl_extended"]
)
self.assert_in(
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read",
self.get_acl(file2.name, numeric_ids=True)["acl_extended"],
)
@unittest.skipUnless(sys.platform.startswith(('linux', 'freebsd', 'darwin')), 'POSIX only tests')
@unittest.skipUnless(sys.platform.startswith(("linux", "freebsd", "darwin")), "POSIX only tests")
class PlatformPosixTestCase(BaseTestCase):
def test_swidth_ascii(self):
self.assert_equal(swidth("borg"), 4)
@ -197,7 +225,7 @@ def test_process_alive(free_pid):
id = get_process_id()
assert process_alive(*id)
host, pid, tid = id
assert process_alive(host + 'abc', pid, tid)
assert process_alive(host + "abc", pid, tid)
assert process_alive(host, pid, tid + 1)
assert not process_alive(host, free_pid, tid)

44
src/borg/testsuite/remote.py
View File

@ -72,10 +72,10 @@ def test_write(self, monkeypatch):
class TestRepositoryCache:
@pytest.fixture
def repository(self, tmpdir):
self.repository_location = os.path.join(str(tmpdir), 'repository')
self.repository_location = os.path.join(str(tmpdir), "repository")
with Repository(self.repository_location, exclusive=True, create=True) as repository:
repository.put(H(1), b'1234')
repository.put(H(2), b'5678')
repository.put(H(1), b"1234")
repository.put(H(2), b"5678")
repository.put(H(3), bytes(100))
yield repository
@ -85,19 +85,19 @@ def cache(self, repository):
def test_simple(self, cache: RepositoryCache):
# Single get()s are not cached, since they are used for unique objects like archives.
assert cache.get(H(1)) == b'1234'
assert cache.get(H(1)) == b"1234"
assert cache.misses == 1
assert cache.hits == 0
assert list(cache.get_many([H(1)])) == [b'1234']
assert list(cache.get_many([H(1)])) == [b"1234"]
assert cache.misses == 2
assert cache.hits == 0
assert list(cache.get_many([H(1)])) == [b'1234']
assert list(cache.get_many([H(1)])) == [b"1234"]
assert cache.misses == 2
assert cache.hits == 1
assert cache.get(H(1)) == b'1234'
assert cache.get(H(1)) == b"1234"
assert cache.misses == 2
assert cache.hits == 2
@ -105,11 +105,11 @@ def test_backoff(self, cache: RepositoryCache):
def query_size_limit():
cache.size_limit = 0
assert list(cache.get_many([H(1), H(2)])) == [b'1234', b'5678']
assert list(cache.get_many([H(1), H(2)])) == [b"1234", b"5678"]
assert cache.misses == 2
assert cache.evictions == 0
iterator = cache.get_many([H(1), H(3), H(2)])
assert next(iterator) == b'1234'
assert next(iterator) == b"1234"
# Force cache to back off
qsl = cache.query_size_limit
@ -124,7 +124,7 @@ def query_size_limit():
assert cache.slow_misses == 0
# Since H(2) was in the cache when we called get_many(), but has
# been evicted during iterating the generator, it will be a slow miss.
assert next(iterator) == b'5678'
assert next(iterator) == b"5678"
assert cache.slow_misses == 1
def test_enospc(self, cache: RepositoryCache):
@ -139,16 +139,16 @@ def __exit__(self, exc_type, exc_val, exc_tb):
pass
def write(self, data):
raise OSError(errno.ENOSPC, 'foo')
raise OSError(errno.ENOSPC, "foo")
def truncate(self, n=None):
pass
iterator = cache.get_many([H(1), H(2), H(3)])
assert next(iterator) == b'1234'
assert next(iterator) == b"1234"
with patch('builtins.open', enospc_open):
assert next(iterator) == b'5678'
with patch("builtins.open", enospc_open):
assert next(iterator) == b"5678"
assert cache.enospc == 1
# We didn't patch query_size_limit which would set size_limit to some low
# value, so nothing was actually evicted.
@ -158,9 +158,9 @@ def truncate(self, n=None):
@pytest.fixture
def key(self, repository, monkeypatch):
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
monkeypatch.setenv("BORG_PASSPHRASE", "test")
key = PlaintextKey.create(repository, TestKey.MockArgs())
key.compressor = CompressionSpec('none').compressor
key.compressor = CompressionSpec("none").compressor
return key
def _put_encrypted_object(self, key, repository, data):
@ -170,11 +170,11 @@ def _put_encrypted_object(self, key, repository, data):
@pytest.fixture
def H1(self, key, repository):
return self._put_encrypted_object(key, repository, b'1234')
return self._put_encrypted_object(key, repository, b"1234")
@pytest.fixture
def H2(self, key, repository):
return self._put_encrypted_object(key, repository, b'5678')
return self._put_encrypted_object(key, repository, b"5678")
@pytest.fixture
def H3(self, key, repository):
@ -188,14 +188,14 @@ def test_cache_corruption(self, decrypted_cache: RepositoryCache, H1, H2, H3):
list(decrypted_cache.get_many([H1, H2, H3]))
iterator = decrypted_cache.get_many([H1, H2, H3])
assert next(iterator) == (7, b'1234')
assert next(iterator) == (7, b"1234")
with open(decrypted_cache.key_filename(H2), 'a+b') as fd:
with open(decrypted_cache.key_filename(H2), "a+b") as fd:
fd.seek(-1, io.SEEK_END)
corrupted = (int.from_bytes(fd.read(), 'little') ^ 2).to_bytes(1, 'little')
corrupted = (int.from_bytes(fd.read(), "little") ^ 2).to_bytes(1, "little")
fd.seek(-1, io.SEEK_END)
fd.write(corrupted)
fd.truncate()
with pytest.raises(IntegrityError):
assert next(iterator) == (7, b'5678')
assert next(iterator) == (7, b"5678")

453
src/borg/testsuite/repository.py
View File

@ -29,7 +29,7 @@ class RepositoryTestCaseBase(BaseTestCase):
def open(self, create=False, exclusive=UNSPECIFIED):
if exclusive is UNSPECIFIED:
exclusive = self.exclusive
return Repository(os.path.join(self.tmppath, 'repository'), exclusive=exclusive, create=create)
return Repository(os.path.join(self.tmppath, "repository"), exclusive=exclusive, create=create)
def setUp(self):
self.tmppath = tempfile.mkdtemp()
@ -46,19 +46,19 @@ def reopen(self, exclusive=UNSPECIFIED):
self.repository = self.open(exclusive=exclusive)
def add_keys(self):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'bar')
self.repository.put(H(3), b'bar')
self.repository.put(H(0), b"foo")
self.repository.put(H(1), b"bar")
self.repository.put(H(3), b"bar")
self.repository.commit(compact=False)
self.repository.put(H(1), b'bar2')
self.repository.put(H(2), b'boo')
self.repository.put(H(1), b"bar2")
self.repository.put(H(2), b"boo")
self.repository.delete(H(3))
def repo_dump(self, label=None):
label = label + ': ' if label is not None else ''
label = label + ": " if label is not None else ""
H_trans = {H(i): i for i in range(10)}
H_trans[None] = -1 # key == None appears in commits
tag_trans = {TAG_PUT2: 'put2', TAG_PUT: 'put', TAG_DELETE: 'del', TAG_COMMIT: 'comm'}
tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"}
for segment, fn in self.repository.io.segment_iterator():
for tag, key, offset, size in self.repository.io.iter_objects(segment):
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
@ -66,12 +66,11 @@ def repo_dump(self, label=None):
class RepositoryTestCase(RepositoryTestCaseBase):
def test1(self):
for x in range(100):
self.repository.put(H(x), b'SOMEDATA')
self.repository.put(H(x), b"SOMEDATA")
key50 = H(50)
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
self.assert_equal(self.repository.get(key50), b"SOMEDATA")
self.repository.delete(key50)
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
self.repository.commit(compact=False)
@ -81,59 +80,55 @@ def test1(self):
for x in range(100):
if x == 50:
continue
self.assert_equal(repository2.get(H(x)), b'SOMEDATA')
self.assert_equal(repository2.get(H(x)), b"SOMEDATA")
def test2(self):
"""Test multiple sequential transactions
"""
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'foo')
"""Test multiple sequential transactions"""
self.repository.put(H(0), b"foo")
self.repository.put(H(1), b"foo")
self.repository.commit(compact=False)
self.repository.delete(H(0))
self.repository.put(H(1), b'bar')
self.repository.put(H(1), b"bar")
self.repository.commit(compact=False)
self.assert_equal(self.repository.get(H(1)), b'bar')
self.assert_equal(self.repository.get(H(1)), b"bar")
def test_consistency(self):
"""Test cache consistency
"""
self.repository.put(H(0), b'foo')
self.assert_equal(self.repository.get(H(0)), b'foo')
self.repository.put(H(0), b'foo2')
self.assert_equal(self.repository.get(H(0)), b'foo2')
self.repository.put(H(0), b'bar')
self.assert_equal(self.repository.get(H(0)), b'bar')
"""Test cache consistency"""
self.repository.put(H(0), b"foo")
self.assert_equal(self.repository.get(H(0)), b"foo")
self.repository.put(H(0), b"foo2")
self.assert_equal(self.repository.get(H(0)), b"foo2")
self.repository.put(H(0), b"bar")
self.assert_equal(self.repository.get(H(0)), b"bar")
self.repository.delete(H(0))
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0)))
def test_consistency2(self):
"""Test cache consistency2
"""
self.repository.put(H(0), b'foo')
self.assert_equal(self.repository.get(H(0)), b'foo')
"""Test cache consistency2"""
self.repository.put(H(0), b"foo")
self.assert_equal(self.repository.get(H(0)), b"foo")
self.repository.commit(compact=False)
self.repository.put(H(0), b'foo2')
self.assert_equal(self.repository.get(H(0)), b'foo2')
self.repository.put(H(0), b"foo2")
self.assert_equal(self.repository.get(H(0)), b"foo2")
self.repository.rollback()
self.assert_equal(self.repository.get(H(0)), b'foo')
self.assert_equal(self.repository.get(H(0)), b"foo")
def test_overwrite_in_same_transaction(self):
"""Test cache consistency2
"""
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b'foo2')
"""Test cache consistency2"""
self.repository.put(H(0), b"foo")
self.repository.put(H(0), b"foo2")
self.repository.commit(compact=False)
self.assert_equal(self.repository.get(H(0)), b'foo2')
self.assert_equal(self.repository.get(H(0)), b"foo2")
def test_single_kind_transactions(self):
# put
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
self.repository.close()
# replace
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar')
self.repository.put(H(0), b"bar")
self.repository.commit(compact=False)
# delete
self.repository = self.open()
@ -143,7 +138,7 @@ def test_single_kind_transactions(self):
def test_list(self):
for x in range(100):
self.repository.put(H(x), b'SOMEDATA')
self.repository.put(H(x), b"SOMEDATA")
self.repository.commit(compact=False)
all = self.repository.list()
self.assert_equal(len(all), 100)
@ -157,7 +152,7 @@ def test_list(self):
def test_scan(self):
for x in range(100):
self.repository.put(H(x), b'SOMEDATA')
self.repository.put(H(x), b"SOMEDATA")
self.repository.commit(compact=False)
all = self.repository.scan()
assert len(all) == 100
@ -173,11 +168,10 @@ def test_scan(self):
assert all[x] == H(x)
def test_max_data_size(self):
max_data = b'x' * MAX_DATA_SIZE
max_data = b"x" * MAX_DATA_SIZE
self.repository.put(H(0), max_data)
self.assert_equal(self.repository.get(H(0)), max_data)
self.assert_raises(IntegrityError,
lambda: self.repository.put(H(1), max_data + b'x'))
self.assert_raises(IntegrityError, lambda: self.repository.put(H(1), max_data + b"x"))
class LocalRepositoryTestCase(RepositoryTestCaseBase):
@ -194,21 +188,21 @@ def _assert_sparse(self):
assert self.repository.compact[0] == 41 + 8 + 9
def test_sparse1(self):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'123456789')
self.repository.put(H(0), b"foo")
self.repository.put(H(1), b"123456789")
self.repository.commit(compact=False)
self.repository.put(H(1), b'bar')
self.repository.put(H(1), b"bar")
self._assert_sparse()
def test_sparse2(self):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'123456789')
self.repository.put(H(0), b"foo")
self.repository.put(H(1), b"123456789")
self.repository.commit(compact=False)
self.repository.delete(H(1))
self._assert_sparse()
def test_sparse_delete(self):
self.repository.put(H(0), b'1245')
self.repository.put(H(0), b"1245")
self.repository.delete(H(0))
self.repository.io._write_fd.sync()
@ -224,27 +218,26 @@ def test_sparse_delete(self):
def test_uncommitted_garbage(self):
# uncommitted garbage should be no problem, it is cleaned up automatically.
# we just have to be careful with invalidation of cached FDs in LoggedIO.
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
# write some crap to a uncommitted segment file
last_segment = self.repository.io.get_latest_segment()
with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f:
f.write(MAGIC + b'crapcrapcrap')
with open(self.repository.io.segment_filename(last_segment + 1), "wb") as f:
f.write(MAGIC + b"crapcrapcrap")
self.repository.close()
# usually, opening the repo and starting a transaction should trigger a cleanup.
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar') # this may trigger compact_segments()
self.repository.put(H(0), b"bar") # this may trigger compact_segments()
self.repository.commit(compact=True)
# the point here is that nothing blows up with an exception.
class RepositoryCommitTestCase(RepositoryTestCaseBase):
def test_replay_of_missing_index(self):
self.add_keys()
for name in os.listdir(self.repository.path):
if name.startswith('index.'):
if name.startswith("index."):
os.unlink(os.path.join(self.repository.path, name))
self.reopen()
with self.repository:
@ -278,9 +271,9 @@ def test_crash_before_write_index(self):
def test_replay_lock_upgrade_old(self):
self.add_keys()
for name in os.listdir(self.repository.path):
if name.startswith('index.'):
if name.startswith("index."):
os.unlink(os.path.join(self.repository.path, name))
with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade:
with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade:
self.reopen(exclusive=None) # simulate old client that always does lock upgrades
with self.repository:
# the repo is only locked by a shared read lock, but to replay segments,
@ -291,9 +284,9 @@ def test_replay_lock_upgrade_old(self):
def test_replay_lock_upgrade(self):
self.add_keys()
for name in os.listdir(self.repository.path):
if name.startswith('index.'):
if name.startswith("index."):
os.unlink(os.path.join(self.repository.path, name))
with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade:
with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade:
self.reopen(exclusive=False) # current client usually does not do lock upgrade, except for replay
with self.repository:
# the repo is only locked by a shared read lock, but to replay segments,
@ -322,13 +315,13 @@ def test_ignores_commit_tag_in_data(self):
assert not io.is_committed_segment(io.get_latest_segment())
def test_moved_deletes_are_tracked(self):
self.repository.put(H(1), b'1')
self.repository.put(H(2), b'2')
self.repository.put(H(1), b"1")
self.repository.put(H(2), b"2")
self.repository.commit(compact=False)
self.repo_dump('p1 p2 c')
self.repo_dump("p1 p2 c")
self.repository.delete(H(1))
self.repository.commit(compact=True)
self.repo_dump('d1 cc')
self.repo_dump("d1 cc")
last_segment = self.repository.io.get_latest_segment() - 1
num_deletes = 0
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
@ -337,9 +330,9 @@ def test_moved_deletes_are_tracked(self):
num_deletes += 1
assert num_deletes == 1
assert last_segment in self.repository.compact
self.repository.put(H(3), b'3')
self.repository.put(H(3), b"3")
self.repository.commit(compact=True)
self.repo_dump('p3 cc')
self.repo_dump("p3 cc")
assert last_segment not in self.repository.compact
assert not self.repository.io.segment_exists(last_segment)
for segment, _ in self.repository.io.segment_iterator():
@ -352,7 +345,7 @@ def test_moved_deletes_are_tracked(self):
def test_shadowed_entries_are_preserved(self):
get_latest_segment = self.repository.io.get_latest_segment
self.repository.put(H(1), b'1')
self.repository.put(H(1), b"1")
# This is the segment with our original PUT of interest
put_segment = get_latest_segment()
self.repository.commit(compact=False)
@ -360,7 +353,7 @@ def test_shadowed_entries_are_preserved(self):
# We now delete H(1), and force this segment to not be compacted, which can happen
# if it's not sparse enough (symbolized by H(2) here).
self.repository.delete(H(1))
self.repository.put(H(2), b'1')
self.repository.put(H(2), b"1")
delete_segment = get_latest_segment()
# We pretend these are mostly dense (not sparse) and won't be compacted
@ -380,33 +373,33 @@ def test_shadowed_entries_are_preserved(self):
# Basic case, since the index survived this must be ok
assert H(1) not in self.repository
# Nuke index, force replay
os.unlink(os.path.join(self.repository.path, 'index.%d' % get_latest_segment()))
os.unlink(os.path.join(self.repository.path, "index.%d" % get_latest_segment()))
# Must not reappear
assert H(1) not in self.repository
def test_shadow_index_rollback(self):
self.repository.put(H(1), b'1')
self.repository.put(H(1), b"1")
self.repository.delete(H(1))
assert self.repository.shadow_index[H(1)] == [0]
self.repository.commit(compact=True)
self.repo_dump('p1 d1 cc')
self.repo_dump("p1 d1 cc")
# note how an empty list means that nothing is shadowed for sure
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
self.repository.put(H(1), b'1')
self.repository.put(H(1), b"1")
self.repository.delete(H(1))
self.repo_dump('p1 d1')
self.repo_dump("p1 d1")
# 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
assert self.repository.shadow_index[H(1)] == [4]
self.repository.rollback()
self.repo_dump('r')
self.repository.put(H(2), b'1')
self.repo_dump("r")
self.repository.put(H(2), b"1")
# After the rollback segment 4 shouldn't be considered anymore
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
def open(self, create=False):
return Repository(os.path.join(self.tmppath, 'repository'), exclusive=True, create=create, append_only=True)
return Repository(os.path.join(self.tmppath, "repository"), exclusive=True, create=create, append_only=True)
def test_destroy_append_only(self):
# Can't destroy append only repo (via the API)
@ -417,19 +410,20 @@ def test_destroy_append_only(self):
def test_append_only(self):
def segments_in_repository():
return len(list(self.repository.io.segment_iterator()))
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
self.repository.append_only = False
assert segments_in_repository() == 2
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=True)
# normal: compact squashes the data together, only one segment
assert segments_in_repository() == 2
self.repository.append_only = True
assert segments_in_repository() == 2
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
# append only: does not compact, only new segments written
assert segments_in_repository() == 4
@ -438,12 +432,12 @@ def segments_in_repository():
class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
def test_additional_free_space(self):
self.add_keys()
self.repository.config.set('repository', 'additional_free_space', '1000T')
self.repository.save_key(b'shortcut to save_config')
self.repository.config.set("repository", "additional_free_space", "1000T")
self.repository.save_key(b"shortcut to save_config")
self.reopen()
with self.repository:
self.repository.put(H(0), b'foobar')
self.repository.put(H(0), b"foobar")
with pytest.raises(Repository.InsufficientFreeSpaceError):
self.repository.commit(compact=False)
assert os.path.exists(self.repository.path)
@ -469,7 +463,7 @@ def test_tracking(self):
self.reopen()
with self.repository:
# Open new transaction; hints and thus quota data is not loaded unless needed.
self.repository.put(H(3), b'')
self.repository.put(H(3), b"")
self.repository.delete(H(3))
assert self.repository.storage_quota_use == 1234 + 5678 + 3 * (41 + 8) # we have not compacted yet
self.repository.commit(compact=True)
@ -478,11 +472,11 @@ def test_tracking(self):
def test_exceed_quota(self):
assert self.repository.storage_quota_use == 0
self.repository.storage_quota = 80
self.repository.put(H(1), b'')
self.repository.put(H(1), b"")
assert self.repository.storage_quota_use == 41 + 8
self.repository.commit(compact=False)
with pytest.raises(Repository.StorageQuotaExceeded):
self.repository.put(H(2), b'')
self.repository.put(H(2), b"")
assert self.repository.storage_quota_use == (41 + 8) * 2
with pytest.raises(Repository.StorageQuotaExceeded):
self.repository.commit(compact=False)
@ -491,8 +485,10 @@ def test_exceed_quota(self):
with self.repository:
self.repository.storage_quota = 150
# Open new transaction; hints and thus quota data is not loaded unless needed.
self.repository.put(H(1), b'')
assert self.repository.storage_quota_use == (41 + 8) * 2 # we have 2 puts for H(1) here and not yet compacted.
self.repository.put(H(1), b"")
assert (
self.repository.storage_quota_use == (41 + 8) * 2
) # we have 2 puts for H(1) here and not yet compacted.
self.repository.commit(compact=True)
assert self.repository.storage_quota_use == 41 + 8 # now we have compacted.
@ -542,54 +538,54 @@ def test_commit_nonce_reservation(self):
class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
def setUp(self):
super().setUp()
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
self.repository.close()
def do_commit(self):
with self.repository:
self.repository.put(H(0), b'fox')
self.repository.put(H(0), b"fox")
self.repository.commit(compact=False)
def test_corrupted_hints(self):
with open(os.path.join(self.repository.path, 'hints.1'), 'ab') as fd:
fd.write(b'123456789')
with open(os.path.join(self.repository.path, "hints.1"), "ab") as fd:
fd.write(b"123456789")
self.do_commit()
def test_deleted_hints(self):
os.unlink(os.path.join(self.repository.path, 'hints.1'))
os.unlink(os.path.join(self.repository.path, "hints.1"))
self.do_commit()
def test_deleted_index(self):
os.unlink(os.path.join(self.repository.path, 'index.1'))
os.unlink(os.path.join(self.repository.path, "index.1"))
self.do_commit()
def test_unreadable_hints(self):
hints = os.path.join(self.repository.path, 'hints.1')
hints = os.path.join(self.repository.path, "hints.1")
os.unlink(hints)
os.mkdir(hints)
with self.assert_raises(OSError):
self.do_commit()
def test_index(self):
with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd:
fd.write(b'123456789')
with open(os.path.join(self.repository.path, "index.1"), "wb") as fd:
fd.write(b"123456789")
self.do_commit()
def test_index_outside_transaction(self):
with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd:
fd.write(b'123456789')
with open(os.path.join(self.repository.path, "index.1"), "wb") as fd:
fd.write(b"123456789")
with self.repository:
assert len(self.repository) == 1
def _corrupt_index(self):
# HashIndex is able to detect incorrect headers and file lengths,
# but on its own it can't tell if the data is correct.
index_path = os.path.join(self.repository.path, 'index.1')
with open(index_path, 'r+b') as fd:
index_path = os.path.join(self.repository.path, "index.1")
with open(index_path, "r+b") as fd:
index_data = fd.read()
# Flip one bit in a key stored in the index
corrupted_key = (int.from_bytes(H(0), 'little') ^ 1).to_bytes(32, 'little')
corrupted_key = (int.from_bytes(H(0), "little") ^ 1).to_bytes(32, "little")
corrupted_index_data = index_data.replace(H(0), corrupted_key)
assert corrupted_index_data != index_data
assert len(corrupted_index_data) == len(index_data)
@ -604,11 +600,11 @@ def test_index_corrupted(self):
# Data corruption is detected due to mismatching checksums
# and fixed by rebuilding the index.
assert len(self.repository) == 1
assert self.repository.get(H(0)) == b'foo'
assert self.repository.get(H(0)) == b"foo"
def test_index_corrupted_without_integrity(self):
self._corrupt_index()
integrity_path = os.path.join(self.repository.path, 'integrity.1')
integrity_path = os.path.join(self.repository.path, "integrity.1")
os.unlink(integrity_path)
with self.repository:
# Since the corrupted key is not noticed, the repository still thinks
@ -619,7 +615,7 @@ def test_index_corrupted_without_integrity(self):
self.repository.get(H(0))
def test_unreadable_index(self):
index = os.path.join(self.repository.path, 'index.1')
index = os.path.join(self.repository.path, "index.1")
os.unlink(index)
os.mkdir(index)
with self.assert_raises(OSError):
@ -627,36 +623,39 @@ def test_unreadable_index(self):
def test_unknown_integrity_version(self):
# For now an unknown integrity data version is ignored and not an error.
integrity_path = os.path.join(self.repository.path, 'integrity.1')
with open(integrity_path, 'r+b') as fd:
msgpack.pack({
# Borg only understands version 2
b'version': 4.7,
}, fd)
integrity_path = os.path.join(self.repository.path, "integrity.1")
with open(integrity_path, "r+b") as fd:
msgpack.pack(
{
# Borg only understands version 2
b"version": 4.7
},
fd,
)
fd.truncate()
with self.repository:
# No issues accessing the repository
assert len(self.repository) == 1
assert self.repository.get(H(0)) == b'foo'
assert self.repository.get(H(0)) == b"foo"
def _subtly_corrupted_hints_setup(self):
with self.repository:
self.repository.append_only = True
assert len(self.repository) == 1
assert self.repository.get(H(0)) == b'foo'
self.repository.put(H(1), b'bar')
self.repository.put(H(2), b'baz')
assert self.repository.get(H(0)) == b"foo"
self.repository.put(H(1), b"bar")
self.repository.put(H(2), b"baz")
self.repository.commit(compact=False)
self.repository.put(H(2), b'bazz')
self.repository.put(H(2), b"bazz")
self.repository.commit(compact=False)
hints_path = os.path.join(self.repository.path, 'hints.5')
with open(hints_path, 'r+b') as fd:
hints_path = os.path.join(self.repository.path, "hints.5")
with open(hints_path, "r+b") as fd:
hints = msgpack.unpack(fd)
fd.seek(0)
# Corrupt segment refcount
assert hints['segments'][2] == 1
hints['segments'][2] = 0
assert hints["segments"][2] == 1
hints["segments"][2] = 0
msgpack.pack(hints, fd)
fd.truncate()
@ -664,37 +663,40 @@ def test_subtly_corrupted_hints(self):
self._subtly_corrupted_hints_setup()
with self.repository:
self.repository.append_only = False
self.repository.put(H(3), b'1234')
self.repository.put(H(3), b"1234")
# Do a compaction run. Succeeds, since the failed checksum prompted a rebuild of the index+hints.
self.repository.commit(compact=True)
assert len(self.repository) == 4
assert self.repository.get(H(0)) == b'foo'
assert self.repository.get(H(1)) == b'bar'
assert self.repository.get(H(2)) == b'bazz'
assert self.repository.get(H(0)) == b"foo"
assert self.repository.get(H(1)) == b"bar"
assert self.repository.get(H(2)) == b"bazz"
def test_subtly_corrupted_hints_without_integrity(self):
self._subtly_corrupted_hints_setup()
integrity_path = os.path.join(self.repository.path, 'integrity.5')
integrity_path = os.path.join(self.repository.path, "integrity.5")
os.unlink(integrity_path)
with self.repository:
self.repository.append_only = False
self.repository.put(H(3), b'1234')
self.repository.put(H(3), b"1234")
# Do a compaction run. Fails, since the corrupted refcount was not detected and leads to an assertion failure.
with pytest.raises(AssertionError) as exc_info:
self.repository.commit(compact=True)
assert 'Corrupted segment reference count' in str(exc_info.value)
assert "Corrupted segment reference count" in str(exc_info.value)
class RepositoryCheckTestCase(RepositoryTestCaseBase):
def list_indices(self):
return [name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if name.startswith('index.')]
return [name for name in os.listdir(os.path.join(self.tmppath, "repository")) if name.startswith("index.")]
def check(self, repair=False, status=True):
self.assert_equal(self.repository.check(repair=repair), status)
# Make sure no tmp files are left behind
self.assert_equal([name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if 'tmp' in name], [], 'Found tmp files')
self.assert_equal(
[name for name in os.listdir(os.path.join(self.tmppath, "repository")) if "tmp" in name],
[],
"Found tmp files",
)
def get_objects(self, *ids):
for id_ in ids:
@ -703,31 +705,35 @@ def get_objects(self, *ids):
def add_objects(self, segments):
for ids in segments:
for id_ in ids:
self.repository.put(H(id_), b'data')
self.repository.put(H(id_), b"data")
self.repository.commit(compact=False)
def get_head(self):
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, "repository", "data", "0")) if n.isdigit())[
-1
]
def open_index(self):
return NSIndex.read(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'))
return NSIndex.read(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"))
def corrupt_object(self, id_):
idx = self.open_index()
segment, offset, _ = idx[H(id_)]
with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
with open(os.path.join(self.tmppath, "repository", "data", "0", str(segment)), "r+b") as fd:
fd.seek(offset)
fd.write(b'BOOM')
fd.write(b"BOOM")
def delete_segment(self, segment):
os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)))
os.unlink(os.path.join(self.tmppath, "repository", "data", "0", str(segment)))
def delete_index(self):
os.unlink(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'))
os.unlink(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"))
def rename_index(self, new_name):
os.rename(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'),
os.path.join(self.tmppath, 'repository', new_name))
os.rename(
os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"),
os.path.join(self.tmppath, "repository", new_name),
)
def list_objects(self):
return {int(key) for key in self.repository.list()}
@ -765,9 +771,9 @@ def test_repair_missing_commit_segment(self):
def test_repair_corrupted_commit_segment(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '3'), 'r+b') as fd:
with open(os.path.join(self.tmppath, "repository", "data", "0", "3"), "r+b") as fd:
fd.seek(-1, os.SEEK_END)
fd.write(b'X')
fd.write(b"X")
self.assert_raises(Repository.ObjectNotFound, lambda: self.get_objects(4))
self.check(status=True)
self.get_objects(3)
@ -775,15 +781,15 @@ def test_repair_corrupted_commit_segment(self):
def test_repair_no_commits(self):
self.add_objects([[1, 2, 3]])
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd:
with open(os.path.join(self.tmppath, "repository", "data", "0", "1"), "r+b") as fd:
fd.seek(-1, os.SEEK_END)
fd.write(b'X')
fd.write(b"X")
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(status=False)
self.assert_equal(self.list_indices(), ['index.1'])
self.assert_equal(self.list_indices(), ["index.1"])
self.check(repair=True, status=True)
self.assert_equal(self.list_indices(), ['index.2'])
self.assert_equal(self.list_indices(), ["index.2"])
self.check(status=True)
self.get_objects(3)
self.assert_equal({1, 2, 3}, self.list_objects())
@ -797,30 +803,29 @@ def test_repair_missing_index(self):
def test_repair_index_too_new(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.assert_equal(self.list_indices(), ['index.3'])
self.rename_index('index.100')
self.assert_equal(self.list_indices(), ["index.3"])
self.rename_index("index.100")
self.check(status=True)
self.assert_equal(self.list_indices(), ['index.3'])
self.assert_equal(self.list_indices(), ["index.3"])
self.get_objects(4)
self.assert_equal({1, 2, 3, 4, 5, 6}, self.list_objects())
def test_crash_before_compact(self):
self.repository.put(H(0), b'data')
self.repository.put(H(0), b'data2')
self.repository.put(H(0), b"data")
self.repository.put(H(0), b"data2")
# Simulate a crash before compact
with patch.object(Repository, 'compact_segments') as compact:
with patch.object(Repository, "compact_segments") as compact:
self.repository.commit(compact=True)
compact.assert_called_once_with(0.1)
self.reopen()
with self.repository:
self.check(repair=True)
self.assert_equal(self.repository.get(H(0)), b'data2')
self.assert_equal(self.repository.get(H(0)), b"data2")
class RepositoryHintsTestCase(RepositoryTestCaseBase):
def test_hints_persistence(self):
self.repository.put(H(0), b'data')
self.repository.put(H(0), b"data")
self.repository.delete(H(0))
self.repository.commit(compact=False)
shadow_index_expected = self.repository.shadow_index
@ -831,7 +836,7 @@ def test_hints_persistence(self):
self.reopen()
with self.repository:
# see also do_compact()
self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data
self.repository.put(H(42), b"foobar") # this will call prepare_txn() and load the hints data
# check if hints persistence worked:
self.assert_equal(shadow_index_expected, self.repository.shadow_index)
self.assert_equal(compact_expected, self.repository.compact)
@ -839,7 +844,7 @@ def test_hints_persistence(self):
self.assert_equal(segments_expected, self.repository.segments)
def test_hints_behaviour(self):
self.repository.put(H(0), b'data')
self.repository.put(H(0), b"data")
self.assert_equal(self.repository.shadow_index, {})
assert len(self.repository.compact) == 0
self.repository.delete(H(0))
@ -848,7 +853,7 @@ def test_hints_behaviour(self):
self.assert_in(H(0), self.repository.shadow_index)
self.assert_equal(len(self.repository.shadow_index[H(0)]), 1)
self.assert_in(0, self.repository.compact) # segment 0 can be compacted
self.repository.put(H(42), b'foobar') # see also do_compact()
self.repository.put(H(42), b"foobar") # see also do_compact()
self.repository.commit(compact=True, threshold=0.0) # compact completely!
# nothing to compact any more! no info left about stuff that does not exist any more:
self.assert_not_in(H(0), self.repository.shadow_index)
@ -861,12 +866,13 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
repository = None # type: RemoteRepository
def open(self, create=False):
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
exclusive=True, create=create)
return RemoteRepository(
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create
)
def _get_mock_args(self):
class MockArgs:
remote_path = 'borg'
remote_path = "borg"
umask = 0o077
debug_topics = []
rsh = None
@ -878,111 +884,122 @@ def __contains__(self, item):
return MockArgs()
def test_invalid_rpc(self):
self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', {}))
self.assert_raises(InvalidRPCMethod, lambda: self.repository.call("__init__", {}))
def test_rpc_exception_transport(self):
s1 = 'test string'
s1 = "test string"
try:
self.repository.call('inject_exception', {'kind': 'DoesNotExist'})
self.repository.call("inject_exception", {"kind": "DoesNotExist"})
except Repository.DoesNotExist as e:
assert len(e.args) == 1
assert e.args[0] == self.repository.location.processed
try:
self.repository.call('inject_exception', {'kind': 'AlreadyExists'})
self.repository.call("inject_exception", {"kind": "AlreadyExists"})
except Repository.AlreadyExists as e:
assert len(e.args) == 1
assert e.args[0] == self.repository.location.processed
try:
self.repository.call('inject_exception', {'kind': 'CheckNeeded'})
self.repository.call("inject_exception", {"kind": "CheckNeeded"})
except Repository.CheckNeeded as e:
assert len(e.args) == 1
assert e.args[0] == self.repository.location.processed
try:
self.repository.call('inject_exception', {'kind': 'IntegrityError'})
self.repository.call("inject_exception", {"kind": "IntegrityError"})
except IntegrityError as e:
assert len(e.args) == 1
assert e.args[0] == s1
try:
self.repository.call('inject_exception', {'kind': 'PathNotAllowed'})
self.repository.call("inject_exception", {"kind": "PathNotAllowed"})
except PathNotAllowed as e:
assert len(e.args) == 1
assert e.args[0] == 'foo'
assert e.args[0] == "foo"
try:
self.repository.call('inject_exception', {'kind': 'ObjectNotFound'})
self.repository.call("inject_exception", {"kind": "ObjectNotFound"})
except Repository.ObjectNotFound as e:
assert len(e.args) == 2
assert e.args[0] == s1
assert e.args[1] == self.repository.location.processed
try:
self.repository.call('inject_exception', {'kind': 'InvalidRPCMethod'})
self.repository.call("inject_exception", {"kind": "InvalidRPCMethod"})
except InvalidRPCMethod as e:
assert len(e.args) == 1
assert e.args[0] == s1
try:
self.repository.call('inject_exception', {'kind': 'divide'})
self.repository.call("inject_exception", {"kind": "divide"})
except RemoteRepository.RPCError as e:
assert e.unpacked
assert e.get_message() == 'ZeroDivisionError: integer division or modulo by zero\n'
assert e.exception_class == 'ZeroDivisionError'
assert e.get_message() == "ZeroDivisionError: integer division or modulo by zero\n"
assert e.exception_class == "ZeroDivisionError"
assert len(e.exception_full) > 0
def test_ssh_cmd(self):
args = self._get_mock_args()
self.repository._args = args
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com']
assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com']
assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com']
os.environ['BORG_RSH'] = 'ssh --foo'
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '--foo', 'example.com']
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "example.com"]
assert self.repository.ssh_cmd(Location("ssh://user@example.com/foo")) == ["ssh", "user@example.com"]
assert self.repository.ssh_cmd(Location("ssh://user@example.com:1234/foo")) == [
"ssh",
"-p",
"1234",
"user@example.com",
]
os.environ["BORG_RSH"] = "ssh --foo"
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "--foo", "example.com"]
def test_borg_cmd(self):
assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve']
assert self.repository.borg_cmd(None, testing=True) == [sys.executable, "-m", "borg.archiver", "serve"]
args = self._get_mock_args()
# XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
logging.getLogger().setLevel(logging.INFO)
# note: test logger is on info log level, so --info gets added automagically
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info']
args.remote_path = 'borg-0.28.2'
assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info']
args.debug_topics = ['something_client_side', 'repository_compaction']
assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info',
'--debug-topic=borg.debug.repository_compaction']
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
args.remote_path = "borg-0.28.2"
assert self.repository.borg_cmd(args, testing=False) == ["borg-0.28.2", "serve", "--info"]
args.debug_topics = ["something_client_side", "repository_compaction"]
assert self.repository.borg_cmd(args, testing=False) == [
"borg-0.28.2",
"serve",
"--info",
"--debug-topic=borg.debug.repository_compaction",
]
args = self._get_mock_args()
args.storage_quota = 0
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info']
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
args.storage_quota = 314159265
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info',
'--storage-quota=314159265']
args.rsh = 'ssh -i foo'
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info", "--storage-quota=314159265"]
args.rsh = "ssh -i foo"
self.repository._args = args
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '-i', 'foo', 'example.com']
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "-i", "foo", "example.com"]
class RemoteLegacyFree(RepositoryTestCaseBase):
# Keep testing this so we can someday safely remove the legacy tuple format.
def open(self, create=False):
with patch.object(RemoteRepository, 'dictFormat', True):
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
exclusive=True, create=create)
with patch.object(RemoteRepository, "dictFormat", True):
return RemoteRepository(
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")),
exclusive=True,
create=create,
)
def test_legacy_free(self):
# put
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b"foo")
self.repository.commit(compact=False)
self.repository.close()
# replace
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar')
self.repository.put(H(0), b"bar")
self.repository.commit(compact=False)
# delete
self.repository = self.open()
@ -992,10 +1009,10 @@ def test_legacy_free(self):
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
def open(self, create=False):
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
exclusive=True, create=create)
return RemoteRepository(
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create
)
def test_crash_before_compact(self):
# skip this test, we can't mock-patch a Repository class in another process!
@ -1007,8 +1024,8 @@ def setUp(self):
self.stream = io.StringIO()
self.handler = logging.StreamHandler(self.stream)
logging.getLogger().handlers[:] = [self.handler]
logging.getLogger('borg.repository').handlers[:] = []
logging.getLogger('borg.repository.foo').handlers[:] = []
logging.getLogger("borg.repository").handlers[:] = []
logging.getLogger("borg.repository.foo").handlers[:] = []
# capture stderr
sys.stderr.flush()
self.old_stderr = sys.stderr
@ -1019,31 +1036,31 @@ def tearDown(self):
def test_stderr_messages(self):
handle_remote_line("unstructured stderr message\n")
self.assert_equal(self.stream.getvalue(), '')
self.assert_equal(self.stream.getvalue(), "")
# stderr messages don't get an implicit newline
self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message\n')
self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr message\n")
def test_stderr_progress_messages(self):
handle_remote_line("unstructured stderr progress message\r")
self.assert_equal(self.stream.getvalue(), '')
self.assert_equal(self.stream.getvalue(), "")
# stderr messages don't get an implicit newline
self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr progress message\r')
self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr progress message\r")
def test_pre11_format_messages(self):
self.handler.setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
handle_remote_line("$LOG INFO Remote: borg < 1.1 format message\n")
self.assert_equal(self.stream.getvalue(), 'Remote: borg < 1.1 format message\n')
self.assert_equal(self.stderr.getvalue(), '')
self.assert_equal(self.stream.getvalue(), "Remote: borg < 1.1 format message\n")
self.assert_equal(self.stderr.getvalue(), "")
def test_post11_format_messages(self):
self.handler.setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message\n")
self.assert_equal(self.stream.getvalue(), 'Remote: borg >= 1.1 format message\n')
self.assert_equal(self.stderr.getvalue(), '')
self.assert_equal(self.stream.getvalue(), "Remote: borg >= 1.1 format message\n")
self.assert_equal(self.stderr.getvalue(), "")
def test_remote_messages_screened(self):
# default borg config for root logger
@ -1051,12 +1068,12 @@ def test_remote_messages_screened(self):
logging.getLogger().setLevel(logging.WARNING)
handle_remote_line("$LOG INFO borg.repository Remote: new format info message\n")
self.assert_equal(self.stream.getvalue(), '')
self.assert_equal(self.stderr.getvalue(), '')
self.assert_equal(self.stream.getvalue(), "")
self.assert_equal(self.stderr.getvalue(), "")
def test_info_to_correct_local_child(self):
logging.getLogger('borg.repository').setLevel(logging.INFO)
logging.getLogger('borg.repository.foo').setLevel(logging.INFO)
logging.getLogger("borg.repository").setLevel(logging.INFO)
logging.getLogger("borg.repository.foo").setLevel(logging.INFO)
# default borg config for root logger
self.handler.setLevel(logging.WARNING)
logging.getLogger().setLevel(logging.WARNING)
@ -1064,14 +1081,14 @@ def test_info_to_correct_local_child(self):
child_stream = io.StringIO()
child_handler = logging.StreamHandler(child_stream)
child_handler.setLevel(logging.INFO)
logging.getLogger('borg.repository').handlers[:] = [child_handler]
logging.getLogger("borg.repository").handlers[:] = [child_handler]
foo_stream = io.StringIO()
foo_handler = logging.StreamHandler(foo_stream)
foo_handler.setLevel(logging.INFO)
logging.getLogger('borg.repository.foo').handlers[:] = [foo_handler]
logging.getLogger("borg.repository.foo").handlers[:] = [foo_handler]
handle_remote_line("$LOG INFO borg.repository Remote: new format child message\n")
self.assert_equal(foo_stream.getvalue(), '')
self.assert_equal(child_stream.getvalue(), 'Remote: new format child message\n')
self.assert_equal(self.stream.getvalue(), '')
self.assert_equal(self.stderr.getvalue(), '')
self.assert_equal(foo_stream.getvalue(), "")
self.assert_equal(child_stream.getvalue(), "Remote: new format child message\n")
self.assert_equal(self.stream.getvalue(), "")
self.assert_equal(self.stderr.getvalue(), "")

187
src/borg/testsuite/shellpattern.py
View File

@ -11,103 +11,96 @@ def check(path, pattern):
return bool(compiled.match(path))
@pytest.mark.parametrize("path, patterns", [
# Literal string
("foo/bar", ["foo/bar"]),
("foo\\bar", ["foo\\bar"]),
# Non-ASCII
("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]),
("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]),
# Question mark
("foo", ["fo?"]),
("foo", ["f?o"]),
("foo", ["f??"]),
("foo", ["?oo"]),
("foo", ["?o?"]),
("foo", ["??o"]),
("foo", ["???"]),
# Single asterisk
("", ["*"]),
("foo", ["*", "**", "***"]),
("foo", ["foo*"]),
("foobar", ["foo*"]),
("foobar", ["foo*bar"]),
("foobarbaz", ["foo*baz"]),
("bar", ["*bar"]),
("foobar", ["*bar"]),
("foo/bar", ["foo/*bar"]),
("foo/bar", ["foo/*ar"]),
("foo/bar", ["foo/*r"]),
("foo/bar", ["foo/*"]),
("foo/bar", ["foo*/bar"]),
("foo/bar", ["fo*/bar"]),
("foo/bar", ["f*/bar"]),
("foo/bar", ["*/bar"]),
# Double asterisk (matches 0..n directory layers)
("foo/bar", ["foo/**/bar"]),
("foo/1/bar", ["foo/**/bar"]),
("foo/1/22/333/bar", ["foo/**/bar"]),
("foo/", ["foo/**/"]),
("foo/1/", ["foo/**/"]),
("foo/1/22/333/", ["foo/**/"]),
("bar", ["**/bar"]),
("1/bar", ["**/bar"]),
("1/22/333/bar", ["**/bar"]),
("foo/bar/baz", ["foo/**/*"]),
# Set
("foo1", ["foo[12]"]),
("foo2", ["foo[12]"]),
("foo2/bar", ["foo[12]/*"]),
("f??f", ["f??f", "f[?][?]f"]),
("foo]", ["foo[]]"]),
# Inverted set
("foo3", ["foo[!12]"]),
("foo^", ["foo[^!]"]),
("foo!", ["foo[^!]"]),
])
@pytest.mark.parametrize(
"path, patterns",
[
# Literal string
("foo/bar", ["foo/bar"]),
("foo\\bar", ["foo\\bar"]),
# Non-ASCII
("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]),
("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]),
# Question mark
("foo", ["fo?"]),
("foo", ["f?o"]),
("foo", ["f??"]),
("foo", ["?oo"]),
("foo", ["?o?"]),
("foo", ["??o"]),
("foo", ["???"]),
# Single asterisk
("", ["*"]),
("foo", ["*", "**", "***"]),
("foo", ["foo*"]),
("foobar", ["foo*"]),
("foobar", ["foo*bar"]),
("foobarbaz", ["foo*baz"]),
("bar", ["*bar"]),
("foobar", ["*bar"]),
("foo/bar", ["foo/*bar"]),
("foo/bar", ["foo/*ar"]),
("foo/bar", ["foo/*r"]),
("foo/bar", ["foo/*"]),
("foo/bar", ["foo*/bar"]),
("foo/bar", ["fo*/bar"]),
("foo/bar", ["f*/bar"]),
("foo/bar", ["*/bar"]),
# Double asterisk (matches 0..n directory layers)
("foo/bar", ["foo/**/bar"]),
("foo/1/bar", ["foo/**/bar"]),
("foo/1/22/333/bar", ["foo/**/bar"]),
("foo/", ["foo/**/"]),
("foo/1/", ["foo/**/"]),
("foo/1/22/333/", ["foo/**/"]),
("bar", ["**/bar"]),
("1/bar", ["**/bar"]),
("1/22/333/bar", ["**/bar"]),
("foo/bar/baz", ["foo/**/*"]),
# Set
("foo1", ["foo[12]"]),
("foo2", ["foo[12]"]),
("foo2/bar", ["foo[12]/*"]),
("f??f", ["f??f", "f[?][?]f"]),
("foo]", ["foo[]]"]),
# Inverted set
("foo3", ["foo[!12]"]),
("foo^", ["foo[^!]"]),
("foo!", ["foo[^!]"]),
],
)
def test_match(path, patterns):
for p in patterns:
assert check(path, p)
@pytest.mark.parametrize("path, patterns", [
("", ["?", "[]"]),
("foo", ["foo?"]),
("foo", ["?foo"]),
("foo", ["f?oo"]),
# do not match path separator
("foo/ar", ["foo?ar"]),
# do not match/cross over os.path.sep
("foo/bar", ["*"]),
("foo/bar", ["foo*bar"]),
("foo/bar", ["foo*ar"]),
("foo/bar", ["fo*bar"]),
("foo/bar", ["fo*ar"]),
# Double asterisk
("foobar", ["foo/**/bar"]),
# Two asterisks without slash do not match directory separator
("foo/bar", ["**"]),
# Double asterisk not matching filename
("foo/bar", ["**/"]),
# Set
("foo3", ["foo[12]"]),
# Inverted set
("foo1", ["foo[!12]"]),
("foo2", ["foo[!12]"]),
])
@pytest.mark.parametrize(
"path, patterns",
[
("", ["?", "[]"]),
("foo", ["foo?"]),
("foo", ["?foo"]),
("foo", ["f?oo"]),
# do not match path separator
("foo/ar", ["foo?ar"]),
# do not match/cross over os.path.sep
("foo/bar", ["*"]),
("foo/bar", ["foo*bar"]),
("foo/bar", ["foo*ar"]),
("foo/bar", ["fo*bar"]),
("foo/bar", ["fo*ar"]),
# Double asterisk
("foobar", ["foo/**/bar"]),
# Two asterisks without slash do not match directory separator
("foo/bar", ["**"]),
# Double asterisk not matching filename
("foo/bar", ["**/"]),
# Set
("foo3", ["foo[12]"]),
# Inverted set
("foo1", ["foo[!12]"]),
("foo2", ["foo[!12]"]),
],
)
def test_mismatch(path, patterns):
for p in patterns:
assert not check(path, p)
@ -115,10 +108,10 @@ def test_mismatch(path, patterns):
def test_match_end():
regex = shellpattern.translate("*-home") # default is match_end == string end
assert re.match(regex, '2017-07-03-home')
assert not re.match(regex, '2017-07-03-home.checkpoint')
assert re.match(regex, "2017-07-03-home")
assert not re.match(regex, "2017-07-03-home.checkpoint")
match_end = r'(%s)?\Z' % r'\.checkpoint(\.\d+)?' # with/without checkpoint ending
match_end = r"(%s)?\Z" % r"\.checkpoint(\.\d+)?" # with/without checkpoint ending
regex = shellpattern.translate("*-home", match_end=match_end)
assert re.match(regex, '2017-07-03-home')
assert re.match(regex, '2017-07-03-home.checkpoint')
assert re.match(regex, "2017-07-03-home")
assert re.match(regex, "2017-07-03-home.checkpoint")

76
src/borg/testsuite/version.py
View File

@ -3,51 +3,57 @@
from ..version import parse_version, format_version
@pytest.mark.parametrize("version_str, version_tuple", [
# setuptools < 8.0 uses "-"
('1.0.0a1.dev204-g8866961.d20170606', (1, 0, 0, -4, 1)),
('1.0.0a1.dev204-g8866961', (1, 0, 0, -4, 1)),
('1.0.0-d20170606', (1, 0, 0, -1)),
# setuptools >= 8.0 uses "+"
('1.0.0a1.dev204+g8866961.d20170606', (1, 0, 0, -4, 1)),
('1.0.0a1.dev204+g8866961', (1, 0, 0, -4, 1)),
('1.0.0+d20170606', (1, 0, 0, -1)),
# pre-release versions:
('1.0.0a1', (1, 0, 0, -4, 1)),
('1.0.0a2', (1, 0, 0, -4, 2)),
('1.0.0b3', (1, 0, 0, -3, 3)),
('1.0.0rc4', (1, 0, 0, -2, 4)),
# release versions:
('0.0.0', (0, 0, 0, -1)),
('0.0.11', (0, 0, 11, -1)),
('0.11.0', (0, 11, 0, -1)),
('11.0.0', (11, 0, 0, -1)),
])
@pytest.mark.parametrize(
"version_str, version_tuple",
[
# setuptools < 8.0 uses "-"
("1.0.0a1.dev204-g8866961.d20170606", (1, 0, 0, -4, 1)),
("1.0.0a1.dev204-g8866961", (1, 0, 0, -4, 1)),
("1.0.0-d20170606", (1, 0, 0, -1)),
# setuptools >= 8.0 uses "+"
("1.0.0a1.dev204+g8866961.d20170606", (1, 0, 0, -4, 1)),
("1.0.0a1.dev204+g8866961", (1, 0, 0, -4, 1)),
("1.0.0+d20170606", (1, 0, 0, -1)),
# pre-release versions:
("1.0.0a1", (1, 0, 0, -4, 1)),
("1.0.0a2", (1, 0, 0, -4, 2)),
("1.0.0b3", (1, 0, 0, -3, 3)),
("1.0.0rc4", (1, 0, 0, -2, 4)),
# release versions:
("0.0.0", (0, 0, 0, -1)),
("0.0.11", (0, 0, 11, -1)),
("0.11.0", (0, 11, 0, -1)),
("11.0.0", (11, 0, 0, -1)),
],
)
def test_parse_version(version_str, version_tuple):
assert parse_version(version_str) == version_tuple
def test_parse_version_invalid():
with pytest.raises(ValueError):
assert parse_version('') # we require x.y.z versions
assert parse_version("") # we require x.y.z versions
with pytest.raises(ValueError):
assert parse_version('1') # we require x.y.z versions
assert parse_version("1") # we require x.y.z versions
with pytest.raises(ValueError):
assert parse_version('1.2') # we require x.y.z versions
assert parse_version("1.2") # we require x.y.z versions
with pytest.raises(ValueError):
assert parse_version('crap')
assert parse_version("crap")
@pytest.mark.parametrize("version_str, version_tuple", [
('1.0.0a1', (1, 0, 0, -4, 1)),
('1.0.0', (1, 0, 0, -1)),
('1.0.0a2', (1, 0, 0, -4, 2)),
('1.0.0b3', (1, 0, 0, -3, 3)),
('1.0.0rc4', (1, 0, 0, -2, 4)),
('0.0.0', (0, 0, 0, -1)),
('0.0.11', (0, 0, 11, -1)),
('0.11.0', (0, 11, 0, -1)),
('11.0.0', (11, 0, 0, -1)),
])
@pytest.mark.parametrize(
"version_str, version_tuple",
[
("1.0.0a1", (1, 0, 0, -4, 1)),
("1.0.0", (1, 0, 0, -1)),
("1.0.0a2", (1, 0, 0, -4, 2)),
("1.0.0b3", (1, 0, 0, -3, 3)),
("1.0.0rc4", (1, 0, 0, -2, 4)),
("0.0.0", (0, 0, 0, -1)),
("0.0.11", (0, 0, 11, -1)),
("0.11.0", (0, 11, 0, -1)),
("11.0.0", (11, 0, 0, -1)),
],
)
def test_format_version(version_str, version_tuple):
assert format_version(version_tuple) == version_str

52
src/borg/testsuite/xattr.py
View File

@ -10,12 +10,11 @@
from . import BaseTestCase
@unittest.skipUnless(is_enabled(), 'xattr not enabled on filesystem')
@unittest.skipUnless(is_enabled(), "xattr not enabled on filesystem")
class XattrTestCase(BaseTestCase):
def setUp(self):
self.tmpfile = tempfile.NamedTemporaryFile()
self.symlink = self.tmpfile.name + '.symlink'
self.symlink = self.tmpfile.name + ".symlink"
os.symlink(self.tmpfile.name, self.symlink)
def tearDown(self):
@ -23,7 +22,7 @@ def tearDown(self):
def assert_equal_se(self, is_x, want_x):
# check 2 xattr lists for equality, but ignore security.selinux attr
is_x = set(is_x) - {b'security.selinux'}
is_x = set(is_x) - {b"security.selinux"}
want_x = set(want_x)
self.assert_equal(is_x, want_x)
@ -34,32 +33,32 @@ def test(self):
self.assert_equal_se(listxattr(tmp_fn), [])
self.assert_equal_se(listxattr(tmp_fd), [])
self.assert_equal_se(listxattr(tmp_lfn), [])
setxattr(tmp_fn, b'user.foo', b'bar')
setxattr(tmp_fd, b'user.bar', b'foo')
setxattr(tmp_fn, b'user.empty', b'')
setxattr(tmp_fn, b"user.foo", b"bar")
setxattr(tmp_fd, b"user.bar", b"foo")
setxattr(tmp_fn, b"user.empty", b"")
if not is_linux:
# linux does not allow setting user.* xattrs on symlinks
setxattr(tmp_lfn, b'user.linkxattr', b'baz')
self.assert_equal_se(listxattr(tmp_fn), [b'user.foo', b'user.bar', b'user.empty'])
self.assert_equal_se(listxattr(tmp_fd), [b'user.foo', b'user.bar', b'user.empty'])
self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b'user.foo', b'user.bar', b'user.empty'])
setxattr(tmp_lfn, b"user.linkxattr", b"baz")
self.assert_equal_se(listxattr(tmp_fn), [b"user.foo", b"user.bar", b"user.empty"])
self.assert_equal_se(listxattr(tmp_fd), [b"user.foo", b"user.bar", b"user.empty"])
self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b"user.foo", b"user.bar", b"user.empty"])
if not is_linux:
self.assert_equal_se(listxattr(tmp_lfn), [b'user.linkxattr'])
self.assert_equal(getxattr(tmp_fn, b'user.foo'), b'bar')
self.assert_equal(getxattr(tmp_fd, b'user.foo'), b'bar')
self.assert_equal(getxattr(tmp_lfn, b'user.foo', follow_symlinks=True), b'bar')
self.assert_equal_se(listxattr(tmp_lfn), [b"user.linkxattr"])
self.assert_equal(getxattr(tmp_fn, b"user.foo"), b"bar")
self.assert_equal(getxattr(tmp_fd, b"user.foo"), b"bar")
self.assert_equal(getxattr(tmp_lfn, b"user.foo", follow_symlinks=True), b"bar")
if not is_linux:
self.assert_equal(getxattr(tmp_lfn, b'user.linkxattr'), b'baz')
self.assert_equal(getxattr(tmp_fn, b'user.empty'), b'')
self.assert_equal(getxattr(tmp_lfn, b"user.linkxattr"), b"baz")
self.assert_equal(getxattr(tmp_fn, b"user.empty"), b"")
def test_listxattr_buffer_growth(self):
tmp_fn = os.fsencode(self.tmpfile.name)
# make it work even with ext4, which imposes rather low limits
buffer.resize(size=64, init=True)
# xattr raw key list will be > 64
keys = [b'user.attr%d' % i for i in range(20)]
keys = [b"user.attr%d" % i for i in range(20)]
for key in keys:
setxattr(tmp_fn, key, b'x')
setxattr(tmp_fn, key, b"x")
got_keys = listxattr(tmp_fn)
self.assert_equal_se(got_keys, keys)
assert len(buffer) > 64
@ -68,18 +67,15 @@ def test_getxattr_buffer_growth(self):
tmp_fn = os.fsencode(self.tmpfile.name)
# make it work even with ext4, which imposes rather low limits
buffer.resize(size=64, init=True)
value = b'x' * 126
setxattr(tmp_fn, b'user.big', value)
got_value = getxattr(tmp_fn, b'user.big')
value = b"x" * 126
setxattr(tmp_fn, b"user.big", value)
got_value = getxattr(tmp_fn, b"user.big")
self.assert_equal(value, got_value)
self.assert_equal(len(buffer), 128)
@pytest.mark.parametrize('lstring, splitted', (
(b'', []),
(b'\x00', [b'']),
(b'\x01a', [b'a']),
(b'\x01a\x02cd', [b'a', b'cd']),
))
@pytest.mark.parametrize(
"lstring, splitted", ((b"", []), (b"\x00", [b""]), (b"\x01a", [b"a"]), (b"\x01a\x02cd", [b"a", b"cd"]))
)
def test_split_lstring(lstring, splitted):
assert split_lstring(lstring) == splitted

73
src/borg/upgrade.py
View File

@ -25,8 +25,16 @@ def upgrade_compressed_chunk(self, *, chunk):
def upgrade_archive_metadata(self, *, metadata):
new_metadata = {}
# keep all metadata except archive version and stats.
for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment',
'chunker_params', 'recreate_cmdline'):
for attr in (
"cmdline",
"hostname",
"username",
"time",
"time_end",
"comment",
"chunker_params",
"recreate_cmdline",
):
if hasattr(metadata, attr):
new_metadata[attr] = getattr(metadata, attr)
return new_metadata
@ -42,24 +50,45 @@ def new_archive(self, *, archive):
def upgrade_item(self, *, item):
"""upgrade item as needed, get rid of legacy crap"""
ITEM_KEY_WHITELIST = {'path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hlid',
'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size',
'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
'part'}
ITEM_KEY_WHITELIST = {
"path",
"source",
"rdev",
"chunks",
"chunks_healthy",
"hlid",
"mode",
"user",
"group",
"uid",
"gid",
"mtime",
"atime",
"ctime",
"birthtime",
"size",
"xattrs",
"bsdflags",
"acl_nfs4",
"acl_access",
"acl_default",
"acl_extended",
"part",
}
if self.hlm.borg1_hardlink_master(item):
item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['path'])
self.hlm.remember(id=hlid, info=(item._dict.get('chunks'), item._dict.get('chunks_healthy')))
item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["path"])
self.hlm.remember(id=hlid, info=(item._dict.get("chunks"), item._dict.get("chunks_healthy")))
elif self.hlm.borg1_hardlink_slave(item):
item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['source'])
item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["source"])
chunks, chunks_healthy = self.hlm.retrieve(id=hlid, default=(None, None))
if chunks is not None:
item._dict['chunks'] = chunks
item._dict["chunks"] = chunks
for chunk_id, _ in chunks:
self.cache.chunk_incref(chunk_id, self.archive.stats)
if chunks_healthy is not None:
item._dict['chunks_healthy'] = chunks
item._dict.pop('source') # not used for hardlinks any more, replaced by hlid
item._dict["chunks_healthy"] = chunks
item._dict.pop("source") # not used for hardlinks any more, replaced by hlid
# make sure we only have desired stuff in the new item. specifically, make sure to get rid of:
# - 'acl' remnants of bug in attic <= 0.13
# - 'hardlink_master' (superseded by hlid)
@ -80,17 +109,17 @@ def upgrade_zlib_and_level(chunk):
return chunk
ctype = chunk[0:1]
level = b'\xFF' # FF means unknown compression level
level = b"\xFF" # FF means unknown compression level
if ctype == ObfuscateSize.ID:
# in older borg, we used unusual byte order
old_header_fmt = Struct('>I')
old_header_fmt = Struct(">I")
new_header_fmt = ObfuscateSize.header_fmt
length = ObfuscateSize.header_len
size_bytes = chunk[2:2+length]
size_bytes = chunk[2 : 2 + length]
size = old_header_fmt.unpack(size_bytes)
size_bytes = new_header_fmt.pack(size)
compressed = chunk[2+length:]
compressed = chunk[2 + length :]
compressed = upgrade_zlib_and_level(compressed)
chunk = ctype + level + size_bytes + compressed
else:
@ -101,8 +130,16 @@ def upgrade_archive_metadata(self, *, metadata):
new_metadata = {}
# keep all metadata except archive version and stats. also do not keep
# recreate_source_id, recreate_args, recreate_partial_chunks which were used only in 1.1.0b1 .. b2.
for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment',
'chunker_params', 'recreate_cmdline'):
for attr in (
"cmdline",
"hostname",
"username",
"time",
"time_end",
"comment",
"chunker_params",
"recreate_cmdline",
):
if hasattr(metadata, attr):
new_metadata[attr] = getattr(metadata, attr)
return new_metadata

14
src/borg/version.py
View File

@ -21,12 +21,12 @@ def parse_version(version):
"""
m = re.match(version_re, version, re.VERBOSE)
if m is None:
raise ValueError('Invalid version string %s' % version)
raise ValueError("Invalid version string %s" % version)
gd = m.groupdict()
version = [int(gd['major']), int(gd['minor']), int(gd['patch'])]
if m.lastgroup == 'prerelease':
p_type = {'a': -4, 'b': -3, 'rc': -2}[gd['ptype']]
p_num = int(gd['pnum'])
version = [int(gd["major"]), int(gd["minor"]), int(gd["patch"])]
if m.lastgroup == "prerelease":
p_type = {"a": -4, "b": -3, "rc": -2}[gd["ptype"]]
p_num = int(gd["pnum"])
version += [p_type, p_num]
else:
version += [-1]
@ -44,6 +44,6 @@ def format_version(version):
elif part == -1:
break
else:
f[-1] = f[-1] + {-2: 'rc', -3: 'b', -4: 'a'}[part] + str(next(it))
f[-1] = f[-1] + {-2: "rc", -3: "b", -4: "a"}[part] + str(next(it))
break
return '.'.join(f)
return ".".join(f)

32
src/borg/xattr.py
View File

@ -22,14 +22,14 @@
# TODO: Check whether fakeroot supports xattrs on all platforms supported below.
# TODO: If that's the case then we can make Borg fakeroot-xattr-compatible on these as well.
XATTR_FAKEROOT = False
if sys.platform.startswith('linux'):
LD_PRELOAD = os.environ.get('LD_PRELOAD', '')
if sys.platform.startswith("linux"):
LD_PRELOAD = os.environ.get("LD_PRELOAD", "")
preloads = re.split("[ :]", LD_PRELOAD)
for preload in preloads:
if preload.startswith("libfakeroot"):
env = prepare_subprocess_env(system=True)
fakeroot_output = subprocess.check_output(['fakeroot', '-v'], env=env)
fakeroot_version = parse_version(fakeroot_output.decode('ascii').split()[-1])
fakeroot_output = subprocess.check_output(["fakeroot", "-v"], env=env)
fakeroot_version = parse_version(fakeroot_output.decode("ascii").split()[-1])
if fakeroot_version >= parse_version("1.20.2"):
# 1.20.2 has been confirmed to have xattr support
# 1.18.2 has been confirmed not to have xattr support
@ -39,11 +39,10 @@
def is_enabled(path=None):
"""Determine if xattr is enabled on the filesystem
"""
with tempfile.NamedTemporaryFile(dir=path, prefix='borg-tmp') as f:
"""Determine if xattr is enabled on the filesystem"""
with tempfile.NamedTemporaryFile(dir=path, prefix="borg-tmp") as f:
fd = f.fileno()
name, value = b'user.name', b'value'
name, value = b"user.name", b"value"
try:
setxattr(fd, name, value)
except OSError:
@ -80,7 +79,7 @@ def get_all(path, follow_symlinks=False):
except OSError as e:
name_str = name.decode()
if isinstance(path, int):
path_str = '<FD %d>' % path
path_str = "<FD %d>" % path
else:
path_str = os.fsdecode(path)
if e.errno == ENOATTR:
@ -89,8 +88,9 @@ def get_all(path, follow_symlinks=False):
pass
elif e.errno == errno.EPERM:
# we were not permitted to read this attribute, still can continue trying to read others
logger.warning('{}: Operation not permitted when reading extended attribute {}'.format(
path_str, name_str))
logger.warning(
"{}: Operation not permitted when reading extended attribute {}".format(path_str, name_str)
)
else:
raise
except OSError as e:
@ -125,21 +125,21 @@ def set_all(path, xattrs, follow_symlinks=False):
warning = True
k_str = k.decode()
if isinstance(path, int):
path_str = '<FD %d>' % path
path_str = "<FD %d>" % path
else:
path_str = os.fsdecode(path)
if e.errno == errno.E2BIG:
err_str = 'too big for this filesystem'
err_str = "too big for this filesystem"
elif e.errno == errno.ENOTSUP:
err_str = 'xattrs not supported on this filesystem'
err_str = "xattrs not supported on this filesystem"
elif e.errno == errno.ENOSPC:
# ext4 reports ENOSPC when trying to set an xattr with >4kiB while ext4 can only support 4kiB xattrs
# (in this case, this is NOT a "disk full" error, just a ext4 limitation).
err_str = 'no space left on device [xattr len = %d]' % (len(v),)
err_str = "no space left on device [xattr len = %d]" % (len(v),)
else:
# generic handler
# EACCES: permission denied to set this specific xattr (this may happen related to security.* keys)
# EPERM: operation not permitted
err_str = os.strerror(e.errno)
logger.warning('%s: when setting extended attribute %s: %s', path_str, k_str, err_str)
logger.warning("%s: when setting extended attribute %s: %s", path_str, k_str, err_str)
return warning