mirror of
https://github.com/borgbackup/borg.git
synced 2024-10-26 12:41:29 +03:00
parent
7f8f671102
commit
7957af562d
21
conftest.py
21
conftest.py
@ -3,8 +3,8 @@
|
||||
import pytest
|
||||
|
||||
# needed to get pretty assertion failures in unit tests:
|
||||
if hasattr(pytest, 'register_assert_rewrite'):
|
||||
pytest.register_assert_rewrite('borg.testsuite')
|
||||
if hasattr(pytest, "register_assert_rewrite"):
|
||||
pytest.register_assert_rewrite("borg.testsuite")
|
||||
|
||||
|
||||
import borg.cache # noqa: E402
|
||||
@ -21,11 +21,10 @@
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_env(tmpdir_factory, monkeypatch):
|
||||
# avoid that we access / modify the user's normal .config / .cache directory:
|
||||
monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home')))
|
||||
monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home')))
|
||||
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir_factory.mktemp("xdg-config-home")))
|
||||
monkeypatch.setenv("XDG_CACHE_HOME", str(tmpdir_factory.mktemp("xdg-cache-home")))
|
||||
# also avoid to use anything from the outside environment:
|
||||
keys = [key for key in os.environ
|
||||
if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )]
|
||||
keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)]
|
||||
for key in keys:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
# Speed up tests
|
||||
@ -41,7 +40,7 @@ def pytest_report_header(config, startdir):
|
||||
"symlinks": are_symlinks_supported(),
|
||||
"hardlinks": are_hardlinks_supported(),
|
||||
"atime/mtime": is_utime_fully_supported(),
|
||||
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ
|
||||
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ,
|
||||
}
|
||||
enabled = []
|
||||
disabled = []
|
||||
@ -60,9 +59,11 @@ def __init__(self, request):
|
||||
self.org_cache_wipe_cache = borg.cache.LocalCache.wipe_cache
|
||||
|
||||
def wipe_should_not_be_called(*a, **kw):
|
||||
raise AssertionError("Cache wipe was triggered, if this is part of the test add "
|
||||
"@pytest.mark.allow_cache_wipe")
|
||||
if 'allow_cache_wipe' not in request.keywords:
|
||||
raise AssertionError(
|
||||
"Cache wipe was triggered, if this is part of the test add " "@pytest.mark.allow_cache_wipe"
|
||||
)
|
||||
|
||||
if "allow_cache_wipe" not in request.keywords:
|
||||
borg.cache.LocalCache.wipe_cache = wipe_should_not_be_called
|
||||
request.addfinalizer(self.undo)
|
||||
|
||||
|
148
docs/conf.py
148
docs/conf.py
@ -13,84 +13,85 @@
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.abspath('../src'))
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../src"))
|
||||
|
||||
from borg import __version__ as sw_version
|
||||
|
||||
# -- General configuration -----------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#needs_sphinx = '1.0'
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = []
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
templates_path = ["_templates"]
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
source_suffix = ".rst"
|
||||
|
||||
# The encoding of source files.
|
||||
#source_encoding = 'utf-8-sig'
|
||||
# source_encoding = 'utf-8-sig'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = 'Borg - Deduplicating Archiver'
|
||||
copyright = '2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)'
|
||||
project = "Borg - Deduplicating Archiver"
|
||||
copyright = "2010-2014 Jonas Borgström, 2015-2022 The Borg Collective (see AUTHORS file)"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
split_char = '+' if '+' in sw_version else '-'
|
||||
split_char = "+" if "+" in sw_version else "-"
|
||||
version = sw_version.split(split_char)[0]
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = version
|
||||
|
||||
suppress_warnings = ['image.nonlocal_uri']
|
||||
suppress_warnings = ["image.nonlocal_uri"]
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#language = None
|
||||
# language = None
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
#today = ''
|
||||
# today = ''
|
||||
# Else, today_fmt is used as the format for a strftime call.
|
||||
today_fmt = '%Y-%m-%d'
|
||||
today_fmt = "%Y-%m-%d"
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = ['_build']
|
||||
exclude_patterns = ["_build"]
|
||||
|
||||
# The reST default role (used for this markup: `text`) to use for all documents.
|
||||
#default_role = None
|
||||
# default_role = None
|
||||
|
||||
# The Borg docs contain no or very little Python docs.
|
||||
# Thus, the primary domain is rst.
|
||||
primary_domain = 'rst'
|
||||
primary_domain = "rst"
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#add_function_parentheses = True
|
||||
# add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
#add_module_names = True
|
||||
# add_module_names = True
|
||||
|
||||
# If true, sectionauthor and moduleauthor directives will be shown in the
|
||||
# output. They are ignored by default.
|
||||
#show_authors = False
|
||||
# show_authors = False
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
pygments_style = "sphinx"
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
#modindex_common_prefix = []
|
||||
# modindex_common_prefix = []
|
||||
|
||||
|
||||
# -- Options for HTML output ---------------------------------------------------
|
||||
@ -100,79 +101,73 @@
|
||||
import guzzle_sphinx_theme
|
||||
|
||||
html_theme_path = guzzle_sphinx_theme.html_theme_path()
|
||||
html_theme = 'guzzle_sphinx_theme'
|
||||
html_theme = "guzzle_sphinx_theme"
|
||||
|
||||
|
||||
def set_rst_settings(app):
|
||||
app.env.settings.update({
|
||||
'field_name_limit': 0,
|
||||
'option_limit': 0,
|
||||
})
|
||||
app.env.settings.update({"field_name_limit": 0, "option_limit": 0})
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.add_css_file('css/borg.css')
|
||||
app.connect('builder-inited', set_rst_settings)
|
||||
app.add_css_file("css/borg.css")
|
||||
app.connect("builder-inited", set_rst_settings)
|
||||
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
html_theme_options = {
|
||||
'project_nav_name': 'Borg %s' % version,
|
||||
}
|
||||
html_theme_options = {"project_nav_name": "Borg %s" % version}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = ['_themes']
|
||||
# html_theme_path = ['_themes']
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
#html_title = None
|
||||
# html_title = None
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#html_short_title = None
|
||||
# html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
html_logo = '_static/logo.svg'
|
||||
html_logo = "_static/logo.svg"
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
html_favicon = '_static/favicon.ico'
|
||||
html_favicon = "_static/favicon.ico"
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['borg_theme']
|
||||
html_static_path = ["borg_theme"]
|
||||
|
||||
html_extra_path = ['../src/borg/paperkey.html']
|
||||
html_extra_path = ["../src/borg/paperkey.html"]
|
||||
|
||||
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
||||
# using the given strftime format.
|
||||
html_last_updated_fmt = '%Y-%m-%d'
|
||||
html_last_updated_fmt = "%Y-%m-%d"
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
html_use_smartypants = True
|
||||
smartquotes_action = 'qe' # no D in there means "do not transform -- and ---"
|
||||
smartquotes_action = "qe" # no D in there means "do not transform -- and ---"
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
html_sidebars = {
|
||||
'**': ['logo-text.html', 'searchbox.html', 'globaltoc.html'],
|
||||
}
|
||||
html_sidebars = {"**": ["logo-text.html", "searchbox.html", "globaltoc.html"]}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#html_additional_pages = {}
|
||||
# html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_domain_indices = True
|
||||
# html_domain_indices = True
|
||||
|
||||
# If false, no index is generated.
|
||||
html_use_index = False
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
#html_split_index = False
|
||||
# html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
html_show_sourcelink = False
|
||||
@ -186,57 +181,45 @@ def setup(app):
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
#html_use_opensearch = ''
|
||||
# html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
#html_file_suffix = None
|
||||
# html_file_suffix = None
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'borgdoc'
|
||||
htmlhelp_basename = "borgdoc"
|
||||
|
||||
|
||||
# -- Options for LaTeX output --------------------------------------------------
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass [howto/manual]).
|
||||
latex_documents = [
|
||||
('book', 'Borg.tex', 'Borg Documentation',
|
||||
'The Borg Collective', 'manual'),
|
||||
]
|
||||
latex_documents = [("book", "Borg.tex", "Borg Documentation", "The Borg Collective", "manual")]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
latex_logo = '_static/logo.pdf'
|
||||
latex_logo = "_static/logo.pdf"
|
||||
|
||||
latex_elements = {
|
||||
'papersize': 'a4paper',
|
||||
'pointsize': '10pt',
|
||||
'figure_align': 'H',
|
||||
}
|
||||
latex_elements = {"papersize": "a4paper", "pointsize": "10pt", "figure_align": "H"}
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
#latex_use_parts = False
|
||||
# latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
#latex_show_pagerefs = False
|
||||
# latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
latex_show_urls = 'footnote'
|
||||
latex_show_urls = "footnote"
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#latex_preamble = ''
|
||||
# latex_preamble = ''
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
latex_appendices = [
|
||||
'support',
|
||||
'resources',
|
||||
'changes',
|
||||
'authors',
|
||||
]
|
||||
latex_appendices = ["support", "resources", "changes", "authors"]
|
||||
|
||||
# If false, no module index is generated.
|
||||
#latex_domain_indices = True
|
||||
# latex_domain_indices = True
|
||||
|
||||
|
||||
# -- Options for manual page output --------------------------------------------
|
||||
@ -244,21 +227,24 @@ def setup(app):
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('usage', 'borg',
|
||||
'BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.',
|
||||
['The Borg Collective (see AUTHORS file)'],
|
||||
1),
|
||||
(
|
||||
"usage",
|
||||
"borg",
|
||||
"BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.",
|
||||
["The Borg Collective (see AUTHORS file)"],
|
||||
1,
|
||||
)
|
||||
]
|
||||
|
||||
extensions = [
|
||||
'sphinx.ext.extlinks',
|
||||
'sphinx.ext.autodoc',
|
||||
'sphinx.ext.todo',
|
||||
'sphinx.ext.coverage',
|
||||
'sphinx.ext.viewcode',
|
||||
"sphinx.ext.extlinks",
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.todo",
|
||||
"sphinx.ext.coverage",
|
||||
"sphinx.ext.viewcode",
|
||||
]
|
||||
|
||||
extlinks = {
|
||||
'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),
|
||||
'targz_url': ('https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz' % version, None),
|
||||
"issue": ("https://github.com/borgbackup/borg/issues/%s", "#"),
|
||||
"targz_url": ("https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz" % version, None),
|
||||
}
|
||||
|
@ -10,5 +10,5 @@
|
||||
for cls in sorted(classes, key=lambda cls: (cls.__module__, cls.__qualname__)):
|
||||
if cls is ErrorWithTraceback:
|
||||
continue
|
||||
print(' ', cls.__qualname__)
|
||||
print(indent(cls.__doc__, ' ' * 8))
|
||||
print(" ", cls.__qualname__)
|
||||
print(indent(cls.__doc__, " " * 8))
|
||||
|
@ -13,11 +13,11 @@
|
||||
|
||||
verbose = True
|
||||
objdump = "objdump -T %s"
|
||||
glibc_re = re.compile(r'GLIBC_([0-9]\.[0-9]+)')
|
||||
glibc_re = re.compile(r"GLIBC_([0-9]\.[0-9]+)")
|
||||
|
||||
|
||||
def parse_version(v):
|
||||
major, minor = v.split('.')
|
||||
major, minor = v.split(".")
|
||||
return int(major), int(minor)
|
||||
|
||||
|
||||
@ -32,11 +32,9 @@ def main():
|
||||
overall_versions = set()
|
||||
for filename in filenames:
|
||||
try:
|
||||
output = subprocess.check_output(objdump % filename, shell=True,
|
||||
stderr=subprocess.STDOUT)
|
||||
output = subprocess.check_output(objdump % filename, shell=True, stderr=subprocess.STDOUT)
|
||||
output = output.decode()
|
||||
versions = {parse_version(match.group(1))
|
||||
for match in glibc_re.finditer(output)}
|
||||
versions = {parse_version(match.group(1)) for match in glibc_re.finditer(output)}
|
||||
requires_glibc = max(versions)
|
||||
overall_versions.add(requires_glibc)
|
||||
if verbose:
|
||||
@ -50,14 +48,15 @@ def main():
|
||||
|
||||
if verbose:
|
||||
if ok:
|
||||
print("The binaries work with the given glibc %s." %
|
||||
format_version(given))
|
||||
print("The binaries work with the given glibc %s." % format_version(given))
|
||||
else:
|
||||
print("The binaries do not work with the given glibc %s. "
|
||||
"Minimum is: %s" % (format_version(given), format_version(wanted)))
|
||||
print(
|
||||
"The binaries do not work with the given glibc %s. "
|
||||
"Minimum is: %s" % (format_version(given), format_version(wanted))
|
||||
)
|
||||
return ok
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
ok = main()
|
||||
sys.exit(0 if ok else 1)
|
||||
|
@ -23,11 +23,11 @@
|
||||
# which growth factor to use when growing a hashtable of size < upto
|
||||
# grow fast (*2.0) at the start so we do not have to resize too often (expensive).
|
||||
# grow slow (*1.1) for huge hash tables (do not jump too much in memory usage)
|
||||
Policy(256*K, 2.0),
|
||||
Policy(2*M, 1.7),
|
||||
Policy(16*M, 1.4),
|
||||
Policy(128*M, 1.2),
|
||||
Policy(2*G-1, 1.1),
|
||||
Policy(256 * K, 2.0),
|
||||
Policy(2 * M, 1.7),
|
||||
Policy(16 * M, 1.4),
|
||||
Policy(128 * M, 1.2),
|
||||
Policy(2 * G - 1, 1.1),
|
||||
]
|
||||
|
||||
|
||||
@ -92,12 +92,15 @@ def main():
|
||||
sizes.append(p)
|
||||
i = int(i * grow_factor)
|
||||
|
||||
print("""\
|
||||
print(
|
||||
"""\
|
||||
static int hash_sizes[] = {
|
||||
%s
|
||||
};
|
||||
""" % ', '.join(str(size) for size in sizes))
|
||||
"""
|
||||
% ", ".join(str(size) for size in sizes)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
142
setup.py
142
setup.py
@ -22,11 +22,11 @@
|
||||
sys.path += [os.path.dirname(__file__)]
|
||||
import setup_docs
|
||||
|
||||
is_win32 = sys.platform.startswith('win32')
|
||||
is_openbsd = sys.platform.startswith('openbsd')
|
||||
is_win32 = sys.platform.startswith("win32")
|
||||
is_openbsd = sys.platform.startswith("openbsd")
|
||||
|
||||
# Number of threads to use for cythonize, not used on windows
|
||||
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != 'spawn' else None
|
||||
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None
|
||||
|
||||
# How the build process finds the system libs:
|
||||
#
|
||||
@ -38,27 +38,23 @@
|
||||
# 3. otherwise raise a fatal error.
|
||||
|
||||
# Are we building on ReadTheDocs?
|
||||
on_rtd = os.environ.get('READTHEDOCS')
|
||||
on_rtd = os.environ.get("READTHEDOCS")
|
||||
|
||||
# Extra cflags for all extensions, usually just warnings we want to explicitly enable
|
||||
cflags = [
|
||||
'-Wall',
|
||||
'-Wextra',
|
||||
'-Wpointer-arith',
|
||||
]
|
||||
cflags = ["-Wall", "-Wextra", "-Wpointer-arith"]
|
||||
|
||||
compress_source = 'src/borg/compress.pyx'
|
||||
crypto_ll_source = 'src/borg/crypto/low_level.pyx'
|
||||
chunker_source = 'src/borg/chunker.pyx'
|
||||
hashindex_source = 'src/borg/hashindex.pyx'
|
||||
item_source = 'src/borg/item.pyx'
|
||||
checksums_source = 'src/borg/checksums.pyx'
|
||||
platform_posix_source = 'src/borg/platform/posix.pyx'
|
||||
platform_linux_source = 'src/borg/platform/linux.pyx'
|
||||
platform_syncfilerange_source = 'src/borg/platform/syncfilerange.pyx'
|
||||
platform_darwin_source = 'src/borg/platform/darwin.pyx'
|
||||
platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
|
||||
platform_windows_source = 'src/borg/platform/windows.pyx'
|
||||
compress_source = "src/borg/compress.pyx"
|
||||
crypto_ll_source = "src/borg/crypto/low_level.pyx"
|
||||
chunker_source = "src/borg/chunker.pyx"
|
||||
hashindex_source = "src/borg/hashindex.pyx"
|
||||
item_source = "src/borg/item.pyx"
|
||||
checksums_source = "src/borg/checksums.pyx"
|
||||
platform_posix_source = "src/borg/platform/posix.pyx"
|
||||
platform_linux_source = "src/borg/platform/linux.pyx"
|
||||
platform_syncfilerange_source = "src/borg/platform/syncfilerange.pyx"
|
||||
platform_darwin_source = "src/borg/platform/darwin.pyx"
|
||||
platform_freebsd_source = "src/borg/platform/freebsd.pyx"
|
||||
platform_windows_source = "src/borg/platform/windows.pyx"
|
||||
|
||||
cython_sources = [
|
||||
compress_source,
|
||||
@ -67,7 +63,6 @@
|
||||
hashindex_source,
|
||||
item_source,
|
||||
checksums_source,
|
||||
|
||||
platform_posix_source,
|
||||
platform_linux_source,
|
||||
platform_syncfilerange_source,
|
||||
@ -79,19 +74,20 @@
|
||||
if cythonize:
|
||||
Sdist = sdist
|
||||
else:
|
||||
|
||||
class Sdist(sdist):
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise Exception('Cython is required to run sdist')
|
||||
raise Exception("Cython is required to run sdist")
|
||||
|
||||
cython_c_files = [fn.replace('.pyx', '.c') for fn in cython_sources]
|
||||
cython_c_files = [fn.replace(".pyx", ".c") for fn in cython_sources]
|
||||
if not on_rtd and not all(os.path.exists(path) for path in cython_c_files):
|
||||
raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
|
||||
raise ImportError("The GIT version of Borg needs Cython. Install Cython or use a released version.")
|
||||
|
||||
|
||||
def rm(file):
|
||||
try:
|
||||
os.unlink(file)
|
||||
print('rm', file)
|
||||
print("rm", file)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
@ -107,19 +103,19 @@ def finalize_options(self):
|
||||
|
||||
def run(self):
|
||||
for source in cython_sources:
|
||||
genc = source.replace('.pyx', '.c')
|
||||
genc = source.replace(".pyx", ".c")
|
||||
rm(genc)
|
||||
compiled_glob = source.replace('.pyx', '.cpython*')
|
||||
compiled_glob = source.replace(".pyx", ".cpython*")
|
||||
for compiled in sorted(glob(compiled_glob)):
|
||||
rm(compiled)
|
||||
|
||||
|
||||
cmdclass = {
|
||||
'build_ext': build_ext,
|
||||
'build_usage': setup_docs.build_usage,
|
||||
'build_man': setup_docs.build_man,
|
||||
'sdist': Sdist,
|
||||
'clean2': Clean,
|
||||
"build_ext": build_ext,
|
||||
"build_usage": setup_docs.build_usage,
|
||||
"build_man": setup_docs.build_man,
|
||||
"sdist": Sdist,
|
||||
"clean2": Clean,
|
||||
}
|
||||
|
||||
|
||||
@ -137,16 +133,18 @@ def members_appended(*ds):
|
||||
try:
|
||||
import pkgconfig as pc
|
||||
except ImportError:
|
||||
print('Warning: can not import pkgconfig python package.')
|
||||
print("Warning: can not import pkgconfig python package.")
|
||||
pc = None
|
||||
|
||||
def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir='lib'):
|
||||
def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_subdir="lib"):
|
||||
system_prefix = os.environ.get(prefix_env_var)
|
||||
if system_prefix:
|
||||
print(f"Detected and preferring {lib_pkg_name} [via {prefix_env_var}]")
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, lib_subdir)],
|
||||
libraries=[lib_name])
|
||||
return dict(
|
||||
include_dirs=[os.path.join(system_prefix, "include")],
|
||||
library_dirs=[os.path.join(system_prefix, lib_subdir)],
|
||||
libraries=[lib_name],
|
||||
)
|
||||
|
||||
if pc and pc.installed(lib_pkg_name, pc_version):
|
||||
print(f"Detected and preferring {lib_pkg_name} [via pkg-config]")
|
||||
@ -158,16 +156,13 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
|
||||
|
||||
crypto_ldflags = []
|
||||
if is_win32:
|
||||
crypto_ext_lib = lib_ext_kwargs(
|
||||
pc, 'BORG_OPENSSL_PREFIX', 'libcrypto', 'libcrypto', '>=1.1.1', lib_subdir='')
|
||||
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="")
|
||||
elif is_openbsd:
|
||||
# use openssl (not libressl) because we need AES-OCB and CHACHA20-POLY1305 via EVP api
|
||||
crypto_ext_lib = lib_ext_kwargs(
|
||||
pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libecrypto11', '>=1.1.1')
|
||||
crypto_ldflags += ['-Wl,-rpath=/usr/local/lib/eopenssl11']
|
||||
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libecrypto11", ">=1.1.1")
|
||||
crypto_ldflags += ["-Wl,-rpath=/usr/local/lib/eopenssl11"]
|
||||
else:
|
||||
crypto_ext_lib = lib_ext_kwargs(
|
||||
pc, 'BORG_OPENSSL_PREFIX', 'crypto', 'libcrypto', '>=1.1.1')
|
||||
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1")
|
||||
|
||||
crypto_ext_kwargs = members_appended(
|
||||
dict(sources=[crypto_ll_source]),
|
||||
@ -178,57 +173,60 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
|
||||
|
||||
compress_ext_kwargs = members_appended(
|
||||
dict(sources=[compress_source]),
|
||||
lib_ext_kwargs(pc, 'BORG_LIBLZ4_PREFIX', 'lz4', 'liblz4', '>= 1.7.0'),
|
||||
lib_ext_kwargs(pc, 'BORG_LIBZSTD_PREFIX', 'zstd', 'libzstd', '>= 1.3.0'),
|
||||
lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
|
||||
lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"),
|
||||
dict(extra_compile_args=cflags),
|
||||
)
|
||||
|
||||
checksums_ext_kwargs = members_appended(
|
||||
dict(sources=[checksums_source]),
|
||||
lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'),
|
||||
lib_ext_kwargs(pc, "BORG_LIBXXHASH_PREFIX", "xxhash", "libxxhash", ">= 0.7.3"),
|
||||
dict(extra_compile_args=cflags),
|
||||
)
|
||||
|
||||
ext_modules += [
|
||||
Extension('borg.crypto.low_level', **crypto_ext_kwargs),
|
||||
Extension('borg.compress', **compress_ext_kwargs),
|
||||
Extension('borg.hashindex', [hashindex_source], extra_compile_args=cflags),
|
||||
Extension('borg.item', [item_source], extra_compile_args=cflags),
|
||||
Extension('borg.chunker', [chunker_source], extra_compile_args=cflags),
|
||||
Extension('borg.checksums', **checksums_ext_kwargs),
|
||||
Extension("borg.crypto.low_level", **crypto_ext_kwargs),
|
||||
Extension("borg.compress", **compress_ext_kwargs),
|
||||
Extension("borg.hashindex", [hashindex_source], extra_compile_args=cflags),
|
||||
Extension("borg.item", [item_source], extra_compile_args=cflags),
|
||||
Extension("borg.chunker", [chunker_source], extra_compile_args=cflags),
|
||||
Extension("borg.checksums", **checksums_ext_kwargs),
|
||||
]
|
||||
|
||||
posix_ext = Extension('borg.platform.posix', [platform_posix_source], extra_compile_args=cflags)
|
||||
linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'], extra_compile_args=cflags)
|
||||
syncfilerange_ext = Extension('borg.platform.syncfilerange', [platform_syncfilerange_source], extra_compile_args=cflags)
|
||||
freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source], extra_compile_args=cflags)
|
||||
darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source], extra_compile_args=cflags)
|
||||
windows_ext = Extension('borg.platform.windows', [platform_windows_source], extra_compile_args=cflags)
|
||||
posix_ext = Extension("borg.platform.posix", [platform_posix_source], extra_compile_args=cflags)
|
||||
linux_ext = Extension("borg.platform.linux", [platform_linux_source], libraries=["acl"], extra_compile_args=cflags)
|
||||
syncfilerange_ext = Extension(
|
||||
"borg.platform.syncfilerange", [platform_syncfilerange_source], extra_compile_args=cflags
|
||||
)
|
||||
freebsd_ext = Extension("borg.platform.freebsd", [platform_freebsd_source], extra_compile_args=cflags)
|
||||
darwin_ext = Extension("borg.platform.darwin", [platform_darwin_source], extra_compile_args=cflags)
|
||||
windows_ext = Extension("borg.platform.windows", [platform_windows_source], extra_compile_args=cflags)
|
||||
|
||||
if not is_win32:
|
||||
ext_modules.append(posix_ext)
|
||||
else:
|
||||
ext_modules.append(windows_ext)
|
||||
if sys.platform == 'linux':
|
||||
if sys.platform == "linux":
|
||||
ext_modules.append(linux_ext)
|
||||
ext_modules.append(syncfilerange_ext)
|
||||
elif sys.platform.startswith('freebsd'):
|
||||
elif sys.platform.startswith("freebsd"):
|
||||
ext_modules.append(freebsd_ext)
|
||||
elif sys.platform == 'darwin':
|
||||
elif sys.platform == "darwin":
|
||||
ext_modules.append(darwin_ext)
|
||||
|
||||
# sometimes there's no need to cythonize
|
||||
# this breaks chained commands like 'clean sdist'
|
||||
cythonizing = len(sys.argv) > 1 and sys.argv[1] not in (
|
||||
('clean', 'clean2', 'egg_info', '--help-commands', '--version')) and '--help' not in sys.argv[1:]
|
||||
cythonizing = (
|
||||
len(sys.argv) > 1
|
||||
and sys.argv[1] not in (("clean", "clean2", "egg_info", "--help-commands", "--version"))
|
||||
and "--help" not in sys.argv[1:]
|
||||
)
|
||||
|
||||
if cythonize and cythonizing:
|
||||
cython_opts = dict(
|
||||
compiler_directives={'language_level': '3str'},
|
||||
)
|
||||
cython_opts = dict(compiler_directives={"language_level": "3str"})
|
||||
if not is_win32:
|
||||
# compile .pyx extensions to .c in parallel, does not work on windows
|
||||
cython_opts['nthreads'] = cpu_threads
|
||||
cython_opts["nthreads"] = cpu_threads
|
||||
|
||||
# generate C code from Cython for ALL supported platforms, so we have them in the sdist.
|
||||
# the sdist does not require Cython at install time, so we need all as C.
|
||||
@ -237,8 +235,4 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
|
||||
ext_modules = cythonize(ext_modules, **cython_opts)
|
||||
|
||||
|
||||
setup(
|
||||
cmdclass=cmdclass,
|
||||
ext_modules=ext_modules,
|
||||
long_description=setup_docs.long_desc_from_readme()
|
||||
)
|
||||
setup(cmdclass=cmdclass, ext_modules=ext_modules, long_description=setup_docs.long_desc_from_readme())
|
||||
|
380
setup_docs.py
380
setup_docs.py
@ -12,36 +12,34 @@
|
||||
|
||||
|
||||
def long_desc_from_readme():
|
||||
with open('README.rst') as fd:
|
||||
with open("README.rst") as fd:
|
||||
long_description = fd.read()
|
||||
# remove header, but have one \n before first headline
|
||||
start = long_description.find('What is BorgBackup?')
|
||||
start = long_description.find("What is BorgBackup?")
|
||||
assert start >= 0
|
||||
long_description = '\n' + long_description[start:]
|
||||
long_description = "\n" + long_description[start:]
|
||||
# remove badges
|
||||
long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
|
||||
long_description = re.compile(r"^\.\. start-badges.*^\.\. end-badges", re.M | re.S).sub("", long_description)
|
||||
# remove unknown directives
|
||||
long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
|
||||
long_description = re.compile(r"^\.\. highlight:: \w+$", re.M).sub("", long_description)
|
||||
return long_description
|
||||
|
||||
|
||||
def format_metavar(option):
|
||||
if option.nargs in ('*', '...'):
|
||||
return '[%s...]' % option.metavar
|
||||
elif option.nargs == '?':
|
||||
return '[%s]' % option.metavar
|
||||
if option.nargs in ("*", "..."):
|
||||
return "[%s...]" % option.metavar
|
||||
elif option.nargs == "?":
|
||||
return "[%s]" % option.metavar
|
||||
elif option.nargs is None:
|
||||
return option.metavar
|
||||
else:
|
||||
raise ValueError(f'Can\'t format metavar {option.metavar}, unknown nargs {option.nargs}!')
|
||||
raise ValueError(f"Can't format metavar {option.metavar}, unknown nargs {option.nargs}!")
|
||||
|
||||
|
||||
class build_usage(Command):
|
||||
description = "generate usage for each command"
|
||||
|
||||
user_options = [
|
||||
('output=', 'O', 'output directory'),
|
||||
]
|
||||
user_options = [("output=", "O", "output directory")]
|
||||
|
||||
def initialize_options(self):
|
||||
pass
|
||||
@ -50,17 +48,19 @@ def finalize_options(self):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
print('generating usage docs')
|
||||
print("generating usage docs")
|
||||
import borg
|
||||
borg.doc_mode = 'build_man'
|
||||
if not os.path.exists('docs/usage'):
|
||||
os.mkdir('docs/usage')
|
||||
|
||||
borg.doc_mode = "build_man"
|
||||
if not os.path.exists("docs/usage"):
|
||||
os.mkdir("docs/usage")
|
||||
# allows us to build docs without the C modules fully loaded during help generation
|
||||
from borg.archiver import Archiver
|
||||
parser = Archiver(prog='borg').build_parser()
|
||||
|
||||
parser = Archiver(prog="borg").build_parser()
|
||||
# borgfs has a separate man page to satisfy debian's "every program from a package
|
||||
# must have a man page" requirement, but it doesn't need a separate HTML docs page
|
||||
#borgfs_parser = Archiver(prog='borgfs').build_parser()
|
||||
# borgfs_parser = Archiver(prog='borgfs').build_parser()
|
||||
|
||||
self.generate_level("", parser, Archiver)
|
||||
|
||||
@ -68,7 +68,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
is_subcommand = False
|
||||
choices = {}
|
||||
for action in parser._actions:
|
||||
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
|
||||
if action.choices is not None and "SubParsersAction" in str(action.__class__):
|
||||
is_subcommand = True
|
||||
for cmd, parser in action.choices.items():
|
||||
choices[prefix + cmd] = parser
|
||||
@ -76,32 +76,37 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
choices.update(extra_choices)
|
||||
if prefix and not choices:
|
||||
return
|
||||
print('found commands: %s' % list(choices.keys()))
|
||||
print("found commands: %s" % list(choices.keys()))
|
||||
|
||||
for command, parser in sorted(choices.items()):
|
||||
if command.startswith('debug'):
|
||||
print('skipping', command)
|
||||
if command.startswith("debug"):
|
||||
print("skipping", command)
|
||||
continue
|
||||
print('generating help for %s' % command)
|
||||
print("generating help for %s" % command)
|
||||
|
||||
if self.generate_level(command + " ", parser, Archiver):
|
||||
continue
|
||||
|
||||
with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
|
||||
with open("docs/usage/%s.rst.inc" % command.replace(" ", "_"), "w") as doc:
|
||||
doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")
|
||||
if command == 'help':
|
||||
if command == "help":
|
||||
for topic in Archiver.helptext:
|
||||
params = {"topic": topic,
|
||||
"underline": '~' * len('borg help ' + topic)}
|
||||
params = {"topic": topic, "underline": "~" * len("borg help " + topic)}
|
||||
doc.write(".. _borg_{topic}:\n\n".format(**params))
|
||||
doc.write("borg help {topic}\n{underline}\n\n".format(**params))
|
||||
doc.write(Archiver.helptext[topic])
|
||||
else:
|
||||
params = {"command": command,
|
||||
"command_": command.replace(' ', '_'),
|
||||
"underline": '-' * len('borg ' + command)}
|
||||
params = {
|
||||
"command": command,
|
||||
"command_": command.replace(" ", "_"),
|
||||
"underline": "-" * len("borg " + command),
|
||||
}
|
||||
doc.write(".. _borg_{command_}:\n\n".format(**params))
|
||||
doc.write("borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(**params))
|
||||
doc.write(
|
||||
"borg {command}\n{underline}\n.. code-block:: none\n\n borg [common options] {command}".format(
|
||||
**params
|
||||
)
|
||||
)
|
||||
self.write_usage(parser, doc)
|
||||
epilog = parser.epilog
|
||||
parser.epilog = None
|
||||
@ -109,21 +114,21 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
doc.write("\n\nDescription\n~~~~~~~~~~~\n")
|
||||
doc.write(epilog)
|
||||
|
||||
if 'create' in choices:
|
||||
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
|
||||
with open('docs/usage/common-options.rst.inc', 'w') as doc:
|
||||
if "create" in choices:
|
||||
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
|
||||
with open("docs/usage/common-options.rst.inc", "w") as doc:
|
||||
self.write_options_group(common_options, doc, False, base_indent=0)
|
||||
|
||||
return is_subcommand
|
||||
|
||||
def write_usage(self, parser, fp):
|
||||
if any(len(o.option_strings) for o in parser._actions):
|
||||
fp.write(' [options]')
|
||||
fp.write(" [options]")
|
||||
for option in parser._actions:
|
||||
if option.option_strings:
|
||||
continue
|
||||
fp.write(' ' + format_metavar(option))
|
||||
fp.write('\n\n')
|
||||
fp.write(" " + format_metavar(option))
|
||||
fp.write("\n\n")
|
||||
|
||||
def write_options(self, parser, fp):
|
||||
def is_positional_group(group):
|
||||
@ -134,58 +139,58 @@ def is_positional_group(group):
|
||||
|
||||
def html_write(s):
|
||||
for line in s.splitlines():
|
||||
fp.write(' ' + line + '\n')
|
||||
fp.write(" " + line + "\n")
|
||||
|
||||
rows = []
|
||||
for group in parser._action_groups:
|
||||
if group.title == 'Common options':
|
||||
if group.title == "Common options":
|
||||
# (no of columns used, columns, ...)
|
||||
rows.append((1, '.. class:: borg-common-opt-ref\n\n:ref:`common_options`'))
|
||||
rows.append((1, ".. class:: borg-common-opt-ref\n\n:ref:`common_options`"))
|
||||
else:
|
||||
if not group._group_actions:
|
||||
continue
|
||||
group_header = '**%s**' % group.title
|
||||
group_header = "**%s**" % group.title
|
||||
if group.description:
|
||||
group_header += ' — ' + group.description
|
||||
group_header += " — " + group.description
|
||||
rows.append((1, group_header))
|
||||
if is_positional_group(group):
|
||||
for option in group._group_actions:
|
||||
rows.append((3, '', '``%s``' % option.metavar, option.help or ''))
|
||||
rows.append((3, "", "``%s``" % option.metavar, option.help or ""))
|
||||
else:
|
||||
for option in group._group_actions:
|
||||
if option.metavar:
|
||||
option_fmt = '``%s ' + option.metavar + '``'
|
||||
option_fmt = "``%s " + option.metavar + "``"
|
||||
else:
|
||||
option_fmt = '``%s``'
|
||||
option_str = ', '.join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
|
||||
rows.append((3, '', option_str, option_desc))
|
||||
option_fmt = "``%s``"
|
||||
option_str = ", ".join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
|
||||
rows.append((3, "", option_str, option_desc))
|
||||
|
||||
fp.write('.. only:: html\n\n')
|
||||
fp.write(".. only:: html\n\n")
|
||||
table = io.StringIO()
|
||||
table.write('.. class:: borg-options-table\n\n')
|
||||
table.write(".. class:: borg-options-table\n\n")
|
||||
self.rows_to_table(rows, table.write)
|
||||
fp.write(textwrap.indent(table.getvalue(), ' ' * 4))
|
||||
fp.write(textwrap.indent(table.getvalue(), " " * 4))
|
||||
|
||||
# LaTeX output:
|
||||
# Regular rST option lists (irregular column widths)
|
||||
latex_options = io.StringIO()
|
||||
for group in parser._action_groups:
|
||||
if group.title == 'Common options':
|
||||
latex_options.write('\n\n:ref:`common_options`\n')
|
||||
latex_options.write(' |')
|
||||
if group.title == "Common options":
|
||||
latex_options.write("\n\n:ref:`common_options`\n")
|
||||
latex_options.write(" |")
|
||||
else:
|
||||
self.write_options_group(group, latex_options)
|
||||
fp.write('\n.. only:: latex\n\n')
|
||||
fp.write(textwrap.indent(latex_options.getvalue(), ' ' * 4))
|
||||
fp.write("\n.. only:: latex\n\n")
|
||||
fp.write(textwrap.indent(latex_options.getvalue(), " " * 4))
|
||||
|
||||
def rows_to_table(self, rows, write):
|
||||
def write_row_separator():
|
||||
write('+')
|
||||
write("+")
|
||||
for column_width in column_widths:
|
||||
write('-' * (column_width + 1))
|
||||
write('+')
|
||||
write('\n')
|
||||
write("-" * (column_width + 1))
|
||||
write("+")
|
||||
write("\n")
|
||||
|
||||
# Find column count and width
|
||||
column_count = max(columns for columns, *_ in rows)
|
||||
@ -201,22 +206,22 @@ def write_row_separator():
|
||||
# where each cell contains no newline.
|
||||
rowspanning_cells = []
|
||||
original_cells = list(original_cells)
|
||||
while any('\n' in cell for cell in original_cells):
|
||||
while any("\n" in cell for cell in original_cells):
|
||||
cell_bloc = []
|
||||
for i, cell in enumerate(original_cells):
|
||||
pre, _, original_cells[i] = cell.partition('\n')
|
||||
pre, _, original_cells[i] = cell.partition("\n")
|
||||
cell_bloc.append(pre)
|
||||
rowspanning_cells.append(cell_bloc)
|
||||
rowspanning_cells.append(original_cells)
|
||||
for cells in rowspanning_cells:
|
||||
for i, column_width in enumerate(column_widths):
|
||||
if i < columns:
|
||||
write('| ')
|
||||
write("| ")
|
||||
write(cells[i].ljust(column_width))
|
||||
else:
|
||||
write(' ')
|
||||
write(''.ljust(column_width))
|
||||
write('|\n')
|
||||
write(" ")
|
||||
write("".ljust(column_width))
|
||||
write("|\n")
|
||||
|
||||
write_row_separator()
|
||||
# This bit of JavaScript kills the <colgroup> that is invariably inserted by docutils,
|
||||
@ -224,7 +229,9 @@ def write_row_separator():
|
||||
# with CSS alone.
|
||||
# Since this is HTML-only output, it would be possible to just generate a <table> directly,
|
||||
# but then we'd lose rST formatting.
|
||||
write(textwrap.dedent("""
|
||||
write(
|
||||
textwrap.dedent(
|
||||
"""
|
||||
.. raw:: html
|
||||
|
||||
<script type='text/javascript'>
|
||||
@ -232,88 +239,88 @@ def write_row_separator():
|
||||
$('.borg-options-table colgroup').remove();
|
||||
})
|
||||
</script>
|
||||
"""))
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
def write_options_group(self, group, fp, with_title=True, base_indent=4):
|
||||
def is_positional_group(group):
|
||||
return any(not o.option_strings for o in group._group_actions)
|
||||
|
||||
indent = ' ' * base_indent
|
||||
indent = " " * base_indent
|
||||
|
||||
if is_positional_group(group):
|
||||
for option in group._group_actions:
|
||||
fp.write(option.metavar + '\n')
|
||||
fp.write(textwrap.indent(option.help or '', ' ' * base_indent) + '\n')
|
||||
fp.write(option.metavar + "\n")
|
||||
fp.write(textwrap.indent(option.help or "", " " * base_indent) + "\n")
|
||||
return
|
||||
|
||||
if not group._group_actions:
|
||||
return
|
||||
|
||||
if with_title:
|
||||
fp.write('\n\n')
|
||||
fp.write(group.title + '\n')
|
||||
fp.write("\n\n")
|
||||
fp.write(group.title + "\n")
|
||||
|
||||
opts = OrderedDict()
|
||||
|
||||
for option in group._group_actions:
|
||||
if option.metavar:
|
||||
option_fmt = '%s ' + option.metavar
|
||||
option_fmt = "%s " + option.metavar
|
||||
else:
|
||||
option_fmt = '%s'
|
||||
option_str = ', '.join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
|
||||
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
|
||||
option_fmt = "%s"
|
||||
option_str = ", ".join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
|
||||
opts[option_str] = textwrap.indent(option_desc, " " * 4)
|
||||
|
||||
padding = len(max(opts)) + 1
|
||||
|
||||
for option, desc in opts.items():
|
||||
fp.write(indent + option.ljust(padding) + desc + '\n')
|
||||
fp.write(indent + option.ljust(padding) + desc + "\n")
|
||||
|
||||
|
||||
class build_man(Command):
|
||||
description = 'build man pages'
|
||||
description = "build man pages"
|
||||
|
||||
user_options = []
|
||||
|
||||
see_also = {
|
||||
'create': ('delete', 'prune', 'check', 'patterns', 'placeholders', 'compression'),
|
||||
'recreate': ('patterns', 'placeholders', 'compression'),
|
||||
'list': ('info', 'diff', 'prune', 'patterns'),
|
||||
'info': ('list', 'diff'),
|
||||
'rcreate': ('rdelete', 'rlist', 'check', 'key-import', 'key-export', 'key-change-passphrase'),
|
||||
'key-import': ('key-export', ),
|
||||
'key-export': ('key-import', ),
|
||||
'mount': ('umount', 'extract'), # Would be cooler if these two were on the same page
|
||||
'umount': ('mount', ),
|
||||
'extract': ('mount', ),
|
||||
'delete': ('compact', ),
|
||||
'prune': ('compact', ),
|
||||
"create": ("delete", "prune", "check", "patterns", "placeholders", "compression"),
|
||||
"recreate": ("patterns", "placeholders", "compression"),
|
||||
"list": ("info", "diff", "prune", "patterns"),
|
||||
"info": ("list", "diff"),
|
||||
"rcreate": ("rdelete", "rlist", "check", "key-import", "key-export", "key-change-passphrase"),
|
||||
"key-import": ("key-export",),
|
||||
"key-export": ("key-import",),
|
||||
"mount": ("umount", "extract"), # Would be cooler if these two were on the same page
|
||||
"umount": ("mount",),
|
||||
"extract": ("mount",),
|
||||
"delete": ("compact",),
|
||||
"prune": ("compact",),
|
||||
}
|
||||
|
||||
rst_prelude = textwrap.dedent("""
|
||||
rst_prelude = textwrap.dedent(
|
||||
"""
|
||||
.. role:: ref(title)
|
||||
|
||||
.. |project_name| replace:: Borg
|
||||
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
usage_group = {
|
||||
'break-lock': 'lock',
|
||||
'with-lock': 'lock',
|
||||
|
||||
'key_change-passphrase': 'key',
|
||||
'key_change-location': 'key',
|
||||
'key_export': 'key',
|
||||
'key_import': 'key',
|
||||
'key_migrate-to-repokey': 'key',
|
||||
|
||||
'export-tar': 'tar',
|
||||
'import-tar': 'tar',
|
||||
|
||||
'benchmark_crud': 'benchmark',
|
||||
'benchmark_cpu': 'benchmark',
|
||||
|
||||
'umount': 'mount',
|
||||
"break-lock": "lock",
|
||||
"with-lock": "lock",
|
||||
"key_change-passphrase": "key",
|
||||
"key_change-location": "key",
|
||||
"key_export": "key",
|
||||
"key_import": "key",
|
||||
"key_migrate-to-repokey": "key",
|
||||
"export-tar": "tar",
|
||||
"import-tar": "tar",
|
||||
"benchmark_crud": "benchmark",
|
||||
"benchmark_cpu": "benchmark",
|
||||
"umount": "mount",
|
||||
}
|
||||
|
||||
def initialize_options(self):
|
||||
@ -323,16 +330,18 @@ def finalize_options(self):
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
print('building man pages (in docs/man)', file=sys.stderr)
|
||||
print("building man pages (in docs/man)", file=sys.stderr)
|
||||
import borg
|
||||
borg.doc_mode = 'build_man'
|
||||
os.makedirs('docs/man', exist_ok=True)
|
||||
|
||||
borg.doc_mode = "build_man"
|
||||
os.makedirs("docs/man", exist_ok=True)
|
||||
# allows us to build docs without the C modules fully loaded during help generation
|
||||
from borg.archiver import Archiver
|
||||
parser = Archiver(prog='borg').build_parser()
|
||||
borgfs_parser = Archiver(prog='borgfs').build_parser()
|
||||
|
||||
self.generate_level('', parser, Archiver, {'borgfs': borgfs_parser})
|
||||
parser = Archiver(prog="borg").build_parser()
|
||||
borgfs_parser = Archiver(prog="borgfs").build_parser()
|
||||
|
||||
self.generate_level("", parser, Archiver, {"borgfs": borgfs_parser})
|
||||
self.build_topic_pages(Archiver)
|
||||
self.build_intro_page()
|
||||
|
||||
@ -340,7 +349,7 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
is_subcommand = False
|
||||
choices = {}
|
||||
for action in parser._actions:
|
||||
if action.choices is not None and 'SubParsersAction' in str(action.__class__):
|
||||
if action.choices is not None and "SubParsersAction" in str(action.__class__):
|
||||
is_subcommand = True
|
||||
for cmd, parser in action.choices.items():
|
||||
choices[prefix + cmd] = parser
|
||||
@ -350,50 +359,50 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
return
|
||||
|
||||
for command, parser in sorted(choices.items()):
|
||||
if command.startswith('debug') or command == 'help':
|
||||
if command.startswith("debug") or command == "help":
|
||||
continue
|
||||
|
||||
if command == "borgfs":
|
||||
man_title = command
|
||||
else:
|
||||
man_title = 'borg-' + command.replace(' ', '-')
|
||||
print('building man page', man_title + '(1)', file=sys.stderr)
|
||||
man_title = "borg-" + command.replace(" ", "-")
|
||||
print("building man page", man_title + "(1)", file=sys.stderr)
|
||||
|
||||
is_intermediary = self.generate_level(command + ' ', parser, Archiver)
|
||||
is_intermediary = self.generate_level(command + " ", parser, Archiver)
|
||||
|
||||
doc, write = self.new_doc()
|
||||
self.write_man_header(write, man_title, parser.description)
|
||||
|
||||
self.write_heading(write, 'SYNOPSIS')
|
||||
self.write_heading(write, "SYNOPSIS")
|
||||
if is_intermediary:
|
||||
subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
|
||||
subparsers = [action for action in parser._actions if "SubParsersAction" in str(action.__class__)][0]
|
||||
for subcommand in subparsers.choices:
|
||||
write('| borg', '[common options]', command, subcommand, '...')
|
||||
self.see_also.setdefault(command, []).append(f'{command}-{subcommand}')
|
||||
write("| borg", "[common options]", command, subcommand, "...")
|
||||
self.see_also.setdefault(command, []).append(f"{command}-{subcommand}")
|
||||
else:
|
||||
if command == "borgfs":
|
||||
write(command, end='')
|
||||
write(command, end="")
|
||||
else:
|
||||
write('borg', '[common options]', command, end='')
|
||||
write("borg", "[common options]", command, end="")
|
||||
self.write_usage(write, parser)
|
||||
write('\n')
|
||||
write("\n")
|
||||
|
||||
description, _, notes = parser.epilog.partition('\n.. man NOTES')
|
||||
description, _, notes = parser.epilog.partition("\n.. man NOTES")
|
||||
|
||||
if description:
|
||||
self.write_heading(write, 'DESCRIPTION')
|
||||
self.write_heading(write, "DESCRIPTION")
|
||||
write(description)
|
||||
|
||||
if not is_intermediary:
|
||||
self.write_heading(write, 'OPTIONS')
|
||||
write('See `borg-common(1)` for common options of Borg commands.')
|
||||
self.write_heading(write, "OPTIONS")
|
||||
write("See `borg-common(1)` for common options of Borg commands.")
|
||||
write()
|
||||
self.write_options(write, parser)
|
||||
|
||||
self.write_examples(write, command)
|
||||
|
||||
if notes:
|
||||
self.write_heading(write, 'NOTES')
|
||||
self.write_heading(write, "NOTES")
|
||||
write(notes)
|
||||
|
||||
self.write_see_also(write, man_title)
|
||||
@ -401,14 +410,14 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
self.gen_man_page(man_title, doc.getvalue())
|
||||
|
||||
# Generate the borg-common(1) man page with the common options.
|
||||
if 'create' in choices:
|
||||
if "create" in choices:
|
||||
doc, write = self.new_doc()
|
||||
man_title = 'borg-common'
|
||||
self.write_man_header(write, man_title, 'Common options of Borg commands')
|
||||
man_title = "borg-common"
|
||||
self.write_man_header(write, man_title, "Common options of Borg commands")
|
||||
|
||||
common_options = [group for group in choices['create']._action_groups if group.title == 'Common options'][0]
|
||||
common_options = [group for group in choices["create"]._action_groups if group.title == "Common options"][0]
|
||||
|
||||
self.write_heading(write, 'SYNOPSIS')
|
||||
self.write_heading(write, "SYNOPSIS")
|
||||
self.write_options_group(write, common_options)
|
||||
self.write_see_also(write, man_title)
|
||||
self.gen_man_page(man_title, doc.getvalue())
|
||||
@ -418,20 +427,20 @@ def generate_level(self, prefix, parser, Archiver, extra_choices=None):
|
||||
def build_topic_pages(self, Archiver):
|
||||
for topic, text in Archiver.helptext.items():
|
||||
doc, write = self.new_doc()
|
||||
man_title = 'borg-' + topic
|
||||
print('building man page', man_title + '(1)', file=sys.stderr)
|
||||
man_title = "borg-" + topic
|
||||
print("building man page", man_title + "(1)", file=sys.stderr)
|
||||
|
||||
self.write_man_header(write, man_title, 'Details regarding ' + topic)
|
||||
self.write_heading(write, 'DESCRIPTION')
|
||||
self.write_man_header(write, man_title, "Details regarding " + topic)
|
||||
self.write_heading(write, "DESCRIPTION")
|
||||
write(text)
|
||||
self.gen_man_page(man_title, doc.getvalue())
|
||||
|
||||
def build_intro_page(self):
|
||||
doc, write = self.new_doc()
|
||||
man_title = 'borg'
|
||||
print('building man page borg(1)', file=sys.stderr)
|
||||
man_title = "borg"
|
||||
print("building man page borg(1)", file=sys.stderr)
|
||||
|
||||
with open('docs/man_intro.rst') as fd:
|
||||
with open("docs/man_intro.rst") as fd:
|
||||
man_intro = fd.read()
|
||||
|
||||
self.write_man_header(write, man_title, "deduplicating and encrypting backup tool")
|
||||
@ -446,9 +455,10 @@ def new_doc(self):
|
||||
def printer(self, fd):
|
||||
def write(*args, **kwargs):
|
||||
print(*args, file=fd, **kwargs)
|
||||
|
||||
return write
|
||||
|
||||
def write_heading(self, write, header, char='-', double_sided=False):
|
||||
def write_heading(self, write, header, char="-", double_sided=False):
|
||||
write()
|
||||
if double_sided:
|
||||
write(char * len(header))
|
||||
@ -457,43 +467,43 @@ def write_heading(self, write, header, char='-', double_sided=False):
|
||||
write()
|
||||
|
||||
def write_man_header(self, write, title, description):
|
||||
self.write_heading(write, title, '=', double_sided=True)
|
||||
self.write_heading(write, title, "=", double_sided=True)
|
||||
self.write_heading(write, description, double_sided=True)
|
||||
# man page metadata
|
||||
write(':Author: The Borg Collective')
|
||||
write(':Date:', datetime.utcnow().date().isoformat())
|
||||
write(':Manual section: 1')
|
||||
write(':Manual group: borg backup tool')
|
||||
write(":Author: The Borg Collective")
|
||||
write(":Date:", datetime.utcnow().date().isoformat())
|
||||
write(":Manual section: 1")
|
||||
write(":Manual group: borg backup tool")
|
||||
write()
|
||||
|
||||
def write_examples(self, write, command):
|
||||
command = command.replace(' ', '_')
|
||||
with open('docs/usage/%s.rst' % self.usage_group.get(command, command)) as fd:
|
||||
command = command.replace(" ", "_")
|
||||
with open("docs/usage/%s.rst" % self.usage_group.get(command, command)) as fd:
|
||||
usage = fd.read()
|
||||
usage_include = '.. include:: %s.rst.inc' % command
|
||||
usage_include = ".. include:: %s.rst.inc" % command
|
||||
begin = usage.find(usage_include)
|
||||
end = usage.find('.. include', begin + 1)
|
||||
end = usage.find(".. include", begin + 1)
|
||||
# If a command has a dedicated anchor, it will occur before the command's include.
|
||||
if 0 < usage.find('.. _', begin + 1) < end:
|
||||
end = usage.find('.. _', begin + 1)
|
||||
if 0 < usage.find(".. _", begin + 1) < end:
|
||||
end = usage.find(".. _", begin + 1)
|
||||
examples = usage[begin:end]
|
||||
examples = examples.replace(usage_include, '')
|
||||
examples = examples.replace('Examples\n~~~~~~~~', '')
|
||||
examples = examples.replace('Miscellaneous Help\n------------------', '')
|
||||
examples = examples.replace('``docs/misc/prune-example.txt``:', '``docs/misc/prune-example.txt``.')
|
||||
examples = examples.replace('.. highlight:: none\n', '') # we don't support highlight
|
||||
examples = re.sub('^(~+)$', lambda matches: '+' * len(matches.group(0)), examples, flags=re.MULTILINE)
|
||||
examples = examples.replace(usage_include, "")
|
||||
examples = examples.replace("Examples\n~~~~~~~~", "")
|
||||
examples = examples.replace("Miscellaneous Help\n------------------", "")
|
||||
examples = examples.replace("``docs/misc/prune-example.txt``:", "``docs/misc/prune-example.txt``.")
|
||||
examples = examples.replace(".. highlight:: none\n", "") # we don't support highlight
|
||||
examples = re.sub("^(~+)$", lambda matches: "+" * len(matches.group(0)), examples, flags=re.MULTILINE)
|
||||
examples = examples.strip()
|
||||
if examples:
|
||||
self.write_heading(write, 'EXAMPLES', '-')
|
||||
self.write_heading(write, "EXAMPLES", "-")
|
||||
write(examples)
|
||||
|
||||
def write_see_also(self, write, man_title):
|
||||
see_also = self.see_also.get(man_title.replace('borg-', ''), ())
|
||||
see_also = ['`borg-%s(1)`' % s for s in see_also]
|
||||
see_also.insert(0, '`borg-common(1)`')
|
||||
self.write_heading(write, 'SEE ALSO')
|
||||
write(', '.join(see_also))
|
||||
see_also = self.see_also.get(man_title.replace("borg-", ""), ())
|
||||
see_also = ["`borg-%s(1)`" % s for s in see_also]
|
||||
see_also.insert(0, "`borg-common(1)`")
|
||||
self.write_heading(write, "SEE ALSO")
|
||||
write(", ".join(see_also))
|
||||
|
||||
def gen_man_page(self, name, rst):
|
||||
from docutils.writers import manpage
|
||||
@ -502,29 +512,29 @@ def gen_man_page(self, name, rst):
|
||||
from docutils.parsers.rst import roles
|
||||
|
||||
def issue(name, rawtext, text, lineno, inliner, options={}, content=[]):
|
||||
return [inline(rawtext, '#' + text)], []
|
||||
return [inline(rawtext, "#" + text)], []
|
||||
|
||||
roles.register_local_role('issue', issue)
|
||||
roles.register_local_role("issue", issue)
|
||||
# We give the source_path so that docutils can find relative includes
|
||||
# as-if the document where located in the docs/ directory.
|
||||
man_page = publish_string(source=rst, source_path='docs/%s.rst' % name, writer=manpage.Writer())
|
||||
with open('docs/man/%s.1' % name, 'wb') as fd:
|
||||
man_page = publish_string(source=rst, source_path="docs/%s.rst" % name, writer=manpage.Writer())
|
||||
with open("docs/man/%s.1" % name, "wb") as fd:
|
||||
fd.write(man_page)
|
||||
|
||||
def write_usage(self, write, parser):
|
||||
if any(len(o.option_strings) for o in parser._actions):
|
||||
write(' [options] ', end='')
|
||||
write(" [options] ", end="")
|
||||
for option in parser._actions:
|
||||
if option.option_strings:
|
||||
continue
|
||||
write(format_metavar(option), end=' ')
|
||||
write(format_metavar(option), end=" ")
|
||||
|
||||
def write_options(self, write, parser):
|
||||
for group in parser._action_groups:
|
||||
if group.title == 'Common options' or not group._group_actions:
|
||||
if group.title == "Common options" or not group._group_actions:
|
||||
continue
|
||||
title = 'arguments' if group.title == 'positional arguments' else group.title
|
||||
self.write_heading(write, title, '+')
|
||||
title = "arguments" if group.title == "positional arguments" else group.title
|
||||
self.write_heading(write, title, "+")
|
||||
self.write_options_group(write, group)
|
||||
|
||||
def write_options_group(self, write, group):
|
||||
@ -534,19 +544,19 @@ def is_positional_group(group):
|
||||
if is_positional_group(group):
|
||||
for option in group._group_actions:
|
||||
write(option.metavar)
|
||||
write(textwrap.indent(option.help or '', ' ' * 4))
|
||||
write(textwrap.indent(option.help or "", " " * 4))
|
||||
return
|
||||
|
||||
opts = OrderedDict()
|
||||
|
||||
for option in group._group_actions:
|
||||
if option.metavar:
|
||||
option_fmt = '%s ' + option.metavar
|
||||
option_fmt = "%s " + option.metavar
|
||||
else:
|
||||
option_fmt = '%s'
|
||||
option_str = ', '.join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or '') % option.__dict__)
|
||||
opts[option_str] = textwrap.indent(option_desc, ' ' * 4)
|
||||
option_fmt = "%s"
|
||||
option_str = ", ".join(option_fmt % s for s in option.option_strings)
|
||||
option_desc = textwrap.dedent((option.help or "") % option.__dict__)
|
||||
opts[option_str] = textwrap.indent(option_desc, " " * 4)
|
||||
|
||||
padding = len(max(opts)) + 1
|
||||
|
||||
|
@ -9,11 +9,13 @@
|
||||
# assert that all semver components are integers
|
||||
# this is mainly to show errors when people repackage poorly
|
||||
# and setuptools_scm determines a 0.1.dev... version
|
||||
assert all(isinstance(v, int) for v in __version_tuple__), \
|
||||
assert all(isinstance(v, int) for v in __version_tuple__), (
|
||||
"""\
|
||||
broken borgbackup version metadata: %r
|
||||
|
||||
version metadata is obtained dynamically on installation via setuptools_scm,
|
||||
please ensure your git repo has the correct tags or you provide the version
|
||||
using SETUPTOOLS_SCM_PRETEND_VERSION in your build script.
|
||||
""" % __version__
|
||||
"""
|
||||
% __version__
|
||||
)
|
||||
|
@ -5,11 +5,12 @@
|
||||
# containing the dll is not in the search path. The dll is shipped
|
||||
# with python in the "DLLs" folder, so let's add this folder
|
||||
# to the path. The folder is always in sys.path, get it from there.
|
||||
if sys.platform.startswith('win32'):
|
||||
if sys.platform.startswith("win32"):
|
||||
# Keep it an iterable to support multiple folder which contain "DLLs".
|
||||
dll_path = (p for p in sys.path if 'DLLs' in os.path.normpath(p).split(os.path.sep))
|
||||
os.environ['PATH'] = os.pathsep.join(dll_path) + os.pathsep + os.environ['PATH']
|
||||
dll_path = (p for p in sys.path if "DLLs" in os.path.normpath(p).split(os.path.sep))
|
||||
os.environ["PATH"] = os.pathsep.join(dll_path) + os.pathsep + os.environ["PATH"]
|
||||
|
||||
|
||||
from borg.archiver import main
|
||||
|
||||
main()
|
||||
|
File diff suppressed because it is too large
Load Diff
4193
src/borg/archiver.py
4193
src/borg/archiver.py
File diff suppressed because it is too large
Load Diff
@ -10,7 +10,7 @@
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
files_cache_logger = create_logger('borg.debug.files_cache')
|
||||
files_cache_logger = create_logger("borg.debug.files_cache")
|
||||
|
||||
from .constants import CACHE_README, FILES_CACHE_MODE_DISABLED
|
||||
from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
|
||||
@ -37,7 +37,7 @@
|
||||
from .repository import LIST_SCAN_LIMIT
|
||||
|
||||
# note: cmtime might me either a ctime or a mtime timestamp
|
||||
FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size cmtime chunk_ids')
|
||||
FileCacheEntry = namedtuple("FileCacheEntry", "age inode size cmtime chunk_ids")
|
||||
|
||||
|
||||
class SecurityManager:
|
||||
@ -64,9 +64,9 @@ def __init__(self, repository):
|
||||
self.repository = repository
|
||||
self.dir = get_security_dir(repository.id_str)
|
||||
self.cache_dir = cache_dir(repository)
|
||||
self.key_type_file = os.path.join(self.dir, 'key-type')
|
||||
self.location_file = os.path.join(self.dir, 'location')
|
||||
self.manifest_ts_file = os.path.join(self.dir, 'manifest-timestamp')
|
||||
self.key_type_file = os.path.join(self.dir, "key-type")
|
||||
self.location_file = os.path.join(self.dir, "location")
|
||||
self.manifest_ts_file = os.path.join(self.dir, "manifest-timestamp")
|
||||
|
||||
@staticmethod
|
||||
def destroy(repository, path=None):
|
||||
@ -76,8 +76,7 @@ def destroy(repository, path=None):
|
||||
shutil.rmtree(path)
|
||||
|
||||
def known(self):
|
||||
return all(os.path.exists(f)
|
||||
for f in (self.key_type_file, self.location_file, self.manifest_ts_file))
|
||||
return all(os.path.exists(f) for f in (self.key_type_file, self.location_file, self.manifest_ts_file))
|
||||
|
||||
def key_matches(self, key):
|
||||
if not self.known():
|
||||
@ -87,14 +86,14 @@ def key_matches(self, key):
|
||||
type = fd.read()
|
||||
return type == str(key.TYPE)
|
||||
except OSError as exc:
|
||||
logger.warning('Could not read/parse key type file: %s', exc)
|
||||
logger.warning("Could not read/parse key type file: %s", exc)
|
||||
|
||||
def save(self, manifest, key):
|
||||
logger.debug('security: saving state for %s to %s', self.repository.id_str, self.dir)
|
||||
logger.debug("security: saving state for %s to %s", self.repository.id_str, self.dir)
|
||||
current_location = self.repository._location.canonical_path()
|
||||
logger.debug('security: current location %s', current_location)
|
||||
logger.debug('security: key type %s', str(key.TYPE))
|
||||
logger.debug('security: manifest timestamp %s', manifest.timestamp)
|
||||
logger.debug("security: current location %s", current_location)
|
||||
logger.debug("security: key type %s", str(key.TYPE))
|
||||
logger.debug("security: manifest timestamp %s", manifest.timestamp)
|
||||
with SaveFile(self.location_file) as fd:
|
||||
fd.write(current_location)
|
||||
with SaveFile(self.key_type_file) as fd:
|
||||
@ -107,28 +106,36 @@ def assert_location_matches(self, cache_config=None):
|
||||
try:
|
||||
with open(self.location_file) as fd:
|
||||
previous_location = fd.read()
|
||||
logger.debug('security: read previous location %r', previous_location)
|
||||
logger.debug("security: read previous location %r", previous_location)
|
||||
except FileNotFoundError:
|
||||
logger.debug('security: previous location file %s not found', self.location_file)
|
||||
logger.debug("security: previous location file %s not found", self.location_file)
|
||||
previous_location = None
|
||||
except OSError as exc:
|
||||
logger.warning('Could not read previous location file: %s', exc)
|
||||
logger.warning("Could not read previous location file: %s", exc)
|
||||
previous_location = None
|
||||
if cache_config and cache_config.previous_location and previous_location != cache_config.previous_location:
|
||||
# Reconcile cache and security dir; we take the cache location.
|
||||
previous_location = cache_config.previous_location
|
||||
logger.debug('security: using previous_location of cache: %r', previous_location)
|
||||
logger.debug("security: using previous_location of cache: %r", previous_location)
|
||||
|
||||
repository_location = self.repository._location.canonical_path()
|
||||
if previous_location and previous_location != repository_location:
|
||||
msg = ("Warning: The repository at location {} was previously located at {}\n".format(
|
||||
repository_location, previous_location) +
|
||||
"Do you want to continue? [yN] ")
|
||||
if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
|
||||
retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
|
||||
msg = (
|
||||
"Warning: The repository at location {} was previously located at {}\n".format(
|
||||
repository_location, previous_location
|
||||
)
|
||||
+ "Do you want to continue? [yN] "
|
||||
)
|
||||
if not yes(
|
||||
msg,
|
||||
false_msg="Aborting.",
|
||||
invalid_msg="Invalid answer, aborting.",
|
||||
retry=False,
|
||||
env_var_override="BORG_RELOCATED_REPO_ACCESS_IS_OK",
|
||||
):
|
||||
raise Cache.RepositoryAccessAborted()
|
||||
# adapt on-disk config immediately if the new location was accepted
|
||||
logger.debug('security: updating location stored in cache and security dir')
|
||||
logger.debug("security: updating location stored in cache and security dir")
|
||||
with SaveFile(self.location_file) as fd:
|
||||
fd.write(repository_location)
|
||||
if cache_config:
|
||||
@ -138,16 +145,16 @@ def assert_no_manifest_replay(self, manifest, key, cache_config=None):
|
||||
try:
|
||||
with open(self.manifest_ts_file) as fd:
|
||||
timestamp = fd.read()
|
||||
logger.debug('security: read manifest timestamp %r', timestamp)
|
||||
logger.debug("security: read manifest timestamp %r", timestamp)
|
||||
except FileNotFoundError:
|
||||
logger.debug('security: manifest timestamp file %s not found', self.manifest_ts_file)
|
||||
timestamp = ''
|
||||
logger.debug("security: manifest timestamp file %s not found", self.manifest_ts_file)
|
||||
timestamp = ""
|
||||
except OSError as exc:
|
||||
logger.warning('Could not read previous location file: %s', exc)
|
||||
timestamp = ''
|
||||
logger.warning("Could not read previous location file: %s", exc)
|
||||
timestamp = ""
|
||||
if cache_config:
|
||||
timestamp = max(timestamp, cache_config.timestamp or '')
|
||||
logger.debug('security: determined newest manifest timestamp as %s', timestamp)
|
||||
timestamp = max(timestamp, cache_config.timestamp or "")
|
||||
logger.debug("security: determined newest manifest timestamp as %s", timestamp)
|
||||
# If repository is older than the cache or security dir something fishy is going on
|
||||
if timestamp and timestamp > manifest.timestamp:
|
||||
if isinstance(key, PlaintextKey):
|
||||
@ -175,30 +182,36 @@ def assert_secure(self, manifest, key, *, cache_config=None, warn_if_unencrypted
|
||||
self._assert_secure(manifest, key, cache_config)
|
||||
else:
|
||||
self._assert_secure(manifest, key)
|
||||
logger.debug('security: repository checks ok, allowing access')
|
||||
logger.debug("security: repository checks ok, allowing access")
|
||||
|
||||
def _assert_secure(self, manifest, key, cache_config=None):
|
||||
self.assert_location_matches(cache_config)
|
||||
self.assert_key_type(key, cache_config)
|
||||
self.assert_no_manifest_replay(manifest, key, cache_config)
|
||||
if not self.known():
|
||||
logger.debug('security: remembering previously unknown repository')
|
||||
logger.debug("security: remembering previously unknown repository")
|
||||
self.save(manifest, key)
|
||||
|
||||
def assert_access_unknown(self, warn_if_unencrypted, manifest, key):
|
||||
# warn_if_unencrypted=False is only used for initializing a new repository.
|
||||
# Thus, avoiding asking about a repository that's currently initializing.
|
||||
if not key.logically_encrypted and not self.known():
|
||||
msg = ("Warning: Attempting to access a previously unknown unencrypted repository!\n" +
|
||||
"Do you want to continue? [yN] ")
|
||||
allow_access = not warn_if_unencrypted or yes(msg, false_msg="Aborting.",
|
||||
msg = (
|
||||
"Warning: Attempting to access a previously unknown unencrypted repository!\n"
|
||||
+ "Do you want to continue? [yN] "
|
||||
)
|
||||
allow_access = not warn_if_unencrypted or yes(
|
||||
msg,
|
||||
false_msg="Aborting.",
|
||||
invalid_msg="Invalid answer, aborting.",
|
||||
retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK')
|
||||
retry=False,
|
||||
env_var_override="BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK",
|
||||
)
|
||||
if allow_access:
|
||||
if warn_if_unencrypted:
|
||||
logger.debug('security: remembering unknown unencrypted repository (explicitly allowed)')
|
||||
logger.debug("security: remembering unknown unencrypted repository (explicitly allowed)")
|
||||
else:
|
||||
logger.debug('security: initializing unencrypted repository')
|
||||
logger.debug("security: initializing unencrypted repository")
|
||||
self.save(manifest, key)
|
||||
else:
|
||||
raise Cache.CacheInitAbortedError()
|
||||
@ -214,10 +227,17 @@ def recanonicalize_relative_location(cache_location, repository):
|
||||
repo_location = repository._location.canonical_path()
|
||||
rl = Location(repo_location)
|
||||
cl = Location(cache_location)
|
||||
if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
|
||||
and \
|
||||
cl.path and rl.path and \
|
||||
cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
|
||||
if (
|
||||
cl.proto == rl.proto
|
||||
and cl.user == rl.user
|
||||
and cl.host == rl.host
|
||||
and cl.port == rl.port
|
||||
and cl.path
|
||||
and rl.path
|
||||
and cl.path.startswith("/~/")
|
||||
and rl.path.startswith("/./")
|
||||
and cl.path[3:] == rl.path[3:]
|
||||
):
|
||||
# everything is same except the expected change in relative path canonicalization,
|
||||
# update previous_location to avoid warning / user query about changed location:
|
||||
return repo_location
|
||||
@ -230,19 +250,19 @@ def cache_dir(repository, path=None):
|
||||
|
||||
|
||||
def files_cache_name():
|
||||
suffix = os.environ.get('BORG_FILES_CACHE_SUFFIX', '')
|
||||
return 'files.' + suffix if suffix else 'files'
|
||||
suffix = os.environ.get("BORG_FILES_CACHE_SUFFIX", "")
|
||||
return "files." + suffix if suffix else "files"
|
||||
|
||||
|
||||
def discover_files_cache_name(path):
|
||||
return [fn for fn in os.listdir(path) if fn == 'files' or fn.startswith('files.')][0]
|
||||
return [fn for fn in os.listdir(path) if fn == "files" or fn.startswith("files.")][0]
|
||||
|
||||
|
||||
class CacheConfig:
|
||||
def __init__(self, repository, path=None, lock_wait=None):
|
||||
self.repository = repository
|
||||
self.path = cache_dir(repository, path)
|
||||
self.config_path = os.path.join(self.path, 'config')
|
||||
self.config_path = os.path.join(self.path, "config")
|
||||
self.lock = None
|
||||
self.lock_wait = lock_wait
|
||||
|
||||
@ -259,17 +279,17 @@ def exists(self):
|
||||
def create(self):
|
||||
assert not self.exists()
|
||||
config = configparser.ConfigParser(interpolation=None)
|
||||
config.add_section('cache')
|
||||
config.set('cache', 'version', '1')
|
||||
config.set('cache', 'repository', self.repository.id_str)
|
||||
config.set('cache', 'manifest', '')
|
||||
config.add_section('integrity')
|
||||
config.set('integrity', 'manifest', '')
|
||||
config.add_section("cache")
|
||||
config.set("cache", "version", "1")
|
||||
config.set("cache", "repository", self.repository.id_str)
|
||||
config.set("cache", "manifest", "")
|
||||
config.add_section("integrity")
|
||||
config.set("integrity", "manifest", "")
|
||||
with SaveFile(self.config_path) as fd:
|
||||
config.write(fd)
|
||||
|
||||
def open(self):
|
||||
self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=self.lock_wait).acquire()
|
||||
self.lock = Lock(os.path.join(self.path, "lock"), exclusive=True, timeout=self.lock_wait).acquire()
|
||||
self.load()
|
||||
|
||||
def load(self):
|
||||
@ -277,15 +297,17 @@ def load(self):
|
||||
with open(self.config_path) as fd:
|
||||
self._config.read_file(fd)
|
||||
self._check_upgrade(self.config_path)
|
||||
self.id = self._config.get('cache', 'repository')
|
||||
self.manifest_id = unhexlify(self._config.get('cache', 'manifest'))
|
||||
self.timestamp = self._config.get('cache', 'timestamp', fallback=None)
|
||||
self.key_type = self._config.get('cache', 'key_type', fallback=None)
|
||||
self.ignored_features = set(parse_stringified_list(self._config.get('cache', 'ignored_features', fallback='')))
|
||||
self.mandatory_features = set(parse_stringified_list(self._config.get('cache', 'mandatory_features', fallback='')))
|
||||
self.id = self._config.get("cache", "repository")
|
||||
self.manifest_id = unhexlify(self._config.get("cache", "manifest"))
|
||||
self.timestamp = self._config.get("cache", "timestamp", fallback=None)
|
||||
self.key_type = self._config.get("cache", "key_type", fallback=None)
|
||||
self.ignored_features = set(parse_stringified_list(self._config.get("cache", "ignored_features", fallback="")))
|
||||
self.mandatory_features = set(
|
||||
parse_stringified_list(self._config.get("cache", "mandatory_features", fallback=""))
|
||||
)
|
||||
try:
|
||||
self.integrity = dict(self._config.items('integrity'))
|
||||
if self._config.get('cache', 'manifest') != self.integrity.pop('manifest'):
|
||||
self.integrity = dict(self._config.items("integrity"))
|
||||
if self._config.get("cache", "manifest") != self.integrity.pop("manifest"):
|
||||
# The cache config file is updated (parsed with ConfigParser, the state of the ConfigParser
|
||||
# is modified and then written out.), not re-created.
|
||||
# Thus, older versions will leave our [integrity] section alone, making the section's data invalid.
|
||||
@ -293,30 +315,30 @@ def load(self):
|
||||
# can discern whether an older version interfered by comparing the manifest IDs of this section
|
||||
# and the main [cache] section.
|
||||
self.integrity = {}
|
||||
logger.warning('Cache integrity data not available: old Borg version modified the cache.')
|
||||
logger.warning("Cache integrity data not available: old Borg version modified the cache.")
|
||||
except configparser.NoSectionError:
|
||||
logger.debug('Cache integrity: No integrity data found (files, chunks). Cache is from old version.')
|
||||
logger.debug("Cache integrity: No integrity data found (files, chunks). Cache is from old version.")
|
||||
self.integrity = {}
|
||||
previous_location = self._config.get('cache', 'previous_location', fallback=None)
|
||||
previous_location = self._config.get("cache", "previous_location", fallback=None)
|
||||
if previous_location:
|
||||
self.previous_location = recanonicalize_relative_location(previous_location, self.repository)
|
||||
else:
|
||||
self.previous_location = None
|
||||
self._config.set('cache', 'previous_location', self.repository._location.canonical_path())
|
||||
self._config.set("cache", "previous_location", self.repository._location.canonical_path())
|
||||
|
||||
def save(self, manifest=None, key=None):
|
||||
if manifest:
|
||||
self._config.set('cache', 'manifest', manifest.id_str)
|
||||
self._config.set('cache', 'timestamp', manifest.timestamp)
|
||||
self._config.set('cache', 'ignored_features', ','.join(self.ignored_features))
|
||||
self._config.set('cache', 'mandatory_features', ','.join(self.mandatory_features))
|
||||
if not self._config.has_section('integrity'):
|
||||
self._config.add_section('integrity')
|
||||
self._config.set("cache", "manifest", manifest.id_str)
|
||||
self._config.set("cache", "timestamp", manifest.timestamp)
|
||||
self._config.set("cache", "ignored_features", ",".join(self.ignored_features))
|
||||
self._config.set("cache", "mandatory_features", ",".join(self.mandatory_features))
|
||||
if not self._config.has_section("integrity"):
|
||||
self._config.add_section("integrity")
|
||||
for file, integrity_data in self.integrity.items():
|
||||
self._config.set('integrity', file, integrity_data)
|
||||
self._config.set('integrity', 'manifest', manifest.id_str)
|
||||
self._config.set("integrity", file, integrity_data)
|
||||
self._config.set("integrity", "manifest", manifest.id_str)
|
||||
if key:
|
||||
self._config.set('cache', 'key_type', str(key.TYPE))
|
||||
self._config.set("cache", "key_type", str(key.TYPE))
|
||||
with SaveFile(self.config_path) as fd:
|
||||
self._config.write(fd)
|
||||
|
||||
@ -327,20 +349,21 @@ def close(self):
|
||||
|
||||
def _check_upgrade(self, config_path):
|
||||
try:
|
||||
cache_version = self._config.getint('cache', 'version')
|
||||
cache_version = self._config.getint("cache", "version")
|
||||
wanted_version = 1
|
||||
if cache_version != wanted_version:
|
||||
self.close()
|
||||
raise Exception('%s has unexpected cache version %d (wanted: %d).' %
|
||||
(config_path, cache_version, wanted_version))
|
||||
raise Exception(
|
||||
"%s has unexpected cache version %d (wanted: %d)." % (config_path, cache_version, wanted_version)
|
||||
)
|
||||
except configparser.NoSectionError:
|
||||
self.close()
|
||||
raise Exception('%s does not look like a Borg cache.' % config_path) from None
|
||||
raise Exception("%s does not look like a Borg cache." % config_path) from None
|
||||
|
||||
|
||||
class Cache:
|
||||
"""Client Side cache
|
||||
"""
|
||||
"""Client Side cache"""
|
||||
|
||||
class RepositoryIDNotUnique(Error):
|
||||
"""Cache is newer than repository - do you have multiple, independently updated repos with same ID?"""
|
||||
|
||||
@ -359,29 +382,56 @@ class EncryptionMethodMismatch(Error):
|
||||
@staticmethod
|
||||
def break_lock(repository, path=None):
|
||||
path = cache_dir(repository, path)
|
||||
Lock(os.path.join(path, 'lock'), exclusive=True).break_lock()
|
||||
Lock(os.path.join(path, "lock"), exclusive=True).break_lock()
|
||||
|
||||
@staticmethod
|
||||
def destroy(repository, path=None):
|
||||
"""destroy the cache for ``repository`` or at ``path``"""
|
||||
path = path or os.path.join(get_cache_dir(), repository.id_str)
|
||||
config = os.path.join(path, 'config')
|
||||
config = os.path.join(path, "config")
|
||||
if os.path.exists(config):
|
||||
os.remove(config) # kill config first
|
||||
shutil.rmtree(path)
|
||||
|
||||
def __new__(cls, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
|
||||
progress=False, lock_wait=None, permit_adhoc_cache=False, cache_mode=FILES_CACHE_MODE_DISABLED,
|
||||
consider_part_files=False, iec=False):
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
repository,
|
||||
key,
|
||||
manifest,
|
||||
path=None,
|
||||
sync=True,
|
||||
warn_if_unencrypted=True,
|
||||
progress=False,
|
||||
lock_wait=None,
|
||||
permit_adhoc_cache=False,
|
||||
cache_mode=FILES_CACHE_MODE_DISABLED,
|
||||
consider_part_files=False,
|
||||
iec=False,
|
||||
):
|
||||
def local():
|
||||
return LocalCache(repository=repository, key=key, manifest=manifest, path=path, sync=sync,
|
||||
warn_if_unencrypted=warn_if_unencrypted, progress=progress, iec=iec,
|
||||
lock_wait=lock_wait, cache_mode=cache_mode, consider_part_files=consider_part_files)
|
||||
return LocalCache(
|
||||
repository=repository,
|
||||
key=key,
|
||||
manifest=manifest,
|
||||
path=path,
|
||||
sync=sync,
|
||||
warn_if_unencrypted=warn_if_unencrypted,
|
||||
progress=progress,
|
||||
iec=iec,
|
||||
lock_wait=lock_wait,
|
||||
cache_mode=cache_mode,
|
||||
consider_part_files=consider_part_files,
|
||||
)
|
||||
|
||||
def adhoc():
|
||||
return AdHocCache(repository=repository, key=key, manifest=manifest, lock_wait=lock_wait, iec=iec,
|
||||
consider_part_files=consider_part_files)
|
||||
return AdHocCache(
|
||||
repository=repository,
|
||||
key=key,
|
||||
manifest=manifest,
|
||||
lock_wait=lock_wait,
|
||||
iec=iec,
|
||||
consider_part_files=consider_part_files,
|
||||
)
|
||||
|
||||
if not permit_adhoc_cache:
|
||||
return local()
|
||||
@ -397,9 +447,9 @@ def adhoc():
|
||||
# Don't nest cache locks
|
||||
if cache_in_sync:
|
||||
# Local cache is in sync, use it
|
||||
logger.debug('Cache: choosing local cache (in sync)')
|
||||
logger.debug("Cache: choosing local cache (in sync)")
|
||||
return local()
|
||||
logger.debug('Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)')
|
||||
logger.debug("Cache: choosing ad-hoc cache (local cache does not exist or is not in sync)")
|
||||
return adhoc()
|
||||
|
||||
|
||||
@ -417,10 +467,11 @@ def __init__(self, iec=False):
|
||||
def __str__(self):
|
||||
return self.str_format.format(self.format_tuple())
|
||||
|
||||
Summary = namedtuple('Summary', ['total_size', 'unique_size', 'total_unique_chunks', 'total_chunks'])
|
||||
Summary = namedtuple("Summary", ["total_size", "unique_size", "total_unique_chunks", "total_chunks"])
|
||||
|
||||
def stats(self):
|
||||
from .archive import Archive
|
||||
|
||||
# XXX: this should really be moved down to `hashindex.pyx`
|
||||
total_size, unique_size, total_unique_chunks, total_chunks = self.chunks.summarize()
|
||||
# the above values have the problem that they do not consider part files,
|
||||
@ -430,8 +481,9 @@ def stats(self):
|
||||
# so we can just sum up all archives to get the "all archives" stats:
|
||||
total_size = 0
|
||||
for archive_name in self.manifest.archives:
|
||||
archive = Archive(self.repository, self.key, self.manifest, archive_name,
|
||||
consider_part_files=self.consider_part_files)
|
||||
archive = Archive(
|
||||
self.repository, self.key, self.manifest, archive_name, consider_part_files=self.consider_part_files
|
||||
)
|
||||
stats = archive.calc_stats(self, want_unique=False)
|
||||
total_size += stats.osize
|
||||
stats = self.Summary(total_size, unique_size, total_unique_chunks, total_chunks)._asdict()
|
||||
@ -439,7 +491,7 @@ def stats(self):
|
||||
|
||||
def format_tuple(self):
|
||||
stats = self.stats()
|
||||
for field in ['total_size', 'unique_size']:
|
||||
for field in ["total_size", "unique_size"]:
|
||||
stats[field] = format_file_size(stats[field], iec=self.iec)
|
||||
return self.Summary(**stats)
|
||||
|
||||
@ -449,9 +501,20 @@ class LocalCache(CacheStatsMixin):
|
||||
Persistent, local (client-side) cache.
|
||||
"""
|
||||
|
||||
def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True,
|
||||
progress=False, lock_wait=None, cache_mode=FILES_CACHE_MODE_DISABLED, consider_part_files=False,
|
||||
iec=False):
|
||||
def __init__(
|
||||
self,
|
||||
repository,
|
||||
key,
|
||||
manifest,
|
||||
path=None,
|
||||
sync=True,
|
||||
warn_if_unencrypted=True,
|
||||
progress=False,
|
||||
lock_wait=None,
|
||||
cache_mode=FILES_CACHE_MODE_DISABLED,
|
||||
consider_part_files=False,
|
||||
iec=False,
|
||||
):
|
||||
"""
|
||||
:param warn_if_unencrypted: print warning if accessing unknown unencrypted repository
|
||||
:param lock_wait: timeout for lock acquisition (int [s] or None [wait forever])
|
||||
@ -500,30 +563,32 @@ def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
|
||||
def create(self):
|
||||
"""Create a new empty cache at `self.path`
|
||||
"""
|
||||
"""Create a new empty cache at `self.path`"""
|
||||
os.makedirs(self.path)
|
||||
with open(os.path.join(self.path, 'README'), 'w') as fd:
|
||||
with open(os.path.join(self.path, "README"), "w") as fd:
|
||||
fd.write(CACHE_README)
|
||||
self.cache_config.create()
|
||||
ChunkIndex().write(os.path.join(self.path, 'chunks'))
|
||||
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
|
||||
ChunkIndex().write(os.path.join(self.path, "chunks"))
|
||||
os.makedirs(os.path.join(self.path, "chunks.archive.d"))
|
||||
with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
|
||||
pass # empty file
|
||||
|
||||
def _do_open(self):
|
||||
self.cache_config.load()
|
||||
with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=False,
|
||||
integrity_data=self.cache_config.integrity.get('chunks')) as fd:
|
||||
with IntegrityCheckedFile(
|
||||
path=os.path.join(self.path, "chunks"),
|
||||
write=False,
|
||||
integrity_data=self.cache_config.integrity.get("chunks"),
|
||||
) as fd:
|
||||
self.chunks = ChunkIndex.read(fd)
|
||||
if 'd' in self.cache_mode: # d(isabled)
|
||||
if "d" in self.cache_mode: # d(isabled)
|
||||
self.files = None
|
||||
else:
|
||||
self._read_files()
|
||||
|
||||
def open(self):
|
||||
if not os.path.isdir(self.path):
|
||||
raise Exception('%s Does not look like a Borg cache' % self.path)
|
||||
raise Exception("%s Does not look like a Borg cache" % self.path)
|
||||
self.cache_config.open()
|
||||
self.rollback()
|
||||
|
||||
@ -535,12 +600,15 @@ def close(self):
|
||||
def _read_files(self):
|
||||
self.files = {}
|
||||
self._newest_cmtime = None
|
||||
logger.debug('Reading files cache ...')
|
||||
logger.debug("Reading files cache ...")
|
||||
files_cache_logger.debug("FILES-CACHE-LOAD: starting...")
|
||||
msg = None
|
||||
try:
|
||||
with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=False,
|
||||
integrity_data=self.cache_config.integrity.get(files_cache_name())) as fd:
|
||||
with IntegrityCheckedFile(
|
||||
path=os.path.join(self.path, files_cache_name()),
|
||||
write=False,
|
||||
integrity_data=self.cache_config.integrity.get(files_cache_name()),
|
||||
) as fd:
|
||||
u = msgpack.Unpacker(use_list=True)
|
||||
while True:
|
||||
data = fd.read(64 * 1024)
|
||||
@ -561,43 +629,41 @@ def _read_files(self):
|
||||
msg = "The files cache is corrupted. [%s]" % str(fie)
|
||||
if msg is not None:
|
||||
logger.warning(msg)
|
||||
logger.warning('Continuing without files cache - expect lower performance.')
|
||||
logger.warning("Continuing without files cache - expect lower performance.")
|
||||
self.files = {}
|
||||
files_cache_logger.debug("FILES-CACHE-LOAD: finished, %d entries loaded.", len(self.files))
|
||||
|
||||
def begin_txn(self):
|
||||
# Initialize transaction snapshot
|
||||
pi = ProgressIndicatorMessage(msgid='cache.begin_transaction')
|
||||
txn_dir = os.path.join(self.path, 'txn.tmp')
|
||||
pi = ProgressIndicatorMessage(msgid="cache.begin_transaction")
|
||||
txn_dir = os.path.join(self.path, "txn.tmp")
|
||||
os.mkdir(txn_dir)
|
||||
pi.output('Initializing cache transaction: Reading config')
|
||||
shutil.copy(os.path.join(self.path, 'config'), txn_dir)
|
||||
pi.output('Initializing cache transaction: Reading chunks')
|
||||
shutil.copy(os.path.join(self.path, 'chunks'), txn_dir)
|
||||
pi.output('Initializing cache transaction: Reading files')
|
||||
pi.output("Initializing cache transaction: Reading config")
|
||||
shutil.copy(os.path.join(self.path, "config"), txn_dir)
|
||||
pi.output("Initializing cache transaction: Reading chunks")
|
||||
shutil.copy(os.path.join(self.path, "chunks"), txn_dir)
|
||||
pi.output("Initializing cache transaction: Reading files")
|
||||
try:
|
||||
shutil.copy(os.path.join(self.path, files_cache_name()), txn_dir)
|
||||
except FileNotFoundError:
|
||||
with SaveFile(os.path.join(txn_dir, files_cache_name()), binary=True):
|
||||
pass # empty file
|
||||
os.rename(os.path.join(self.path, 'txn.tmp'),
|
||||
os.path.join(self.path, 'txn.active'))
|
||||
os.rename(os.path.join(self.path, "txn.tmp"), os.path.join(self.path, "txn.active"))
|
||||
self.txn_active = True
|
||||
pi.finish()
|
||||
|
||||
def commit(self):
|
||||
"""Commit transaction
|
||||
"""
|
||||
"""Commit transaction"""
|
||||
if not self.txn_active:
|
||||
return
|
||||
self.security_manager.save(self.manifest, self.key)
|
||||
pi = ProgressIndicatorMessage(msgid='cache.commit')
|
||||
pi = ProgressIndicatorMessage(msgid="cache.commit")
|
||||
if self.files is not None:
|
||||
if self._newest_cmtime is None:
|
||||
# was never set because no files were modified/added
|
||||
self._newest_cmtime = 2 ** 63 - 1 # nanoseconds, good until y2262
|
||||
ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
|
||||
pi.output('Saving files cache')
|
||||
self._newest_cmtime = 2**63 - 1 # nanoseconds, good until y2262
|
||||
ttl = int(os.environ.get("BORG_FILES_CACHE_TTL", 20))
|
||||
pi.output("Saving files cache")
|
||||
files_cache_logger.debug("FILES-CACHE-SAVE: starting...")
|
||||
with IntegrityCheckedFile(path=os.path.join(self.path, files_cache_name()), write=True) as fd:
|
||||
entry_count = 0
|
||||
@ -606,41 +672,45 @@ def commit(self):
|
||||
# this is to avoid issues with filesystem snapshots and cmtime granularity.
|
||||
# Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
|
||||
entry = FileCacheEntry(*msgpack.unpackb(item))
|
||||
if entry.age == 0 and timestamp_to_int(entry.cmtime) < self._newest_cmtime or \
|
||||
entry.age > 0 and entry.age < ttl:
|
||||
if (
|
||||
entry.age == 0
|
||||
and timestamp_to_int(entry.cmtime) < self._newest_cmtime
|
||||
or entry.age > 0
|
||||
and entry.age < ttl
|
||||
):
|
||||
msgpack.pack((path_hash, entry), fd)
|
||||
entry_count += 1
|
||||
files_cache_logger.debug("FILES-CACHE-KILL: removed all old entries with age >= TTL [%d]", ttl)
|
||||
files_cache_logger.debug("FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime)
|
||||
files_cache_logger.debug(
|
||||
"FILES-CACHE-KILL: removed all current entries with newest cmtime %d", self._newest_cmtime
|
||||
)
|
||||
files_cache_logger.debug("FILES-CACHE-SAVE: finished, %d remaining entries saved.", entry_count)
|
||||
self.cache_config.integrity[files_cache_name()] = fd.integrity_data
|
||||
pi.output('Saving chunks cache')
|
||||
with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=True) as fd:
|
||||
pi.output("Saving chunks cache")
|
||||
with IntegrityCheckedFile(path=os.path.join(self.path, "chunks"), write=True) as fd:
|
||||
self.chunks.write(fd)
|
||||
self.cache_config.integrity['chunks'] = fd.integrity_data
|
||||
pi.output('Saving cache config')
|
||||
self.cache_config.integrity["chunks"] = fd.integrity_data
|
||||
pi.output("Saving cache config")
|
||||
self.cache_config.save(self.manifest, self.key)
|
||||
os.rename(os.path.join(self.path, 'txn.active'),
|
||||
os.path.join(self.path, 'txn.tmp'))
|
||||
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
|
||||
os.rename(os.path.join(self.path, "txn.active"), os.path.join(self.path, "txn.tmp"))
|
||||
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
|
||||
self.txn_active = False
|
||||
pi.finish()
|
||||
|
||||
def rollback(self):
|
||||
"""Roll back partial and aborted transactions
|
||||
"""
|
||||
"""Roll back partial and aborted transactions"""
|
||||
# Remove partial transaction
|
||||
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
|
||||
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
|
||||
if os.path.exists(os.path.join(self.path, "txn.tmp")):
|
||||
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
|
||||
# Roll back active transaction
|
||||
txn_dir = os.path.join(self.path, 'txn.active')
|
||||
txn_dir = os.path.join(self.path, "txn.active")
|
||||
if os.path.exists(txn_dir):
|
||||
shutil.copy(os.path.join(txn_dir, 'config'), self.path)
|
||||
shutil.copy(os.path.join(txn_dir, 'chunks'), self.path)
|
||||
shutil.copy(os.path.join(txn_dir, "config"), self.path)
|
||||
shutil.copy(os.path.join(txn_dir, "chunks"), self.path)
|
||||
shutil.copy(os.path.join(txn_dir, discover_files_cache_name(txn_dir)), self.path)
|
||||
os.rename(txn_dir, os.path.join(self.path, 'txn.tmp'))
|
||||
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
|
||||
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
|
||||
os.rename(txn_dir, os.path.join(self.path, "txn.tmp"))
|
||||
if os.path.exists(os.path.join(self.path, "txn.tmp")):
|
||||
shutil.rmtree(os.path.join(self.path, "txn.tmp"))
|
||||
self.txn_active = False
|
||||
self._do_open()
|
||||
|
||||
@ -654,13 +724,13 @@ def sync(self):
|
||||
get removed and a new master chunks index is built by merging all
|
||||
archive indexes.
|
||||
"""
|
||||
archive_path = os.path.join(self.path, 'chunks.archive.d')
|
||||
archive_path = os.path.join(self.path, "chunks.archive.d")
|
||||
# Instrumentation
|
||||
processed_item_metadata_bytes = 0
|
||||
processed_item_metadata_chunks = 0
|
||||
compact_chunks_archive_saved_space = 0
|
||||
|
||||
def mkpath(id, suffix=''):
|
||||
def mkpath(id, suffix=""):
|
||||
id_hex = bin_to_hex(id)
|
||||
path = os.path.join(archive_path, id_hex + suffix)
|
||||
return path
|
||||
@ -670,8 +740,9 @@ def cached_archives():
|
||||
fns = os.listdir(archive_path)
|
||||
# filenames with 64 hex digits == 256bit,
|
||||
# or compact indices which are 64 hex digits + ".compact"
|
||||
return {unhexlify(fn) for fn in fns if len(fn) == 64} | \
|
||||
{unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith('.compact')}
|
||||
return {unhexlify(fn) for fn in fns if len(fn) == 64} | {
|
||||
unhexlify(fn[:64]) for fn in fns if len(fn) == 72 and fn.endswith(".compact")
|
||||
}
|
||||
else:
|
||||
return set()
|
||||
|
||||
@ -685,14 +756,14 @@ def cleanup_outdated(ids):
|
||||
def cleanup_cached_archive(id, cleanup_compact=True):
|
||||
try:
|
||||
os.unlink(mkpath(id))
|
||||
os.unlink(mkpath(id) + '.integrity')
|
||||
os.unlink(mkpath(id) + ".integrity")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
if not cleanup_compact:
|
||||
return
|
||||
try:
|
||||
os.unlink(mkpath(id, suffix='.compact'))
|
||||
os.unlink(mkpath(id, suffix='.compact') + '.integrity')
|
||||
os.unlink(mkpath(id, suffix=".compact"))
|
||||
os.unlink(mkpath(id, suffix=".compact") + ".integrity")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
@ -703,7 +774,7 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx):
|
||||
chunk_idx.add(archive_id, 1, len(data))
|
||||
archive = ArchiveItem(internal_dict=msgpack.unpackb(data))
|
||||
if archive.version not in (1, 2): # legacy
|
||||
raise Exception('Unknown archive metadata version')
|
||||
raise Exception("Unknown archive metadata version")
|
||||
sync = CacheSynchronizer(chunk_idx)
|
||||
for item_id, (csize, data) in zip(archive.items, decrypted_repository.get_many(archive.items)):
|
||||
chunk_idx.add(item_id, 1, len(data))
|
||||
@ -716,11 +787,12 @@ def fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx):
|
||||
def write_archive_index(archive_id, chunk_idx):
|
||||
nonlocal compact_chunks_archive_saved_space
|
||||
compact_chunks_archive_saved_space += chunk_idx.compact()
|
||||
fn = mkpath(archive_id, suffix='.compact')
|
||||
fn_tmp = mkpath(archive_id, suffix='.tmp')
|
||||
fn = mkpath(archive_id, suffix=".compact")
|
||||
fn_tmp = mkpath(archive_id, suffix=".tmp")
|
||||
try:
|
||||
with DetachedIntegrityCheckedFile(path=fn_tmp, write=True,
|
||||
filename=bin_to_hex(archive_id) + '.compact') as fd:
|
||||
with DetachedIntegrityCheckedFile(
|
||||
path=fn_tmp, write=True, filename=bin_to_hex(archive_id) + ".compact"
|
||||
) as fd:
|
||||
chunk_idx.write(fd)
|
||||
except Exception:
|
||||
safe_unlink(fn_tmp)
|
||||
@ -733,7 +805,7 @@ def read_archive_index(archive_id, archive_name):
|
||||
try:
|
||||
try:
|
||||
# Attempt to load compact index first
|
||||
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + '.compact', write=False) as fd:
|
||||
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path + ".compact", write=False) as fd:
|
||||
archive_chunk_idx = ChunkIndex.read(fd, permit_compact=True)
|
||||
# In case a non-compact index exists, delete it.
|
||||
cleanup_cached_archive(archive_id, cleanup_compact=False)
|
||||
@ -744,14 +816,14 @@ def read_archive_index(archive_id, archive_name):
|
||||
with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
|
||||
archive_chunk_idx = ChunkIndex.read(fd)
|
||||
except FileIntegrityError as fie:
|
||||
logger.error('Cached archive chunk index of %s is corrupted: %s', archive_name, fie)
|
||||
logger.error("Cached archive chunk index of %s is corrupted: %s", archive_name, fie)
|
||||
# Delete corrupted index, set warning. A new index must be build.
|
||||
cleanup_cached_archive(archive_id)
|
||||
set_ec(EXIT_WARNING)
|
||||
return None
|
||||
|
||||
# Convert to compact index. Delete the existing index first.
|
||||
logger.debug('Found non-compact index for %s, converting to compact.', archive_name)
|
||||
logger.debug("Found non-compact index for %s, converting to compact.", archive_name)
|
||||
cleanup_cached_archive(archive_id)
|
||||
write_archive_index(archive_id, archive_chunk_idx)
|
||||
return archive_chunk_idx
|
||||
@ -769,12 +841,16 @@ def get_archive_ids_to_names(archive_ids):
|
||||
return archive_names
|
||||
|
||||
def create_master_idx(chunk_idx):
|
||||
logger.info('Synchronizing chunks cache...')
|
||||
logger.info("Synchronizing chunks cache...")
|
||||
cached_ids = cached_archives()
|
||||
archive_ids = repo_archives()
|
||||
logger.info('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.',
|
||||
len(archive_ids), len(cached_ids),
|
||||
len(cached_ids - archive_ids), len(archive_ids - cached_ids))
|
||||
logger.info(
|
||||
"Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.",
|
||||
len(archive_ids),
|
||||
len(cached_ids),
|
||||
len(cached_ids - archive_ids),
|
||||
len(archive_ids - cached_ids),
|
||||
)
|
||||
# deallocates old hashindex, creates empty hashindex:
|
||||
chunk_idx.clear()
|
||||
cleanup_outdated(cached_ids - archive_ids)
|
||||
@ -783,9 +859,12 @@ def create_master_idx(chunk_idx):
|
||||
master_index_capacity = len(self.repository)
|
||||
if archive_ids:
|
||||
chunk_idx = None if not self.do_cache else ChunkIndex(usable=master_index_capacity)
|
||||
pi = ProgressIndicatorPercent(total=len(archive_ids), step=0.1,
|
||||
msg='%3.0f%% Syncing chunks cache. Processing archive %s',
|
||||
msgid='cache.sync')
|
||||
pi = ProgressIndicatorPercent(
|
||||
total=len(archive_ids),
|
||||
step=0.1,
|
||||
msg="%3.0f%% Syncing chunks cache. Processing archive %s",
|
||||
msgid="cache.sync",
|
||||
)
|
||||
archive_ids_to_names = get_archive_ids_to_names(archive_ids)
|
||||
for archive_id, archive_name in archive_ids_to_names.items():
|
||||
pi.show(info=[remove_surrogates(archive_name)])
|
||||
@ -797,31 +876,36 @@ def create_master_idx(chunk_idx):
|
||||
if archive_id not in cached_ids:
|
||||
# Do not make this an else branch; the FileIntegrityError exception handler
|
||||
# above can remove *archive_id* from *cached_ids*.
|
||||
logger.info('Fetching and building archive index for %s ...', archive_name)
|
||||
logger.info("Fetching and building archive index for %s ...", archive_name)
|
||||
archive_chunk_idx = ChunkIndex()
|
||||
fetch_and_build_idx(archive_id, decrypted_repository, archive_chunk_idx)
|
||||
logger.info("Merging into master chunks index ...")
|
||||
chunk_idx.merge(archive_chunk_idx)
|
||||
else:
|
||||
chunk_idx = chunk_idx or ChunkIndex(usable=master_index_capacity)
|
||||
logger.info('Fetching archive index for %s ...', archive_name)
|
||||
logger.info("Fetching archive index for %s ...", archive_name)
|
||||
fetch_and_build_idx(archive_id, decrypted_repository, chunk_idx)
|
||||
pi.finish()
|
||||
logger.debug('Cache sync: processed %s (%d chunks) of metadata',
|
||||
format_file_size(processed_item_metadata_bytes), processed_item_metadata_chunks)
|
||||
logger.debug('Cache sync: compact chunks.archive.d storage saved %s bytes',
|
||||
format_file_size(compact_chunks_archive_saved_space))
|
||||
logger.info('Done.')
|
||||
logger.debug(
|
||||
"Cache sync: processed %s (%d chunks) of metadata",
|
||||
format_file_size(processed_item_metadata_bytes),
|
||||
processed_item_metadata_chunks,
|
||||
)
|
||||
logger.debug(
|
||||
"Cache sync: compact chunks.archive.d storage saved %s bytes",
|
||||
format_file_size(compact_chunks_archive_saved_space),
|
||||
)
|
||||
logger.info("Done.")
|
||||
return chunk_idx
|
||||
|
||||
def legacy_cleanup():
|
||||
"""bring old cache dirs into the desired state (cleanup and adapt)"""
|
||||
try:
|
||||
os.unlink(os.path.join(self.path, 'chunks.archive'))
|
||||
os.unlink(os.path.join(self.path, "chunks.archive"))
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
os.unlink(os.path.join(self.path, 'chunks.archive.tmp'))
|
||||
os.unlink(os.path.join(self.path, "chunks.archive.tmp"))
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
@ -832,7 +916,7 @@ def legacy_cleanup():
|
||||
# The cache can be used by a command that e.g. only checks against Manifest.Operation.WRITE,
|
||||
# which does not have to include all flags from Manifest.Operation.READ.
|
||||
# Since the sync will attempt to read archives, check compatibility with Manifest.Operation.READ.
|
||||
self.manifest.check_repository_compatibility((Manifest.Operation.READ, ))
|
||||
self.manifest.check_repository_compatibility((Manifest.Operation.READ,))
|
||||
|
||||
self.begin_txn()
|
||||
with cache_if_remote(self.repository, decrypted_cache=self.key) as decrypted_repository:
|
||||
@ -856,15 +940,15 @@ def check_cache_compatibility(self):
|
||||
|
||||
def wipe_cache(self):
|
||||
logger.warning("Discarding incompatible cache and forcing a cache rebuild")
|
||||
archive_path = os.path.join(self.path, 'chunks.archive.d')
|
||||
archive_path = os.path.join(self.path, "chunks.archive.d")
|
||||
if os.path.isdir(archive_path):
|
||||
shutil.rmtree(os.path.join(self.path, 'chunks.archive.d'))
|
||||
os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
|
||||
shutil.rmtree(os.path.join(self.path, "chunks.archive.d"))
|
||||
os.makedirs(os.path.join(self.path, "chunks.archive.d"))
|
||||
self.chunks = ChunkIndex()
|
||||
with SaveFile(os.path.join(self.path, files_cache_name()), binary=True):
|
||||
pass # empty file
|
||||
self.cache_config.manifest_id = ''
|
||||
self.cache_config._config.set('cache', 'manifest', '')
|
||||
self.cache_config.manifest_id = ""
|
||||
self.cache_config._config.set("cache", "manifest", "")
|
||||
|
||||
self.cache_config.ignored_features = set()
|
||||
self.cache_config.mandatory_features = set()
|
||||
@ -900,8 +984,9 @@ def seen_chunk(self, id, size=None):
|
||||
if size is not None and stored_size is not None and size != stored_size:
|
||||
# we already have a chunk with that id, but different size.
|
||||
# this is either a hash collision (unlikely) or corruption or a bug.
|
||||
raise Exception("chunk has same id [%r], but different size (stored: %d new: %d)!" % (
|
||||
id, stored_size, size))
|
||||
raise Exception(
|
||||
"chunk has same id [%r], but different size (stored: %d new: %d)!" % (id, stored_size, size)
|
||||
)
|
||||
return refcount
|
||||
|
||||
def chunk_incref(self, id, stats, size=None, part=False):
|
||||
@ -936,32 +1021,32 @@ def file_known_and_unchanged(self, hashed_path, path_hash, st):
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
return False, None
|
||||
cache_mode = self.cache_mode
|
||||
if 'd' in cache_mode: # d(isabled)
|
||||
files_cache_logger.debug('UNKNOWN: files cache disabled')
|
||||
if "d" in cache_mode: # d(isabled)
|
||||
files_cache_logger.debug("UNKNOWN: files cache disabled")
|
||||
return False, None
|
||||
# note: r(echunk) does not need the files cache in this method, but the files cache will
|
||||
# be updated and saved to disk to memorize the files. To preserve previous generations in
|
||||
# the cache, this means that it also needs to get loaded from disk first.
|
||||
if 'r' in cache_mode: # r(echunk)
|
||||
files_cache_logger.debug('UNKNOWN: rechunking enforced')
|
||||
if "r" in cache_mode: # r(echunk)
|
||||
files_cache_logger.debug("UNKNOWN: rechunking enforced")
|
||||
return False, None
|
||||
entry = self.files.get(path_hash)
|
||||
if not entry:
|
||||
files_cache_logger.debug('UNKNOWN: no file metadata in cache for: %r', hashed_path)
|
||||
files_cache_logger.debug("UNKNOWN: no file metadata in cache for: %r", hashed_path)
|
||||
return False, None
|
||||
# we know the file!
|
||||
entry = FileCacheEntry(*msgpack.unpackb(entry))
|
||||
if 's' in cache_mode and entry.size != st.st_size:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file size has changed: %r', hashed_path)
|
||||
if "s" in cache_mode and entry.size != st.st_size:
|
||||
files_cache_logger.debug("KNOWN-CHANGED: file size has changed: %r", hashed_path)
|
||||
return True, None
|
||||
if 'i' in cache_mode and entry.inode != st.st_ino:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file inode number has changed: %r', hashed_path)
|
||||
if "i" in cache_mode and entry.inode != st.st_ino:
|
||||
files_cache_logger.debug("KNOWN-CHANGED: file inode number has changed: %r", hashed_path)
|
||||
return True, None
|
||||
if 'c' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file ctime has changed: %r', hashed_path)
|
||||
if "c" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns:
|
||||
files_cache_logger.debug("KNOWN-CHANGED: file ctime has changed: %r", hashed_path)
|
||||
return True, None
|
||||
elif 'm' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file mtime has changed: %r', hashed_path)
|
||||
elif "m" in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns:
|
||||
files_cache_logger.debug("KNOWN-CHANGED: file mtime has changed: %r", hashed_path)
|
||||
return True, None
|
||||
# we ignored the inode number in the comparison above or it is still same.
|
||||
# if it is still the same, replacing it in the tuple doesn't change it.
|
||||
@ -979,21 +1064,26 @@ def memorize_file(self, hashed_path, path_hash, st, ids):
|
||||
return
|
||||
cache_mode = self.cache_mode
|
||||
# note: r(echunk) modes will update the files cache, d(isabled) mode won't
|
||||
if 'd' in cache_mode:
|
||||
files_cache_logger.debug('FILES-CACHE-NOUPDATE: files cache disabled')
|
||||
if "d" in cache_mode:
|
||||
files_cache_logger.debug("FILES-CACHE-NOUPDATE: files cache disabled")
|
||||
return
|
||||
if 'c' in cache_mode:
|
||||
cmtime_type = 'ctime'
|
||||
if "c" in cache_mode:
|
||||
cmtime_type = "ctime"
|
||||
cmtime_ns = safe_ns(st.st_ctime_ns)
|
||||
elif 'm' in cache_mode:
|
||||
cmtime_type = 'mtime'
|
||||
elif "m" in cache_mode:
|
||||
cmtime_type = "mtime"
|
||||
cmtime_ns = safe_ns(st.st_mtime_ns)
|
||||
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids)
|
||||
entry = FileCacheEntry(
|
||||
age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids
|
||||
)
|
||||
self.files[path_hash] = msgpack.packb(entry)
|
||||
self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns)
|
||||
files_cache_logger.debug('FILES-CACHE-UPDATE: put %r [has %s] <- %r',
|
||||
entry._replace(chunk_ids='[%d entries]' % len(entry.chunk_ids)),
|
||||
cmtime_type, hashed_path)
|
||||
files_cache_logger.debug(
|
||||
"FILES-CACHE-UPDATE: put %r [has %s] <- %r",
|
||||
entry._replace(chunk_ids="[%d entries]" % len(entry.chunk_ids)),
|
||||
cmtime_type,
|
||||
hashed_path,
|
||||
)
|
||||
|
||||
|
||||
class AdHocCache(CacheStatsMixin):
|
||||
@ -1012,8 +1102,9 @@ class AdHocCache(CacheStatsMixin):
|
||||
Unique chunks Total chunks
|
||||
Chunk index: {0.total_unique_chunks:20d} unknown"""
|
||||
|
||||
def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False,
|
||||
iec=False):
|
||||
def __init__(
|
||||
self, repository, key, manifest, warn_if_unencrypted=True, lock_wait=None, consider_part_files=False, iec=False
|
||||
):
|
||||
CacheStatsMixin.__init__(self, iec=iec)
|
||||
self.repository = repository
|
||||
self.key = key
|
||||
@ -1024,7 +1115,7 @@ def __init__(self, repository, key, manifest, warn_if_unencrypted=True, lock_wai
|
||||
self.security_manager = SecurityManager(repository)
|
||||
self.security_manager.assert_secure(manifest, key, lock_wait=lock_wait)
|
||||
|
||||
logger.warning('Note: --no-cache-sync is an experimental feature.')
|
||||
logger.warning("Note: --no-cache-sync is an experimental feature.")
|
||||
|
||||
# Public API
|
||||
|
||||
@ -1035,7 +1126,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
files = None
|
||||
cache_mode = 'd'
|
||||
cache_mode = "d"
|
||||
|
||||
def file_known_and_unchanged(self, hashed_path, path_hash, st):
|
||||
files_cache_logger.debug("UNKNOWN: files cache not implemented")
|
||||
@ -1045,7 +1136,7 @@ def memorize_file(self, hashed_path, path_hash, st, ids):
|
||||
pass
|
||||
|
||||
def add_chunk(self, id, chunk, stats, *, overwrite=False, wait=True, compress=True, size=None):
|
||||
assert not overwrite, 'AdHocCache does not permit overwrites — trying to use it for recreate?'
|
||||
assert not overwrite, "AdHocCache does not permit overwrites — trying to use it for recreate?"
|
||||
if not self._txn_active:
|
||||
self.begin_txn()
|
||||
if size is None and compress:
|
||||
@ -1111,8 +1202,9 @@ def begin_txn(self):
|
||||
# Since we're creating an archive, add 10 % from the start.
|
||||
num_chunks = len(self.repository)
|
||||
self.chunks = ChunkIndex(usable=num_chunks * 1.1)
|
||||
pi = ProgressIndicatorPercent(total=num_chunks, msg='Downloading chunk list... %3.0f%%',
|
||||
msgid='cache.download_chunks')
|
||||
pi = ProgressIndicatorPercent(
|
||||
total=num_chunks, msg="Downloading chunk list... %3.0f%%", msgid="cache.download_chunks"
|
||||
)
|
||||
t0 = perf_counter()
|
||||
num_requests = 0
|
||||
marker = None
|
||||
@ -1134,7 +1226,12 @@ def begin_txn(self):
|
||||
del self.chunks[self.manifest.MANIFEST_ID]
|
||||
duration = perf_counter() - t0 or 0.01
|
||||
pi.finish()
|
||||
logger.debug('AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s',
|
||||
num_chunks, duration, num_requests, format_file_size(num_chunks * 34 / duration))
|
||||
logger.debug(
|
||||
"AdHocCache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s",
|
||||
num_chunks,
|
||||
duration,
|
||||
num_requests,
|
||||
format_file_size(num_chunks * 34 / duration),
|
||||
)
|
||||
# Chunk IDs in a list are encoded in 34 bytes: 1 byte msgpack header, 1 byte length, 32 ID bytes.
|
||||
# Protocol overhead is neglected in this calculation.
|
||||
|
@ -7,7 +7,7 @@
|
||||
# fmt: on
|
||||
|
||||
# this is the set of keys that are always present in items:
|
||||
REQUIRED_ITEM_KEYS = frozenset(['path', 'mtime', ])
|
||||
REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"])
|
||||
|
||||
# this set must be kept complete, otherwise rebuild_manifest might malfunction:
|
||||
# fmt: off
|
||||
@ -19,7 +19,7 @@
|
||||
# fmt: on
|
||||
|
||||
# this is the set of keys that are always present in archives:
|
||||
REQUIRED_ARCHIVE_KEYS = frozenset(['version', 'name', 'items', 'cmdline', 'time', ])
|
||||
REQUIRED_ARCHIVE_KEYS = frozenset(["version", "name", "items", "cmdline", "time"])
|
||||
|
||||
# default umask, overridden by --umask, defaults to read/write only for owner
|
||||
UMASK_DEFAULT = 0o077
|
||||
@ -28,8 +28,8 @@
|
||||
# forcing to 0o100XXX later
|
||||
STDIN_MODE_DEFAULT = 0o660
|
||||
|
||||
CACHE_TAG_NAME = 'CACHEDIR.TAG'
|
||||
CACHE_TAG_CONTENTS = b'Signature: 8a477f597d28d172789f06886806bc55'
|
||||
CACHE_TAG_NAME = "CACHEDIR.TAG"
|
||||
CACHE_TAG_CONTENTS = b"Signature: 8a477f597d28d172789f06886806bc55"
|
||||
|
||||
# A large, but not unreasonably large segment size. Always less than 2 GiB (for legacy file systems). We choose
|
||||
# 500 MiB which means that no indirection from the inode is needed for typical Linux file systems.
|
||||
@ -48,7 +48,7 @@
|
||||
MAX_OBJECT_SIZE = MAX_DATA_SIZE + 41 + 8 # see assertion at end of repository module
|
||||
|
||||
# repo config max_segment_size value must be below this limit to stay within uint32 offsets:
|
||||
MAX_SEGMENT_SIZE_LIMIT = 2 ** 32 - MAX_OBJECT_SIZE
|
||||
MAX_SEGMENT_SIZE_LIMIT = 2**32 - MAX_OBJECT_SIZE
|
||||
|
||||
# have one all-zero bytes object
|
||||
# we use it at all places where we need to detect or create all-zero buffers
|
||||
@ -71,12 +71,12 @@
|
||||
|
||||
CHUNK_MIN_EXP = 19 # 2**19 == 512kiB
|
||||
CHUNK_MAX_EXP = 23 # 2**23 == 8MiB
|
||||
HASH_WINDOW_SIZE = 0xfff # 4095B
|
||||
HASH_WINDOW_SIZE = 0xFFF # 4095B
|
||||
HASH_MASK_BITS = 21 # results in ~2MiB chunks statistically
|
||||
|
||||
# chunker algorithms
|
||||
CH_BUZHASH = 'buzhash'
|
||||
CH_FIXED = 'fixed'
|
||||
CH_BUZHASH = "buzhash"
|
||||
CH_FIXED = "fixed"
|
||||
|
||||
# defaults, use --chunker-params to override
|
||||
CHUNKER_PARAMS = (CH_BUZHASH, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)
|
||||
@ -88,8 +88,8 @@
|
||||
CH_DATA, CH_ALLOC, CH_HOLE = 0, 1, 2
|
||||
|
||||
# operating mode of the files cache (for fast skipping of unchanged files)
|
||||
FILES_CACHE_MODE_UI_DEFAULT = 'ctime,size,inode' # default for "borg create" command (CLI UI)
|
||||
FILES_CACHE_MODE_DISABLED = 'd' # most borg commands do not use the files cache at all (disable)
|
||||
FILES_CACHE_MODE_UI_DEFAULT = "ctime,size,inode" # default for "borg create" command (CLI UI)
|
||||
FILES_CACHE_MODE_DISABLED = "d" # most borg commands do not use the files cache at all (disable)
|
||||
|
||||
# return codes returned by borg command
|
||||
# when borg is killed by signal N, rc = 128 + N
|
||||
@ -101,30 +101,30 @@
|
||||
# never use datetime.isoformat(), it is evil. always use one of these:
|
||||
# datetime.strftime(ISO_FORMAT) # output always includes .microseconds
|
||||
# datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds
|
||||
ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S'
|
||||
ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f'
|
||||
ISO_FORMAT_NO_USECS = "%Y-%m-%dT%H:%M:%S"
|
||||
ISO_FORMAT = ISO_FORMAT_NO_USECS + ".%f"
|
||||
|
||||
DASHES = '-' * 78
|
||||
DASHES = "-" * 78
|
||||
|
||||
PBKDF2_ITERATIONS = 100000
|
||||
|
||||
# https://www.rfc-editor.org/rfc/rfc9106.html#section-4-6.2
|
||||
ARGON2_ARGS = {'time_cost': 3, 'memory_cost': 2**16, 'parallelism': 4, 'type': 'id'}
|
||||
ARGON2_ARGS = {"time_cost": 3, "memory_cost": 2**16, "parallelism": 4, "type": "id"}
|
||||
ARGON2_SALT_BYTES = 16
|
||||
|
||||
# Maps the CLI argument to our internal identifier for the format
|
||||
KEY_ALGORITHMS = {
|
||||
# encrypt-and-MAC, kdf: PBKDF2(HMAC−SHA256), encryption: AES256-CTR, authentication: HMAC-SHA256
|
||||
'pbkdf2': 'sha256',
|
||||
"pbkdf2": "sha256",
|
||||
# encrypt-then-MAC, kdf: argon2, encryption: chacha20, authentication: poly1305
|
||||
'argon2': 'argon2 chacha20-poly1305',
|
||||
"argon2": "argon2 chacha20-poly1305",
|
||||
}
|
||||
|
||||
|
||||
class KeyBlobStorage:
|
||||
NO_STORAGE = 'no_storage'
|
||||
KEYFILE = 'keyfile'
|
||||
REPO = 'repository'
|
||||
NO_STORAGE = "no_storage"
|
||||
KEYFILE = "keyfile"
|
||||
REPO = "repository"
|
||||
|
||||
|
||||
class KeyType:
|
||||
|
@ -102,12 +102,12 @@ def hash_length(self, seek_to_end=False):
|
||||
|
||||
|
||||
class SHA512FileHashingWrapper(FileHashingWrapper):
|
||||
ALGORITHM = 'SHA512'
|
||||
ALGORITHM = "SHA512"
|
||||
FACTORY = hashlib.sha512
|
||||
|
||||
|
||||
class XXH64FileHashingWrapper(FileHashingWrapper):
|
||||
ALGORITHM = 'XXH64'
|
||||
ALGORITHM = "XXH64"
|
||||
FACTORY = StreamingXXH64
|
||||
|
||||
|
||||
@ -125,7 +125,7 @@ class IntegrityCheckedFile(FileLikeWrapper):
|
||||
def __init__(self, path, write, filename=None, override_fd=None, integrity_data=None):
|
||||
self.path = path
|
||||
self.writing = write
|
||||
mode = 'wb' if write else 'rb'
|
||||
mode = "wb" if write else "rb"
|
||||
self.file_fd = override_fd or open(path, mode)
|
||||
self.digests = {}
|
||||
|
||||
@ -155,7 +155,7 @@ def hash_filename(self, filename=None):
|
||||
# While Borg does not use anything except ASCII in these file names, it's important to use
|
||||
# the same encoding everywhere for portability. Using os.fsencode() would be wrong.
|
||||
filename = os.path.basename(filename or self.path)
|
||||
self.hasher.update(('%10d' % len(filename)).encode())
|
||||
self.hasher.update(("%10d" % len(filename)).encode())
|
||||
self.hasher.update(filename.encode())
|
||||
|
||||
@classmethod
|
||||
@ -163,44 +163,41 @@ def parse_integrity_data(cls, path: str, data: str):
|
||||
try:
|
||||
integrity_data = json.loads(data)
|
||||
# Provisions for agility now, implementation later, but make sure the on-disk joint is oiled.
|
||||
algorithm = integrity_data['algorithm']
|
||||
algorithm = integrity_data["algorithm"]
|
||||
if algorithm not in SUPPORTED_ALGORITHMS:
|
||||
logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm)
|
||||
logger.warning("Cannot verify integrity of %s: Unknown algorithm %r", path, algorithm)
|
||||
return
|
||||
digests = integrity_data['digests']
|
||||
digests = integrity_data["digests"]
|
||||
# Require at least presence of the final digest
|
||||
digests['final']
|
||||
digests["final"]
|
||||
return algorithm, digests
|
||||
except (ValueError, TypeError, KeyError) as e:
|
||||
logger.warning('Could not parse integrity data for %s: %s', path, e)
|
||||
logger.warning("Could not parse integrity data for %s: %s", path, e)
|
||||
raise FileIntegrityError(path)
|
||||
|
||||
def hash_part(self, partname, is_final=False):
|
||||
if not self.writing and not self.digests:
|
||||
return
|
||||
self.hasher.update(('%10d' % len(partname)).encode())
|
||||
self.hasher.update(("%10d" % len(partname)).encode())
|
||||
self.hasher.update(partname.encode())
|
||||
self.hasher.hash_length(seek_to_end=is_final)
|
||||
digest = self.hasher.hexdigest()
|
||||
if self.writing:
|
||||
self.digests[partname] = digest
|
||||
elif self.digests and not compare_digest(self.digests.get(partname, ''), digest):
|
||||
elif self.digests and not compare_digest(self.digests.get(partname, ""), digest):
|
||||
raise FileIntegrityError(self.path)
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
exception = exc_type is not None
|
||||
if not exception:
|
||||
self.hash_part('final', is_final=True)
|
||||
self.hash_part("final", is_final=True)
|
||||
self.hasher.__exit__(exc_type, exc_val, exc_tb)
|
||||
if exception:
|
||||
return
|
||||
if self.writing:
|
||||
self.store_integrity_data(json.dumps({
|
||||
'algorithm': self.hasher.ALGORITHM,
|
||||
'digests': self.digests,
|
||||
}))
|
||||
self.store_integrity_data(json.dumps({"algorithm": self.hasher.ALGORITHM, "digests": self.digests}))
|
||||
elif self.digests:
|
||||
logger.debug('Verified integrity of %s', self.path)
|
||||
logger.debug("Verified integrity of %s", self.path)
|
||||
|
||||
def store_integrity_data(self, data: str):
|
||||
self.integrity_data = data
|
||||
@ -214,12 +211,12 @@ def __init__(self, path, write, filename=None, override_fd=None):
|
||||
self.output_integrity_file = self.integrity_file_path(os.path.join(output_dir, filename))
|
||||
|
||||
def load_integrity_data(self, path, integrity_data):
|
||||
assert not integrity_data, 'Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile'
|
||||
assert not integrity_data, "Cannot pass explicit integrity_data to DetachedIntegrityCheckedFile"
|
||||
return self.read_integrity_file(self.path)
|
||||
|
||||
@staticmethod
|
||||
def integrity_file_path(path):
|
||||
return path + '.integrity'
|
||||
return path + ".integrity"
|
||||
|
||||
@classmethod
|
||||
def read_integrity_file(cls, path):
|
||||
@ -227,11 +224,11 @@ def read_integrity_file(cls, path):
|
||||
with open(cls.integrity_file_path(path)) as fd:
|
||||
return cls.parse_integrity_data(path, fd.read())
|
||||
except FileNotFoundError:
|
||||
logger.info('No integrity file found for %s', path)
|
||||
logger.info("No integrity file found for %s", path)
|
||||
except OSError as e:
|
||||
logger.warning('Could not read integrity file for %s: %s', path, e)
|
||||
logger.warning("Could not read integrity file for %s: %s", path, e)
|
||||
raise FileIntegrityError(path)
|
||||
|
||||
def store_integrity_data(self, data: str):
|
||||
with open(self.output_integrity_file, 'w') as fd:
|
||||
with open(self.output_integrity_file, "w") as fd:
|
||||
fd.write(data)
|
||||
|
@ -59,9 +59,11 @@ class UnsupportedKeyFormatError(Error):
|
||||
|
||||
|
||||
class TAMRequiredError(IntegrityError):
|
||||
__doc__ = textwrap.dedent("""
|
||||
__doc__ = textwrap.dedent(
|
||||
"""
|
||||
Manifest is unauthenticated, but it is required for this repository. Is somebody attacking you?
|
||||
""").strip()
|
||||
"""
|
||||
).strip()
|
||||
traceback = False
|
||||
|
||||
|
||||
@ -71,11 +73,12 @@ class TAMInvalid(IntegrityError):
|
||||
|
||||
def __init__(self):
|
||||
# Error message becomes: "Data integrity error: Manifest authentication did not verify"
|
||||
super().__init__('Manifest authentication did not verify')
|
||||
super().__init__("Manifest authentication did not verify")
|
||||
|
||||
|
||||
class TAMUnsupportedSuiteError(IntegrityError):
|
||||
"""Could not verify manifest: Unsupported suite {!r}; a newer version is needed."""
|
||||
|
||||
traceback = False
|
||||
|
||||
|
||||
@ -110,7 +113,7 @@ def key_factory(repository, manifest_data):
|
||||
|
||||
def tam_required_file(repository):
|
||||
security_dir = get_security_dir(bin_to_hex(repository.id))
|
||||
return os.path.join(security_dir, 'tam_required')
|
||||
return os.path.join(security_dir, "tam_required")
|
||||
|
||||
|
||||
def tam_required(repository):
|
||||
@ -126,9 +129,10 @@ def uses_same_id_hash(other_key, key):
|
||||
old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey)
|
||||
new_blake2_ids = (Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey)
|
||||
same_ids = (
|
||||
isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids) and isinstance(key, new_hmac_sha256_ids)
|
||||
or
|
||||
isinstance(other_key, old_blake2_ids + new_blake2_ids) and isinstance(key, new_blake2_ids)
|
||||
isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids)
|
||||
and isinstance(key, new_hmac_sha256_ids)
|
||||
or isinstance(other_key, old_blake2_ids + new_blake2_ids)
|
||||
and isinstance(key, new_blake2_ids)
|
||||
)
|
||||
return same_ids
|
||||
|
||||
@ -140,10 +144,10 @@ class KeyBase:
|
||||
TYPES_ACCEPTABLE = None # override in subclasses
|
||||
|
||||
# Human-readable name
|
||||
NAME = 'UNDEFINED'
|
||||
NAME = "UNDEFINED"
|
||||
|
||||
# Name used in command line / API (e.g. borg init --encryption=...)
|
||||
ARG_NAME = 'UNDEFINED'
|
||||
ARG_NAME = "UNDEFINED"
|
||||
|
||||
# Storage type (no key blob storage / keyfile / repo)
|
||||
STORAGE = KeyBlobStorage.NO_STORAGE
|
||||
@ -167,13 +171,12 @@ def __init__(self, repository):
|
||||
self.target = None # key location file path / repo obj
|
||||
# Some commands write new chunks (e.g. rename) but don't take a --compression argument. This duplicates
|
||||
# the default used by those commands who do take a --compression argument.
|
||||
self.compressor = Compressor('lz4')
|
||||
self.compressor = Compressor("lz4")
|
||||
self.decompress = self.compressor.decompress
|
||||
self.tam_required = True
|
||||
|
||||
def id_hash(self, data):
|
||||
"""Return HMAC hash using the "id" HMAC key
|
||||
"""
|
||||
"""Return HMAC hash using the "id" HMAC key"""
|
||||
raise NotImplementedError
|
||||
|
||||
def encrypt(self, id, data, compress=True):
|
||||
@ -186,83 +189,79 @@ def assert_id(self, id, data):
|
||||
if id and id != Manifest.MANIFEST_ID:
|
||||
id_computed = self.id_hash(data)
|
||||
if not hmac.compare_digest(id_computed, id):
|
||||
raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
|
||||
raise IntegrityError("Chunk %s: id verification failed" % bin_to_hex(id))
|
||||
|
||||
def assert_type(self, type_byte, id=None):
|
||||
if type_byte not in self.TYPES_ACCEPTABLE:
|
||||
id_str = bin_to_hex(id) if id is not None else '(unknown)'
|
||||
raise IntegrityError(f'Chunk {id_str}: Invalid encryption envelope')
|
||||
id_str = bin_to_hex(id) if id is not None else "(unknown)"
|
||||
raise IntegrityError(f"Chunk {id_str}: Invalid encryption envelope")
|
||||
|
||||
def _tam_key(self, salt, context):
|
||||
return hkdf_hmac_sha512(
|
||||
ikm=self.id_key + self.enc_key + self.enc_hmac_key,
|
||||
salt=salt,
|
||||
info=b'borg-metadata-authentication-' + context,
|
||||
output_length=64
|
||||
info=b"borg-metadata-authentication-" + context,
|
||||
output_length=64,
|
||||
)
|
||||
|
||||
def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'):
|
||||
def pack_and_authenticate_metadata(self, metadata_dict, context=b"manifest"):
|
||||
metadata_dict = StableDict(metadata_dict)
|
||||
tam = metadata_dict['tam'] = StableDict({
|
||||
'type': 'HKDF_HMAC_SHA512',
|
||||
'hmac': bytes(64),
|
||||
'salt': os.urandom(64),
|
||||
})
|
||||
tam = metadata_dict["tam"] = StableDict({"type": "HKDF_HMAC_SHA512", "hmac": bytes(64), "salt": os.urandom(64)})
|
||||
packed = msgpack.packb(metadata_dict)
|
||||
tam_key = self._tam_key(tam['salt'], context)
|
||||
tam['hmac'] = hmac.digest(tam_key, packed, 'sha512')
|
||||
tam_key = self._tam_key(tam["salt"], context)
|
||||
tam["hmac"] = hmac.digest(tam_key, packed, "sha512")
|
||||
return msgpack.packb(metadata_dict)
|
||||
|
||||
def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
|
||||
"""Unpack msgpacked *data* and return (object, did_verify)."""
|
||||
if data.startswith(b'\xc1' * 4):
|
||||
if data.startswith(b"\xc1" * 4):
|
||||
# This is a manifest from the future, we can't read it.
|
||||
raise UnsupportedManifestError()
|
||||
tam_required = self.tam_required
|
||||
if force_tam_not_required and tam_required:
|
||||
logger.warning('Manifest authentication DISABLED.')
|
||||
logger.warning("Manifest authentication DISABLED.")
|
||||
tam_required = False
|
||||
data = bytearray(data)
|
||||
unpacker = get_limited_unpacker('manifest')
|
||||
unpacker = get_limited_unpacker("manifest")
|
||||
unpacker.feed(data)
|
||||
unpacked = unpacker.unpack()
|
||||
if 'tam' not in unpacked:
|
||||
if "tam" not in unpacked:
|
||||
if tam_required:
|
||||
raise TAMRequiredError(self.repository._location.canonical_path())
|
||||
else:
|
||||
logger.debug('TAM not found and not required')
|
||||
logger.debug("TAM not found and not required")
|
||||
return unpacked, False
|
||||
tam = unpacked.pop('tam', None)
|
||||
tam = unpacked.pop("tam", None)
|
||||
if not isinstance(tam, dict):
|
||||
raise TAMInvalid()
|
||||
tam_type = tam.get('type', '<none>')
|
||||
if tam_type != 'HKDF_HMAC_SHA512':
|
||||
tam_type = tam.get("type", "<none>")
|
||||
if tam_type != "HKDF_HMAC_SHA512":
|
||||
if tam_required:
|
||||
raise TAMUnsupportedSuiteError(repr(tam_type))
|
||||
else:
|
||||
logger.debug('Ignoring TAM made with unsupported suite, since TAM is not required: %r', tam_type)
|
||||
logger.debug("Ignoring TAM made with unsupported suite, since TAM is not required: %r", tam_type)
|
||||
return unpacked, False
|
||||
tam_hmac = tam.get('hmac')
|
||||
tam_salt = tam.get('salt')
|
||||
tam_hmac = tam.get("hmac")
|
||||
tam_salt = tam.get("salt")
|
||||
if not isinstance(tam_salt, (bytes, str)) or not isinstance(tam_hmac, (bytes, str)):
|
||||
raise TAMInvalid()
|
||||
tam_hmac = want_bytes(tam_hmac) # legacy
|
||||
tam_salt = want_bytes(tam_salt) # legacy
|
||||
offset = data.index(tam_hmac)
|
||||
data[offset:offset + 64] = bytes(64)
|
||||
tam_key = self._tam_key(tam_salt, context=b'manifest')
|
||||
calculated_hmac = hmac.digest(tam_key, data, 'sha512')
|
||||
data[offset : offset + 64] = bytes(64)
|
||||
tam_key = self._tam_key(tam_salt, context=b"manifest")
|
||||
calculated_hmac = hmac.digest(tam_key, data, "sha512")
|
||||
if not hmac.compare_digest(calculated_hmac, tam_hmac):
|
||||
raise TAMInvalid()
|
||||
logger.debug('TAM-verified manifest')
|
||||
logger.debug("TAM-verified manifest")
|
||||
return unpacked, True
|
||||
|
||||
|
||||
class PlaintextKey(KeyBase):
|
||||
TYPE = KeyType.PLAINTEXT
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = 'plaintext'
|
||||
ARG_NAME = 'none'
|
||||
NAME = "plaintext"
|
||||
ARG_NAME = "none"
|
||||
STORAGE = KeyBlobStorage.NO_STORAGE
|
||||
|
||||
chunk_seed = 0
|
||||
@ -287,7 +286,7 @@ def id_hash(self, data):
|
||||
def encrypt(self, id, data, compress=True):
|
||||
if compress:
|
||||
data = self.compressor.compress(data)
|
||||
return b''.join([self.TYPE_STR, data])
|
||||
return b"".join([self.TYPE_STR, data])
|
||||
|
||||
def decrypt(self, id, data, decompress=True):
|
||||
self.assert_type(data[0], id)
|
||||
@ -364,8 +363,7 @@ class AESKeyBase(KeyBase):
|
||||
def encrypt(self, id, data, compress=True):
|
||||
if compress:
|
||||
data = self.compressor.compress(data)
|
||||
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(),
|
||||
self.cipher.block_count(len(data)))
|
||||
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data)))
|
||||
return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)
|
||||
|
||||
def decrypt(self, id, data, decompress=True):
|
||||
@ -395,12 +393,10 @@ def init_from_random_data(self):
|
||||
chunk_seed = bytes_to_int(data[96:100])
|
||||
# Convert to signed int32
|
||||
if chunk_seed & 0x80000000:
|
||||
chunk_seed = chunk_seed - 0xffffffff - 1
|
||||
chunk_seed = chunk_seed - 0xFFFFFFFF - 1
|
||||
self.init_from_given_data(
|
||||
enc_key=data[0:32],
|
||||
enc_hmac_key=data[32:64],
|
||||
id_key=data[64:96],
|
||||
chunk_seed=chunk_seed)
|
||||
enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed
|
||||
)
|
||||
|
||||
def init_ciphers(self, manifest_data=None):
|
||||
self.cipher = self.CIPHERSUITE(mac_key=self.enc_hmac_key, enc_key=self.enc_key, header_len=1, aad_offset=1)
|
||||
@ -418,13 +414,13 @@ def init_ciphers(self, manifest_data=None):
|
||||
|
||||
|
||||
class FlexiKey:
|
||||
FILE_ID = 'BORG_KEY'
|
||||
FILE_ID = "BORG_KEY"
|
||||
|
||||
@classmethod
|
||||
def detect(cls, repository, manifest_data):
|
||||
key = cls(repository)
|
||||
target = key.find_key()
|
||||
prompt = 'Enter passphrase for key %s: ' % target
|
||||
prompt = "Enter passphrase for key %s: " % target
|
||||
passphrase = Passphrase.env_passphrase()
|
||||
if passphrase is None:
|
||||
passphrase = Passphrase()
|
||||
@ -449,18 +445,18 @@ def _load(self, key_data, passphrase):
|
||||
data = msgpack.unpackb(data)
|
||||
key = Key(internal_dict=data)
|
||||
if key.version != 1:
|
||||
raise IntegrityError('Invalid key file header')
|
||||
raise IntegrityError("Invalid key file header")
|
||||
self.repository_id = key.repository_id
|
||||
self.enc_key = key.enc_key
|
||||
self.enc_hmac_key = key.enc_hmac_key
|
||||
self.id_key = key.id_key
|
||||
self.chunk_seed = key.chunk_seed
|
||||
self.tam_required = key.get('tam_required', tam_required(self.repository))
|
||||
self.tam_required = key.get("tam_required", tam_required(self.repository))
|
||||
return True
|
||||
return False
|
||||
|
||||
def decrypt_key_file(self, data, passphrase):
|
||||
unpacker = get_limited_unpacker('key')
|
||||
unpacker = get_limited_unpacker("key")
|
||||
unpacker.feed(data)
|
||||
data = unpacker.unpack()
|
||||
encrypted_key = EncryptedKey(internal_dict=data)
|
||||
@ -468,9 +464,9 @@ def decrypt_key_file(self, data, passphrase):
|
||||
raise UnsupportedKeyFormatError()
|
||||
else:
|
||||
self._encrypted_key_algorithm = encrypted_key.algorithm
|
||||
if encrypted_key.algorithm == 'sha256':
|
||||
if encrypted_key.algorithm == "sha256":
|
||||
return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
|
||||
elif encrypted_key.algorithm == 'argon2 chacha20-poly1305':
|
||||
elif encrypted_key.algorithm == "argon2 chacha20-poly1305":
|
||||
return self.decrypt_key_file_argon2(encrypted_key, passphrase)
|
||||
else:
|
||||
raise UnsupportedKeyFormatError()
|
||||
@ -479,7 +475,7 @@ def decrypt_key_file(self, data, passphrase):
|
||||
def pbkdf2(passphrase, salt, iterations, output_len_in_bytes):
|
||||
if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
|
||||
iterations = 1
|
||||
return pbkdf2_hmac('sha256', passphrase.encode('utf-8'), salt, iterations, output_len_in_bytes)
|
||||
return pbkdf2_hmac("sha256", passphrase.encode("utf-8"), salt, iterations, output_len_in_bytes)
|
||||
|
||||
@staticmethod
|
||||
def argon2(
|
||||
@ -489,18 +485,14 @@ def argon2(
|
||||
time_cost: int,
|
||||
memory_cost: int,
|
||||
parallelism: int,
|
||||
type: Literal['i', 'd', 'id']
|
||||
type: Literal["i", "d", "id"],
|
||||
) -> bytes:
|
||||
if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
|
||||
time_cost = 1
|
||||
parallelism = 1
|
||||
# 8 is the smallest value that avoids the "Memory cost is too small" exception
|
||||
memory_cost = 8
|
||||
type_map = {
|
||||
'i': argon2.low_level.Type.I,
|
||||
'd': argon2.low_level.Type.D,
|
||||
'id': argon2.low_level.Type.ID,
|
||||
}
|
||||
type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID}
|
||||
key = argon2.low_level.hash_secret_raw(
|
||||
secret=passphrase.encode("utf-8"),
|
||||
hash_len=output_len_in_bytes,
|
||||
@ -514,7 +506,7 @@ def argon2(
|
||||
|
||||
def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
|
||||
key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)
|
||||
data = AES(key, b'\0'*16).decrypt(encrypted_key.data)
|
||||
data = AES(key, b"\0" * 16).decrypt(encrypted_key.data)
|
||||
if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
|
||||
return data
|
||||
return None
|
||||
@ -536,44 +528,32 @@ def decrypt_key_file_argon2(self, encrypted_key, passphrase):
|
||||
return None
|
||||
|
||||
def encrypt_key_file(self, data, passphrase, algorithm):
|
||||
if algorithm == 'sha256':
|
||||
if algorithm == "sha256":
|
||||
return self.encrypt_key_file_pbkdf2(data, passphrase)
|
||||
elif algorithm == 'argon2 chacha20-poly1305':
|
||||
elif algorithm == "argon2 chacha20-poly1305":
|
||||
return self.encrypt_key_file_argon2(data, passphrase)
|
||||
else:
|
||||
raise ValueError(f'Unexpected algorithm: {algorithm}')
|
||||
raise ValueError(f"Unexpected algorithm: {algorithm}")
|
||||
|
||||
def encrypt_key_file_pbkdf2(self, data, passphrase):
|
||||
salt = os.urandom(32)
|
||||
iterations = PBKDF2_ITERATIONS
|
||||
key = self.pbkdf2(passphrase, salt, iterations, 32)
|
||||
hash = hmac_sha256(key, data)
|
||||
cdata = AES(key, b'\0'*16).encrypt(data)
|
||||
enc_key = EncryptedKey(
|
||||
version=1,
|
||||
salt=salt,
|
||||
iterations=iterations,
|
||||
algorithm='sha256',
|
||||
hash=hash,
|
||||
data=cdata,
|
||||
)
|
||||
cdata = AES(key, b"\0" * 16).encrypt(data)
|
||||
enc_key = EncryptedKey(version=1, salt=salt, iterations=iterations, algorithm="sha256", hash=hash, data=cdata)
|
||||
return msgpack.packb(enc_key.as_dict())
|
||||
|
||||
def encrypt_key_file_argon2(self, data, passphrase):
|
||||
salt = os.urandom(ARGON2_SALT_BYTES)
|
||||
key = self.argon2(
|
||||
passphrase,
|
||||
output_len_in_bytes=32,
|
||||
salt=salt,
|
||||
**ARGON2_ARGS,
|
||||
)
|
||||
key = self.argon2(passphrase, output_len_in_bytes=32, salt=salt, **ARGON2_ARGS)
|
||||
ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0)
|
||||
encrypted_key = EncryptedKey(
|
||||
version=1,
|
||||
algorithm='argon2 chacha20-poly1305',
|
||||
algorithm="argon2 chacha20-poly1305",
|
||||
salt=salt,
|
||||
data=ae_cipher.encrypt(data),
|
||||
**{'argon2_' + k: v for k, v in ARGON2_ARGS.items()},
|
||||
**{"argon2_" + k: v for k, v in ARGON2_ARGS.items()},
|
||||
)
|
||||
return msgpack.packb(encrypted_key.as_dict())
|
||||
|
||||
@ -588,7 +568,7 @@ def _save(self, passphrase, algorithm):
|
||||
tam_required=self.tam_required,
|
||||
)
|
||||
data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm)
|
||||
key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
|
||||
key_data = "\n".join(textwrap.wrap(b2a_base64(data).decode("ascii")))
|
||||
return key_data
|
||||
|
||||
def change_passphrase(self, passphrase=None):
|
||||
@ -612,22 +592,23 @@ def create(cls, repository, args, *, other_key=None):
|
||||
enc_key=other_key.enc_key,
|
||||
enc_hmac_key=other_key.enc_hmac_key,
|
||||
id_key=other_key.id_key,
|
||||
chunk_seed=other_key.chunk_seed)
|
||||
chunk_seed=other_key.chunk_seed,
|
||||
)
|
||||
passphrase = other_key._passphrase
|
||||
else:
|
||||
key.init_from_random_data()
|
||||
passphrase = Passphrase.new(allow_empty=True)
|
||||
key.init_ciphers()
|
||||
target = key.get_new_target(args)
|
||||
key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS['argon2'])
|
||||
key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS["argon2"])
|
||||
logger.info('Key in "%s" created.' % target)
|
||||
logger.info('Keep this key safe. Your data will be inaccessible without it.')
|
||||
logger.info("Keep this key safe. Your data will be inaccessible without it.")
|
||||
return key
|
||||
|
||||
def sanity_check(self, filename, id):
|
||||
file_id = self.FILE_ID.encode() + b' '
|
||||
file_id = self.FILE_ID.encode() + b" "
|
||||
repo_id = hexlify(id)
|
||||
with open(filename, 'rb') as fd:
|
||||
with open(filename, "rb") as fd:
|
||||
# we do the magic / id check in binary mode to avoid stumbling over
|
||||
# decoding errors if somebody has binary files in the keys dir for some reason.
|
||||
if fd.read(len(file_id)) != file_id:
|
||||
@ -653,7 +634,7 @@ def find_key(self):
|
||||
raise RepoKeyNotFoundError(loc) from None
|
||||
return loc
|
||||
else:
|
||||
raise TypeError('Unsupported borg key storage type')
|
||||
raise TypeError("Unsupported borg key storage type")
|
||||
|
||||
def get_existing_or_new_target(self, args):
|
||||
keyfile = self._find_key_file_from_environment()
|
||||
@ -683,10 +664,10 @@ def get_new_target(self, args):
|
||||
elif self.STORAGE == KeyBlobStorage.REPO:
|
||||
return self.repository
|
||||
else:
|
||||
raise TypeError('Unsupported borg key storage type')
|
||||
raise TypeError("Unsupported borg key storage type")
|
||||
|
||||
def _find_key_file_from_environment(self):
|
||||
keyfile = os.environ.get('BORG_KEY_FILE')
|
||||
keyfile = os.environ.get("BORG_KEY_FILE")
|
||||
if keyfile:
|
||||
return os.path.abspath(keyfile)
|
||||
|
||||
@ -696,17 +677,17 @@ def _get_new_target_in_keys_dir(self, args):
|
||||
i = 1
|
||||
while os.path.exists(path):
|
||||
i += 1
|
||||
path = filename + '.%d' % i
|
||||
path = filename + ".%d" % i
|
||||
return path
|
||||
|
||||
def load(self, target, passphrase):
|
||||
if self.STORAGE == KeyBlobStorage.KEYFILE:
|
||||
with open(target) as fd:
|
||||
key_data = ''.join(fd.readlines()[1:])
|
||||
key_data = "".join(fd.readlines()[1:])
|
||||
elif self.STORAGE == KeyBlobStorage.REPO:
|
||||
# While the repository is encrypted, we consider a repokey repository with a blank
|
||||
# passphrase an unencrypted repository.
|
||||
self.logically_encrypted = passphrase != ''
|
||||
self.logically_encrypted = passphrase != ""
|
||||
|
||||
# what we get in target is just a repo location, but we already have the repo obj:
|
||||
target = self.repository
|
||||
@ -715,9 +696,9 @@ def load(self, target, passphrase):
|
||||
# if we got an empty key, it means there is no key.
|
||||
loc = target._location.canonical_path()
|
||||
raise RepoKeyNotFoundError(loc) from None
|
||||
key_data = key_data.decode('utf-8') # remote repo: msgpack issue #99, getting bytes
|
||||
key_data = key_data.decode("utf-8") # remote repo: msgpack issue #99, getting bytes
|
||||
else:
|
||||
raise TypeError('Unsupported borg key storage type')
|
||||
raise TypeError("Unsupported borg key storage type")
|
||||
success = self._load(key_data, passphrase)
|
||||
if success:
|
||||
self.target = target
|
||||
@ -732,31 +713,31 @@ def save(self, target, passphrase, algorithm, create=False):
|
||||
# see issue #6036
|
||||
raise Error('Aborting because key in "%s" already exists.' % target)
|
||||
with SaveFile(target) as fd:
|
||||
fd.write(f'{self.FILE_ID} {bin_to_hex(self.repository_id)}\n')
|
||||
fd.write(f"{self.FILE_ID} {bin_to_hex(self.repository_id)}\n")
|
||||
fd.write(key_data)
|
||||
fd.write('\n')
|
||||
fd.write("\n")
|
||||
elif self.STORAGE == KeyBlobStorage.REPO:
|
||||
self.logically_encrypted = passphrase != ''
|
||||
key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes
|
||||
self.logically_encrypted = passphrase != ""
|
||||
key_data = key_data.encode("utf-8") # remote repo: msgpack issue #99, giving bytes
|
||||
target.save_key(key_data)
|
||||
else:
|
||||
raise TypeError('Unsupported borg key storage type')
|
||||
raise TypeError("Unsupported borg key storage type")
|
||||
self.target = target
|
||||
|
||||
def remove(self, target):
|
||||
if self.STORAGE == KeyBlobStorage.KEYFILE:
|
||||
os.remove(target)
|
||||
elif self.STORAGE == KeyBlobStorage.REPO:
|
||||
target.save_key(b'') # save empty key (no new api at remote repo necessary)
|
||||
target.save_key(b"") # save empty key (no new api at remote repo necessary)
|
||||
else:
|
||||
raise TypeError('Unsupported borg key storage type')
|
||||
raise TypeError("Unsupported borg key storage type")
|
||||
|
||||
|
||||
class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
|
||||
TYPE = KeyType.KEYFILE
|
||||
NAME = 'key file'
|
||||
ARG_NAME = 'keyfile'
|
||||
NAME = "key file"
|
||||
ARG_NAME = "keyfile"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = AES256_CTR_HMAC_SHA256
|
||||
|
||||
@ -764,8 +745,8 @@ class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
|
||||
class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.KEYFILE, KeyType.REPO, KeyType.PASSPHRASE}
|
||||
TYPE = KeyType.REPO
|
||||
NAME = 'repokey'
|
||||
ARG_NAME = 'repokey'
|
||||
NAME = "repokey"
|
||||
ARG_NAME = "repokey"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = AES256_CTR_HMAC_SHA256
|
||||
|
||||
@ -773,8 +754,8 @@ class RepoKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey):
|
||||
class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
|
||||
TYPE = KeyType.BLAKE2KEYFILE
|
||||
NAME = 'key file BLAKE2b'
|
||||
ARG_NAME = 'keyfile-blake2'
|
||||
NAME = "key file BLAKE2b"
|
||||
ARG_NAME = "keyfile-blake2"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = AES256_CTR_BLAKE2b
|
||||
|
||||
@ -782,8 +763,8 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
|
||||
class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2KEYFILE, KeyType.BLAKE2REPO}
|
||||
TYPE = KeyType.BLAKE2REPO
|
||||
NAME = 'repokey BLAKE2b'
|
||||
ARG_NAME = 'repokey-blake2'
|
||||
NAME = "repokey BLAKE2b"
|
||||
ARG_NAME = "repokey-blake2"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = AES256_CTR_BLAKE2b
|
||||
|
||||
@ -810,7 +791,7 @@ def init_ciphers(self, manifest_data=None):
|
||||
def encrypt(self, id, data, compress=True):
|
||||
if compress:
|
||||
data = self.compressor.compress(data)
|
||||
return b''.join([self.TYPE_STR, data])
|
||||
return b"".join([self.TYPE_STR, data])
|
||||
|
||||
def decrypt(self, id, data, decompress=True):
|
||||
self.assert_type(data[0], id)
|
||||
@ -825,15 +806,15 @@ def decrypt(self, id, data, decompress=True):
|
||||
class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase):
|
||||
TYPE = KeyType.AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = 'authenticated'
|
||||
ARG_NAME = 'authenticated'
|
||||
NAME = "authenticated"
|
||||
ARG_NAME = "authenticated"
|
||||
|
||||
|
||||
class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase):
|
||||
TYPE = KeyType.BLAKE2AUTHENTICATED
|
||||
TYPES_ACCEPTABLE = {TYPE}
|
||||
NAME = 'authenticated BLAKE2b'
|
||||
ARG_NAME = 'authenticated-blake2'
|
||||
NAME = "authenticated BLAKE2b"
|
||||
ARG_NAME = "authenticated-blake2"
|
||||
|
||||
|
||||
# ------------ new crypto ------------
|
||||
@ -862,17 +843,17 @@ class AEADKeyBase(KeyBase):
|
||||
|
||||
logically_encrypted = True
|
||||
|
||||
MAX_IV = 2 ** 48 - 1
|
||||
MAX_IV = 2**48 - 1
|
||||
|
||||
def encrypt(self, id, data, compress=True):
|
||||
# to encrypt new data in this session we use always self.cipher and self.sessionid
|
||||
if compress:
|
||||
data = self.compressor.compress(data)
|
||||
reserved = b'\0'
|
||||
reserved = b"\0"
|
||||
iv = self.cipher.next_iv()
|
||||
if iv > self.MAX_IV: # see the data-structures docs about why the IV range is enough
|
||||
raise IntegrityError("IV overflow, should never happen.")
|
||||
iv_48bit = iv.to_bytes(6, 'big')
|
||||
iv_48bit = iv.to_bytes(6, "big")
|
||||
header = self.TYPE_STR + reserved + iv_48bit + self.sessionid
|
||||
return self.cipher.encrypt(data, header=header, iv=iv, aad=id)
|
||||
|
||||
@ -881,7 +862,7 @@ def decrypt(self, id, data, decompress=True):
|
||||
self.assert_type(data[0], id)
|
||||
iv_48bit = data[2:8]
|
||||
sessionid = data[8:32]
|
||||
iv = int.from_bytes(iv_48bit, 'big')
|
||||
iv = int.from_bytes(iv_48bit, "big")
|
||||
cipher = self._get_cipher(sessionid, iv)
|
||||
try:
|
||||
payload = cipher.decrypt(data, aad=id)
|
||||
@ -908,27 +889,25 @@ def init_from_random_data(self):
|
||||
chunk_seed = bytes_to_int(data[96:100])
|
||||
# Convert to signed int32
|
||||
if chunk_seed & 0x80000000:
|
||||
chunk_seed = chunk_seed - 0xffffffff - 1
|
||||
chunk_seed = chunk_seed - 0xFFFFFFFF - 1
|
||||
self.init_from_given_data(
|
||||
enc_key=data[0:32],
|
||||
enc_hmac_key=data[32:64],
|
||||
id_key=data[64:96],
|
||||
chunk_seed=chunk_seed)
|
||||
enc_key=data[0:32], enc_hmac_key=data[32:64], id_key=data[64:96], chunk_seed=chunk_seed
|
||||
)
|
||||
|
||||
def _get_session_key(self, sessionid):
|
||||
assert len(sessionid) == 24 # 192bit
|
||||
key = hkdf_hmac_sha512(
|
||||
ikm=self.enc_key + self.enc_hmac_key,
|
||||
salt=sessionid,
|
||||
info=b'borg-session-key-' + self.CIPHERSUITE.__name__.encode(),
|
||||
output_length=32
|
||||
info=b"borg-session-key-" + self.CIPHERSUITE.__name__.encode(),
|
||||
output_length=32,
|
||||
)
|
||||
return key
|
||||
|
||||
def _get_cipher(self, sessionid, iv):
|
||||
assert isinstance(iv, int)
|
||||
key = self._get_session_key(sessionid)
|
||||
cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1+1+6+24, aad_offset=0)
|
||||
cipher = self.CIPHERSUITE(key=key, iv=iv, header_len=1 + 1 + 6 + 24, aad_offset=0)
|
||||
return cipher
|
||||
|
||||
def init_ciphers(self, manifest_data=None, iv=0):
|
||||
@ -940,8 +919,8 @@ def init_ciphers(self, manifest_data=None, iv=0):
|
||||
class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO}
|
||||
TYPE = KeyType.AESOCBKEYFILE
|
||||
NAME = 'key file AES-OCB'
|
||||
ARG_NAME = 'keyfile-aes-ocb'
|
||||
NAME = "key file AES-OCB"
|
||||
ARG_NAME = "keyfile-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
@ -949,8 +928,8 @@ class AESOCBKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.AESOCBKEYFILE, KeyType.AESOCBREPO}
|
||||
TYPE = KeyType.AESOCBREPO
|
||||
NAME = 'repokey AES-OCB'
|
||||
ARG_NAME = 'repokey-aes-ocb'
|
||||
NAME = "repokey AES-OCB"
|
||||
ARG_NAME = "repokey-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
@ -958,8 +937,8 @@ class AESOCBRepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO}
|
||||
TYPE = KeyType.CHPOKEYFILE
|
||||
NAME = 'key file ChaCha20-Poly1305'
|
||||
ARG_NAME = 'keyfile-chacha20-poly1305'
|
||||
NAME = "key file ChaCha20-Poly1305"
|
||||
ARG_NAME = "keyfile-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
@ -967,8 +946,8 @@ class CHPOKeyfileKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.CHPOKEYFILE, KeyType.CHPOREPO}
|
||||
TYPE = KeyType.CHPOREPO
|
||||
NAME = 'repokey ChaCha20-Poly1305'
|
||||
ARG_NAME = 'repokey-chacha20-poly1305'
|
||||
NAME = "repokey ChaCha20-Poly1305"
|
||||
ARG_NAME = "repokey-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
@ -976,8 +955,8 @@ class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey):
|
||||
class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE2AESOCBKEYFILE
|
||||
NAME = 'key file BLAKE2b AES-OCB'
|
||||
ARG_NAME = 'keyfile-blake2-aes-ocb'
|
||||
NAME = "key file BLAKE2b AES-OCB"
|
||||
ARG_NAME = "keyfile-blake2-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
@ -985,8 +964,8 @@ class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO}
|
||||
TYPE = KeyType.BLAKE2AESOCBREPO
|
||||
NAME = 'repokey BLAKE2b AES-OCB'
|
||||
ARG_NAME = 'repokey-blake2-aes-ocb'
|
||||
NAME = "repokey BLAKE2b AES-OCB"
|
||||
ARG_NAME = "repokey-blake2-aes-ocb"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = AES256_OCB
|
||||
|
||||
@ -994,8 +973,8 @@ class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
|
||||
TYPE = KeyType.BLAKE2CHPOKEYFILE
|
||||
NAME = 'key file BLAKE2b ChaCha20-Poly1305'
|
||||
ARG_NAME = 'keyfile-blake2-chacha20-poly1305'
|
||||
NAME = "key file BLAKE2b ChaCha20-Poly1305"
|
||||
ARG_NAME = "keyfile-blake2-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.KEYFILE
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
@ -1003,26 +982,33 @@ class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey):
|
||||
TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO}
|
||||
TYPE = KeyType.BLAKE2CHPOREPO
|
||||
NAME = 'repokey BLAKE2b ChaCha20-Poly1305'
|
||||
ARG_NAME = 'repokey-blake2-chacha20-poly1305'
|
||||
NAME = "repokey BLAKE2b ChaCha20-Poly1305"
|
||||
ARG_NAME = "repokey-blake2-chacha20-poly1305"
|
||||
STORAGE = KeyBlobStorage.REPO
|
||||
CIPHERSUITE = CHACHA20_POLY1305
|
||||
|
||||
|
||||
LEGACY_KEY_TYPES = (
|
||||
# legacy (AES-CTR based) crypto
|
||||
KeyfileKey, RepoKey,
|
||||
Blake2KeyfileKey, Blake2RepoKey,
|
||||
KeyfileKey,
|
||||
RepoKey,
|
||||
Blake2KeyfileKey,
|
||||
Blake2RepoKey,
|
||||
)
|
||||
|
||||
AVAILABLE_KEY_TYPES = (
|
||||
# these are available encryption modes for new repositories
|
||||
# not encrypted modes
|
||||
PlaintextKey,
|
||||
AuthenticatedKey, Blake2AuthenticatedKey,
|
||||
AuthenticatedKey,
|
||||
Blake2AuthenticatedKey,
|
||||
# new crypto
|
||||
AESOCBKeyfileKey, AESOCBRepoKey,
|
||||
CHPOKeyfileKey, CHPORepoKey,
|
||||
Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey,
|
||||
Blake2CHPOKeyfileKey, Blake2CHPORepoKey,
|
||||
AESOCBKeyfileKey,
|
||||
AESOCBRepoKey,
|
||||
CHPOKeyfileKey,
|
||||
CHPORepoKey,
|
||||
Blake2AESOCBKeyfileKey,
|
||||
Blake2AESOCBRepoKey,
|
||||
Blake2CHPOKeyfileKey,
|
||||
Blake2CHPORepoKey,
|
||||
)
|
||||
|
@ -53,7 +53,7 @@ def load_keyblob(self):
|
||||
k = CHPOKeyfileKey(self.repository)
|
||||
target = k.find_key()
|
||||
with open(target) as fd:
|
||||
self.keyblob = ''.join(fd.readlines()[1:])
|
||||
self.keyblob = "".join(fd.readlines()[1:])
|
||||
|
||||
elif self.keyblob_storage == KeyBlobStorage.REPO:
|
||||
key_data = self.repository.load_key().decode()
|
||||
@ -70,75 +70,77 @@ def store_keyblob(self, args):
|
||||
|
||||
self.store_keyfile(target)
|
||||
elif self.keyblob_storage == KeyBlobStorage.REPO:
|
||||
self.repository.save_key(self.keyblob.encode('utf-8'))
|
||||
self.repository.save_key(self.keyblob.encode("utf-8"))
|
||||
|
||||
def get_keyfile_data(self):
|
||||
data = f'{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n'
|
||||
data = f"{CHPOKeyfileKey.FILE_ID} {bin_to_hex(self.repository.id)}\n"
|
||||
data += self.keyblob
|
||||
if not self.keyblob.endswith('\n'):
|
||||
data += '\n'
|
||||
if not self.keyblob.endswith("\n"):
|
||||
data += "\n"
|
||||
return data
|
||||
|
||||
def store_keyfile(self, target):
|
||||
with dash_open(target, 'w') as fd:
|
||||
with dash_open(target, "w") as fd:
|
||||
fd.write(self.get_keyfile_data())
|
||||
|
||||
def export(self, path):
|
||||
if path is None:
|
||||
path = '-'
|
||||
path = "-"
|
||||
|
||||
self.store_keyfile(path)
|
||||
|
||||
def export_qr(self, path):
|
||||
if path is None:
|
||||
path = '-'
|
||||
path = "-"
|
||||
|
||||
with dash_open(path, 'wb') as fd:
|
||||
with dash_open(path, "wb") as fd:
|
||||
key_data = self.get_keyfile_data()
|
||||
html = pkgutil.get_data('borg', 'paperkey.html')
|
||||
html = html.replace(b'</textarea>', key_data.encode() + b'</textarea>')
|
||||
html = pkgutil.get_data("borg", "paperkey.html")
|
||||
html = html.replace(b"</textarea>", key_data.encode() + b"</textarea>")
|
||||
fd.write(html)
|
||||
|
||||
def export_paperkey(self, path):
|
||||
if path is None:
|
||||
path = '-'
|
||||
path = "-"
|
||||
|
||||
def grouped(s):
|
||||
ret = ''
|
||||
ret = ""
|
||||
i = 0
|
||||
for ch in s:
|
||||
if i and i % 6 == 0:
|
||||
ret += ' '
|
||||
ret += " "
|
||||
ret += ch
|
||||
i += 1
|
||||
return ret
|
||||
|
||||
export = 'To restore key use borg key import --paper /path/to/repo\n\n'
|
||||
export = "To restore key use borg key import --paper /path/to/repo\n\n"
|
||||
|
||||
binary = a2b_base64(self.keyblob)
|
||||
export += 'BORG PAPER KEY v1\n'
|
||||
export += "BORG PAPER KEY v1\n"
|
||||
lines = (len(binary) + 17) // 18
|
||||
repoid = bin_to_hex(self.repository.id)[:18]
|
||||
complete_checksum = sha256_truncated(binary, 12)
|
||||
export += 'id: {:d} / {} / {} - {}\n'.format(lines,
|
||||
grouped(repoid),
|
||||
grouped(complete_checksum),
|
||||
sha256_truncated((str(lines) + '/' + repoid + '/' + complete_checksum).encode('ascii'), 2))
|
||||
export += "id: {:d} / {} / {} - {}\n".format(
|
||||
lines,
|
||||
grouped(repoid),
|
||||
grouped(complete_checksum),
|
||||
sha256_truncated((str(lines) + "/" + repoid + "/" + complete_checksum).encode("ascii"), 2),
|
||||
)
|
||||
idx = 0
|
||||
while len(binary):
|
||||
idx += 1
|
||||
binline = binary[:18]
|
||||
checksum = sha256_truncated(idx.to_bytes(2, byteorder='big') + binline, 2)
|
||||
export += f'{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n'
|
||||
checksum = sha256_truncated(idx.to_bytes(2, byteorder="big") + binline, 2)
|
||||
export += f"{idx:2d}: {grouped(bin_to_hex(binline))} - {checksum}\n"
|
||||
binary = binary[18:]
|
||||
|
||||
with dash_open(path, 'w') as fd:
|
||||
with dash_open(path, "w") as fd:
|
||||
fd.write(export)
|
||||
|
||||
def import_keyfile(self, args):
|
||||
file_id = CHPOKeyfileKey.FILE_ID
|
||||
first_line = file_id + ' ' + bin_to_hex(self.repository.id) + '\n'
|
||||
with dash_open(args.path, 'r') as fd:
|
||||
first_line = file_id + " " + bin_to_hex(self.repository.id) + "\n"
|
||||
with dash_open(args.path, "r") as fd:
|
||||
file_first_line = fd.read(len(first_line))
|
||||
if file_first_line != first_line:
|
||||
if not file_first_line.startswith(file_id):
|
||||
@ -154,52 +156,52 @@ def import_paperkey(self, args):
|
||||
# imported here because it has global side effects
|
||||
import readline
|
||||
except ImportError:
|
||||
print('Note: No line editing available due to missing readline support')
|
||||
print("Note: No line editing available due to missing readline support")
|
||||
|
||||
repoid = bin_to_hex(self.repository.id)[:18]
|
||||
try:
|
||||
while True: # used for repeating on overall checksum mismatch
|
||||
# id line input
|
||||
while True:
|
||||
idline = input('id: ').replace(' ', '')
|
||||
if idline == '':
|
||||
if yes('Abort import? [yN]:'):
|
||||
idline = input("id: ").replace(" ", "")
|
||||
if idline == "":
|
||||
if yes("Abort import? [yN]:"):
|
||||
raise EOFError()
|
||||
|
||||
try:
|
||||
(data, checksum) = idline.split('-')
|
||||
(data, checksum) = idline.split("-")
|
||||
except ValueError:
|
||||
print("each line must contain exactly one '-', try again")
|
||||
continue
|
||||
try:
|
||||
(id_lines, id_repoid, id_complete_checksum) = data.split('/')
|
||||
(id_lines, id_repoid, id_complete_checksum) = data.split("/")
|
||||
except ValueError:
|
||||
print("the id line must contain exactly three '/', try again")
|
||||
continue
|
||||
if sha256_truncated(data.lower().encode('ascii'), 2) != checksum:
|
||||
print('line checksum did not match, try same line again')
|
||||
if sha256_truncated(data.lower().encode("ascii"), 2) != checksum:
|
||||
print("line checksum did not match, try same line again")
|
||||
continue
|
||||
try:
|
||||
lines = int(id_lines)
|
||||
except ValueError:
|
||||
print('internal error while parsing length')
|
||||
print("internal error while parsing length")
|
||||
|
||||
break
|
||||
|
||||
if repoid != id_repoid:
|
||||
raise RepoIdMismatch()
|
||||
|
||||
result = b''
|
||||
result = b""
|
||||
idx = 1
|
||||
# body line input
|
||||
while True:
|
||||
inline = input(f'{idx:2d}: ')
|
||||
inline = inline.replace(' ', '')
|
||||
if inline == '':
|
||||
if yes('Abort import? [yN]:'):
|
||||
inline = input(f"{idx:2d}: ")
|
||||
inline = inline.replace(" ", "")
|
||||
if inline == "":
|
||||
if yes("Abort import? [yN]:"):
|
||||
raise EOFError()
|
||||
try:
|
||||
(data, checksum) = inline.split('-')
|
||||
(data, checksum) = inline.split("-")
|
||||
except ValueError:
|
||||
print("each line must contain exactly one '-', try again")
|
||||
continue
|
||||
@ -208,8 +210,8 @@ def import_paperkey(self, args):
|
||||
except binascii.Error:
|
||||
print("only characters 0-9 and a-f and '-' are valid, try again")
|
||||
continue
|
||||
if sha256_truncated(idx.to_bytes(2, byteorder='big') + part, 2) != checksum:
|
||||
print(f'line checksum did not match, try line {idx} again')
|
||||
if sha256_truncated(idx.to_bytes(2, byteorder="big") + part, 2) != checksum:
|
||||
print(f"line checksum did not match, try line {idx} again")
|
||||
continue
|
||||
result += part
|
||||
if idx == lines:
|
||||
@ -217,13 +219,13 @@ def import_paperkey(self, args):
|
||||
idx += 1
|
||||
|
||||
if sha256_truncated(result, 12) != id_complete_checksum:
|
||||
print('The overall checksum did not match, retry or enter a blank line to abort.')
|
||||
print("The overall checksum did not match, retry or enter a blank line to abort.")
|
||||
continue
|
||||
|
||||
self.keyblob = '\n'.join(textwrap.wrap(b2a_base64(result).decode('ascii'))) + '\n'
|
||||
self.keyblob = "\n".join(textwrap.wrap(b2a_base64(result).decode("ascii"))) + "\n"
|
||||
self.store_keyblob(args)
|
||||
break
|
||||
|
||||
except EOFError:
|
||||
print('\n - aborted')
|
||||
print("\n - aborted")
|
||||
return
|
||||
|
@ -18,7 +18,7 @@ def __init__(self, repository, manifest_nonce):
|
||||
self.repository = repository
|
||||
self.end_of_nonce_reservation = None
|
||||
self.manifest_nonce = manifest_nonce
|
||||
self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), 'nonce')
|
||||
self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce")
|
||||
|
||||
def get_local_free_nonce(self):
|
||||
try:
|
||||
@ -78,7 +78,11 @@ def ensure_reservation(self, nonce, nonce_space_needed):
|
||||
|
||||
repo_free_nonce = self.get_repo_free_nonce()
|
||||
local_free_nonce = self.get_local_free_nonce()
|
||||
free_nonce_space = max(x for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation) if x is not None)
|
||||
free_nonce_space = max(
|
||||
x
|
||||
for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation)
|
||||
if x is not None
|
||||
)
|
||||
reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION
|
||||
assert reservation_end < MAX_REPRESENTABLE_NONCE
|
||||
self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce)
|
||||
|
213
src/borg/fuse.py
213
src/borg/fuse.py
@ -20,7 +20,9 @@ def async_wrapper(fn):
|
||||
@functools.wraps(fn)
|
||||
async def wrapper(*args, **kwargs):
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
else:
|
||||
trio = None
|
||||
|
||||
@ -29,6 +31,7 @@ def async_wrapper(fn):
|
||||
|
||||
|
||||
from .logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from .crypto.low_level import blake2b_128
|
||||
@ -79,7 +82,7 @@ class ItemCache:
|
||||
# to resize it in the first place; that's free).
|
||||
GROW_META_BY = 2 * 1024 * 1024
|
||||
|
||||
indirect_entry_struct = struct.Struct('=cII')
|
||||
indirect_entry_struct = struct.Struct("=cII")
|
||||
assert indirect_entry_struct.size == 9
|
||||
|
||||
def __init__(self, decrypted_repository):
|
||||
@ -105,7 +108,7 @@ def __init__(self, decrypted_repository):
|
||||
# These are items that span more than one chunk and thus cannot be efficiently cached
|
||||
# by the object cache (self.decrypted_repository), which would require variable-length structures;
|
||||
# possible but not worth the effort, see iter_archive_items.
|
||||
self.fd = tempfile.TemporaryFile(prefix='borg-tmp')
|
||||
self.fd = tempfile.TemporaryFile(prefix="borg-tmp")
|
||||
|
||||
# A small LRU cache for chunks requested by ItemCache.get() from the object cache,
|
||||
# this significantly speeds up directory traversal and similar operations which
|
||||
@ -123,12 +126,12 @@ def __init__(self, decrypted_repository):
|
||||
def get(self, inode):
|
||||
offset = inode - self.offset
|
||||
if offset < 0:
|
||||
raise ValueError('ItemCache.get() called with an invalid inode number')
|
||||
if self.meta[offset] == ord(b'I'):
|
||||
raise ValueError("ItemCache.get() called with an invalid inode number")
|
||||
if self.meta[offset] == ord(b"I"):
|
||||
_, chunk_id_relative_offset, chunk_offset = self.indirect_entry_struct.unpack_from(self.meta, offset)
|
||||
chunk_id_offset = offset - chunk_id_relative_offset
|
||||
# bytearray slices are bytearrays as well, explicitly convert to bytes()
|
||||
chunk_id = bytes(self.meta[chunk_id_offset:chunk_id_offset + 32])
|
||||
chunk_id = bytes(self.meta[chunk_id_offset : chunk_id_offset + 32])
|
||||
chunk = self.chunks.get(chunk_id)
|
||||
if not chunk:
|
||||
csize, chunk = next(self.decrypted_repository.get_many([chunk_id]))
|
||||
@ -137,12 +140,12 @@ def get(self, inode):
|
||||
unpacker = msgpack.Unpacker()
|
||||
unpacker.feed(data)
|
||||
return Item(internal_dict=next(unpacker))
|
||||
elif self.meta[offset] == ord(b'S'):
|
||||
fd_offset = int.from_bytes(self.meta[offset + 1:offset + 9], 'little')
|
||||
elif self.meta[offset] == ord(b"S"):
|
||||
fd_offset = int.from_bytes(self.meta[offset + 1 : offset + 9], "little")
|
||||
self.fd.seek(fd_offset, io.SEEK_SET)
|
||||
return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024)))
|
||||
else:
|
||||
raise ValueError('Invalid entry type in self.meta')
|
||||
raise ValueError("Invalid entry type in self.meta")
|
||||
|
||||
def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=False):
|
||||
unpacker = msgpack.Unpacker()
|
||||
@ -153,7 +156,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
chunk_begin = 0
|
||||
# Length of the chunk preceding the current chunk
|
||||
last_chunk_length = 0
|
||||
msgpacked_bytes = b''
|
||||
msgpacked_bytes = b""
|
||||
|
||||
write_offset = self.write_offset
|
||||
meta = self.meta
|
||||
@ -163,7 +166,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
# Store the chunk ID in the meta-array
|
||||
if write_offset + 32 >= len(meta):
|
||||
self.meta = meta = meta + bytes(self.GROW_META_BY)
|
||||
meta[write_offset:write_offset + 32] = key
|
||||
meta[write_offset : write_offset + 32] = key
|
||||
current_id_offset = write_offset
|
||||
write_offset += 32
|
||||
|
||||
@ -182,7 +185,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
# tell() is not helpful for the need_more_data case, but we know it is the remainder
|
||||
# of the data in that case. in the other case, tell() works as expected.
|
||||
length = (len(data) - start) if need_more_data else (unpacker.tell() - stream_offset)
|
||||
msgpacked_bytes += data[start:start+length]
|
||||
msgpacked_bytes += data[start : start + length]
|
||||
stream_offset += length
|
||||
|
||||
if need_more_data:
|
||||
@ -190,14 +193,14 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
break
|
||||
|
||||
item = Item(internal_dict=item)
|
||||
if filter and not filter(item) or not consider_part_files and 'part' in item:
|
||||
msgpacked_bytes = b''
|
||||
if filter and not filter(item) or not consider_part_files and "part" in item:
|
||||
msgpacked_bytes = b""
|
||||
continue
|
||||
|
||||
current_item = msgpacked_bytes
|
||||
current_item_length = len(current_item)
|
||||
current_spans_chunks = stream_offset - current_item_length < chunk_begin
|
||||
msgpacked_bytes = b''
|
||||
msgpacked_bytes = b""
|
||||
|
||||
if write_offset + 9 >= len(meta):
|
||||
self.meta = meta = meta + bytes(self.GROW_META_BY)
|
||||
@ -221,11 +224,11 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
if current_spans_chunks:
|
||||
pos = self.fd.seek(0, io.SEEK_END)
|
||||
self.fd.write(current_item)
|
||||
meta[write_offset:write_offset + 9] = b'S' + pos.to_bytes(8, 'little')
|
||||
meta[write_offset : write_offset + 9] = b"S" + pos.to_bytes(8, "little")
|
||||
self.direct_items += 1
|
||||
else:
|
||||
item_offset = stream_offset - current_item_length - chunk_begin
|
||||
pack_indirect_into(meta, write_offset, b'I', write_offset - current_id_offset, item_offset)
|
||||
pack_indirect_into(meta, write_offset, b"I", write_offset - current_id_offset, item_offset)
|
||||
self.indirect_items += 1
|
||||
inode = write_offset + self.offset
|
||||
write_offset += 9
|
||||
@ -236,8 +239,7 @@ def iter_archive_items(self, archive_item_ids, filter=None, consider_part_files=
|
||||
|
||||
|
||||
class FuseBackend:
|
||||
"""Virtual filesystem based on archive(s) to provide information to fuse
|
||||
"""
|
||||
"""Virtual filesystem based on archive(s) to provide information to fuse"""
|
||||
|
||||
def __init__(self, key, manifest, repository, args, decrypted_repository):
|
||||
self.repository_uncached = repository
|
||||
@ -307,8 +309,7 @@ def _allocate_inode(self):
|
||||
return self.inode_count
|
||||
|
||||
def _create_dir(self, parent, mtime=None):
|
||||
"""Create directory
|
||||
"""
|
||||
"""Create directory"""
|
||||
ino = self._allocate_inode()
|
||||
if mtime is not None:
|
||||
self._items[ino] = Item(internal_dict=self.default_dir.as_dict())
|
||||
@ -319,26 +320,31 @@ def _create_dir(self, parent, mtime=None):
|
||||
return ino
|
||||
|
||||
def find_inode(self, path, prefix=[]):
|
||||
segments = prefix + path.split(b'/')
|
||||
segments = prefix + path.split(b"/")
|
||||
inode = 1
|
||||
for segment in segments:
|
||||
inode = self.contents[inode][segment]
|
||||
return inode
|
||||
|
||||
def _process_archive(self, archive_name, prefix=[]):
|
||||
"""Build FUSE inode hierarchy from archive metadata
|
||||
"""
|
||||
"""Build FUSE inode hierarchy from archive metadata"""
|
||||
self.file_versions = {} # for versions mode: original path -> version
|
||||
t0 = time.perf_counter()
|
||||
archive = Archive(self.repository_uncached, self.key, self._manifest, archive_name,
|
||||
consider_part_files=self._args.consider_part_files)
|
||||
archive = Archive(
|
||||
self.repository_uncached,
|
||||
self.key,
|
||||
self._manifest,
|
||||
archive_name,
|
||||
consider_part_files=self._args.consider_part_files,
|
||||
)
|
||||
strip_components = self._args.strip_components
|
||||
matcher = Archiver.build_matcher(self._args.patterns, self._args.paths)
|
||||
hlm = HardLinkManager(id_type=bytes, info_type=str) # hlid -> path
|
||||
|
||||
filter = Archiver.build_filter(matcher, strip_components)
|
||||
for item_inode, item in self.cache.iter_archive_items(archive.metadata.items, filter=filter,
|
||||
consider_part_files=self._args.consider_part_files):
|
||||
for item_inode, item in self.cache.iter_archive_items(
|
||||
archive.metadata.items, filter=filter, consider_part_files=self._args.consider_part_files
|
||||
):
|
||||
if strip_components:
|
||||
item.path = os.sep.join(item.path.split(os.sep)[strip_components:])
|
||||
path = os.fsencode(item.path)
|
||||
@ -354,24 +360,24 @@ def _process_archive(self, archive_name, prefix=[]):
|
||||
else:
|
||||
self._items[inode] = item
|
||||
continue
|
||||
segments = prefix + path.split(b'/')
|
||||
segments = prefix + path.split(b"/")
|
||||
parent = 1
|
||||
for segment in segments[:-1]:
|
||||
parent = self._process_inner(segment, parent)
|
||||
self._process_leaf(segments[-1], item, parent, prefix, is_dir, item_inode, hlm)
|
||||
duration = time.perf_counter() - t0
|
||||
logger.debug('fuse: _process_archive completed in %.1f s for archive %s', duration, archive.name)
|
||||
logger.debug("fuse: _process_archive completed in %.1f s for archive %s", duration, archive.name)
|
||||
|
||||
def _process_leaf(self, name, item, parent, prefix, is_dir, item_inode, hlm):
|
||||
path = item.path
|
||||
del item.path # save some space
|
||||
|
||||
def file_version(item, path):
|
||||
if 'chunks' in item:
|
||||
if "chunks" in item:
|
||||
file_id = blake2b_128(path)
|
||||
current_version, previous_id = self.versions_index.get(file_id, (0, None))
|
||||
|
||||
contents_id = blake2b_128(b''.join(chunk_id for chunk_id, _ in item.chunks))
|
||||
contents_id = blake2b_128(b"".join(chunk_id for chunk_id, _ in item.chunks))
|
||||
|
||||
if contents_id != previous_id:
|
||||
current_version += 1
|
||||
@ -382,14 +388,14 @@ def file_version(item, path):
|
||||
def make_versioned_name(name, version, add_dir=False):
|
||||
if add_dir:
|
||||
# add intermediate directory with same name as filename
|
||||
path_fname = name.rsplit(b'/', 1)
|
||||
name += b'/' + path_fname[-1]
|
||||
path_fname = name.rsplit(b"/", 1)
|
||||
name += b"/" + path_fname[-1]
|
||||
# keep original extension at end to avoid confusing tools
|
||||
name, ext = os.path.splitext(name)
|
||||
version_enc = os.fsencode('.%05d' % version)
|
||||
version_enc = os.fsencode(".%05d" % version)
|
||||
return name + version_enc + ext
|
||||
|
||||
if 'hlid' in item:
|
||||
if "hlid" in item:
|
||||
link_target = hlm.retrieve(id=item.hlid, default=None)
|
||||
if link_target is not None:
|
||||
# Hard link was extracted previously, just link
|
||||
@ -401,10 +407,10 @@ def make_versioned_name(name, version, add_dir=False):
|
||||
try:
|
||||
inode = self.find_inode(link_target, prefix)
|
||||
except KeyError:
|
||||
logger.warning('Skipping broken hard link: %s -> %s', path, link_target)
|
||||
logger.warning("Skipping broken hard link: %s -> %s", path, link_target)
|
||||
return
|
||||
item = self.get_item(inode)
|
||||
item.nlink = item.get('nlink', 1) + 1
|
||||
item.nlink = item.get("nlink", 1) + 1
|
||||
self._items[inode] = item
|
||||
else:
|
||||
inode = item_inode
|
||||
@ -439,31 +445,41 @@ def _process_inner(self, name, parent_inode):
|
||||
|
||||
|
||||
class FuseOperations(llfuse.Operations, FuseBackend):
|
||||
"""Export archive as a FUSE filesystem
|
||||
"""
|
||||
"""Export archive as a FUSE filesystem"""
|
||||
|
||||
def __init__(self, key, repository, manifest, args, decrypted_repository):
|
||||
llfuse.Operations.__init__(self)
|
||||
FuseBackend.__init__(self, key, manifest, repository, args, decrypted_repository)
|
||||
self.decrypted_repository = decrypted_repository
|
||||
data_cache_capacity = int(os.environ.get('BORG_MOUNT_DATA_CACHE_ENTRIES', os.cpu_count() or 1))
|
||||
logger.debug('mount data cache capacity: %d chunks', data_cache_capacity)
|
||||
data_cache_capacity = int(os.environ.get("BORG_MOUNT_DATA_CACHE_ENTRIES", os.cpu_count() or 1))
|
||||
logger.debug("mount data cache capacity: %d chunks", data_cache_capacity)
|
||||
self.data_cache = LRUCache(capacity=data_cache_capacity, dispose=lambda _: None)
|
||||
self._last_pos = LRUCache(capacity=FILES, dispose=lambda _: None)
|
||||
|
||||
def sig_info_handler(self, sig_no, stack):
|
||||
logger.debug('fuse: %d synth inodes, %d edges (%s)',
|
||||
self.inode_count, len(self.parent),
|
||||
# getsizeof is the size of the dict itself; key and value are two small-ish integers,
|
||||
# which are shared due to code structure (this has been verified).
|
||||
format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)))
|
||||
logger.debug('fuse: %d pending archives', len(self.pending_archives))
|
||||
logger.debug('fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s',
|
||||
self.cache.direct_items + self.cache.indirect_items, self.cache.direct_items, self.cache.indirect_items,
|
||||
format_file_size(sys.getsizeof(self.cache.meta)),
|
||||
format_file_size(os.stat(self.cache.fd.fileno()).st_size))
|
||||
logger.debug('fuse: data cache: %d/%d entries, %s', len(self.data_cache.items()), self.data_cache._capacity,
|
||||
format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())))
|
||||
logger.debug(
|
||||
"fuse: %d synth inodes, %d edges (%s)",
|
||||
self.inode_count,
|
||||
len(self.parent),
|
||||
# getsizeof is the size of the dict itself; key and value are two small-ish integers,
|
||||
# which are shared due to code structure (this has been verified).
|
||||
format_file_size(sys.getsizeof(self.parent) + len(self.parent) * sys.getsizeof(self.inode_count)),
|
||||
)
|
||||
logger.debug("fuse: %d pending archives", len(self.pending_archives))
|
||||
logger.debug(
|
||||
"fuse: ItemCache %d entries (%d direct, %d indirect), meta-array size %s, direct items size %s",
|
||||
self.cache.direct_items + self.cache.indirect_items,
|
||||
self.cache.direct_items,
|
||||
self.cache.indirect_items,
|
||||
format_file_size(sys.getsizeof(self.cache.meta)),
|
||||
format_file_size(os.stat(self.cache.fd.fileno()).st_size),
|
||||
)
|
||||
logger.debug(
|
||||
"fuse: data cache: %d/%d entries, %s",
|
||||
len(self.data_cache.items()),
|
||||
self.data_cache._capacity,
|
||||
format_file_size(sum(len(chunk) for key, chunk in self.data_cache.items())),
|
||||
)
|
||||
self.decrypted_repository.log_instrumentation()
|
||||
|
||||
def mount(self, mountpoint, mount_options, foreground=False):
|
||||
@ -475,25 +491,25 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
|
||||
if option == key:
|
||||
options.pop(idx)
|
||||
return present
|
||||
if option.startswith(key + '='):
|
||||
if option.startswith(key + "="):
|
||||
options.pop(idx)
|
||||
value = option.split('=', 1)[1]
|
||||
value = option.split("=", 1)[1]
|
||||
if wanted_type is bool:
|
||||
v = value.lower()
|
||||
if v in ('y', 'yes', 'true', '1'):
|
||||
if v in ("y", "yes", "true", "1"):
|
||||
return True
|
||||
if v in ('n', 'no', 'false', '0'):
|
||||
if v in ("n", "no", "false", "0"):
|
||||
return False
|
||||
raise ValueError('unsupported value in option: %s' % option)
|
||||
raise ValueError("unsupported value in option: %s" % option)
|
||||
if wanted_type is int:
|
||||
try:
|
||||
return int(value, base=int_base)
|
||||
except ValueError:
|
||||
raise ValueError('unsupported value in option: %s' % option) from None
|
||||
raise ValueError("unsupported value in option: %s" % option) from None
|
||||
try:
|
||||
return wanted_type(value)
|
||||
except ValueError:
|
||||
raise ValueError('unsupported value in option: %s' % option) from None
|
||||
raise ValueError("unsupported value in option: %s" % option) from None
|
||||
else:
|
||||
return not_present
|
||||
|
||||
@ -502,20 +518,20 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
|
||||
# cause security issues if used with allow_other mount option.
|
||||
# When not using allow_other or allow_root, access is limited to the
|
||||
# mounting user anyway.
|
||||
options = ['fsname=borgfs', 'ro', 'default_permissions']
|
||||
options = ["fsname=borgfs", "ro", "default_permissions"]
|
||||
if mount_options:
|
||||
options.extend(mount_options.split(','))
|
||||
ignore_permissions = pop_option(options, 'ignore_permissions', True, False, bool)
|
||||
options.extend(mount_options.split(","))
|
||||
ignore_permissions = pop_option(options, "ignore_permissions", True, False, bool)
|
||||
if ignore_permissions:
|
||||
# in case users have a use-case that requires NOT giving "default_permissions",
|
||||
# this is enabled by the custom "ignore_permissions" mount option which just
|
||||
# removes "default_permissions" again:
|
||||
pop_option(options, 'default_permissions', True, False, bool)
|
||||
self.allow_damaged_files = pop_option(options, 'allow_damaged_files', True, False, bool)
|
||||
self.versions = pop_option(options, 'versions', True, False, bool)
|
||||
self.uid_forced = pop_option(options, 'uid', None, None, int)
|
||||
self.gid_forced = pop_option(options, 'gid', None, None, int)
|
||||
self.umask = pop_option(options, 'umask', 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222
|
||||
pop_option(options, "default_permissions", True, False, bool)
|
||||
self.allow_damaged_files = pop_option(options, "allow_damaged_files", True, False, bool)
|
||||
self.versions = pop_option(options, "versions", True, False, bool)
|
||||
self.uid_forced = pop_option(options, "uid", None, None, int)
|
||||
self.gid_forced = pop_option(options, "gid", None, None, int)
|
||||
self.umask = pop_option(options, "umask", 0, 0, int, int_base=8) # umask is octal, e.g. 222 or 0222
|
||||
dir_uid = self.uid_forced if self.uid_forced is not None else self.default_uid
|
||||
dir_gid = self.gid_forced if self.gid_forced is not None else self.default_gid
|
||||
dir_user = uid2user(dir_uid)
|
||||
@ -523,8 +539,9 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
|
||||
assert isinstance(dir_user, str)
|
||||
assert isinstance(dir_group, str)
|
||||
dir_mode = 0o40755 & ~self.umask
|
||||
self.default_dir = Item(mode=dir_mode, mtime=int(time.time() * 1e9),
|
||||
user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid)
|
||||
self.default_dir = Item(
|
||||
mode=dir_mode, mtime=int(time.time() * 1e9), user=dir_user, group=dir_group, uid=dir_uid, gid=dir_gid
|
||||
)
|
||||
self._create_filesystem()
|
||||
llfuse.init(self, mountpoint, options)
|
||||
if not foreground:
|
||||
@ -533,7 +550,7 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
|
||||
else:
|
||||
with daemonizing() as (old_id, new_id):
|
||||
# local repo: the locking process' PID is changing, migrate it:
|
||||
logger.debug('fuse: mount local repo, going to background: migrating lock.')
|
||||
logger.debug("fuse: mount local repo, going to background: migrating lock.")
|
||||
self.repository_uncached.migrate_lock(old_id, new_id)
|
||||
|
||||
# If the file system crashes, we do not want to umount because in that
|
||||
@ -543,11 +560,10 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
|
||||
# mirror.
|
||||
umount = False
|
||||
try:
|
||||
with signal_handler('SIGUSR1', self.sig_info_handler), \
|
||||
signal_handler('SIGINFO', self.sig_info_handler):
|
||||
with signal_handler("SIGUSR1", self.sig_info_handler), signal_handler("SIGINFO", self.sig_info_handler):
|
||||
signal = fuse_main()
|
||||
# no crash and no signal (or it's ^C and we're in the foreground) -> umount request
|
||||
umount = (signal is None or (signal == SIGINT and foreground))
|
||||
umount = signal is None or (signal == SIGINT and foreground)
|
||||
finally:
|
||||
llfuse.close(umount)
|
||||
|
||||
@ -573,19 +589,24 @@ def _getattr(self, inode, ctx=None):
|
||||
entry.entry_timeout = 300
|
||||
entry.attr_timeout = 300
|
||||
entry.st_mode = item.mode & ~self.umask
|
||||
entry.st_nlink = item.get('nlink', 1)
|
||||
entry.st_uid, entry.st_gid = get_item_uid_gid(item, numeric=self.numeric_ids,
|
||||
uid_default=self.default_uid, gid_default=self.default_gid,
|
||||
uid_forced=self.uid_forced, gid_forced=self.gid_forced)
|
||||
entry.st_rdev = item.get('rdev', 0)
|
||||
entry.st_nlink = item.get("nlink", 1)
|
||||
entry.st_uid, entry.st_gid = get_item_uid_gid(
|
||||
item,
|
||||
numeric=self.numeric_ids,
|
||||
uid_default=self.default_uid,
|
||||
gid_default=self.default_gid,
|
||||
uid_forced=self.uid_forced,
|
||||
gid_forced=self.gid_forced,
|
||||
)
|
||||
entry.st_rdev = item.get("rdev", 0)
|
||||
entry.st_size = item.get_size()
|
||||
entry.st_blksize = 512
|
||||
entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize
|
||||
# note: older archives only have mtime (not atime nor ctime)
|
||||
entry.st_mtime_ns = mtime_ns = item.mtime
|
||||
entry.st_atime_ns = item.get('atime', mtime_ns)
|
||||
entry.st_ctime_ns = item.get('ctime', mtime_ns)
|
||||
entry.st_birthtime_ns = item.get('birthtime', mtime_ns)
|
||||
entry.st_atime_ns = item.get("atime", mtime_ns)
|
||||
entry.st_ctime_ns = item.get("ctime", mtime_ns)
|
||||
entry.st_birthtime_ns = item.get("birthtime", mtime_ns)
|
||||
return entry
|
||||
|
||||
@async_wrapper
|
||||
@ -595,22 +616,22 @@ def getattr(self, inode, ctx=None):
|
||||
@async_wrapper
|
||||
def listxattr(self, inode, ctx=None):
|
||||
item = self.get_item(inode)
|
||||
return item.get('xattrs', {}).keys()
|
||||
return item.get("xattrs", {}).keys()
|
||||
|
||||
@async_wrapper
|
||||
def getxattr(self, inode, name, ctx=None):
|
||||
item = self.get_item(inode)
|
||||
try:
|
||||
return item.get('xattrs', {})[name] or b''
|
||||
return item.get("xattrs", {})[name] or b""
|
||||
except KeyError:
|
||||
raise llfuse.FUSEError(llfuse.ENOATTR) from None
|
||||
|
||||
@async_wrapper
|
||||
def lookup(self, parent_inode, name, ctx=None):
|
||||
self.check_pending_archive(parent_inode)
|
||||
if name == b'.':
|
||||
if name == b".":
|
||||
inode = parent_inode
|
||||
elif name == b'..':
|
||||
elif name == b"..":
|
||||
inode = self.parent[parent_inode]
|
||||
else:
|
||||
inode = self.contents[parent_inode].get(name)
|
||||
@ -622,12 +643,14 @@ def lookup(self, parent_inode, name, ctx=None):
|
||||
def open(self, inode, flags, ctx=None):
|
||||
if not self.allow_damaged_files:
|
||||
item = self.get_item(inode)
|
||||
if 'chunks_healthy' in item:
|
||||
if "chunks_healthy" in item:
|
||||
# Processed archive items don't carry the path anymore; for converting the inode
|
||||
# to the path we'd either have to store the inverse of the current structure,
|
||||
# or search the entire archive. So we just don't print it. It's easy to correlate anyway.
|
||||
logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. '
|
||||
'Mount with allow_damaged_files to read damaged files.')
|
||||
logger.warning(
|
||||
"File has damaged (all-zero) chunks. Try running borg check --repair. "
|
||||
"Mount with allow_damaged_files to read damaged files."
|
||||
)
|
||||
raise llfuse.FUSEError(errno.EIO)
|
||||
return llfuse.FileInfo(fh=inode) if has_pyfuse3 else inode
|
||||
|
||||
@ -669,7 +692,7 @@ def read(self, fh, offset, size):
|
||||
if offset + n < len(data):
|
||||
# chunk was only partially read, cache it
|
||||
self.data_cache[id] = data
|
||||
parts.append(data[offset:offset + n])
|
||||
parts.append(data[offset : offset + n])
|
||||
offset = 0
|
||||
size -= n
|
||||
if not size:
|
||||
@ -678,12 +701,13 @@ def read(self, fh, offset, size):
|
||||
else:
|
||||
self._last_pos[fh] = (chunk_no, chunk_offset)
|
||||
break
|
||||
return b''.join(parts)
|
||||
return b"".join(parts)
|
||||
|
||||
# note: we can't have a generator (with yield) and not a generator (async) in the same method
|
||||
if has_pyfuse3:
|
||||
|
||||
async def readdir(self, fh, off, token):
|
||||
entries = [(b'.', fh), (b'..', self.parent[fh])]
|
||||
entries = [(b".", fh), (b"..", self.parent[fh])]
|
||||
entries.extend(self.contents[fh].items())
|
||||
for i, (name, inode) in enumerate(entries[off:], off):
|
||||
attrs = self._getattr(inode)
|
||||
@ -691,8 +715,9 @@ async def readdir(self, fh, off, token):
|
||||
break
|
||||
|
||||
else:
|
||||
|
||||
def readdir(self, fh, off):
|
||||
entries = [(b'.', fh), (b'..', self.parent[fh])]
|
||||
entries = [(b".", fh), (b"..", self.parent[fh])]
|
||||
entries.extend(self.contents[fh].items())
|
||||
for i, (name, inode) in enumerate(entries[off:], off):
|
||||
attrs = self._getattr(inode)
|
||||
|
@ -4,11 +4,11 @@
|
||||
|
||||
import os
|
||||
|
||||
BORG_FUSE_IMPL = os.environ.get('BORG_FUSE_IMPL', 'pyfuse3,llfuse')
|
||||
BORG_FUSE_IMPL = os.environ.get("BORG_FUSE_IMPL", "pyfuse3,llfuse")
|
||||
|
||||
for FUSE_IMPL in BORG_FUSE_IMPL.split(','):
|
||||
for FUSE_IMPL in BORG_FUSE_IMPL.split(","):
|
||||
FUSE_IMPL = FUSE_IMPL.strip()
|
||||
if FUSE_IMPL == 'pyfuse3':
|
||||
if FUSE_IMPL == "pyfuse3":
|
||||
try:
|
||||
import pyfuse3 as llfuse
|
||||
except ImportError:
|
||||
@ -17,7 +17,7 @@
|
||||
has_llfuse = False
|
||||
has_pyfuse3 = True
|
||||
break
|
||||
elif FUSE_IMPL == 'llfuse':
|
||||
elif FUSE_IMPL == "llfuse":
|
||||
try:
|
||||
import llfuse
|
||||
except ImportError:
|
||||
@ -26,7 +26,7 @@
|
||||
has_llfuse = True
|
||||
has_pyfuse3 = False
|
||||
break
|
||||
elif FUSE_IMPL == 'none':
|
||||
elif FUSE_IMPL == "none":
|
||||
pass
|
||||
else:
|
||||
raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL)
|
||||
|
@ -24,7 +24,7 @@
|
||||
# generic mechanism to enable users to invoke workarounds by setting the
|
||||
# BORG_WORKAROUNDS environment variable to a list of comma-separated strings.
|
||||
# see the docs for a list of known workaround strings.
|
||||
workarounds = tuple(os.environ.get('BORG_WORKAROUNDS', '').split(','))
|
||||
workarounds = tuple(os.environ.get("BORG_WORKAROUNDS", "").split(","))
|
||||
|
||||
"""
|
||||
The global exit_code variable is used so that modules other than archiver can increase the program exit code if a
|
||||
|
@ -23,15 +23,16 @@ class ExtensionModuleError(Error):
|
||||
|
||||
def check_extension_modules():
|
||||
from .. import platform, compress, crypto, item, chunker, hashindex
|
||||
if hashindex.API_VERSION != '1.2_01':
|
||||
|
||||
if hashindex.API_VERSION != "1.2_01":
|
||||
raise ExtensionModuleError
|
||||
if chunker.API_VERSION != '1.2_01':
|
||||
if chunker.API_VERSION != "1.2_01":
|
||||
raise ExtensionModuleError
|
||||
if compress.API_VERSION != '1.2_02':
|
||||
if compress.API_VERSION != "1.2_02":
|
||||
raise ExtensionModuleError
|
||||
if crypto.low_level.API_VERSION != '1.3_01':
|
||||
if crypto.low_level.API_VERSION != "1.3_01":
|
||||
raise ExtensionModuleError
|
||||
if item.API_VERSION != '1.2_01':
|
||||
if item.API_VERSION != "1.2_01":
|
||||
raise ExtensionModuleError
|
||||
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.2_05':
|
||||
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != "1.2_05":
|
||||
raise ExtensionModuleError
|
||||
|
@ -3,6 +3,7 @@
|
||||
|
||||
class StableDict(dict):
|
||||
"""A dict subclass with stable items() ordering"""
|
||||
|
||||
def items(self):
|
||||
return sorted(super().items())
|
||||
|
||||
@ -20,8 +21,8 @@ def __init__(self, allocator, size=4096, limit=None):
|
||||
Initialize the buffer: use allocator(size) call to allocate a buffer.
|
||||
Optionally, set the upper <limit> for the buffer size.
|
||||
"""
|
||||
assert callable(allocator), 'must give alloc(size) function as first param'
|
||||
assert limit is None or size <= limit, 'initial size must be <= limit'
|
||||
assert callable(allocator), "must give alloc(size) function as first param"
|
||||
assert limit is None or size <= limit, "initial size must be <= limit"
|
||||
self.allocator = allocator
|
||||
self.limit = limit
|
||||
self.resize(size, init=True)
|
||||
|
@ -5,6 +5,7 @@
|
||||
|
||||
class Error(Exception):
|
||||
"""Error: {}"""
|
||||
|
||||
# Error base class
|
||||
|
||||
# if we raise such an Error and it is only caught by the uppermost
|
||||
@ -26,6 +27,7 @@ def get_message(self):
|
||||
|
||||
class ErrorWithTraceback(Error):
|
||||
"""Error: {}"""
|
||||
|
||||
# like Error, but show a traceback also
|
||||
traceback = True
|
||||
|
||||
|
@ -16,6 +16,7 @@
|
||||
from ..constants import * # NOQA
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
|
||||
@ -50,32 +51,32 @@ def get_base_dir():
|
||||
- ~$USER, if USER is set
|
||||
- ~
|
||||
"""
|
||||
base_dir = os.environ.get('BORG_BASE_DIR') or os.environ.get('HOME')
|
||||
base_dir = os.environ.get("BORG_BASE_DIR") or os.environ.get("HOME")
|
||||
# os.path.expanduser() behaves differently for '~' and '~someuser' as
|
||||
# parameters: when called with an explicit username, the possibly set
|
||||
# environment variable HOME is no longer respected. So we have to check if
|
||||
# it is set and only expand the user's home directory if HOME is unset.
|
||||
if not base_dir:
|
||||
base_dir = os.path.expanduser('~%s' % os.environ.get('USER', ''))
|
||||
base_dir = os.path.expanduser("~%s" % os.environ.get("USER", ""))
|
||||
return base_dir
|
||||
|
||||
|
||||
def get_keys_dir():
|
||||
"""Determine where to repository keys and cache"""
|
||||
keys_dir = os.environ.get('BORG_KEYS_DIR')
|
||||
keys_dir = os.environ.get("BORG_KEYS_DIR")
|
||||
if keys_dir is None:
|
||||
# note: do not just give this as default to the environment.get(), see issue #5979.
|
||||
keys_dir = os.path.join(get_config_dir(), 'keys')
|
||||
keys_dir = os.path.join(get_config_dir(), "keys")
|
||||
ensure_dir(keys_dir)
|
||||
return keys_dir
|
||||
|
||||
|
||||
def get_security_dir(repository_id=None):
|
||||
"""Determine where to store local security information."""
|
||||
security_dir = os.environ.get('BORG_SECURITY_DIR')
|
||||
security_dir = os.environ.get("BORG_SECURITY_DIR")
|
||||
if security_dir is None:
|
||||
# note: do not just give this as default to the environment.get(), see issue #5979.
|
||||
security_dir = os.path.join(get_config_dir(), 'security')
|
||||
security_dir = os.path.join(get_config_dir(), "security")
|
||||
if repository_id:
|
||||
security_dir = os.path.join(security_dir, repository_id)
|
||||
ensure_dir(security_dir)
|
||||
@ -85,22 +86,28 @@ def get_security_dir(repository_id=None):
|
||||
def get_cache_dir():
|
||||
"""Determine where to repository keys and cache"""
|
||||
# Get cache home path
|
||||
cache_home = os.path.join(get_base_dir(), '.cache')
|
||||
cache_home = os.path.join(get_base_dir(), ".cache")
|
||||
# Try to use XDG_CACHE_HOME instead if BORG_BASE_DIR isn't explicitly set
|
||||
if not os.environ.get('BORG_BASE_DIR'):
|
||||
cache_home = os.environ.get('XDG_CACHE_HOME', cache_home)
|
||||
if not os.environ.get("BORG_BASE_DIR"):
|
||||
cache_home = os.environ.get("XDG_CACHE_HOME", cache_home)
|
||||
# Use BORG_CACHE_DIR if set, otherwise assemble final path from cache home path
|
||||
cache_dir = os.environ.get('BORG_CACHE_DIR', os.path.join(cache_home, 'borg'))
|
||||
cache_dir = os.environ.get("BORG_CACHE_DIR", os.path.join(cache_home, "borg"))
|
||||
# Create path if it doesn't exist yet
|
||||
ensure_dir(cache_dir)
|
||||
cache_tag_fn = os.path.join(cache_dir, CACHE_TAG_NAME)
|
||||
if not os.path.exists(cache_tag_fn):
|
||||
cache_tag_contents = CACHE_TAG_CONTENTS + textwrap.dedent("""
|
||||
cache_tag_contents = (
|
||||
CACHE_TAG_CONTENTS
|
||||
+ textwrap.dedent(
|
||||
"""
|
||||
# This file is a cache directory tag created by Borg.
|
||||
# For information about cache directory tags, see:
|
||||
# http://www.bford.info/cachedir/spec.html
|
||||
""").encode('ascii')
|
||||
"""
|
||||
).encode("ascii")
|
||||
)
|
||||
from ..platform import SaveFile
|
||||
|
||||
with SaveFile(cache_tag_fn, binary=True) as fd:
|
||||
fd.write(cache_tag_contents)
|
||||
return cache_dir
|
||||
@ -109,12 +116,12 @@ def get_cache_dir():
|
||||
def get_config_dir():
|
||||
"""Determine where to store whole config"""
|
||||
# Get config home path
|
||||
config_home = os.path.join(get_base_dir(), '.config')
|
||||
config_home = os.path.join(get_base_dir(), ".config")
|
||||
# Try to use XDG_CONFIG_HOME instead if BORG_BASE_DIR isn't explicitly set
|
||||
if not os.environ.get('BORG_BASE_DIR'):
|
||||
config_home = os.environ.get('XDG_CONFIG_HOME', config_home)
|
||||
if not os.environ.get("BORG_BASE_DIR"):
|
||||
config_home = os.environ.get("XDG_CONFIG_HOME", config_home)
|
||||
# Use BORG_CONFIG_DIR if set, otherwise assemble final path from config home path
|
||||
config_dir = os.environ.get('BORG_CONFIG_DIR', os.path.join(config_home, 'borg'))
|
||||
config_dir = os.environ.get("BORG_CONFIG_DIR", os.path.join(config_home, "borg"))
|
||||
# Create path if it doesn't exist yet
|
||||
ensure_dir(config_dir)
|
||||
return config_dir
|
||||
@ -130,7 +137,7 @@ def dir_is_cachedir(path):
|
||||
tag_path = os.path.join(path, CACHE_TAG_NAME)
|
||||
try:
|
||||
if os.path.exists(tag_path):
|
||||
with open(tag_path, 'rb') as tag_file:
|
||||
with open(tag_path, "rb") as tag_file:
|
||||
tag_data = tag_file.read(len(CACHE_TAG_CONTENTS))
|
||||
if tag_data == CACHE_TAG_CONTENTS:
|
||||
return True
|
||||
@ -157,13 +164,12 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present):
|
||||
return tag_names
|
||||
|
||||
|
||||
_safe_re = re.compile(r'^((\.\.)?/+)+')
|
||||
_safe_re = re.compile(r"^((\.\.)?/+)+")
|
||||
|
||||
|
||||
def make_path_safe(path):
|
||||
"""Make path safe by making it relative and local
|
||||
"""
|
||||
return _safe_re.sub('', path) or '.'
|
||||
"""Make path safe by making it relative and local"""
|
||||
return _safe_re.sub("", path) or "."
|
||||
|
||||
|
||||
class HardLinkManager:
|
||||
@ -189,6 +195,7 @@ class HardLinkManager:
|
||||
For better hardlink support (including the very first hardlink item for each group of same-target hardlinks),
|
||||
we would need a 2-pass processing, which is not yet implemented.
|
||||
"""
|
||||
|
||||
def __init__(self, *, id_type, info_type):
|
||||
self._map = {}
|
||||
self.id_type = id_type
|
||||
@ -198,21 +205,21 @@ def borg1_hardlinkable(self, mode): # legacy
|
||||
return stat.S_ISREG(mode) or stat.S_ISBLK(mode) or stat.S_ISCHR(mode) or stat.S_ISFIFO(mode)
|
||||
|
||||
def borg1_hardlink_master(self, item): # legacy
|
||||
return item.get('hardlink_master', True) and 'source' not in item and self.borg1_hardlinkable(item.mode)
|
||||
return item.get("hardlink_master", True) and "source" not in item and self.borg1_hardlinkable(item.mode)
|
||||
|
||||
def borg1_hardlink_slave(self, item): # legacy
|
||||
return 'source' in item and self.borg1_hardlinkable(item.mode)
|
||||
return "source" in item and self.borg1_hardlinkable(item.mode)
|
||||
|
||||
def hardlink_id_from_path(self, path):
|
||||
"""compute a hardlink id from a path"""
|
||||
assert isinstance(path, str)
|
||||
return hashlib.sha256(path.encode('utf-8', errors='surrogateescape')).digest()
|
||||
return hashlib.sha256(path.encode("utf-8", errors="surrogateescape")).digest()
|
||||
|
||||
def hardlink_id_from_inode(self, *, ino, dev):
|
||||
"""compute a hardlink id from an inode"""
|
||||
assert isinstance(ino, int)
|
||||
assert isinstance(dev, int)
|
||||
return hashlib.sha256(f'{ino}/{dev}'.encode()).digest()
|
||||
return hashlib.sha256(f"{ino}/{dev}".encode()).digest()
|
||||
|
||||
def remember(self, *, id, info):
|
||||
"""
|
||||
@ -243,7 +250,7 @@ def scandir_keyfunc(dirent):
|
||||
return (0, dirent.inode())
|
||||
except OSError as e:
|
||||
# maybe a permission denied error while doing a stat() on the dirent
|
||||
logger.debug('scandir_inorder: Unable to stat %s: %s', dirent.path, e)
|
||||
logger.debug("scandir_inorder: Unable to stat %s: %s", dirent.path, e)
|
||||
# order this dirent after all the others lexically by file name
|
||||
# we may not break the whole scandir just because of an exception in one dirent
|
||||
# ignore the exception for now, since another stat will be done later anyways
|
||||
@ -268,7 +275,7 @@ def secure_erase(path, *, avoid_collateral_damage):
|
||||
If avoid_collateral_damage is False, we always secure erase.
|
||||
If there are hardlinks pointing to the same inode as <path>, they will contain random garbage afterwards.
|
||||
"""
|
||||
with open(path, 'r+b') as fd:
|
||||
with open(path, "r+b") as fd:
|
||||
st = os.stat(fd.fileno())
|
||||
if not (st.st_nlink > 1 and avoid_collateral_damage):
|
||||
fd.write(os.urandom(st.st_size))
|
||||
@ -303,7 +310,7 @@ def safe_unlink(path):
|
||||
# no other hardlink! try to recover free space by truncating this file.
|
||||
try:
|
||||
# Do not create *path* if it does not exist, open for truncation in r+b mode (=O_RDWR|O_BINARY).
|
||||
with open(path, 'r+b') as fd:
|
||||
with open(path, "r+b") as fd:
|
||||
fd.truncate()
|
||||
except OSError:
|
||||
# truncate didn't work, so we still have the original unlink issue - give up:
|
||||
@ -314,10 +321,10 @@ def safe_unlink(path):
|
||||
|
||||
|
||||
def dash_open(path, mode):
|
||||
assert '+' not in mode # the streams are either r or w, but never both
|
||||
if path == '-':
|
||||
stream = sys.stdin if 'r' in mode else sys.stdout
|
||||
return stream.buffer if 'b' in mode else stream
|
||||
assert "+" not in mode # the streams are either r or w, but never both
|
||||
if path == "-":
|
||||
stream = sys.stdin if "r" in mode else sys.stdout
|
||||
return stream.buffer if "b" in mode else stream
|
||||
else:
|
||||
return open(path, mode)
|
||||
|
||||
@ -325,17 +332,17 @@ def dash_open(path, mode):
|
||||
def O_(*flags):
|
||||
result = 0
|
||||
for flag in flags:
|
||||
result |= getattr(os, 'O_' + flag, 0)
|
||||
result |= getattr(os, "O_" + flag, 0)
|
||||
return result
|
||||
|
||||
|
||||
flags_base = O_('BINARY', 'NOCTTY', 'RDONLY')
|
||||
flags_special = flags_base | O_('NOFOLLOW') # BLOCK == wait when reading devices or fifos
|
||||
flags_base = O_("BINARY", "NOCTTY", "RDONLY")
|
||||
flags_special = flags_base | O_("NOFOLLOW") # BLOCK == wait when reading devices or fifos
|
||||
flags_special_follow = flags_base # BLOCK == wait when reading symlinked devices or fifos
|
||||
flags_normal = flags_base | O_('NONBLOCK', 'NOFOLLOW')
|
||||
flags_noatime = flags_normal | O_('NOATIME')
|
||||
flags_root = O_('RDONLY')
|
||||
flags_dir = O_('DIRECTORY', 'RDONLY', 'NOFOLLOW')
|
||||
flags_normal = flags_base | O_("NONBLOCK", "NOFOLLOW")
|
||||
flags_noatime = flags_normal | O_("NOATIME")
|
||||
flags_root = O_("RDONLY")
|
||||
flags_dir = O_("DIRECTORY", "RDONLY", "NOFOLLOW")
|
||||
|
||||
|
||||
def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
|
||||
@ -362,7 +369,7 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
|
||||
return None
|
||||
_flags_normal = flags
|
||||
if noatime:
|
||||
_flags_noatime = _flags_normal | O_('NOATIME')
|
||||
_flags_noatime = _flags_normal | O_("NOATIME")
|
||||
try:
|
||||
# if we have O_NOATIME, this likely will succeed if we are root or owner of file:
|
||||
fd = os.open(fname, _flags_noatime, dir_fd=parent_fd)
|
||||
@ -375,7 +382,8 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
|
||||
except OSError as exc:
|
||||
# O_NOATIME causes EROFS when accessing a volume shadow copy in WSL1
|
||||
from . import workarounds
|
||||
if 'retry_erofs' in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal:
|
||||
|
||||
if "retry_erofs" in workarounds and exc.errno == errno.EROFS and _flags_noatime != _flags_normal:
|
||||
fd = os.open(fname, _flags_normal, dir_fd=parent_fd)
|
||||
else:
|
||||
raise
|
||||
@ -407,6 +415,6 @@ def os_stat(*, path=None, parent_fd=None, name=None, follow_symlinks=False):
|
||||
def umount(mountpoint):
|
||||
env = prepare_subprocess_env(system=True)
|
||||
try:
|
||||
return subprocess.call(['fusermount', '-u', mountpoint], env=env)
|
||||
return subprocess.call(["fusermount", "-u", mountpoint], env=env)
|
||||
except FileNotFoundError:
|
||||
return subprocess.call(['umount', mountpoint], env=env)
|
||||
return subprocess.call(["umount", mountpoint], env=env)
|
||||
|
@ -9,6 +9,7 @@
|
||||
from .errors import Error
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from .datastruct import StableDict
|
||||
@ -26,10 +27,10 @@ class MandatoryFeatureUnsupported(Error):
|
||||
"""Unsupported repository feature(s) {}. A newer version of borg is required to access this repository."""
|
||||
|
||||
|
||||
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')
|
||||
ArchiveInfo = namedtuple("ArchiveInfo", "name id ts")
|
||||
|
||||
AI_HUMAN_SORT_KEYS = ['timestamp'] + list(ArchiveInfo._fields)
|
||||
AI_HUMAN_SORT_KEYS.remove('ts')
|
||||
AI_HUMAN_SORT_KEYS = ["timestamp"] + list(ArchiveInfo._fields)
|
||||
AI_HUMAN_SORT_KEYS.remove("ts")
|
||||
|
||||
|
||||
class Archives(abc.MutableMapping):
|
||||
@ -38,6 +39,7 @@ class Archives(abc.MutableMapping):
|
||||
and we can deal with str keys (and it internally encodes to byte keys) and either
|
||||
str timestamps or datetime timestamps.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# key: str archive name, value: dict('id': bytes_id, 'time': str_iso_ts)
|
||||
self._archives = {}
|
||||
@ -53,8 +55,8 @@ def __getitem__(self, name):
|
||||
values = self._archives.get(name)
|
||||
if values is None:
|
||||
raise KeyError
|
||||
ts = parse_timestamp(values['time'])
|
||||
return ArchiveInfo(name=name, id=values['id'], ts=ts)
|
||||
ts = parse_timestamp(values["time"])
|
||||
return ArchiveInfo(name=name, id=values["id"], ts=ts)
|
||||
|
||||
def __setitem__(self, name, info):
|
||||
assert isinstance(name, str)
|
||||
@ -64,13 +66,15 @@ def __setitem__(self, name, info):
|
||||
if isinstance(ts, datetime):
|
||||
ts = ts.replace(tzinfo=None).strftime(ISO_FORMAT)
|
||||
assert isinstance(ts, str)
|
||||
self._archives[name] = {'id': id, 'time': ts}
|
||||
self._archives[name] = {"id": id, "time": ts}
|
||||
|
||||
def __delitem__(self, name):
|
||||
assert isinstance(name, str)
|
||||
del self._archives[name]
|
||||
|
||||
def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False):
|
||||
def list(
|
||||
self, *, glob=None, match_end=r"\Z", sort_by=(), consider_checkpoints=True, first=None, last=None, reverse=False
|
||||
):
|
||||
"""
|
||||
Return list of ArchiveInfo instances according to the parameters.
|
||||
|
||||
@ -84,17 +88,17 @@ def list(self, *, glob=None, match_end=r'\Z', sort_by=(), consider_checkpoints=T
|
||||
some callers EXPECT to iterate over all archives in a repo for correct operation.
|
||||
"""
|
||||
if isinstance(sort_by, (str, bytes)):
|
||||
raise TypeError('sort_by must be a sequence of str')
|
||||
regex = re.compile(shellpattern.translate(glob or '*', match_end=match_end))
|
||||
raise TypeError("sort_by must be a sequence of str")
|
||||
regex = re.compile(shellpattern.translate(glob or "*", match_end=match_end))
|
||||
archives = [x for x in self.values() if regex.match(x.name) is not None]
|
||||
if not consider_checkpoints:
|
||||
archives = [x for x in archives if '.checkpoint' not in x.name]
|
||||
archives = [x for x in archives if ".checkpoint" not in x.name]
|
||||
for sortkey in reversed(sort_by):
|
||||
archives.sort(key=attrgetter(sortkey))
|
||||
if first:
|
||||
archives = archives[:first]
|
||||
elif last:
|
||||
archives = archives[max(len(archives) - last, 0):]
|
||||
archives = archives[max(len(archives) - last, 0) :]
|
||||
if reverse:
|
||||
archives.reverse()
|
||||
return archives
|
||||
@ -103,17 +107,25 @@ def list_considering(self, args):
|
||||
"""
|
||||
get a list of archives, considering --first/last/prefix/glob-archives/sort/consider-checkpoints cmdline args
|
||||
"""
|
||||
name = getattr(args, 'name', None)
|
||||
consider_checkpoints = getattr(args, 'consider_checkpoints', None)
|
||||
name = getattr(args, "name", None)
|
||||
consider_checkpoints = getattr(args, "consider_checkpoints", None)
|
||||
if name is not None:
|
||||
raise Error('Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints.')
|
||||
return self.list(sort_by=args.sort_by.split(','), consider_checkpoints=consider_checkpoints, glob=args.glob_archives, first=args.first, last=args.last)
|
||||
raise Error(
|
||||
"Giving a specific name is incompatible with options --first, --last, -a / --glob-archives, and --consider-checkpoints."
|
||||
)
|
||||
return self.list(
|
||||
sort_by=args.sort_by.split(","),
|
||||
consider_checkpoints=consider_checkpoints,
|
||||
glob=args.glob_archives,
|
||||
first=args.first,
|
||||
last=args.last,
|
||||
)
|
||||
|
||||
def set_raw_dict(self, d):
|
||||
"""set the dict we get from the msgpack unpacker"""
|
||||
for k, v in d.items():
|
||||
assert isinstance(k, str)
|
||||
assert isinstance(v, dict) and 'id' in v and 'time' in v
|
||||
assert isinstance(v, dict) and "id" in v and "time" in v
|
||||
self._archives[k] = v
|
||||
|
||||
def get_raw_dict(self):
|
||||
@ -122,7 +134,6 @@ def get_raw_dict(self):
|
||||
|
||||
|
||||
class Manifest:
|
||||
|
||||
@enum.unique
|
||||
class Operation(enum.Enum):
|
||||
# The comments here only roughly describe the scope of each feature. In the end, additions need to be
|
||||
@ -133,25 +144,25 @@ class Operation(enum.Enum):
|
||||
|
||||
# The READ operation describes which features are needed to safely list and extract the archives in the
|
||||
# repository.
|
||||
READ = 'read'
|
||||
READ = "read"
|
||||
# The CHECK operation is for all operations that need either to understand every detail
|
||||
# of the repository (for consistency checks and repairs) or are seldom used functions that just
|
||||
# should use the most restrictive feature set because more fine grained compatibility tracking is
|
||||
# not needed.
|
||||
CHECK = 'check'
|
||||
CHECK = "check"
|
||||
# The WRITE operation is for adding archives. Features here ensure that older clients don't add archives
|
||||
# in an old format, or is used to lock out clients that for other reasons can no longer safely add new
|
||||
# archives.
|
||||
WRITE = 'write'
|
||||
WRITE = "write"
|
||||
# The DELETE operation is for all operations (like archive deletion) that need a 100% correct reference
|
||||
# count and the need to be able to find all (directly and indirectly) referenced chunks of a given archive.
|
||||
DELETE = 'delete'
|
||||
DELETE = "delete"
|
||||
|
||||
NO_OPERATION_CHECK = tuple()
|
||||
|
||||
SUPPORTED_REPO_FEATURES = frozenset([])
|
||||
|
||||
MANIFEST_ID = b'\0' * 32
|
||||
MANIFEST_ID = b"\0" * 32
|
||||
|
||||
def __init__(self, key, repository, item_keys=None):
|
||||
self.archives = Archives()
|
||||
@ -175,6 +186,7 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
|
||||
from ..item import ManifestItem
|
||||
from ..crypto.key import key_factory, tam_required_file, tam_required
|
||||
from ..repository import Repository
|
||||
|
||||
try:
|
||||
cdata = repository.get(cls.MANIFEST_ID)
|
||||
except Repository.ObjectNotFound:
|
||||
@ -183,26 +195,28 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
|
||||
key = key_factory(repository, cdata)
|
||||
manifest = cls(key, repository)
|
||||
data = key.decrypt(cls.MANIFEST_ID, cdata)
|
||||
manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(data, force_tam_not_required=force_tam_not_required)
|
||||
manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(
|
||||
data, force_tam_not_required=force_tam_not_required
|
||||
)
|
||||
m = ManifestItem(internal_dict=manifest_dict)
|
||||
manifest.id = key.id_hash(data)
|
||||
if m.get('version') not in (1, 2):
|
||||
raise ValueError('Invalid manifest version')
|
||||
if m.get("version") not in (1, 2):
|
||||
raise ValueError("Invalid manifest version")
|
||||
manifest.archives.set_raw_dict(m.archives)
|
||||
manifest.timestamp = m.get('timestamp')
|
||||
manifest.timestamp = m.get("timestamp")
|
||||
manifest.config = m.config
|
||||
# valid item keys are whatever is known in the repo or every key we know
|
||||
manifest.item_keys = ITEM_KEYS | frozenset(m.get('item_keys', []))
|
||||
manifest.item_keys = ITEM_KEYS | frozenset(m.get("item_keys", []))
|
||||
|
||||
if manifest.tam_verified:
|
||||
manifest_required = manifest.config.get('tam_required', False)
|
||||
manifest_required = manifest.config.get("tam_required", False)
|
||||
security_required = tam_required(repository)
|
||||
if manifest_required and not security_required:
|
||||
logger.debug('Manifest is TAM verified and says TAM is required, updating security database...')
|
||||
logger.debug("Manifest is TAM verified and says TAM is required, updating security database...")
|
||||
file = tam_required_file(repository)
|
||||
open(file, 'w').close()
|
||||
open(file, "w").close()
|
||||
if not manifest_required and security_required:
|
||||
logger.debug('Manifest is TAM verified and says TAM is *not* required, updating security database...')
|
||||
logger.debug("Manifest is TAM verified and says TAM is *not* required, updating security database...")
|
||||
os.unlink(tam_required_file(repository))
|
||||
manifest.check_repository_compatibility(operations)
|
||||
return manifest, key
|
||||
@ -210,32 +224,33 @@ def load(cls, repository, operations, key=None, force_tam_not_required=False):
|
||||
def check_repository_compatibility(self, operations):
|
||||
for operation in operations:
|
||||
assert isinstance(operation, self.Operation)
|
||||
feature_flags = self.config.get('feature_flags', None)
|
||||
feature_flags = self.config.get("feature_flags", None)
|
||||
if feature_flags is None:
|
||||
return
|
||||
if operation.value not in feature_flags:
|
||||
continue
|
||||
requirements = feature_flags[operation.value]
|
||||
if 'mandatory' in requirements:
|
||||
unsupported = set(requirements['mandatory']) - self.SUPPORTED_REPO_FEATURES
|
||||
if "mandatory" in requirements:
|
||||
unsupported = set(requirements["mandatory"]) - self.SUPPORTED_REPO_FEATURES
|
||||
if unsupported:
|
||||
raise MandatoryFeatureUnsupported(list(unsupported))
|
||||
|
||||
def get_all_mandatory_features(self):
|
||||
result = {}
|
||||
feature_flags = self.config.get('feature_flags', None)
|
||||
feature_flags = self.config.get("feature_flags", None)
|
||||
if feature_flags is None:
|
||||
return result
|
||||
|
||||
for operation, requirements in feature_flags.items():
|
||||
if 'mandatory' in requirements:
|
||||
result[operation] = set(requirements['mandatory'])
|
||||
if "mandatory" in requirements:
|
||||
result[operation] = set(requirements["mandatory"])
|
||||
return result
|
||||
|
||||
def write(self):
|
||||
from ..item import ManifestItem
|
||||
|
||||
if self.key.tam_required:
|
||||
self.config['tam_required'] = True
|
||||
self.config["tam_required"] = True
|
||||
# self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
|
||||
if self.timestamp is None:
|
||||
self.timestamp = datetime.utcnow().strftime(ISO_FORMAT)
|
||||
|
@ -10,6 +10,7 @@
|
||||
from operator import attrgetter
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from .time import to_localtime
|
||||
@ -30,15 +31,17 @@ def prune_within(archives, hours, kept_because):
|
||||
return result
|
||||
|
||||
|
||||
PRUNING_PATTERNS = OrderedDict([
|
||||
("secondly", '%Y-%m-%d %H:%M:%S'),
|
||||
("minutely", '%Y-%m-%d %H:%M'),
|
||||
("hourly", '%Y-%m-%d %H'),
|
||||
("daily", '%Y-%m-%d'),
|
||||
("weekly", '%G-%V'),
|
||||
("monthly", '%Y-%m'),
|
||||
("yearly", '%Y'),
|
||||
])
|
||||
PRUNING_PATTERNS = OrderedDict(
|
||||
[
|
||||
("secondly", "%Y-%m-%d %H:%M:%S"),
|
||||
("minutely", "%Y-%m-%d %H:%M"),
|
||||
("hourly", "%Y-%m-%d %H"),
|
||||
("daily", "%Y-%m-%d"),
|
||||
("weekly", "%G-%V"),
|
||||
("monthly", "%Y-%m"),
|
||||
("yearly", "%Y"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def prune_split(archives, rule, n, kept_because=None):
|
||||
@ -51,7 +54,7 @@ def prune_split(archives, rule, n, kept_because=None):
|
||||
return keep
|
||||
|
||||
a = None
|
||||
for a in sorted(archives, key=attrgetter('ts'), reverse=True):
|
||||
for a in sorted(archives, key=attrgetter("ts"), reverse=True):
|
||||
period = to_localtime(a.ts).strftime(pattern)
|
||||
if period != last:
|
||||
last = period
|
||||
@ -63,14 +66,14 @@ def prune_split(archives, rule, n, kept_because=None):
|
||||
# Keep oldest archive if we didn't reach the target retention count
|
||||
if a is not None and len(keep) < n and a.id not in kept_because:
|
||||
keep.append(a)
|
||||
kept_because[a.id] = (rule+"[oldest]", len(keep))
|
||||
kept_because[a.id] = (rule + "[oldest]", len(keep))
|
||||
return keep
|
||||
|
||||
|
||||
def sysinfo():
|
||||
show_sysinfo = os.environ.get('BORG_SHOW_SYSINFO', 'yes').lower()
|
||||
if show_sysinfo == 'no':
|
||||
return ''
|
||||
show_sysinfo = os.environ.get("BORG_SHOW_SYSINFO", "yes").lower()
|
||||
if show_sysinfo == "no":
|
||||
return ""
|
||||
|
||||
python_implementation = platform.python_implementation()
|
||||
python_version = platform.python_version()
|
||||
@ -80,30 +83,34 @@ def sysinfo():
|
||||
uname = os.uname()
|
||||
except AttributeError:
|
||||
uname = None
|
||||
if sys.platform.startswith('linux'):
|
||||
linux_distribution = ('Unknown Linux', '', '')
|
||||
if sys.platform.startswith("linux"):
|
||||
linux_distribution = ("Unknown Linux", "", "")
|
||||
else:
|
||||
linux_distribution = None
|
||||
try:
|
||||
msgpack_version = '.'.join(str(v) for v in msgpack.version)
|
||||
msgpack_version = ".".join(str(v) for v in msgpack.version)
|
||||
except:
|
||||
msgpack_version = 'unknown'
|
||||
msgpack_version = "unknown"
|
||||
from ..fuse_impl import llfuse, BORG_FUSE_IMPL
|
||||
llfuse_name = llfuse.__name__ if llfuse else 'None'
|
||||
llfuse_version = (' %s' % llfuse.__version__) if llfuse else ''
|
||||
llfuse_info = f'{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]'
|
||||
|
||||
llfuse_name = llfuse.__name__ if llfuse else "None"
|
||||
llfuse_version = (" %s" % llfuse.__version__) if llfuse else ""
|
||||
llfuse_info = f"{llfuse_name}{llfuse_version} [{BORG_FUSE_IMPL}]"
|
||||
info = []
|
||||
if uname is not None:
|
||||
info.append('Platform: {}'.format(' '.join(uname)))
|
||||
info.append("Platform: {}".format(" ".join(uname)))
|
||||
if linux_distribution is not None:
|
||||
info.append('Linux: %s %s %s' % linux_distribution)
|
||||
info.append('Borg: {} Python: {} {} msgpack: {} fuse: {}'.format(
|
||||
borg_version, python_implementation, python_version, msgpack_version, llfuse_info))
|
||||
info.append('PID: %d CWD: %s' % (os.getpid(), os.getcwd()))
|
||||
info.append('sys.argv: %r' % sys.argv)
|
||||
info.append('SSH_ORIGINAL_COMMAND: %r' % os.environ.get('SSH_ORIGINAL_COMMAND'))
|
||||
info.append('')
|
||||
return '\n'.join(info)
|
||||
info.append("Linux: %s %s %s" % linux_distribution)
|
||||
info.append(
|
||||
"Borg: {} Python: {} {} msgpack: {} fuse: {}".format(
|
||||
borg_version, python_implementation, python_version, msgpack_version, llfuse_info
|
||||
)
|
||||
)
|
||||
info.append("PID: %d CWD: %s" % (os.getpid(), os.getcwd()))
|
||||
info.append("sys.argv: %r" % sys.argv)
|
||||
info.append("SSH_ORIGINAL_COMMAND: %r" % os.environ.get("SSH_ORIGINAL_COMMAND"))
|
||||
info.append("")
|
||||
return "\n".join(info)
|
||||
|
||||
|
||||
def log_multi(*msgs, level=logging.INFO, logger=logger):
|
||||
@ -133,7 +140,7 @@ def __init__(self, chunk_iterator, read_callback=None):
|
||||
"""
|
||||
self.chunk_iterator = chunk_iterator
|
||||
self.chunk_offset = 0
|
||||
self.chunk = b''
|
||||
self.chunk = b""
|
||||
self.exhausted = False
|
||||
self.read_callback = read_callback
|
||||
|
||||
@ -152,11 +159,11 @@ def _refill(self):
|
||||
|
||||
def _read(self, nbytes):
|
||||
if not nbytes:
|
||||
return b''
|
||||
return b""
|
||||
remaining = self._refill()
|
||||
will_read = min(remaining, nbytes)
|
||||
self.chunk_offset += will_read
|
||||
return self.chunk[self.chunk_offset - will_read:self.chunk_offset]
|
||||
return self.chunk[self.chunk_offset - will_read : self.chunk_offset]
|
||||
|
||||
def read(self, nbytes):
|
||||
parts = []
|
||||
@ -166,7 +173,7 @@ def read(self, nbytes):
|
||||
parts.append(read_data)
|
||||
if self.read_callback:
|
||||
self.read_callback(read_data)
|
||||
return b''.join(parts)
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def open_item(archive, item):
|
||||
@ -207,7 +214,7 @@ def read(self, n):
|
||||
super().close()
|
||||
except OSError:
|
||||
pass
|
||||
return ''
|
||||
return ""
|
||||
|
||||
def write(self, s):
|
||||
if not self.closed:
|
||||
@ -225,8 +232,8 @@ def iter_separated(fd, sep=None, read_size=4096):
|
||||
"""Iter over chunks of open file ``fd`` delimited by ``sep``. Doesn't trim."""
|
||||
buf = fd.read(read_size)
|
||||
is_str = isinstance(buf, str)
|
||||
part = '' if is_str else b''
|
||||
sep = sep or ('\n' if is_str else b'\n')
|
||||
part = "" if is_str else b""
|
||||
sep = sep or ("\n" if is_str else b"\n")
|
||||
while len(buf) > 0:
|
||||
part2, *items = buf.split(sep)
|
||||
*full, part = (part + part2, *items)
|
||||
@ -240,17 +247,17 @@ def iter_separated(fd, sep=None, read_size=4096):
|
||||
|
||||
def get_tar_filter(fname, decompress):
|
||||
# Note that filter is None if fname is '-'.
|
||||
if fname.endswith(('.tar.gz', '.tgz')):
|
||||
filter = 'gzip -d' if decompress else 'gzip'
|
||||
elif fname.endswith(('.tar.bz2', '.tbz')):
|
||||
filter = 'bzip2 -d' if decompress else 'bzip2'
|
||||
elif fname.endswith(('.tar.xz', '.txz')):
|
||||
filter = 'xz -d' if decompress else 'xz'
|
||||
elif fname.endswith(('.tar.lz4', )):
|
||||
filter = 'lz4 -d' if decompress else 'lz4'
|
||||
elif fname.endswith(('.tar.zstd', )):
|
||||
filter = 'zstd -d' if decompress else 'zstd'
|
||||
if fname.endswith((".tar.gz", ".tgz")):
|
||||
filter = "gzip -d" if decompress else "gzip"
|
||||
elif fname.endswith((".tar.bz2", ".tbz")):
|
||||
filter = "bzip2 -d" if decompress else "bzip2"
|
||||
elif fname.endswith((".tar.xz", ".txz")):
|
||||
filter = "xz -d" if decompress else "xz"
|
||||
elif fname.endswith((".tar.lz4",)):
|
||||
filter = "lz4 -d" if decompress else "lz4"
|
||||
elif fname.endswith((".tar.zstd",)):
|
||||
filter = "zstd -d" if decompress else "zstd"
|
||||
else:
|
||||
filter = None
|
||||
logger.debug('Automatically determined tar filter: %s', filter)
|
||||
logger.debug("Automatically determined tar filter: %s", filter)
|
||||
return filter
|
||||
|
@ -64,7 +64,7 @@
|
||||
|
||||
USE_BIN_TYPE = True
|
||||
RAW = False
|
||||
UNICODE_ERRORS = 'surrogateescape'
|
||||
UNICODE_ERRORS = "surrogateescape"
|
||||
|
||||
|
||||
class PackException(Exception):
|
||||
@ -76,13 +76,25 @@ class UnpackException(Exception):
|
||||
|
||||
|
||||
class Packer(mp_Packer):
|
||||
def __init__(self, *, default=None, unicode_errors=UNICODE_ERRORS,
|
||||
use_single_float=False, autoreset=True, use_bin_type=USE_BIN_TYPE,
|
||||
strict_types=False):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
default=None,
|
||||
unicode_errors=UNICODE_ERRORS,
|
||||
use_single_float=False,
|
||||
autoreset=True,
|
||||
use_bin_type=USE_BIN_TYPE,
|
||||
strict_types=False
|
||||
):
|
||||
assert unicode_errors == UNICODE_ERRORS
|
||||
super().__init__(default=default, unicode_errors=unicode_errors,
|
||||
use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type,
|
||||
strict_types=strict_types)
|
||||
super().__init__(
|
||||
default=default,
|
||||
unicode_errors=unicode_errors,
|
||||
use_single_float=use_single_float,
|
||||
autoreset=autoreset,
|
||||
use_bin_type=use_bin_type,
|
||||
strict_types=strict_types,
|
||||
)
|
||||
|
||||
def pack(self, obj):
|
||||
try:
|
||||
@ -108,18 +120,36 @@ def pack(o, stream, *, use_bin_type=USE_BIN_TYPE, unicode_errors=UNICODE_ERRORS,
|
||||
|
||||
|
||||
class Unpacker(mp_Unpacker):
|
||||
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=RAW,
|
||||
object_hook=None, object_pairs_hook=None, list_hook=None,
|
||||
unicode_errors=UNICODE_ERRORS, max_buffer_size=0,
|
||||
ext_hook=ExtType,
|
||||
strict_map_key=False):
|
||||
def __init__(
|
||||
self,
|
||||
file_like=None,
|
||||
*,
|
||||
read_size=0,
|
||||
use_list=True,
|
||||
raw=RAW,
|
||||
object_hook=None,
|
||||
object_pairs_hook=None,
|
||||
list_hook=None,
|
||||
unicode_errors=UNICODE_ERRORS,
|
||||
max_buffer_size=0,
|
||||
ext_hook=ExtType,
|
||||
strict_map_key=False
|
||||
):
|
||||
assert raw == RAW
|
||||
assert unicode_errors == UNICODE_ERRORS
|
||||
kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
|
||||
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
|
||||
unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
|
||||
ext_hook=ext_hook,
|
||||
strict_map_key=strict_map_key)
|
||||
kw = dict(
|
||||
file_like=file_like,
|
||||
read_size=read_size,
|
||||
use_list=use_list,
|
||||
raw=raw,
|
||||
object_hook=object_hook,
|
||||
object_pairs_hook=object_pairs_hook,
|
||||
list_hook=list_hook,
|
||||
unicode_errors=unicode_errors,
|
||||
max_buffer_size=max_buffer_size,
|
||||
ext_hook=ext_hook,
|
||||
strict_map_key=strict_map_key,
|
||||
)
|
||||
super().__init__(**kw)
|
||||
|
||||
def unpack(self):
|
||||
@ -141,28 +171,22 @@ def __next__(self):
|
||||
next = __next__
|
||||
|
||||
|
||||
def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
|
||||
strict_map_key=False,
|
||||
**kwargs):
|
||||
def unpackb(packed, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs):
|
||||
assert raw == RAW
|
||||
assert unicode_errors == UNICODE_ERRORS
|
||||
try:
|
||||
kw = dict(raw=raw, unicode_errors=unicode_errors,
|
||||
strict_map_key=strict_map_key)
|
||||
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
|
||||
kw.update(kwargs)
|
||||
return mp_unpackb(packed, **kw)
|
||||
except Exception as e:
|
||||
raise UnpackException(e)
|
||||
|
||||
|
||||
def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
|
||||
strict_map_key=False,
|
||||
**kwargs):
|
||||
def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS, strict_map_key=False, **kwargs):
|
||||
assert raw == RAW
|
||||
assert unicode_errors == UNICODE_ERRORS
|
||||
try:
|
||||
kw = dict(raw=raw, unicode_errors=unicode_errors,
|
||||
strict_map_key=strict_map_key)
|
||||
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
|
||||
kw.update(kwargs)
|
||||
return mp_unpack(stream, **kw)
|
||||
except Exception as e:
|
||||
@ -171,32 +195,34 @@ def unpack(stream, *, raw=RAW, unicode_errors=UNICODE_ERRORS,
|
||||
|
||||
# msgpacking related utilities -----------------------------------------------
|
||||
|
||||
|
||||
def is_slow_msgpack():
|
||||
import msgpack
|
||||
import msgpack.fallback
|
||||
|
||||
return msgpack.Packer is msgpack.fallback.Packer
|
||||
|
||||
|
||||
def is_supported_msgpack():
|
||||
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
|
||||
import msgpack
|
||||
return (1, 0, 3) <= msgpack.version <= (1, 0, 4) and \
|
||||
msgpack.version not in [] # < add bad releases here to deny list
|
||||
|
||||
return (1, 0, 3) <= msgpack.version <= (
|
||||
1,
|
||||
0,
|
||||
4,
|
||||
) and msgpack.version not in [] # < add bad releases here to deny list
|
||||
|
||||
|
||||
def get_limited_unpacker(kind):
|
||||
"""return a limited Unpacker because we should not trust msgpack data received from remote"""
|
||||
# Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for
|
||||
# unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N).
|
||||
args = dict(use_list=False, # return tuples, not lists
|
||||
max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
|
||||
)
|
||||
if kind in ('server', 'client'):
|
||||
args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE)) # return tuples, not lists
|
||||
if kind in ("server", "client"):
|
||||
pass # nothing special
|
||||
elif kind in ('manifest', 'key'):
|
||||
args.update(dict(use_list=True, # default value
|
||||
object_hook=StableDict,
|
||||
))
|
||||
elif kind in ("manifest", "key"):
|
||||
args.update(dict(use_list=True, object_hook=StableDict)) # default value
|
||||
else:
|
||||
raise ValueError('kind must be "server", "client", "manifest" or "key"')
|
||||
return Unpacker(**args)
|
||||
|
@ -15,6 +15,7 @@
|
||||
from string import Formatter
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from .errors import Error
|
||||
@ -28,34 +29,34 @@
|
||||
|
||||
|
||||
def bin_to_hex(binary):
|
||||
return hexlify(binary).decode('ascii')
|
||||
return hexlify(binary).decode("ascii")
|
||||
|
||||
|
||||
def safe_decode(s, coding='utf-8', errors='surrogateescape'):
|
||||
def safe_decode(s, coding="utf-8", errors="surrogateescape"):
|
||||
"""decode bytes to str, with round-tripping "invalid" bytes"""
|
||||
if s is None:
|
||||
return None
|
||||
return s.decode(coding, errors)
|
||||
|
||||
|
||||
def safe_encode(s, coding='utf-8', errors='surrogateescape'):
|
||||
def safe_encode(s, coding="utf-8", errors="surrogateescape"):
|
||||
"""encode str to bytes, with round-tripping "invalid" bytes"""
|
||||
if s is None:
|
||||
return None
|
||||
return s.encode(coding, errors)
|
||||
|
||||
|
||||
def remove_surrogates(s, errors='replace'):
|
||||
def remove_surrogates(s, errors="replace"):
|
||||
"""Replace surrogates generated by fsdecode with '?'"""
|
||||
return s.encode('utf-8', errors).decode('utf-8')
|
||||
return s.encode("utf-8", errors).decode("utf-8")
|
||||
|
||||
|
||||
def eval_escapes(s):
|
||||
"""Evaluate literal escape sequences in a string (eg `\\n` -> `\n`)."""
|
||||
return s.encode('ascii', 'backslashreplace').decode('unicode-escape')
|
||||
return s.encode("ascii", "backslashreplace").decode("unicode-escape")
|
||||
|
||||
|
||||
def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'):
|
||||
def decode_dict(d, keys, encoding="utf-8", errors="surrogateescape"):
|
||||
for key in keys:
|
||||
if isinstance(d.get(key), bytes):
|
||||
d[key] = d[key].decode(encoding, errors)
|
||||
@ -66,13 +67,13 @@ def positive_int_validator(value):
|
||||
"""argparse type for positive integers"""
|
||||
int_value = int(value)
|
||||
if int_value <= 0:
|
||||
raise argparse.ArgumentTypeError('A positive integer is required: %s' % value)
|
||||
raise argparse.ArgumentTypeError("A positive integer is required: %s" % value)
|
||||
return int_value
|
||||
|
||||
|
||||
def interval(s):
|
||||
"""Convert a string representing a valid interval to a number of hours."""
|
||||
multiplier = {'H': 1, 'd': 24, 'w': 24 * 7, 'm': 24 * 31, 'y': 24 * 365}
|
||||
multiplier = {"H": 1, "d": 24, "w": 24 * 7, "m": 24 * 31, "y": 24 * 365}
|
||||
|
||||
if s.endswith(tuple(multiplier.keys())):
|
||||
number = s[:-1]
|
||||
@ -80,8 +81,7 @@ def interval(s):
|
||||
else:
|
||||
# range suffixes in ascending multiplier order
|
||||
ranges = [k for k, v in sorted(multiplier.items(), key=lambda t: t[1])]
|
||||
raise argparse.ArgumentTypeError(
|
||||
f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}')
|
||||
raise argparse.ArgumentTypeError(f'Unexpected interval time unit "{s[-1]}": expected one of {ranges!r}')
|
||||
|
||||
try:
|
||||
hours = int(number) * multiplier[suffix]
|
||||
@ -89,17 +89,16 @@ def interval(s):
|
||||
hours = -1
|
||||
|
||||
if hours <= 0:
|
||||
raise argparse.ArgumentTypeError(
|
||||
'Unexpected interval number "%s": expected an integer greater than 0' % number)
|
||||
raise argparse.ArgumentTypeError('Unexpected interval number "%s": expected an integer greater than 0' % number)
|
||||
|
||||
return hours
|
||||
|
||||
|
||||
def ChunkerParams(s):
|
||||
params = s.strip().split(',')
|
||||
params = s.strip().split(",")
|
||||
count = len(params)
|
||||
if count == 0:
|
||||
raise ValueError('no chunker params given')
|
||||
raise ValueError("no chunker params given")
|
||||
algo = params[0].lower()
|
||||
if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size]
|
||||
block_size = int(params[1])
|
||||
@ -110,36 +109,36 @@ def ChunkerParams(s):
|
||||
# or in-memory chunk management.
|
||||
# choose the block (chunk) size wisely: if you have a lot of data and you cut
|
||||
# it into very small chunks, you are asking for trouble!
|
||||
raise ValueError('block_size must not be less than 64 Bytes')
|
||||
raise ValueError("block_size must not be less than 64 Bytes")
|
||||
if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
|
||||
raise ValueError('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE)
|
||||
raise ValueError("block_size and header_size must not exceed MAX_DATA_SIZE [%d]" % MAX_DATA_SIZE)
|
||||
return algo, block_size, header_size
|
||||
if algo == 'default' and count == 1: # default
|
||||
if algo == "default" and count == 1: # default
|
||||
return CHUNKER_PARAMS
|
||||
# this must stay last as it deals with old-style compat mode (no algorithm, 4 params, buzhash):
|
||||
if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
|
||||
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:])
|
||||
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4 :])
|
||||
if not (chunk_min <= chunk_mask <= chunk_max):
|
||||
raise ValueError('required: chunk_min <= chunk_mask <= chunk_max')
|
||||
raise ValueError("required: chunk_min <= chunk_mask <= chunk_max")
|
||||
if chunk_min < 6:
|
||||
# see comment in 'fixed' algo check
|
||||
raise ValueError('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)')
|
||||
raise ValueError("min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)")
|
||||
if chunk_max > 23:
|
||||
raise ValueError('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)')
|
||||
raise ValueError("max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)")
|
||||
return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
|
||||
raise ValueError('invalid chunker params')
|
||||
raise ValueError("invalid chunker params")
|
||||
|
||||
|
||||
def FilesCacheMode(s):
|
||||
ENTRIES_MAP = dict(ctime='c', mtime='m', size='s', inode='i', rechunk='r', disabled='d')
|
||||
VALID_MODES = ('cis', 'ims', 'cs', 'ms', 'cr', 'mr', 'd', 's') # letters in alpha order
|
||||
entries = set(s.strip().split(','))
|
||||
ENTRIES_MAP = dict(ctime="c", mtime="m", size="s", inode="i", rechunk="r", disabled="d")
|
||||
VALID_MODES = ("cis", "ims", "cs", "ms", "cr", "mr", "d", "s") # letters in alpha order
|
||||
entries = set(s.strip().split(","))
|
||||
if not entries <= set(ENTRIES_MAP):
|
||||
raise ValueError('cache mode must be a comma-separated list of: %s' % ','.join(sorted(ENTRIES_MAP)))
|
||||
raise ValueError("cache mode must be a comma-separated list of: %s" % ",".join(sorted(ENTRIES_MAP)))
|
||||
short_entries = {ENTRIES_MAP[entry] for entry in entries}
|
||||
mode = ''.join(sorted(short_entries))
|
||||
mode = "".join(sorted(short_entries))
|
||||
if mode not in VALID_MODES:
|
||||
raise ValueError('cache mode short must be one of: %s' % ','.join(VALID_MODES))
|
||||
raise ValueError("cache mode short must be one of: %s" % ",".join(VALID_MODES))
|
||||
return mode
|
||||
|
||||
|
||||
@ -151,9 +150,9 @@ def partial_format(format, mapping):
|
||||
"""
|
||||
for key, value in mapping.items():
|
||||
key = re.escape(key)
|
||||
format = re.sub(fr'(?<!\{{)((\{{{key}\}})|(\{{{key}:[^\}}]*\}}))',
|
||||
lambda match: match.group(1).format_map(mapping),
|
||||
format)
|
||||
format = re.sub(
|
||||
rf"(?<!\{{)((\{{{key}\}})|(\{{{key}:[^\}}]*\}}))", lambda match: match.group(1).format_map(mapping), format
|
||||
)
|
||||
return format
|
||||
|
||||
|
||||
@ -162,7 +161,7 @@ def __init__(self, dt):
|
||||
self.dt = dt
|
||||
|
||||
def __format__(self, format_spec):
|
||||
if format_spec == '':
|
||||
if format_spec == "":
|
||||
format_spec = ISO_FORMAT_NO_USECS
|
||||
return self.dt.__format__(format_spec)
|
||||
|
||||
@ -190,20 +189,21 @@ def format_line(format, data):
|
||||
def replace_placeholders(text, overrides={}):
|
||||
"""Replace placeholders in text with their values."""
|
||||
from ..platform import fqdn, hostname, getosusername
|
||||
|
||||
current_time = datetime.now(timezone.utc)
|
||||
data = {
|
||||
'pid': os.getpid(),
|
||||
'fqdn': fqdn,
|
||||
'reverse-fqdn': '.'.join(reversed(fqdn.split('.'))),
|
||||
'hostname': hostname,
|
||||
'now': DatetimeWrapper(current_time.astimezone(None)),
|
||||
'utcnow': DatetimeWrapper(current_time),
|
||||
'user': getosusername(),
|
||||
'uuid4': str(uuid.uuid4()),
|
||||
'borgversion': borg_version,
|
||||
'borgmajor': '%d' % borg_version_tuple[:1],
|
||||
'borgminor': '%d.%d' % borg_version_tuple[:2],
|
||||
'borgpatch': '%d.%d.%d' % borg_version_tuple[:3],
|
||||
"pid": os.getpid(),
|
||||
"fqdn": fqdn,
|
||||
"reverse-fqdn": ".".join(reversed(fqdn.split("."))),
|
||||
"hostname": hostname,
|
||||
"now": DatetimeWrapper(current_time.astimezone(None)),
|
||||
"utcnow": DatetimeWrapper(current_time),
|
||||
"user": getosusername(),
|
||||
"uuid4": str(uuid.uuid4()),
|
||||
"borgversion": borg_version,
|
||||
"borgmajor": "%d" % borg_version_tuple[:1],
|
||||
"borgminor": "%d.%d" % borg_version_tuple[:2],
|
||||
"borgpatch": "%d.%d.%d" % borg_version_tuple[:3],
|
||||
**overrides,
|
||||
}
|
||||
return format_line(text, data)
|
||||
@ -220,17 +220,17 @@ def replace_placeholders(text, overrides={}):
|
||||
|
||||
def SortBySpec(text):
|
||||
from .manifest import AI_HUMAN_SORT_KEYS
|
||||
for token in text.split(','):
|
||||
|
||||
for token in text.split(","):
|
||||
if token not in AI_HUMAN_SORT_KEYS:
|
||||
raise ValueError('Invalid sort key: %s' % token)
|
||||
return text.replace('timestamp', 'ts')
|
||||
raise ValueError("Invalid sort key: %s" % token)
|
||||
return text.replace("timestamp", "ts")
|
||||
|
||||
|
||||
def format_file_size(v, precision=2, sign=False, iec=False):
|
||||
"""Format file size into a human friendly format
|
||||
"""
|
||||
"""Format file size into a human friendly format"""
|
||||
fn = sizeof_fmt_iec if iec else sizeof_fmt_decimal
|
||||
return fn(v, suffix='B', sep=' ', precision=precision, sign=sign)
|
||||
return fn(v, suffix="B", sep=" ", precision=precision, sign=sign)
|
||||
|
||||
|
||||
class FileSize(int):
|
||||
@ -250,22 +250,16 @@ def parse_file_size(s):
|
||||
suffix = s[-1]
|
||||
power = 1000
|
||||
try:
|
||||
factor = {
|
||||
'K': power,
|
||||
'M': power**2,
|
||||
'G': power**3,
|
||||
'T': power**4,
|
||||
'P': power**5,
|
||||
}[suffix]
|
||||
factor = {"K": power, "M": power**2, "G": power**3, "T": power**4, "P": power**5}[suffix]
|
||||
s = s[:-1]
|
||||
except KeyError:
|
||||
factor = 1
|
||||
return int(float(s) * factor)
|
||||
|
||||
|
||||
def sizeof_fmt(num, suffix='B', units=None, power=None, sep='', precision=2, sign=False):
|
||||
sign = '+' if sign and num > 0 else ''
|
||||
fmt = '{0:{1}.{2}f}{3}{4}{5}'
|
||||
def sizeof_fmt(num, suffix="B", units=None, power=None, sep="", precision=2, sign=False):
|
||||
sign = "+" if sign and num > 0 else ""
|
||||
fmt = "{0:{1}.{2}f}{3}{4}{5}"
|
||||
prec = 0
|
||||
for unit in units[:-1]:
|
||||
if abs(round(num, precision)) < power:
|
||||
@ -277,27 +271,37 @@ def sizeof_fmt(num, suffix='B', units=None, power=None, sep='', precision=2, sig
|
||||
return fmt.format(num, sign, prec, sep, unit, suffix)
|
||||
|
||||
|
||||
def sizeof_fmt_iec(num, suffix='B', sep='', precision=2, sign=False):
|
||||
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign,
|
||||
units=['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'], power=1024)
|
||||
def sizeof_fmt_iec(num, suffix="B", sep="", precision=2, sign=False):
|
||||
return sizeof_fmt(
|
||||
num,
|
||||
suffix=suffix,
|
||||
sep=sep,
|
||||
precision=precision,
|
||||
sign=sign,
|
||||
units=["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"],
|
||||
power=1024,
|
||||
)
|
||||
|
||||
|
||||
def sizeof_fmt_decimal(num, suffix='B', sep='', precision=2, sign=False):
|
||||
return sizeof_fmt(num, suffix=suffix, sep=sep, precision=precision, sign=sign,
|
||||
units=['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'], power=1000)
|
||||
def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False):
|
||||
return sizeof_fmt(
|
||||
num,
|
||||
suffix=suffix,
|
||||
sep=sep,
|
||||
precision=precision,
|
||||
sign=sign,
|
||||
units=["", "k", "M", "G", "T", "P", "E", "Z", "Y"],
|
||||
power=1000,
|
||||
)
|
||||
|
||||
|
||||
def format_archive(archive):
|
||||
return '%-36s %s [%s]' % (
|
||||
archive.name,
|
||||
format_time(to_localtime(archive.ts)),
|
||||
bin_to_hex(archive.id),
|
||||
)
|
||||
return "%-36s %s [%s]" % (archive.name, format_time(to_localtime(archive.ts)), bin_to_hex(archive.id))
|
||||
|
||||
|
||||
def parse_stringified_list(s):
|
||||
l = re.split(" *, *", s)
|
||||
return [item for item in l if item != '']
|
||||
return [item for item in l if item != ""]
|
||||
|
||||
|
||||
class Location:
|
||||
@ -343,28 +347,42 @@ class Location:
|
||||
"""
|
||||
|
||||
# regexes for misc. kinds of supported location specifiers:
|
||||
ssh_re = re.compile(r"""
|
||||
ssh_re = re.compile(
|
||||
r"""
|
||||
(?P<proto>ssh):// # ssh://
|
||||
""" + optional_user_re + host_re + r""" # user@ (optional), host name or address
|
||||
"""
|
||||
+ optional_user_re
|
||||
+ host_re
|
||||
+ r""" # user@ (optional), host name or address
|
||||
(?::(?P<port>\d+))? # :port (optional)
|
||||
""" + abs_path_re, re.VERBOSE) # path
|
||||
"""
|
||||
+ abs_path_re,
|
||||
re.VERBOSE,
|
||||
) # path
|
||||
|
||||
file_re = re.compile(r"""
|
||||
file_re = re.compile(
|
||||
r"""
|
||||
(?P<proto>file):// # file://
|
||||
""" + file_path_re, re.VERBOSE) # servername/path or path
|
||||
"""
|
||||
+ file_path_re,
|
||||
re.VERBOSE,
|
||||
) # servername/path or path
|
||||
|
||||
local_re = re.compile(local_path_re, re.VERBOSE) # local path
|
||||
local_re = re.compile(local_path_re, re.VERBOSE) # local path
|
||||
|
||||
win_file_re = re.compile(r"""
|
||||
win_file_re = re.compile(
|
||||
r"""
|
||||
(?:file://)? # optional file protocol
|
||||
(?P<path>
|
||||
(?:[a-zA-Z]:)? # Drive letter followed by a colon (optional)
|
||||
(?:[^:]+) # Anything which does not contain a :, at least one character
|
||||
)
|
||||
""", re.VERBOSE)
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
def __init__(self, text='', overrides={}, other=False):
|
||||
self.repo_env_var = 'BORG_OTHER_REPO' if other else 'BORG_REPO'
|
||||
def __init__(self, text="", overrides={}, other=False):
|
||||
self.repo_env_var = "BORG_OTHER_REPO" if other else "BORG_REPO"
|
||||
self.valid = False
|
||||
self.proto = None
|
||||
self.user = None
|
||||
@ -393,15 +411,15 @@ def parse(self, text, overrides={}):
|
||||
def _parse(self, text):
|
||||
def normpath_special(p):
|
||||
# avoid that normpath strips away our relative path hack and even makes p absolute
|
||||
relative = p.startswith('/./')
|
||||
relative = p.startswith("/./")
|
||||
p = os.path.normpath(p)
|
||||
return ('/.' + p) if relative else p
|
||||
return ("/." + p) if relative else p
|
||||
|
||||
if is_win32:
|
||||
m = self.win_file_re.match(text)
|
||||
if m:
|
||||
self.proto = 'file'
|
||||
self.path = m.group('path')
|
||||
self.proto = "file"
|
||||
self.path = m.group("path")
|
||||
return True
|
||||
|
||||
# On windows we currently only support windows paths.
|
||||
@ -409,38 +427,38 @@ def normpath_special(p):
|
||||
|
||||
m = self.ssh_re.match(text)
|
||||
if m:
|
||||
self.proto = m.group('proto')
|
||||
self.user = m.group('user')
|
||||
self._host = m.group('host')
|
||||
self.port = m.group('port') and int(m.group('port')) or None
|
||||
self.path = normpath_special(m.group('path'))
|
||||
self.proto = m.group("proto")
|
||||
self.user = m.group("user")
|
||||
self._host = m.group("host")
|
||||
self.port = m.group("port") and int(m.group("port")) or None
|
||||
self.path = normpath_special(m.group("path"))
|
||||
return True
|
||||
m = self.file_re.match(text)
|
||||
if m:
|
||||
self.proto = m.group('proto')
|
||||
self.path = normpath_special(m.group('path'))
|
||||
self.proto = m.group("proto")
|
||||
self.path = normpath_special(m.group("path"))
|
||||
return True
|
||||
m = self.local_re.match(text)
|
||||
if m:
|
||||
self.proto = 'file'
|
||||
self.path = normpath_special(m.group('path'))
|
||||
self.proto = "file"
|
||||
self.path = normpath_special(m.group("path"))
|
||||
return True
|
||||
return False
|
||||
|
||||
def __str__(self):
|
||||
items = [
|
||||
'proto=%r' % self.proto,
|
||||
'user=%r' % self.user,
|
||||
'host=%r' % self.host,
|
||||
'port=%r' % self.port,
|
||||
'path=%r' % self.path,
|
||||
"proto=%r" % self.proto,
|
||||
"user=%r" % self.user,
|
||||
"host=%r" % self.host,
|
||||
"port=%r" % self.port,
|
||||
"path=%r" % self.path,
|
||||
]
|
||||
return ', '.join(items)
|
||||
return ", ".join(items)
|
||||
|
||||
def to_key_filename(self):
|
||||
name = re.sub(r'[^\w]', '_', self.path).strip('_')
|
||||
if self.proto != 'file':
|
||||
name = re.sub(r'[^\w]', '_', self.host) + '__' + name
|
||||
name = re.sub(r"[^\w]", "_", self.path).strip("_")
|
||||
if self.proto != "file":
|
||||
name = re.sub(r"[^\w]", "_", self.host) + "__" + name
|
||||
if len(name) > 100:
|
||||
# Limit file names to some reasonable length. Most file systems
|
||||
# limit them to 255 [unit of choice]; due to variations in unicode
|
||||
@ -455,28 +473,30 @@ def __repr__(self):
|
||||
def host(self):
|
||||
# strip square brackets used for IPv6 addrs
|
||||
if self._host is not None:
|
||||
return self._host.lstrip('[').rstrip(']')
|
||||
return self._host.lstrip("[").rstrip("]")
|
||||
|
||||
def canonical_path(self):
|
||||
if self.proto == 'file':
|
||||
if self.proto == "file":
|
||||
return self.path
|
||||
else:
|
||||
if self.path and self.path.startswith('~'):
|
||||
path = '/' + self.path # /~/x = path x relative to home dir
|
||||
elif self.path and not self.path.startswith('/'):
|
||||
path = '/./' + self.path # /./x = path x relative to cwd
|
||||
if self.path and self.path.startswith("~"):
|
||||
path = "/" + self.path # /~/x = path x relative to home dir
|
||||
elif self.path and not self.path.startswith("/"):
|
||||
path = "/./" + self.path # /./x = path x relative to cwd
|
||||
else:
|
||||
path = self.path
|
||||
return 'ssh://{}{}{}{}'.format(f'{self.user}@' if self.user else '',
|
||||
self._host, # needed for ipv6 addrs
|
||||
f':{self.port}' if self.port else '',
|
||||
path)
|
||||
return "ssh://{}{}{}{}".format(
|
||||
f"{self.user}@" if self.user else "",
|
||||
self._host, # needed for ipv6 addrs
|
||||
f":{self.port}" if self.port else "",
|
||||
path,
|
||||
)
|
||||
|
||||
def with_timestamp(self, timestamp):
|
||||
return Location(self.raw, overrides={
|
||||
'now': DatetimeWrapper(timestamp.astimezone(None)),
|
||||
'utcnow': DatetimeWrapper(timestamp),
|
||||
})
|
||||
return Location(
|
||||
self.raw,
|
||||
overrides={"now": DatetimeWrapper(timestamp.astimezone(None)), "utcnow": DatetimeWrapper(timestamp)},
|
||||
)
|
||||
|
||||
|
||||
def location_validator(proto=None, other=False):
|
||||
@ -486,33 +506,35 @@ def validator(text):
|
||||
except ValueError as err:
|
||||
raise argparse.ArgumentTypeError(str(err)) from None
|
||||
if proto is not None and loc.proto != proto:
|
||||
if proto == 'file':
|
||||
if proto == "file":
|
||||
raise argparse.ArgumentTypeError('"%s": Repository must be local' % text)
|
||||
else:
|
||||
raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text)
|
||||
return loc
|
||||
|
||||
return validator
|
||||
|
||||
|
||||
def archivename_validator():
|
||||
def validator(text):
|
||||
text = replace_placeholders(text)
|
||||
if '/' in text or '::' in text or not text:
|
||||
if "/" in text or "::" in text or not text:
|
||||
raise argparse.ArgumentTypeError('Invalid archive name: "%s"' % text)
|
||||
return text
|
||||
|
||||
return validator
|
||||
|
||||
|
||||
class BaseFormatter:
|
||||
FIXED_KEYS = {
|
||||
# Formatting aids
|
||||
'LF': '\n',
|
||||
'SPACE': ' ',
|
||||
'TAB': '\t',
|
||||
'CR': '\r',
|
||||
'NUL': '\0',
|
||||
'NEWLINE': os.linesep,
|
||||
'NL': os.linesep,
|
||||
"LF": "\n",
|
||||
"SPACE": " ",
|
||||
"TAB": "\t",
|
||||
"CR": "\r",
|
||||
"NUL": "\0",
|
||||
"NEWLINE": os.linesep,
|
||||
"NL": os.linesep,
|
||||
}
|
||||
|
||||
def get_item_data(self, item):
|
||||
@ -523,42 +545,45 @@ def format_item(self, item):
|
||||
|
||||
@staticmethod
|
||||
def keys_help():
|
||||
return "- NEWLINE: OS dependent line separator\n" \
|
||||
"- NL: alias of NEWLINE\n" \
|
||||
"- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n" \
|
||||
"- SPACE\n" \
|
||||
"- TAB\n" \
|
||||
"- CR\n" \
|
||||
"- LF"
|
||||
return (
|
||||
"- NEWLINE: OS dependent line separator\n"
|
||||
"- NL: alias of NEWLINE\n"
|
||||
"- NUL: NUL character for creating print0 / xargs -0 like output, see barchive and bpath keys below\n"
|
||||
"- SPACE\n"
|
||||
"- TAB\n"
|
||||
"- CR\n"
|
||||
"- LF"
|
||||
)
|
||||
|
||||
|
||||
class ArchiveFormatter(BaseFormatter):
|
||||
KEY_DESCRIPTIONS = {
|
||||
'archive': 'archive name interpreted as text (might be missing non-text characters, see barchive)',
|
||||
'name': 'alias of "archive"',
|
||||
'barchive': 'verbatim archive name, can contain any character except NUL',
|
||||
'comment': 'archive comment interpreted as text (might be missing non-text characters, see bcomment)',
|
||||
'bcomment': 'verbatim archive comment, can contain any character except NUL',
|
||||
"archive": "archive name interpreted as text (might be missing non-text characters, see barchive)",
|
||||
"name": 'alias of "archive"',
|
||||
"barchive": "verbatim archive name, can contain any character except NUL",
|
||||
"comment": "archive comment interpreted as text (might be missing non-text characters, see bcomment)",
|
||||
"bcomment": "verbatim archive comment, can contain any character except NUL",
|
||||
# *start* is the key used by borg-info for this timestamp, this makes the formats more compatible
|
||||
'start': 'time (start) of creation of the archive',
|
||||
'time': 'alias of "start"',
|
||||
'end': 'time (end) of creation of the archive',
|
||||
'command_line': 'command line which was used to create the archive',
|
||||
'id': 'internal ID of the archive',
|
||||
'hostname': 'hostname of host on which this archive was created',
|
||||
'username': 'username of user who created this archive',
|
||||
"start": "time (start) of creation of the archive",
|
||||
"time": 'alias of "start"',
|
||||
"end": "time (end) of creation of the archive",
|
||||
"command_line": "command line which was used to create the archive",
|
||||
"id": "internal ID of the archive",
|
||||
"hostname": "hostname of host on which this archive was created",
|
||||
"username": "username of user who created this archive",
|
||||
}
|
||||
KEY_GROUPS = (
|
||||
('archive', 'name', 'barchive', 'comment', 'bcomment', 'id'),
|
||||
('start', 'time', 'end', 'command_line'),
|
||||
('hostname', 'username'),
|
||||
("archive", "name", "barchive", "comment", "bcomment", "id"),
|
||||
("start", "time", "end", "command_line"),
|
||||
("hostname", "username"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def available_keys(cls):
|
||||
from .manifest import ArchiveInfo
|
||||
fake_archive_info = ArchiveInfo('archivename', b'\1'*32, datetime(1970, 1, 1, tzinfo=timezone.utc))
|
||||
formatter = cls('', None, None, None)
|
||||
|
||||
fake_archive_info = ArchiveInfo("archivename", b"\1" * 32, datetime(1970, 1, 1, tzinfo=timezone.utc))
|
||||
formatter = cls("", None, None, None)
|
||||
keys = []
|
||||
keys.extend(formatter.call_keys.keys())
|
||||
keys.extend(formatter.get_item_data(fake_archive_info).keys())
|
||||
@ -596,12 +621,12 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False):
|
||||
self.format = partial_format(format, static_keys)
|
||||
self.format_keys = {f[1] for f in Formatter().parse(format)}
|
||||
self.call_keys = {
|
||||
'hostname': partial(self.get_meta, 'hostname', rs=True),
|
||||
'username': partial(self.get_meta, 'username', rs=True),
|
||||
'comment': partial(self.get_meta, 'comment', rs=True),
|
||||
'bcomment': partial(self.get_meta, 'comment', rs=False),
|
||||
'end': self.get_ts_end,
|
||||
'command_line': self.get_cmdline,
|
||||
"hostname": partial(self.get_meta, "hostname", rs=True),
|
||||
"username": partial(self.get_meta, "username", rs=True),
|
||||
"comment": partial(self.get_meta, "comment", rs=True),
|
||||
"bcomment": partial(self.get_meta, "comment", rs=False),
|
||||
"end": self.get_ts_end,
|
||||
"command_line": self.get_cmdline,
|
||||
}
|
||||
self.used_call_keys = set(self.call_keys) & self.format_keys
|
||||
if self.json:
|
||||
@ -611,21 +636,23 @@ def __init__(self, format, repository, manifest, key, *, json=False, iec=False):
|
||||
self.item_data = static_keys
|
||||
|
||||
def format_item_json(self, item):
|
||||
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
|
||||
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n"
|
||||
|
||||
def get_item_data(self, archive_info):
|
||||
self.name = archive_info.name
|
||||
self.id = archive_info.id
|
||||
item_data = {}
|
||||
item_data.update(self.item_data)
|
||||
item_data.update({
|
||||
'name': remove_surrogates(archive_info.name),
|
||||
'archive': remove_surrogates(archive_info.name),
|
||||
'barchive': archive_info.name,
|
||||
'id': bin_to_hex(archive_info.id),
|
||||
'time': self.format_time(archive_info.ts),
|
||||
'start': self.format_time(archive_info.ts),
|
||||
})
|
||||
item_data.update(
|
||||
{
|
||||
"name": remove_surrogates(archive_info.name),
|
||||
"archive": remove_surrogates(archive_info.name),
|
||||
"barchive": archive_info.name,
|
||||
"id": bin_to_hex(archive_info.id),
|
||||
"time": self.format_time(archive_info.ts),
|
||||
"start": self.format_time(archive_info.ts),
|
||||
}
|
||||
)
|
||||
for key in self.used_call_keys:
|
||||
item_data[key] = self.call_keys[key]()
|
||||
return item_data
|
||||
@ -635,19 +662,20 @@ def archive(self):
|
||||
"""lazy load / update loaded archive"""
|
||||
if self._archive is None or self._archive.id != self.id:
|
||||
from ..archive import Archive
|
||||
|
||||
self._archive = Archive(self.repository, self.key, self.manifest, self.name, iec=self.iec)
|
||||
return self._archive
|
||||
|
||||
def get_meta(self, key, rs):
|
||||
value = self.archive.metadata.get(key, '')
|
||||
value = self.archive.metadata.get(key, "")
|
||||
return remove_surrogates(value) if rs else value
|
||||
|
||||
def get_cmdline(self):
|
||||
cmdline = map(remove_surrogates, self.archive.metadata.get('cmdline', []))
|
||||
cmdline = map(remove_surrogates, self.archive.metadata.get("cmdline", []))
|
||||
if self.json:
|
||||
return list(cmdline)
|
||||
else:
|
||||
return ' '.join(map(shlex.quote, cmdline))
|
||||
return " ".join(map(shlex.quote, cmdline))
|
||||
|
||||
def get_ts_end(self):
|
||||
return self.format_time(self.archive.ts_end)
|
||||
@ -659,31 +687,29 @@ def format_time(self, ts):
|
||||
class ItemFormatter(BaseFormatter):
|
||||
# we provide the hash algos from python stdlib (except shake_*) and additionally xxh64.
|
||||
# shake_* is not provided because it uses an incompatible .digest() method to support variable length.
|
||||
hash_algorithms = hashlib.algorithms_guaranteed.union({'xxh64'}).difference({'shake_128', 'shake_256'})
|
||||
hash_algorithms = hashlib.algorithms_guaranteed.union({"xxh64"}).difference({"shake_128", "shake_256"})
|
||||
KEY_DESCRIPTIONS = {
|
||||
'bpath': 'verbatim POSIX path, can contain any character except NUL',
|
||||
'path': 'path interpreted as text (might be missing non-text characters, see bpath)',
|
||||
'source': 'link target for symlinks (identical to linktarget)',
|
||||
'hlid': 'hard link identity (same if hardlinking same fs object)',
|
||||
'extra': 'prepends {source} with " -> " for soft links and " link to " for hard links',
|
||||
'dsize': 'deduplicated size',
|
||||
'num_chunks': 'number of chunks in this file',
|
||||
'unique_chunks': 'number of unique chunks in this file',
|
||||
'xxh64': 'XXH64 checksum of this file (note: this is NOT a cryptographic hash!)',
|
||||
'health': 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)',
|
||||
"bpath": "verbatim POSIX path, can contain any character except NUL",
|
||||
"path": "path interpreted as text (might be missing non-text characters, see bpath)",
|
||||
"source": "link target for symlinks (identical to linktarget)",
|
||||
"hlid": "hard link identity (same if hardlinking same fs object)",
|
||||
"extra": 'prepends {source} with " -> " for soft links and " link to " for hard links',
|
||||
"dsize": "deduplicated size",
|
||||
"num_chunks": "number of chunks in this file",
|
||||
"unique_chunks": "number of unique chunks in this file",
|
||||
"xxh64": "XXH64 checksum of this file (note: this is NOT a cryptographic hash!)",
|
||||
"health": 'either "healthy" (file ok) or "broken" (if file has all-zero replacement chunks)',
|
||||
}
|
||||
KEY_GROUPS = (
|
||||
('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'hlid', 'flags'),
|
||||
('size', 'dsize', 'num_chunks', 'unique_chunks'),
|
||||
('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'),
|
||||
("type", "mode", "uid", "gid", "user", "group", "path", "bpath", "source", "linktarget", "hlid", "flags"),
|
||||
("size", "dsize", "num_chunks", "unique_chunks"),
|
||||
("mtime", "ctime", "atime", "isomtime", "isoctime", "isoatime"),
|
||||
tuple(sorted(hash_algorithms)),
|
||||
('archiveid', 'archivename', 'extra'),
|
||||
('health', )
|
||||
("archiveid", "archivename", "extra"),
|
||||
("health",),
|
||||
)
|
||||
|
||||
KEYS_REQUIRING_CACHE = (
|
||||
'dsize', 'unique_chunks',
|
||||
)
|
||||
KEYS_REQUIRING_CACHE = ("dsize", "unique_chunks")
|
||||
|
||||
@classmethod
|
||||
def available_keys(cls):
|
||||
@ -691,7 +717,8 @@ class FakeArchive:
|
||||
fpr = name = ""
|
||||
|
||||
from ..item import Item
|
||||
fake_item = Item(mode=0, path='', user='', group='', mtime=0, uid=0, gid=0)
|
||||
|
||||
fake_item = Item(mode=0, path="", user="", group="", mtime=0, uid=0, gid=0)
|
||||
formatter = cls(FakeArchive, "")
|
||||
keys = []
|
||||
keys.extend(formatter.call_keys.keys())
|
||||
@ -723,13 +750,11 @@ def format_needs_cache(cls, format):
|
||||
|
||||
def __init__(self, archive, format, *, json_lines=False):
|
||||
from ..checksums import StreamingXXH64
|
||||
|
||||
self.xxh64 = StreamingXXH64
|
||||
self.archive = archive
|
||||
self.json_lines = json_lines
|
||||
static_keys = {
|
||||
'archivename': archive.name,
|
||||
'archiveid': archive.fpr,
|
||||
}
|
||||
static_keys = {"archivename": archive.name, "archiveid": archive.fpr}
|
||||
static_keys.update(self.FIXED_KEYS)
|
||||
if self.json_lines:
|
||||
self.item_data = {}
|
||||
@ -739,23 +764,23 @@ def __init__(self, archive, format, *, json_lines=False):
|
||||
self.format = partial_format(format, static_keys)
|
||||
self.format_keys = {f[1] for f in Formatter().parse(format)}
|
||||
self.call_keys = {
|
||||
'size': self.calculate_size,
|
||||
'dsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size),
|
||||
'num_chunks': self.calculate_num_chunks,
|
||||
'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
|
||||
'isomtime': partial(self.format_iso_time, 'mtime'),
|
||||
'isoctime': partial(self.format_iso_time, 'ctime'),
|
||||
'isoatime': partial(self.format_iso_time, 'atime'),
|
||||
'mtime': partial(self.format_time, 'mtime'),
|
||||
'ctime': partial(self.format_time, 'ctime'),
|
||||
'atime': partial(self.format_time, 'atime'),
|
||||
"size": self.calculate_size,
|
||||
"dsize": partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.size),
|
||||
"num_chunks": self.calculate_num_chunks,
|
||||
"unique_chunks": partial(self.sum_unique_chunks_metadata, lambda chunk: 1),
|
||||
"isomtime": partial(self.format_iso_time, "mtime"),
|
||||
"isoctime": partial(self.format_iso_time, "ctime"),
|
||||
"isoatime": partial(self.format_iso_time, "atime"),
|
||||
"mtime": partial(self.format_time, "mtime"),
|
||||
"ctime": partial(self.format_time, "ctime"),
|
||||
"atime": partial(self.format_time, "atime"),
|
||||
}
|
||||
for hash_function in self.hash_algorithms:
|
||||
self.call_keys[hash_function] = partial(self.hash_item, hash_function)
|
||||
self.used_call_keys = set(self.call_keys) & self.format_keys
|
||||
|
||||
def format_item_json(self, item):
|
||||
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n'
|
||||
return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + "\n"
|
||||
|
||||
def get_item_data(self, item):
|
||||
item_data = {}
|
||||
@ -763,30 +788,30 @@ def get_item_data(self, item):
|
||||
mode = stat.filemode(item.mode)
|
||||
item_type = mode[0]
|
||||
|
||||
source = item.get('source', '')
|
||||
extra = ''
|
||||
source = item.get("source", "")
|
||||
extra = ""
|
||||
if source:
|
||||
source = remove_surrogates(source)
|
||||
extra = ' -> %s' % source
|
||||
hlid = item.get('hlid')
|
||||
hlid = bin_to_hex(hlid) if hlid else ''
|
||||
item_data['type'] = item_type
|
||||
item_data['mode'] = mode
|
||||
item_data['user'] = item.get('user', str(item.uid))
|
||||
item_data['group'] = item.get('group', str(item.gid))
|
||||
item_data['uid'] = item.uid
|
||||
item_data['gid'] = item.gid
|
||||
item_data['path'] = remove_surrogates(item.path)
|
||||
extra = " -> %s" % source
|
||||
hlid = item.get("hlid")
|
||||
hlid = bin_to_hex(hlid) if hlid else ""
|
||||
item_data["type"] = item_type
|
||||
item_data["mode"] = mode
|
||||
item_data["user"] = item.get("user", str(item.uid))
|
||||
item_data["group"] = item.get("group", str(item.gid))
|
||||
item_data["uid"] = item.uid
|
||||
item_data["gid"] = item.gid
|
||||
item_data["path"] = remove_surrogates(item.path)
|
||||
if self.json_lines:
|
||||
item_data['healthy'] = 'chunks_healthy' not in item
|
||||
item_data["healthy"] = "chunks_healthy" not in item
|
||||
else:
|
||||
item_data['bpath'] = item.path
|
||||
item_data['extra'] = extra
|
||||
item_data['health'] = 'broken' if 'chunks_healthy' in item else 'healthy'
|
||||
item_data['source'] = source
|
||||
item_data['linktarget'] = source
|
||||
item_data['hlid'] = hlid
|
||||
item_data['flags'] = item.get('bsdflags')
|
||||
item_data["bpath"] = item.path
|
||||
item_data["extra"] = extra
|
||||
item_data["health"] = "broken" if "chunks_healthy" in item else "healthy"
|
||||
item_data["source"] = source
|
||||
item_data["linktarget"] = source
|
||||
item_data["hlid"] = hlid
|
||||
item_data["flags"] = item.get("bsdflags")
|
||||
for key in self.used_call_keys:
|
||||
item_data[key] = self.call_keys[key](item)
|
||||
return item_data
|
||||
@ -801,21 +826,21 @@ def sum_unique_chunks_metadata(self, metadata_func, item):
|
||||
the metadata needed from the chunk
|
||||
"""
|
||||
chunk_index = self.archive.cache.chunks
|
||||
chunks = item.get('chunks', [])
|
||||
chunks = item.get("chunks", [])
|
||||
chunks_counter = Counter(c.id for c in chunks)
|
||||
return sum(metadata_func(c) for c in chunks if chunk_index[c.id].refcount == chunks_counter[c.id])
|
||||
|
||||
def calculate_num_chunks(self, item):
|
||||
return len(item.get('chunks', []))
|
||||
return len(item.get("chunks", []))
|
||||
|
||||
def calculate_size(self, item):
|
||||
# note: does not support hardlink slaves, they will be size 0
|
||||
return item.get_size()
|
||||
|
||||
def hash_item(self, hash_function, item):
|
||||
if 'chunks' not in item:
|
||||
if "chunks" not in item:
|
||||
return ""
|
||||
if hash_function == 'xxh64':
|
||||
if hash_function == "xxh64":
|
||||
hash = self.xxh64()
|
||||
elif hash_function in self.hash_algorithms:
|
||||
hash = hashlib.new(hash_function)
|
||||
@ -832,18 +857,18 @@ def format_iso_time(self, key, item):
|
||||
|
||||
def file_status(mode):
|
||||
if stat.S_ISREG(mode):
|
||||
return 'A'
|
||||
return "A"
|
||||
elif stat.S_ISDIR(mode):
|
||||
return 'd'
|
||||
return "d"
|
||||
elif stat.S_ISBLK(mode):
|
||||
return 'b'
|
||||
return "b"
|
||||
elif stat.S_ISCHR(mode):
|
||||
return 'c'
|
||||
return "c"
|
||||
elif stat.S_ISLNK(mode):
|
||||
return 's'
|
||||
return "s"
|
||||
elif stat.S_ISFIFO(mode):
|
||||
return 'f'
|
||||
return '?'
|
||||
return "f"
|
||||
return "?"
|
||||
|
||||
|
||||
def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True):
|
||||
@ -868,7 +893,7 @@ def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comme
|
||||
line = line.rstrip(rstrip)
|
||||
if remove_empty and not line:
|
||||
continue
|
||||
if remove_comments and line.startswith('#'):
|
||||
if remove_comments and line.startswith("#"):
|
||||
continue
|
||||
yield line
|
||||
|
||||
@ -883,6 +908,7 @@ def swidth_slice(string, max_width):
|
||||
Latin characters are usually one cell wide, many CJK characters are two cells wide.
|
||||
"""
|
||||
from ..platform import swidth
|
||||
|
||||
reverse = max_width < 0
|
||||
max_width = abs(max_width)
|
||||
if reverse:
|
||||
@ -896,7 +922,7 @@ def swidth_slice(string, max_width):
|
||||
result.append(character)
|
||||
if reverse:
|
||||
result.reverse()
|
||||
return ''.join(result)
|
||||
return "".join(result)
|
||||
|
||||
|
||||
def ellipsis_truncate(msg, space):
|
||||
@ -905,15 +931,15 @@ def ellipsis_truncate(msg, space):
|
||||
this_is_a_very_long_string -------> this_is..._string
|
||||
"""
|
||||
from ..platform import swidth
|
||||
ellipsis_width = swidth('...')
|
||||
|
||||
ellipsis_width = swidth("...")
|
||||
msg_width = swidth(msg)
|
||||
if space < 8:
|
||||
# if there is very little space, just show ...
|
||||
return '...' + ' ' * (space - ellipsis_width)
|
||||
return "..." + " " * (space - ellipsis_width)
|
||||
if space < ellipsis_width + msg_width:
|
||||
return '{}...{}'.format(swidth_slice(msg, space // 2 - ellipsis_width),
|
||||
swidth_slice(msg, -space // 2))
|
||||
return msg + ' ' * (space - msg_width)
|
||||
return "{}...{}".format(swidth_slice(msg, space // 2 - ellipsis_width), swidth_slice(msg, -space // 2))
|
||||
return msg + " " * (space - msg_width)
|
||||
|
||||
|
||||
class BorgJsonEncoder(json.JSONEncoder):
|
||||
@ -922,23 +948,16 @@ def default(self, o):
|
||||
from ..remote import RemoteRepository
|
||||
from ..archive import Archive
|
||||
from ..cache import LocalCache, AdHocCache
|
||||
|
||||
if isinstance(o, Repository) or isinstance(o, RemoteRepository):
|
||||
return {
|
||||
'id': bin_to_hex(o.id),
|
||||
'location': o._location.canonical_path(),
|
||||
}
|
||||
return {"id": bin_to_hex(o.id), "location": o._location.canonical_path()}
|
||||
if isinstance(o, Archive):
|
||||
return o.info()
|
||||
if isinstance(o, LocalCache):
|
||||
return {
|
||||
'path': o.path,
|
||||
'stats': o.stats(),
|
||||
}
|
||||
return {"path": o.path, "stats": o.stats()}
|
||||
if isinstance(o, AdHocCache):
|
||||
return {
|
||||
'stats': o.stats(),
|
||||
}
|
||||
if callable(getattr(o, 'to_json', None)):
|
||||
return {"stats": o.stats()}
|
||||
if callable(getattr(o, "to_json", None)):
|
||||
return o.to_json()
|
||||
return super().default(o)
|
||||
|
||||
@ -946,17 +965,12 @@ def default(self, o):
|
||||
def basic_json_data(manifest, *, cache=None, extra=None):
|
||||
key = manifest.key
|
||||
data = extra or {}
|
||||
data.update({
|
||||
'repository': BorgJsonEncoder().default(manifest.repository),
|
||||
'encryption': {
|
||||
'mode': key.ARG_NAME,
|
||||
},
|
||||
})
|
||||
data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
|
||||
if key.NAME.startswith('key file'):
|
||||
data['encryption']['keyfile'] = key.find_key()
|
||||
data.update({"repository": BorgJsonEncoder().default(manifest.repository), "encryption": {"mode": key.ARG_NAME}})
|
||||
data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
|
||||
if key.NAME.startswith("key file"):
|
||||
data["encryption"]["keyfile"] = key.find_key()
|
||||
if cache:
|
||||
data['cache'] = cache
|
||||
data["cache"] = cache
|
||||
return data
|
||||
|
||||
|
||||
@ -975,13 +989,13 @@ def decode_bytes(value):
|
||||
# look nice and chunk ids should mostly show in hex. Use a special
|
||||
# inband signaling character (ASCII DEL) to distinguish between
|
||||
# decoded and hex mode.
|
||||
if not value.startswith(b'\x7f'):
|
||||
if not value.startswith(b"\x7f"):
|
||||
try:
|
||||
value = value.decode()
|
||||
return value
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
return '\u007f' + bin_to_hex(value)
|
||||
return "\u007f" + bin_to_hex(value)
|
||||
|
||||
def decode_tuple(t):
|
||||
res = []
|
||||
|
@ -39,7 +39,7 @@ def _env_passphrase(cls, env_var, default=None):
|
||||
|
||||
@classmethod
|
||||
def env_passphrase(cls, default=None):
|
||||
passphrase = cls._env_passphrase('BORG_PASSPHRASE', default)
|
||||
passphrase = cls._env_passphrase("BORG_PASSPHRASE", default)
|
||||
if passphrase is not None:
|
||||
return passphrase
|
||||
passphrase = cls.env_passcommand()
|
||||
@ -51,7 +51,7 @@ def env_passphrase(cls, default=None):
|
||||
|
||||
@classmethod
|
||||
def env_passcommand(cls, default=None):
|
||||
passcommand = os.environ.get('BORG_PASSCOMMAND', None)
|
||||
passcommand = os.environ.get("BORG_PASSCOMMAND", None)
|
||||
if passcommand is not None:
|
||||
# passcommand is a system command (not inside pyinstaller env)
|
||||
env = prepare_subprocess_env(system=True)
|
||||
@ -59,21 +59,21 @@ def env_passcommand(cls, default=None):
|
||||
passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True, env=env)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||||
raise PasscommandFailure(e)
|
||||
return cls(passphrase.rstrip('\n'))
|
||||
return cls(passphrase.rstrip("\n"))
|
||||
|
||||
@classmethod
|
||||
def fd_passphrase(cls):
|
||||
try:
|
||||
fd = int(os.environ.get('BORG_PASSPHRASE_FD'))
|
||||
fd = int(os.environ.get("BORG_PASSPHRASE_FD"))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
with os.fdopen(fd, mode='r') as f:
|
||||
with os.fdopen(fd, mode="r") as f:
|
||||
passphrase = f.read()
|
||||
return cls(passphrase.rstrip('\n'))
|
||||
return cls(passphrase.rstrip("\n"))
|
||||
|
||||
@classmethod
|
||||
def env_new_passphrase(cls, default=None):
|
||||
return cls._env_passphrase('BORG_NEW_PASSPHRASE', default)
|
||||
return cls._env_passphrase("BORG_NEW_PASSPHRASE", default)
|
||||
|
||||
@classmethod
|
||||
def getpass(cls, prompt):
|
||||
@ -83,32 +83,38 @@ def getpass(cls, prompt):
|
||||
if prompt:
|
||||
print() # avoid err msg appearing right of prompt
|
||||
msg = []
|
||||
for env_var in 'BORG_PASSPHRASE', 'BORG_PASSCOMMAND':
|
||||
for env_var in "BORG_PASSPHRASE", "BORG_PASSCOMMAND":
|
||||
env_var_set = os.environ.get(env_var) is not None
|
||||
msg.append('{} is {}.'.format(env_var, 'set' if env_var_set else 'not set'))
|
||||
msg.append('Interactive password query failed.')
|
||||
raise NoPassphraseFailure(' '.join(msg)) from None
|
||||
msg.append("{} is {}.".format(env_var, "set" if env_var_set else "not set"))
|
||||
msg.append("Interactive password query failed.")
|
||||
raise NoPassphraseFailure(" ".join(msg)) from None
|
||||
else:
|
||||
return cls(pw)
|
||||
|
||||
@classmethod
|
||||
def verification(cls, passphrase):
|
||||
msg = 'Do you want your passphrase to be displayed for verification? [yN]: '
|
||||
if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.',
|
||||
retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'):
|
||||
print('Your passphrase (between double-quotes): "%s"' % passphrase,
|
||||
file=sys.stderr)
|
||||
print('Make sure the passphrase displayed above is exactly what you wanted.',
|
||||
file=sys.stderr)
|
||||
msg = "Do you want your passphrase to be displayed for verification? [yN]: "
|
||||
if yes(
|
||||
msg,
|
||||
retry_msg=msg,
|
||||
invalid_msg="Invalid answer, try again.",
|
||||
retry=True,
|
||||
env_var_override="BORG_DISPLAY_PASSPHRASE",
|
||||
):
|
||||
print('Your passphrase (between double-quotes): "%s"' % passphrase, file=sys.stderr)
|
||||
print("Make sure the passphrase displayed above is exactly what you wanted.", file=sys.stderr)
|
||||
try:
|
||||
passphrase.encode('ascii')
|
||||
passphrase.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
print('Your passphrase (UTF-8 encoding in hex): %s' %
|
||||
bin_to_hex(passphrase.encode('utf-8')),
|
||||
file=sys.stderr)
|
||||
print('As you have a non-ASCII passphrase, it is recommended to keep the '
|
||||
'UTF-8 encoding in hex together with the passphrase at a safe place.',
|
||||
file=sys.stderr)
|
||||
print(
|
||||
"Your passphrase (UTF-8 encoding in hex): %s" % bin_to_hex(passphrase.encode("utf-8")),
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
"As you have a non-ASCII passphrase, it is recommended to keep the "
|
||||
"UTF-8 encoding in hex together with the passphrase at a safe place.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def new(cls, allow_empty=False):
|
||||
@ -119,17 +125,17 @@ def new(cls, allow_empty=False):
|
||||
if passphrase is not None:
|
||||
return passphrase
|
||||
for retry in range(1, 11):
|
||||
passphrase = cls.getpass('Enter new passphrase: ')
|
||||
passphrase = cls.getpass("Enter new passphrase: ")
|
||||
if allow_empty or passphrase:
|
||||
passphrase2 = cls.getpass('Enter same passphrase again: ')
|
||||
passphrase2 = cls.getpass("Enter same passphrase again: ")
|
||||
if passphrase == passphrase2:
|
||||
cls.verification(passphrase)
|
||||
logger.info('Remember your passphrase. Your data will be inaccessible without it.')
|
||||
logger.info("Remember your passphrase. Your data will be inaccessible without it.")
|
||||
return passphrase
|
||||
else:
|
||||
print('Passphrases do not match', file=sys.stderr)
|
||||
print("Passphrases do not match", file=sys.stderr)
|
||||
else:
|
||||
print('Passphrase must not be blank', file=sys.stderr)
|
||||
print("Passphrase must not be blank", file=sys.stderr)
|
||||
else:
|
||||
raise PasswordRetriesExceeded
|
||||
|
||||
|
@ -13,6 +13,7 @@
|
||||
|
||||
from ..platformflags import is_win32, is_linux, is_freebsd, is_darwin
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_SIGNAL_BASE, Error
|
||||
@ -21,6 +22,7 @@
|
||||
@contextlib.contextmanager
|
||||
def _daemonize():
|
||||
from ..platform import get_process_id
|
||||
|
||||
old_id = get_process_id()
|
||||
pid = os.fork()
|
||||
if pid:
|
||||
@ -30,13 +32,13 @@ def _daemonize():
|
||||
except _ExitCodeException as e:
|
||||
exit_code = e.exit_code
|
||||
finally:
|
||||
logger.debug('Daemonizing: Foreground process (%s, %s, %s) is now dying.' % old_id)
|
||||
logger.debug("Daemonizing: Foreground process (%s, %s, %s) is now dying." % old_id)
|
||||
os._exit(exit_code)
|
||||
os.setsid()
|
||||
pid = os.fork()
|
||||
if pid:
|
||||
os._exit(0)
|
||||
os.chdir('/')
|
||||
os.chdir("/")
|
||||
os.close(0)
|
||||
os.close(1)
|
||||
fd = os.open(os.devnull, os.O_RDWR)
|
||||
@ -78,12 +80,12 @@ def daemonizing(*, timeout=5):
|
||||
with _daemonize() as (old_id, new_id):
|
||||
if new_id is None:
|
||||
# The original / parent process, waiting for a signal to die.
|
||||
logger.debug('Daemonizing: Foreground process (%s, %s, %s) is waiting for background process...' % old_id)
|
||||
logger.debug("Daemonizing: Foreground process (%s, %s, %s) is waiting for background process..." % old_id)
|
||||
exit_code = EXIT_SUCCESS
|
||||
# Indeed, SIGHUP and SIGTERM handlers should have been set on archiver.run(). Just in case...
|
||||
with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \
|
||||
signal_handler('SIGHUP', raising_signal_handler(SigHup)), \
|
||||
signal_handler('SIGTERM', raising_signal_handler(SigTerm)):
|
||||
with signal_handler("SIGINT", raising_signal_handler(KeyboardInterrupt)), signal_handler(
|
||||
"SIGHUP", raising_signal_handler(SigHup)
|
||||
), signal_handler("SIGTERM", raising_signal_handler(SigTerm)):
|
||||
try:
|
||||
if timeout > 0:
|
||||
time.sleep(timeout)
|
||||
@ -96,15 +98,17 @@ def daemonizing(*, timeout=5):
|
||||
exit_code = EXIT_WARNING
|
||||
except KeyboardInterrupt:
|
||||
# Manual termination.
|
||||
logger.debug('Daemonizing: Foreground process (%s, %s, %s) received SIGINT.' % old_id)
|
||||
logger.debug("Daemonizing: Foreground process (%s, %s, %s) received SIGINT." % old_id)
|
||||
exit_code = EXIT_SIGNAL_BASE + 2
|
||||
except BaseException as e:
|
||||
# Just in case...
|
||||
logger.warning('Daemonizing: Foreground process received an exception while waiting:\n' +
|
||||
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
|
||||
logger.warning(
|
||||
"Daemonizing: Foreground process received an exception while waiting:\n"
|
||||
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
|
||||
)
|
||||
exit_code = EXIT_WARNING
|
||||
else:
|
||||
logger.warning('Daemonizing: Background process did not respond (timeout). Is it alive?')
|
||||
logger.warning("Daemonizing: Background process did not respond (timeout). Is it alive?")
|
||||
exit_code = EXIT_WARNING
|
||||
finally:
|
||||
# Don't call with-body, but die immediately!
|
||||
@ -113,22 +117,26 @@ def daemonizing(*, timeout=5):
|
||||
|
||||
# The background / grandchild process.
|
||||
sig_to_foreground = signal.SIGTERM
|
||||
logger.debug('Daemonizing: Background process (%s, %s, %s) is starting...' % new_id)
|
||||
logger.debug("Daemonizing: Background process (%s, %s, %s) is starting..." % new_id)
|
||||
try:
|
||||
yield old_id, new_id
|
||||
except BaseException as e:
|
||||
sig_to_foreground = signal.SIGHUP
|
||||
logger.warning('Daemonizing: Background process raised an exception while starting:\n' +
|
||||
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
|
||||
logger.warning(
|
||||
"Daemonizing: Background process raised an exception while starting:\n"
|
||||
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
|
||||
)
|
||||
raise e
|
||||
else:
|
||||
logger.debug('Daemonizing: Background process (%s, %s, %s) has started.' % new_id)
|
||||
logger.debug("Daemonizing: Background process (%s, %s, %s) has started." % new_id)
|
||||
finally:
|
||||
try:
|
||||
os.kill(old_id[1], sig_to_foreground)
|
||||
except BaseException as e:
|
||||
logger.error('Daemonizing: Trying to kill the foreground process raised an exception:\n' +
|
||||
''.join(traceback.format_exception(e.__class__, e, e.__traceback__)))
|
||||
logger.error(
|
||||
"Daemonizing: Trying to kill the foreground process raised an exception:\n"
|
||||
+ "".join(traceback.format_exception(e.__class__, e, e.__traceback__))
|
||||
)
|
||||
|
||||
|
||||
class _ExitCodeException(BaseException):
|
||||
@ -187,7 +195,7 @@ def __init__(self):
|
||||
self._sig_int_triggered = False
|
||||
self._action_triggered = False
|
||||
self._action_done = False
|
||||
self.ctx = signal_handler('SIGINT', self.handler)
|
||||
self.ctx = signal_handler("SIGINT", self.handler)
|
||||
|
||||
def __bool__(self):
|
||||
# this will be True (and stay True) after the first Ctrl-C/SIGINT
|
||||
@ -228,7 +236,7 @@ def __exit__(self, exception_type, exception_value, traceback):
|
||||
sig_int = SigIntManager()
|
||||
|
||||
|
||||
def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
|
||||
def popen_with_error_handling(cmd_line: str, log_prefix="", **kwargs):
|
||||
"""
|
||||
Handle typical errors raised by subprocess.Popen. Return None if an error occurred,
|
||||
otherwise return the Popen object.
|
||||
@ -240,27 +248,27 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
|
||||
|
||||
Does not change the exit code.
|
||||
"""
|
||||
assert not kwargs.get('shell'), 'Sorry pal, shell mode is a no-no'
|
||||
assert not kwargs.get("shell"), "Sorry pal, shell mode is a no-no"
|
||||
try:
|
||||
command = shlex.split(cmd_line)
|
||||
if not command:
|
||||
raise ValueError('an empty command line is not permitted')
|
||||
raise ValueError("an empty command line is not permitted")
|
||||
except ValueError as ve:
|
||||
logger.error('%s%s', log_prefix, ve)
|
||||
logger.error("%s%s", log_prefix, ve)
|
||||
return
|
||||
logger.debug('%scommand line: %s', log_prefix, command)
|
||||
logger.debug("%scommand line: %s", log_prefix, command)
|
||||
try:
|
||||
return subprocess.Popen(command, **kwargs)
|
||||
except FileNotFoundError:
|
||||
logger.error('%sexecutable not found: %s', log_prefix, command[0])
|
||||
logger.error("%sexecutable not found: %s", log_prefix, command[0])
|
||||
return
|
||||
except PermissionError:
|
||||
logger.error('%spermission denied: %s', log_prefix, command[0])
|
||||
logger.error("%spermission denied: %s", log_prefix, command[0])
|
||||
return
|
||||
|
||||
|
||||
def is_terminal(fd=sys.stdout):
|
||||
return hasattr(fd, 'isatty') and fd.isatty() and (not is_win32 or 'ANSICON' in os.environ)
|
||||
return hasattr(fd, "isatty") and fd.isatty() and (not is_win32 or "ANSICON" in os.environ)
|
||||
|
||||
|
||||
def prepare_subprocess_env(system, env=None):
|
||||
@ -278,8 +286,8 @@ def prepare_subprocess_env(system, env=None):
|
||||
# but we do not want that system binaries (like ssh or other) pick up
|
||||
# (non-matching) libraries from there.
|
||||
# thus we install the original LDLP, before pyinstaller has modified it:
|
||||
lp_key = 'LD_LIBRARY_PATH'
|
||||
lp_orig = env.get(lp_key + '_ORIG') # pyinstaller >= 20160820 / v3.2.1 has this
|
||||
lp_key = "LD_LIBRARY_PATH"
|
||||
lp_orig = env.get(lp_key + "_ORIG") # pyinstaller >= 20160820 / v3.2.1 has this
|
||||
if lp_orig is not None:
|
||||
env[lp_key] = lp_orig
|
||||
else:
|
||||
@ -292,12 +300,12 @@ def prepare_subprocess_env(system, env=None):
|
||||
# in this case, we must kill LDLP.
|
||||
# We can recognize this via sys.frozen and sys._MEIPASS being set.
|
||||
lp = env.get(lp_key)
|
||||
if lp is not None and getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'):
|
||||
if lp is not None and getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
|
||||
env.pop(lp_key)
|
||||
# security: do not give secrets to subprocess
|
||||
env.pop('BORG_PASSPHRASE', None)
|
||||
env.pop("BORG_PASSPHRASE", None)
|
||||
# for information, give borg version to the subprocess
|
||||
env['BORG_VERSION'] = __version__
|
||||
env["BORG_VERSION"] = __version__
|
||||
return env
|
||||
|
||||
|
||||
@ -314,13 +322,15 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
|
||||
# communication with the process is a one-way road, i.e. the process can never block
|
||||
# for us to do something while we block on the process for something different.
|
||||
if inbound:
|
||||
proc = popen_with_error_handling(cmd, stdout=subprocess.PIPE, stdin=filter_stream,
|
||||
log_prefix='filter-process: ', env=env)
|
||||
proc = popen_with_error_handling(
|
||||
cmd, stdout=subprocess.PIPE, stdin=filter_stream, log_prefix="filter-process: ", env=env
|
||||
)
|
||||
else:
|
||||
proc = popen_with_error_handling(cmd, stdin=subprocess.PIPE, stdout=filter_stream,
|
||||
log_prefix='filter-process: ', env=env)
|
||||
proc = popen_with_error_handling(
|
||||
cmd, stdin=subprocess.PIPE, stdout=filter_stream, log_prefix="filter-process: ", env=env
|
||||
)
|
||||
if not proc:
|
||||
raise Error(f'filter {cmd}: process creation failed')
|
||||
raise Error(f"filter {cmd}: process creation failed")
|
||||
stream = proc.stdout if inbound else proc.stdin
|
||||
# inbound: do not close the pipe (this is the task of the filter process [== writer])
|
||||
# outbound: close the pipe, otherwise the filter process would not notice when we are done.
|
||||
@ -331,7 +341,7 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
|
||||
|
||||
except Exception:
|
||||
# something went wrong with processing the stream by borg
|
||||
logger.debug('Exception, killing the filter...')
|
||||
logger.debug("Exception, killing the filter...")
|
||||
if cmd:
|
||||
proc.kill()
|
||||
borg_succeeded = False
|
||||
@ -343,11 +353,11 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
|
||||
stream.close()
|
||||
|
||||
if cmd:
|
||||
logger.debug('Done, waiting for filter to die...')
|
||||
logger.debug("Done, waiting for filter to die...")
|
||||
rc = proc.wait()
|
||||
logger.debug('filter cmd exited with code %d', rc)
|
||||
logger.debug("filter cmd exited with code %d", rc)
|
||||
if filter_stream_close:
|
||||
filter_stream.close()
|
||||
if borg_succeeded and rc:
|
||||
# if borg did not succeed, we know that we killed the filter process
|
||||
raise Error('filter %s failed, rc=%d' % (cmd, rc))
|
||||
raise Error("filter %s failed, rc=%d" % (cmd, rc))
|
||||
|
@ -5,6 +5,7 @@
|
||||
from shutil import get_terminal_size
|
||||
|
||||
from ..logger import create_logger
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
from .parseformat import ellipsis_truncate
|
||||
@ -19,7 +20,7 @@ def justify_to_terminal_size(message):
|
||||
|
||||
|
||||
class ProgressIndicatorBase:
|
||||
LOGGER = 'borg.output.progress'
|
||||
LOGGER = "borg.output.progress"
|
||||
JSON_TYPE = None
|
||||
json = False
|
||||
|
||||
@ -43,15 +44,15 @@ def __init__(self, msgid=None):
|
||||
if not self.logger.handlers:
|
||||
self.handler = logging.StreamHandler(stream=sys.stderr)
|
||||
self.handler.setLevel(logging.INFO)
|
||||
logger = logging.getLogger('borg')
|
||||
logger = logging.getLogger("borg")
|
||||
# Some special attributes on the borg logger, created by setup_logging
|
||||
# But also be able to work without that
|
||||
try:
|
||||
formatter = logger.formatter
|
||||
terminator = '\n' if logger.json else '\r'
|
||||
terminator = "\n" if logger.json else "\r"
|
||||
self.json = logger.json
|
||||
except AttributeError:
|
||||
terminator = '\r'
|
||||
terminator = "\r"
|
||||
else:
|
||||
self.handler.setFormatter(formatter)
|
||||
self.handler.terminator = terminator
|
||||
@ -79,24 +80,20 @@ def output_json(self, *, finished=False, **kwargs):
|
||||
assert self.json
|
||||
if not self.emit:
|
||||
return
|
||||
kwargs.update(dict(
|
||||
operation=self.id,
|
||||
msgid=self.msgid,
|
||||
type=self.JSON_TYPE,
|
||||
finished=finished,
|
||||
time=time.time(),
|
||||
))
|
||||
kwargs.update(
|
||||
dict(operation=self.id, msgid=self.msgid, type=self.JSON_TYPE, finished=finished, time=time.time())
|
||||
)
|
||||
print(json.dumps(kwargs), file=sys.stderr, flush=True)
|
||||
|
||||
def finish(self):
|
||||
if self.json:
|
||||
self.output_json(finished=True)
|
||||
else:
|
||||
self.output('')
|
||||
self.output("")
|
||||
|
||||
|
||||
class ProgressIndicatorMessage(ProgressIndicatorBase):
|
||||
JSON_TYPE = 'progress_message'
|
||||
JSON_TYPE = "progress_message"
|
||||
|
||||
def output(self, msg):
|
||||
if self.json:
|
||||
@ -106,7 +103,7 @@ def output(self, msg):
|
||||
|
||||
|
||||
class ProgressIndicatorPercent(ProgressIndicatorBase):
|
||||
JSON_TYPE = 'progress_percent'
|
||||
JSON_TYPE = "progress_percent"
|
||||
|
||||
def __init__(self, total=0, step=5, start=0, msg="%3.0f%%", msgid=None):
|
||||
"""
|
||||
@ -150,7 +147,7 @@ def show(self, current=None, increase=1, info=None):
|
||||
# no need to truncate if we're not outputting to a terminal
|
||||
terminal_space = get_terminal_size(fallback=(-1, -1))[0]
|
||||
if terminal_space != -1:
|
||||
space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + ['']))
|
||||
space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [""]))
|
||||
info[-1] = ellipsis_truncate(info[-1], space)
|
||||
return self.output(self.msg % tuple([pct] + info), justify=False, info=info)
|
||||
|
||||
@ -193,7 +190,7 @@ def show(self):
|
||||
return self.output(self.triggered)
|
||||
|
||||
def output(self, triggered):
|
||||
print('.', end='', file=self.file, flush=True)
|
||||
print(".", end="", file=self.file, flush=True)
|
||||
|
||||
def finish(self):
|
||||
print(file=self.file)
|
||||
|
@ -12,7 +12,7 @@ def to_localtime(ts):
|
||||
|
||||
def parse_timestamp(timestamp, tzinfo=timezone.utc):
|
||||
"""Parse a ISO 8601 timestamp string"""
|
||||
fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS
|
||||
fmt = ISO_FORMAT if "." in timestamp else ISO_FORMAT_NO_USECS
|
||||
dt = datetime.strptime(timestamp, fmt)
|
||||
if tzinfo is not None:
|
||||
dt = dt.replace(tzinfo=tzinfo)
|
||||
@ -27,11 +27,16 @@ def timestamp(s):
|
||||
return datetime.fromtimestamp(ts, tz=timezone.utc)
|
||||
except OSError:
|
||||
# didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support.
|
||||
for format in ('%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S+00:00',
|
||||
'%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S',
|
||||
'%Y-%m-%dT%H:%M', '%Y-%m-%d %H:%M',
|
||||
'%Y-%m-%d', '%Y-%j',
|
||||
):
|
||||
for format in (
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
"%Y-%m-%dT%H:%M:%S+00:00",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M",
|
||||
"%Y-%m-%d %H:%M",
|
||||
"%Y-%m-%d",
|
||||
"%Y-%j",
|
||||
):
|
||||
try:
|
||||
return datetime.strptime(s, format).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
@ -54,7 +59,7 @@ def timestamp(s):
|
||||
# subtract last 48h to avoid any issues that could be caused by tz calculations.
|
||||
# this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit).
|
||||
# msgpack can pack up to uint64.
|
||||
MAX_S = 2**31-1 - 48*3600
|
||||
MAX_S = 2**31 - 1 - 48 * 3600
|
||||
MAX_NS = MAX_S * 1000000000
|
||||
else:
|
||||
# nanosecond timestamps will fit into a signed int64.
|
||||
@ -62,7 +67,7 @@ def timestamp(s):
|
||||
# this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit).
|
||||
# round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S.
|
||||
# msgpack can pack up to uint64.
|
||||
MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000
|
||||
MAX_NS = (2**63 - 1 - 48 * 3600 * 1000000000) // 1000000000 * 1000000000
|
||||
MAX_S = MAX_NS // 1000000000
|
||||
|
||||
|
||||
@ -89,11 +94,11 @@ def safe_timestamp(item_timestamp_ns):
|
||||
return datetime.fromtimestamp(t_ns / 1e9)
|
||||
|
||||
|
||||
def format_time(ts: datetime, format_spec=''):
|
||||
def format_time(ts: datetime, format_spec=""):
|
||||
"""
|
||||
Convert *ts* to a human-friendly format with textual weekday.
|
||||
"""
|
||||
return ts.strftime('%a, %Y-%m-%d %H:%M:%S' if format_spec == '' else format_spec)
|
||||
return ts.strftime("%a, %Y-%m-%d %H:%M:%S" if format_spec == "" else format_spec)
|
||||
|
||||
|
||||
def isoformat_time(ts: datetime):
|
||||
@ -105,19 +110,18 @@ def isoformat_time(ts: datetime):
|
||||
|
||||
|
||||
def format_timedelta(td):
|
||||
"""Format timedelta in a human friendly format
|
||||
"""
|
||||
"""Format timedelta in a human friendly format"""
|
||||
ts = td.total_seconds()
|
||||
s = ts % 60
|
||||
m = int(ts / 60) % 60
|
||||
h = int(ts / 3600) % 24
|
||||
txt = '%.2f seconds' % s
|
||||
txt = "%.2f seconds" % s
|
||||
if m:
|
||||
txt = '%d minutes %s' % (m, txt)
|
||||
txt = "%d minutes %s" % (m, txt)
|
||||
if h:
|
||||
txt = '%d hours %s' % (h, txt)
|
||||
txt = "%d hours %s" % (h, txt)
|
||||
if td.days:
|
||||
txt = '%d days %s' % (td.days, txt)
|
||||
txt = "%d days %s" % (td.days, txt)
|
||||
return txt
|
||||
|
||||
|
||||
@ -131,7 +135,7 @@ def __format__(self, format_spec):
|
||||
return format_time(self.ts, format_spec=format_spec)
|
||||
|
||||
def __str__(self):
|
||||
return f'{self}'
|
||||
return f"{self}"
|
||||
|
||||
def isoformat(self):
|
||||
return isoformat_time(self.ts)
|
||||
|
@ -4,16 +4,30 @@
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', )
|
||||
TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', )
|
||||
DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', )
|
||||
FALSISH = ("No", "NO", "no", "N", "n", "0")
|
||||
TRUISH = ("Yes", "YES", "yes", "Y", "y", "1")
|
||||
DEFAULTISH = ("Default", "DEFAULT", "default", "D", "d", "")
|
||||
|
||||
|
||||
def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
|
||||
retry_msg=None, invalid_msg=None, env_msg='{} (from {})',
|
||||
falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH,
|
||||
default=False, retry=True, env_var_override=None, ofile=None, input=input, prompt=True,
|
||||
msgid=None):
|
||||
def yes(
|
||||
msg=None,
|
||||
false_msg=None,
|
||||
true_msg=None,
|
||||
default_msg=None,
|
||||
retry_msg=None,
|
||||
invalid_msg=None,
|
||||
env_msg="{} (from {})",
|
||||
falsish=FALSISH,
|
||||
truish=TRUISH,
|
||||
defaultish=DEFAULTISH,
|
||||
default=False,
|
||||
retry=True,
|
||||
env_var_override=None,
|
||||
ofile=None,
|
||||
input=input,
|
||||
prompt=True,
|
||||
msgid=None,
|
||||
):
|
||||
"""Output <msg> (usually a question) and let user input an answer.
|
||||
Qualifies the answer according to falsish, truish and defaultish as True, False or <default>.
|
||||
If it didn't qualify and retry is False (no retries wanted), return the default [which
|
||||
@ -43,18 +57,15 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None,
|
||||
:param input: input function [input from builtins]
|
||||
:return: boolean answer value, True or False
|
||||
"""
|
||||
|
||||
def output(msg, msg_type, is_prompt=False, **kwargs):
|
||||
json_output = getattr(logging.getLogger('borg'), 'json', False)
|
||||
json_output = getattr(logging.getLogger("borg"), "json", False)
|
||||
if json_output:
|
||||
kwargs.update(dict(
|
||||
type='question_%s' % msg_type,
|
||||
msgid=msgid,
|
||||
message=msg,
|
||||
))
|
||||
kwargs.update(dict(type="question_%s" % msg_type, msgid=msgid, message=msg))
|
||||
print(json.dumps(kwargs), file=sys.stderr)
|
||||
else:
|
||||
if is_prompt:
|
||||
print(msg, file=ofile, end='', flush=True)
|
||||
print(msg, file=ofile, end="", flush=True)
|
||||
else:
|
||||
print(msg, file=ofile)
|
||||
|
||||
@ -66,13 +77,13 @@ def output(msg, msg_type, is_prompt=False, **kwargs):
|
||||
if default not in (True, False):
|
||||
raise ValueError("invalid default value, must be True or False")
|
||||
if msg:
|
||||
output(msg, 'prompt', is_prompt=True)
|
||||
output(msg, "prompt", is_prompt=True)
|
||||
while True:
|
||||
answer = None
|
||||
if env_var_override:
|
||||
answer = os.environ.get(env_var_override)
|
||||
if answer is not None and env_msg:
|
||||
output(env_msg.format(answer, env_var_override), 'env_answer', env_var=env_var_override)
|
||||
output(env_msg.format(answer, env_var_override), "env_answer", env_var=env_var_override)
|
||||
if answer is None:
|
||||
if not prompt:
|
||||
return default
|
||||
@ -83,22 +94,22 @@ def output(msg, msg_type, is_prompt=False, **kwargs):
|
||||
answer = truish[0] if default else falsish[0]
|
||||
if answer in defaultish:
|
||||
if default_msg:
|
||||
output(default_msg, 'accepted_default')
|
||||
output(default_msg, "accepted_default")
|
||||
return default
|
||||
if answer in truish:
|
||||
if true_msg:
|
||||
output(true_msg, 'accepted_true')
|
||||
output(true_msg, "accepted_true")
|
||||
return True
|
||||
if answer in falsish:
|
||||
if false_msg:
|
||||
output(false_msg, 'accepted_false')
|
||||
output(false_msg, "accepted_false")
|
||||
return False
|
||||
# if we get here, the answer was invalid
|
||||
if invalid_msg:
|
||||
output(invalid_msg, 'invalid_answer')
|
||||
output(invalid_msg, "invalid_answer")
|
||||
if not retry:
|
||||
return default
|
||||
if retry_msg:
|
||||
output(retry_msg, 'prompt_retry', is_prompt=True)
|
||||
output(retry_msg, "prompt_retry", is_prompt=True)
|
||||
# in case we used an environment variable and it gave an invalid answer, do not use it again:
|
||||
env_var_override = None
|
||||
|
@ -8,8 +8,8 @@
|
||||
from .helpers import Error, ErrorWithTraceback
|
||||
from .logger import create_logger
|
||||
|
||||
ADD, REMOVE = 'add', 'remove'
|
||||
SHARED, EXCLUSIVE = 'shared', 'exclusive'
|
||||
ADD, REMOVE = "add", "remove"
|
||||
SHARED, EXCLUSIVE = "shared", "exclusive"
|
||||
|
||||
logger = create_logger(__name__)
|
||||
|
||||
@ -20,6 +20,7 @@ class TimeoutTimer:
|
||||
It can also compute and optionally execute a reasonable sleep time (e.g. to avoid
|
||||
polling too often or to support thread/process rescheduling).
|
||||
"""
|
||||
|
||||
def __init__(self, timeout=None, sleep=None):
|
||||
"""
|
||||
Initialize a timer.
|
||||
@ -43,8 +44,8 @@ def __init__(self, timeout=None, sleep=None):
|
||||
|
||||
def __repr__(self):
|
||||
return "<{}: start={!r} end={!r} timeout={!r} sleep={!r}>".format(
|
||||
self.__class__.__name__, self.start_time, self.end_time,
|
||||
self.timeout_interval, self.sleep_interval)
|
||||
self.__class__.__name__, self.start_time, self.end_time, self.timeout_interval, self.sleep_interval
|
||||
)
|
||||
|
||||
def start(self):
|
||||
self.start_time = time.time()
|
||||
@ -102,6 +103,7 @@ class ExclusiveLock:
|
||||
This makes sure the lock is released again if the block is left, no
|
||||
matter how (e.g. if an exception occurred).
|
||||
"""
|
||||
|
||||
def __init__(self, path, timeout=None, sleep=None, id=None):
|
||||
self.timeout = timeout
|
||||
self.sleep = sleep
|
||||
@ -129,7 +131,7 @@ def acquire(self, timeout=None, sleep=None):
|
||||
unique_base_name = os.path.basename(self.unique_name)
|
||||
temp_path = None
|
||||
try:
|
||||
temp_path = tempfile.mkdtemp(".tmp", base_name + '.', parent_path)
|
||||
temp_path = tempfile.mkdtemp(".tmp", base_name + ".", parent_path)
|
||||
temp_unique_name = os.path.join(temp_path, unique_base_name)
|
||||
with open(temp_unique_name, "wb"):
|
||||
pass
|
||||
@ -192,8 +194,8 @@ def kill_stale_lock(self):
|
||||
else:
|
||||
for name in names:
|
||||
try:
|
||||
host_pid, thread_str = name.rsplit('-', 1)
|
||||
host, pid_str = host_pid.rsplit('.', 1)
|
||||
host_pid, thread_str = name.rsplit("-", 1)
|
||||
host, pid_str = host_pid.rsplit(".", 1)
|
||||
pid = int(pid_str)
|
||||
thread = int(thread_str)
|
||||
except ValueError:
|
||||
@ -207,17 +209,19 @@ def kill_stale_lock(self):
|
||||
if not self.kill_stale_locks:
|
||||
if not self.stale_warning_printed:
|
||||
# Log this at warning level to hint the user at the ability
|
||||
logger.warning("Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name)
|
||||
logger.warning(
|
||||
"Found stale lock %s, but not deleting because self.kill_stale_locks = False.", name
|
||||
)
|
||||
self.stale_warning_printed = True
|
||||
return False
|
||||
|
||||
try:
|
||||
os.unlink(os.path.join(self.path, name))
|
||||
logger.warning('Killed stale lock %s.', name)
|
||||
logger.warning("Killed stale lock %s.", name)
|
||||
except OSError as err:
|
||||
if not self.stale_warning_printed:
|
||||
# This error will bubble up and likely result in locking failure
|
||||
logger.error('Found stale lock %s, but cannot delete due to %s', name, str(err))
|
||||
logger.error("Found stale lock %s, but cannot delete due to %s", name, str(err))
|
||||
self.stale_warning_printed = True
|
||||
return False
|
||||
|
||||
@ -228,7 +232,7 @@ def kill_stale_lock(self):
|
||||
# Directory is not empty or doesn't exist any more = we lost the race to somebody else--which is ok.
|
||||
return False
|
||||
# EACCES or EIO or ... = we cannot operate anyway
|
||||
logger.error('Failed to remove lock dir: %s', str(err))
|
||||
logger.error("Failed to remove lock dir: %s", str(err))
|
||||
return False
|
||||
|
||||
return True
|
||||
@ -257,6 +261,7 @@ class LockRoster:
|
||||
Note: you usually should call the methods with an exclusive lock held,
|
||||
to avoid conflicting access by multiple threads/processes/machines.
|
||||
"""
|
||||
|
||||
def __init__(self, path, id=None):
|
||||
self.path = path
|
||||
self.id = id or platform.get_process_id()
|
||||
@ -279,8 +284,9 @@ def load(self):
|
||||
if platform.process_alive(host, pid, thread):
|
||||
elements.add((host, pid, thread))
|
||||
else:
|
||||
logger.warning('Removed stale %s roster lock for host %s pid %d thread %d.',
|
||||
key, host, pid, thread)
|
||||
logger.warning(
|
||||
"Removed stale %s roster lock for host %s pid %d thread %d.", key, host, pid, thread
|
||||
)
|
||||
data[key] = list(elements)
|
||||
except (FileNotFoundError, ValueError):
|
||||
# no or corrupt/empty roster file?
|
||||
@ -315,7 +321,7 @@ def modify(self, key, op):
|
||||
elif op == REMOVE:
|
||||
elements.remove(self.id)
|
||||
else:
|
||||
raise ValueError('Unknown LockRoster op %r' % op)
|
||||
raise ValueError("Unknown LockRoster op %r" % op)
|
||||
roster[key] = list(list(e) for e in elements)
|
||||
self.save(roster)
|
||||
|
||||
@ -354,6 +360,7 @@ class Lock:
|
||||
This makes sure the lock is released again if the block is left, no
|
||||
matter how (e.g. if an exception occurred).
|
||||
"""
|
||||
|
||||
def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None):
|
||||
self.path = path
|
||||
self.is_exclusive = exclusive
|
||||
@ -361,11 +368,11 @@ def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None):
|
||||
self.timeout = timeout
|
||||
self.id = id or platform.get_process_id()
|
||||
# globally keeping track of shared and exclusive lockers:
|
||||
self._roster = LockRoster(path + '.roster', id=id)
|
||||
self._roster = LockRoster(path + ".roster", id=id)
|
||||
# an exclusive lock, used for:
|
||||
# - holding while doing roster queries / updates
|
||||
# - holding while the Lock itself is exclusive
|
||||
self._lock = ExclusiveLock(path + '.exclusive', id=id, timeout=timeout)
|
||||
self._lock = ExclusiveLock(path + ".exclusive", id=id, timeout=timeout)
|
||||
|
||||
def __enter__(self):
|
||||
return self.acquire()
|
||||
|
@ -53,7 +53,7 @@ def _log_warning(message, category, filename, lineno, file=None, line=None):
|
||||
logger.warning(msg)
|
||||
|
||||
|
||||
def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info', is_serve=False, json=False):
|
||||
def setup_logging(stream=None, conf_fname=None, env_var="BORG_LOGGING_CONF", level="info", is_serve=False, json=False):
|
||||
"""setup logging module according to the arguments provided
|
||||
|
||||
if conf_fname is given (or the config file name can be determined via
|
||||
@ -80,7 +80,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
|
||||
logging.config.fileConfig(f)
|
||||
configured = True
|
||||
logger = logging.getLogger(__name__)
|
||||
borg_logger = logging.getLogger('borg')
|
||||
borg_logger = logging.getLogger("borg")
|
||||
borg_logger.json = json
|
||||
logger.debug(f'using logging configuration read from "{conf_fname}"')
|
||||
warnings.showwarning = _log_warning
|
||||
@ -88,15 +88,15 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
|
||||
except Exception as err: # XXX be more precise
|
||||
err_msg = str(err)
|
||||
# if we did not / not successfully load a logging configuration, fallback to this:
|
||||
logger = logging.getLogger('')
|
||||
logger = logging.getLogger("")
|
||||
handler = logging.StreamHandler(stream)
|
||||
if is_serve and not json:
|
||||
fmt = '$LOG %(levelname)s %(name)s Remote: %(message)s'
|
||||
fmt = "$LOG %(levelname)s %(name)s Remote: %(message)s"
|
||||
else:
|
||||
fmt = '%(message)s'
|
||||
fmt = "%(message)s"
|
||||
formatter = JsonFormatter(fmt) if json else logging.Formatter(fmt)
|
||||
handler.setFormatter(formatter)
|
||||
borg_logger = logging.getLogger('borg')
|
||||
borg_logger = logging.getLogger("borg")
|
||||
borg_logger.formatter = formatter
|
||||
borg_logger.json = json
|
||||
if configured and logger.handlers:
|
||||
@ -111,7 +111,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev
|
||||
logger = logging.getLogger(__name__)
|
||||
if err_msg:
|
||||
logger.warning(f'setup_logging for "{conf_fname}" failed with "{err_msg}".')
|
||||
logger.debug('using builtin fallback logging configuration')
|
||||
logger.debug("using builtin fallback logging configuration")
|
||||
warnings.showwarning = _log_warning
|
||||
return handler
|
||||
|
||||
@ -151,6 +151,7 @@ def create_logger(name=None):
|
||||
be careful not to call any logger methods before the setup_logging() call.
|
||||
If you try, you'll get an exception.
|
||||
"""
|
||||
|
||||
class LazyLogger:
|
||||
def __init__(self, name=None):
|
||||
self.__name = name or find_parent_module()
|
||||
@ -162,49 +163,49 @@ def __logger(self):
|
||||
if not configured:
|
||||
raise Exception("tried to call a logger before setup_logging() was called")
|
||||
self.__real_logger = logging.getLogger(self.__name)
|
||||
if self.__name.startswith('borg.debug.') and self.__real_logger.level == logging.NOTSET:
|
||||
self.__real_logger.setLevel('WARNING')
|
||||
if self.__name.startswith("borg.debug.") and self.__real_logger.level == logging.NOTSET:
|
||||
self.__real_logger.setLevel("WARNING")
|
||||
return self.__real_logger
|
||||
|
||||
def getChild(self, suffix):
|
||||
return LazyLogger(self.__name + '.' + suffix)
|
||||
return LazyLogger(self.__name + "." + suffix)
|
||||
|
||||
def setLevel(self, *args, **kw):
|
||||
return self.__logger.setLevel(*args, **kw)
|
||||
|
||||
def log(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.log(*args, **kw)
|
||||
|
||||
def exception(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.exception(*args, **kw)
|
||||
|
||||
def debug(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.debug(*args, **kw)
|
||||
|
||||
def info(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.info(*args, **kw)
|
||||
|
||||
def warning(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.warning(*args, **kw)
|
||||
|
||||
def error(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.error(*args, **kw)
|
||||
|
||||
def critical(self, *args, **kw):
|
||||
if 'msgid' in kw:
|
||||
kw.setdefault('extra', {})['msgid'] = kw.pop('msgid')
|
||||
if "msgid" in kw:
|
||||
kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
|
||||
return self.__logger.critical(*args, **kw)
|
||||
|
||||
return LazyLogger(name)
|
||||
@ -212,11 +213,11 @@ def critical(self, *args, **kw):
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
RECORD_ATTRIBUTES = (
|
||||
'levelname',
|
||||
'name',
|
||||
'message',
|
||||
"levelname",
|
||||
"name",
|
||||
"message",
|
||||
# msgid is an attribute we made up in Borg to expose a non-changing handle for log messages
|
||||
'msgid',
|
||||
"msgid",
|
||||
)
|
||||
|
||||
# Other attributes that are not very useful but do exist:
|
||||
@ -229,12 +230,7 @@ class JsonFormatter(logging.Formatter):
|
||||
|
||||
def format(self, record):
|
||||
super().format(record)
|
||||
data = {
|
||||
'type': 'log_message',
|
||||
'time': record.created,
|
||||
'message': '',
|
||||
'levelname': 'CRITICAL',
|
||||
}
|
||||
data = {"type": "log_message", "time": record.created, "message": "", "levelname": "CRITICAL"}
|
||||
for attr in self.RECORD_ATTRIBUTES:
|
||||
value = getattr(record, attr, None)
|
||||
if value:
|
||||
|
@ -10,8 +10,8 @@ def __init__(self, capacity, dispose):
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
assert key not in self._cache, (
|
||||
"Unexpected attempt to replace a cached item,"
|
||||
" without first deleting the old item.")
|
||||
"Unexpected attempt to replace a cached item," " without first deleting the old item."
|
||||
)
|
||||
self._lru.append(key)
|
||||
while len(self._lru) > self._capacity:
|
||||
del self[self._lru[0]]
|
||||
|
@ -11,18 +11,18 @@ def __init__(self, s):
|
||||
|
||||
def read(self, n):
|
||||
self.i += n
|
||||
return self.str[self.i - n:self.i]
|
||||
return self.str[self.i - n : self.i]
|
||||
|
||||
def peek(self, n):
|
||||
if n >= 0:
|
||||
return self.str[self.i:self.i + n]
|
||||
return self.str[self.i : self.i + n]
|
||||
else:
|
||||
return self.str[self.i + n - 1:self.i - 1]
|
||||
return self.str[self.i + n - 1 : self.i - 1]
|
||||
|
||||
def peekline(self):
|
||||
out = ''
|
||||
out = ""
|
||||
i = self.i
|
||||
while i < len(self.str) and self.str[i] != '\n':
|
||||
while i < len(self.str) and self.str[i] != "\n":
|
||||
out += self.str[i]
|
||||
i += 1
|
||||
return out
|
||||
@ -34,18 +34,18 @@ def readline(self):
|
||||
|
||||
|
||||
def process_directive(directive, arguments, out, state_hook):
|
||||
if directive == 'container' and arguments == 'experimental':
|
||||
state_hook('text', '**', out)
|
||||
out.write('++ Experimental ++')
|
||||
state_hook('**', 'text', out)
|
||||
if directive == "container" and arguments == "experimental":
|
||||
state_hook("text", "**", out)
|
||||
out.write("++ Experimental ++")
|
||||
state_hook("**", "text", out)
|
||||
else:
|
||||
state_hook('text', '**', out)
|
||||
state_hook("text", "**", out)
|
||||
out.write(directive.title())
|
||||
out.write(':\n')
|
||||
state_hook('**', 'text', out)
|
||||
out.write(":\n")
|
||||
state_hook("**", "text", out)
|
||||
if arguments:
|
||||
out.write(arguments)
|
||||
out.write('\n')
|
||||
out.write("\n")
|
||||
|
||||
|
||||
def rst_to_text(text, state_hook=None, references=None):
|
||||
@ -58,12 +58,12 @@ def rst_to_text(text, state_hook=None, references=None):
|
||||
"""
|
||||
state_hook = state_hook or (lambda old_state, new_state, out: None)
|
||||
references = references or {}
|
||||
state = 'text'
|
||||
inline_mode = 'replace'
|
||||
state = "text"
|
||||
inline_mode = "replace"
|
||||
text = TextPecker(text)
|
||||
out = io.StringIO()
|
||||
|
||||
inline_single = ('*', '`')
|
||||
inline_single = ("*", "`")
|
||||
|
||||
while True:
|
||||
char = text.read(1)
|
||||
@ -71,81 +71,83 @@ def rst_to_text(text, state_hook=None, references=None):
|
||||
break
|
||||
next = text.peek(1) # type: str
|
||||
|
||||
if state == 'text':
|
||||
if char == '\\' and text.peek(1) in inline_single:
|
||||
if state == "text":
|
||||
if char == "\\" and text.peek(1) in inline_single:
|
||||
continue
|
||||
if text.peek(-1) != '\\':
|
||||
if text.peek(-1) != "\\":
|
||||
if char in inline_single and next != char:
|
||||
state_hook(state, char, out)
|
||||
state = char
|
||||
continue
|
||||
if char == next == '*':
|
||||
state_hook(state, '**', out)
|
||||
state = '**'
|
||||
if char == next == "*":
|
||||
state_hook(state, "**", out)
|
||||
state = "**"
|
||||
text.read(1)
|
||||
continue
|
||||
if char == next == '`':
|
||||
state_hook(state, '``', out)
|
||||
state = '``'
|
||||
if char == next == "`":
|
||||
state_hook(state, "``", out)
|
||||
state = "``"
|
||||
text.read(1)
|
||||
continue
|
||||
if text.peek(-1).isspace() and char == ':' and text.peek(5) == 'ref:`':
|
||||
if text.peek(-1).isspace() and char == ":" and text.peek(5) == "ref:`":
|
||||
# translate reference
|
||||
text.read(5)
|
||||
ref = ''
|
||||
ref = ""
|
||||
while True:
|
||||
char = text.peek(1)
|
||||
if char == '`':
|
||||
if char == "`":
|
||||
text.read(1)
|
||||
break
|
||||
if char == '\n':
|
||||
if char == "\n":
|
||||
text.read(1)
|
||||
continue # merge line breaks in :ref:`...\n...`
|
||||
ref += text.read(1)
|
||||
try:
|
||||
out.write(references[ref])
|
||||
except KeyError:
|
||||
raise ValueError("Undefined reference in Archiver help: %r — please add reference "
|
||||
"substitution to 'rst_plain_text_references'" % ref)
|
||||
raise ValueError(
|
||||
"Undefined reference in Archiver help: %r — please add reference "
|
||||
"substitution to 'rst_plain_text_references'" % ref
|
||||
)
|
||||
continue
|
||||
if char == ':' and text.peek(2) == ':\n': # End of line code block
|
||||
if char == ":" and text.peek(2) == ":\n": # End of line code block
|
||||
text.read(2)
|
||||
state_hook(state, 'code-block', out)
|
||||
state = 'code-block'
|
||||
out.write(':\n')
|
||||
state_hook(state, "code-block", out)
|
||||
state = "code-block"
|
||||
out.write(":\n")
|
||||
continue
|
||||
if text.peek(-2) in ('\n\n', '') and char == next == '.':
|
||||
if text.peek(-2) in ("\n\n", "") and char == next == ".":
|
||||
text.read(2)
|
||||
directive, is_directive, arguments = text.readline().partition('::')
|
||||
directive, is_directive, arguments = text.readline().partition("::")
|
||||
text.read(1)
|
||||
if not is_directive:
|
||||
# partition: if the separator is not in the text, the leftmost output is the entire input
|
||||
if directive == 'nanorst: inline-fill':
|
||||
inline_mode = 'fill'
|
||||
elif directive == 'nanorst: inline-replace':
|
||||
inline_mode = 'replace'
|
||||
if directive == "nanorst: inline-fill":
|
||||
inline_mode = "fill"
|
||||
elif directive == "nanorst: inline-replace":
|
||||
inline_mode = "replace"
|
||||
continue
|
||||
process_directive(directive, arguments.strip(), out, state_hook)
|
||||
continue
|
||||
if state in inline_single and char == state:
|
||||
state_hook(state, 'text', out)
|
||||
state = 'text'
|
||||
if inline_mode == 'fill':
|
||||
out.write(2 * ' ')
|
||||
state_hook(state, "text", out)
|
||||
state = "text"
|
||||
if inline_mode == "fill":
|
||||
out.write(2 * " ")
|
||||
continue
|
||||
if state == '``' and char == next == '`':
|
||||
state_hook(state, 'text', out)
|
||||
state = 'text'
|
||||
if state == "``" and char == next == "`":
|
||||
state_hook(state, "text", out)
|
||||
state = "text"
|
||||
text.read(1)
|
||||
if inline_mode == 'fill':
|
||||
out.write(4 * ' ')
|
||||
if inline_mode == "fill":
|
||||
out.write(4 * " ")
|
||||
continue
|
||||
if state == '**' and char == next == '*':
|
||||
state_hook(state, 'text', out)
|
||||
state = 'text'
|
||||
if state == "**" and char == next == "*":
|
||||
state_hook(state, "text", out)
|
||||
state = "text"
|
||||
text.read(1)
|
||||
continue
|
||||
if state == 'code-block' and char == next == '\n' and text.peek(5)[1:] != ' ':
|
||||
if state == "code-block" and char == next == "\n" and text.peek(5)[1:] != " ":
|
||||
# Foo::
|
||||
#
|
||||
# *stuff* *code* *ignore .. all markup*
|
||||
@ -153,11 +155,11 @@ def rst_to_text(text, state_hook=None, references=None):
|
||||
# More arcane stuff
|
||||
#
|
||||
# Regular text...
|
||||
state_hook(state, 'text', out)
|
||||
state = 'text'
|
||||
state_hook(state, "text", out)
|
||||
state = "text"
|
||||
out.write(char)
|
||||
|
||||
assert state == 'text', 'Invalid final state %r (This usually indicates unmatched */**)' % state
|
||||
assert state == "text", "Invalid final state %r (This usually indicates unmatched */**)" % state
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
@ -191,12 +193,12 @@ def __contains__(self, item):
|
||||
|
||||
|
||||
def ansi_escapes(old_state, new_state, out):
|
||||
if old_state == 'text' and new_state in ('*', '`', '``'):
|
||||
out.write('\033[4m')
|
||||
if old_state == 'text' and new_state == '**':
|
||||
out.write('\033[1m')
|
||||
if old_state in ('*', '`', '``', '**') and new_state == 'text':
|
||||
out.write('\033[0m')
|
||||
if old_state == "text" and new_state in ("*", "`", "``"):
|
||||
out.write("\033[4m")
|
||||
if old_state == "text" and new_state == "**":
|
||||
out.write("\033[1m")
|
||||
if old_state in ("*", "`", "``", "**") and new_state == "text":
|
||||
out.write("\033[0m")
|
||||
|
||||
|
||||
def rst_to_terminal(rst, references=None, destination=sys.stdout):
|
||||
|
@ -75,6 +75,7 @@ class PatternMatcher:
|
||||
*fallback* is a boolean value that *match()* returns if no matching patterns are found.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, fallback=None):
|
||||
self._items = []
|
||||
|
||||
@ -96,18 +97,13 @@ def __init__(self, fallback=None):
|
||||
self.include_patterns = []
|
||||
|
||||
# TODO: move this info to parse_inclexcl_command and store in PatternBase subclass?
|
||||
self.is_include_cmd = {
|
||||
IECommand.Exclude: False,
|
||||
IECommand.ExcludeNoRecurse: False,
|
||||
IECommand.Include: True
|
||||
}
|
||||
self.is_include_cmd = {IECommand.Exclude: False, IECommand.ExcludeNoRecurse: False, IECommand.Include: True}
|
||||
|
||||
def empty(self):
|
||||
return not len(self._items) and not len(self._path_full_patterns)
|
||||
|
||||
def _add(self, pattern, cmd):
|
||||
"""*cmd* is an IECommand value.
|
||||
"""
|
||||
"""*cmd* is an IECommand value."""
|
||||
if isinstance(pattern, PathFullPattern):
|
||||
key = pattern.pattern # full, normalized path
|
||||
self._path_full_patterns[key] = cmd
|
||||
@ -123,8 +119,7 @@ def add(self, patterns, cmd):
|
||||
self._add(pattern, cmd)
|
||||
|
||||
def add_includepaths(self, include_paths):
|
||||
"""Used to add inclusion-paths from args.paths (from commandline).
|
||||
"""
|
||||
"""Used to add inclusion-paths from args.paths (from commandline)."""
|
||||
include_patterns = [parse_pattern(p, PathPrefixPattern) for p in include_paths]
|
||||
self.add(include_patterns, IECommand.Include)
|
||||
self.fallback = not include_patterns
|
||||
@ -135,8 +130,7 @@ def get_unmatched_include_patterns(self):
|
||||
return [p for p in self.include_patterns if p.match_count == 0]
|
||||
|
||||
def add_inclexcl(self, patterns):
|
||||
"""Add list of patterns (of type CmdTuple) to internal list.
|
||||
"""
|
||||
"""Add list of patterns (of type CmdTuple) to internal list."""
|
||||
for pattern, cmd in patterns:
|
||||
self._add(pattern, cmd)
|
||||
|
||||
@ -172,12 +166,12 @@ def normalize_path(path):
|
||||
"""normalize paths for MacOS (but do nothing on other platforms)"""
|
||||
# HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match.
|
||||
# Windows and Unix filesystems allow different forms, so users always have to enter an exact match.
|
||||
return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path
|
||||
return unicodedata.normalize("NFD", path) if sys.platform == "darwin" else path
|
||||
|
||||
|
||||
class PatternBase:
|
||||
"""Shared logic for inclusion/exclusion patterns.
|
||||
"""
|
||||
"""Shared logic for inclusion/exclusion patterns."""
|
||||
|
||||
PREFIX = NotImplemented
|
||||
|
||||
def __init__(self, pattern, recurse_dir=False):
|
||||
@ -201,7 +195,7 @@ def match(self, path, normalize=True):
|
||||
return matches
|
||||
|
||||
def __repr__(self):
|
||||
return f'{type(self)}({self.pattern})'
|
||||
return f"{type(self)}({self.pattern})"
|
||||
|
||||
def __str__(self):
|
||||
return self.pattern_orig
|
||||
@ -216,6 +210,7 @@ def _match(self, path):
|
||||
|
||||
class PathFullPattern(PatternBase):
|
||||
"""Full match of a path."""
|
||||
|
||||
PREFIX = "pf"
|
||||
|
||||
def _prepare(self, pattern):
|
||||
@ -236,6 +231,7 @@ class PathPrefixPattern(PatternBase):
|
||||
If a directory is specified, all paths that start with that
|
||||
path match as well. A trailing slash makes no difference.
|
||||
"""
|
||||
|
||||
PREFIX = "pp"
|
||||
|
||||
def _prepare(self, pattern):
|
||||
@ -251,13 +247,14 @@ class FnmatchPattern(PatternBase):
|
||||
"""Shell glob patterns to exclude. A trailing slash means to
|
||||
exclude the contents of a directory, but not the directory itself.
|
||||
"""
|
||||
|
||||
PREFIX = "fm"
|
||||
|
||||
def _prepare(self, pattern):
|
||||
if pattern.endswith(os.path.sep):
|
||||
pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep
|
||||
pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + "*" + os.path.sep
|
||||
else:
|
||||
pattern = os.path.normpath(pattern) + os.path.sep + '*'
|
||||
pattern = os.path.normpath(pattern) + os.path.sep + "*"
|
||||
|
||||
self.pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed
|
||||
|
||||
@ -266,13 +263,14 @@ def _prepare(self, pattern):
|
||||
self.regex = re.compile(fnmatch.translate(self.pattern))
|
||||
|
||||
def _match(self, path):
|
||||
return (self.regex.match(path + os.path.sep) is not None)
|
||||
return self.regex.match(path + os.path.sep) is not None
|
||||
|
||||
|
||||
class ShellPattern(PatternBase):
|
||||
"""Shell glob patterns to exclude. A trailing slash means to
|
||||
exclude the contents of a directory, but not the directory itself.
|
||||
"""
|
||||
|
||||
PREFIX = "sh"
|
||||
|
||||
def _prepare(self, pattern):
|
||||
@ -287,12 +285,12 @@ def _prepare(self, pattern):
|
||||
self.regex = re.compile(shellpattern.translate(self.pattern))
|
||||
|
||||
def _match(self, path):
|
||||
return (self.regex.match(path + os.path.sep) is not None)
|
||||
return self.regex.match(path + os.path.sep) is not None
|
||||
|
||||
|
||||
class RegexPattern(PatternBase):
|
||||
"""Regular expression to exclude.
|
||||
"""
|
||||
"""Regular expression to exclude."""
|
||||
|
||||
PREFIX = "re"
|
||||
|
||||
def _prepare(self, pattern):
|
||||
@ -301,28 +299,22 @@ def _prepare(self, pattern):
|
||||
|
||||
def _match(self, path):
|
||||
# Normalize path separators
|
||||
if os.path.sep != '/':
|
||||
path = path.replace(os.path.sep, '/')
|
||||
if os.path.sep != "/":
|
||||
path = path.replace(os.path.sep, "/")
|
||||
|
||||
return (self.regex.search(path) is not None)
|
||||
return self.regex.search(path) is not None
|
||||
|
||||
|
||||
_PATTERN_CLASSES = {
|
||||
FnmatchPattern,
|
||||
PathFullPattern,
|
||||
PathPrefixPattern,
|
||||
RegexPattern,
|
||||
ShellPattern,
|
||||
}
|
||||
_PATTERN_CLASSES = {FnmatchPattern, PathFullPattern, PathPrefixPattern, RegexPattern, ShellPattern}
|
||||
|
||||
_PATTERN_CLASS_BY_PREFIX = {i.PREFIX: i for i in _PATTERN_CLASSES}
|
||||
|
||||
CmdTuple = namedtuple('CmdTuple', 'val cmd')
|
||||
CmdTuple = namedtuple("CmdTuple", "val cmd")
|
||||
|
||||
|
||||
class IECommand(Enum):
|
||||
"""A command that an InclExcl file line can represent.
|
||||
"""
|
||||
"""A command that an InclExcl file line can represent."""
|
||||
|
||||
RootPath = 1
|
||||
PatternStyle = 2
|
||||
Include = 3
|
||||
@ -343,9 +335,7 @@ def get_pattern_class(prefix):
|
||||
|
||||
|
||||
def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True):
|
||||
"""Read pattern from string and return an instance of the appropriate implementation class.
|
||||
|
||||
"""
|
||||
"""Read pattern from string and return an instance of the appropriate implementation class."""
|
||||
if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum():
|
||||
(style, pattern) = (pattern[:2], pattern[3:])
|
||||
cls = get_pattern_class(style)
|
||||
@ -355,8 +345,7 @@ def parse_pattern(pattern, fallback=FnmatchPattern, recurse_dir=True):
|
||||
|
||||
|
||||
def parse_exclude_pattern(pattern_str, fallback=FnmatchPattern):
|
||||
"""Read pattern from string and return an instance of the appropriate implementation class.
|
||||
"""
|
||||
"""Read pattern from string and return an instance of the appropriate implementation class."""
|
||||
epattern_obj = parse_pattern(pattern_str, fallback, recurse_dir=False)
|
||||
return CmdTuple(epattern_obj, IECommand.ExcludeNoRecurse)
|
||||
|
||||
@ -365,21 +354,20 @@ def parse_inclexcl_command(cmd_line_str, fallback=ShellPattern):
|
||||
"""Read a --patterns-from command from string and return a CmdTuple object."""
|
||||
|
||||
cmd_prefix_map = {
|
||||
'-': IECommand.Exclude,
|
||||
'!': IECommand.ExcludeNoRecurse,
|
||||
'+': IECommand.Include,
|
||||
'R': IECommand.RootPath,
|
||||
'r': IECommand.RootPath,
|
||||
'P': IECommand.PatternStyle,
|
||||
'p': IECommand.PatternStyle,
|
||||
"-": IECommand.Exclude,
|
||||
"!": IECommand.ExcludeNoRecurse,
|
||||
"+": IECommand.Include,
|
||||
"R": IECommand.RootPath,
|
||||
"r": IECommand.RootPath,
|
||||
"P": IECommand.PatternStyle,
|
||||
"p": IECommand.PatternStyle,
|
||||
}
|
||||
if not cmd_line_str:
|
||||
raise argparse.ArgumentTypeError("A pattern/command must not be empty.")
|
||||
|
||||
cmd = cmd_prefix_map.get(cmd_line_str[0])
|
||||
if cmd is None:
|
||||
raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" %
|
||||
', '.join(cmd_prefix_map))
|
||||
raise argparse.ArgumentTypeError("A pattern/command must start with anyone of: %s" % ", ".join(cmd_prefix_map))
|
||||
|
||||
# remaining text on command-line following the command character
|
||||
remainder_str = cmd_line_str[1:].lstrip()
|
||||
|
@ -17,6 +17,7 @@
|
||||
|
||||
if not is_win32:
|
||||
from .posix import process_alive, local_pid_alive
|
||||
|
||||
# posix swidth implementation works for: linux, freebsd, darwin, openindiana, cygwin
|
||||
from .posix import swidth
|
||||
from .posix import get_errno
|
||||
|
@ -20,9 +20,9 @@
|
||||
are correctly composed into the base functionality.
|
||||
"""
|
||||
|
||||
API_VERSION = '1.2_05'
|
||||
API_VERSION = "1.2_05"
|
||||
|
||||
fdatasync = getattr(os, 'fdatasync', os.fsync)
|
||||
fdatasync = getattr(os, "fdatasync", os.fsync)
|
||||
|
||||
from .xattr import ENOATTR
|
||||
|
||||
@ -86,14 +86,16 @@ def acl_set(path, item, numeric_ids=False, fd=None):
|
||||
|
||||
def set_flags(path, bsd_flags, fd=None):
|
||||
lchflags(path, bsd_flags)
|
||||
|
||||
except ImportError:
|
||||
|
||||
def set_flags(path, bsd_flags, fd=None):
|
||||
pass
|
||||
|
||||
|
||||
def get_flags(path, st, fd=None):
|
||||
"""Return BSD-style file flags for path or stat without following symlinks."""
|
||||
return getattr(st, 'st_flags', 0)
|
||||
return getattr(st, "st_flags", 0)
|
||||
|
||||
|
||||
def sync_dir(path):
|
||||
@ -114,8 +116,8 @@ def sync_dir(path):
|
||||
|
||||
|
||||
def safe_fadvise(fd, offset, len, advice):
|
||||
if hasattr(os, 'posix_fadvise'):
|
||||
advice = getattr(os, 'POSIX_FADV_' + advice)
|
||||
if hasattr(os, "posix_fadvise"):
|
||||
advice = getattr(os, "POSIX_FADV_" + advice)
|
||||
try:
|
||||
os.posix_fadvise(fd, offset, len, advice)
|
||||
except OSError:
|
||||
@ -158,7 +160,7 @@ def __init__(self, path, *, fd=None, binary=False):
|
||||
that corresponds to path (like from os.open(path, ...) or os.mkstemp(...))
|
||||
:param binary: whether to open in binary mode, default is False.
|
||||
"""
|
||||
mode = 'xb' if binary else 'x' # x -> raise FileExists exception in open() if file exists already
|
||||
mode = "xb" if binary else "x" # x -> raise FileExists exception in open() if file exists already
|
||||
self.path = path
|
||||
if fd is None:
|
||||
self.f = open(path, mode=mode) # python file object
|
||||
@ -181,15 +183,17 @@ def sync(self):
|
||||
after sync().
|
||||
"""
|
||||
from .. import platform
|
||||
|
||||
self.f.flush()
|
||||
platform.fdatasync(self.fd)
|
||||
# tell the OS that it does not need to cache what we just wrote,
|
||||
# avoids spoiling the cache for the OS and other processes.
|
||||
safe_fadvise(self.fd, 0, 0, 'DONTNEED')
|
||||
safe_fadvise(self.fd, 0, 0, "DONTNEED")
|
||||
|
||||
def close(self):
|
||||
"""sync() and close."""
|
||||
from .. import platform
|
||||
|
||||
dirname = None
|
||||
try:
|
||||
dirname = os.path.dirname(self.path)
|
||||
@ -216,23 +220,26 @@ class SaveFile:
|
||||
Internally used temporary files are created in the target directory and are
|
||||
named <BASENAME>-<RANDOMCHARS>.tmp and cleaned up in normal and error conditions.
|
||||
"""
|
||||
|
||||
def __init__(self, path, binary=False):
|
||||
self.binary = binary
|
||||
self.path = path
|
||||
self.dir = os.path.dirname(path)
|
||||
self.tmp_prefix = os.path.basename(path) + '-'
|
||||
self.tmp_prefix = os.path.basename(path) + "-"
|
||||
self.tmp_fd = None # OS-level fd
|
||||
self.tmp_fname = None # full path/filename corresponding to self.tmp_fd
|
||||
self.f = None # python-file-like SyncFile
|
||||
|
||||
def __enter__(self):
|
||||
from .. import platform
|
||||
self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix='.tmp', dir=self.dir)
|
||||
|
||||
self.tmp_fd, self.tmp_fname = tempfile.mkstemp(prefix=self.tmp_prefix, suffix=".tmp", dir=self.dir)
|
||||
self.f = platform.SyncFile(self.tmp_fname, fd=self.tmp_fd, binary=self.binary)
|
||||
return self.f
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
from .. import platform
|
||||
|
||||
self.f.close() # this indirectly also closes self.tmp_fd
|
||||
self.tmp_fd = None
|
||||
if exc_type is not None:
|
||||
@ -246,7 +253,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
# thanks to the crappy os.umask api, we can't query the umask without setting it. :-(
|
||||
umask = os.umask(UMASK_DEFAULT)
|
||||
os.umask(umask)
|
||||
os.chmod(self.tmp_fname, mode=0o666 & ~ umask)
|
||||
os.chmod(self.tmp_fname, mode=0o666 & ~umask)
|
||||
except OSError:
|
||||
# chmod might fail if the fs does not support it.
|
||||
# this is not harmful, the file will still have permissions for the owner.
|
||||
@ -270,13 +277,13 @@ def swidth(s):
|
||||
|
||||
|
||||
# patched socket.getfqdn() - see https://bugs.python.org/issue5004
|
||||
def getfqdn(name=''):
|
||||
def getfqdn(name=""):
|
||||
"""Get fully qualified domain name from name.
|
||||
|
||||
An empty argument is interpreted as meaning the local host.
|
||||
"""
|
||||
name = name.strip()
|
||||
if not name or name == '0.0.0.0':
|
||||
if not name or name == "0.0.0.0":
|
||||
name = socket.gethostname()
|
||||
try:
|
||||
addrs = socket.getaddrinfo(name, None, 0, socket.SOCK_DGRAM, 0, socket.AI_CANONNAME)
|
||||
@ -296,14 +303,14 @@ def getfqdn(name=''):
|
||||
fqdn = getfqdn(hostname)
|
||||
# some people put the fqdn into /etc/hostname (which is wrong, should be the short hostname)
|
||||
# fix this (do the same as "hostname --short" cli command does internally):
|
||||
hostname = hostname.split('.')[0]
|
||||
hostname = hostname.split(".")[0]
|
||||
|
||||
# uuid.getnode() is problematic in some environments (e.g. OpenVZ, see #3968) where the virtual MAC address
|
||||
# is all-zero. uuid.getnode falls back to returning a random value in that case, which is not what we want.
|
||||
# thus, we offer BORG_HOST_ID where a user can set an own, unique id for each of his hosts.
|
||||
hostid = os.environ.get('BORG_HOST_ID')
|
||||
hostid = os.environ.get("BORG_HOST_ID")
|
||||
if not hostid:
|
||||
hostid = f'{fqdn}@{uuid.getnode()}'
|
||||
hostid = f"{fqdn}@{uuid.getnode()}"
|
||||
|
||||
|
||||
def get_process_id():
|
||||
|
@ -18,7 +18,7 @@ def split_string0(buf):
|
||||
"""split a list of zero-terminated strings into python not-zero-terminated bytes"""
|
||||
if isinstance(buf, bytearray):
|
||||
buf = bytes(buf) # use a bytes object, so we return a list of bytes objects
|
||||
return buf.split(b'\0')[:-1]
|
||||
return buf.split(b"\0")[:-1]
|
||||
|
||||
|
||||
def split_lstring(buf):
|
||||
@ -27,8 +27,8 @@ def split_lstring(buf):
|
||||
mv = memoryview(buf)
|
||||
while mv:
|
||||
length = mv[0]
|
||||
result.append(bytes(mv[1:1 + length]))
|
||||
mv = mv[1 + length:]
|
||||
result.append(bytes(mv[1 : 1 + length]))
|
||||
mv = mv[1 + length :]
|
||||
return result
|
||||
|
||||
|
||||
@ -38,6 +38,7 @@ class BufferTooSmallError(Exception):
|
||||
|
||||
def _check(rv, path=None, detect_buffer_too_small=False):
|
||||
from . import get_errno
|
||||
|
||||
if rv < 0:
|
||||
e = get_errno()
|
||||
if detect_buffer_too_small and e == errno.ERANGE:
|
||||
@ -48,9 +49,9 @@ def _check(rv, path=None, detect_buffer_too_small=False):
|
||||
try:
|
||||
msg = os.strerror(e)
|
||||
except ValueError:
|
||||
msg = ''
|
||||
msg = ""
|
||||
if isinstance(path, int):
|
||||
path = '<FD %d>' % path
|
||||
path = "<FD %d>" % path
|
||||
raise OSError(e, msg, path)
|
||||
if detect_buffer_too_small and rv >= len(buffer):
|
||||
# freebsd does not error with ERANGE if the buffer is too small,
|
||||
|
@ -6,7 +6,7 @@
|
||||
|
||||
import sys
|
||||
|
||||
is_win32 = sys.platform.startswith('win32')
|
||||
is_linux = sys.platform.startswith('linux')
|
||||
is_freebsd = sys.platform.startswith('freebsd')
|
||||
is_darwin = sys.platform.startswith('darwin')
|
||||
is_win32 = sys.platform.startswith("win32")
|
||||
is_linux = sys.platform.startswith("linux")
|
||||
is_freebsd = sys.platform.startswith("freebsd")
|
||||
is_darwin = sys.platform.startswith("darwin")
|
||||
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -50,16 +50,16 @@ def test_name(self, test):
|
||||
|
||||
def log_results(self, logger):
|
||||
for test, failure in self.errors + self.failures + self.unexpectedSuccesses:
|
||||
logger.error('self test %s FAILED:\n%s', self.test_name(test), failure)
|
||||
logger.error("self test %s FAILED:\n%s", self.test_name(test), failure)
|
||||
for test, reason in self.skipped:
|
||||
logger.warning('self test %s skipped: %s', self.test_name(test), reason)
|
||||
logger.warning("self test %s skipped: %s", self.test_name(test), reason)
|
||||
|
||||
def successful_test_count(self):
|
||||
return len(self.successes)
|
||||
|
||||
|
||||
def selftest(logger):
|
||||
if os.environ.get('BORG_SELFTEST') == 'disabled':
|
||||
if os.environ.get("BORG_SELFTEST") == "disabled":
|
||||
logger.debug("borg selftest disabled via BORG_SELFTEST env variable")
|
||||
return
|
||||
selftest_started = time.perf_counter()
|
||||
@ -69,7 +69,7 @@ def selftest(logger):
|
||||
module = sys.modules[test_case.__module__]
|
||||
# a normal borg user does not have pytest installed, we must not require it in the test modules used here.
|
||||
# note: this only detects the usual toplevel import
|
||||
assert 'pytest' not in dir(module), "pytest must not be imported in %s" % module.__name__
|
||||
assert "pytest" not in dir(module), "pytest must not be imported in %s" % module.__name__
|
||||
test_suite.addTest(defaultTestLoader.loadTestsFromTestCase(test_case))
|
||||
test_suite.run(result)
|
||||
result.log_results(logger)
|
||||
@ -77,12 +77,17 @@ def selftest(logger):
|
||||
count_mismatch = successful_tests != SELFTEST_COUNT
|
||||
if result.wasSuccessful() and count_mismatch:
|
||||
# only print this if all tests succeeded
|
||||
logger.error("self test count (%d != %d) mismatch, either test discovery is broken or a test was added "
|
||||
"without updating borg.selftest",
|
||||
successful_tests, SELFTEST_COUNT)
|
||||
logger.error(
|
||||
"self test count (%d != %d) mismatch, either test discovery is broken or a test was added "
|
||||
"without updating borg.selftest",
|
||||
successful_tests,
|
||||
SELFTEST_COUNT,
|
||||
)
|
||||
if not result.wasSuccessful() or count_mismatch:
|
||||
logger.error("self test failed\n"
|
||||
"Could be a bug either in Borg, the package / distribution you use, your OS or your hardware.")
|
||||
logger.error(
|
||||
"self test failed\n"
|
||||
"Could be a bug either in Borg, the package / distribution you use, your OS or your hardware."
|
||||
)
|
||||
sys.exit(2)
|
||||
assert False, "sanity assertion failed: ran beyond sys.exit()"
|
||||
selftest_elapsed = time.perf_counter() - selftest_started
|
||||
|
@ -33,7 +33,7 @@ def translate(pat, match_end=r"\Z"):
|
||||
if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep:
|
||||
# **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands
|
||||
# for the platform-specific path separator
|
||||
res += fr"(?:[^\{sep}]*\{sep})*"
|
||||
res += rf"(?:[^\{sep}]*\{sep})*"
|
||||
i += 2
|
||||
else:
|
||||
# * == wildcard for name parts (does not cross path separator)
|
||||
|
@ -2,6 +2,7 @@
|
||||
import filecmp
|
||||
import functools
|
||||
import os
|
||||
|
||||
try:
|
||||
import posix
|
||||
except ImportError:
|
||||
@ -25,14 +26,14 @@
|
||||
from ..fuse_impl import llfuse, has_pyfuse3, has_llfuse
|
||||
|
||||
# Does this version of llfuse support ns precision?
|
||||
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False
|
||||
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, "st_mtime_ns") if llfuse else False
|
||||
|
||||
try:
|
||||
from pytest import raises
|
||||
except: # noqa
|
||||
raises = None
|
||||
|
||||
has_lchflags = hasattr(os, 'lchflags') or sys.platform.startswith('linux')
|
||||
has_lchflags = hasattr(os, "lchflags") or sys.platform.startswith("linux")
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile() as file:
|
||||
platform.set_flags(file.name, stat.UF_NODUMP)
|
||||
@ -40,14 +41,14 @@
|
||||
has_lchflags = False
|
||||
|
||||
# The mtime get/set precision varies on different OS and Python versions
|
||||
if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
|
||||
if posix and "HAVE_FUTIMENS" in getattr(posix, "_have_functions", []):
|
||||
st_mtime_ns_round = 0
|
||||
elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
|
||||
elif "HAVE_UTIMES" in sysconfig.get_config_vars():
|
||||
st_mtime_ns_round = -6
|
||||
else:
|
||||
st_mtime_ns_round = -9
|
||||
|
||||
if sys.platform.startswith('netbsd'):
|
||||
if sys.platform.startswith("netbsd"):
|
||||
st_mtime_ns_round = -4 # only >1 microsecond resolution here?
|
||||
|
||||
|
||||
@ -61,8 +62,8 @@ def unopened_tempfile():
|
||||
def are_symlinks_supported():
|
||||
with unopened_tempfile() as filepath:
|
||||
try:
|
||||
os.symlink('somewhere', filepath)
|
||||
if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == 'somewhere':
|
||||
os.symlink("somewhere", filepath)
|
||||
if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == "somewhere":
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
@ -71,12 +72,12 @@ def are_symlinks_supported():
|
||||
|
||||
@functools.lru_cache
|
||||
def are_hardlinks_supported():
|
||||
if not hasattr(os, 'link'):
|
||||
if not hasattr(os, "link"):
|
||||
# some pythons do not have os.link
|
||||
return False
|
||||
|
||||
with unopened_tempfile() as file1path, unopened_tempfile() as file2path:
|
||||
open(file1path, 'w').close()
|
||||
open(file1path, "w").close()
|
||||
try:
|
||||
os.link(file1path, file2path)
|
||||
stat1 = os.stat(file1path)
|
||||
@ -108,9 +109,9 @@ def is_utime_fully_supported():
|
||||
with unopened_tempfile() as filepath:
|
||||
# Some filesystems (such as SSHFS) don't support utime on symlinks
|
||||
if are_symlinks_supported():
|
||||
os.symlink('something', filepath)
|
||||
os.symlink("something", filepath)
|
||||
else:
|
||||
open(filepath, 'w').close()
|
||||
open(filepath, "w").close()
|
||||
try:
|
||||
os.utime(filepath, (1000, 2000), follow_symlinks=False)
|
||||
new_stats = os.stat(filepath, follow_symlinks=False)
|
||||
@ -125,14 +126,14 @@ def is_utime_fully_supported():
|
||||
|
||||
@functools.lru_cache
|
||||
def is_birthtime_fully_supported():
|
||||
if not hasattr(os.stat_result, 'st_birthtime'):
|
||||
if not hasattr(os.stat_result, "st_birthtime"):
|
||||
return False
|
||||
with unopened_tempfile() as filepath:
|
||||
# Some filesystems (such as SSHFS) don't support utime on symlinks
|
||||
if are_symlinks_supported():
|
||||
os.symlink('something', filepath)
|
||||
os.symlink("something", filepath)
|
||||
else:
|
||||
open(filepath, 'w').close()
|
||||
open(filepath, "w").close()
|
||||
try:
|
||||
birthtime, mtime, atime = 946598400, 946684800, 946771200
|
||||
os.utime(filepath, (atime, birthtime), follow_symlinks=False)
|
||||
@ -149,7 +150,7 @@ def is_birthtime_fully_supported():
|
||||
|
||||
def no_selinux(x):
|
||||
# selinux fails our FUSE tests, thus ignore selinux xattrs
|
||||
SELINUX_KEY = b'security.selinux'
|
||||
SELINUX_KEY = b"security.selinux"
|
||||
if isinstance(x, dict):
|
||||
return {k: v for k, v in x.items() if k != SELINUX_KEY}
|
||||
if isinstance(x, list):
|
||||
@ -157,8 +158,8 @@ def no_selinux(x):
|
||||
|
||||
|
||||
class BaseTestCase(unittest.TestCase):
|
||||
"""
|
||||
"""
|
||||
""" """
|
||||
|
||||
assert_in = unittest.TestCase.assertIn
|
||||
assert_not_in = unittest.TestCase.assertNotIn
|
||||
assert_equal = unittest.TestCase.assertEqual
|
||||
@ -171,9 +172,9 @@ class BaseTestCase(unittest.TestCase):
|
||||
|
||||
@contextmanager
|
||||
def assert_creates_file(self, path):
|
||||
assert not os.path.exists(path), f'{path} should not exist'
|
||||
assert not os.path.exists(path), f"{path} should not exist"
|
||||
yield
|
||||
assert os.path.exists(path), f'{path} should exist'
|
||||
assert os.path.exists(path), f"{path} should exist"
|
||||
|
||||
def assert_dirs_equal(self, dir1, dir2, **kwargs):
|
||||
diff = filecmp.dircmp(dir1, dir2)
|
||||
@ -191,10 +192,10 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False,
|
||||
s2 = os.stat(path2, follow_symlinks=False)
|
||||
# Assume path2 is on FUSE if st_dev is different
|
||||
fuse = s1.st_dev != s2.st_dev
|
||||
attrs = ['st_uid', 'st_gid', 'st_rdev']
|
||||
attrs = ["st_uid", "st_gid", "st_rdev"]
|
||||
if not fuse or not os.path.isdir(path1):
|
||||
# dir nlink is always 1 on our FUSE filesystem
|
||||
attrs.append('st_nlink')
|
||||
attrs.append("st_nlink")
|
||||
d1 = [filename] + [getattr(s1, a) for a in attrs]
|
||||
d2 = [filename] + [getattr(s2, a) for a in attrs]
|
||||
d1.insert(1, oct(s1.st_mode))
|
||||
@ -225,7 +226,9 @@ def _assert_dirs_equal_cmp(self, diff, ignore_flags=False, ignore_xattrs=False,
|
||||
d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
|
||||
self.assert_equal(d1, d2)
|
||||
for sub_diff in diff.subdirs.values():
|
||||
self._assert_dirs_equal_cmp(sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns)
|
||||
self._assert_dirs_equal_cmp(
|
||||
sub_diff, ignore_flags=ignore_flags, ignore_xattrs=ignore_xattrs, ignore_ns=ignore_ns
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=False, **kwargs):
|
||||
@ -247,7 +250,7 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal
|
||||
mountpoint = tempfile.mkdtemp()
|
||||
else:
|
||||
os.mkdir(mountpoint)
|
||||
args = [f'--repo={location}', 'mount', mountpoint] + list(options)
|
||||
args = [f"--repo={location}", "mount", mountpoint] + list(options)
|
||||
if os_fork:
|
||||
# Do not spawn, but actually (OS) fork.
|
||||
if os.fork() == 0:
|
||||
@ -264,12 +267,11 @@ def fuse_mount(self, location, mountpoint=None, *options, fork=True, os_fork=Fal
|
||||
# This should never be reached, since it daemonizes,
|
||||
# and the grandchild process exits before cmd() returns.
|
||||
# However, just in case...
|
||||
print('Fatal: borg mount did not daemonize properly. Force exiting.',
|
||||
file=sys.stderr, flush=True)
|
||||
print("Fatal: borg mount did not daemonize properly. Force exiting.", file=sys.stderr, flush=True)
|
||||
os._exit(0)
|
||||
else:
|
||||
self.cmd(*args, fork=fork, **kwargs)
|
||||
if kwargs.get('exit_code', EXIT_SUCCESS) == EXIT_ERROR:
|
||||
if kwargs.get("exit_code", EXIT_SUCCESS) == EXIT_ERROR:
|
||||
# If argument `exit_code = EXIT_ERROR`, then this call
|
||||
# is testing the behavior of an unsuccessful mount and
|
||||
# we must not continue, as there is no mount to work
|
||||
@ -292,7 +294,7 @@ def wait_for_mountstate(self, mountpoint, *, mounted, timeout=5):
|
||||
if os.path.ismount(mountpoint) == mounted:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
message = 'Waiting for {} of {}'.format('mount' if mounted else 'umount', mountpoint)
|
||||
message = "Waiting for {} of {}".format("mount" if mounted else "umount", mountpoint)
|
||||
raise TimeoutError(message)
|
||||
|
||||
@contextmanager
|
||||
@ -308,17 +310,17 @@ def read_only(self, path):
|
||||
tests are running with root privileges. Instead, the folder is
|
||||
rendered immutable with chattr or chflags, respectively.
|
||||
"""
|
||||
if sys.platform.startswith('linux'):
|
||||
if sys.platform.startswith("linux"):
|
||||
cmd_immutable = 'chattr +i "%s"' % path
|
||||
cmd_mutable = 'chattr -i "%s"' % path
|
||||
elif sys.platform.startswith(('darwin', 'freebsd', 'netbsd', 'openbsd')):
|
||||
elif sys.platform.startswith(("darwin", "freebsd", "netbsd", "openbsd")):
|
||||
cmd_immutable = 'chflags uchg "%s"' % path
|
||||
cmd_mutable = 'chflags nouchg "%s"' % path
|
||||
elif sys.platform.startswith('sunos'): # openindiana
|
||||
elif sys.platform.startswith("sunos"): # openindiana
|
||||
cmd_immutable = 'chmod S+vimmutable "%s"' % path
|
||||
cmd_mutable = 'chmod S-vimmutable "%s"' % path
|
||||
else:
|
||||
message = 'Testing read-only repos is not supported on platform %s' % sys.platform
|
||||
message = "Testing read-only repos is not supported on platform %s" % sys.platform
|
||||
self.skipTest(message)
|
||||
try:
|
||||
os.system('LD_PRELOAD= chmod -R ugo-w "%s"' % path)
|
||||
@ -365,12 +367,13 @@ def __exit__(self, *args, **kw):
|
||||
|
||||
class FakeInputs:
|
||||
"""Simulate multiple user inputs, can be used as input() replacement"""
|
||||
|
||||
def __init__(self, inputs):
|
||||
self.inputs = inputs
|
||||
|
||||
def __call__(self, prompt=None):
|
||||
if prompt is not None:
|
||||
print(prompt, end='')
|
||||
print(prompt, end="")
|
||||
try:
|
||||
return self.inputs.pop(0)
|
||||
except IndexError:
|
||||
|
@ -33,64 +33,66 @@ def test_stats_basic(stats):
|
||||
|
||||
|
||||
def tests_stats_progress(stats, monkeypatch, columns=80):
|
||||
monkeypatch.setenv('COLUMNS', str(columns))
|
||||
monkeypatch.setenv("COLUMNS", str(columns))
|
||||
out = StringIO()
|
||||
stats.show_progress(stream=out)
|
||||
s = '20 B O 20 B U 1 N '
|
||||
buf = ' ' * (columns - len(s))
|
||||
s = "20 B O 20 B U 1 N "
|
||||
buf = " " * (columns - len(s))
|
||||
assert out.getvalue() == s + buf + "\r"
|
||||
|
||||
out = StringIO()
|
||||
stats.update(10 ** 3, unique=False)
|
||||
stats.show_progress(item=Item(path='foo'), final=False, stream=out)
|
||||
s = '1.02 kB O 20 B U 1 N foo'
|
||||
buf = ' ' * (columns - len(s))
|
||||
stats.update(10**3, unique=False)
|
||||
stats.show_progress(item=Item(path="foo"), final=False, stream=out)
|
||||
s = "1.02 kB O 20 B U 1 N foo"
|
||||
buf = " " * (columns - len(s))
|
||||
assert out.getvalue() == s + buf + "\r"
|
||||
out = StringIO()
|
||||
stats.show_progress(item=Item(path='foo'*40), final=False, stream=out)
|
||||
s = '1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo'
|
||||
buf = ' ' * (columns - len(s))
|
||||
stats.show_progress(item=Item(path="foo" * 40), final=False, stream=out)
|
||||
s = "1.02 kB O 20 B U 1 N foofoofoofoofoofoofoofoofo...foofoofoofoofoofoofoofoofoofoo"
|
||||
buf = " " * (columns - len(s))
|
||||
assert out.getvalue() == s + buf + "\r"
|
||||
|
||||
|
||||
def test_stats_format(stats):
|
||||
assert str(stats) == """\
|
||||
assert (
|
||||
str(stats)
|
||||
== """\
|
||||
Number of files: 1
|
||||
Original size: 20 B
|
||||
Deduplicated size: 20 B
|
||||
"""
|
||||
)
|
||||
s = f"{stats.osize_fmt}"
|
||||
assert s == "20 B"
|
||||
# kind of redundant, but id is variable so we can't match reliably
|
||||
assert repr(stats) == f'<Statistics object at {id(stats):#x} (20, 20)>'
|
||||
assert repr(stats) == f"<Statistics object at {id(stats):#x} (20, 20)>"
|
||||
|
||||
|
||||
def test_stats_progress_json(stats):
|
||||
stats.output_json = True
|
||||
|
||||
out = StringIO()
|
||||
stats.show_progress(item=Item(path='foo'), stream=out)
|
||||
stats.show_progress(item=Item(path="foo"), stream=out)
|
||||
result = json.loads(out.getvalue())
|
||||
assert result['type'] == 'archive_progress'
|
||||
assert isinstance(result['time'], float)
|
||||
assert result['finished'] is False
|
||||
assert result['path'] == 'foo'
|
||||
assert result['original_size'] == 20
|
||||
assert result['nfiles'] == 1
|
||||
assert result["type"] == "archive_progress"
|
||||
assert isinstance(result["time"], float)
|
||||
assert result["finished"] is False
|
||||
assert result["path"] == "foo"
|
||||
assert result["original_size"] == 20
|
||||
assert result["nfiles"] == 1
|
||||
|
||||
out = StringIO()
|
||||
stats.show_progress(stream=out, final=True)
|
||||
result = json.loads(out.getvalue())
|
||||
assert result['type'] == 'archive_progress'
|
||||
assert isinstance(result['time'], float)
|
||||
assert result['finished'] is True # see #6570
|
||||
assert 'path' not in result
|
||||
assert 'original_size' not in result
|
||||
assert 'nfiles' not in result
|
||||
assert result["type"] == "archive_progress"
|
||||
assert isinstance(result["time"], float)
|
||||
assert result["finished"] is True # see #6570
|
||||
assert "path" not in result
|
||||
assert "original_size" not in result
|
||||
assert "nfiles" not in result
|
||||
|
||||
|
||||
class MockCache:
|
||||
|
||||
class MockRepo:
|
||||
def async_response(self, wait=True):
|
||||
pass
|
||||
@ -105,30 +107,24 @@ def add_chunk(self, id, chunk, stats=None, wait=True):
|
||||
|
||||
|
||||
class ArchiveTimestampTestCase(BaseTestCase):
|
||||
|
||||
def _test_timestamp_parsing(self, isoformat, expected):
|
||||
repository = Mock()
|
||||
key = PlaintextKey(repository)
|
||||
manifest = Manifest(repository, key)
|
||||
a = Archive(repository, key, manifest, 'test', create=True)
|
||||
a = Archive(repository, key, manifest, "test", create=True)
|
||||
a.metadata = ArchiveItem(time=isoformat)
|
||||
self.assert_equal(a.ts, expected)
|
||||
|
||||
def test_with_microseconds(self):
|
||||
self._test_timestamp_parsing(
|
||||
'1970-01-01T00:00:01.000001',
|
||||
datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc))
|
||||
self._test_timestamp_parsing("1970-01-01T00:00:01.000001", datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc))
|
||||
|
||||
def test_without_microseconds(self):
|
||||
self._test_timestamp_parsing(
|
||||
'1970-01-01T00:00:01',
|
||||
datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc))
|
||||
self._test_timestamp_parsing("1970-01-01T00:00:01", datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc))
|
||||
|
||||
|
||||
class ChunkBufferTestCase(BaseTestCase):
|
||||
|
||||
def test(self):
|
||||
data = [Item(path='p1'), Item(path='p2')]
|
||||
data = [Item(path="p1"), Item(path="p2")]
|
||||
cache = MockCache()
|
||||
key = PlaintextKey(None)
|
||||
chunks = CacheChunkBuffer(cache, key, None)
|
||||
@ -144,7 +140,7 @@ def test(self):
|
||||
|
||||
def test_partial(self):
|
||||
big = "0123456789abcdefghijklmnopqrstuvwxyz" * 25000
|
||||
data = [Item(path='full', source=big), Item(path='partial', source=big)]
|
||||
data = [Item(path="full", source=big), Item(path="partial", source=big)]
|
||||
cache = MockCache()
|
||||
key = PlaintextKey(None)
|
||||
chunks = CacheChunkBuffer(cache, key, None)
|
||||
@ -165,12 +161,11 @@ def test_partial(self):
|
||||
|
||||
|
||||
class RobustUnpackerTestCase(BaseTestCase):
|
||||
|
||||
def make_chunks(self, items):
|
||||
return b''.join(msgpack.packb({'path': item}) for item in items)
|
||||
return b"".join(msgpack.packb({"path": item}) for item in items)
|
||||
|
||||
def _validator(self, value):
|
||||
return isinstance(value, dict) and value.get('path') in ('foo', 'bar', 'boo', 'baz')
|
||||
return isinstance(value, dict) and value.get("path") in ("foo", "bar", "boo", "baz")
|
||||
|
||||
def process(self, input):
|
||||
unpacker = RobustUnpacker(validator=self._validator, item_keys=ITEM_KEYS)
|
||||
@ -185,14 +180,14 @@ def process(self, input):
|
||||
return result
|
||||
|
||||
def test_extra_garbage_no_sync(self):
|
||||
chunks = [(False, [self.make_chunks(['foo', 'bar'])]),
|
||||
(False, [b'garbage'] + [self.make_chunks(['boo', 'baz'])])]
|
||||
chunks = [
|
||||
(False, [self.make_chunks(["foo", "bar"])]),
|
||||
(False, [b"garbage"] + [self.make_chunks(["boo", "baz"])]),
|
||||
]
|
||||
result = self.process(chunks)
|
||||
self.assert_equal(result, [
|
||||
{'path': 'foo'}, {'path': 'bar'},
|
||||
103, 97, 114, 98, 97, 103, 101,
|
||||
{'path': 'boo'},
|
||||
{'path': 'baz'}])
|
||||
self.assert_equal(
|
||||
result, [{"path": "foo"}, {"path": "bar"}, 103, 97, 114, 98, 97, 103, 101, {"path": "boo"}, {"path": "baz"}]
|
||||
)
|
||||
|
||||
def split(self, left, length):
|
||||
parts = []
|
||||
@ -202,22 +197,22 @@ def split(self, left, length):
|
||||
return parts
|
||||
|
||||
def test_correct_stream(self):
|
||||
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 2)
|
||||
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 2)
|
||||
input = [(False, chunks)]
|
||||
result = self.process(input)
|
||||
self.assert_equal(result, [{'path': 'foo'}, {'path': 'bar'}, {'path': 'boo'}, {'path': 'baz'}])
|
||||
self.assert_equal(result, [{"path": "foo"}, {"path": "bar"}, {"path": "boo"}, {"path": "baz"}])
|
||||
|
||||
def test_missing_chunk(self):
|
||||
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4)
|
||||
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4)
|
||||
input = [(False, chunks[:3]), (True, chunks[4:])]
|
||||
result = self.process(input)
|
||||
self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}])
|
||||
self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}])
|
||||
|
||||
def test_corrupt_chunk(self):
|
||||
chunks = self.split(self.make_chunks(['foo', 'bar', 'boo', 'baz']), 4)
|
||||
input = [(False, chunks[:3]), (True, [b'gar', b'bage'] + chunks[3:])]
|
||||
chunks = self.split(self.make_chunks(["foo", "bar", "boo", "baz"]), 4)
|
||||
input = [(False, chunks[:3]), (True, [b"gar", b"bage"] + chunks[3:])]
|
||||
result = self.process(input)
|
||||
self.assert_equal(result, [{'path': 'foo'}, {'path': 'boo'}, {'path': 'baz'}])
|
||||
self.assert_equal(result, [{"path": "foo"}, {"path": "boo"}, {"path": "baz"}])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@ -225,12 +220,17 @@ def item_keys_serialized():
|
||||
return [msgpack.packb(name) for name in ITEM_KEYS]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('packed',
|
||||
[b'', b'x', b'foobar', ] +
|
||||
[msgpack.packb(o) for o in (
|
||||
[None, 0, 0.0, False, '', {}, [], ()] +
|
||||
[42, 23.42, True, b'foobar', {b'foo': b'bar'}, [b'foo', b'bar'], (b'foo', b'bar')]
|
||||
)])
|
||||
@pytest.mark.parametrize(
|
||||
"packed",
|
||||
[b"", b"x", b"foobar"]
|
||||
+ [
|
||||
msgpack.packb(o)
|
||||
for o in (
|
||||
[None, 0, 0.0, False, "", {}, [], ()]
|
||||
+ [42, 23.42, True, b"foobar", {b"foo": b"bar"}, [b"foo", b"bar"], (b"foo", b"bar")]
|
||||
)
|
||||
],
|
||||
)
|
||||
def test_invalid_msgpacked_item(packed, item_keys_serialized):
|
||||
assert not valid_msgpacked_dict(packed, item_keys_serialized)
|
||||
|
||||
@ -239,20 +239,25 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized):
|
||||
IK = sorted(list(ITEM_KEYS))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('packed',
|
||||
[msgpack.packb(o) for o in [
|
||||
{'path': b'/a/b/c'}, # small (different msgpack mapping type!)
|
||||
OrderedDict((k, b'') for k in IK), # as big (key count) as it gets
|
||||
OrderedDict((k, b'x' * 1000) for k in IK), # as big (key count and volume) as it gets
|
||||
]])
|
||||
@pytest.mark.parametrize(
|
||||
"packed",
|
||||
[
|
||||
msgpack.packb(o)
|
||||
for o in [
|
||||
{"path": b"/a/b/c"}, # small (different msgpack mapping type!)
|
||||
OrderedDict((k, b"") for k in IK), # as big (key count) as it gets
|
||||
OrderedDict((k, b"x" * 1000) for k in IK), # as big (key count and volume) as it gets
|
||||
]
|
||||
],
|
||||
)
|
||||
def test_valid_msgpacked_items(packed, item_keys_serialized):
|
||||
assert valid_msgpacked_dict(packed, item_keys_serialized)
|
||||
|
||||
|
||||
def test_key_length_msgpacked_items():
|
||||
key = 'x' * 32 # 31 bytes is the limit for fixstr msgpack type
|
||||
data = {key: b''}
|
||||
item_keys_serialized = [msgpack.packb(key), ]
|
||||
key = "x" * 32 # 31 bytes is the limit for fixstr msgpack type
|
||||
data = {key: b""}
|
||||
item_keys_serialized = [msgpack.packb(key)]
|
||||
assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
|
||||
|
||||
|
||||
@ -277,7 +282,7 @@ def __next__(self):
|
||||
|
||||
normal_iterator = Iterator(StopIteration)
|
||||
for _ in backup_io_iter(normal_iterator):
|
||||
assert False, 'StopIteration handled incorrectly'
|
||||
assert False, "StopIteration handled incorrectly"
|
||||
|
||||
|
||||
def test_get_item_uid_gid():
|
||||
@ -288,7 +293,7 @@ def test_get_item_uid_gid():
|
||||
user0, group0 = uid2user(0), gid2group(0)
|
||||
|
||||
# this is intentionally a "strange" item, with not matching ids/names.
|
||||
item = Item(path='filename', uid=1, gid=2, user=user0, group=group0)
|
||||
item = Item(path="filename", uid=1, gid=2, user=user0, group=group0)
|
||||
|
||||
uid, gid = get_item_uid_gid(item, numeric=False)
|
||||
# these are found via a name-to-id lookup
|
||||
@ -306,7 +311,7 @@ def test_get_item_uid_gid():
|
||||
assert gid == 4
|
||||
|
||||
# item metadata broken, has negative ids.
|
||||
item = Item(path='filename', uid=-1, gid=-2, user=user0, group=group0)
|
||||
item = Item(path="filename", uid=-1, gid=-2, user=user0, group=group0)
|
||||
|
||||
uid, gid = get_item_uid_gid(item, numeric=True)
|
||||
# use the uid/gid defaults (which both default to 0).
|
||||
@ -319,7 +324,7 @@ def test_get_item_uid_gid():
|
||||
assert gid == 6
|
||||
|
||||
# item metadata broken, has negative ids and non-existing user/group names.
|
||||
item = Item(path='filename', uid=-3, gid=-4, user='udoesnotexist', group='gdoesnotexist')
|
||||
item = Item(path="filename", uid=-3, gid=-4, user="udoesnotexist", group="gdoesnotexist")
|
||||
|
||||
uid, gid = get_item_uid_gid(item, numeric=False)
|
||||
# use the uid/gid defaults (which both default to 0).
|
||||
@ -332,7 +337,7 @@ def test_get_item_uid_gid():
|
||||
assert gid == 8
|
||||
|
||||
# item metadata has valid uid/gid, but non-existing user/group names.
|
||||
item = Item(path='filename', uid=9, gid=10, user='udoesnotexist', group='gdoesnotexist')
|
||||
item = Item(path="filename", uid=9, gid=10, user="udoesnotexist", group="gdoesnotexist")
|
||||
|
||||
uid, gid = get_item_uid_gid(item, numeric=False)
|
||||
# because user/group name does not exist here, use valid numeric ids from item metadata.
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -16,35 +16,38 @@
|
||||
|
||||
@pytest.fixture
|
||||
def repo_url(request, tmpdir, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', '123456')
|
||||
monkeypatch.setenv('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', 'YES')
|
||||
monkeypatch.setenv('BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', 'YES')
|
||||
monkeypatch.setenv('BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', 'yes')
|
||||
monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir.join('keys')))
|
||||
monkeypatch.setenv('BORG_CACHE_DIR', str(tmpdir.join('cache')))
|
||||
yield str(tmpdir.join('repository'))
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "123456")
|
||||
monkeypatch.setenv("BORG_CHECK_I_KNOW_WHAT_I_AM_DOING", "YES")
|
||||
monkeypatch.setenv("BORG_DELETE_I_KNOW_WHAT_I_AM_DOING", "YES")
|
||||
monkeypatch.setenv("BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK", "yes")
|
||||
monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir.join("keys")))
|
||||
monkeypatch.setenv("BORG_CACHE_DIR", str(tmpdir.join("cache")))
|
||||
yield str(tmpdir.join("repository"))
|
||||
tmpdir.remove(rec=1)
|
||||
|
||||
|
||||
@pytest.fixture(params=["none", "repokey-aes-ocb"])
|
||||
def repo(request, cmd, repo_url):
|
||||
cmd(f'--repo={repo_url}', 'rcreate', '--encryption', request.param)
|
||||
cmd(f"--repo={repo_url}", "rcreate", "--encryption", request.param)
|
||||
return repo_url
|
||||
|
||||
|
||||
@pytest.fixture(scope='session', params=["zeros", "random"])
|
||||
@pytest.fixture(scope="session", params=["zeros", "random"])
|
||||
def testdata(request, tmpdir_factory):
|
||||
count, size = 10, 1000*1000
|
||||
count, size = 10, 1000 * 1000
|
||||
assert size <= len(zeros)
|
||||
p = tmpdir_factory.mktemp('data')
|
||||
p = tmpdir_factory.mktemp("data")
|
||||
data_type = request.param
|
||||
if data_type == 'zeros':
|
||||
if data_type == "zeros":
|
||||
# do not use a binary zero (\0) to avoid sparse detection
|
||||
def data(size):
|
||||
return memoryview(zeros)[:size]
|
||||
elif data_type == 'random':
|
||||
|
||||
elif data_type == "random":
|
||||
|
||||
def data(size):
|
||||
return os.urandom(size)
|
||||
|
||||
else:
|
||||
raise ValueError("data_type must be 'random' or 'zeros'.")
|
||||
for i in range(count):
|
||||
@ -54,56 +57,54 @@ def data(size):
|
||||
p.remove(rec=1)
|
||||
|
||||
|
||||
@pytest.fixture(params=['none', 'lz4'])
|
||||
@pytest.fixture(params=["none", "lz4"])
|
||||
def repo_archive(request, cmd, repo, testdata):
|
||||
archive = 'test'
|
||||
cmd(f'--repo={repo}', 'create', '--compression', request.param, archive, testdata)
|
||||
archive = "test"
|
||||
cmd(f"--repo={repo}", "create", "--compression", request.param, archive, testdata)
|
||||
return repo, archive
|
||||
|
||||
|
||||
def test_create_none(benchmark, cmd, repo, testdata):
|
||||
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'none',
|
||||
'test', testdata))
|
||||
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "none", "test", testdata))
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_create_lz4(benchmark, cmd, repo, testdata):
|
||||
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'create', '--compression', 'lz4',
|
||||
'test', testdata))
|
||||
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "create", "--compression", "lz4", "test", testdata))
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_extract(benchmark, cmd, repo_archive, tmpdir):
|
||||
repo, archive = repo_archive
|
||||
with changedir(str(tmpdir)):
|
||||
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'extract', archive))
|
||||
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "extract", archive))
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_delete(benchmark, cmd, repo_archive):
|
||||
repo, archive = repo_archive
|
||||
result, out = benchmark.pedantic(cmd, (f'--repo={repo}', 'delete', '-a', archive))
|
||||
result, out = benchmark.pedantic(cmd, (f"--repo={repo}", "delete", "-a", archive))
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_list(benchmark, cmd, repo_archive):
|
||||
repo, archive = repo_archive
|
||||
result, out = benchmark(cmd, f'--repo={repo}', 'list', archive)
|
||||
result, out = benchmark(cmd, f"--repo={repo}", "list", archive)
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_info(benchmark, cmd, repo_archive):
|
||||
repo, archive = repo_archive
|
||||
result, out = benchmark(cmd, f'--repo={repo}', 'info', '-a', archive)
|
||||
result, out = benchmark(cmd, f"--repo={repo}", "info", "-a", archive)
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_check(benchmark, cmd, repo_archive):
|
||||
repo, archive = repo_archive
|
||||
result, out = benchmark(cmd, f'--repo={repo}', 'check')
|
||||
result, out = benchmark(cmd, f"--repo={repo}", "check")
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_help(benchmark, cmd):
|
||||
result, out = benchmark(cmd, 'help')
|
||||
result, out = benchmark(cmd, "help")
|
||||
assert result == 0
|
||||
|
@ -26,75 +26,29 @@ def sync(self, index):
|
||||
return CacheSynchronizer(index)
|
||||
|
||||
def test_no_chunks(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': []
|
||||
})
|
||||
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": []})
|
||||
sync.feed(data)
|
||||
assert not len(index)
|
||||
|
||||
def test_simple(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': [
|
||||
(H(1), 1),
|
||||
(H(2), 2),
|
||||
]
|
||||
})
|
||||
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
|
||||
sync.feed(data)
|
||||
assert len(index) == 2
|
||||
assert index[H(1)] == (1, 1)
|
||||
assert index[H(2)] == (1, 2)
|
||||
|
||||
def test_multiple(self, index, sync):
|
||||
data = packb({
|
||||
'foo': 'bar',
|
||||
'baz': 1234,
|
||||
'bar': 5678,
|
||||
'user': 'chunks',
|
||||
'chunks': [
|
||||
(H(1), 1),
|
||||
(H(2), 2),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'xattrs': {
|
||||
'security.foo': 'bar',
|
||||
'chunks': '123456',
|
||||
},
|
||||
'stuff': [
|
||||
(1, 2, 3),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'xattrs': {
|
||||
'security.foo': 'bar',
|
||||
'chunks': '123456',
|
||||
},
|
||||
'chunks': [
|
||||
(H(1), 1),
|
||||
(H(2), 2),
|
||||
],
|
||||
'stuff': [
|
||||
(1, 2, 3),
|
||||
]
|
||||
})
|
||||
data += packb({
|
||||
'chunks': [
|
||||
(H(3), 1),
|
||||
],
|
||||
})
|
||||
data += packb({
|
||||
'chunks': [
|
||||
(H(1), 1),
|
||||
],
|
||||
})
|
||||
data = packb({"foo": "bar", "baz": 1234, "bar": 5678, "user": "chunks", "chunks": [(H(1), 1), (H(2), 2)]})
|
||||
data += packb({"xattrs": {"security.foo": "bar", "chunks": "123456"}, "stuff": [(1, 2, 3)]})
|
||||
data += packb(
|
||||
{
|
||||
"xattrs": {"security.foo": "bar", "chunks": "123456"},
|
||||
"chunks": [(H(1), 1), (H(2), 2)],
|
||||
"stuff": [(1, 2, 3)],
|
||||
}
|
||||
)
|
||||
data += packb({"chunks": [(H(3), 1)]})
|
||||
data += packb({"chunks": [(H(1), 1)]})
|
||||
|
||||
part1 = data[:70]
|
||||
part2 = data[70:120]
|
||||
@ -107,62 +61,68 @@ def test_multiple(self, index, sync):
|
||||
assert index[H(2)] == (2, 2)
|
||||
assert index[H(3)] == (1, 1)
|
||||
|
||||
@pytest.mark.parametrize('elem,error', (
|
||||
({1: 2}, 'Unexpected object: map'),
|
||||
(bytes(213), [
|
||||
'Unexpected bytes in chunks structure', # structure 2/3
|
||||
'Incorrect key length']), # structure 3/3
|
||||
(1, 'Unexpected object: integer'),
|
||||
(1.0, 'Unexpected object: double'),
|
||||
(True, 'Unexpected object: true'),
|
||||
(False, 'Unexpected object: false'),
|
||||
(None, 'Unexpected object: nil'),
|
||||
))
|
||||
@pytest.mark.parametrize('structure', (
|
||||
lambda elem: {'chunks': elem},
|
||||
lambda elem: {'chunks': [elem]},
|
||||
lambda elem: {'chunks': [(elem, 1)]},
|
||||
))
|
||||
@pytest.mark.parametrize(
|
||||
"elem,error",
|
||||
(
|
||||
({1: 2}, "Unexpected object: map"),
|
||||
(
|
||||
bytes(213),
|
||||
["Unexpected bytes in chunks structure", "Incorrect key length"], # structure 2/3
|
||||
), # structure 3/3
|
||||
(1, "Unexpected object: integer"),
|
||||
(1.0, "Unexpected object: double"),
|
||||
(True, "Unexpected object: true"),
|
||||
(False, "Unexpected object: false"),
|
||||
(None, "Unexpected object: nil"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"structure",
|
||||
(lambda elem: {"chunks": elem}, lambda elem: {"chunks": [elem]}, lambda elem: {"chunks": [(elem, 1)]}),
|
||||
)
|
||||
def test_corrupted(self, sync, structure, elem, error):
|
||||
packed = packb(structure(elem))
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
sync.feed(packed)
|
||||
if isinstance(error, str):
|
||||
error = [error]
|
||||
possible_errors = ['cache_sync_feed failed: ' + error for error in error]
|
||||
possible_errors = ["cache_sync_feed failed: " + error for error in error]
|
||||
assert str(excinfo.value) in possible_errors
|
||||
|
||||
@pytest.mark.parametrize('data,error', (
|
||||
# Incorrect tuple length
|
||||
({'chunks': [(bytes(32), 2, 3, 4)]}, 'Invalid chunk list entry length'),
|
||||
({'chunks': [(bytes(32), )]}, 'Invalid chunk list entry length'),
|
||||
# Incorrect types
|
||||
({'chunks': [(1, 2)]}, 'Unexpected object: integer'),
|
||||
({'chunks': [(1, bytes(32))]}, 'Unexpected object: integer'),
|
||||
({'chunks': [(bytes(32), 1.0)]}, 'Unexpected object: double'),
|
||||
))
|
||||
@pytest.mark.parametrize(
|
||||
"data,error",
|
||||
(
|
||||
# Incorrect tuple length
|
||||
({"chunks": [(bytes(32), 2, 3, 4)]}, "Invalid chunk list entry length"),
|
||||
({"chunks": [(bytes(32),)]}, "Invalid chunk list entry length"),
|
||||
# Incorrect types
|
||||
({"chunks": [(1, 2)]}, "Unexpected object: integer"),
|
||||
({"chunks": [(1, bytes(32))]}, "Unexpected object: integer"),
|
||||
({"chunks": [(bytes(32), 1.0)]}, "Unexpected object: double"),
|
||||
),
|
||||
)
|
||||
def test_corrupted_ancillary(self, index, sync, data, error):
|
||||
packed = packb(data)
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
sync.feed(packed)
|
||||
assert str(excinfo.value) == 'cache_sync_feed failed: ' + error
|
||||
assert str(excinfo.value) == "cache_sync_feed failed: " + error
|
||||
|
||||
def make_index_with_refcount(self, refcount):
|
||||
index_data = io.BytesIO()
|
||||
index_data.write(b'BORG_IDX')
|
||||
index_data.write(b"BORG_IDX")
|
||||
# num_entries
|
||||
index_data.write((1).to_bytes(4, 'little'))
|
||||
index_data.write((1).to_bytes(4, "little"))
|
||||
# num_buckets
|
||||
index_data.write((1).to_bytes(4, 'little'))
|
||||
index_data.write((1).to_bytes(4, "little"))
|
||||
# key_size
|
||||
index_data.write((32).to_bytes(1, 'little'))
|
||||
index_data.write((32).to_bytes(1, "little"))
|
||||
# value_size
|
||||
index_data.write((3 * 4).to_bytes(1, 'little'))
|
||||
index_data.write((3 * 4).to_bytes(1, "little"))
|
||||
|
||||
index_data.write(H(0))
|
||||
index_data.write(refcount.to_bytes(4, 'little'))
|
||||
index_data.write((1234).to_bytes(4, 'little'))
|
||||
index_data.write((5678).to_bytes(4, 'little'))
|
||||
index_data.write(refcount.to_bytes(4, "little"))
|
||||
index_data.write((1234).to_bytes(4, "little"))
|
||||
index_data.write((5678).to_bytes(4, "little"))
|
||||
|
||||
index_data.seek(0)
|
||||
index = ChunkIndex.read(index_data)
|
||||
@ -171,34 +131,22 @@ def make_index_with_refcount(self, refcount):
|
||||
def test_corrupted_refcount(self):
|
||||
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE + 1)
|
||||
sync = CacheSynchronizer(index)
|
||||
data = packb({
|
||||
'chunks': [
|
||||
(H(0), 1),
|
||||
]
|
||||
})
|
||||
data = packb({"chunks": [(H(0), 1)]})
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
sync.feed(data)
|
||||
assert str(excinfo.value) == 'cache_sync_feed failed: invalid reference count'
|
||||
assert str(excinfo.value) == "cache_sync_feed failed: invalid reference count"
|
||||
|
||||
def test_refcount_max_value(self):
|
||||
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE)
|
||||
sync = CacheSynchronizer(index)
|
||||
data = packb({
|
||||
'chunks': [
|
||||
(H(0), 1),
|
||||
]
|
||||
})
|
||||
data = packb({"chunks": [(H(0), 1)]})
|
||||
sync.feed(data)
|
||||
assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
|
||||
|
||||
def test_refcount_one_below_max_value(self):
|
||||
index = self.make_index_with_refcount(ChunkIndex.MAX_VALUE - 1)
|
||||
sync = CacheSynchronizer(index)
|
||||
data = packb({
|
||||
'chunks': [
|
||||
(H(0), 1),
|
||||
]
|
||||
})
|
||||
data = packb({"chunks": [(H(0), 1)]})
|
||||
sync.feed(data)
|
||||
# Incremented to maximum
|
||||
assert index[H(0)] == (ChunkIndex.MAX_VALUE, 1234)
|
||||
@ -209,17 +157,17 @@ def test_refcount_one_below_max_value(self):
|
||||
class TestAdHocCache:
|
||||
@pytest.fixture
|
||||
def repository(self, tmpdir):
|
||||
self.repository_location = os.path.join(str(tmpdir), 'repository')
|
||||
self.repository_location = os.path.join(str(tmpdir), "repository")
|
||||
with Repository(self.repository_location, exclusive=True, create=True) as repository:
|
||||
repository.put(H(1), b'1234')
|
||||
repository.put(Manifest.MANIFEST_ID, b'5678')
|
||||
repository.put(H(1), b"1234")
|
||||
repository.put(Manifest.MANIFEST_ID, b"5678")
|
||||
yield repository
|
||||
|
||||
@pytest.fixture
|
||||
def key(self, repository, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = AESOCBRepoKey.create(repository, TestKey.MockArgs())
|
||||
key.compressor = CompressionSpec('none').compressor
|
||||
key.compressor = CompressionSpec("none").compressor
|
||||
return key
|
||||
|
||||
@pytest.fixture
|
||||
@ -237,18 +185,18 @@ def test_does_not_contain_manifest(self, cache):
|
||||
def test_does_not_delete_existing_chunks(self, repository, cache):
|
||||
assert cache.seen_chunk(H(1)) == ChunkIndex.MAX_VALUE
|
||||
cache.chunk_decref(H(1), Statistics())
|
||||
assert repository.get(H(1)) == b'1234'
|
||||
assert repository.get(H(1)) == b"1234"
|
||||
|
||||
def test_does_not_overwrite(self, cache):
|
||||
with pytest.raises(AssertionError):
|
||||
cache.add_chunk(H(1), b'5678', Statistics(), overwrite=True)
|
||||
cache.add_chunk(H(1), b"5678", Statistics(), overwrite=True)
|
||||
|
||||
def test_seen_chunk_add_chunk_size(self, cache):
|
||||
assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4)
|
||||
assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4)
|
||||
|
||||
def test_deletes_chunks_during_lifetime(self, cache, repository):
|
||||
"""E.g. checkpoint archives"""
|
||||
cache.add_chunk(H(5), b'1010', Statistics())
|
||||
cache.add_chunk(H(5), b"1010", Statistics())
|
||||
assert cache.seen_chunk(H(5)) == 1
|
||||
cache.chunk_decref(H(5), Statistics())
|
||||
assert not cache.seen_chunk(H(5))
|
||||
@ -256,8 +204,8 @@ def test_deletes_chunks_during_lifetime(self, cache, repository):
|
||||
repository.get(H(5))
|
||||
|
||||
def test_files_cache(self, cache):
|
||||
assert cache.file_known_and_unchanged(b'foo', bytes(32), None) == (False, None)
|
||||
assert cache.cache_mode == 'd'
|
||||
assert cache.file_known_and_unchanged(b"foo", bytes(32), None) == (False, None)
|
||||
assert cache.cache_mode == "d"
|
||||
assert cache.files is None
|
||||
|
||||
def test_txn(self, cache):
|
||||
@ -267,13 +215,13 @@ def test_txn(self, cache):
|
||||
assert cache.chunks
|
||||
cache.rollback()
|
||||
assert not cache._txn_active
|
||||
assert not hasattr(cache, 'chunks')
|
||||
assert not hasattr(cache, "chunks")
|
||||
|
||||
def test_incref_after_add_chunk(self, cache):
|
||||
assert cache.add_chunk(H(3), b'5678', Statistics()) == (H(3), 4)
|
||||
assert cache.add_chunk(H(3), b"5678", Statistics()) == (H(3), 4)
|
||||
assert cache.chunk_incref(H(3), Statistics()) == (H(3), 4)
|
||||
|
||||
def test_existing_incref_after_add_chunk(self, cache):
|
||||
"""This case occurs with part files, see Archive.chunk_file."""
|
||||
assert cache.add_chunk(H(1), b'5678', Statistics()) == (H(1), 4)
|
||||
assert cache.add_chunk(H(1), b"5678", Statistics()) == (H(1), 4)
|
||||
assert cache.chunk_incref(H(1), Statistics()) == (H(1), 4)
|
||||
|
@ -5,16 +5,24 @@
|
||||
|
||||
|
||||
def test_xxh64():
|
||||
assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'
|
||||
assert bin_to_hex(checksums.xxh64(b'test')) == '4fdcca5ddb678139'
|
||||
assert bin_to_hex(checksums.xxh64(unhexlify(
|
||||
'6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724'
|
||||
'fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d'
|
||||
'cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331'))) == '35d5d2f545d9511a'
|
||||
assert bin_to_hex(checksums.xxh64(b"test", 123)) == "2b81b9401bef86cf"
|
||||
assert bin_to_hex(checksums.xxh64(b"test")) == "4fdcca5ddb678139"
|
||||
assert (
|
||||
bin_to_hex(
|
||||
checksums.xxh64(
|
||||
unhexlify(
|
||||
"6f663f01c118abdea553373d5eae44e7dac3b6829b46b9bbeff202b6c592c22d724"
|
||||
"fb3d25a347cca6c5b8f20d567e4bb04b9cfa85d17f691590f9a9d32e8ccc9102e9d"
|
||||
"cf8a7e6716280cd642ce48d03fdf114c9f57c20d9472bb0f81c147645e6fa3d331"
|
||||
)
|
||||
)
|
||||
)
|
||||
== "35d5d2f545d9511a"
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_xxh64():
|
||||
hasher = checksums.StreamingXXH64(123)
|
||||
hasher.update(b'te')
|
||||
hasher.update(b'st')
|
||||
assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == '2b81b9401bef86cf'
|
||||
hasher.update(b"te")
|
||||
hasher.update(b"st")
|
||||
assert bin_to_hex(hasher.digest()) == hasher.hexdigest() == "2b81b9401bef86cf"
|
||||
|
@ -12,113 +12,129 @@ def cf(chunks):
|
||||
"""chunk filter"""
|
||||
# this is to simplify testing: either return the data piece (bytes) or the hole length (int).
|
||||
def _cf(chunk):
|
||||
if chunk.meta['allocation'] == CH_DATA:
|
||||
assert len(chunk.data) == chunk.meta['size']
|
||||
if chunk.meta["allocation"] == CH_DATA:
|
||||
assert len(chunk.data) == chunk.meta["size"]
|
||||
return bytes(chunk.data) # make sure we have bytes, not memoryview
|
||||
if chunk.meta['allocation'] in (CH_HOLE, CH_ALLOC):
|
||||
if chunk.meta["allocation"] in (CH_HOLE, CH_ALLOC):
|
||||
assert chunk.data is None
|
||||
return chunk.meta['size']
|
||||
return chunk.meta["size"]
|
||||
assert False, "unexpected allocation value"
|
||||
|
||||
return [_cf(chunk) for chunk in chunks]
|
||||
|
||||
|
||||
class ChunkerFixedTestCase(BaseTestCase):
|
||||
|
||||
def test_chunkify_just_blocks(self):
|
||||
data = b'foobar' * 1500
|
||||
data = b"foobar" * 1500
|
||||
chunker = ChunkerFixed(4096)
|
||||
parts = cf(chunker.chunkify(BytesIO(data)))
|
||||
self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]])
|
||||
|
||||
def test_chunkify_header_and_blocks(self):
|
||||
data = b'foobar' * 1500
|
||||
data = b"foobar" * 1500
|
||||
chunker = ChunkerFixed(4096, 123)
|
||||
parts = cf(chunker.chunkify(BytesIO(data)))
|
||||
self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]])
|
||||
self.assert_equal(
|
||||
parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]]
|
||||
)
|
||||
|
||||
def test_chunkify_just_blocks_fmap_complete(self):
|
||||
data = b'foobar' * 1500
|
||||
data = b"foobar" * 1500
|
||||
chunker = ChunkerFixed(4096)
|
||||
fmap = [
|
||||
(0, 4096, True),
|
||||
(4096, 8192, True),
|
||||
(8192, 99999999, True),
|
||||
]
|
||||
fmap = [(0, 4096, True), (4096, 8192, True), (8192, 99999999, True)]
|
||||
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
|
||||
self.assert_equal(parts, [data[0:4096], data[4096:8192], data[8192:]])
|
||||
|
||||
def test_chunkify_header_and_blocks_fmap_complete(self):
|
||||
data = b'foobar' * 1500
|
||||
data = b"foobar" * 1500
|
||||
chunker = ChunkerFixed(4096, 123)
|
||||
fmap = [
|
||||
(0, 123, True),
|
||||
(123, 4096, True),
|
||||
(123+4096, 4096, True),
|
||||
(123+8192, 4096, True),
|
||||
]
|
||||
fmap = [(0, 123, True), (123, 4096, True), (123 + 4096, 4096, True), (123 + 8192, 4096, True)]
|
||||
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
|
||||
self.assert_equal(parts, [data[0:123], data[123:123+4096], data[123+4096:123+8192], data[123+8192:]])
|
||||
self.assert_equal(
|
||||
parts, [data[0:123], data[123 : 123 + 4096], data[123 + 4096 : 123 + 8192], data[123 + 8192 :]]
|
||||
)
|
||||
|
||||
def test_chunkify_header_and_blocks_fmap_zeros(self):
|
||||
data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096
|
||||
data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096
|
||||
chunker = ChunkerFixed(4096, 123)
|
||||
fmap = [
|
||||
(0, 123, True),
|
||||
(123, 4096, False),
|
||||
(123+4096, 4096, True),
|
||||
(123+8192, 4096, False),
|
||||
]
|
||||
fmap = [(0, 123, True), (123, 4096, False), (123 + 4096, 4096, True), (123 + 8192, 4096, False)]
|
||||
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
|
||||
# because we marked the '_' ranges as holes, we will get hole ranges instead!
|
||||
self.assert_equal(parts, [data[0:123], 4096, data[123+4096:123+8192], 4096])
|
||||
self.assert_equal(parts, [data[0:123], 4096, data[123 + 4096 : 123 + 8192], 4096])
|
||||
|
||||
def test_chunkify_header_and_blocks_fmap_partial(self):
|
||||
data = b'H' * 123 + b'_' * 4096 + b'X' * 4096 + b'_' * 4096
|
||||
data = b"H" * 123 + b"_" * 4096 + b"X" * 4096 + b"_" * 4096
|
||||
chunker = ChunkerFixed(4096, 123)
|
||||
fmap = [
|
||||
(0, 123, True),
|
||||
# (123, 4096, False),
|
||||
(123+4096, 4096, True),
|
||||
(123 + 4096, 4096, True),
|
||||
# (123+8192, 4096, False),
|
||||
]
|
||||
parts = cf(chunker.chunkify(BytesIO(data), fmap=fmap))
|
||||
# because we left out the '_' ranges from the fmap, we will not get them at all!
|
||||
self.assert_equal(parts, [data[0:123], data[123+4096:123+8192]])
|
||||
self.assert_equal(parts, [data[0:123], data[123 + 4096 : 123 + 8192]])
|
||||
|
||||
|
||||
class ChunkerTestCase(BaseTestCase):
|
||||
|
||||
def test_chunkify(self):
|
||||
data = b'0' * int(1.5 * (1 << CHUNK_MAX_EXP)) + b'Y'
|
||||
data = b"0" * int(1.5 * (1 << CHUNK_MAX_EXP)) + b"Y"
|
||||
parts = cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(data)))
|
||||
self.assert_equal(len(parts), 2)
|
||||
self.assert_equal(b''.join(parts), data)
|
||||
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b''))), [])
|
||||
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz'])
|
||||
self.assert_equal(cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz'])
|
||||
self.assert_equal(cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz'])
|
||||
self.assert_equal(cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3])
|
||||
self.assert_equal(cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz'])
|
||||
self.assert_equal(cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz'])
|
||||
self.assert_equal(cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz' * 3])
|
||||
self.assert_equal(cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarbo', b'obazfoobar', b'boobazfo', b'obarboobaz'])
|
||||
self.assert_equal(cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))), [b'foobarboobaz', b'foobarboobaz', b'foobarboobaz'])
|
||||
self.assert_equal(b"".join(parts), data)
|
||||
self.assert_equal(cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b""))), [])
|
||||
self.assert_equal(
|
||||
cf(Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"fooba", b"rboobaz", b"fooba", b"rboobaz", b"fooba", b"rboobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"fo", b"obarb", b"oob", b"azf", b"oobarb", b"oob", b"azf", b"oobarb", b"oobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"foob", b"ar", b"boobazfoob", b"ar", b"boobazfoob", b"ar", b"boobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3]
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"foobar", b"boobazfo", b"obar", b"boobazfo", b"obar", b"boobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"foob", b"arboobaz", b"foob", b"arboobaz", b"foob", b"arboobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))), [b"foobarboobaz" * 3]
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"foobarbo", b"obazfoobar", b"boobazfo", b"obarboobaz"],
|
||||
)
|
||||
self.assert_equal(
|
||||
cf(Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b"foobarboobaz" * 3))),
|
||||
[b"foobarboobaz", b"foobarboobaz", b"foobarboobaz"],
|
||||
)
|
||||
|
||||
def test_buzhash(self):
|
||||
self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769)
|
||||
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502)
|
||||
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1))
|
||||
self.assert_equal(buzhash(b"abcdefghijklmnop", 0), 3795437769)
|
||||
self.assert_equal(buzhash(b"abcdefghijklmnop", 1), 3795400502)
|
||||
self.assert_equal(
|
||||
buzhash(b"abcdefghijklmnop", 1), buzhash_update(buzhash(b"Xabcdefghijklmno", 1), ord("X"), ord("p"), 16, 1)
|
||||
)
|
||||
# Test with more than 31 bytes to make sure our barrel_shift macro works correctly
|
||||
self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248)
|
||||
self.assert_equal(buzhash(b"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz", 0), 566521248)
|
||||
|
||||
def test_small_reads(self):
|
||||
class SmallReadFile:
|
||||
input = b'a' * (20 + 1)
|
||||
input = b"a" * (20 + 1)
|
||||
|
||||
def read(self, nbytes):
|
||||
self.input = self.input[:-1]
|
||||
return self.input[:1]
|
||||
|
||||
chunker = get_chunker(*CHUNKER_PARAMS, seed=0)
|
||||
reconstructed = b''.join(cf(chunker.chunkify(SmallReadFile())))
|
||||
assert reconstructed == b'a' * 20
|
||||
reconstructed = b"".join(cf(chunker.chunkify(SmallReadFile())))
|
||||
assert reconstructed == b"a" * 20
|
||||
|
@ -12,37 +12,27 @@
|
||||
|
||||
# some sparse files. X = content blocks, _ = sparse blocks.
|
||||
# X__XXX____
|
||||
map_sparse1 = [
|
||||
(0 * BS, 1 * BS, True),
|
||||
(1 * BS, 2 * BS, False),
|
||||
(3 * BS, 3 * BS, True),
|
||||
(6 * BS, 4 * BS, False),
|
||||
]
|
||||
map_sparse1 = [(0 * BS, 1 * BS, True), (1 * BS, 2 * BS, False), (3 * BS, 3 * BS, True), (6 * BS, 4 * BS, False)]
|
||||
|
||||
# _XX___XXXX
|
||||
map_sparse2 = [
|
||||
(0 * BS, 1 * BS, False),
|
||||
(1 * BS, 2 * BS, True),
|
||||
(3 * BS, 3 * BS, False),
|
||||
(6 * BS, 4 * BS, True),
|
||||
]
|
||||
map_sparse2 = [(0 * BS, 1 * BS, False), (1 * BS, 2 * BS, True), (3 * BS, 3 * BS, False), (6 * BS, 4 * BS, True)]
|
||||
|
||||
# XXX
|
||||
map_notsparse = [(0 * BS, 3 * BS, True), ]
|
||||
map_notsparse = [(0 * BS, 3 * BS, True)]
|
||||
|
||||
# ___
|
||||
map_onlysparse = [(0 * BS, 3 * BS, False), ]
|
||||
map_onlysparse = [(0 * BS, 3 * BS, False)]
|
||||
|
||||
|
||||
def make_sparsefile(fname, sparsemap, header_size=0):
|
||||
with open(fname, 'wb') as fd:
|
||||
with open(fname, "wb") as fd:
|
||||
total = 0
|
||||
if header_size:
|
||||
fd.write(b'H' * header_size)
|
||||
fd.write(b"H" * header_size)
|
||||
total += header_size
|
||||
for offset, size, is_data in sparsemap:
|
||||
if is_data:
|
||||
fd.write(b'X' * size)
|
||||
fd.write(b"X" * size)
|
||||
else:
|
||||
fd.seek(size, os.SEEK_CUR)
|
||||
total += size
|
||||
@ -54,11 +44,11 @@ def make_content(sparsemap, header_size=0):
|
||||
result = []
|
||||
total = 0
|
||||
if header_size:
|
||||
result.append(b'H' * header_size)
|
||||
result.append(b"H" * header_size)
|
||||
total += header_size
|
||||
for offset, size, is_data in sparsemap:
|
||||
if is_data:
|
||||
result.append(b'X' * size) # bytes!
|
||||
result.append(b"X" * size) # bytes!
|
||||
else:
|
||||
result.append(size) # int!
|
||||
total += size
|
||||
@ -69,9 +59,9 @@ def fs_supports_sparse():
|
||||
if not has_seek_hole:
|
||||
return False
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
fn = os.path.join(tmpdir, 'test_sparse')
|
||||
fn = os.path.join(tmpdir, "test_sparse")
|
||||
make_sparsefile(fn, [(0, BS, False), (BS, BS, True)])
|
||||
with open(fn, 'rb') as f:
|
||||
with open(fn, "rb") as f:
|
||||
try:
|
||||
offset_hole = f.seek(0, os.SEEK_HOLE)
|
||||
offset_data = f.seek(0, os.SEEK_DATA)
|
||||
@ -81,15 +71,12 @@ def fs_supports_sparse():
|
||||
return offset_hole == 0 and offset_data == BS
|
||||
|
||||
|
||||
@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files')
|
||||
@pytest.mark.parametrize("fname, sparse_map", [
|
||||
('sparse1', map_sparse1),
|
||||
('sparse2', map_sparse2),
|
||||
('onlysparse', map_onlysparse),
|
||||
('notsparse', map_notsparse),
|
||||
])
|
||||
@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files")
|
||||
@pytest.mark.parametrize(
|
||||
"fname, sparse_map",
|
||||
[("sparse1", map_sparse1), ("sparse2", map_sparse2), ("onlysparse", map_onlysparse), ("notsparse", map_notsparse)],
|
||||
)
|
||||
def test_sparsemap(tmpdir, fname, sparse_map):
|
||||
|
||||
def get_sparsemap_fh(fname):
|
||||
fh = os.open(fname, flags=os.O_RDONLY)
|
||||
try:
|
||||
@ -98,7 +85,7 @@ def get_sparsemap_fh(fname):
|
||||
os.close(fh)
|
||||
|
||||
def get_sparsemap_fd(fname):
|
||||
with open(fname, 'rb') as fd:
|
||||
with open(fname, "rb") as fd:
|
||||
return list(sparsemap(fd=fd))
|
||||
|
||||
fn = str(tmpdir / fname)
|
||||
@ -107,30 +94,32 @@ def get_sparsemap_fd(fname):
|
||||
assert get_sparsemap_fd(fn) == sparse_map
|
||||
|
||||
|
||||
@pytest.mark.skipif(not fs_supports_sparse(), reason='fs does not support sparse files')
|
||||
@pytest.mark.parametrize("fname, sparse_map, header_size, sparse", [
|
||||
('sparse1', map_sparse1, 0, False),
|
||||
('sparse1', map_sparse1, 0, True),
|
||||
('sparse1', map_sparse1, BS, False),
|
||||
('sparse1', map_sparse1, BS, True),
|
||||
('sparse2', map_sparse2, 0, False),
|
||||
('sparse2', map_sparse2, 0, True),
|
||||
('sparse2', map_sparse2, BS, False),
|
||||
('sparse2', map_sparse2, BS, True),
|
||||
('onlysparse', map_onlysparse, 0, False),
|
||||
('onlysparse', map_onlysparse, 0, True),
|
||||
('onlysparse', map_onlysparse, BS, False),
|
||||
('onlysparse', map_onlysparse, BS, True),
|
||||
('notsparse', map_notsparse, 0, False),
|
||||
('notsparse', map_notsparse, 0, True),
|
||||
('notsparse', map_notsparse, BS, False),
|
||||
('notsparse', map_notsparse, BS, True),
|
||||
])
|
||||
@pytest.mark.skipif(not fs_supports_sparse(), reason="fs does not support sparse files")
|
||||
@pytest.mark.parametrize(
|
||||
"fname, sparse_map, header_size, sparse",
|
||||
[
|
||||
("sparse1", map_sparse1, 0, False),
|
||||
("sparse1", map_sparse1, 0, True),
|
||||
("sparse1", map_sparse1, BS, False),
|
||||
("sparse1", map_sparse1, BS, True),
|
||||
("sparse2", map_sparse2, 0, False),
|
||||
("sparse2", map_sparse2, 0, True),
|
||||
("sparse2", map_sparse2, BS, False),
|
||||
("sparse2", map_sparse2, BS, True),
|
||||
("onlysparse", map_onlysparse, 0, False),
|
||||
("onlysparse", map_onlysparse, 0, True),
|
||||
("onlysparse", map_onlysparse, BS, False),
|
||||
("onlysparse", map_onlysparse, BS, True),
|
||||
("notsparse", map_notsparse, 0, False),
|
||||
("notsparse", map_notsparse, 0, True),
|
||||
("notsparse", map_notsparse, BS, False),
|
||||
("notsparse", map_notsparse, BS, True),
|
||||
],
|
||||
)
|
||||
def test_chunkify_sparse(tmpdir, fname, sparse_map, header_size, sparse):
|
||||
|
||||
def get_chunks(fname, sparse, header_size):
|
||||
chunker = ChunkerFixed(4096, header_size=header_size, sparse=sparse)
|
||||
with open(fname, 'rb') as fd:
|
||||
with open(fname, "rb") as fd:
|
||||
return cf(chunker.chunkify(fd))
|
||||
|
||||
fn = str(tmpdir / fname)
|
||||
|
@ -9,7 +9,6 @@
|
||||
|
||||
|
||||
class ChunkerRegressionTestCase(BaseTestCase):
|
||||
|
||||
def test_chunkpoints_unchanged(self):
|
||||
def twist(size):
|
||||
x = 1
|
||||
@ -31,10 +30,10 @@ def twist(size):
|
||||
for seed in (1849058162, 1234567653):
|
||||
fh = BytesIO(data)
|
||||
chunker = Chunker(seed, minexp, maxexp, maskbits, winsize)
|
||||
chunks = [blake2b_256(b'', c) for c in cf(chunker.chunkify(fh, -1))]
|
||||
runs.append(blake2b_256(b'', b''.join(chunks)))
|
||||
chunks = [blake2b_256(b"", c) for c in cf(chunker.chunkify(fh, -1))]
|
||||
runs.append(blake2b_256(b"", b"".join(chunks)))
|
||||
|
||||
# The "correct" hash below matches the existing chunker behavior.
|
||||
# Future chunker optimisations must not change this, or existing repos will bloat.
|
||||
overall_hash = blake2b_256(b'', b''.join(runs))
|
||||
overall_hash = blake2b_256(b"", b"".join(runs))
|
||||
self.assert_equal(overall_hash, unhexlify("b559b0ac8df8daaa221201d018815114241ea5c6609d98913cd2246a702af4e3"))
|
||||
|
@ -1,5 +1,6 @@
|
||||
import os
|
||||
import zlib
|
||||
|
||||
try:
|
||||
import lzma
|
||||
except ImportError:
|
||||
@ -11,23 +12,23 @@
|
||||
|
||||
|
||||
buffer = bytes(2**16)
|
||||
data = b'fooooooooobaaaaaaaar' * 10
|
||||
params = dict(name='zlib', level=6)
|
||||
data = b"fooooooooobaaaaaaaar" * 10
|
||||
params = dict(name="zlib", level=6)
|
||||
|
||||
|
||||
def test_get_compressor():
|
||||
c = get_compressor(name='none')
|
||||
c = get_compressor(name="none")
|
||||
assert isinstance(c, CNONE)
|
||||
c = get_compressor(name='lz4')
|
||||
c = get_compressor(name="lz4")
|
||||
assert isinstance(c, LZ4)
|
||||
c = get_compressor(name='zlib')
|
||||
c = get_compressor(name="zlib")
|
||||
assert isinstance(c, ZLIB)
|
||||
with pytest.raises(KeyError):
|
||||
get_compressor(name='foobar')
|
||||
get_compressor(name="foobar")
|
||||
|
||||
|
||||
def test_cnull():
|
||||
c = get_compressor(name='none')
|
||||
c = get_compressor(name="none")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) > len(data)
|
||||
assert data in cdata # it's not compressed and just in there 1:1
|
||||
@ -36,7 +37,7 @@ def test_cnull():
|
||||
|
||||
|
||||
def test_lz4():
|
||||
c = get_compressor(name='lz4')
|
||||
c = get_compressor(name="lz4")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) < len(data)
|
||||
assert data == c.decompress(cdata)
|
||||
@ -45,18 +46,18 @@ def test_lz4():
|
||||
|
||||
def test_lz4_buffer_allocation(monkeypatch):
|
||||
# disable fallback to no compression on incompressible data
|
||||
monkeypatch.setattr(LZ4, 'decide', lambda always_compress: LZ4)
|
||||
monkeypatch.setattr(LZ4, "decide", lambda always_compress: LZ4)
|
||||
# test with a rather huge data object to see if buffer allocation / resizing works
|
||||
data = os.urandom(5 * 2**20) * 10 # 50MiB badly compressible data
|
||||
assert len(data) == 50 * 2**20
|
||||
c = Compressor('lz4')
|
||||
c = Compressor("lz4")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) > len(data)
|
||||
assert data == c.decompress(cdata)
|
||||
|
||||
|
||||
def test_zlib():
|
||||
c = get_compressor(name='zlib')
|
||||
c = get_compressor(name="zlib")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) < len(data)
|
||||
assert data == c.decompress(cdata)
|
||||
@ -66,7 +67,7 @@ def test_zlib():
|
||||
def test_lzma():
|
||||
if lzma is None:
|
||||
pytest.skip("No lzma support found.")
|
||||
c = get_compressor(name='lzma')
|
||||
c = get_compressor(name="lzma")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) < len(data)
|
||||
assert data == c.decompress(cdata)
|
||||
@ -74,7 +75,7 @@ def test_lzma():
|
||||
|
||||
|
||||
def test_zstd():
|
||||
c = get_compressor(name='zstd')
|
||||
c = get_compressor(name="zstd")
|
||||
cdata = c.compress(data)
|
||||
assert len(cdata) < len(data)
|
||||
assert data == c.decompress(cdata)
|
||||
@ -83,16 +84,16 @@ def test_zstd():
|
||||
|
||||
def test_autodetect_invalid():
|
||||
with pytest.raises(ValueError):
|
||||
Compressor(**params).decompress(b'\xff\xfftotalcrap')
|
||||
Compressor(**params).decompress(b"\xff\xfftotalcrap")
|
||||
with pytest.raises(ValueError):
|
||||
Compressor(**params).decompress(b'\x08\x00notreallyzlib')
|
||||
Compressor(**params).decompress(b"\x08\x00notreallyzlib")
|
||||
|
||||
|
||||
def test_zlib_legacy_compat():
|
||||
# for compatibility reasons, we do not add an extra header for zlib,
|
||||
# nor do we expect one when decompressing / autodetecting
|
||||
for level in range(10):
|
||||
c = get_compressor(name='zlib_legacy', level=level)
|
||||
c = get_compressor(name="zlib_legacy", level=level)
|
||||
cdata1 = c.compress(data)
|
||||
cdata2 = zlib.compress(data, level)
|
||||
assert cdata1 == cdata2
|
||||
@ -104,19 +105,19 @@ def test_zlib_legacy_compat():
|
||||
|
||||
def test_compressor():
|
||||
params_list = [
|
||||
dict(name='none'),
|
||||
dict(name='lz4'),
|
||||
dict(name='zstd', level=1),
|
||||
dict(name='zstd', level=3),
|
||||
dict(name="none"),
|
||||
dict(name="lz4"),
|
||||
dict(name="zstd", level=1),
|
||||
dict(name="zstd", level=3),
|
||||
# avoiding high zstd levels, memory needs unclear
|
||||
dict(name='zlib', level=0),
|
||||
dict(name='zlib', level=6),
|
||||
dict(name='zlib', level=9),
|
||||
dict(name="zlib", level=0),
|
||||
dict(name="zlib", level=6),
|
||||
dict(name="zlib", level=9),
|
||||
]
|
||||
if lzma:
|
||||
params_list += [
|
||||
dict(name='lzma', level=0),
|
||||
dict(name='lzma', level=6),
|
||||
dict(name="lzma", level=0),
|
||||
dict(name="lzma", level=6),
|
||||
# we do not test lzma on level 9 because of the huge memory needs
|
||||
]
|
||||
for params in params_list:
|
||||
@ -125,9 +126,9 @@ def test_compressor():
|
||||
|
||||
|
||||
def test_auto():
|
||||
compressor_auto_zlib = CompressionSpec('auto,zlib,9').compressor
|
||||
compressor_lz4 = CompressionSpec('lz4').compressor
|
||||
compressor_zlib = CompressionSpec('zlib,9').compressor
|
||||
compressor_auto_zlib = CompressionSpec("auto,zlib,9").compressor
|
||||
compressor_lz4 = CompressionSpec("lz4").compressor
|
||||
compressor_zlib = CompressionSpec("zlib,9").compressor
|
||||
data = bytes(500)
|
||||
compressed_auto_zlib = compressor_auto_zlib.compress(data)
|
||||
compressed_lz4 = compressor_lz4.compress(data)
|
||||
@ -135,13 +136,13 @@ def test_auto():
|
||||
ratio = len(compressed_zlib) / len(compressed_lz4)
|
||||
assert Compressor.detect(compressed_auto_zlib)[0] == ZLIB if ratio < 0.99 else LZ4
|
||||
|
||||
data = b'\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~'
|
||||
data = b"\x00\xb8\xa3\xa2-O\xe1i\xb6\x12\x03\xc21\xf3\x8a\xf78\\\x01\xa5b\x07\x95\xbeE\xf8\xa3\x9ahm\xb1~"
|
||||
compressed = compressor_auto_zlib.compress(data)
|
||||
assert Compressor.detect(compressed)[0] == CNONE
|
||||
|
||||
|
||||
def test_obfuscate():
|
||||
compressor = CompressionSpec('obfuscate,1,none').compressor
|
||||
compressor = CompressionSpec("obfuscate,1,none").compressor
|
||||
data = bytes(10000)
|
||||
compressed = compressor.compress(data)
|
||||
# 2 id bytes compression, 2 id bytes obfuscator. 4 length bytes
|
||||
@ -149,7 +150,7 @@ def test_obfuscate():
|
||||
# compressing 100 times the same data should give at least 50 different result sizes
|
||||
assert len({len(compressor.compress(data)) for i in range(100)}) > 50
|
||||
|
||||
cs = CompressionSpec('obfuscate,2,lz4')
|
||||
cs = CompressionSpec("obfuscate,2,lz4")
|
||||
assert isinstance(cs.inner.compressor, LZ4)
|
||||
compressor = cs.compressor
|
||||
data = bytes(10000)
|
||||
@ -160,7 +161,7 @@ def test_obfuscate():
|
||||
# compressing 100 times the same data should give multiple different result sizes
|
||||
assert len({len(compressor.compress(data)) for i in range(100)}) > 10
|
||||
|
||||
cs = CompressionSpec('obfuscate,6,zstd,3')
|
||||
cs = CompressionSpec("obfuscate,6,zstd,3")
|
||||
assert isinstance(cs.inner.compressor, ZSTD)
|
||||
compressor = cs.compressor
|
||||
data = bytes(10000)
|
||||
@ -171,7 +172,7 @@ def test_obfuscate():
|
||||
# compressing 100 times the same data should give multiple different result sizes
|
||||
assert len({len(compressor.compress(data)) for i in range(100)}) > 90
|
||||
|
||||
cs = CompressionSpec('obfuscate,2,auto,zstd,10')
|
||||
cs = CompressionSpec("obfuscate,2,auto,zstd,10")
|
||||
assert isinstance(cs.inner.compressor, Auto)
|
||||
compressor = cs.compressor
|
||||
data = bytes(10000)
|
||||
@ -182,7 +183,7 @@ def test_obfuscate():
|
||||
# compressing 100 times the same data should give multiple different result sizes
|
||||
assert len({len(compressor.compress(data)) for i in range(100)}) > 10
|
||||
|
||||
cs = CompressionSpec('obfuscate,110,none')
|
||||
cs = CompressionSpec("obfuscate,110,none")
|
||||
assert isinstance(cs.inner.compressor, CNONE)
|
||||
compressor = cs.compressor
|
||||
data = bytes(1000)
|
||||
@ -199,44 +200,44 @@ def test_obfuscate():
|
||||
|
||||
def test_compression_specs():
|
||||
with pytest.raises(ValueError):
|
||||
CompressionSpec('')
|
||||
CompressionSpec("")
|
||||
|
||||
assert isinstance(CompressionSpec('none').compressor, CNONE)
|
||||
assert isinstance(CompressionSpec('lz4').compressor, LZ4)
|
||||
assert isinstance(CompressionSpec("none").compressor, CNONE)
|
||||
assert isinstance(CompressionSpec("lz4").compressor, LZ4)
|
||||
|
||||
zlib = CompressionSpec('zlib').compressor
|
||||
zlib = CompressionSpec("zlib").compressor
|
||||
assert isinstance(zlib, ZLIB)
|
||||
assert zlib.level == 6
|
||||
zlib = CompressionSpec('zlib,0').compressor
|
||||
zlib = CompressionSpec("zlib,0").compressor
|
||||
assert isinstance(zlib, ZLIB)
|
||||
assert zlib.level == 0
|
||||
zlib = CompressionSpec('zlib,9').compressor
|
||||
zlib = CompressionSpec("zlib,9").compressor
|
||||
assert isinstance(zlib, ZLIB)
|
||||
assert zlib.level == 9
|
||||
with pytest.raises(ValueError):
|
||||
CompressionSpec('zlib,9,invalid')
|
||||
CompressionSpec("zlib,9,invalid")
|
||||
|
||||
lzma = CompressionSpec('lzma').compressor
|
||||
lzma = CompressionSpec("lzma").compressor
|
||||
assert isinstance(lzma, LZMA)
|
||||
assert lzma.level == 6
|
||||
lzma = CompressionSpec('lzma,0').compressor
|
||||
lzma = CompressionSpec("lzma,0").compressor
|
||||
assert isinstance(lzma, LZMA)
|
||||
assert lzma.level == 0
|
||||
lzma = CompressionSpec('lzma,9').compressor
|
||||
lzma = CompressionSpec("lzma,9").compressor
|
||||
assert isinstance(lzma, LZMA)
|
||||
assert lzma.level == 9
|
||||
|
||||
zstd = CompressionSpec('zstd').compressor
|
||||
zstd = CompressionSpec("zstd").compressor
|
||||
assert isinstance(zstd, ZSTD)
|
||||
assert zstd.level == 3
|
||||
zstd = CompressionSpec('zstd,1').compressor
|
||||
zstd = CompressionSpec("zstd,1").compressor
|
||||
assert isinstance(zstd, ZSTD)
|
||||
assert zstd.level == 1
|
||||
zstd = CompressionSpec('zstd,22').compressor
|
||||
zstd = CompressionSpec("zstd,22").compressor
|
||||
assert isinstance(zstd, ZSTD)
|
||||
assert zstd.level == 22
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
CompressionSpec('lzma,9,invalid')
|
||||
CompressionSpec("lzma,9,invalid")
|
||||
with pytest.raises(ValueError):
|
||||
CompressionSpec('invalid')
|
||||
CompressionSpec("invalid")
|
||||
|
@ -16,18 +16,17 @@
|
||||
|
||||
|
||||
class CryptoTestCase(BaseTestCase):
|
||||
|
||||
def test_bytes_to_int(self):
|
||||
self.assert_equal(bytes_to_int(b'\0\0\0\1'), 1)
|
||||
self.assert_equal(bytes_to_int(b"\0\0\0\1"), 1)
|
||||
|
||||
def test_bytes_to_long(self):
|
||||
self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1)
|
||||
self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1')
|
||||
self.assert_equal(bytes_to_long(b"\0\0\0\0\0\0\0\1"), 1)
|
||||
self.assert_equal(long_to_bytes(1), b"\0\0\0\0\0\0\0\1")
|
||||
|
||||
def test_UNENCRYPTED(self):
|
||||
iv = b'' # any IV is ok, it just must be set and not None
|
||||
data = b'data'
|
||||
header = b'header'
|
||||
iv = b"" # any IV is ok, it just must be set and not None
|
||||
data = b"data"
|
||||
header = b"header"
|
||||
cs = UNENCRYPTED(None, None, iv, header_len=6)
|
||||
envelope = cs.encrypt(data, header=header)
|
||||
self.assert_equal(envelope, header + data)
|
||||
@ -36,11 +35,11 @@ def test_UNENCRYPTED(self):
|
||||
|
||||
def test_AES256_CTR_HMAC_SHA256(self):
|
||||
# this tests the layout as in attic / borg < 1.2 (1 type byte, no aad)
|
||||
mac_key = b'Y' * 32
|
||||
enc_key = b'X' * 32
|
||||
mac_key = b"Y" * 32
|
||||
enc_key = b"X" * 32
|
||||
iv = 0
|
||||
data = b'foo' * 10
|
||||
header = b'\x42'
|
||||
data = b"foo" * 10
|
||||
header = b"\x42"
|
||||
# encrypt-then-mac
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=1, aad_offset=1)
|
||||
hdr_mac_iv_cdata = cs.encrypt(data, header=header)
|
||||
@ -48,10 +47,10 @@ def test_AES256_CTR_HMAC_SHA256(self):
|
||||
mac = hdr_mac_iv_cdata[1:33]
|
||||
iv = hdr_mac_iv_cdata[33:41]
|
||||
cdata = hdr_mac_iv_cdata[41:]
|
||||
self.assert_equal(hexlify(hdr), b'42')
|
||||
self.assert_equal(hexlify(mac), b'af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8')
|
||||
self.assert_equal(hexlify(iv), b'0000000000000000')
|
||||
self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466')
|
||||
self.assert_equal(hexlify(hdr), b"42")
|
||||
self.assert_equal(hexlify(mac), b"af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8")
|
||||
self.assert_equal(hexlify(iv), b"0000000000000000")
|
||||
self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466")
|
||||
self.assert_equal(cs.next_iv(), 2)
|
||||
# auth-then-decrypt
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
|
||||
@ -60,16 +59,15 @@ def test_AES256_CTR_HMAC_SHA256(self):
|
||||
self.assert_equal(cs.next_iv(), 2)
|
||||
# auth-failure due to corruption (corrupted data)
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b'\0' + hdr_mac_iv_cdata[42:]
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b"\0" + hdr_mac_iv_cdata[42:]
|
||||
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
|
||||
def test_AES256_CTR_HMAC_SHA256_aad(self):
|
||||
mac_key = b'Y' * 32
|
||||
enc_key = b'X' * 32
|
||||
mac_key = b"Y" * 32
|
||||
enc_key = b"X" * 32
|
||||
iv = 0
|
||||
data = b'foo' * 10
|
||||
header = b'\x12\x34\x56'
|
||||
data = b"foo" * 10
|
||||
header = b"\x12\x34\x56"
|
||||
# encrypt-then-mac
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv, header_len=3, aad_offset=1)
|
||||
hdr_mac_iv_cdata = cs.encrypt(data, header=header)
|
||||
@ -77,10 +75,10 @@ def test_AES256_CTR_HMAC_SHA256_aad(self):
|
||||
mac = hdr_mac_iv_cdata[3:35]
|
||||
iv = hdr_mac_iv_cdata[35:43]
|
||||
cdata = hdr_mac_iv_cdata[43:]
|
||||
self.assert_equal(hexlify(hdr), b'123456')
|
||||
self.assert_equal(hexlify(mac), b'7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138')
|
||||
self.assert_equal(hexlify(iv), b'0000000000000000')
|
||||
self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466')
|
||||
self.assert_equal(hexlify(hdr), b"123456")
|
||||
self.assert_equal(hexlify(mac), b"7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138")
|
||||
self.assert_equal(hexlify(iv), b"0000000000000000")
|
||||
self.assert_equal(hexlify(cdata), b"c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466")
|
||||
self.assert_equal(cs.next_iv(), 2)
|
||||
# auth-then-decrypt
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
|
||||
@ -89,24 +87,27 @@ def test_AES256_CTR_HMAC_SHA256_aad(self):
|
||||
self.assert_equal(cs.next_iv(), 2)
|
||||
# auth-failure due to corruption (corrupted aad)
|
||||
cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, header_len=len(header), aad_offset=1)
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:]
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:]
|
||||
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
|
||||
def test_AE(self):
|
||||
# used in legacy-like layout (1 type byte, no aad)
|
||||
key = b'X' * 32
|
||||
key = b"X" * 32
|
||||
iv_int = 0
|
||||
data = b'foo' * 10
|
||||
header = b'\x23' + iv_int.to_bytes(12, 'big')
|
||||
data = b"foo" * 10
|
||||
header = b"\x23" + iv_int.to_bytes(12, "big")
|
||||
tests = [
|
||||
# (ciphersuite class, exp_mac, exp_cdata)
|
||||
(AES256_OCB,
|
||||
b'b6909c23c9aaebd9abbe1ff42097652d',
|
||||
b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ),
|
||||
(CHACHA20_POLY1305,
|
||||
b'fd08594796e0706cde1e8b461e3e0555',
|
||||
b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', )
|
||||
(
|
||||
AES256_OCB,
|
||||
b"b6909c23c9aaebd9abbe1ff42097652d",
|
||||
b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493",
|
||||
),
|
||||
(
|
||||
CHACHA20_POLY1305,
|
||||
b"fd08594796e0706cde1e8b461e3e0555",
|
||||
b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775",
|
||||
),
|
||||
]
|
||||
for cs_cls, exp_mac, exp_cdata in tests:
|
||||
# print(repr(cs_cls))
|
||||
@ -117,9 +118,9 @@ def test_AE(self):
|
||||
iv = hdr_mac_iv_cdata[1:13]
|
||||
mac = hdr_mac_iv_cdata[13:29]
|
||||
cdata = hdr_mac_iv_cdata[29:]
|
||||
self.assert_equal(hexlify(hdr), b'23')
|
||||
self.assert_equal(hexlify(hdr), b"23")
|
||||
self.assert_equal(hexlify(mac), exp_mac)
|
||||
self.assert_equal(hexlify(iv), b'000000000000000000000000')
|
||||
self.assert_equal(hexlify(iv), b"000000000000000000000000")
|
||||
self.assert_equal(hexlify(cdata), exp_cdata)
|
||||
self.assert_equal(cs.next_iv(), 1)
|
||||
# auth/decrypt
|
||||
@ -129,24 +130,27 @@ def test_AE(self):
|
||||
self.assert_equal(cs.next_iv(), 1)
|
||||
# auth-failure due to corruption (corrupted data)
|
||||
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1)
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:]
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b"\0" + hdr_mac_iv_cdata[30:]
|
||||
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
|
||||
def test_AEAD(self):
|
||||
# test with aad
|
||||
key = b'X' * 32
|
||||
key = b"X" * 32
|
||||
iv_int = 0
|
||||
data = b'foo' * 10
|
||||
header = b'\x12\x34\x56' + iv_int.to_bytes(12, 'big')
|
||||
data = b"foo" * 10
|
||||
header = b"\x12\x34\x56" + iv_int.to_bytes(12, "big")
|
||||
tests = [
|
||||
# (ciphersuite class, exp_mac, exp_cdata)
|
||||
(AES256_OCB,
|
||||
b'f2748c412af1c7ead81863a18c2c1893',
|
||||
b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ),
|
||||
(CHACHA20_POLY1305,
|
||||
b'b7e7c9a79f2404e14f9aad156bf091dd',
|
||||
b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', )
|
||||
(
|
||||
AES256_OCB,
|
||||
b"f2748c412af1c7ead81863a18c2c1893",
|
||||
b"877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493",
|
||||
),
|
||||
(
|
||||
CHACHA20_POLY1305,
|
||||
b"b7e7c9a79f2404e14f9aad156bf091dd",
|
||||
b"a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775",
|
||||
),
|
||||
]
|
||||
for cs_cls, exp_mac, exp_cdata in tests:
|
||||
# print(repr(cs_cls))
|
||||
@ -157,9 +161,9 @@ def test_AEAD(self):
|
||||
iv = hdr_mac_iv_cdata[3:15]
|
||||
mac = hdr_mac_iv_cdata[15:31]
|
||||
cdata = hdr_mac_iv_cdata[31:]
|
||||
self.assert_equal(hexlify(hdr), b'123456')
|
||||
self.assert_equal(hexlify(hdr), b"123456")
|
||||
self.assert_equal(hexlify(mac), exp_mac)
|
||||
self.assert_equal(hexlify(iv), b'000000000000000000000000')
|
||||
self.assert_equal(hexlify(iv), b"000000000000000000000000")
|
||||
self.assert_equal(hexlify(cdata), exp_cdata)
|
||||
self.assert_equal(cs.next_iv(), 1)
|
||||
# auth/decrypt
|
||||
@ -169,101 +173,117 @@ def test_AEAD(self):
|
||||
self.assert_equal(cs.next_iv(), 1)
|
||||
# auth-failure due to corruption (corrupted aad)
|
||||
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=1)
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:]
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b"\0" + hdr_mac_iv_cdata[2:]
|
||||
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
|
||||
|
||||
def test_AEAD_with_more_AAD(self):
|
||||
# test giving extra aad to the .encrypt() and .decrypt() calls
|
||||
key = b'X' * 32
|
||||
key = b"X" * 32
|
||||
iv_int = 0
|
||||
data = b'foo' * 10
|
||||
header = b'\x12\x34'
|
||||
data = b"foo" * 10
|
||||
header = b"\x12\x34"
|
||||
tests = [AES256_OCB, CHACHA20_POLY1305]
|
||||
for cs_cls in tests:
|
||||
# encrypt/mac
|
||||
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
|
||||
hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b'correct_chunkid')
|
||||
hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad=b"correct_chunkid")
|
||||
# successful auth/decrypt (correct aad)
|
||||
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
|
||||
pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b'correct_chunkid')
|
||||
pdata = cs.decrypt(hdr_mac_iv_cdata, aad=b"correct_chunkid")
|
||||
self.assert_equal(data, pdata)
|
||||
# unsuccessful auth (incorrect aad)
|
||||
cs = cs_cls(key, iv_int, header_len=len(header), aad_offset=0)
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b'incorrect_chunkid'))
|
||||
self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata, aad=b"incorrect_chunkid"))
|
||||
|
||||
# These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/
|
||||
# who claims to have verified these against independent Python and C++ implementations.
|
||||
|
||||
def test_hkdf_hmac_sha512(self):
|
||||
ikm = b'\x0b' * 22
|
||||
salt = bytes.fromhex('000102030405060708090a0b0c')
|
||||
info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9')
|
||||
ikm = b"\x0b" * 22
|
||||
salt = bytes.fromhex("000102030405060708090a0b0c")
|
||||
info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9")
|
||||
l = 42
|
||||
|
||||
okm = hkdf_hmac_sha512(ikm, salt, info, l)
|
||||
assert okm == bytes.fromhex('832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb')
|
||||
assert okm == bytes.fromhex(
|
||||
"832390086cda71fb47625bb5ceb168e4c8e26a1a16ed34d9fc7fe92c1481579338da362cb8d9f925d7cb"
|
||||
)
|
||||
|
||||
def test_hkdf_hmac_sha512_2(self):
|
||||
ikm = bytes.fromhex('000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627'
|
||||
'28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f')
|
||||
salt = bytes.fromhex('606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868'
|
||||
'788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf')
|
||||
info = bytes.fromhex('b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7'
|
||||
'd8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff')
|
||||
ikm = bytes.fromhex(
|
||||
"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627"
|
||||
"28292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f"
|
||||
)
|
||||
salt = bytes.fromhex(
|
||||
"606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868"
|
||||
"788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
|
||||
)
|
||||
info = bytes.fromhex(
|
||||
"b0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7"
|
||||
"d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
|
||||
)
|
||||
l = 82
|
||||
|
||||
okm = hkdf_hmac_sha512(ikm, salt, info, l)
|
||||
assert okm == bytes.fromhex('ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844'
|
||||
'1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93')
|
||||
assert okm == bytes.fromhex(
|
||||
"ce6c97192805b346e6161e821ed165673b84f400a2b514b2fe23d84cd189ddf1b695b48cbd1c838844"
|
||||
"1137b3ce28f16aa64ba33ba466b24df6cfcb021ecff235f6a2056ce3af1de44d572097a8505d9e7a93"
|
||||
)
|
||||
|
||||
def test_hkdf_hmac_sha512_3(self):
|
||||
ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b')
|
||||
ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b")
|
||||
salt = None
|
||||
info = b''
|
||||
info = b""
|
||||
l = 42
|
||||
|
||||
okm = hkdf_hmac_sha512(ikm, salt, info, l)
|
||||
assert okm == bytes.fromhex('f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac')
|
||||
assert okm == bytes.fromhex(
|
||||
"f5fa02b18298a72a8c23898a8703472c6eb179dc204c03425c970e3b164bf90fff22d04836d0e2343bac"
|
||||
)
|
||||
|
||||
def test_hkdf_hmac_sha512_4(self):
|
||||
ikm = bytes.fromhex('0b0b0b0b0b0b0b0b0b0b0b')
|
||||
salt = bytes.fromhex('000102030405060708090a0b0c')
|
||||
info = bytes.fromhex('f0f1f2f3f4f5f6f7f8f9')
|
||||
ikm = bytes.fromhex("0b0b0b0b0b0b0b0b0b0b0b")
|
||||
salt = bytes.fromhex("000102030405060708090a0b0c")
|
||||
info = bytes.fromhex("f0f1f2f3f4f5f6f7f8f9")
|
||||
l = 42
|
||||
|
||||
okm = hkdf_hmac_sha512(ikm, salt, info, l)
|
||||
assert okm == bytes.fromhex('7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068')
|
||||
assert okm == bytes.fromhex(
|
||||
"7413e8997e020610fbf6823f2ce14bff01875db1ca55f68cfcf3954dc8aff53559bd5e3028b080f7c068"
|
||||
)
|
||||
|
||||
def test_hkdf_hmac_sha512_5(self):
|
||||
ikm = bytes.fromhex('0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c')
|
||||
ikm = bytes.fromhex("0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c0c")
|
||||
salt = None
|
||||
info = b''
|
||||
info = b""
|
||||
l = 42
|
||||
|
||||
okm = hkdf_hmac_sha512(ikm, salt, info, l)
|
||||
assert okm == bytes.fromhex('1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb')
|
||||
assert okm == bytes.fromhex(
|
||||
"1407d46013d98bc6decefcfee55f0f90b0c7f63d68eb1a80eaf07e953cfc0a3a5240a155d6e4daa965bb"
|
||||
)
|
||||
|
||||
|
||||
def test_decrypt_key_file_argon2_chacha20_poly1305():
|
||||
plain = b'hello'
|
||||
plain = b"hello"
|
||||
# echo -n "hello, pass phrase" | argon2 saltsaltsaltsalt -id -t 1 -k 8 -p 1 -l 32 -r
|
||||
key = bytes.fromhex('a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec')
|
||||
key = bytes.fromhex("a1b0cba145c154fbd8960996c5ce3428e9920cfe53c84ef08b4102a70832bcec")
|
||||
ae_cipher = CHACHA20_POLY1305(key=key, iv=0, header_len=0, aad_offset=0)
|
||||
|
||||
envelope = ae_cipher.encrypt(plain)
|
||||
|
||||
encrypted = msgpack.packb({
|
||||
'version': 1,
|
||||
'salt': b'salt'*4,
|
||||
'argon2_time_cost': 1,
|
||||
'argon2_memory_cost': 8,
|
||||
'argon2_parallelism': 1,
|
||||
'argon2_type': b'id',
|
||||
'algorithm': 'argon2 chacha20-poly1305',
|
||||
'data': envelope,
|
||||
})
|
||||
encrypted = msgpack.packb(
|
||||
{
|
||||
"version": 1,
|
||||
"salt": b"salt" * 4,
|
||||
"argon2_time_cost": 1,
|
||||
"argon2_memory_cost": 8,
|
||||
"argon2_parallelism": 1,
|
||||
"argon2_type": b"id",
|
||||
"algorithm": "argon2 chacha20-poly1305",
|
||||
"data": envelope,
|
||||
}
|
||||
)
|
||||
key = CHPOKeyfileKey(None)
|
||||
|
||||
decrypted = key.decrypt_key_file(encrypted, "hello, pass phrase")
|
||||
@ -272,20 +292,15 @@ def test_decrypt_key_file_argon2_chacha20_poly1305():
|
||||
|
||||
|
||||
def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256():
|
||||
plain = b'hello'
|
||||
salt = b'salt'*4
|
||||
plain = b"hello"
|
||||
salt = b"salt" * 4
|
||||
passphrase = "hello, pass phrase"
|
||||
key = FlexiKey.pbkdf2(passphrase, salt, 1, 32)
|
||||
hash = hmac_sha256(key, plain)
|
||||
data = AES(key, b'\0'*16).encrypt(plain)
|
||||
encrypted = msgpack.packb({
|
||||
'version': 1,
|
||||
'algorithm': 'sha256',
|
||||
'iterations': 1,
|
||||
'salt': salt,
|
||||
'data': data,
|
||||
'hash': hash,
|
||||
})
|
||||
data = AES(key, b"\0" * 16).encrypt(plain)
|
||||
encrypted = msgpack.packb(
|
||||
{"version": 1, "algorithm": "sha256", "iterations": 1, "salt": salt, "data": data, "hash": hash}
|
||||
)
|
||||
key = CHPOKeyfileKey(None)
|
||||
|
||||
decrypted = key.decrypt_key_file(encrypted, passphrase)
|
||||
@ -293,7 +308,7 @@ def test_decrypt_key_file_pbkdf2_sha256_aes256_ctr_hmac_sha256():
|
||||
assert decrypted == plain
|
||||
|
||||
|
||||
@unittest.mock.patch('getpass.getpass')
|
||||
@unittest.mock.patch("getpass.getpass")
|
||||
def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch):
|
||||
"""https://github.com/borgbackup/borg/pull/6469#discussion_r832670411
|
||||
|
||||
@ -322,10 +337,10 @@ def test_repo_key_detect_does_not_raise_integrity_error(getpass, monkeypatch):
|
||||
2. FlexiKey.detect() relies on that interface - it tries an empty passphrase before prompting the user
|
||||
3. my initial implementation of decrypt_key_file_argon2() was simply passing through the IntegrityError() from AES256_CTR_BASE.decrypt()
|
||||
"""
|
||||
repository = MagicMock(id=b'repository_id')
|
||||
repository = MagicMock(id=b"repository_id")
|
||||
getpass.return_value = "hello, pass phrase"
|
||||
monkeypatch.setenv('BORG_DISPLAY_PASSPHRASE', 'no')
|
||||
AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2'))
|
||||
monkeypatch.setenv("BORG_DISPLAY_PASSPHRASE", "no")
|
||||
AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2"))
|
||||
repository.load_key.return_value = repository.save_key.call_args.args[0]
|
||||
|
||||
AESOCBRepoKey.detect(repository, manifest_data=None)
|
||||
|
@ -6,13 +6,13 @@
|
||||
class TestEfficientQueue:
|
||||
def test_base_usage(self):
|
||||
queue = EfficientCollectionQueue(100, bytes)
|
||||
assert queue.peek_front() == b''
|
||||
queue.push_back(b'1234')
|
||||
assert queue.peek_front() == b'1234'
|
||||
assert queue.peek_front() == b""
|
||||
queue.push_back(b"1234")
|
||||
assert queue.peek_front() == b"1234"
|
||||
assert len(queue) == 4
|
||||
assert queue
|
||||
queue.pop_front(4)
|
||||
assert queue.peek_front() == b''
|
||||
assert queue.peek_front() == b""
|
||||
assert len(queue) == 0
|
||||
assert not queue
|
||||
|
||||
@ -30,22 +30,22 @@ def test_usage_with_arrays(self):
|
||||
|
||||
def test_chunking(self):
|
||||
queue = EfficientCollectionQueue(2, bytes)
|
||||
queue.push_back(b'1')
|
||||
queue.push_back(b'23')
|
||||
queue.push_back(b'4567')
|
||||
queue.push_back(b"1")
|
||||
queue.push_back(b"23")
|
||||
queue.push_back(b"4567")
|
||||
assert len(queue) == 7
|
||||
assert queue.peek_front() == b'12'
|
||||
assert queue.peek_front() == b"12"
|
||||
queue.pop_front(3)
|
||||
assert queue.peek_front() == b'4'
|
||||
assert queue.peek_front() == b"4"
|
||||
queue.pop_front(1)
|
||||
assert queue.peek_front() == b'56'
|
||||
assert queue.peek_front() == b"56"
|
||||
queue.pop_front(2)
|
||||
assert len(queue) == 1
|
||||
assert queue
|
||||
with pytest.raises(EfficientCollectionQueue.SizeUnderflow):
|
||||
queue.pop_front(2)
|
||||
assert queue.peek_front() == b'7'
|
||||
assert queue.peek_front() == b"7"
|
||||
queue.pop_front(1)
|
||||
assert queue.peek_front() == b''
|
||||
assert queue.peek_front() == b""
|
||||
assert len(queue) == 0
|
||||
assert not queue
|
||||
|
@ -5,34 +5,30 @@
|
||||
|
||||
class TestReadIntegrityFile:
|
||||
def test_no_integrity(self, tmpdir):
|
||||
protected_file = tmpdir.join('file')
|
||||
protected_file.write('1234')
|
||||
protected_file = tmpdir.join("file")
|
||||
protected_file.write("1234")
|
||||
assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None
|
||||
|
||||
def test_truncated_integrity(self, tmpdir):
|
||||
protected_file = tmpdir.join('file')
|
||||
protected_file.write('1234')
|
||||
tmpdir.join('file.integrity').write('')
|
||||
protected_file = tmpdir.join("file")
|
||||
protected_file.write("1234")
|
||||
tmpdir.join("file.integrity").write("")
|
||||
with pytest.raises(FileIntegrityError):
|
||||
DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file))
|
||||
|
||||
def test_unknown_algorithm(self, tmpdir):
|
||||
protected_file = tmpdir.join('file')
|
||||
protected_file.write('1234')
|
||||
tmpdir.join('file.integrity').write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}')
|
||||
protected_file = tmpdir.join("file")
|
||||
protected_file.write("1234")
|
||||
tmpdir.join("file.integrity").write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}')
|
||||
assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file)) is None
|
||||
|
||||
@pytest.mark.parametrize('json', (
|
||||
'{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}',
|
||||
'[]',
|
||||
'1234.5',
|
||||
'"A string"',
|
||||
'Invalid JSON',
|
||||
))
|
||||
@pytest.mark.parametrize(
|
||||
"json", ('{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}', "[]", "1234.5", '"A string"', "Invalid JSON")
|
||||
)
|
||||
def test_malformed(self, tmpdir, json):
|
||||
protected_file = tmpdir.join('file')
|
||||
protected_file.write('1234')
|
||||
tmpdir.join('file.integrity').write(json)
|
||||
protected_file = tmpdir.join("file")
|
||||
protected_file.write("1234")
|
||||
tmpdir.join("file.integrity").write(json)
|
||||
with pytest.raises(FileIntegrityError):
|
||||
DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file))
|
||||
|
||||
@ -40,74 +36,71 @@ def test_malformed(self, tmpdir, json):
|
||||
class TestDetachedIntegrityCheckedFile:
|
||||
@pytest.fixture
|
||||
def integrity_protected_file(self, tmpdir):
|
||||
path = str(tmpdir.join('file'))
|
||||
path = str(tmpdir.join("file"))
|
||||
with DetachedIntegrityCheckedFile(path, write=True) as fd:
|
||||
fd.write(b'foo and bar')
|
||||
fd.write(b"foo and bar")
|
||||
return path
|
||||
|
||||
def test_simple(self, tmpdir, integrity_protected_file):
|
||||
assert tmpdir.join('file').check(file=True)
|
||||
assert tmpdir.join('file.integrity').check(file=True)
|
||||
assert tmpdir.join("file").check(file=True)
|
||||
assert tmpdir.join("file.integrity").check(file=True)
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
assert fd.read() == b'foo and bar'
|
||||
assert fd.read() == b"foo and bar"
|
||||
|
||||
def test_corrupted_file(self, integrity_protected_file):
|
||||
with open(integrity_protected_file, 'ab') as fd:
|
||||
fd.write(b' extra data')
|
||||
with open(integrity_protected_file, "ab") as fd:
|
||||
fd.write(b" extra data")
|
||||
with pytest.raises(FileIntegrityError):
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
assert fd.read() == b'foo and bar extra data'
|
||||
assert fd.read() == b"foo and bar extra data"
|
||||
|
||||
def test_corrupted_file_partial_read(self, integrity_protected_file):
|
||||
with open(integrity_protected_file, 'ab') as fd:
|
||||
fd.write(b' extra data')
|
||||
with open(integrity_protected_file, "ab") as fd:
|
||||
fd.write(b" extra data")
|
||||
with pytest.raises(FileIntegrityError):
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
data = b'foo and bar'
|
||||
data = b"foo and bar"
|
||||
assert fd.read(len(data)) == data
|
||||
|
||||
@pytest.mark.parametrize('new_name', (
|
||||
'different_file',
|
||||
'different_file.different_ext',
|
||||
))
|
||||
@pytest.mark.parametrize("new_name", ("different_file", "different_file.different_ext"))
|
||||
def test_renamed_file(self, tmpdir, integrity_protected_file, new_name):
|
||||
new_path = tmpdir.join(new_name)
|
||||
tmpdir.join('file').move(new_path)
|
||||
tmpdir.join('file.integrity').move(new_path + '.integrity')
|
||||
tmpdir.join("file").move(new_path)
|
||||
tmpdir.join("file.integrity").move(new_path + ".integrity")
|
||||
with pytest.raises(FileIntegrityError):
|
||||
with DetachedIntegrityCheckedFile(str(new_path), write=False) as fd:
|
||||
assert fd.read() == b'foo and bar'
|
||||
assert fd.read() == b"foo and bar"
|
||||
|
||||
def test_moved_file(self, tmpdir, integrity_protected_file):
|
||||
new_dir = tmpdir.mkdir('another_directory')
|
||||
tmpdir.join('file').move(new_dir.join('file'))
|
||||
tmpdir.join('file.integrity').move(new_dir.join('file.integrity'))
|
||||
new_path = str(new_dir.join('file'))
|
||||
new_dir = tmpdir.mkdir("another_directory")
|
||||
tmpdir.join("file").move(new_dir.join("file"))
|
||||
tmpdir.join("file.integrity").move(new_dir.join("file.integrity"))
|
||||
new_path = str(new_dir.join("file"))
|
||||
with DetachedIntegrityCheckedFile(new_path, write=False) as fd:
|
||||
assert fd.read() == b'foo and bar'
|
||||
assert fd.read() == b"foo and bar"
|
||||
|
||||
def test_no_integrity(self, tmpdir, integrity_protected_file):
|
||||
tmpdir.join('file.integrity').remove()
|
||||
tmpdir.join("file.integrity").remove()
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
assert fd.read() == b'foo and bar'
|
||||
assert fd.read() == b"foo and bar"
|
||||
|
||||
|
||||
class TestDetachedIntegrityCheckedFileParts:
|
||||
@pytest.fixture
|
||||
def integrity_protected_file(self, tmpdir):
|
||||
path = str(tmpdir.join('file'))
|
||||
path = str(tmpdir.join("file"))
|
||||
with DetachedIntegrityCheckedFile(path, write=True) as fd:
|
||||
fd.write(b'foo and bar')
|
||||
fd.hash_part('foopart')
|
||||
fd.write(b' other data')
|
||||
fd.write(b"foo and bar")
|
||||
fd.hash_part("foopart")
|
||||
fd.write(b" other data")
|
||||
return path
|
||||
|
||||
def test_simple(self, integrity_protected_file):
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
data1 = b'foo and bar'
|
||||
data1 = b"foo and bar"
|
||||
assert fd.read(len(data1)) == data1
|
||||
fd.hash_part('foopart')
|
||||
assert fd.read() == b' other data'
|
||||
fd.hash_part("foopart")
|
||||
assert fd.read() == b" other data"
|
||||
|
||||
def test_wrong_part_name(self, integrity_protected_file):
|
||||
with pytest.raises(FileIntegrityError):
|
||||
@ -115,25 +108,25 @@ def test_wrong_part_name(self, integrity_protected_file):
|
||||
# the failing hash_part. This is intentional: (1) it makes the code simpler (2) it's a good fail-safe
|
||||
# against overly broad exception handling.
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
data1 = b'foo and bar'
|
||||
data1 = b"foo and bar"
|
||||
assert fd.read(len(data1)) == data1
|
||||
with pytest.raises(FileIntegrityError):
|
||||
# This specific bit raises it directly
|
||||
fd.hash_part('barpart')
|
||||
fd.hash_part("barpart")
|
||||
# Still explodes in the end.
|
||||
|
||||
@pytest.mark.parametrize('partial_read', (False, True))
|
||||
@pytest.mark.parametrize("partial_read", (False, True))
|
||||
def test_part_independence(self, integrity_protected_file, partial_read):
|
||||
with open(integrity_protected_file, 'ab') as fd:
|
||||
fd.write(b'some extra stuff that does not belong')
|
||||
with open(integrity_protected_file, "ab") as fd:
|
||||
fd.write(b"some extra stuff that does not belong")
|
||||
with pytest.raises(FileIntegrityError):
|
||||
with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
|
||||
data1 = b'foo and bar'
|
||||
data1 = b"foo and bar"
|
||||
try:
|
||||
assert fd.read(len(data1)) == data1
|
||||
fd.hash_part('foopart')
|
||||
fd.hash_part("foopart")
|
||||
except FileIntegrityError:
|
||||
assert False, 'This part must not raise, since this part is still valid.'
|
||||
assert False, "This part must not raise, since this part is still valid."
|
||||
if not partial_read:
|
||||
fd.read()
|
||||
# But overall it explodes with the final digest. Neat, eh?
|
||||
|
@ -15,7 +15,7 @@
|
||||
|
||||
def H(x):
|
||||
# make some 32byte long thing that depends on x
|
||||
return bytes('%-0.32d' % x, 'ascii')
|
||||
return bytes("%-0.32d" % x, "ascii")
|
||||
|
||||
|
||||
def H2(x):
|
||||
@ -24,7 +24,6 @@ def H2(x):
|
||||
|
||||
|
||||
class HashIndexTestCase(BaseTestCase):
|
||||
|
||||
def _generic_test(self, cls, make_value, sha):
|
||||
idx = cls()
|
||||
self.assert_equal(len(idx), 0)
|
||||
@ -57,7 +56,7 @@ def _generic_test(self, cls, make_value, sha):
|
||||
idx.write(filepath)
|
||||
del idx
|
||||
# Verify file contents
|
||||
with open(filepath, 'rb') as fd:
|
||||
with open(filepath, "rb") as fd:
|
||||
self.assert_equal(hashlib.sha256(fd.read()).hexdigest(), sha)
|
||||
# Make sure we can open the file
|
||||
idx = cls.read(filepath)
|
||||
@ -86,12 +85,14 @@ def _generic_test(self, cls, make_value, sha):
|
||||
del idx
|
||||
|
||||
def test_nsindex(self):
|
||||
self._generic_test(NSIndex, lambda x: (x, x, x),
|
||||
'7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256')
|
||||
self._generic_test(
|
||||
NSIndex, lambda x: (x, x, x), "7d70671d0b7e9d2f51b2691ecf35184b9f8ecc1202cceb2748c905c8fc04c256"
|
||||
)
|
||||
|
||||
def test_chunkindex(self):
|
||||
self._generic_test(ChunkIndex, lambda x: (x, x),
|
||||
'85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef')
|
||||
self._generic_test(
|
||||
ChunkIndex, lambda x: (x, x), "85f72b036c692c8266e4f51ccf0cff2147204282b5e316ae508d30a448d88fef"
|
||||
)
|
||||
|
||||
def test_resize(self):
|
||||
n = 2000 # Must be >= MIN_BUCKETS
|
||||
@ -218,8 +219,8 @@ def test_flags_iteritems(self):
|
||||
|
||||
|
||||
class HashIndexExtraTestCase(BaseTestCase):
|
||||
"""These tests are separate because they should not become part of the selftest.
|
||||
"""
|
||||
"""These tests are separate because they should not become part of the selftest."""
|
||||
|
||||
def test_chunk_indexer(self):
|
||||
# see _hashindex.c hash_sizes, we want to be close to the max. load
|
||||
# because interesting errors happen there.
|
||||
@ -227,7 +228,7 @@ def test_chunk_indexer(self):
|
||||
index = ChunkIndex(key_count)
|
||||
all_keys = [hashlib.sha256(H(k)).digest() for k in range(key_count)]
|
||||
# we're gonna delete 1/3 of all_keys, so let's split them 2/3 and 1/3:
|
||||
keys, to_delete_keys = all_keys[0:(2*key_count//3)], all_keys[(2*key_count//3):]
|
||||
keys, to_delete_keys = all_keys[0 : (2 * key_count // 3)], all_keys[(2 * key_count // 3) :]
|
||||
|
||||
for i, key in enumerate(keys):
|
||||
index[key] = (i, i)
|
||||
@ -286,6 +287,7 @@ def merge(refcount1, refcount2):
|
||||
idx1.merge(idx2)
|
||||
refcount, *_ = idx1[H(1)]
|
||||
return refcount
|
||||
|
||||
result = merge(refcounta, refcountb)
|
||||
# check for commutativity
|
||||
assert result == merge(refcountb, refcounta)
|
||||
@ -367,22 +369,24 @@ def test_keyerror(self):
|
||||
|
||||
class HashIndexDataTestCase(BaseTestCase):
|
||||
# This bytestring was created with borg2-pre 2022-06-10
|
||||
HASHINDEX = b'eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4' \
|
||||
b'juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4' \
|
||||
b'juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO' \
|
||||
b'4ziO4ziO4ziO4ziO4ziO437LHbSVHGw='
|
||||
HASHINDEX = (
|
||||
b"eJzt0LEJg1AYhdE/JqBjOEJMNhBBrAQrO9ewc+HsoG+CPMsEz1cfbnHbceqXoZvvEVE+IuoqMu2pnOE4"
|
||||
b"juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4juM4"
|
||||
b"juM4juM4juM4jruie36vuSVT5N0rzW0n9t7r5z9+4TiO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO4ziO"
|
||||
b"4ziO4ziO4ziO4ziO4ziO437LHbSVHGw="
|
||||
)
|
||||
|
||||
def _serialize_hashindex(self, idx):
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
file = os.path.join(tempdir, 'idx')
|
||||
file = os.path.join(tempdir, "idx")
|
||||
idx.write(file)
|
||||
with open(file, 'rb') as f:
|
||||
with open(file, "rb") as f:
|
||||
return self._pack(f.read())
|
||||
|
||||
def _deserialize_hashindex(self, bytestring):
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
file = os.path.join(tempdir, 'idx')
|
||||
with open(file, 'wb') as f:
|
||||
file = os.path.join(tempdir, "idx")
|
||||
with open(file, "wb") as f:
|
||||
f.write(self._unpack(bytestring))
|
||||
return ChunkIndex.read(file)
|
||||
|
||||
@ -416,19 +420,19 @@ def test_read_known_good(self):
|
||||
class HashIndexIntegrityTestCase(HashIndexDataTestCase):
|
||||
def write_integrity_checked_index(self, tempdir):
|
||||
idx = self._deserialize_hashindex(self.HASHINDEX)
|
||||
file = os.path.join(tempdir, 'idx')
|
||||
file = os.path.join(tempdir, "idx")
|
||||
with IntegrityCheckedFile(path=file, write=True) as fd:
|
||||
idx.write(fd)
|
||||
integrity_data = fd.integrity_data
|
||||
assert 'final' in integrity_data
|
||||
assert 'HashHeader' in integrity_data
|
||||
assert "final" in integrity_data
|
||||
assert "HashHeader" in integrity_data
|
||||
return file, integrity_data
|
||||
|
||||
def test_integrity_checked_file(self):
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
file, integrity_data = self.write_integrity_checked_index(tempdir)
|
||||
with open(file, 'r+b') as fd:
|
||||
fd.write(b'Foo')
|
||||
with open(file, "r+b") as fd:
|
||||
fd.write(b"Foo")
|
||||
with self.assert_raises(FileIntegrityError):
|
||||
with IntegrityCheckedFile(path=file, write=False, integrity_data=integrity_data) as fd:
|
||||
ChunkIndex.read(fd)
|
||||
@ -437,15 +441,15 @@ def test_integrity_checked_file(self):
|
||||
class HashIndexCompactTestCase(HashIndexDataTestCase):
|
||||
def index(self, num_entries, num_buckets):
|
||||
index_data = io.BytesIO()
|
||||
index_data.write(b'BORG_IDX')
|
||||
index_data.write(b"BORG_IDX")
|
||||
# num_entries
|
||||
index_data.write(num_entries.to_bytes(4, 'little'))
|
||||
index_data.write(num_entries.to_bytes(4, "little"))
|
||||
# num_buckets
|
||||
index_data.write(num_buckets.to_bytes(4, 'little'))
|
||||
index_data.write(num_buckets.to_bytes(4, "little"))
|
||||
# key_size
|
||||
index_data.write((32).to_bytes(1, 'little'))
|
||||
index_data.write((32).to_bytes(1, "little"))
|
||||
# value_size
|
||||
index_data.write((3 * 4).to_bytes(1, 'little'))
|
||||
index_data.write((3 * 4).to_bytes(1, "little"))
|
||||
|
||||
self.index_data = index_data
|
||||
|
||||
@ -468,13 +472,13 @@ def index_from_data_compact_to_data(self):
|
||||
def write_entry(self, key, *values):
|
||||
self.index_data.write(key)
|
||||
for value in values:
|
||||
self.index_data.write(value.to_bytes(4, 'little'))
|
||||
self.index_data.write(value.to_bytes(4, "little"))
|
||||
|
||||
def write_empty(self, key):
|
||||
self.write_entry(key, 0xffffffff, 0, 0)
|
||||
self.write_entry(key, 0xFFFFFFFF, 0, 0)
|
||||
|
||||
def write_deleted(self, key):
|
||||
self.write_entry(key, 0xfffffffe, 0, 0)
|
||||
self.write_entry(key, 0xFFFFFFFE, 0, 0)
|
||||
|
||||
def test_simple(self):
|
||||
self.index(num_entries=3, num_buckets=6)
|
||||
@ -600,7 +604,7 @@ def HH(x, y, z):
|
||||
# first 4 bytes. giving a specific x targets bucket index x.
|
||||
# y is to create different keys and does not go into the bucket index calculation.
|
||||
# so, same x + different y --> collision
|
||||
return pack('<IIIIIIII', x, y, z, 0, 0, 0, 0, 0) # 8 * 4 == 32
|
||||
return pack("<IIIIIIII", x, y, z, 0, 0, 0, 0, 0) # 8 * 4 == 32
|
||||
|
||||
idx = NSIndex()
|
||||
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -11,20 +11,20 @@ def test_item_empty():
|
||||
|
||||
assert item.as_dict() == {}
|
||||
|
||||
assert 'path' not in item
|
||||
assert "path" not in item
|
||||
with pytest.raises(ValueError):
|
||||
'invalid-key' in item
|
||||
"invalid-key" in item
|
||||
with pytest.raises(TypeError):
|
||||
b'path' in item
|
||||
b"path" in item
|
||||
with pytest.raises(TypeError):
|
||||
42 in item
|
||||
|
||||
assert item.get('mode') is None
|
||||
assert item.get('mode', 0o666) == 0o666
|
||||
assert item.get("mode") is None
|
||||
assert item.get("mode", 0o666) == 0o666
|
||||
with pytest.raises(ValueError):
|
||||
item.get('invalid-key')
|
||||
item.get("invalid-key")
|
||||
with pytest.raises(TypeError):
|
||||
item.get(b'mode')
|
||||
item.get(b"mode")
|
||||
with pytest.raises(TypeError):
|
||||
item.get(42)
|
||||
|
||||
@ -37,16 +37,16 @@ def test_item_empty():
|
||||
|
||||
def test_item_from_dict():
|
||||
# does not matter whether we get str or bytes keys
|
||||
item = Item({b'path': '/a/b/c', b'mode': 0o666})
|
||||
assert item.path == '/a/b/c'
|
||||
item = Item({b"path": "/a/b/c", b"mode": 0o666})
|
||||
assert item.path == "/a/b/c"
|
||||
assert item.mode == 0o666
|
||||
assert 'path' in item
|
||||
assert "path" in item
|
||||
|
||||
# does not matter whether we get str or bytes keys
|
||||
item = Item({'path': '/a/b/c', 'mode': 0o666})
|
||||
assert item.path == '/a/b/c'
|
||||
item = Item({"path": "/a/b/c", "mode": 0o666})
|
||||
assert item.path == "/a/b/c"
|
||||
assert item.mode == 0o666
|
||||
assert 'mode' in item
|
||||
assert "mode" in item
|
||||
|
||||
# invalid - no dict
|
||||
with pytest.raises(TypeError):
|
||||
@ -58,12 +58,12 @@ def test_item_from_dict():
|
||||
|
||||
# invalid - unknown key
|
||||
with pytest.raises(ValueError):
|
||||
Item({'foobar': 'baz'})
|
||||
Item({"foobar": "baz"})
|
||||
|
||||
|
||||
def test_item_from_kw():
|
||||
item = Item(path='/a/b/c', mode=0o666)
|
||||
assert item.path == '/a/b/c'
|
||||
item = Item(path="/a/b/c", mode=0o666)
|
||||
assert item.path == "/a/b/c"
|
||||
assert item.mode == 0o666
|
||||
|
||||
|
||||
@ -71,7 +71,7 @@ def test_item_int_property():
|
||||
item = Item()
|
||||
item.mode = 0o666
|
||||
assert item.mode == 0o666
|
||||
assert item.as_dict() == {'mode': 0o666}
|
||||
assert item.as_dict() == {"mode": 0o666}
|
||||
del item.mode
|
||||
assert item.as_dict() == {}
|
||||
with pytest.raises(TypeError):
|
||||
@ -80,34 +80,34 @@ def test_item_int_property():
|
||||
|
||||
def test_item_mptimestamp_property():
|
||||
item = Item()
|
||||
small, big = 42, 2 ** 65
|
||||
small, big = 42, 2**65
|
||||
item.atime = small
|
||||
assert item.atime == small
|
||||
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(small)}
|
||||
assert item.as_dict() == {"atime": Timestamp.from_unix_nano(small)}
|
||||
item.atime = big
|
||||
assert item.atime == big
|
||||
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(big)}
|
||||
assert item.as_dict() == {"atime": Timestamp.from_unix_nano(big)}
|
||||
|
||||
|
||||
def test_item_se_str_property():
|
||||
# start simple
|
||||
item = Item()
|
||||
item.path = '/a/b/c'
|
||||
assert item.path == '/a/b/c'
|
||||
assert item.as_dict() == {'path': '/a/b/c'}
|
||||
item.path = "/a/b/c"
|
||||
assert item.path == "/a/b/c"
|
||||
assert item.as_dict() == {"path": "/a/b/c"}
|
||||
del item.path
|
||||
assert item.as_dict() == {}
|
||||
with pytest.raises(TypeError):
|
||||
item.path = 42
|
||||
|
||||
# non-utf-8 path, needing surrogate-escaping for latin-1 u-umlaut
|
||||
item = Item(internal_dict={'path': b'/a/\xfc/c'})
|
||||
assert item.path == '/a/\udcfc/c' # getting a surrogate-escaped representation
|
||||
assert item.as_dict() == {'path': '/a/\udcfc/c'}
|
||||
item = Item(internal_dict={"path": b"/a/\xfc/c"})
|
||||
assert item.path == "/a/\udcfc/c" # getting a surrogate-escaped representation
|
||||
assert item.as_dict() == {"path": "/a/\udcfc/c"}
|
||||
del item.path
|
||||
assert 'path' not in item
|
||||
item.path = '/a/\udcfc/c' # setting using a surrogate-escaped representation
|
||||
assert item.as_dict() == {'path': '/a/\udcfc/c'}
|
||||
assert "path" not in item
|
||||
item.path = "/a/\udcfc/c" # setting using a surrogate-escaped representation
|
||||
assert item.as_dict() == {"path": "/a/\udcfc/c"}
|
||||
|
||||
|
||||
def test_item_list_property():
|
||||
@ -118,18 +118,18 @@ def test_item_list_property():
|
||||
assert item.chunks == [0]
|
||||
item.chunks.append(1)
|
||||
assert item.chunks == [0, 1]
|
||||
assert item.as_dict() == {'chunks': [0, 1]}
|
||||
assert item.as_dict() == {"chunks": [0, 1]}
|
||||
|
||||
|
||||
def test_item_dict_property():
|
||||
item = Item()
|
||||
item.xattrs = StableDict()
|
||||
assert item.xattrs == StableDict()
|
||||
item.xattrs['foo'] = 'bar'
|
||||
assert item.xattrs['foo'] == 'bar'
|
||||
item.xattrs['bar'] = 'baz'
|
||||
assert item.xattrs == StableDict({'foo': 'bar', 'bar': 'baz'})
|
||||
assert item.as_dict() == {'xattrs': {'foo': 'bar', 'bar': 'baz'}}
|
||||
item.xattrs["foo"] = "bar"
|
||||
assert item.xattrs["foo"] == "bar"
|
||||
item.xattrs["bar"] = "baz"
|
||||
assert item.xattrs == StableDict({"foo": "bar", "bar": "baz"})
|
||||
assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}}
|
||||
|
||||
|
||||
def test_unknown_property():
|
||||
@ -142,10 +142,7 @@ def test_unknown_property():
|
||||
|
||||
|
||||
def test_item_file_size():
|
||||
item = Item(mode=0o100666, chunks=[
|
||||
ChunkListEntry(size=1000, id=None),
|
||||
ChunkListEntry(size=2000, id=None),
|
||||
])
|
||||
item = Item(mode=0o100666, chunks=[ChunkListEntry(size=1000, id=None), ChunkListEntry(size=2000, id=None)])
|
||||
assert item.get_size() == 3000
|
||||
item.get_size(memorize=True)
|
||||
assert item.size == 3000
|
||||
|
@ -11,7 +11,13 @@
|
||||
from ..crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey
|
||||
from ..crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey
|
||||
from ..crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256
|
||||
from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError, UnsupportedKeyFormatError
|
||||
from ..crypto.key import (
|
||||
TAMRequiredError,
|
||||
TAMInvalid,
|
||||
TAMUnsupportedSuiteError,
|
||||
UnsupportedManifestError,
|
||||
UnsupportedKeyFormatError,
|
||||
)
|
||||
from ..crypto.key import identify_key
|
||||
from ..crypto.low_level import IntegrityError as IntegrityErrorBase
|
||||
from ..helpers import IntegrityError
|
||||
@ -36,11 +42,17 @@ class MockArgs:
|
||||
F84MsMMiqpbz4KVICeBZhfAaTPs4W7BC63qml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgLENQ
|
||||
2uVCoR7EnAoiRzn8J+orbojKtJlNCnQ31SSC8rendmVyc2lvbgE=""".strip()
|
||||
|
||||
keyfile2_cdata = unhexlify(re.sub(r'\W', '', """
|
||||
keyfile2_cdata = unhexlify(
|
||||
re.sub(
|
||||
r"\W",
|
||||
"",
|
||||
"""
|
||||
0055f161493fcfc16276e8c31493c4641e1eb19a79d0326fad0291e5a9c98e5933
|
||||
00000000000003e8d21eaf9b86c297a8cd56432e1915bb
|
||||
"""))
|
||||
keyfile2_id = unhexlify('c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314')
|
||||
""",
|
||||
)
|
||||
)
|
||||
keyfile2_id = unhexlify("c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314")
|
||||
|
||||
keyfile_blake2_key_file = """
|
||||
BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000
|
||||
@ -56,8 +68,9 @@ class MockArgs:
|
||||
UTHFJg343jqml0ZXJhdGlvbnPOAAGGoKRzYWx02gAgz3YaUZZ/s+UWywj97EY5b4KhtJYi
|
||||
qkPqtDDxs2j/T7+ndmVyc2lvbgE=""".strip()
|
||||
|
||||
keyfile_blake2_cdata = bytes.fromhex('04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448'
|
||||
'0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947')
|
||||
keyfile_blake2_cdata = bytes.fromhex(
|
||||
"04fdf9475cf2323c0ba7a99ddc011064f2e7d039f539f2e448" "0e6f5fc6ff9993d604040404040404098c8cee1c6db8c28947"
|
||||
)
|
||||
# Verified against b2sum. Entire string passed to BLAKE2, including the padded 64 byte key contained in
|
||||
# keyfile_blake2_key_file above is
|
||||
# 19280471de95185ec27ecb6fc9edbb4f4db26974c315ede1cd505fab4250ce7cd0d081ea66946c
|
||||
@ -65,33 +78,42 @@ class MockArgs:
|
||||
# 000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
# 00000000000000000000007061796c6f6164
|
||||
# p a y l o a d
|
||||
keyfile_blake2_id = bytes.fromhex('d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb')
|
||||
keyfile_blake2_id = bytes.fromhex("d8bc68e961c79f99be39061589e5179b2113cd9226e07b08ddd4a1fef7ce93fb")
|
||||
|
||||
@pytest.fixture
|
||||
def keys_dir(self, request, monkeypatch, tmpdir):
|
||||
monkeypatch.setenv('BORG_KEYS_DIR', str(tmpdir))
|
||||
monkeypatch.setenv("BORG_KEYS_DIR", str(tmpdir))
|
||||
return tmpdir
|
||||
|
||||
@pytest.fixture(params=(
|
||||
# not encrypted
|
||||
PlaintextKey,
|
||||
AuthenticatedKey, Blake2AuthenticatedKey,
|
||||
# legacy crypto
|
||||
KeyfileKey, Blake2KeyfileKey,
|
||||
RepoKey, Blake2RepoKey,
|
||||
# new crypto
|
||||
AESOCBKeyfileKey, AESOCBRepoKey,
|
||||
Blake2AESOCBKeyfileKey, Blake2AESOCBRepoKey,
|
||||
CHPOKeyfileKey, CHPORepoKey,
|
||||
Blake2CHPOKeyfileKey, Blake2CHPORepoKey,
|
||||
))
|
||||
@pytest.fixture(
|
||||
params=(
|
||||
# not encrypted
|
||||
PlaintextKey,
|
||||
AuthenticatedKey,
|
||||
Blake2AuthenticatedKey,
|
||||
# legacy crypto
|
||||
KeyfileKey,
|
||||
Blake2KeyfileKey,
|
||||
RepoKey,
|
||||
Blake2RepoKey,
|
||||
# new crypto
|
||||
AESOCBKeyfileKey,
|
||||
AESOCBRepoKey,
|
||||
Blake2AESOCBKeyfileKey,
|
||||
Blake2AESOCBRepoKey,
|
||||
CHPOKeyfileKey,
|
||||
CHPORepoKey,
|
||||
Blake2CHPOKeyfileKey,
|
||||
Blake2CHPORepoKey,
|
||||
)
|
||||
)
|
||||
def key(self, request, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
return request.param.create(self.MockRepository(), self.MockArgs())
|
||||
|
||||
class MockRepository:
|
||||
class _Location:
|
||||
raw = processed = '/some/place'
|
||||
raw = processed = "/some/place"
|
||||
|
||||
def canonical_path(self):
|
||||
return self.processed
|
||||
@ -114,16 +136,16 @@ def load_key(self):
|
||||
|
||||
def test_plaintext(self):
|
||||
key = PlaintextKey.create(None, None)
|
||||
chunk = b'foo'
|
||||
chunk = b"foo"
|
||||
id = key.id_hash(chunk)
|
||||
assert hexlify(id) == b'2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'
|
||||
assert hexlify(id) == b"2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"
|
||||
assert chunk == key.decrypt(id, key.encrypt(id, chunk))
|
||||
|
||||
def test_keyfile(self, monkeypatch, keys_dir):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = KeyfileKey.create(self.MockRepository(), self.MockArgs())
|
||||
assert key.cipher.next_iv() == 0
|
||||
chunk = b'ABC'
|
||||
chunk = b"ABC"
|
||||
id = key.id_hash(chunk)
|
||||
manifest = key.encrypt(id, chunk)
|
||||
assert key.cipher.extract_iv(manifest) == 0
|
||||
@ -137,18 +159,18 @@ def test_keyfile(self, monkeypatch, keys_dir):
|
||||
# Key data sanity check
|
||||
assert len({key2.id_key, key2.enc_key, key2.enc_hmac_key}) == 3
|
||||
assert key2.chunk_seed != 0
|
||||
chunk = b'foo'
|
||||
chunk = b"foo"
|
||||
id = key.id_hash(chunk)
|
||||
assert chunk == key2.decrypt(id, key.encrypt(id, chunk))
|
||||
|
||||
def test_keyfile_kfenv(self, tmpdir, monkeypatch):
|
||||
keyfile = tmpdir.join('keyfile')
|
||||
monkeypatch.setenv('BORG_KEY_FILE', str(keyfile))
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'testkf')
|
||||
keyfile = tmpdir.join("keyfile")
|
||||
monkeypatch.setenv("BORG_KEY_FILE", str(keyfile))
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "testkf")
|
||||
assert not keyfile.exists()
|
||||
key = CHPOKeyfileKey.create(self.MockRepository(), self.MockArgs())
|
||||
assert keyfile.exists()
|
||||
chunk = b'ABC'
|
||||
chunk = b"ABC"
|
||||
chunk_id = key.id_hash(chunk)
|
||||
chunk_cdata = key.encrypt(chunk_id, chunk)
|
||||
key = CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata)
|
||||
@ -158,27 +180,27 @@ def test_keyfile_kfenv(self, tmpdir, monkeypatch):
|
||||
CHPOKeyfileKey.detect(self.MockRepository(), chunk_cdata)
|
||||
|
||||
def test_keyfile2(self, monkeypatch, keys_dir):
|
||||
with keys_dir.join('keyfile').open('w') as fd:
|
||||
with keys_dir.join("keyfile").open("w") as fd:
|
||||
fd.write(self.keyfile2_key_file)
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
|
||||
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
|
||||
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload'
|
||||
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload"
|
||||
|
||||
def test_keyfile2_kfenv(self, tmpdir, monkeypatch):
|
||||
keyfile = tmpdir.join('keyfile')
|
||||
with keyfile.open('w') as fd:
|
||||
keyfile = tmpdir.join("keyfile")
|
||||
with keyfile.open("w") as fd:
|
||||
fd.write(self.keyfile2_key_file)
|
||||
monkeypatch.setenv('BORG_KEY_FILE', str(keyfile))
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
|
||||
monkeypatch.setenv("BORG_KEY_FILE", str(keyfile))
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
|
||||
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
|
||||
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b'payload'
|
||||
assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata) == b"payload"
|
||||
|
||||
def test_keyfile_blake2(self, monkeypatch, keys_dir):
|
||||
with keys_dir.join('keyfile').open('w') as fd:
|
||||
with keys_dir.join("keyfile").open("w") as fd:
|
||||
fd.write(self.keyfile_blake2_key_file)
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
|
||||
key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata)
|
||||
assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b'payload'
|
||||
assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata) == b"payload"
|
||||
|
||||
def _corrupt_byte(self, key, data, offset):
|
||||
data = bytearray(data)
|
||||
@ -186,12 +208,12 @@ def _corrupt_byte(self, key, data, offset):
|
||||
# will trigger an IntegrityError (does not happen while we stay within TYPES_ACCEPTABLE).
|
||||
data[offset] ^= 64
|
||||
with pytest.raises(IntegrityErrorBase):
|
||||
key.decrypt(b'', data)
|
||||
key.decrypt(b"", data)
|
||||
|
||||
def test_decrypt_integrity(self, monkeypatch, keys_dir):
|
||||
with keys_dir.join('keyfile').open('w') as fd:
|
||||
with keys_dir.join("keyfile").open("w") as fd:
|
||||
fd.write(self.keyfile2_key_file)
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "passphrase")
|
||||
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
|
||||
|
||||
data = self.keyfile2_cdata
|
||||
@ -206,7 +228,7 @@ def test_decrypt_integrity(self, monkeypatch, keys_dir):
|
||||
|
||||
def test_roundtrip(self, key):
|
||||
repository = key.repository
|
||||
plaintext = b'foo'
|
||||
plaintext = b"foo"
|
||||
id = key.id_hash(plaintext)
|
||||
encrypted = key.encrypt(id, plaintext)
|
||||
identified_key_class = identify_key(encrypted)
|
||||
@ -216,59 +238,59 @@ def test_roundtrip(self, key):
|
||||
assert decrypted == plaintext
|
||||
|
||||
def test_decrypt_decompress(self, key):
|
||||
plaintext = b'123456789'
|
||||
plaintext = b"123456789"
|
||||
id = key.id_hash(plaintext)
|
||||
encrypted = key.encrypt(id, plaintext)
|
||||
assert key.decrypt(id, encrypted, decompress=False) != plaintext
|
||||
assert key.decrypt(id, encrypted) == plaintext
|
||||
|
||||
def test_assert_id(self, key):
|
||||
plaintext = b'123456789'
|
||||
plaintext = b"123456789"
|
||||
id = key.id_hash(plaintext)
|
||||
key.assert_id(id, plaintext)
|
||||
id_changed = bytearray(id)
|
||||
id_changed[0] ^= 1
|
||||
with pytest.raises(IntegrityError):
|
||||
key.assert_id(id_changed, plaintext)
|
||||
plaintext_changed = plaintext + b'1'
|
||||
plaintext_changed = plaintext + b"1"
|
||||
with pytest.raises(IntegrityError):
|
||||
key.assert_id(id, plaintext_changed)
|
||||
|
||||
def test_authenticated_encrypt(self, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
|
||||
assert AuthenticatedKey.id_hash is ID_HMAC_SHA_256.id_hash
|
||||
assert len(key.id_key) == 32
|
||||
plaintext = b'123456789'
|
||||
plaintext = b"123456789"
|
||||
id = key.id_hash(plaintext)
|
||||
authenticated = key.encrypt(id, plaintext)
|
||||
# 0x07 is the key TYPE, \x00ff identifies no compression / unknown level.
|
||||
assert authenticated == b'\x07\x00\xff' + plaintext
|
||||
assert authenticated == b"\x07\x00\xff" + plaintext
|
||||
|
||||
def test_blake2_authenticated_encrypt(self, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = Blake2AuthenticatedKey.create(self.MockRepository(), self.MockArgs())
|
||||
assert Blake2AuthenticatedKey.id_hash is ID_BLAKE2b_256.id_hash
|
||||
assert len(key.id_key) == 128
|
||||
plaintext = b'123456789'
|
||||
plaintext = b"123456789"
|
||||
id = key.id_hash(plaintext)
|
||||
authenticated = key.encrypt(id, plaintext)
|
||||
# 0x06 is the key TYPE, 0x00ff identifies no compression / unknown level.
|
||||
assert authenticated == b'\x06\x00\xff' + plaintext
|
||||
assert authenticated == b"\x06\x00\xff" + plaintext
|
||||
|
||||
|
||||
class TestTAM:
|
||||
@pytest.fixture
|
||||
def key(self, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
return CHPOKeyfileKey.create(TestKey.MockRepository(), TestKey.MockArgs())
|
||||
|
||||
def test_unpack_future(self, key):
|
||||
blob = b'\xc1\xc1\xc1\xc1foobar'
|
||||
blob = b"\xc1\xc1\xc1\xc1foobar"
|
||||
with pytest.raises(UnsupportedManifestError):
|
||||
key.unpack_and_verify_manifest(blob)
|
||||
|
||||
blob = b'\xc1\xc1\xc1'
|
||||
blob = b"\xc1\xc1\xc1"
|
||||
with pytest.raises(msgpack.UnpackException):
|
||||
key.unpack_and_verify_manifest(blob)
|
||||
|
||||
@ -285,84 +307,66 @@ def test_missing(self, key):
|
||||
assert not verified
|
||||
|
||||
def test_unknown_type_when_required(self, key):
|
||||
blob = msgpack.packb({
|
||||
'tam': {
|
||||
'type': 'HMAC_VOLLBIT',
|
||||
},
|
||||
})
|
||||
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
|
||||
with pytest.raises(TAMUnsupportedSuiteError):
|
||||
key.unpack_and_verify_manifest(blob)
|
||||
|
||||
def test_unknown_type(self, key):
|
||||
blob = msgpack.packb({
|
||||
'tam': {
|
||||
'type': 'HMAC_VOLLBIT',
|
||||
},
|
||||
})
|
||||
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
|
||||
key.tam_required = False
|
||||
unpacked, verified = key.unpack_and_verify_manifest(blob)
|
||||
assert unpacked == {}
|
||||
assert not verified
|
||||
|
||||
@pytest.mark.parametrize('tam, exc', (
|
||||
({}, TAMUnsupportedSuiteError),
|
||||
({'type': b'\xff'}, TAMUnsupportedSuiteError),
|
||||
(None, TAMInvalid),
|
||||
(1234, TAMInvalid),
|
||||
))
|
||||
@pytest.mark.parametrize(
|
||||
"tam, exc",
|
||||
(
|
||||
({}, TAMUnsupportedSuiteError),
|
||||
({"type": b"\xff"}, TAMUnsupportedSuiteError),
|
||||
(None, TAMInvalid),
|
||||
(1234, TAMInvalid),
|
||||
),
|
||||
)
|
||||
def test_invalid(self, key, tam, exc):
|
||||
blob = msgpack.packb({
|
||||
'tam': tam,
|
||||
})
|
||||
blob = msgpack.packb({"tam": tam})
|
||||
with pytest.raises(exc):
|
||||
key.unpack_and_verify_manifest(blob)
|
||||
|
||||
@pytest.mark.parametrize('hmac, salt', (
|
||||
({}, bytes(64)),
|
||||
(bytes(64), {}),
|
||||
(None, bytes(64)),
|
||||
(bytes(64), None),
|
||||
))
|
||||
@pytest.mark.parametrize("hmac, salt", (({}, bytes(64)), (bytes(64), {}), (None, bytes(64)), (bytes(64), None)))
|
||||
def test_wrong_types(self, key, hmac, salt):
|
||||
data = {
|
||||
'tam': {
|
||||
'type': 'HKDF_HMAC_SHA512',
|
||||
'hmac': hmac,
|
||||
'salt': salt
|
||||
},
|
||||
}
|
||||
tam = data['tam']
|
||||
data = {"tam": {"type": "HKDF_HMAC_SHA512", "hmac": hmac, "salt": salt}}
|
||||
tam = data["tam"]
|
||||
if hmac is None:
|
||||
del tam['hmac']
|
||||
del tam["hmac"]
|
||||
if salt is None:
|
||||
del tam['salt']
|
||||
del tam["salt"]
|
||||
blob = msgpack.packb(data)
|
||||
with pytest.raises(TAMInvalid):
|
||||
key.unpack_and_verify_manifest(blob)
|
||||
|
||||
def test_round_trip(self, key):
|
||||
data = {'foo': 'bar'}
|
||||
data = {"foo": "bar"}
|
||||
blob = key.pack_and_authenticate_metadata(data)
|
||||
assert blob.startswith(b'\x82')
|
||||
assert blob.startswith(b"\x82")
|
||||
|
||||
unpacked = msgpack.unpackb(blob)
|
||||
assert unpacked['tam']['type'] == 'HKDF_HMAC_SHA512'
|
||||
assert unpacked["tam"]["type"] == "HKDF_HMAC_SHA512"
|
||||
|
||||
unpacked, verified = key.unpack_and_verify_manifest(blob)
|
||||
assert verified
|
||||
assert unpacked['foo'] == 'bar'
|
||||
assert 'tam' not in unpacked
|
||||
assert unpacked["foo"] == "bar"
|
||||
assert "tam" not in unpacked
|
||||
|
||||
@pytest.mark.parametrize('which', ('hmac', 'salt'))
|
||||
@pytest.mark.parametrize("which", ("hmac", "salt"))
|
||||
def test_tampered(self, key, which):
|
||||
data = {'foo': 'bar'}
|
||||
data = {"foo": "bar"}
|
||||
blob = key.pack_and_authenticate_metadata(data)
|
||||
assert blob.startswith(b'\x82')
|
||||
assert blob.startswith(b"\x82")
|
||||
|
||||
unpacked = msgpack.unpackb(blob, object_hook=StableDict)
|
||||
assert len(unpacked['tam'][which]) == 64
|
||||
unpacked['tam'][which] = unpacked['tam'][which][0:32] + bytes(32)
|
||||
assert len(unpacked['tam'][which]) == 64
|
||||
assert len(unpacked["tam"][which]) == 64
|
||||
unpacked["tam"][which] = unpacked["tam"][which][0:32] + bytes(32)
|
||||
assert len(unpacked["tam"][which]) == 64
|
||||
blob = msgpack.packb(unpacked)
|
||||
|
||||
with pytest.raises(TAMInvalid):
|
||||
@ -372,10 +376,7 @@ def test_tampered(self, key, which):
|
||||
def test_decrypt_key_file_unsupported_algorithm():
|
||||
"""We will add more algorithms in the future. We should raise a helpful error."""
|
||||
key = CHPOKeyfileKey(None)
|
||||
encrypted = msgpack.packb({
|
||||
'algorithm': 'THIS ALGORITHM IS NOT SUPPORTED',
|
||||
'version': 1,
|
||||
})
|
||||
encrypted = msgpack.packb({"algorithm": "THIS ALGORITHM IS NOT SUPPORTED", "version": 1})
|
||||
|
||||
with pytest.raises(UnsupportedKeyFormatError):
|
||||
key.decrypt_key_file(encrypted, "hello, pass phrase")
|
||||
@ -384,9 +385,7 @@ def test_decrypt_key_file_unsupported_algorithm():
|
||||
def test_decrypt_key_file_v2_is_unsupported():
|
||||
"""There may eventually be a version 2 of the format. For now we should raise a helpful error."""
|
||||
key = CHPOKeyfileKey(None)
|
||||
encrypted = msgpack.packb({
|
||||
'version': 2,
|
||||
})
|
||||
encrypted = msgpack.packb({"version": 2})
|
||||
|
||||
with pytest.raises(UnsupportedKeyFormatError):
|
||||
key.decrypt_key_file(encrypted, "hello, pass phrase")
|
||||
@ -394,16 +393,16 @@ def test_decrypt_key_file_v2_is_unsupported():
|
||||
|
||||
def test_key_file_roundtrip(monkeypatch):
|
||||
def to_dict(key):
|
||||
extract = 'repository_id', 'enc_key', 'enc_hmac_key', 'id_key', 'chunk_seed'
|
||||
extract = "repository_id", "enc_key", "enc_hmac_key", "id_key", "chunk_seed"
|
||||
return {a: getattr(key, a) for a in extract}
|
||||
|
||||
repository = MagicMock(id=b'repository_id')
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', "hello, pass phrase")
|
||||
repository = MagicMock(id=b"repository_id")
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "hello, pass phrase")
|
||||
|
||||
save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm='argon2'))
|
||||
save_me = AESOCBRepoKey.create(repository, args=MagicMock(key_algorithm="argon2"))
|
||||
saved = repository.save_key.call_args.args[0]
|
||||
repository.load_key.return_value = saved
|
||||
load_me = AESOCBRepoKey.detect(repository, manifest_data=None)
|
||||
|
||||
assert to_dict(load_me) == to_dict(save_me)
|
||||
assert msgpack.unpackb(a2b_base64(saved))['algorithm'] == KEY_ALGORITHMS['argon2']
|
||||
assert msgpack.unpackb(a2b_base64(saved))["algorithm"] == KEY_ALGORITHMS["argon2"]
|
||||
|
@ -6,8 +6,19 @@
|
||||
import pytest
|
||||
|
||||
from ..platform import get_process_id, process_alive
|
||||
from ..locking import TimeoutTimer, ExclusiveLock, Lock, LockRoster, \
|
||||
ADD, REMOVE, SHARED, EXCLUSIVE, LockTimeout, NotLocked, NotMyLock
|
||||
from ..locking import (
|
||||
TimeoutTimer,
|
||||
ExclusiveLock,
|
||||
Lock,
|
||||
LockRoster,
|
||||
ADD,
|
||||
REMOVE,
|
||||
SHARED,
|
||||
EXCLUSIVE,
|
||||
LockTimeout,
|
||||
NotLocked,
|
||||
NotMyLock,
|
||||
)
|
||||
|
||||
ID1 = "foo", 1, 1
|
||||
ID2 = "bar", 2, 2
|
||||
@ -45,7 +56,7 @@ def test_notimeout_sleep(self):
|
||||
|
||||
@pytest.fixture()
|
||||
def lockpath(tmpdir):
|
||||
return str(tmpdir.join('lock'))
|
||||
return str(tmpdir.join("lock"))
|
||||
|
||||
|
||||
class TestExclusiveLock:
|
||||
@ -67,7 +78,7 @@ def test_timeout(self, lockpath):
|
||||
def test_kill_stale(self, lockpath, free_pid):
|
||||
host, pid, tid = our_id = get_process_id()
|
||||
dead_id = host, free_pid, tid
|
||||
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
|
||||
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
|
||||
|
||||
dead_lock = ExclusiveLock(lockpath, id=dead_id).acquire()
|
||||
with ExclusiveLock(lockpath, id=our_id):
|
||||
@ -94,9 +105,7 @@ def test_migrate_lock(self, lockpath):
|
||||
assert old_unique_name != new_unique_name # locking filename is different now
|
||||
|
||||
def test_race_condition(self, lockpath):
|
||||
|
||||
class SynchronizedCounter:
|
||||
|
||||
def __init__(self, count=0):
|
||||
self.lock = ThreadingLock()
|
||||
self.count = count
|
||||
@ -126,44 +135,77 @@ def print_locked(msg):
|
||||
with print_lock:
|
||||
print(msg)
|
||||
|
||||
def acquire_release_loop(id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None):
|
||||
print_locked("Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s" % (thread_id, id, timeout, lockpath))
|
||||
def acquire_release_loop(
|
||||
id, timeout, thread_id, lock_owner_counter, exception_counter, print_lock, last_thread=None
|
||||
):
|
||||
print_locked(
|
||||
"Thread %2d: Starting acquire_release_loop(id=%s, timeout=%d); lockpath=%s"
|
||||
% (thread_id, id, timeout, lockpath)
|
||||
)
|
||||
timer = TimeoutTimer(timeout, -1).start()
|
||||
cycle = 0
|
||||
|
||||
while not timer.timed_out():
|
||||
cycle += 1
|
||||
try:
|
||||
with ExclusiveLock(lockpath, id=id, timeout=timeout/20, sleep=-1): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway.
|
||||
with ExclusiveLock(
|
||||
lockpath, id=id, timeout=timeout / 20, sleep=-1
|
||||
): # This timeout is only for not exceeding the given timeout by more than 5%. With sleep<0 it's constantly polling anyway.
|
||||
lock_owner_count = lock_owner_counter.incr()
|
||||
print_locked("Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently." % (thread_id, cycle, lock_owner_count))
|
||||
print_locked(
|
||||
"Thread %2d: Acquired the lock. It's my %d. loop cycle. I am the %d. who has the lock concurrently."
|
||||
% (thread_id, cycle, lock_owner_count)
|
||||
)
|
||||
time.sleep(0.005)
|
||||
lock_owner_count = lock_owner_counter.decr()
|
||||
print_locked("Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock." % (thread_id, cycle, lock_owner_count))
|
||||
print_locked(
|
||||
"Thread %2d: Releasing the lock, finishing my %d. loop cycle. Currently, %d colleagues still have the lock."
|
||||
% (thread_id, cycle, lock_owner_count)
|
||||
)
|
||||
except LockTimeout:
|
||||
print_locked("Thread %2d: Got LockTimeout, finishing my %d. loop cycle." % (thread_id, cycle))
|
||||
except:
|
||||
exception_count = exception_counter.incr()
|
||||
e = format_exc()
|
||||
print_locked("Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s" % (thread_id, cycle, exception_count, e))
|
||||
print_locked(
|
||||
"Thread %2d: Exception thrown, finishing my %d. loop cycle. It's the %d. exception seen until now: %s"
|
||||
% (thread_id, cycle, exception_count, e)
|
||||
)
|
||||
|
||||
print_locked("Thread %2d: Loop timed out--terminating after %d loop cycles." % (thread_id, cycle))
|
||||
if last_thread is not None: # joining its predecessor, if any
|
||||
last_thread.join()
|
||||
|
||||
print('')
|
||||
print("")
|
||||
lock_owner_counter = SynchronizedCounter()
|
||||
exception_counter = SynchronizedCounter()
|
||||
print_lock = ThreadingLock()
|
||||
thread = None
|
||||
for thread_id in range(RACE_TEST_NUM_THREADS):
|
||||
thread = Thread(target=acquire_release_loop, args=(('foo', thread_id, 0), RACE_TEST_DURATION, thread_id, lock_owner_counter, exception_counter, print_lock, thread))
|
||||
thread = Thread(
|
||||
target=acquire_release_loop,
|
||||
args=(
|
||||
("foo", thread_id, 0),
|
||||
RACE_TEST_DURATION,
|
||||
thread_id,
|
||||
lock_owner_counter,
|
||||
exception_counter,
|
||||
print_lock,
|
||||
thread,
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
thread.join() # joining the last thread
|
||||
|
||||
assert lock_owner_counter.maxvalue() > 0, 'Never gained the lock? Something went wrong here...'
|
||||
assert lock_owner_counter.maxvalue() <= 1, "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % (lock_owner_counter.maxvalue())
|
||||
assert exception_counter.value() == 0, "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value())
|
||||
assert lock_owner_counter.maxvalue() > 0, "Never gained the lock? Something went wrong here..."
|
||||
assert (
|
||||
lock_owner_counter.maxvalue() <= 1
|
||||
), "Maximal number of concurrent lock holders was %d. So exclusivity is broken." % (
|
||||
lock_owner_counter.maxvalue()
|
||||
)
|
||||
assert (
|
||||
exception_counter.value() == 0
|
||||
), "ExclusiveLock threw %d exceptions due to unclean concurrency handling." % (exception_counter.value())
|
||||
|
||||
|
||||
class TestLock:
|
||||
@ -228,7 +270,7 @@ def test_timeout(self, lockpath):
|
||||
def test_kill_stale(self, lockpath, free_pid):
|
||||
host, pid, tid = our_id = get_process_id()
|
||||
dead_id = host, free_pid, tid
|
||||
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
|
||||
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
|
||||
|
||||
dead_lock = Lock(lockpath, id=dead_id, exclusive=True).acquire()
|
||||
roster = dead_lock._roster
|
||||
@ -263,7 +305,7 @@ def test_migrate_lock(self, lockpath):
|
||||
|
||||
@pytest.fixture()
|
||||
def rosterpath(tmpdir):
|
||||
return str(tmpdir.join('roster'))
|
||||
return str(tmpdir.join("roster"))
|
||||
|
||||
|
||||
class TestLockRoster:
|
||||
@ -277,13 +319,13 @@ def test_modify_get(self, rosterpath):
|
||||
roster1 = LockRoster(rosterpath, id=ID1)
|
||||
assert roster1.get(SHARED) == set()
|
||||
roster1.modify(SHARED, ADD)
|
||||
assert roster1.get(SHARED) == {ID1, }
|
||||
assert roster1.get(SHARED) == {ID1}
|
||||
roster2 = LockRoster(rosterpath, id=ID2)
|
||||
roster2.modify(SHARED, ADD)
|
||||
assert roster2.get(SHARED) == {ID1, ID2, }
|
||||
assert roster2.get(SHARED) == {ID1, ID2}
|
||||
roster1 = LockRoster(rosterpath, id=ID1)
|
||||
roster1.modify(SHARED, REMOVE)
|
||||
assert roster1.get(SHARED) == {ID2, }
|
||||
assert roster1.get(SHARED) == {ID2}
|
||||
roster2 = LockRoster(rosterpath, id=ID2)
|
||||
roster2.modify(SHARED, REMOVE)
|
||||
assert roster2.get(SHARED) == set()
|
||||
@ -300,7 +342,7 @@ def test_kill_stale(self, rosterpath, free_pid):
|
||||
assert roster1.get(SHARED) == {dead_id}
|
||||
|
||||
# put a unknown-state remote process lock into roster
|
||||
cant_know_if_dead_id = 'foo.bar.example.net', 1, 2
|
||||
cant_know_if_dead_id = "foo.bar.example.net", 1, 2
|
||||
roster1 = LockRoster(rosterpath, id=cant_know_if_dead_id)
|
||||
roster1.kill_stale_locks = False
|
||||
assert roster1.get(SHARED) == {dead_id}
|
||||
|
@ -4,6 +4,7 @@
|
||||
import pytest
|
||||
|
||||
from ..logger import find_parent_module, create_logger, setup_logging
|
||||
|
||||
logger = create_logger()
|
||||
|
||||
|
||||
@ -11,27 +12,27 @@
|
||||
def io_logger():
|
||||
io = StringIO()
|
||||
handler = setup_logging(stream=io, env_var=None)
|
||||
handler.setFormatter(logging.Formatter('%(name)s: %(message)s'))
|
||||
handler.setFormatter(logging.Formatter("%(name)s: %(message)s"))
|
||||
logger.setLevel(logging.DEBUG)
|
||||
return io
|
||||
|
||||
|
||||
def test_setup_logging(io_logger):
|
||||
logger.info('hello world')
|
||||
logger.info("hello world")
|
||||
assert io_logger.getvalue() == "borg.testsuite.logger: hello world\n"
|
||||
|
||||
|
||||
def test_multiple_loggers(io_logger):
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info('hello world 1')
|
||||
logger.info("hello world 1")
|
||||
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\n"
|
||||
logger = logging.getLogger('borg.testsuite.logger')
|
||||
logger.info('hello world 2')
|
||||
logger = logging.getLogger("borg.testsuite.logger")
|
||||
logger.info("hello world 2")
|
||||
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 1\nborg.testsuite.logger: hello world 2\n"
|
||||
io_logger.truncate(0)
|
||||
io_logger.seek(0)
|
||||
logger = logging.getLogger('borg.testsuite.logger')
|
||||
logger.info('hello world 2')
|
||||
logger = logging.getLogger("borg.testsuite.logger")
|
||||
logger.info("hello world 2")
|
||||
assert io_logger.getvalue() == "borg.testsuite.logger: hello world 2\n"
|
||||
|
||||
|
||||
|
@ -6,33 +6,32 @@
|
||||
|
||||
|
||||
class TestLRUCache:
|
||||
|
||||
def test_lrucache(self):
|
||||
c = LRUCache(2, dispose=lambda _: None)
|
||||
assert len(c) == 0
|
||||
assert c.items() == set()
|
||||
for i, x in enumerate('abc'):
|
||||
for i, x in enumerate("abc"):
|
||||
c[x] = i
|
||||
assert len(c) == 2
|
||||
assert c.items() == {('b', 1), ('c', 2)}
|
||||
assert 'a' not in c
|
||||
assert 'b' in c
|
||||
assert c.items() == {("b", 1), ("c", 2)}
|
||||
assert "a" not in c
|
||||
assert "b" in c
|
||||
with pytest.raises(KeyError):
|
||||
c['a']
|
||||
assert c.get('a') is None
|
||||
assert c.get('a', 'foo') == 'foo'
|
||||
assert c['b'] == 1
|
||||
assert c.get('b') == 1
|
||||
assert c['c'] == 2
|
||||
c['d'] = 3
|
||||
c["a"]
|
||||
assert c.get("a") is None
|
||||
assert c.get("a", "foo") == "foo"
|
||||
assert c["b"] == 1
|
||||
assert c.get("b") == 1
|
||||
assert c["c"] == 2
|
||||
c["d"] = 3
|
||||
assert len(c) == 2
|
||||
assert c['c'] == 2
|
||||
assert c['d'] == 3
|
||||
del c['c']
|
||||
assert c["c"] == 2
|
||||
assert c["d"] == 3
|
||||
del c["c"]
|
||||
assert len(c) == 1
|
||||
with pytest.raises(KeyError):
|
||||
c['c']
|
||||
assert c['d'] == 3
|
||||
c["c"]
|
||||
assert c["d"] == 3
|
||||
c.clear()
|
||||
assert c.items() == set()
|
||||
|
||||
|
@ -4,15 +4,15 @@
|
||||
|
||||
|
||||
def test_inline():
|
||||
assert rst_to_text('*foo* and ``bar``.') == 'foo and bar.'
|
||||
assert rst_to_text("*foo* and ``bar``.") == "foo and bar."
|
||||
|
||||
|
||||
def test_inline_spread():
|
||||
assert rst_to_text('*foo and bar, thusly\nfoobar*.') == 'foo and bar, thusly\nfoobar.'
|
||||
assert rst_to_text("*foo and bar, thusly\nfoobar*.") == "foo and bar, thusly\nfoobar."
|
||||
|
||||
|
||||
def test_comment_inline():
|
||||
assert rst_to_text('Foo and Bar\n.. foo\nbar') == 'Foo and Bar\n.. foo\nbar'
|
||||
assert rst_to_text("Foo and Bar\n.. foo\nbar") == "Foo and Bar\n.. foo\nbar"
|
||||
|
||||
|
||||
def test_inline_escape():
|
||||
@ -20,21 +20,19 @@ def test_inline_escape():
|
||||
|
||||
|
||||
def test_comment():
|
||||
assert rst_to_text('Foo and Bar\n\n.. foo\nbar') == 'Foo and Bar\n\nbar'
|
||||
assert rst_to_text("Foo and Bar\n\n.. foo\nbar") == "Foo and Bar\n\nbar"
|
||||
|
||||
|
||||
def test_directive_note():
|
||||
assert rst_to_text('.. note::\n Note this and that') == 'Note:\n Note this and that'
|
||||
assert rst_to_text(".. note::\n Note this and that") == "Note:\n Note this and that"
|
||||
|
||||
|
||||
def test_ref():
|
||||
references = {
|
||||
'foo': 'baz'
|
||||
}
|
||||
assert rst_to_text('See :ref:`fo\no`.', references=references) == 'See baz.'
|
||||
references = {"foo": "baz"}
|
||||
assert rst_to_text("See :ref:`fo\no`.", references=references) == "See baz."
|
||||
|
||||
|
||||
def test_undefined_ref():
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
rst_to_text('See :ref:`foo`.')
|
||||
assert 'Undefined reference' in str(exc_info.value)
|
||||
rst_to_text("See :ref:`foo`.")
|
||||
assert "Undefined reference" in str(exc_info.value)
|
||||
|
@ -10,10 +10,9 @@
|
||||
|
||||
|
||||
class TestNonceManager:
|
||||
|
||||
class MockRepository:
|
||||
class _Location:
|
||||
orig = '/some/place'
|
||||
orig = "/some/place"
|
||||
|
||||
_location = _Location()
|
||||
id = bytes(32)
|
||||
@ -37,15 +36,15 @@ def setUp(self):
|
||||
self.repository = None
|
||||
|
||||
def cache_nonce(self):
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce')) as fd:
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd:
|
||||
return fd.read()
|
||||
|
||||
def set_cache_nonce(self, nonce):
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce'), "w") as fd:
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd:
|
||||
assert fd.write(nonce)
|
||||
|
||||
def test_empty_cache_and_old_server(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockOldRepository()
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
@ -55,7 +54,7 @@ def test_empty_cache_and_old_server(self, monkeypatch):
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_empty_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
@ -66,7 +65,7 @@ def test_empty_cache(self, monkeypatch):
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_empty_nonce(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = None
|
||||
@ -99,10 +98,10 @@ def test_empty_nonce(self, monkeypatch):
|
||||
next_nonce = manager.ensure_reservation(0x2043, 64)
|
||||
assert next_nonce == 0x2063
|
||||
assert self.cache_nonce() == "00000000000020c3"
|
||||
assert self.repository.next_free == 0x20c3
|
||||
assert self.repository.next_free == 0x20C3
|
||||
|
||||
def test_sync_nonce(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
@ -116,7 +115,7 @@ def test_sync_nonce(self, monkeypatch):
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_server_just_upgraded(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = None
|
||||
@ -130,7 +129,7 @@ def test_server_just_upgraded(self, monkeypatch):
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_transaction_abort_no_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
@ -143,7 +142,7 @@ def test_transaction_abort_no_cache(self, monkeypatch):
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_transaction_abort_old_server(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockOldRepository()
|
||||
self.set_cache_nonce("0000000000002000")
|
||||
@ -155,7 +154,7 @@ def test_transaction_abort_old_server(self, monkeypatch):
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_transaction_abort_on_other_client(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
@ -169,7 +168,7 @@ def test_transaction_abort_on_other_client(self, monkeypatch):
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_interleaved(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, 'NONCE_SPACE_RESERVATION', 0x20)
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
@ -192,7 +191,7 @@ def test_interleaved(self, monkeypatch):
|
||||
assert self.repository.next_free == 0x4000
|
||||
|
||||
# spans reservation boundary
|
||||
next_nonce = manager.ensure_reservation(0x201f, 21)
|
||||
next_nonce = manager.ensure_reservation(0x201F, 21)
|
||||
assert next_nonce == 0x4000
|
||||
assert self.cache_nonce() == "0000000000004035"
|
||||
assert self.repository.next_free == 0x4035
|
||||
|
@ -11,8 +11,7 @@
|
||||
|
||||
|
||||
def check_patterns(files, pattern, expected):
|
||||
"""Utility for testing patterns.
|
||||
"""
|
||||
"""Utility for testing patterns."""
|
||||
assert all([f == os.path.normpath(f) for f in files]), "Pattern matchers expect normalized input paths"
|
||||
|
||||
matched = [f for f in files if pattern.match(f)]
|
||||
@ -20,167 +19,291 @@ def check_patterns(files, pattern, expected):
|
||||
assert matched == (files if expected is None else expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/", []),
|
||||
("/home", ["home"]),
|
||||
("/home///", ["home"]),
|
||||
("/./home", ["home"]),
|
||||
("/home/user", ["home/user"]),
|
||||
("/home/user2", ["home/user2"]),
|
||||
("/home/user/.bashrc", ["home/user/.bashrc"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/", []),
|
||||
("/home", ["home"]),
|
||||
("/home///", ["home"]),
|
||||
("/./home", ["home"]),
|
||||
("/home/user", ["home/user"]),
|
||||
("/home/user2", ["home/user2"]),
|
||||
("/home/user/.bashrc", ["home/user/.bashrc"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_full(pattern, expected):
|
||||
files = ["home", "home/user", "home/user2", "home/user/.bashrc", ]
|
||||
files = ["home", "home/user", "home/user2", "home/user/.bashrc"]
|
||||
|
||||
check_patterns(files, PathFullPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", []),
|
||||
("relative", []),
|
||||
("relative/path/", ["relative/path"]),
|
||||
("relative/path", ["relative/path"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", []),
|
||||
("relative", []),
|
||||
("relative/path/", ["relative/path"]),
|
||||
("relative/path", ["relative/path"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_full_relative(pattern, expected):
|
||||
files = ["relative/path", "relative/path2", ]
|
||||
files = ["relative/path", "relative/path2"]
|
||||
|
||||
check_patterns(files, PathFullPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/", None),
|
||||
("/./", None),
|
||||
("", []),
|
||||
("/home/u", []),
|
||||
("/home/user", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc", ["etc/server/config", "etc/server/hosts"]),
|
||||
("///etc//////", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv", ["srv/messages", "srv/dmesg"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/", None),
|
||||
("/./", None),
|
||||
("", []),
|
||||
("/home/u", []),
|
||||
("/home/user", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc", ["etc/server/config", "etc/server/hosts"]),
|
||||
("///etc//////", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv", ["srv/messages", "srv/dmesg"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_prefix(pattern, expected):
|
||||
files = [
|
||||
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
]
|
||||
|
||||
check_patterns(files, PathPrefixPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", []),
|
||||
("foo", []),
|
||||
("relative", ["relative/path1", "relative/two"]),
|
||||
("more", ["more/relative"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", []),
|
||||
("foo", []),
|
||||
("relative", ["relative/path1", "relative/two"]),
|
||||
("more", ["more/relative"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_prefix_relative(pattern, expected):
|
||||
files = ["relative/path1", "relative/two", "more/relative"]
|
||||
|
||||
check_patterns(files, PathPrefixPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/*", None),
|
||||
("/./*", None),
|
||||
("*", None),
|
||||
("*/*",
|
||||
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("*///*",
|
||||
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("/home/u", []),
|
||||
("/home/*",
|
||||
["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html",
|
||||
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
|
||||
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv*", ["srv/messages", "srv/dmesg"]),
|
||||
("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("/*", None),
|
||||
("/./*", None),
|
||||
("*", None),
|
||||
(
|
||||
"*/*",
|
||||
[
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
(
|
||||
"*///*",
|
||||
[
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
("/home/u", []),
|
||||
(
|
||||
"/home/*",
|
||||
[
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("*/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
|
||||
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv*", ["srv/messages", "srv/dmesg"]),
|
||||
("/home/*/.thumbnails", ["home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_fnmatch(pattern, expected):
|
||||
files = [
|
||||
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"home/foo/.thumbnails", "home/foo/bar/.thumbnails",
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
]
|
||||
|
||||
check_patterns(files, FnmatchPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("*", None),
|
||||
("**/*", None),
|
||||
("/**/*", None),
|
||||
("/./*", None),
|
||||
("*/*",
|
||||
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("*///*",
|
||||
["etc/server/config", "etc/server/hosts", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv/messages", "srv/dmesg",
|
||||
"srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("/home/u", []),
|
||||
("/home/*",
|
||||
["home/user/.profile", "home/user/.bashrc", "home/user2/.profile", "home/user2/public_html/index.html",
|
||||
"home/foo/.thumbnails", "home/foo/bar/.thumbnails"]),
|
||||
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc/*/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/etc/**/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("*/.pr????e", []),
|
||||
("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
|
||||
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]),
|
||||
("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]),
|
||||
("/srv/*", ["srv/messages", "srv/dmesg"]),
|
||||
("/srv2/**", ["srv2", "srv2/blafasel"]),
|
||||
("/srv2/**/", ["srv2/blafasel"]),
|
||||
("/home/*/.thumbnails", ["home/foo/.thumbnails"]),
|
||||
("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("*", None),
|
||||
("**/*", None),
|
||||
("/**/*", None),
|
||||
("/./*", None),
|
||||
(
|
||||
"*/*",
|
||||
[
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"srv2/blafasel",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
(
|
||||
"*///*",
|
||||
[
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"srv2/blafasel",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
("/home/u", []),
|
||||
(
|
||||
"/home/*",
|
||||
[
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
("/home/user/*", ["home/user/.profile", "home/user/.bashrc"]),
|
||||
("/etc/*/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/etc/**/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/etc/**/*/*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("*/.pr????e", []),
|
||||
("**/.pr????e", ["home/user/.profile", "home/user2/.profile"]),
|
||||
("///etc//////*", ["etc/server/config", "etc/server/hosts"]),
|
||||
("/./home//..//home/user2/", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/./home//..//home/user2/**/*", ["home/user2/.profile", "home/user2/public_html/index.html"]),
|
||||
("/srv*/", ["srv/messages", "srv/dmesg", "srv2/blafasel"]),
|
||||
("/srv*", ["srv", "srv/messages", "srv/dmesg", "srv2", "srv2/blafasel"]),
|
||||
("/srv/*", ["srv/messages", "srv/dmesg"]),
|
||||
("/srv2/**", ["srv2", "srv2/blafasel"]),
|
||||
("/srv2/**/", ["srv2/blafasel"]),
|
||||
("/home/*/.thumbnails", ["home/foo/.thumbnails"]),
|
||||
("/home/*/*/.thumbnails", ["home/foo/bar/.thumbnails"]),
|
||||
],
|
||||
)
|
||||
def test_patterns_shell(pattern, expected):
|
||||
files = [
|
||||
"etc/server/config", "etc/server/hosts", "home", "home/user/.profile", "home/user/.bashrc",
|
||||
"home/user2/.profile", "home/user2/public_html/index.html", "srv", "srv/messages", "srv/dmesg",
|
||||
"srv2", "srv2/blafasel", "home/foo/.thumbnails", "home/foo/bar/.thumbnails",
|
||||
"etc/server/config",
|
||||
"etc/server/hosts",
|
||||
"home",
|
||||
"home/user/.profile",
|
||||
"home/user/.bashrc",
|
||||
"home/user2/.profile",
|
||||
"home/user2/public_html/index.html",
|
||||
"srv",
|
||||
"srv/messages",
|
||||
"srv/dmesg",
|
||||
"srv2",
|
||||
"srv2/blafasel",
|
||||
"home/foo/.thumbnails",
|
||||
"home/foo/bar/.thumbnails",
|
||||
]
|
||||
|
||||
check_patterns(files, ShellPattern(pattern), expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, expected", [
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", None),
|
||||
(".*", None),
|
||||
("^/", None),
|
||||
("^abc$", []),
|
||||
("^[^/]", []),
|
||||
("^(?!/srv|/foo|/opt)",
|
||||
["/home", "/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile",
|
||||
"/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", ]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, expected",
|
||||
[
|
||||
# "None" means all files, i.e. all match the given pattern
|
||||
("", None),
|
||||
(".*", None),
|
||||
("^/", None),
|
||||
("^abc$", []),
|
||||
("^[^/]", []),
|
||||
(
|
||||
"^(?!/srv|/foo|/opt)",
|
||||
[
|
||||
"/home",
|
||||
"/home/user/.profile",
|
||||
"/home/user/.bashrc",
|
||||
"/home/user2/.profile",
|
||||
"/home/user2/public_html/index.html",
|
||||
"/home/foo/.thumbnails",
|
||||
"/home/foo/bar/.thumbnails",
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_patterns_regex(pattern, expected):
|
||||
files = [
|
||||
'/srv/data', '/foo/bar', '/home',
|
||||
'/home/user/.profile', '/home/user/.bashrc',
|
||||
'/home/user2/.profile', '/home/user2/public_html/index.html',
|
||||
'/opt/log/messages.txt', '/opt/log/dmesg.txt',
|
||||
"/home/foo/.thumbnails", "/home/foo/bar/.thumbnails",
|
||||
"/srv/data",
|
||||
"/foo/bar",
|
||||
"/home",
|
||||
"/home/user/.profile",
|
||||
"/home/user/.bashrc",
|
||||
"/home/user2/.profile",
|
||||
"/home/user2/public_html/index.html",
|
||||
"/opt/log/messages.txt",
|
||||
"/opt/log/dmesg.txt",
|
||||
"/home/foo/.thumbnails",
|
||||
"/home/foo/bar/.thumbnails",
|
||||
]
|
||||
|
||||
obj = RegexPattern(pattern)
|
||||
@ -202,11 +325,12 @@ def use_normalized_unicode():
|
||||
|
||||
|
||||
def _make_test_patterns(pattern):
|
||||
return [PathPrefixPattern(pattern),
|
||||
FnmatchPattern(pattern),
|
||||
RegexPattern(f"^{pattern}/foo$"),
|
||||
ShellPattern(pattern),
|
||||
]
|
||||
return [
|
||||
PathPrefixPattern(pattern),
|
||||
FnmatchPattern(pattern),
|
||||
RegexPattern(f"^{pattern}/foo$"),
|
||||
ShellPattern(pattern),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", _make_test_patterns("b\N{LATIN SMALL LETTER A WITH ACUTE}"))
|
||||
@ -227,51 +351,63 @@ def test_invalid_unicode_pattern(pattern):
|
||||
assert pattern.match(str(b"ba\x80/foo", "latin1"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lines, expected", [
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], None),
|
||||
(["# Comment only"], None),
|
||||
(["*"], []),
|
||||
(["# Comment",
|
||||
"*/something00.txt",
|
||||
" *whitespace* ",
|
||||
# Whitespace before comment
|
||||
" #/ws*",
|
||||
# Empty line
|
||||
"",
|
||||
"# EOF"],
|
||||
["more/data", "home", " #/wsfoobar"]),
|
||||
([r"re:.*"], []),
|
||||
([r"re:\s"], ["data/something00.txt", "more/data", "home"]),
|
||||
([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]),
|
||||
(["", "", "",
|
||||
"# This is a test with mixed pattern styles",
|
||||
# Case-insensitive pattern
|
||||
r"re:(?i)BAR|ME$",
|
||||
"",
|
||||
"*whitespace*",
|
||||
"fm:*/something00*"],
|
||||
["more/data"]),
|
||||
([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]),
|
||||
([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]),
|
||||
(["pp:./"], None),
|
||||
# leading slash is removed
|
||||
(["pp:/"], []),
|
||||
(["pp:aaabbb"], None),
|
||||
(["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]),
|
||||
(["/nomatch", "/more/*"],
|
||||
['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']),
|
||||
# the order of exclude patterns shouldn't matter
|
||||
(["/more/*", "/nomatch"],
|
||||
['data/something00.txt', 'home', ' #/wsfoobar', '\tstart/whitespace', 'whitespace/end\t']),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"lines, expected",
|
||||
[
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], None),
|
||||
(["# Comment only"], None),
|
||||
(["*"], []),
|
||||
(
|
||||
[
|
||||
"# Comment",
|
||||
"*/something00.txt",
|
||||
" *whitespace* ",
|
||||
# Whitespace before comment
|
||||
" #/ws*",
|
||||
# Empty line
|
||||
"",
|
||||
"# EOF",
|
||||
],
|
||||
["more/data", "home", " #/wsfoobar"],
|
||||
),
|
||||
([r"re:.*"], []),
|
||||
([r"re:\s"], ["data/something00.txt", "more/data", "home"]),
|
||||
([r"re:(.)(\1)"], ["more/data", "home", "\tstart/whitespace", "whitespace/end\t"]),
|
||||
(
|
||||
[
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"# This is a test with mixed pattern styles",
|
||||
# Case-insensitive pattern
|
||||
r"re:(?i)BAR|ME$",
|
||||
"",
|
||||
"*whitespace*",
|
||||
"fm:*/something00*",
|
||||
],
|
||||
["more/data"],
|
||||
),
|
||||
([r" re:^\s "], ["data/something00.txt", "more/data", "home", "whitespace/end\t"]),
|
||||
([r" re:\s$ "], ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace"]),
|
||||
(["pp:./"], None),
|
||||
# leading slash is removed
|
||||
(["pp:/"], []),
|
||||
(["pp:aaabbb"], None),
|
||||
(["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["more/data", "home"]),
|
||||
(
|
||||
["/nomatch", "/more/*"],
|
||||
["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"],
|
||||
),
|
||||
# the order of exclude patterns shouldn't matter
|
||||
(
|
||||
["/more/*", "/nomatch"],
|
||||
["data/something00.txt", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_exclude_patterns_from_file(tmpdir, lines, expected):
|
||||
files = [
|
||||
'data/something00.txt', 'more/data', 'home',
|
||||
' #/wsfoobar',
|
||||
'\tstart/whitespace',
|
||||
'whitespace/end\t',
|
||||
]
|
||||
files = ["data/something00.txt", "more/data", "home", " #/wsfoobar", "\tstart/whitespace", "whitespace/end\t"]
|
||||
|
||||
def evaluate(filename):
|
||||
patterns = []
|
||||
@ -288,26 +424,26 @@ def evaluate(filename):
|
||||
assert evaluate(str(exclfile)) == (files if expected is None else expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], [], 0),
|
||||
(["# Comment only"], [], 0),
|
||||
(["- *"], [], 1),
|
||||
(["+fm:*/something00.txt",
|
||||
"-/data"], [], 2),
|
||||
(["R /"], ["/"], 0),
|
||||
(["R /",
|
||||
"# comment"], ["/"], 0),
|
||||
(["# comment",
|
||||
"- /data",
|
||||
"R /home"], ["/home"], 1),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"lines, expected_roots, expected_numpatterns",
|
||||
[
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], [], 0),
|
||||
(["# Comment only"], [], 0),
|
||||
(["- *"], [], 1),
|
||||
(["+fm:*/something00.txt", "-/data"], [], 2),
|
||||
(["R /"], ["/"], 0),
|
||||
(["R /", "# comment"], ["/"], 0),
|
||||
(["# comment", "- /data", "R /home"], ["/home"], 1),
|
||||
],
|
||||
)
|
||||
def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns):
|
||||
def evaluate(filename):
|
||||
roots = []
|
||||
inclexclpatterns = []
|
||||
load_pattern_file(open(filename), roots, inclexclpatterns)
|
||||
return roots, len(inclexclpatterns)
|
||||
|
||||
patternfile = tmpdir.join("patterns.txt")
|
||||
|
||||
with patternfile.open("wt") as fh:
|
||||
@ -344,10 +480,9 @@ def test_switch_patterns_style():
|
||||
assert isinstance(patterns[5].val, ShellPattern)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lines", [
|
||||
(["X /data"]), # illegal pattern type prefix
|
||||
(["/data"]), # need a pattern type prefix
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"lines", [(["X /data"]), (["/data"])] # illegal pattern type prefix # need a pattern type prefix
|
||||
)
|
||||
def test_load_invalid_patterns_from_file(tmpdir, lines):
|
||||
patternfile = tmpdir.join("patterns.txt")
|
||||
with patternfile.open("wt") as fh:
|
||||
@ -359,41 +494,47 @@ def test_load_invalid_patterns_from_file(tmpdir, lines):
|
||||
load_pattern_file(open(filename), roots, inclexclpatterns)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lines, expected", [
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], None),
|
||||
(["# Comment only"], None),
|
||||
(["- *"], []),
|
||||
# default match type is sh: for patterns -> * doesn't match a /
|
||||
(["-*/something0?.txt"],
|
||||
['data', 'data/subdir/something01.txt',
|
||||
'home', 'home/leo', 'home/leo/t', 'home/other']),
|
||||
(["-fm:*/something00.txt"],
|
||||
['data', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t', 'home/other']),
|
||||
(["-fm:*/something0?.txt"],
|
||||
["data", 'home', 'home/leo', 'home/leo/t', 'home/other']),
|
||||
(["+/*/something0?.txt",
|
||||
"-/data"],
|
||||
["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']),
|
||||
(["+fm:*/something00.txt",
|
||||
"-/data"],
|
||||
["data/something00.txt", 'home', 'home/leo', 'home/leo/t', 'home/other']),
|
||||
# include /home/leo and exclude the rest of /home:
|
||||
(["+/home/leo",
|
||||
"-/home/*"],
|
||||
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']),
|
||||
# wrong order, /home/leo is already excluded by -/home/*:
|
||||
(["-/home/*",
|
||||
"+/home/leo"],
|
||||
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home']),
|
||||
(["+fm:/home/leo",
|
||||
"-/home/"],
|
||||
['data', 'data/something00.txt', 'data/subdir/something01.txt', 'home', 'home/leo', 'home/leo/t']),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"lines, expected",
|
||||
[
|
||||
# "None" means all files, i.e. none excluded
|
||||
([], None),
|
||||
(["# Comment only"], None),
|
||||
(["- *"], []),
|
||||
# default match type is sh: for patterns -> * doesn't match a /
|
||||
(
|
||||
["-*/something0?.txt"],
|
||||
["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"],
|
||||
),
|
||||
(
|
||||
["-fm:*/something00.txt"],
|
||||
["data", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t", "home/other"],
|
||||
),
|
||||
(["-fm:*/something0?.txt"], ["data", "home", "home/leo", "home/leo/t", "home/other"]),
|
||||
(["+/*/something0?.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]),
|
||||
(["+fm:*/something00.txt", "-/data"], ["data/something00.txt", "home", "home/leo", "home/leo/t", "home/other"]),
|
||||
# include /home/leo and exclude the rest of /home:
|
||||
(
|
||||
["+/home/leo", "-/home/*"],
|
||||
["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"],
|
||||
),
|
||||
# wrong order, /home/leo is already excluded by -/home/*:
|
||||
(["-/home/*", "+/home/leo"], ["data", "data/something00.txt", "data/subdir/something01.txt", "home"]),
|
||||
(
|
||||
["+fm:/home/leo", "-/home/"],
|
||||
["data", "data/something00.txt", "data/subdir/something01.txt", "home", "home/leo", "home/leo/t"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_inclexcl_patterns_from_file(tmpdir, lines, expected):
|
||||
files = [
|
||||
'data', 'data/something00.txt', 'data/subdir/something01.txt',
|
||||
'home', 'home/leo', 'home/leo/t', 'home/other'
|
||||
"data",
|
||||
"data/something00.txt",
|
||||
"data/subdir/something01.txt",
|
||||
"home",
|
||||
"home/leo",
|
||||
"home/leo/t",
|
||||
"home/other",
|
||||
]
|
||||
|
||||
def evaluate(filename):
|
||||
@ -412,37 +553,35 @@ def evaluate(filename):
|
||||
assert evaluate(str(patternfile)) == (files if expected is None else expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern, cls", [
|
||||
("", FnmatchPattern),
|
||||
|
||||
# Default style
|
||||
("*", FnmatchPattern),
|
||||
("/data/*", FnmatchPattern),
|
||||
|
||||
# fnmatch style
|
||||
("fm:", FnmatchPattern),
|
||||
("fm:*", FnmatchPattern),
|
||||
("fm:/data/*", FnmatchPattern),
|
||||
("fm:fm:/data/*", FnmatchPattern),
|
||||
|
||||
# Regular expression
|
||||
("re:", RegexPattern),
|
||||
("re:.*", RegexPattern),
|
||||
("re:^/something/", RegexPattern),
|
||||
("re:re:^/something/", RegexPattern),
|
||||
|
||||
# Path prefix
|
||||
("pp:", PathPrefixPattern),
|
||||
("pp:/", PathPrefixPattern),
|
||||
("pp:/data/", PathPrefixPattern),
|
||||
("pp:pp:/data/", PathPrefixPattern),
|
||||
|
||||
# Shell-pattern style
|
||||
("sh:", ShellPattern),
|
||||
("sh:*", ShellPattern),
|
||||
("sh:/data/*", ShellPattern),
|
||||
("sh:sh:/data/*", ShellPattern),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"pattern, cls",
|
||||
[
|
||||
("", FnmatchPattern),
|
||||
# Default style
|
||||
("*", FnmatchPattern),
|
||||
("/data/*", FnmatchPattern),
|
||||
# fnmatch style
|
||||
("fm:", FnmatchPattern),
|
||||
("fm:*", FnmatchPattern),
|
||||
("fm:/data/*", FnmatchPattern),
|
||||
("fm:fm:/data/*", FnmatchPattern),
|
||||
# Regular expression
|
||||
("re:", RegexPattern),
|
||||
("re:.*", RegexPattern),
|
||||
("re:^/something/", RegexPattern),
|
||||
("re:re:^/something/", RegexPattern),
|
||||
# Path prefix
|
||||
("pp:", PathPrefixPattern),
|
||||
("pp:/", PathPrefixPattern),
|
||||
("pp:/data/", PathPrefixPattern),
|
||||
("pp:pp:/data/", PathPrefixPattern),
|
||||
# Shell-pattern style
|
||||
("sh:", ShellPattern),
|
||||
("sh:*", ShellPattern),
|
||||
("sh:/data/*", ShellPattern),
|
||||
("sh:sh:/data/*", ShellPattern),
|
||||
],
|
||||
)
|
||||
def test_parse_pattern(pattern, cls):
|
||||
assert isinstance(parse_pattern(pattern), cls)
|
||||
|
||||
|
@ -21,7 +21,9 @@
|
||||
group:9999:r--:9999
|
||||
mask::rw-
|
||||
other::r--
|
||||
""".strip().encode('ascii')
|
||||
""".strip().encode(
|
||||
"ascii"
|
||||
)
|
||||
|
||||
DEFAULT_ACL = """
|
||||
user::rw-
|
||||
@ -32,18 +34,21 @@
|
||||
group:8888:r--:8888
|
||||
mask::rw-
|
||||
other::r--
|
||||
""".strip().encode('ascii')
|
||||
""".strip().encode(
|
||||
"ascii"
|
||||
)
|
||||
|
||||
_acls_working = None
|
||||
|
||||
|
||||
def fakeroot_detected():
|
||||
return 'FAKEROOTKEY' in os.environ
|
||||
return "FAKEROOTKEY" in os.environ
|
||||
|
||||
|
||||
def user_exists(username):
|
||||
if not is_win32:
|
||||
import pwd
|
||||
|
||||
try:
|
||||
pwd.getpwnam(username)
|
||||
return True
|
||||
@ -55,25 +60,24 @@ def user_exists(username):
|
||||
@functools.lru_cache
|
||||
def are_acls_working():
|
||||
with unopened_tempfile() as filepath:
|
||||
open(filepath, 'w').close()
|
||||
open(filepath, "w").close()
|
||||
try:
|
||||
access = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n'
|
||||
acl = {'acl_access': access}
|
||||
access = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n"
|
||||
acl = {"acl_access": access}
|
||||
acl_set(filepath, acl)
|
||||
read_acl = {}
|
||||
acl_get(filepath, read_acl, os.stat(filepath))
|
||||
read_acl_access = read_acl.get('acl_access', None)
|
||||
if read_acl_access and b'user::rw-' in read_acl_access:
|
||||
read_acl_access = read_acl.get("acl_access", None)
|
||||
if read_acl_access and b"user::rw-" in read_acl_access:
|
||||
return True
|
||||
except PermissionError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test')
|
||||
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
|
||||
@unittest.skipUnless(sys.platform.startswith("linux"), "linux only test")
|
||||
@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot")
|
||||
class PlatformLinuxTestCase(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
@ -86,72 +90,80 @@ def get_acl(self, path, numeric_ids=False):
|
||||
return item
|
||||
|
||||
def set_acl(self, path, access=None, default=None, numeric_ids=False):
|
||||
item = {'acl_access': access, 'acl_default': default}
|
||||
item = {"acl_access": access, "acl_default": default}
|
||||
acl_set(path, item, numeric_ids=numeric_ids)
|
||||
|
||||
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
|
||||
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
|
||||
def test_access_acl(self):
|
||||
file = tempfile.NamedTemporaryFile()
|
||||
self.assert_equal(self.get_acl(file.name), {})
|
||||
self.set_acl(file.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=False)
|
||||
self.assert_in(b'user:root:rw-:0', self.get_acl(file.name)['acl_access'])
|
||||
self.assert_in(b'group:root:rw-:0', self.get_acl(file.name)['acl_access'])
|
||||
self.assert_in(b'user:0:rw-:0', self.get_acl(file.name, numeric_ids=True)['acl_access'])
|
||||
self.set_acl(
|
||||
file.name,
|
||||
access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n",
|
||||
numeric_ids=False,
|
||||
)
|
||||
self.assert_in(b"user:root:rw-:0", self.get_acl(file.name)["acl_access"])
|
||||
self.assert_in(b"group:root:rw-:0", self.get_acl(file.name)["acl_access"])
|
||||
self.assert_in(b"user:0:rw-:0", self.get_acl(file.name, numeric_ids=True)["acl_access"])
|
||||
file2 = tempfile.NamedTemporaryFile()
|
||||
self.set_acl(file2.name, access=b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n', numeric_ids=True)
|
||||
self.assert_in(b'user:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
|
||||
self.assert_in(b'group:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
|
||||
self.set_acl(
|
||||
file2.name,
|
||||
access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n",
|
||||
numeric_ids=True,
|
||||
)
|
||||
self.assert_in(b"user:9999:rw-:9999", self.get_acl(file2.name)["acl_access"])
|
||||
self.assert_in(b"group:9999:rw-:9999", self.get_acl(file2.name)["acl_access"])
|
||||
|
||||
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
|
||||
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
|
||||
def test_default_acl(self):
|
||||
self.assert_equal(self.get_acl(self.tmpdir), {})
|
||||
self.set_acl(self.tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL)
|
||||
self.assert_equal(self.get_acl(self.tmpdir)['acl_access'], ACCESS_ACL)
|
||||
self.assert_equal(self.get_acl(self.tmpdir)['acl_default'], DEFAULT_ACL)
|
||||
self.assert_equal(self.get_acl(self.tmpdir)["acl_access"], ACCESS_ACL)
|
||||
self.assert_equal(self.get_acl(self.tmpdir)["acl_default"], DEFAULT_ACL)
|
||||
|
||||
@unittest.skipIf(not user_exists('übel'), 'requires übel user')
|
||||
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
|
||||
@unittest.skipIf(not user_exists("übel"), "requires übel user")
|
||||
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
|
||||
def test_non_ascii_acl(self):
|
||||
# Testing non-ascii ACL processing to see whether our code is robust.
|
||||
# I have no idea whether non-ascii ACLs are allowed by the standard,
|
||||
# but in practice they seem to be out there and must not make our code explode.
|
||||
file = tempfile.NamedTemporaryFile()
|
||||
self.assert_equal(self.get_acl(file.name), {})
|
||||
nothing_special = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\n'
|
||||
nothing_special = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\n"
|
||||
# TODO: can this be tested without having an existing system user übel with uid 666 gid 666?
|
||||
user_entry = 'user:übel:rw-:666'.encode()
|
||||
user_entry_numeric = b'user:666:rw-:666'
|
||||
group_entry = 'group:übel:rw-:666'.encode()
|
||||
group_entry_numeric = b'group:666:rw-:666'
|
||||
acl = b'\n'.join([nothing_special, user_entry, group_entry])
|
||||
user_entry = "user:übel:rw-:666".encode()
|
||||
user_entry_numeric = b"user:666:rw-:666"
|
||||
group_entry = "group:übel:rw-:666".encode()
|
||||
group_entry_numeric = b"group:666:rw-:666"
|
||||
acl = b"\n".join([nothing_special, user_entry, group_entry])
|
||||
self.set_acl(file.name, access=acl, numeric_ids=False)
|
||||
acl_access = self.get_acl(file.name, numeric_ids=False)['acl_access']
|
||||
acl_access = self.get_acl(file.name, numeric_ids=False)["acl_access"]
|
||||
self.assert_in(user_entry, acl_access)
|
||||
self.assert_in(group_entry, acl_access)
|
||||
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access']
|
||||
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"]
|
||||
self.assert_in(user_entry_numeric, acl_access_numeric)
|
||||
self.assert_in(group_entry_numeric, acl_access_numeric)
|
||||
file2 = tempfile.NamedTemporaryFile()
|
||||
self.set_acl(file2.name, access=acl, numeric_ids=True)
|
||||
acl_access = self.get_acl(file2.name, numeric_ids=False)['acl_access']
|
||||
acl_access = self.get_acl(file2.name, numeric_ids=False)["acl_access"]
|
||||
self.assert_in(user_entry, acl_access)
|
||||
self.assert_in(group_entry, acl_access)
|
||||
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)['acl_access']
|
||||
acl_access_numeric = self.get_acl(file.name, numeric_ids=True)["acl_access"]
|
||||
self.assert_in(user_entry_numeric, acl_access_numeric)
|
||||
self.assert_in(group_entry_numeric, acl_access_numeric)
|
||||
|
||||
def test_utils(self):
|
||||
from ..platform.linux import acl_use_local_uid_gid
|
||||
self.assert_equal(acl_use_local_uid_gid(b'user:nonexistent1234:rw-:1234'), b'user:1234:rw-')
|
||||
self.assert_equal(acl_use_local_uid_gid(b'group:nonexistent1234:rw-:1234'), b'group:1234:rw-')
|
||||
self.assert_equal(acl_use_local_uid_gid(b'user:root:rw-:0'), b'user:0:rw-')
|
||||
self.assert_equal(acl_use_local_uid_gid(b'group:root:rw-:0'), b'group:0:rw-')
|
||||
|
||||
self.assert_equal(acl_use_local_uid_gid(b"user:nonexistent1234:rw-:1234"), b"user:1234:rw-")
|
||||
self.assert_equal(acl_use_local_uid_gid(b"group:nonexistent1234:rw-:1234"), b"group:1234:rw-")
|
||||
self.assert_equal(acl_use_local_uid_gid(b"user:root:rw-:0"), b"user:0:rw-")
|
||||
self.assert_equal(acl_use_local_uid_gid(b"group:root:rw-:0"), b"group:0:rw-")
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith('darwin'), 'OS X only test')
|
||||
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
|
||||
@unittest.skipUnless(sys.platform.startswith("darwin"), "OS X only test")
|
||||
@unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot")
|
||||
class PlatformDarwinTestCase(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
@ -164,25 +176,41 @@ def get_acl(self, path, numeric_ids=False):
|
||||
return item
|
||||
|
||||
def set_acl(self, path, acl, numeric_ids=False):
|
||||
item = {'acl_extended': acl}
|
||||
item = {"acl_extended": acl}
|
||||
acl_set(path, item, numeric_ids=numeric_ids)
|
||||
|
||||
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
|
||||
@unittest.skipIf(not are_acls_working(), "ACLs do not work")
|
||||
def test_access_acl(self):
|
||||
file = tempfile.NamedTemporaryFile()
|
||||
file2 = tempfile.NamedTemporaryFile()
|
||||
self.assert_equal(self.get_acl(file.name), {})
|
||||
self.set_acl(file.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=False)
|
||||
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read', self.get_acl(file.name)['acl_extended'])
|
||||
self.assert_in(b'user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read', self.get_acl(file.name)['acl_extended'])
|
||||
self.set_acl(file2.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_ids=True)
|
||||
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read', self.get_acl(file2.name)['acl_extended'])
|
||||
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read', self.get_acl(file2.name, numeric_ids=True)['acl_extended'])
|
||||
self.set_acl(
|
||||
file.name,
|
||||
b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n",
|
||||
numeric_ids=False,
|
||||
)
|
||||
self.assert_in(
|
||||
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000014:staff:20:allow:read", self.get_acl(file.name)["acl_extended"]
|
||||
)
|
||||
self.assert_in(
|
||||
b"user:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read", self.get_acl(file.name)["acl_extended"]
|
||||
)
|
||||
self.set_acl(
|
||||
file2.name,
|
||||
b"!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n",
|
||||
numeric_ids=True,
|
||||
)
|
||||
self.assert_in(
|
||||
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read", self.get_acl(file2.name)["acl_extended"]
|
||||
)
|
||||
self.assert_in(
|
||||
b"group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read",
|
||||
self.get_acl(file2.name, numeric_ids=True)["acl_extended"],
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith(('linux', 'freebsd', 'darwin')), 'POSIX only tests')
|
||||
@unittest.skipUnless(sys.platform.startswith(("linux", "freebsd", "darwin")), "POSIX only tests")
|
||||
class PlatformPosixTestCase(BaseTestCase):
|
||||
|
||||
def test_swidth_ascii(self):
|
||||
self.assert_equal(swidth("borg"), 4)
|
||||
|
||||
@ -197,7 +225,7 @@ def test_process_alive(free_pid):
|
||||
id = get_process_id()
|
||||
assert process_alive(*id)
|
||||
host, pid, tid = id
|
||||
assert process_alive(host + 'abc', pid, tid)
|
||||
assert process_alive(host + "abc", pid, tid)
|
||||
assert process_alive(host, pid, tid + 1)
|
||||
assert not process_alive(host, free_pid, tid)
|
||||
|
||||
|
@ -72,10 +72,10 @@ def test_write(self, monkeypatch):
|
||||
class TestRepositoryCache:
|
||||
@pytest.fixture
|
||||
def repository(self, tmpdir):
|
||||
self.repository_location = os.path.join(str(tmpdir), 'repository')
|
||||
self.repository_location = os.path.join(str(tmpdir), "repository")
|
||||
with Repository(self.repository_location, exclusive=True, create=True) as repository:
|
||||
repository.put(H(1), b'1234')
|
||||
repository.put(H(2), b'5678')
|
||||
repository.put(H(1), b"1234")
|
||||
repository.put(H(2), b"5678")
|
||||
repository.put(H(3), bytes(100))
|
||||
yield repository
|
||||
|
||||
@ -85,19 +85,19 @@ def cache(self, repository):
|
||||
|
||||
def test_simple(self, cache: RepositoryCache):
|
||||
# Single get()s are not cached, since they are used for unique objects like archives.
|
||||
assert cache.get(H(1)) == b'1234'
|
||||
assert cache.get(H(1)) == b"1234"
|
||||
assert cache.misses == 1
|
||||
assert cache.hits == 0
|
||||
|
||||
assert list(cache.get_many([H(1)])) == [b'1234']
|
||||
assert list(cache.get_many([H(1)])) == [b"1234"]
|
||||
assert cache.misses == 2
|
||||
assert cache.hits == 0
|
||||
|
||||
assert list(cache.get_many([H(1)])) == [b'1234']
|
||||
assert list(cache.get_many([H(1)])) == [b"1234"]
|
||||
assert cache.misses == 2
|
||||
assert cache.hits == 1
|
||||
|
||||
assert cache.get(H(1)) == b'1234'
|
||||
assert cache.get(H(1)) == b"1234"
|
||||
assert cache.misses == 2
|
||||
assert cache.hits == 2
|
||||
|
||||
@ -105,11 +105,11 @@ def test_backoff(self, cache: RepositoryCache):
|
||||
def query_size_limit():
|
||||
cache.size_limit = 0
|
||||
|
||||
assert list(cache.get_many([H(1), H(2)])) == [b'1234', b'5678']
|
||||
assert list(cache.get_many([H(1), H(2)])) == [b"1234", b"5678"]
|
||||
assert cache.misses == 2
|
||||
assert cache.evictions == 0
|
||||
iterator = cache.get_many([H(1), H(3), H(2)])
|
||||
assert next(iterator) == b'1234'
|
||||
assert next(iterator) == b"1234"
|
||||
|
||||
# Force cache to back off
|
||||
qsl = cache.query_size_limit
|
||||
@ -124,7 +124,7 @@ def query_size_limit():
|
||||
assert cache.slow_misses == 0
|
||||
# Since H(2) was in the cache when we called get_many(), but has
|
||||
# been evicted during iterating the generator, it will be a slow miss.
|
||||
assert next(iterator) == b'5678'
|
||||
assert next(iterator) == b"5678"
|
||||
assert cache.slow_misses == 1
|
||||
|
||||
def test_enospc(self, cache: RepositoryCache):
|
||||
@ -139,16 +139,16 @@ def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
def write(self, data):
|
||||
raise OSError(errno.ENOSPC, 'foo')
|
||||
raise OSError(errno.ENOSPC, "foo")
|
||||
|
||||
def truncate(self, n=None):
|
||||
pass
|
||||
|
||||
iterator = cache.get_many([H(1), H(2), H(3)])
|
||||
assert next(iterator) == b'1234'
|
||||
assert next(iterator) == b"1234"
|
||||
|
||||
with patch('builtins.open', enospc_open):
|
||||
assert next(iterator) == b'5678'
|
||||
with patch("builtins.open", enospc_open):
|
||||
assert next(iterator) == b"5678"
|
||||
assert cache.enospc == 1
|
||||
# We didn't patch query_size_limit which would set size_limit to some low
|
||||
# value, so nothing was actually evicted.
|
||||
@ -158,9 +158,9 @@ def truncate(self, n=None):
|
||||
|
||||
@pytest.fixture
|
||||
def key(self, repository, monkeypatch):
|
||||
monkeypatch.setenv('BORG_PASSPHRASE', 'test')
|
||||
monkeypatch.setenv("BORG_PASSPHRASE", "test")
|
||||
key = PlaintextKey.create(repository, TestKey.MockArgs())
|
||||
key.compressor = CompressionSpec('none').compressor
|
||||
key.compressor = CompressionSpec("none").compressor
|
||||
return key
|
||||
|
||||
def _put_encrypted_object(self, key, repository, data):
|
||||
@ -170,11 +170,11 @@ def _put_encrypted_object(self, key, repository, data):
|
||||
|
||||
@pytest.fixture
|
||||
def H1(self, key, repository):
|
||||
return self._put_encrypted_object(key, repository, b'1234')
|
||||
return self._put_encrypted_object(key, repository, b"1234")
|
||||
|
||||
@pytest.fixture
|
||||
def H2(self, key, repository):
|
||||
return self._put_encrypted_object(key, repository, b'5678')
|
||||
return self._put_encrypted_object(key, repository, b"5678")
|
||||
|
||||
@pytest.fixture
|
||||
def H3(self, key, repository):
|
||||
@ -188,14 +188,14 @@ def test_cache_corruption(self, decrypted_cache: RepositoryCache, H1, H2, H3):
|
||||
list(decrypted_cache.get_many([H1, H2, H3]))
|
||||
|
||||
iterator = decrypted_cache.get_many([H1, H2, H3])
|
||||
assert next(iterator) == (7, b'1234')
|
||||
assert next(iterator) == (7, b"1234")
|
||||
|
||||
with open(decrypted_cache.key_filename(H2), 'a+b') as fd:
|
||||
with open(decrypted_cache.key_filename(H2), "a+b") as fd:
|
||||
fd.seek(-1, io.SEEK_END)
|
||||
corrupted = (int.from_bytes(fd.read(), 'little') ^ 2).to_bytes(1, 'little')
|
||||
corrupted = (int.from_bytes(fd.read(), "little") ^ 2).to_bytes(1, "little")
|
||||
fd.seek(-1, io.SEEK_END)
|
||||
fd.write(corrupted)
|
||||
fd.truncate()
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
assert next(iterator) == (7, b'5678')
|
||||
assert next(iterator) == (7, b"5678")
|
||||
|
@ -29,7 +29,7 @@ class RepositoryTestCaseBase(BaseTestCase):
|
||||
def open(self, create=False, exclusive=UNSPECIFIED):
|
||||
if exclusive is UNSPECIFIED:
|
||||
exclusive = self.exclusive
|
||||
return Repository(os.path.join(self.tmppath, 'repository'), exclusive=exclusive, create=create)
|
||||
return Repository(os.path.join(self.tmppath, "repository"), exclusive=exclusive, create=create)
|
||||
|
||||
def setUp(self):
|
||||
self.tmppath = tempfile.mkdtemp()
|
||||
@ -46,19 +46,19 @@ def reopen(self, exclusive=UNSPECIFIED):
|
||||
self.repository = self.open(exclusive=exclusive)
|
||||
|
||||
def add_keys(self):
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.put(H(3), b'bar')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.put(H(1), b"bar")
|
||||
self.repository.put(H(3), b"bar")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.put(H(1), b'bar2')
|
||||
self.repository.put(H(2), b'boo')
|
||||
self.repository.put(H(1), b"bar2")
|
||||
self.repository.put(H(2), b"boo")
|
||||
self.repository.delete(H(3))
|
||||
|
||||
def repo_dump(self, label=None):
|
||||
label = label + ': ' if label is not None else ''
|
||||
label = label + ": " if label is not None else ""
|
||||
H_trans = {H(i): i for i in range(10)}
|
||||
H_trans[None] = -1 # key == None appears in commits
|
||||
tag_trans = {TAG_PUT2: 'put2', TAG_PUT: 'put', TAG_DELETE: 'del', TAG_COMMIT: 'comm'}
|
||||
tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"}
|
||||
for segment, fn in self.repository.io.segment_iterator():
|
||||
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
||||
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
|
||||
@ -66,12 +66,11 @@ def repo_dump(self, label=None):
|
||||
|
||||
|
||||
class RepositoryTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def test1(self):
|
||||
for x in range(100):
|
||||
self.repository.put(H(x), b'SOMEDATA')
|
||||
self.repository.put(H(x), b"SOMEDATA")
|
||||
key50 = H(50)
|
||||
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
|
||||
self.assert_equal(self.repository.get(key50), b"SOMEDATA")
|
||||
self.repository.delete(key50)
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
|
||||
self.repository.commit(compact=False)
|
||||
@ -81,59 +80,55 @@ def test1(self):
|
||||
for x in range(100):
|
||||
if x == 50:
|
||||
continue
|
||||
self.assert_equal(repository2.get(H(x)), b'SOMEDATA')
|
||||
self.assert_equal(repository2.get(H(x)), b"SOMEDATA")
|
||||
|
||||
def test2(self):
|
||||
"""Test multiple sequential transactions
|
||||
"""
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'foo')
|
||||
"""Test multiple sequential transactions"""
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.put(H(1), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.delete(H(0))
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.put(H(1), b"bar")
|
||||
self.repository.commit(compact=False)
|
||||
self.assert_equal(self.repository.get(H(1)), b'bar')
|
||||
self.assert_equal(self.repository.get(H(1)), b"bar")
|
||||
|
||||
def test_consistency(self):
|
||||
"""Test cache consistency
|
||||
"""
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
self.repository.put(H(0), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
self.repository.put(H(0), b'bar')
|
||||
self.assert_equal(self.repository.get(H(0)), b'bar')
|
||||
"""Test cache consistency"""
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo")
|
||||
self.repository.put(H(0), b"foo2")
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo2")
|
||||
self.repository.put(H(0), b"bar")
|
||||
self.assert_equal(self.repository.get(H(0)), b"bar")
|
||||
self.repository.delete(H(0))
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0)))
|
||||
|
||||
def test_consistency2(self):
|
||||
"""Test cache consistency2
|
||||
"""
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
"""Test cache consistency2"""
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.put(H(0), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
self.repository.put(H(0), b"foo2")
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo2")
|
||||
self.repository.rollback()
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo")
|
||||
|
||||
def test_overwrite_in_same_transaction(self):
|
||||
"""Test cache consistency2
|
||||
"""
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b'foo2')
|
||||
"""Test cache consistency2"""
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.put(H(0), b"foo2")
|
||||
self.repository.commit(compact=False)
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b"foo2")
|
||||
|
||||
def test_single_kind_transactions(self):
|
||||
# put
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.close()
|
||||
# replace
|
||||
self.repository = self.open()
|
||||
with self.repository:
|
||||
self.repository.put(H(0), b'bar')
|
||||
self.repository.put(H(0), b"bar")
|
||||
self.repository.commit(compact=False)
|
||||
# delete
|
||||
self.repository = self.open()
|
||||
@ -143,7 +138,7 @@ def test_single_kind_transactions(self):
|
||||
|
||||
def test_list(self):
|
||||
for x in range(100):
|
||||
self.repository.put(H(x), b'SOMEDATA')
|
||||
self.repository.put(H(x), b"SOMEDATA")
|
||||
self.repository.commit(compact=False)
|
||||
all = self.repository.list()
|
||||
self.assert_equal(len(all), 100)
|
||||
@ -157,7 +152,7 @@ def test_list(self):
|
||||
|
||||
def test_scan(self):
|
||||
for x in range(100):
|
||||
self.repository.put(H(x), b'SOMEDATA')
|
||||
self.repository.put(H(x), b"SOMEDATA")
|
||||
self.repository.commit(compact=False)
|
||||
all = self.repository.scan()
|
||||
assert len(all) == 100
|
||||
@ -173,11 +168,10 @@ def test_scan(self):
|
||||
assert all[x] == H(x)
|
||||
|
||||
def test_max_data_size(self):
|
||||
max_data = b'x' * MAX_DATA_SIZE
|
||||
max_data = b"x" * MAX_DATA_SIZE
|
||||
self.repository.put(H(0), max_data)
|
||||
self.assert_equal(self.repository.get(H(0)), max_data)
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: self.repository.put(H(1), max_data + b'x'))
|
||||
self.assert_raises(IntegrityError, lambda: self.repository.put(H(1), max_data + b"x"))
|
||||
|
||||
|
||||
class LocalRepositoryTestCase(RepositoryTestCaseBase):
|
||||
@ -194,21 +188,21 @@ def _assert_sparse(self):
|
||||
assert self.repository.compact[0] == 41 + 8 + 9
|
||||
|
||||
def test_sparse1(self):
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'123456789')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.put(H(1), b"123456789")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.put(H(1), b"bar")
|
||||
self._assert_sparse()
|
||||
|
||||
def test_sparse2(self):
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'123456789')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.put(H(1), b"123456789")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.delete(H(1))
|
||||
self._assert_sparse()
|
||||
|
||||
def test_sparse_delete(self):
|
||||
self.repository.put(H(0), b'1245')
|
||||
self.repository.put(H(0), b"1245")
|
||||
self.repository.delete(H(0))
|
||||
self.repository.io._write_fd.sync()
|
||||
|
||||
@ -224,27 +218,26 @@ def test_sparse_delete(self):
|
||||
def test_uncommitted_garbage(self):
|
||||
# uncommitted garbage should be no problem, it is cleaned up automatically.
|
||||
# we just have to be careful with invalidation of cached FDs in LoggedIO.
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
# write some crap to a uncommitted segment file
|
||||
last_segment = self.repository.io.get_latest_segment()
|
||||
with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f:
|
||||
f.write(MAGIC + b'crapcrapcrap')
|
||||
with open(self.repository.io.segment_filename(last_segment + 1), "wb") as f:
|
||||
f.write(MAGIC + b"crapcrapcrap")
|
||||
self.repository.close()
|
||||
# usually, opening the repo and starting a transaction should trigger a cleanup.
|
||||
self.repository = self.open()
|
||||
with self.repository:
|
||||
self.repository.put(H(0), b'bar') # this may trigger compact_segments()
|
||||
self.repository.put(H(0), b"bar") # this may trigger compact_segments()
|
||||
self.repository.commit(compact=True)
|
||||
# the point here is that nothing blows up with an exception.
|
||||
|
||||
|
||||
class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def test_replay_of_missing_index(self):
|
||||
self.add_keys()
|
||||
for name in os.listdir(self.repository.path):
|
||||
if name.startswith('index.'):
|
||||
if name.startswith("index."):
|
||||
os.unlink(os.path.join(self.repository.path, name))
|
||||
self.reopen()
|
||||
with self.repository:
|
||||
@ -278,9 +271,9 @@ def test_crash_before_write_index(self):
|
||||
def test_replay_lock_upgrade_old(self):
|
||||
self.add_keys()
|
||||
for name in os.listdir(self.repository.path):
|
||||
if name.startswith('index.'):
|
||||
if name.startswith("index."):
|
||||
os.unlink(os.path.join(self.repository.path, name))
|
||||
with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade:
|
||||
with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade:
|
||||
self.reopen(exclusive=None) # simulate old client that always does lock upgrades
|
||||
with self.repository:
|
||||
# the repo is only locked by a shared read lock, but to replay segments,
|
||||
@ -291,9 +284,9 @@ def test_replay_lock_upgrade_old(self):
|
||||
def test_replay_lock_upgrade(self):
|
||||
self.add_keys()
|
||||
for name in os.listdir(self.repository.path):
|
||||
if name.startswith('index.'):
|
||||
if name.startswith("index."):
|
||||
os.unlink(os.path.join(self.repository.path, name))
|
||||
with patch.object(Lock, 'upgrade', side_effect=LockFailed) as upgrade:
|
||||
with patch.object(Lock, "upgrade", side_effect=LockFailed) as upgrade:
|
||||
self.reopen(exclusive=False) # current client usually does not do lock upgrade, except for replay
|
||||
with self.repository:
|
||||
# the repo is only locked by a shared read lock, but to replay segments,
|
||||
@ -322,13 +315,13 @@ def test_ignores_commit_tag_in_data(self):
|
||||
assert not io.is_committed_segment(io.get_latest_segment())
|
||||
|
||||
def test_moved_deletes_are_tracked(self):
|
||||
self.repository.put(H(1), b'1')
|
||||
self.repository.put(H(2), b'2')
|
||||
self.repository.put(H(1), b"1")
|
||||
self.repository.put(H(2), b"2")
|
||||
self.repository.commit(compact=False)
|
||||
self.repo_dump('p1 p2 c')
|
||||
self.repo_dump("p1 p2 c")
|
||||
self.repository.delete(H(1))
|
||||
self.repository.commit(compact=True)
|
||||
self.repo_dump('d1 cc')
|
||||
self.repo_dump("d1 cc")
|
||||
last_segment = self.repository.io.get_latest_segment() - 1
|
||||
num_deletes = 0
|
||||
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
|
||||
@ -337,9 +330,9 @@ def test_moved_deletes_are_tracked(self):
|
||||
num_deletes += 1
|
||||
assert num_deletes == 1
|
||||
assert last_segment in self.repository.compact
|
||||
self.repository.put(H(3), b'3')
|
||||
self.repository.put(H(3), b"3")
|
||||
self.repository.commit(compact=True)
|
||||
self.repo_dump('p3 cc')
|
||||
self.repo_dump("p3 cc")
|
||||
assert last_segment not in self.repository.compact
|
||||
assert not self.repository.io.segment_exists(last_segment)
|
||||
for segment, _ in self.repository.io.segment_iterator():
|
||||
@ -352,7 +345,7 @@ def test_moved_deletes_are_tracked(self):
|
||||
|
||||
def test_shadowed_entries_are_preserved(self):
|
||||
get_latest_segment = self.repository.io.get_latest_segment
|
||||
self.repository.put(H(1), b'1')
|
||||
self.repository.put(H(1), b"1")
|
||||
# This is the segment with our original PUT of interest
|
||||
put_segment = get_latest_segment()
|
||||
self.repository.commit(compact=False)
|
||||
@ -360,7 +353,7 @@ def test_shadowed_entries_are_preserved(self):
|
||||
# We now delete H(1), and force this segment to not be compacted, which can happen
|
||||
# if it's not sparse enough (symbolized by H(2) here).
|
||||
self.repository.delete(H(1))
|
||||
self.repository.put(H(2), b'1')
|
||||
self.repository.put(H(2), b"1")
|
||||
delete_segment = get_latest_segment()
|
||||
|
||||
# We pretend these are mostly dense (not sparse) and won't be compacted
|
||||
@ -380,33 +373,33 @@ def test_shadowed_entries_are_preserved(self):
|
||||
# Basic case, since the index survived this must be ok
|
||||
assert H(1) not in self.repository
|
||||
# Nuke index, force replay
|
||||
os.unlink(os.path.join(self.repository.path, 'index.%d' % get_latest_segment()))
|
||||
os.unlink(os.path.join(self.repository.path, "index.%d" % get_latest_segment()))
|
||||
# Must not reappear
|
||||
assert H(1) not in self.repository
|
||||
|
||||
def test_shadow_index_rollback(self):
|
||||
self.repository.put(H(1), b'1')
|
||||
self.repository.put(H(1), b"1")
|
||||
self.repository.delete(H(1))
|
||||
assert self.repository.shadow_index[H(1)] == [0]
|
||||
self.repository.commit(compact=True)
|
||||
self.repo_dump('p1 d1 cc')
|
||||
self.repo_dump("p1 d1 cc")
|
||||
# note how an empty list means that nothing is shadowed for sure
|
||||
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
|
||||
self.repository.put(H(1), b'1')
|
||||
self.repository.put(H(1), b"1")
|
||||
self.repository.delete(H(1))
|
||||
self.repo_dump('p1 d1')
|
||||
self.repo_dump("p1 d1")
|
||||
# 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
|
||||
assert self.repository.shadow_index[H(1)] == [4]
|
||||
self.repository.rollback()
|
||||
self.repo_dump('r')
|
||||
self.repository.put(H(2), b'1')
|
||||
self.repo_dump("r")
|
||||
self.repository.put(H(2), b"1")
|
||||
# After the rollback segment 4 shouldn't be considered anymore
|
||||
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
|
||||
|
||||
|
||||
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
|
||||
def open(self, create=False):
|
||||
return Repository(os.path.join(self.tmppath, 'repository'), exclusive=True, create=create, append_only=True)
|
||||
return Repository(os.path.join(self.tmppath, "repository"), exclusive=True, create=create, append_only=True)
|
||||
|
||||
def test_destroy_append_only(self):
|
||||
# Can't destroy append only repo (via the API)
|
||||
@ -417,19 +410,20 @@ def test_destroy_append_only(self):
|
||||
def test_append_only(self):
|
||||
def segments_in_repository():
|
||||
return len(list(self.repository.io.segment_iterator()))
|
||||
self.repository.put(H(0), b'foo')
|
||||
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
|
||||
self.repository.append_only = False
|
||||
assert segments_in_repository() == 2
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=True)
|
||||
# normal: compact squashes the data together, only one segment
|
||||
assert segments_in_repository() == 2
|
||||
|
||||
self.repository.append_only = True
|
||||
assert segments_in_repository() == 2
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
# append only: does not compact, only new segments written
|
||||
assert segments_in_repository() == 4
|
||||
@ -438,12 +432,12 @@ def segments_in_repository():
|
||||
class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
|
||||
def test_additional_free_space(self):
|
||||
self.add_keys()
|
||||
self.repository.config.set('repository', 'additional_free_space', '1000T')
|
||||
self.repository.save_key(b'shortcut to save_config')
|
||||
self.repository.config.set("repository", "additional_free_space", "1000T")
|
||||
self.repository.save_key(b"shortcut to save_config")
|
||||
self.reopen()
|
||||
|
||||
with self.repository:
|
||||
self.repository.put(H(0), b'foobar')
|
||||
self.repository.put(H(0), b"foobar")
|
||||
with pytest.raises(Repository.InsufficientFreeSpaceError):
|
||||
self.repository.commit(compact=False)
|
||||
assert os.path.exists(self.repository.path)
|
||||
@ -469,7 +463,7 @@ def test_tracking(self):
|
||||
self.reopen()
|
||||
with self.repository:
|
||||
# Open new transaction; hints and thus quota data is not loaded unless needed.
|
||||
self.repository.put(H(3), b'')
|
||||
self.repository.put(H(3), b"")
|
||||
self.repository.delete(H(3))
|
||||
assert self.repository.storage_quota_use == 1234 + 5678 + 3 * (41 + 8) # we have not compacted yet
|
||||
self.repository.commit(compact=True)
|
||||
@ -478,11 +472,11 @@ def test_tracking(self):
|
||||
def test_exceed_quota(self):
|
||||
assert self.repository.storage_quota_use == 0
|
||||
self.repository.storage_quota = 80
|
||||
self.repository.put(H(1), b'')
|
||||
self.repository.put(H(1), b"")
|
||||
assert self.repository.storage_quota_use == 41 + 8
|
||||
self.repository.commit(compact=False)
|
||||
with pytest.raises(Repository.StorageQuotaExceeded):
|
||||
self.repository.put(H(2), b'')
|
||||
self.repository.put(H(2), b"")
|
||||
assert self.repository.storage_quota_use == (41 + 8) * 2
|
||||
with pytest.raises(Repository.StorageQuotaExceeded):
|
||||
self.repository.commit(compact=False)
|
||||
@ -491,8 +485,10 @@ def test_exceed_quota(self):
|
||||
with self.repository:
|
||||
self.repository.storage_quota = 150
|
||||
# Open new transaction; hints and thus quota data is not loaded unless needed.
|
||||
self.repository.put(H(1), b'')
|
||||
assert self.repository.storage_quota_use == (41 + 8) * 2 # we have 2 puts for H(1) here and not yet compacted.
|
||||
self.repository.put(H(1), b"")
|
||||
assert (
|
||||
self.repository.storage_quota_use == (41 + 8) * 2
|
||||
) # we have 2 puts for H(1) here and not yet compacted.
|
||||
self.repository.commit(compact=True)
|
||||
assert self.repository.storage_quota_use == 41 + 8 # now we have compacted.
|
||||
|
||||
@ -542,54 +538,54 @@ def test_commit_nonce_reservation(self):
|
||||
class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.close()
|
||||
|
||||
def do_commit(self):
|
||||
with self.repository:
|
||||
self.repository.put(H(0), b'fox')
|
||||
self.repository.put(H(0), b"fox")
|
||||
self.repository.commit(compact=False)
|
||||
|
||||
def test_corrupted_hints(self):
|
||||
with open(os.path.join(self.repository.path, 'hints.1'), 'ab') as fd:
|
||||
fd.write(b'123456789')
|
||||
with open(os.path.join(self.repository.path, "hints.1"), "ab") as fd:
|
||||
fd.write(b"123456789")
|
||||
self.do_commit()
|
||||
|
||||
def test_deleted_hints(self):
|
||||
os.unlink(os.path.join(self.repository.path, 'hints.1'))
|
||||
os.unlink(os.path.join(self.repository.path, "hints.1"))
|
||||
self.do_commit()
|
||||
|
||||
def test_deleted_index(self):
|
||||
os.unlink(os.path.join(self.repository.path, 'index.1'))
|
||||
os.unlink(os.path.join(self.repository.path, "index.1"))
|
||||
self.do_commit()
|
||||
|
||||
def test_unreadable_hints(self):
|
||||
hints = os.path.join(self.repository.path, 'hints.1')
|
||||
hints = os.path.join(self.repository.path, "hints.1")
|
||||
os.unlink(hints)
|
||||
os.mkdir(hints)
|
||||
with self.assert_raises(OSError):
|
||||
self.do_commit()
|
||||
|
||||
def test_index(self):
|
||||
with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd:
|
||||
fd.write(b'123456789')
|
||||
with open(os.path.join(self.repository.path, "index.1"), "wb") as fd:
|
||||
fd.write(b"123456789")
|
||||
self.do_commit()
|
||||
|
||||
def test_index_outside_transaction(self):
|
||||
with open(os.path.join(self.repository.path, 'index.1'), 'wb') as fd:
|
||||
fd.write(b'123456789')
|
||||
with open(os.path.join(self.repository.path, "index.1"), "wb") as fd:
|
||||
fd.write(b"123456789")
|
||||
with self.repository:
|
||||
assert len(self.repository) == 1
|
||||
|
||||
def _corrupt_index(self):
|
||||
# HashIndex is able to detect incorrect headers and file lengths,
|
||||
# but on its own it can't tell if the data is correct.
|
||||
index_path = os.path.join(self.repository.path, 'index.1')
|
||||
with open(index_path, 'r+b') as fd:
|
||||
index_path = os.path.join(self.repository.path, "index.1")
|
||||
with open(index_path, "r+b") as fd:
|
||||
index_data = fd.read()
|
||||
# Flip one bit in a key stored in the index
|
||||
corrupted_key = (int.from_bytes(H(0), 'little') ^ 1).to_bytes(32, 'little')
|
||||
corrupted_key = (int.from_bytes(H(0), "little") ^ 1).to_bytes(32, "little")
|
||||
corrupted_index_data = index_data.replace(H(0), corrupted_key)
|
||||
assert corrupted_index_data != index_data
|
||||
assert len(corrupted_index_data) == len(index_data)
|
||||
@ -604,11 +600,11 @@ def test_index_corrupted(self):
|
||||
# Data corruption is detected due to mismatching checksums
|
||||
# and fixed by rebuilding the index.
|
||||
assert len(self.repository) == 1
|
||||
assert self.repository.get(H(0)) == b'foo'
|
||||
assert self.repository.get(H(0)) == b"foo"
|
||||
|
||||
def test_index_corrupted_without_integrity(self):
|
||||
self._corrupt_index()
|
||||
integrity_path = os.path.join(self.repository.path, 'integrity.1')
|
||||
integrity_path = os.path.join(self.repository.path, "integrity.1")
|
||||
os.unlink(integrity_path)
|
||||
with self.repository:
|
||||
# Since the corrupted key is not noticed, the repository still thinks
|
||||
@ -619,7 +615,7 @@ def test_index_corrupted_without_integrity(self):
|
||||
self.repository.get(H(0))
|
||||
|
||||
def test_unreadable_index(self):
|
||||
index = os.path.join(self.repository.path, 'index.1')
|
||||
index = os.path.join(self.repository.path, "index.1")
|
||||
os.unlink(index)
|
||||
os.mkdir(index)
|
||||
with self.assert_raises(OSError):
|
||||
@ -627,36 +623,39 @@ def test_unreadable_index(self):
|
||||
|
||||
def test_unknown_integrity_version(self):
|
||||
# For now an unknown integrity data version is ignored and not an error.
|
||||
integrity_path = os.path.join(self.repository.path, 'integrity.1')
|
||||
with open(integrity_path, 'r+b') as fd:
|
||||
msgpack.pack({
|
||||
# Borg only understands version 2
|
||||
b'version': 4.7,
|
||||
}, fd)
|
||||
integrity_path = os.path.join(self.repository.path, "integrity.1")
|
||||
with open(integrity_path, "r+b") as fd:
|
||||
msgpack.pack(
|
||||
{
|
||||
# Borg only understands version 2
|
||||
b"version": 4.7
|
||||
},
|
||||
fd,
|
||||
)
|
||||
fd.truncate()
|
||||
with self.repository:
|
||||
# No issues accessing the repository
|
||||
assert len(self.repository) == 1
|
||||
assert self.repository.get(H(0)) == b'foo'
|
||||
assert self.repository.get(H(0)) == b"foo"
|
||||
|
||||
def _subtly_corrupted_hints_setup(self):
|
||||
with self.repository:
|
||||
self.repository.append_only = True
|
||||
assert len(self.repository) == 1
|
||||
assert self.repository.get(H(0)) == b'foo'
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.put(H(2), b'baz')
|
||||
assert self.repository.get(H(0)) == b"foo"
|
||||
self.repository.put(H(1), b"bar")
|
||||
self.repository.put(H(2), b"baz")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.put(H(2), b'bazz')
|
||||
self.repository.put(H(2), b"bazz")
|
||||
self.repository.commit(compact=False)
|
||||
|
||||
hints_path = os.path.join(self.repository.path, 'hints.5')
|
||||
with open(hints_path, 'r+b') as fd:
|
||||
hints_path = os.path.join(self.repository.path, "hints.5")
|
||||
with open(hints_path, "r+b") as fd:
|
||||
hints = msgpack.unpack(fd)
|
||||
fd.seek(0)
|
||||
# Corrupt segment refcount
|
||||
assert hints['segments'][2] == 1
|
||||
hints['segments'][2] = 0
|
||||
assert hints["segments"][2] == 1
|
||||
hints["segments"][2] = 0
|
||||
msgpack.pack(hints, fd)
|
||||
fd.truncate()
|
||||
|
||||
@ -664,37 +663,40 @@ def test_subtly_corrupted_hints(self):
|
||||
self._subtly_corrupted_hints_setup()
|
||||
with self.repository:
|
||||
self.repository.append_only = False
|
||||
self.repository.put(H(3), b'1234')
|
||||
self.repository.put(H(3), b"1234")
|
||||
# Do a compaction run. Succeeds, since the failed checksum prompted a rebuild of the index+hints.
|
||||
self.repository.commit(compact=True)
|
||||
|
||||
assert len(self.repository) == 4
|
||||
assert self.repository.get(H(0)) == b'foo'
|
||||
assert self.repository.get(H(1)) == b'bar'
|
||||
assert self.repository.get(H(2)) == b'bazz'
|
||||
assert self.repository.get(H(0)) == b"foo"
|
||||
assert self.repository.get(H(1)) == b"bar"
|
||||
assert self.repository.get(H(2)) == b"bazz"
|
||||
|
||||
def test_subtly_corrupted_hints_without_integrity(self):
|
||||
self._subtly_corrupted_hints_setup()
|
||||
integrity_path = os.path.join(self.repository.path, 'integrity.5')
|
||||
integrity_path = os.path.join(self.repository.path, "integrity.5")
|
||||
os.unlink(integrity_path)
|
||||
with self.repository:
|
||||
self.repository.append_only = False
|
||||
self.repository.put(H(3), b'1234')
|
||||
self.repository.put(H(3), b"1234")
|
||||
# Do a compaction run. Fails, since the corrupted refcount was not detected and leads to an assertion failure.
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
self.repository.commit(compact=True)
|
||||
assert 'Corrupted segment reference count' in str(exc_info.value)
|
||||
assert "Corrupted segment reference count" in str(exc_info.value)
|
||||
|
||||
|
||||
class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def list_indices(self):
|
||||
return [name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if name.startswith('index.')]
|
||||
return [name for name in os.listdir(os.path.join(self.tmppath, "repository")) if name.startswith("index.")]
|
||||
|
||||
def check(self, repair=False, status=True):
|
||||
self.assert_equal(self.repository.check(repair=repair), status)
|
||||
# Make sure no tmp files are left behind
|
||||
self.assert_equal([name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if 'tmp' in name], [], 'Found tmp files')
|
||||
self.assert_equal(
|
||||
[name for name in os.listdir(os.path.join(self.tmppath, "repository")) if "tmp" in name],
|
||||
[],
|
||||
"Found tmp files",
|
||||
)
|
||||
|
||||
def get_objects(self, *ids):
|
||||
for id_ in ids:
|
||||
@ -703,31 +705,35 @@ def get_objects(self, *ids):
|
||||
def add_objects(self, segments):
|
||||
for ids in segments:
|
||||
for id_ in ids:
|
||||
self.repository.put(H(id_), b'data')
|
||||
self.repository.put(H(id_), b"data")
|
||||
self.repository.commit(compact=False)
|
||||
|
||||
def get_head(self):
|
||||
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
|
||||
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, "repository", "data", "0")) if n.isdigit())[
|
||||
-1
|
||||
]
|
||||
|
||||
def open_index(self):
|
||||
return NSIndex.read(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'))
|
||||
return NSIndex.read(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"))
|
||||
|
||||
def corrupt_object(self, id_):
|
||||
idx = self.open_index()
|
||||
segment, offset, _ = idx[H(id_)]
|
||||
with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
|
||||
with open(os.path.join(self.tmppath, "repository", "data", "0", str(segment)), "r+b") as fd:
|
||||
fd.seek(offset)
|
||||
fd.write(b'BOOM')
|
||||
fd.write(b"BOOM")
|
||||
|
||||
def delete_segment(self, segment):
|
||||
os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)))
|
||||
os.unlink(os.path.join(self.tmppath, "repository", "data", "0", str(segment)))
|
||||
|
||||
def delete_index(self):
|
||||
os.unlink(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'))
|
||||
os.unlink(os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"))
|
||||
|
||||
def rename_index(self, new_name):
|
||||
os.rename(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'),
|
||||
os.path.join(self.tmppath, 'repository', new_name))
|
||||
os.rename(
|
||||
os.path.join(self.tmppath, "repository", f"index.{self.get_head()}"),
|
||||
os.path.join(self.tmppath, "repository", new_name),
|
||||
)
|
||||
|
||||
def list_objects(self):
|
||||
return {int(key) for key in self.repository.list()}
|
||||
@ -765,9 +771,9 @@ def test_repair_missing_commit_segment(self):
|
||||
|
||||
def test_repair_corrupted_commit_segment(self):
|
||||
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
||||
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '3'), 'r+b') as fd:
|
||||
with open(os.path.join(self.tmppath, "repository", "data", "0", "3"), "r+b") as fd:
|
||||
fd.seek(-1, os.SEEK_END)
|
||||
fd.write(b'X')
|
||||
fd.write(b"X")
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.get_objects(4))
|
||||
self.check(status=True)
|
||||
self.get_objects(3)
|
||||
@ -775,15 +781,15 @@ def test_repair_corrupted_commit_segment(self):
|
||||
|
||||
def test_repair_no_commits(self):
|
||||
self.add_objects([[1, 2, 3]])
|
||||
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd:
|
||||
with open(os.path.join(self.tmppath, "repository", "data", "0", "1"), "r+b") as fd:
|
||||
fd.seek(-1, os.SEEK_END)
|
||||
fd.write(b'X')
|
||||
fd.write(b"X")
|
||||
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
|
||||
self.check(status=False)
|
||||
self.check(status=False)
|
||||
self.assert_equal(self.list_indices(), ['index.1'])
|
||||
self.assert_equal(self.list_indices(), ["index.1"])
|
||||
self.check(repair=True, status=True)
|
||||
self.assert_equal(self.list_indices(), ['index.2'])
|
||||
self.assert_equal(self.list_indices(), ["index.2"])
|
||||
self.check(status=True)
|
||||
self.get_objects(3)
|
||||
self.assert_equal({1, 2, 3}, self.list_objects())
|
||||
@ -797,30 +803,29 @@ def test_repair_missing_index(self):
|
||||
|
||||
def test_repair_index_too_new(self):
|
||||
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
||||
self.assert_equal(self.list_indices(), ['index.3'])
|
||||
self.rename_index('index.100')
|
||||
self.assert_equal(self.list_indices(), ["index.3"])
|
||||
self.rename_index("index.100")
|
||||
self.check(status=True)
|
||||
self.assert_equal(self.list_indices(), ['index.3'])
|
||||
self.assert_equal(self.list_indices(), ["index.3"])
|
||||
self.get_objects(4)
|
||||
self.assert_equal({1, 2, 3, 4, 5, 6}, self.list_objects())
|
||||
|
||||
def test_crash_before_compact(self):
|
||||
self.repository.put(H(0), b'data')
|
||||
self.repository.put(H(0), b'data2')
|
||||
self.repository.put(H(0), b"data")
|
||||
self.repository.put(H(0), b"data2")
|
||||
# Simulate a crash before compact
|
||||
with patch.object(Repository, 'compact_segments') as compact:
|
||||
with patch.object(Repository, "compact_segments") as compact:
|
||||
self.repository.commit(compact=True)
|
||||
compact.assert_called_once_with(0.1)
|
||||
self.reopen()
|
||||
with self.repository:
|
||||
self.check(repair=True)
|
||||
self.assert_equal(self.repository.get(H(0)), b'data2')
|
||||
self.assert_equal(self.repository.get(H(0)), b"data2")
|
||||
|
||||
|
||||
class RepositoryHintsTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def test_hints_persistence(self):
|
||||
self.repository.put(H(0), b'data')
|
||||
self.repository.put(H(0), b"data")
|
||||
self.repository.delete(H(0))
|
||||
self.repository.commit(compact=False)
|
||||
shadow_index_expected = self.repository.shadow_index
|
||||
@ -831,7 +836,7 @@ def test_hints_persistence(self):
|
||||
self.reopen()
|
||||
with self.repository:
|
||||
# see also do_compact()
|
||||
self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data
|
||||
self.repository.put(H(42), b"foobar") # this will call prepare_txn() and load the hints data
|
||||
# check if hints persistence worked:
|
||||
self.assert_equal(shadow_index_expected, self.repository.shadow_index)
|
||||
self.assert_equal(compact_expected, self.repository.compact)
|
||||
@ -839,7 +844,7 @@ def test_hints_persistence(self):
|
||||
self.assert_equal(segments_expected, self.repository.segments)
|
||||
|
||||
def test_hints_behaviour(self):
|
||||
self.repository.put(H(0), b'data')
|
||||
self.repository.put(H(0), b"data")
|
||||
self.assert_equal(self.repository.shadow_index, {})
|
||||
assert len(self.repository.compact) == 0
|
||||
self.repository.delete(H(0))
|
||||
@ -848,7 +853,7 @@ def test_hints_behaviour(self):
|
||||
self.assert_in(H(0), self.repository.shadow_index)
|
||||
self.assert_equal(len(self.repository.shadow_index[H(0)]), 1)
|
||||
self.assert_in(0, self.repository.compact) # segment 0 can be compacted
|
||||
self.repository.put(H(42), b'foobar') # see also do_compact()
|
||||
self.repository.put(H(42), b"foobar") # see also do_compact()
|
||||
self.repository.commit(compact=True, threshold=0.0) # compact completely!
|
||||
# nothing to compact any more! no info left about stuff that does not exist any more:
|
||||
self.assert_not_in(H(0), self.repository.shadow_index)
|
||||
@ -861,12 +866,13 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
|
||||
repository = None # type: RemoteRepository
|
||||
|
||||
def open(self, create=False):
|
||||
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
|
||||
exclusive=True, create=create)
|
||||
return RemoteRepository(
|
||||
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create
|
||||
)
|
||||
|
||||
def _get_mock_args(self):
|
||||
class MockArgs:
|
||||
remote_path = 'borg'
|
||||
remote_path = "borg"
|
||||
umask = 0o077
|
||||
debug_topics = []
|
||||
rsh = None
|
||||
@ -878,111 +884,122 @@ def __contains__(self, item):
|
||||
return MockArgs()
|
||||
|
||||
def test_invalid_rpc(self):
|
||||
self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', {}))
|
||||
self.assert_raises(InvalidRPCMethod, lambda: self.repository.call("__init__", {}))
|
||||
|
||||
def test_rpc_exception_transport(self):
|
||||
s1 = 'test string'
|
||||
s1 = "test string"
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'DoesNotExist'})
|
||||
self.repository.call("inject_exception", {"kind": "DoesNotExist"})
|
||||
except Repository.DoesNotExist as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == self.repository.location.processed
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'AlreadyExists'})
|
||||
self.repository.call("inject_exception", {"kind": "AlreadyExists"})
|
||||
except Repository.AlreadyExists as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == self.repository.location.processed
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'CheckNeeded'})
|
||||
self.repository.call("inject_exception", {"kind": "CheckNeeded"})
|
||||
except Repository.CheckNeeded as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == self.repository.location.processed
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'IntegrityError'})
|
||||
self.repository.call("inject_exception", {"kind": "IntegrityError"})
|
||||
except IntegrityError as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == s1
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'PathNotAllowed'})
|
||||
self.repository.call("inject_exception", {"kind": "PathNotAllowed"})
|
||||
except PathNotAllowed as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == 'foo'
|
||||
assert e.args[0] == "foo"
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'ObjectNotFound'})
|
||||
self.repository.call("inject_exception", {"kind": "ObjectNotFound"})
|
||||
except Repository.ObjectNotFound as e:
|
||||
assert len(e.args) == 2
|
||||
assert e.args[0] == s1
|
||||
assert e.args[1] == self.repository.location.processed
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'InvalidRPCMethod'})
|
||||
self.repository.call("inject_exception", {"kind": "InvalidRPCMethod"})
|
||||
except InvalidRPCMethod as e:
|
||||
assert len(e.args) == 1
|
||||
assert e.args[0] == s1
|
||||
|
||||
try:
|
||||
self.repository.call('inject_exception', {'kind': 'divide'})
|
||||
self.repository.call("inject_exception", {"kind": "divide"})
|
||||
except RemoteRepository.RPCError as e:
|
||||
assert e.unpacked
|
||||
assert e.get_message() == 'ZeroDivisionError: integer division or modulo by zero\n'
|
||||
assert e.exception_class == 'ZeroDivisionError'
|
||||
assert e.get_message() == "ZeroDivisionError: integer division or modulo by zero\n"
|
||||
assert e.exception_class == "ZeroDivisionError"
|
||||
assert len(e.exception_full) > 0
|
||||
|
||||
def test_ssh_cmd(self):
|
||||
args = self._get_mock_args()
|
||||
self.repository._args = args
|
||||
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com']
|
||||
assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com']
|
||||
assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com']
|
||||
os.environ['BORG_RSH'] = 'ssh --foo'
|
||||
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '--foo', 'example.com']
|
||||
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "example.com"]
|
||||
assert self.repository.ssh_cmd(Location("ssh://user@example.com/foo")) == ["ssh", "user@example.com"]
|
||||
assert self.repository.ssh_cmd(Location("ssh://user@example.com:1234/foo")) == [
|
||||
"ssh",
|
||||
"-p",
|
||||
"1234",
|
||||
"user@example.com",
|
||||
]
|
||||
os.environ["BORG_RSH"] = "ssh --foo"
|
||||
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "--foo", "example.com"]
|
||||
|
||||
def test_borg_cmd(self):
|
||||
assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve']
|
||||
assert self.repository.borg_cmd(None, testing=True) == [sys.executable, "-m", "borg.archiver", "serve"]
|
||||
args = self._get_mock_args()
|
||||
# XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
|
||||
logging.getLogger().setLevel(logging.INFO)
|
||||
# note: test logger is on info log level, so --info gets added automagically
|
||||
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info']
|
||||
args.remote_path = 'borg-0.28.2'
|
||||
assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info']
|
||||
args.debug_topics = ['something_client_side', 'repository_compaction']
|
||||
assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--info',
|
||||
'--debug-topic=borg.debug.repository_compaction']
|
||||
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
|
||||
args.remote_path = "borg-0.28.2"
|
||||
assert self.repository.borg_cmd(args, testing=False) == ["borg-0.28.2", "serve", "--info"]
|
||||
args.debug_topics = ["something_client_side", "repository_compaction"]
|
||||
assert self.repository.borg_cmd(args, testing=False) == [
|
||||
"borg-0.28.2",
|
||||
"serve",
|
||||
"--info",
|
||||
"--debug-topic=borg.debug.repository_compaction",
|
||||
]
|
||||
args = self._get_mock_args()
|
||||
args.storage_quota = 0
|
||||
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info']
|
||||
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
|
||||
args.storage_quota = 314159265
|
||||
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--info',
|
||||
'--storage-quota=314159265']
|
||||
args.rsh = 'ssh -i foo'
|
||||
assert self.repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info", "--storage-quota=314159265"]
|
||||
args.rsh = "ssh -i foo"
|
||||
self.repository._args = args
|
||||
assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', '-i', 'foo', 'example.com']
|
||||
assert self.repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "-i", "foo", "example.com"]
|
||||
|
||||
|
||||
class RemoteLegacyFree(RepositoryTestCaseBase):
|
||||
# Keep testing this so we can someday safely remove the legacy tuple format.
|
||||
|
||||
def open(self, create=False):
|
||||
with patch.object(RemoteRepository, 'dictFormat', True):
|
||||
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
|
||||
exclusive=True, create=create)
|
||||
with patch.object(RemoteRepository, "dictFormat", True):
|
||||
return RemoteRepository(
|
||||
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")),
|
||||
exclusive=True,
|
||||
create=create,
|
||||
)
|
||||
|
||||
def test_legacy_free(self):
|
||||
# put
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b"foo")
|
||||
self.repository.commit(compact=False)
|
||||
self.repository.close()
|
||||
# replace
|
||||
self.repository = self.open()
|
||||
with self.repository:
|
||||
self.repository.put(H(0), b'bar')
|
||||
self.repository.put(H(0), b"bar")
|
||||
self.repository.commit(compact=False)
|
||||
# delete
|
||||
self.repository = self.open()
|
||||
@ -992,10 +1009,10 @@ def test_legacy_free(self):
|
||||
|
||||
|
||||
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
|
||||
|
||||
def open(self, create=False):
|
||||
return RemoteRepository(Location('ssh://__testsuite__' + os.path.join(self.tmppath, 'repository')),
|
||||
exclusive=True, create=create)
|
||||
return RemoteRepository(
|
||||
Location("ssh://__testsuite__" + os.path.join(self.tmppath, "repository")), exclusive=True, create=create
|
||||
)
|
||||
|
||||
def test_crash_before_compact(self):
|
||||
# skip this test, we can't mock-patch a Repository class in another process!
|
||||
@ -1007,8 +1024,8 @@ def setUp(self):
|
||||
self.stream = io.StringIO()
|
||||
self.handler = logging.StreamHandler(self.stream)
|
||||
logging.getLogger().handlers[:] = [self.handler]
|
||||
logging.getLogger('borg.repository').handlers[:] = []
|
||||
logging.getLogger('borg.repository.foo').handlers[:] = []
|
||||
logging.getLogger("borg.repository").handlers[:] = []
|
||||
logging.getLogger("borg.repository.foo").handlers[:] = []
|
||||
# capture stderr
|
||||
sys.stderr.flush()
|
||||
self.old_stderr = sys.stderr
|
||||
@ -1019,31 +1036,31 @@ def tearDown(self):
|
||||
|
||||
def test_stderr_messages(self):
|
||||
handle_remote_line("unstructured stderr message\n")
|
||||
self.assert_equal(self.stream.getvalue(), '')
|
||||
self.assert_equal(self.stream.getvalue(), "")
|
||||
# stderr messages don't get an implicit newline
|
||||
self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr message\n')
|
||||
self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr message\n")
|
||||
|
||||
def test_stderr_progress_messages(self):
|
||||
handle_remote_line("unstructured stderr progress message\r")
|
||||
self.assert_equal(self.stream.getvalue(), '')
|
||||
self.assert_equal(self.stream.getvalue(), "")
|
||||
# stderr messages don't get an implicit newline
|
||||
self.assert_equal(self.stderr.getvalue(), 'Remote: unstructured stderr progress message\r')
|
||||
self.assert_equal(self.stderr.getvalue(), "Remote: unstructured stderr progress message\r")
|
||||
|
||||
def test_pre11_format_messages(self):
|
||||
self.handler.setLevel(logging.DEBUG)
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
handle_remote_line("$LOG INFO Remote: borg < 1.1 format message\n")
|
||||
self.assert_equal(self.stream.getvalue(), 'Remote: borg < 1.1 format message\n')
|
||||
self.assert_equal(self.stderr.getvalue(), '')
|
||||
self.assert_equal(self.stream.getvalue(), "Remote: borg < 1.1 format message\n")
|
||||
self.assert_equal(self.stderr.getvalue(), "")
|
||||
|
||||
def test_post11_format_messages(self):
|
||||
self.handler.setLevel(logging.DEBUG)
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
handle_remote_line("$LOG INFO borg.repository Remote: borg >= 1.1 format message\n")
|
||||
self.assert_equal(self.stream.getvalue(), 'Remote: borg >= 1.1 format message\n')
|
||||
self.assert_equal(self.stderr.getvalue(), '')
|
||||
self.assert_equal(self.stream.getvalue(), "Remote: borg >= 1.1 format message\n")
|
||||
self.assert_equal(self.stderr.getvalue(), "")
|
||||
|
||||
def test_remote_messages_screened(self):
|
||||
# default borg config for root logger
|
||||
@ -1051,12 +1068,12 @@ def test_remote_messages_screened(self):
|
||||
logging.getLogger().setLevel(logging.WARNING)
|
||||
|
||||
handle_remote_line("$LOG INFO borg.repository Remote: new format info message\n")
|
||||
self.assert_equal(self.stream.getvalue(), '')
|
||||
self.assert_equal(self.stderr.getvalue(), '')
|
||||
self.assert_equal(self.stream.getvalue(), "")
|
||||
self.assert_equal(self.stderr.getvalue(), "")
|
||||
|
||||
def test_info_to_correct_local_child(self):
|
||||
logging.getLogger('borg.repository').setLevel(logging.INFO)
|
||||
logging.getLogger('borg.repository.foo').setLevel(logging.INFO)
|
||||
logging.getLogger("borg.repository").setLevel(logging.INFO)
|
||||
logging.getLogger("borg.repository.foo").setLevel(logging.INFO)
|
||||
# default borg config for root logger
|
||||
self.handler.setLevel(logging.WARNING)
|
||||
logging.getLogger().setLevel(logging.WARNING)
|
||||
@ -1064,14 +1081,14 @@ def test_info_to_correct_local_child(self):
|
||||
child_stream = io.StringIO()
|
||||
child_handler = logging.StreamHandler(child_stream)
|
||||
child_handler.setLevel(logging.INFO)
|
||||
logging.getLogger('borg.repository').handlers[:] = [child_handler]
|
||||
logging.getLogger("borg.repository").handlers[:] = [child_handler]
|
||||
foo_stream = io.StringIO()
|
||||
foo_handler = logging.StreamHandler(foo_stream)
|
||||
foo_handler.setLevel(logging.INFO)
|
||||
logging.getLogger('borg.repository.foo').handlers[:] = [foo_handler]
|
||||
logging.getLogger("borg.repository.foo").handlers[:] = [foo_handler]
|
||||
|
||||
handle_remote_line("$LOG INFO borg.repository Remote: new format child message\n")
|
||||
self.assert_equal(foo_stream.getvalue(), '')
|
||||
self.assert_equal(child_stream.getvalue(), 'Remote: new format child message\n')
|
||||
self.assert_equal(self.stream.getvalue(), '')
|
||||
self.assert_equal(self.stderr.getvalue(), '')
|
||||
self.assert_equal(foo_stream.getvalue(), "")
|
||||
self.assert_equal(child_stream.getvalue(), "Remote: new format child message\n")
|
||||
self.assert_equal(self.stream.getvalue(), "")
|
||||
self.assert_equal(self.stderr.getvalue(), "")
|
||||
|
@ -11,103 +11,96 @@ def check(path, pattern):
|
||||
return bool(compiled.match(path))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, patterns", [
|
||||
# Literal string
|
||||
("foo/bar", ["foo/bar"]),
|
||||
("foo\\bar", ["foo\\bar"]),
|
||||
|
||||
# Non-ASCII
|
||||
("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]),
|
||||
("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]),
|
||||
|
||||
# Question mark
|
||||
("foo", ["fo?"]),
|
||||
("foo", ["f?o"]),
|
||||
("foo", ["f??"]),
|
||||
("foo", ["?oo"]),
|
||||
("foo", ["?o?"]),
|
||||
("foo", ["??o"]),
|
||||
("foo", ["???"]),
|
||||
|
||||
# Single asterisk
|
||||
("", ["*"]),
|
||||
("foo", ["*", "**", "***"]),
|
||||
("foo", ["foo*"]),
|
||||
("foobar", ["foo*"]),
|
||||
("foobar", ["foo*bar"]),
|
||||
("foobarbaz", ["foo*baz"]),
|
||||
("bar", ["*bar"]),
|
||||
("foobar", ["*bar"]),
|
||||
("foo/bar", ["foo/*bar"]),
|
||||
("foo/bar", ["foo/*ar"]),
|
||||
("foo/bar", ["foo/*r"]),
|
||||
("foo/bar", ["foo/*"]),
|
||||
("foo/bar", ["foo*/bar"]),
|
||||
("foo/bar", ["fo*/bar"]),
|
||||
("foo/bar", ["f*/bar"]),
|
||||
("foo/bar", ["*/bar"]),
|
||||
|
||||
# Double asterisk (matches 0..n directory layers)
|
||||
("foo/bar", ["foo/**/bar"]),
|
||||
("foo/1/bar", ["foo/**/bar"]),
|
||||
("foo/1/22/333/bar", ["foo/**/bar"]),
|
||||
("foo/", ["foo/**/"]),
|
||||
("foo/1/", ["foo/**/"]),
|
||||
("foo/1/22/333/", ["foo/**/"]),
|
||||
("bar", ["**/bar"]),
|
||||
("1/bar", ["**/bar"]),
|
||||
("1/22/333/bar", ["**/bar"]),
|
||||
("foo/bar/baz", ["foo/**/*"]),
|
||||
|
||||
# Set
|
||||
("foo1", ["foo[12]"]),
|
||||
("foo2", ["foo[12]"]),
|
||||
("foo2/bar", ["foo[12]/*"]),
|
||||
("f??f", ["f??f", "f[?][?]f"]),
|
||||
("foo]", ["foo[]]"]),
|
||||
|
||||
# Inverted set
|
||||
("foo3", ["foo[!12]"]),
|
||||
("foo^", ["foo[^!]"]),
|
||||
("foo!", ["foo[^!]"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"path, patterns",
|
||||
[
|
||||
# Literal string
|
||||
("foo/bar", ["foo/bar"]),
|
||||
("foo\\bar", ["foo\\bar"]),
|
||||
# Non-ASCII
|
||||
("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]),
|
||||
("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]),
|
||||
# Question mark
|
||||
("foo", ["fo?"]),
|
||||
("foo", ["f?o"]),
|
||||
("foo", ["f??"]),
|
||||
("foo", ["?oo"]),
|
||||
("foo", ["?o?"]),
|
||||
("foo", ["??o"]),
|
||||
("foo", ["???"]),
|
||||
# Single asterisk
|
||||
("", ["*"]),
|
||||
("foo", ["*", "**", "***"]),
|
||||
("foo", ["foo*"]),
|
||||
("foobar", ["foo*"]),
|
||||
("foobar", ["foo*bar"]),
|
||||
("foobarbaz", ["foo*baz"]),
|
||||
("bar", ["*bar"]),
|
||||
("foobar", ["*bar"]),
|
||||
("foo/bar", ["foo/*bar"]),
|
||||
("foo/bar", ["foo/*ar"]),
|
||||
("foo/bar", ["foo/*r"]),
|
||||
("foo/bar", ["foo/*"]),
|
||||
("foo/bar", ["foo*/bar"]),
|
||||
("foo/bar", ["fo*/bar"]),
|
||||
("foo/bar", ["f*/bar"]),
|
||||
("foo/bar", ["*/bar"]),
|
||||
# Double asterisk (matches 0..n directory layers)
|
||||
("foo/bar", ["foo/**/bar"]),
|
||||
("foo/1/bar", ["foo/**/bar"]),
|
||||
("foo/1/22/333/bar", ["foo/**/bar"]),
|
||||
("foo/", ["foo/**/"]),
|
||||
("foo/1/", ["foo/**/"]),
|
||||
("foo/1/22/333/", ["foo/**/"]),
|
||||
("bar", ["**/bar"]),
|
||||
("1/bar", ["**/bar"]),
|
||||
("1/22/333/bar", ["**/bar"]),
|
||||
("foo/bar/baz", ["foo/**/*"]),
|
||||
# Set
|
||||
("foo1", ["foo[12]"]),
|
||||
("foo2", ["foo[12]"]),
|
||||
("foo2/bar", ["foo[12]/*"]),
|
||||
("f??f", ["f??f", "f[?][?]f"]),
|
||||
("foo]", ["foo[]]"]),
|
||||
# Inverted set
|
||||
("foo3", ["foo[!12]"]),
|
||||
("foo^", ["foo[^!]"]),
|
||||
("foo!", ["foo[^!]"]),
|
||||
],
|
||||
)
|
||||
def test_match(path, patterns):
|
||||
for p in patterns:
|
||||
assert check(path, p)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, patterns", [
|
||||
("", ["?", "[]"]),
|
||||
("foo", ["foo?"]),
|
||||
("foo", ["?foo"]),
|
||||
("foo", ["f?oo"]),
|
||||
|
||||
# do not match path separator
|
||||
("foo/ar", ["foo?ar"]),
|
||||
|
||||
# do not match/cross over os.path.sep
|
||||
("foo/bar", ["*"]),
|
||||
("foo/bar", ["foo*bar"]),
|
||||
("foo/bar", ["foo*ar"]),
|
||||
("foo/bar", ["fo*bar"]),
|
||||
("foo/bar", ["fo*ar"]),
|
||||
|
||||
# Double asterisk
|
||||
("foobar", ["foo/**/bar"]),
|
||||
|
||||
# Two asterisks without slash do not match directory separator
|
||||
("foo/bar", ["**"]),
|
||||
|
||||
# Double asterisk not matching filename
|
||||
("foo/bar", ["**/"]),
|
||||
|
||||
# Set
|
||||
("foo3", ["foo[12]"]),
|
||||
|
||||
# Inverted set
|
||||
("foo1", ["foo[!12]"]),
|
||||
("foo2", ["foo[!12]"]),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"path, patterns",
|
||||
[
|
||||
("", ["?", "[]"]),
|
||||
("foo", ["foo?"]),
|
||||
("foo", ["?foo"]),
|
||||
("foo", ["f?oo"]),
|
||||
# do not match path separator
|
||||
("foo/ar", ["foo?ar"]),
|
||||
# do not match/cross over os.path.sep
|
||||
("foo/bar", ["*"]),
|
||||
("foo/bar", ["foo*bar"]),
|
||||
("foo/bar", ["foo*ar"]),
|
||||
("foo/bar", ["fo*bar"]),
|
||||
("foo/bar", ["fo*ar"]),
|
||||
# Double asterisk
|
||||
("foobar", ["foo/**/bar"]),
|
||||
# Two asterisks without slash do not match directory separator
|
||||
("foo/bar", ["**"]),
|
||||
# Double asterisk not matching filename
|
||||
("foo/bar", ["**/"]),
|
||||
# Set
|
||||
("foo3", ["foo[12]"]),
|
||||
# Inverted set
|
||||
("foo1", ["foo[!12]"]),
|
||||
("foo2", ["foo[!12]"]),
|
||||
],
|
||||
)
|
||||
def test_mismatch(path, patterns):
|
||||
for p in patterns:
|
||||
assert not check(path, p)
|
||||
@ -115,10 +108,10 @@ def test_mismatch(path, patterns):
|
||||
|
||||
def test_match_end():
|
||||
regex = shellpattern.translate("*-home") # default is match_end == string end
|
||||
assert re.match(regex, '2017-07-03-home')
|
||||
assert not re.match(regex, '2017-07-03-home.checkpoint')
|
||||
assert re.match(regex, "2017-07-03-home")
|
||||
assert not re.match(regex, "2017-07-03-home.checkpoint")
|
||||
|
||||
match_end = r'(%s)?\Z' % r'\.checkpoint(\.\d+)?' # with/without checkpoint ending
|
||||
match_end = r"(%s)?\Z" % r"\.checkpoint(\.\d+)?" # with/without checkpoint ending
|
||||
regex = shellpattern.translate("*-home", match_end=match_end)
|
||||
assert re.match(regex, '2017-07-03-home')
|
||||
assert re.match(regex, '2017-07-03-home.checkpoint')
|
||||
assert re.match(regex, "2017-07-03-home")
|
||||
assert re.match(regex, "2017-07-03-home.checkpoint")
|
||||
|
@ -3,51 +3,57 @@
|
||||
from ..version import parse_version, format_version
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_str, version_tuple", [
|
||||
# setuptools < 8.0 uses "-"
|
||||
('1.0.0a1.dev204-g8866961.d20170606', (1, 0, 0, -4, 1)),
|
||||
('1.0.0a1.dev204-g8866961', (1, 0, 0, -4, 1)),
|
||||
('1.0.0-d20170606', (1, 0, 0, -1)),
|
||||
# setuptools >= 8.0 uses "+"
|
||||
('1.0.0a1.dev204+g8866961.d20170606', (1, 0, 0, -4, 1)),
|
||||
('1.0.0a1.dev204+g8866961', (1, 0, 0, -4, 1)),
|
||||
('1.0.0+d20170606', (1, 0, 0, -1)),
|
||||
# pre-release versions:
|
||||
('1.0.0a1', (1, 0, 0, -4, 1)),
|
||||
('1.0.0a2', (1, 0, 0, -4, 2)),
|
||||
('1.0.0b3', (1, 0, 0, -3, 3)),
|
||||
('1.0.0rc4', (1, 0, 0, -2, 4)),
|
||||
# release versions:
|
||||
('0.0.0', (0, 0, 0, -1)),
|
||||
('0.0.11', (0, 0, 11, -1)),
|
||||
('0.11.0', (0, 11, 0, -1)),
|
||||
('11.0.0', (11, 0, 0, -1)),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_str, version_tuple",
|
||||
[
|
||||
# setuptools < 8.0 uses "-"
|
||||
("1.0.0a1.dev204-g8866961.d20170606", (1, 0, 0, -4, 1)),
|
||||
("1.0.0a1.dev204-g8866961", (1, 0, 0, -4, 1)),
|
||||
("1.0.0-d20170606", (1, 0, 0, -1)),
|
||||
# setuptools >= 8.0 uses "+"
|
||||
("1.0.0a1.dev204+g8866961.d20170606", (1, 0, 0, -4, 1)),
|
||||
("1.0.0a1.dev204+g8866961", (1, 0, 0, -4, 1)),
|
||||
("1.0.0+d20170606", (1, 0, 0, -1)),
|
||||
# pre-release versions:
|
||||
("1.0.0a1", (1, 0, 0, -4, 1)),
|
||||
("1.0.0a2", (1, 0, 0, -4, 2)),
|
||||
("1.0.0b3", (1, 0, 0, -3, 3)),
|
||||
("1.0.0rc4", (1, 0, 0, -2, 4)),
|
||||
# release versions:
|
||||
("0.0.0", (0, 0, 0, -1)),
|
||||
("0.0.11", (0, 0, 11, -1)),
|
||||
("0.11.0", (0, 11, 0, -1)),
|
||||
("11.0.0", (11, 0, 0, -1)),
|
||||
],
|
||||
)
|
||||
def test_parse_version(version_str, version_tuple):
|
||||
assert parse_version(version_str) == version_tuple
|
||||
|
||||
|
||||
def test_parse_version_invalid():
|
||||
with pytest.raises(ValueError):
|
||||
assert parse_version('') # we require x.y.z versions
|
||||
assert parse_version("") # we require x.y.z versions
|
||||
with pytest.raises(ValueError):
|
||||
assert parse_version('1') # we require x.y.z versions
|
||||
assert parse_version("1") # we require x.y.z versions
|
||||
with pytest.raises(ValueError):
|
||||
assert parse_version('1.2') # we require x.y.z versions
|
||||
assert parse_version("1.2") # we require x.y.z versions
|
||||
with pytest.raises(ValueError):
|
||||
assert parse_version('crap')
|
||||
assert parse_version("crap")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version_str, version_tuple", [
|
||||
('1.0.0a1', (1, 0, 0, -4, 1)),
|
||||
('1.0.0', (1, 0, 0, -1)),
|
||||
('1.0.0a2', (1, 0, 0, -4, 2)),
|
||||
('1.0.0b3', (1, 0, 0, -3, 3)),
|
||||
('1.0.0rc4', (1, 0, 0, -2, 4)),
|
||||
('0.0.0', (0, 0, 0, -1)),
|
||||
('0.0.11', (0, 0, 11, -1)),
|
||||
('0.11.0', (0, 11, 0, -1)),
|
||||
('11.0.0', (11, 0, 0, -1)),
|
||||
])
|
||||
@pytest.mark.parametrize(
|
||||
"version_str, version_tuple",
|
||||
[
|
||||
("1.0.0a1", (1, 0, 0, -4, 1)),
|
||||
("1.0.0", (1, 0, 0, -1)),
|
||||
("1.0.0a2", (1, 0, 0, -4, 2)),
|
||||
("1.0.0b3", (1, 0, 0, -3, 3)),
|
||||
("1.0.0rc4", (1, 0, 0, -2, 4)),
|
||||
("0.0.0", (0, 0, 0, -1)),
|
||||
("0.0.11", (0, 0, 11, -1)),
|
||||
("0.11.0", (0, 11, 0, -1)),
|
||||
("11.0.0", (11, 0, 0, -1)),
|
||||
],
|
||||
)
|
||||
def test_format_version(version_str, version_tuple):
|
||||
assert format_version(version_tuple) == version_str
|
||||
|
@ -10,12 +10,11 @@
|
||||
from . import BaseTestCase
|
||||
|
||||
|
||||
@unittest.skipUnless(is_enabled(), 'xattr not enabled on filesystem')
|
||||
@unittest.skipUnless(is_enabled(), "xattr not enabled on filesystem")
|
||||
class XattrTestCase(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpfile = tempfile.NamedTemporaryFile()
|
||||
self.symlink = self.tmpfile.name + '.symlink'
|
||||
self.symlink = self.tmpfile.name + ".symlink"
|
||||
os.symlink(self.tmpfile.name, self.symlink)
|
||||
|
||||
def tearDown(self):
|
||||
@ -23,7 +22,7 @@ def tearDown(self):
|
||||
|
||||
def assert_equal_se(self, is_x, want_x):
|
||||
# check 2 xattr lists for equality, but ignore security.selinux attr
|
||||
is_x = set(is_x) - {b'security.selinux'}
|
||||
is_x = set(is_x) - {b"security.selinux"}
|
||||
want_x = set(want_x)
|
||||
self.assert_equal(is_x, want_x)
|
||||
|
||||
@ -34,32 +33,32 @@ def test(self):
|
||||
self.assert_equal_se(listxattr(tmp_fn), [])
|
||||
self.assert_equal_se(listxattr(tmp_fd), [])
|
||||
self.assert_equal_se(listxattr(tmp_lfn), [])
|
||||
setxattr(tmp_fn, b'user.foo', b'bar')
|
||||
setxattr(tmp_fd, b'user.bar', b'foo')
|
||||
setxattr(tmp_fn, b'user.empty', b'')
|
||||
setxattr(tmp_fn, b"user.foo", b"bar")
|
||||
setxattr(tmp_fd, b"user.bar", b"foo")
|
||||
setxattr(tmp_fn, b"user.empty", b"")
|
||||
if not is_linux:
|
||||
# linux does not allow setting user.* xattrs on symlinks
|
||||
setxattr(tmp_lfn, b'user.linkxattr', b'baz')
|
||||
self.assert_equal_se(listxattr(tmp_fn), [b'user.foo', b'user.bar', b'user.empty'])
|
||||
self.assert_equal_se(listxattr(tmp_fd), [b'user.foo', b'user.bar', b'user.empty'])
|
||||
self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b'user.foo', b'user.bar', b'user.empty'])
|
||||
setxattr(tmp_lfn, b"user.linkxattr", b"baz")
|
||||
self.assert_equal_se(listxattr(tmp_fn), [b"user.foo", b"user.bar", b"user.empty"])
|
||||
self.assert_equal_se(listxattr(tmp_fd), [b"user.foo", b"user.bar", b"user.empty"])
|
||||
self.assert_equal_se(listxattr(tmp_lfn, follow_symlinks=True), [b"user.foo", b"user.bar", b"user.empty"])
|
||||
if not is_linux:
|
||||
self.assert_equal_se(listxattr(tmp_lfn), [b'user.linkxattr'])
|
||||
self.assert_equal(getxattr(tmp_fn, b'user.foo'), b'bar')
|
||||
self.assert_equal(getxattr(tmp_fd, b'user.foo'), b'bar')
|
||||
self.assert_equal(getxattr(tmp_lfn, b'user.foo', follow_symlinks=True), b'bar')
|
||||
self.assert_equal_se(listxattr(tmp_lfn), [b"user.linkxattr"])
|
||||
self.assert_equal(getxattr(tmp_fn, b"user.foo"), b"bar")
|
||||
self.assert_equal(getxattr(tmp_fd, b"user.foo"), b"bar")
|
||||
self.assert_equal(getxattr(tmp_lfn, b"user.foo", follow_symlinks=True), b"bar")
|
||||
if not is_linux:
|
||||
self.assert_equal(getxattr(tmp_lfn, b'user.linkxattr'), b'baz')
|
||||
self.assert_equal(getxattr(tmp_fn, b'user.empty'), b'')
|
||||
self.assert_equal(getxattr(tmp_lfn, b"user.linkxattr"), b"baz")
|
||||
self.assert_equal(getxattr(tmp_fn, b"user.empty"), b"")
|
||||
|
||||
def test_listxattr_buffer_growth(self):
|
||||
tmp_fn = os.fsencode(self.tmpfile.name)
|
||||
# make it work even with ext4, which imposes rather low limits
|
||||
buffer.resize(size=64, init=True)
|
||||
# xattr raw key list will be > 64
|
||||
keys = [b'user.attr%d' % i for i in range(20)]
|
||||
keys = [b"user.attr%d" % i for i in range(20)]
|
||||
for key in keys:
|
||||
setxattr(tmp_fn, key, b'x')
|
||||
setxattr(tmp_fn, key, b"x")
|
||||
got_keys = listxattr(tmp_fn)
|
||||
self.assert_equal_se(got_keys, keys)
|
||||
assert len(buffer) > 64
|
||||
@ -68,18 +67,15 @@ def test_getxattr_buffer_growth(self):
|
||||
tmp_fn = os.fsencode(self.tmpfile.name)
|
||||
# make it work even with ext4, which imposes rather low limits
|
||||
buffer.resize(size=64, init=True)
|
||||
value = b'x' * 126
|
||||
setxattr(tmp_fn, b'user.big', value)
|
||||
got_value = getxattr(tmp_fn, b'user.big')
|
||||
value = b"x" * 126
|
||||
setxattr(tmp_fn, b"user.big", value)
|
||||
got_value = getxattr(tmp_fn, b"user.big")
|
||||
self.assert_equal(value, got_value)
|
||||
self.assert_equal(len(buffer), 128)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('lstring, splitted', (
|
||||
(b'', []),
|
||||
(b'\x00', [b'']),
|
||||
(b'\x01a', [b'a']),
|
||||
(b'\x01a\x02cd', [b'a', b'cd']),
|
||||
))
|
||||
@pytest.mark.parametrize(
|
||||
"lstring, splitted", ((b"", []), (b"\x00", [b""]), (b"\x01a", [b"a"]), (b"\x01a\x02cd", [b"a", b"cd"]))
|
||||
)
|
||||
def test_split_lstring(lstring, splitted):
|
||||
assert split_lstring(lstring) == splitted
|
||||
|
@ -25,8 +25,16 @@ def upgrade_compressed_chunk(self, *, chunk):
|
||||
def upgrade_archive_metadata(self, *, metadata):
|
||||
new_metadata = {}
|
||||
# keep all metadata except archive version and stats.
|
||||
for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment',
|
||||
'chunker_params', 'recreate_cmdline'):
|
||||
for attr in (
|
||||
"cmdline",
|
||||
"hostname",
|
||||
"username",
|
||||
"time",
|
||||
"time_end",
|
||||
"comment",
|
||||
"chunker_params",
|
||||
"recreate_cmdline",
|
||||
):
|
||||
if hasattr(metadata, attr):
|
||||
new_metadata[attr] = getattr(metadata, attr)
|
||||
return new_metadata
|
||||
@ -42,24 +50,45 @@ def new_archive(self, *, archive):
|
||||
|
||||
def upgrade_item(self, *, item):
|
||||
"""upgrade item as needed, get rid of legacy crap"""
|
||||
ITEM_KEY_WHITELIST = {'path', 'source', 'rdev', 'chunks', 'chunks_healthy', 'hlid',
|
||||
'mode', 'user', 'group', 'uid', 'gid', 'mtime', 'atime', 'ctime', 'birthtime', 'size',
|
||||
'xattrs', 'bsdflags', 'acl_nfs4', 'acl_access', 'acl_default', 'acl_extended',
|
||||
'part'}
|
||||
ITEM_KEY_WHITELIST = {
|
||||
"path",
|
||||
"source",
|
||||
"rdev",
|
||||
"chunks",
|
||||
"chunks_healthy",
|
||||
"hlid",
|
||||
"mode",
|
||||
"user",
|
||||
"group",
|
||||
"uid",
|
||||
"gid",
|
||||
"mtime",
|
||||
"atime",
|
||||
"ctime",
|
||||
"birthtime",
|
||||
"size",
|
||||
"xattrs",
|
||||
"bsdflags",
|
||||
"acl_nfs4",
|
||||
"acl_access",
|
||||
"acl_default",
|
||||
"acl_extended",
|
||||
"part",
|
||||
}
|
||||
|
||||
if self.hlm.borg1_hardlink_master(item):
|
||||
item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['path'])
|
||||
self.hlm.remember(id=hlid, info=(item._dict.get('chunks'), item._dict.get('chunks_healthy')))
|
||||
item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["path"])
|
||||
self.hlm.remember(id=hlid, info=(item._dict.get("chunks"), item._dict.get("chunks_healthy")))
|
||||
elif self.hlm.borg1_hardlink_slave(item):
|
||||
item._dict['hlid'] = hlid = self.hlm.hardlink_id_from_path(item._dict['source'])
|
||||
item._dict["hlid"] = hlid = self.hlm.hardlink_id_from_path(item._dict["source"])
|
||||
chunks, chunks_healthy = self.hlm.retrieve(id=hlid, default=(None, None))
|
||||
if chunks is not None:
|
||||
item._dict['chunks'] = chunks
|
||||
item._dict["chunks"] = chunks
|
||||
for chunk_id, _ in chunks:
|
||||
self.cache.chunk_incref(chunk_id, self.archive.stats)
|
||||
if chunks_healthy is not None:
|
||||
item._dict['chunks_healthy'] = chunks
|
||||
item._dict.pop('source') # not used for hardlinks any more, replaced by hlid
|
||||
item._dict["chunks_healthy"] = chunks
|
||||
item._dict.pop("source") # not used for hardlinks any more, replaced by hlid
|
||||
# make sure we only have desired stuff in the new item. specifically, make sure to get rid of:
|
||||
# - 'acl' remnants of bug in attic <= 0.13
|
||||
# - 'hardlink_master' (superseded by hlid)
|
||||
@ -80,17 +109,17 @@ def upgrade_zlib_and_level(chunk):
|
||||
return chunk
|
||||
|
||||
ctype = chunk[0:1]
|
||||
level = b'\xFF' # FF means unknown compression level
|
||||
level = b"\xFF" # FF means unknown compression level
|
||||
|
||||
if ctype == ObfuscateSize.ID:
|
||||
# in older borg, we used unusual byte order
|
||||
old_header_fmt = Struct('>I')
|
||||
old_header_fmt = Struct(">I")
|
||||
new_header_fmt = ObfuscateSize.header_fmt
|
||||
length = ObfuscateSize.header_len
|
||||
size_bytes = chunk[2:2+length]
|
||||
size_bytes = chunk[2 : 2 + length]
|
||||
size = old_header_fmt.unpack(size_bytes)
|
||||
size_bytes = new_header_fmt.pack(size)
|
||||
compressed = chunk[2+length:]
|
||||
compressed = chunk[2 + length :]
|
||||
compressed = upgrade_zlib_and_level(compressed)
|
||||
chunk = ctype + level + size_bytes + compressed
|
||||
else:
|
||||
@ -101,8 +130,16 @@ def upgrade_archive_metadata(self, *, metadata):
|
||||
new_metadata = {}
|
||||
# keep all metadata except archive version and stats. also do not keep
|
||||
# recreate_source_id, recreate_args, recreate_partial_chunks which were used only in 1.1.0b1 .. b2.
|
||||
for attr in ('cmdline', 'hostname', 'username', 'time', 'time_end', 'comment',
|
||||
'chunker_params', 'recreate_cmdline'):
|
||||
for attr in (
|
||||
"cmdline",
|
||||
"hostname",
|
||||
"username",
|
||||
"time",
|
||||
"time_end",
|
||||
"comment",
|
||||
"chunker_params",
|
||||
"recreate_cmdline",
|
||||
):
|
||||
if hasattr(metadata, attr):
|
||||
new_metadata[attr] = getattr(metadata, attr)
|
||||
return new_metadata
|
||||
|
@ -21,12 +21,12 @@ def parse_version(version):
|
||||
"""
|
||||
m = re.match(version_re, version, re.VERBOSE)
|
||||
if m is None:
|
||||
raise ValueError('Invalid version string %s' % version)
|
||||
raise ValueError("Invalid version string %s" % version)
|
||||
gd = m.groupdict()
|
||||
version = [int(gd['major']), int(gd['minor']), int(gd['patch'])]
|
||||
if m.lastgroup == 'prerelease':
|
||||
p_type = {'a': -4, 'b': -3, 'rc': -2}[gd['ptype']]
|
||||
p_num = int(gd['pnum'])
|
||||
version = [int(gd["major"]), int(gd["minor"]), int(gd["patch"])]
|
||||
if m.lastgroup == "prerelease":
|
||||
p_type = {"a": -4, "b": -3, "rc": -2}[gd["ptype"]]
|
||||
p_num = int(gd["pnum"])
|
||||
version += [p_type, p_num]
|
||||
else:
|
||||
version += [-1]
|
||||
@ -44,6 +44,6 @@ def format_version(version):
|
||||
elif part == -1:
|
||||
break
|
||||
else:
|
||||
f[-1] = f[-1] + {-2: 'rc', -3: 'b', -4: 'a'}[part] + str(next(it))
|
||||
f[-1] = f[-1] + {-2: "rc", -3: "b", -4: "a"}[part] + str(next(it))
|
||||
break
|
||||
return '.'.join(f)
|
||||
return ".".join(f)
|
||||
|
@ -22,14 +22,14 @@
|
||||
# TODO: Check whether fakeroot supports xattrs on all platforms supported below.
|
||||
# TODO: If that's the case then we can make Borg fakeroot-xattr-compatible on these as well.
|
||||
XATTR_FAKEROOT = False
|
||||
if sys.platform.startswith('linux'):
|
||||
LD_PRELOAD = os.environ.get('LD_PRELOAD', '')
|
||||
if sys.platform.startswith("linux"):
|
||||
LD_PRELOAD = os.environ.get("LD_PRELOAD", "")
|
||||
preloads = re.split("[ :]", LD_PRELOAD)
|
||||
for preload in preloads:
|
||||
if preload.startswith("libfakeroot"):
|
||||
env = prepare_subprocess_env(system=True)
|
||||
fakeroot_output = subprocess.check_output(['fakeroot', '-v'], env=env)
|
||||
fakeroot_version = parse_version(fakeroot_output.decode('ascii').split()[-1])
|
||||
fakeroot_output = subprocess.check_output(["fakeroot", "-v"], env=env)
|
||||
fakeroot_version = parse_version(fakeroot_output.decode("ascii").split()[-1])
|
||||
if fakeroot_version >= parse_version("1.20.2"):
|
||||
# 1.20.2 has been confirmed to have xattr support
|
||||
# 1.18.2 has been confirmed not to have xattr support
|
||||
@ -39,11 +39,10 @@
|
||||
|
||||
|
||||
def is_enabled(path=None):
|
||||
"""Determine if xattr is enabled on the filesystem
|
||||
"""
|
||||
with tempfile.NamedTemporaryFile(dir=path, prefix='borg-tmp') as f:
|
||||
"""Determine if xattr is enabled on the filesystem"""
|
||||
with tempfile.NamedTemporaryFile(dir=path, prefix="borg-tmp") as f:
|
||||
fd = f.fileno()
|
||||
name, value = b'user.name', b'value'
|
||||
name, value = b"user.name", b"value"
|
||||
try:
|
||||
setxattr(fd, name, value)
|
||||
except OSError:
|
||||
@ -80,7 +79,7 @@ def get_all(path, follow_symlinks=False):
|
||||
except OSError as e:
|
||||
name_str = name.decode()
|
||||
if isinstance(path, int):
|
||||
path_str = '<FD %d>' % path
|
||||
path_str = "<FD %d>" % path
|
||||
else:
|
||||
path_str = os.fsdecode(path)
|
||||
if e.errno == ENOATTR:
|
||||
@ -89,8 +88,9 @@ def get_all(path, follow_symlinks=False):
|
||||
pass
|
||||
elif e.errno == errno.EPERM:
|
||||
# we were not permitted to read this attribute, still can continue trying to read others
|
||||
logger.warning('{}: Operation not permitted when reading extended attribute {}'.format(
|
||||
path_str, name_str))
|
||||
logger.warning(
|
||||
"{}: Operation not permitted when reading extended attribute {}".format(path_str, name_str)
|
||||
)
|
||||
else:
|
||||
raise
|
||||
except OSError as e:
|
||||
@ -125,21 +125,21 @@ def set_all(path, xattrs, follow_symlinks=False):
|
||||
warning = True
|
||||
k_str = k.decode()
|
||||
if isinstance(path, int):
|
||||
path_str = '<FD %d>' % path
|
||||
path_str = "<FD %d>" % path
|
||||
else:
|
||||
path_str = os.fsdecode(path)
|
||||
if e.errno == errno.E2BIG:
|
||||
err_str = 'too big for this filesystem'
|
||||
err_str = "too big for this filesystem"
|
||||
elif e.errno == errno.ENOTSUP:
|
||||
err_str = 'xattrs not supported on this filesystem'
|
||||
err_str = "xattrs not supported on this filesystem"
|
||||
elif e.errno == errno.ENOSPC:
|
||||
# ext4 reports ENOSPC when trying to set an xattr with >4kiB while ext4 can only support 4kiB xattrs
|
||||
# (in this case, this is NOT a "disk full" error, just a ext4 limitation).
|
||||
err_str = 'no space left on device [xattr len = %d]' % (len(v),)
|
||||
err_str = "no space left on device [xattr len = %d]" % (len(v),)
|
||||
else:
|
||||
# generic handler
|
||||
# EACCES: permission denied to set this specific xattr (this may happen related to security.* keys)
|
||||
# EPERM: operation not permitted
|
||||
err_str = os.strerror(e.errno)
|
||||
logger.warning('%s: when setting extended attribute %s: %s', path_str, k_str, err_str)
|
||||
logger.warning("%s: when setting extended attribute %s: %s", path_str, k_str, err_str)
|
||||
return warning
|
||||
|
Loading…
Reference in New Issue
Block a user