From 96d93dcf0e841fa22cb0503ced2f5d974ec76828 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 26 Feb 2022 22:18:12 +0100 Subject: [PATCH] do not support bundled 3rd party code any more, fixes #6316 --- docs/installation.rst | 12 +- setup.py | 35 ++---- setup_checksums.py | 45 ++------ setup_compress.py | 148 ++++--------------------- src/borg/algorithms/checksums.pyx | 2 +- src/borg/algorithms/lz4-libselect.h | 5 - src/borg/algorithms/xxhash-libselect.h | 5 - src/borg/algorithms/zstd-libselect.h | 5 - src/borg/compress.pyx | 4 +- 9 files changed, 52 insertions(+), 209 deletions(-) delete mode 100644 src/borg/algorithms/lz4-libselect.h delete mode 100644 src/borg/algorithms/xxhash-libselect.h delete mode 100644 src/borg/algorithms/zstd-libselect.h diff --git a/docs/installation.rst b/docs/installation.rst index 96100eb3d..f41d1d5bc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -159,20 +159,18 @@ Dependencies To install Borg from a source package (including pip), you have to install the following dependencies first: -* `Python 3`_ >= 3.8.0, plus development headers. -* OpenSSL_ >= 1.1.1, plus development headers. -* libacl_ (which depends on libattr_), both plus development headers. -* We have bundled code of the following packages, but borg by default (see - setup.py if you want to change that) prefers a shared library if it can - be found on the system (lib + dev headers) at build time: +* `Python 3`_ >= 3.8.0 (interpreter plus development headers) +* Libraries (library plus development headers): + - OpenSSL_ >= 1.1.1 + - libacl_ (which depends on libattr_) - liblz4_ >= 1.7.0 (r129) - libzstd_ >= 1.3.0 - libxxhash >= 0.8.1 (0.8.0 might work also) * pkg-config (cli tool) and pkgconfig python package (borg uses these to discover header and library location - if it can't import pkgconfig and is not pointed to header/library locations via env vars [see setup.py], - it will fall back to using the bundled code, see above). + it will raise a fatal error). **These must be present before invoking setup.py!** * some other Python dependencies, pip will automatically install them for you. * optionally, if you wish to mount an archive as a FUSE filesystem, you need diff --git a/setup.py b/setup.py index 370e0801f..9090b9e82 100644 --- a/setup.py +++ b/setup.py @@ -26,35 +26,25 @@ import setup_docs is_win32 = sys.platform.startswith('win32') -# How the build process finds the system libs / uses the bundled code: +# How the build process finds the system libs: # -# 1. it will try to use (system) libs (see 1.1. and 1.2.), -# except if you use these env vars to force using the bundled code: -# BORG_USE_BUNDLED_XXX undefined --> try using system lib -# BORG_USE_BUNDLED_XXX=YES --> use the bundled code -# Note: do not use =NO, that is not supported! -# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there. -# 1.2. if not and pkg-config can locate the lib, the lib located by -# pkg-config will be used. We use the pkg-config tool via the pkgconfig -# python package, which must be installed before invoking setup.py. -# if pkgconfig is not installed, this step is skipped. -# 2. if no system lib could be located via 1.1. or 1.2., it will fall back -# to using the bundled code. +# 1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there. +# 2. if not and pkg-config can locate the lib, the lib located by +# pkg-config will be used. We use the pkg-config tool via the pkgconfig +# python package, which must be installed before invoking setup.py. +# if pkgconfig is not installed, this step is skipped. +# 3. otherwise raise a fatal error. -# OpenSSL is required as a (system) lib in any case as we do not bundle it. -# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.). -# needed: openssl >=1.0.2 or >=1.1.0 (or compatible) +# needed: >=1.1.1 (or compatible) system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX') # needed: lz4 (>= 1.7.0 / r129) -prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4')) system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX') # needed: zstd (>= 1.3.0) -prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD')) system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX') -prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH')) +# needed: xxhash (>= 0.8.1) system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX') # Number of threads to use for cythonize, not used on windows @@ -187,14 +177,13 @@ if not on_rtd: compress_ext_kwargs = members_appended( dict(sources=[compress_source]), - setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4), - setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd, - multithreaded=False, legacy=False), + setup_compress.lz4_ext_kwargs(pc, system_prefix_liblz4), + setup_compress.zstd_ext_kwargs(pc, system_prefix_libzstd), ) checksums_ext_kwargs = members_appended( dict(sources=[checksums_source]), - setup_checksums.xxhash_ext_kwargs(pc, prefer_system_libxxhash, system_prefix_libxxhash), + setup_checksums.xxhash_ext_kwargs(pc, system_prefix_libxxhash), ) ext_modules += [ diff --git a/setup_checksums.py b/setup_checksums.py index c4a2fa38a..c41e35905 100644 --- a/setup_checksums.py +++ b/setup_checksums.py @@ -3,40 +3,15 @@ import os -def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path,))) for path in paths] +def xxhash_ext_kwargs(pc, system_prefix): + if system_prefix: + print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['xxhash']) + if pc and pc.installed('libxxhash', '>= 0.7.3'): + print('Detected and preferring libxxhash [via pkg-config]') + return pc.parse('libxxhash') -# xxhash files, structure as seen in xxhash project repository: - -# path relative (to this file) to the bundled library source code files -xxhash_bundled_path = 'src/borg/algorithms/xxh64' - -xxhash_sources = [ - 'xxhash.c', -] - -xxhash_includes = [ - '', -] - - -def xxhash_ext_kwargs(pc, prefer_system, system_prefix): - if prefer_system: - if system_prefix: - print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['xxhash']) - - if pc and pc.installed('libxxhash', '>= 0.7.3'): - print('Detected and preferring libxxhash [via pkg-config]') - return pc.parse('libxxhash') - - print('Using bundled xxhash') - sources = multi_join(xxhash_sources, xxhash_bundled_path) - include_dirs = multi_join(xxhash_includes, xxhash_bundled_path) - define_macros = [('BORG_USE_BUNDLED_XXHASH', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) - + raise Exception('Could not find xxhash lib/headers, please set BORG_LIBXXHASH_PREFIX') diff --git a/setup_compress.py b/setup_compress.py index 87cb887e0..7d018e9e9 100644 --- a/setup_compress.py +++ b/setup_compress.py @@ -3,133 +3,29 @@ import os -def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path,))) for path in paths] +def zstd_ext_kwargs(pc, system_prefix): + if system_prefix: + print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['zstd']) + + if pc and pc.installed('libzstd', '>= 1.3.0'): + print('Detected and preferring libzstd [via pkg-config]') + return pc.parse('libzstd') + + raise Exception('Could not find zstd lib/headers, please set BORG_LIBZSTD_PREFIX') -# zstd files, structure as seen in zstd project repository: +def lz4_ext_kwargs(pc, system_prefix): + if system_prefix: + print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['lz4']) -# path relative (to this file) to the bundled library source code files -zstd_bundled_path = 'src/borg/algorithms/zstd' + if pc and pc.installed('liblz4', '>= 1.7.0'): + print('Detected and preferring liblz4 [via pkg-config]') + return pc.parse('liblz4') -zstd_sources = [ - 'lib/common/debug.c', - 'lib/common/entropy_common.c', - 'lib/common/error_private.c', - 'lib/common/fse_decompress.c', - 'lib/common/pool.c', - 'lib/common/threading.c', - 'lib/common/xxhash.c', - 'lib/common/zstd_common.c', - 'lib/compress/fse_compress.c', - 'lib/compress/hist.c', - 'lib/compress/huf_compress.c', - 'lib/compress/zstd_compress.c', - 'lib/compress/zstd_compress_literals.c', - 'lib/compress/zstd_compress_sequences.c', - 'lib/compress/zstd_compress_superblock.c', - 'lib/compress/zstd_double_fast.c', - 'lib/compress/zstd_fast.c', - 'lib/compress/zstd_lazy.c', - 'lib/compress/zstd_ldm.c', - 'lib/compress/zstd_opt.c', - 'lib/compress/zstdmt_compress.c', - 'lib/decompress/huf_decompress.c', - 'lib/decompress/zstd_ddict.c', - 'lib/decompress/zstd_decompress.c', - 'lib/decompress/zstd_decompress_block.c', - 'lib/dictBuilder/cover.c', - 'lib/dictBuilder/divsufsort.c', - 'lib/dictBuilder/fastcover.c', - 'lib/dictBuilder/zdict.c', -] - -zstd_sources_legacy = [ - 'lib/deprecated/zbuff_common.c', - 'lib/deprecated/zbuff_compress.c', - 'lib/deprecated/zbuff_decompress.c', - 'lib/legacy/zstd_v01.c', - 'lib/legacy/zstd_v02.c', - 'lib/legacy/zstd_v03.c', - 'lib/legacy/zstd_v04.c', - 'lib/legacy/zstd_v05.c', - 'lib/legacy/zstd_v06.c', - 'lib/legacy/zstd_v07.c', -] - -zstd_includes = [ - 'lib', - 'lib/common', - 'lib/compress', - 'lib/decompress', - 'lib/dictBuilder', -] - -zstd_includes_legacy = [ - 'lib/deprecated', - 'lib/legacy', -] - - -def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False): - if prefer_system: - if system_prefix: - print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['zstd']) - - if pc and pc.installed('libzstd', '>= 1.3.0'): - print('Detected and preferring libzstd [via pkg-config]') - return pc.parse('libzstd') - - print('Using bundled ZSTD') - sources = multi_join(zstd_sources, zstd_bundled_path) - if legacy: - sources += multi_join(zstd_sources_legacy, zstd_bundled_path) - include_dirs = multi_join(zstd_includes, zstd_bundled_path) - if legacy: - include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path) - extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] - # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then - if legacy: - extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] - if multithreaded: - extra_compile_args += ['-DZSTD_MULTITHREAD', ] - define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, - extra_compile_args=extra_compile_args, define_macros=define_macros) - - -# lz4 files, structure as seen in lz4 project repository: - -# path relative (to this file) to the bundled library source code files -lz4_bundled_path = 'src/borg/algorithms/lz4' - -lz4_sources = [ - 'lib/lz4.c', -] - -lz4_includes = [ - 'lib', -] - - -def lz4_ext_kwargs(pc, prefer_system, system_prefix): - if prefer_system: - if system_prefix: - print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['lz4']) - - if pc and pc.installed('liblz4', '>= 1.7.0'): - print('Detected and preferring liblz4 [via pkg-config]') - return pc.parse('liblz4') - - print('Using bundled LZ4') - sources = multi_join(lz4_sources, lz4_bundled_path) - include_dirs = multi_join(lz4_includes, lz4_bundled_path) - define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) + raise Exception('Could not find lz4 lib/headers, please set BORG_LIBLZ4_PREFIX') diff --git a/src/borg/algorithms/checksums.pyx b/src/borg/algorithms/checksums.pyx index 95081649b..c2c773d6d 100644 --- a/src/borg/algorithms/checksums.pyx +++ b/src/borg/algorithms/checksums.pyx @@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c": int _have_clmul "have_clmul"() -cdef extern from "xxhash-libselect.h": +cdef extern from "xxhash.h": ctypedef struct XXH64_canonical_t: char digest[8] diff --git a/src/borg/algorithms/lz4-libselect.h b/src/borg/algorithms/lz4-libselect.h deleted file mode 100644 index 7abab8262..000000000 --- a/src/borg/algorithms/lz4-libselect.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef BORG_USE_BUNDLED_LZ4 -#include "lz4/lib/lz4.h" -#else -#include -#endif diff --git a/src/borg/algorithms/xxhash-libselect.h b/src/borg/algorithms/xxhash-libselect.h deleted file mode 100644 index ff7edffa1..000000000 --- a/src/borg/algorithms/xxhash-libselect.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef BORG_USE_BUNDLED_XXHASH -#include "xxh64/xxhash.h" -#else -#include -#endif diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h deleted file mode 100644 index d505823f6..000000000 --- a/src/borg/algorithms/zstd-libselect.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef BORG_USE_BUNDLED_ZSTD -#include -#else -#include -#endif diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 4beec0782..32ecb0b53 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -30,13 +30,13 @@ from .helpers import Buffer, DecompressionError API_VERSION = '1.2_02' -cdef extern from "algorithms/lz4-libselect.h": +cdef extern from "lz4.h": int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_compressBound(int inputSize) nogil -cdef extern from "algorithms/zstd-libselect.h": +cdef extern from "zstd.h": size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) nogil size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil size_t ZSTD_compressBound(size_t srcSize) nogil