1
1
mirror of https://github.com/rui314/mold.git synced 2024-10-05 17:17:40 +03:00

Update the bundled xxhash to v0.8.2

This commit is contained in:
Rui Ueyama 2023-07-21 19:26:36 +09:00
parent b5b3ffa6fe
commit 44b15cbb35
58 changed files with 50919 additions and 1226 deletions

View File

@ -0,0 +1,7 @@
# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"

View File

@ -30,7 +30,7 @@
# | Item | Section in the right pane |
# | ------------------------- | ------------------------------------- |
# | OS, VM | Set up job |
# | git repo, commit hash | Run actions/checkout@v2 |
# | git repo, commit hash | Run actions/checkout@v3 |
# | gcc, tools | Environment info |
#
# - To fail earlier, order of tests in the same job are roughly sorted by
@ -53,6 +53,13 @@ name: xxHash CI tests
on: [push, pull_request]
concurrency:
group: fast-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
xxhash-c-compilers:
name: CC=${{ matrix.cc }}, ${{ matrix.os }}
@ -73,27 +80,26 @@ jobs:
# gcc
{ pkgs: '', cc: gcc, cxx: g++, avx512: 'true', os: ubuntu-latest, },
{ pkgs: 'gcc-11 g++-11 lib32gcc-11-dev', cc: gcc-11, cxx: g++-11, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'gcc-10 g++-10 lib32gcc-10-dev', cc: gcc-10, cxx: g++-10, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'gcc-9 g++-9 lib32gcc-9-dev', cc: gcc-9, cxx: g++-9, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'gcc-13 g++-13 lib32gcc-13-dev', cc: gcc-13, cxx: g++-13, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'gcc-12 g++-12 lib32gcc-12-dev', cc: gcc-12, cxx: g++-12, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'gcc-11 g++-11 lib32gcc-11-dev', cc: gcc-11, cxx: g++-11, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'gcc-10 g++-10 lib32gcc-10-dev', cc: gcc-10, cxx: g++-10, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'gcc-9 g++-9 lib32gcc-9-dev', cc: gcc-9, cxx: g++-9, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'gcc-8 g++-8 lib32gcc-8-dev', cc: gcc-8, cxx: g++-8, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'gcc-7 g++-7 lib32gcc-7-dev', cc: gcc-7, cxx: g++-7, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'gcc-6 g++-6 lib32gcc-6-dev', cc: gcc-6, cxx: g++-6, avx512: 'true', os: ubuntu-18.04, },
{ pkgs: 'gcc-5 g++-5 lib32gcc-5-dev', cc: gcc-5, cxx: g++-5, avx512: 'true', os: ubuntu-18.04, },
{ pkgs: 'gcc-4.8 g++-4.8 lib32gcc-4.8-dev ', cc: gcc-4.8, cxx: g++-4.8, avx512: 'false', os: ubuntu-18.04, },
# clang
{ pkgs: '', cc: clang, cxx: clang++, avx512: 'true', os: ubuntu-latest, },
{ pkgs: 'clang-12', cc: clang-12, cxx: clang++-12, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-11', cc: clang-11, cxx: clang++-11, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-15', cc: clang-15, cxx: clang++-15, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'clang-14', cc: clang-14, cxx: clang++-14, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'clang-13', cc: clang-13, cxx: clang++-13, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'clang-12', cc: clang-12, cxx: clang++-12, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'clang-11', cc: clang-11, cxx: clang++-11, avx512: 'true', os: ubuntu-22.04, },
{ pkgs: 'clang-10', cc: clang-10, cxx: clang++-10, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-9', cc: clang-9, cxx: clang++-9, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-8', cc: clang-8, cxx: clang++-8, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-7', cc: clang-7, cxx: clang++-7, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-6.0', cc: clang-6.0, cxx: clang++-6.0, avx512: 'true', os: ubuntu-20.04, },
{ pkgs: 'clang-5.0', cc: clang-5.0, cxx: clang++-5.0, avx512: 'true', os: ubuntu-18.04, },
{ pkgs: 'clang-4.0', cc: clang-4.0, cxx: clang++-4.0, avx512: 'true', os: ubuntu-18.04, },
{ pkgs: 'clang-3.9', cc: clang-3.9, cxx: clang++-3.9, avx512: 'true', os: ubuntu-18.04, },
]
runs-on: ${{ matrix.os }}
@ -102,7 +108,7 @@ jobs:
CC: ${{ matrix.cc }}
CXX: ${{ matrix.cxx }}
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: apt-get install
run: |
@ -139,12 +145,23 @@ jobs:
run: |
CFLAGS="-Wall -Wextra -Werror" make DISPATCH=1 clean default
- name: XXH_SIZE_OPT == 2
if: always()
run: |
CFLAGS="-Os -DXXH_SIZE_OPT=2 -Wall -Wextra -Werror" make clean xxhsum
- name: noxxh3test
if: always()
run: |
# check library can be compiled with XXH_NO_XXH3, resulting in no XXH3_* symbol
make clean noxxh3test
- name: nostreamtest
if: always()
run: |
# check library can be compiled with XXH_NO_STREAM, resulting in no streaming symbols
make clean noxxh3test
- name: make avx512f
if: ${{ matrix.avx512 == 'true' }}
run: |
@ -160,7 +177,7 @@ jobs:
name: Linux x64 check results consistency
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Environment info
run: |
@ -195,14 +212,68 @@ jobs:
make -C tests/bench
ubuntu-wasm:
name: Ubuntu Node ${{ matrix.node-version }} WebAssembly/asm.js tests
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [16.x, 17.x, 18.x]
env:
EM_VERSION: 3.1.33 # TODO: more emsdk versions
EM_CACHE_FOLDER: emsdk-cache-${{ matrix.node-version }}
CC: emcc
steps:
- uses: actions/checkout@v3 # https://github.com/actions/checkout
- name: Setup cache
id: cache-system-libraries
uses: actions/cache@v3
with:
path: ${{env.EM_CACHE_FOLDER}}
key: em${{env.EM_VERSION}}-node${{ matrix.node-version }}-${{ runner.os }}
- name: Setup emsdk
uses: mymindstorm/setup-emsdk@v12
with:
version: ${{env.EM_VERSION}}
actions-cache-folder: ${{env.EM_CACHE_FOLDER}}
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.node-version }}
- name: Environment info
run: |
echo && node -p '`node version: ${process.versions.node}, v8 version: ${process.versions.v8}`'
echo && emcc --version
echo && make -v
echo && cat /proc/cpuinfo || echo /proc/cpuinfo is not present
- name: Scalar code path
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" RUN_ENV="node" NODE_JS=1 make clean check
- name: SIMD128 (via NEON SIMDe) code path (XXH_VECTOR=XXH_NEON)
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_NEON -msimd128" RUN_ENV="node" NODE_JS=1 make clean check
- name: Scalar asm.js (-sWASM=0)
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" RUN_ENV="node" NODE_JS=1 LDFLAGS="-sWASM=0" make clean check
ubuntu-misc:
name: Linux x64 misc tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: apt-get install
run: |
sudo apt-get update
sudo apt-get install valgrind cppcheck
- name: Environment info
@ -232,12 +303,20 @@ jobs:
run: |
make lint-unicode
- name: test-filename-escape
# See also issue #695 - https://github.com/Cyan4973/xxHash/issues/695
run: |
make clean test-filename-escape
- name: test-cli-comment-line
run: |
make clean test-cli-comment-line
ubuntu-cmake-unofficial:
name: Linux x64 cmake unofficial build test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Environment info
run: |
@ -249,10 +328,44 @@ jobs:
- name: cmake
run: |
cd cmake_unofficial
mkdir build
cmake -Bbuild
cd build
cmake ..
CFLAGS=-Werror make
mkdir -p test_install_dir
DESTDIR=test_install_dir cmake --install .
- name: cmake pkgconfig generation
run: |
cd cmake_unofficial
pwd
ls
rm -rf build
cmake -Bbuild -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DCMAKE_INSTALL_LIBDIR=/usr/lib
echo "checking content of libxxhash.pc"
cat build/libxxhash.pc | grep "libdir=/usr/lib"
cat build/libxxhash.pc | grep "includedir=/usr/include"
- name: cmake minimum version v2.8.12 test
run: |
mkdir -p cmake_bins
cd cmake_bins
wget https://cmake.org/files/v2.8/cmake-2.8.12.2-Linux-i386.tar.gz
tar xzf cmake-2.8.12.2-Linux-i386.tar.gz
cd ../cmake_unofficial
rm -rf build
pwd
ls
mkdir -p build
cd build
../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --version
../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake ..
../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --build .
mkdir -p test_install_dir
DESTDIR=test_install_dir ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --install .
rm -rf *
../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake -DCMAKE_BUILD_TYPE=Debug ..
../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --build .
# Linux, { ARM, ARM64, PPC64LE, PPC64, S390X }
@ -260,32 +373,64 @@ jobs:
qemu-consistency:
name: QEMU ${{ matrix.name }}
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
include: [
{ name: ARM, xcc_pkg: gcc-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static },
{ name: ARM64, xcc_pkg: gcc-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static },
{ name: PPC64LE-gcc9, xcc_pkg: gcc-9-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc-9, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static },
{ name: PPC64LE-gcc10, xcc_pkg: gcc-10-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc-10, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static },
{ name: PPC64-gcc9, xcc_pkg: gcc-9-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc-9, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static },
{ name: PPC64-gcc10, xcc_pkg: gcc-10-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc-10, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static },
{ name: S390X, xcc_pkg: gcc-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static },
{ name: MIPS, xcc_pkg: gcc-mips-linux-gnu, xcc: mips-linux-gnu-gcc, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static },
{ name: 'ARM', xcc_pkg: gcc-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static, os: ubuntu-latest, },
{ name: 'AARCH64', xcc_pkg: gcc-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static, os: ubuntu-latest, },
{ name: 'PPC64LE', xcc_pkg: gcc-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static, os: ubuntu-latest, },
{ name: 'PPC64', xcc_pkg: gcc-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static, os: ubuntu-latest, },
{ name: 'S390X', xcc_pkg: gcc-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static, os: ubuntu-latest, },
{ name: 'MIPS', xcc_pkg: gcc-mips-linux-gnu, xcc: mips-linux-gnu-gcc, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static, os: ubuntu-latest, },
{ name: 'M68K', xcc_pkg: gcc-m68k-linux-gnu, xcc: m68k-linux-gnu-gcc, xemu_pkg: qemu-system-m68k, xemu: qemu-m68k-static, os: ubuntu-latest, },
{ name: 'RISC-V', xcc_pkg: gcc-riscv64-linux-gnu, xcc: riscv64-linux-gnu-gcc, xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static, os: ubuntu-latest, },
{ name: 'ARM, gcc-10', xcc_pkg: gcc-10-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc-10, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static, os: ubuntu-20.04, },
{ name: 'AARCH64, gcc-10', xcc_pkg: gcc-10-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc-10, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static, os: ubuntu-20.04, },
{ name: 'PPC64LE, gcc-10', xcc_pkg: gcc-10-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc-10, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static, os: ubuntu-20.04, },
{ name: 'PPC64, gcc-10', xcc_pkg: gcc-10-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc-10, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static, os: ubuntu-20.04, },
{ name: 'S390X, gcc-10', xcc_pkg: gcc-10-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc-10, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static, os: ubuntu-20.04, },
{ name: 'MIPS, gcc-10', xcc_pkg: gcc-10-mips-linux-gnu, xcc: mips-linux-gnu-gcc-10, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static, os: ubuntu-20.04, },
{ name: 'ARM, gcc-9', xcc_pkg: gcc-9-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc-9, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static, os: ubuntu-20.04, },
# SVE tests require at least gcc-10.1
# { name: 'AARCH64, gcc-9', xcc_pkg: gcc-9-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc-9, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static, os: ubuntu-20.04, },
{ name: 'PPC64LE, gcc-9', xcc_pkg: gcc-9-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc-9, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static, os: ubuntu-20.04, },
{ name: 'PPC64, gcc-9', xcc_pkg: gcc-9-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc-9, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static, os: ubuntu-20.04, },
{ name: 'S390X, gcc-9', xcc_pkg: gcc-9-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc-9, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static, os: ubuntu-20.04, },
{ name: 'MIPS, gcc-9', xcc_pkg: gcc-9-mips-linux-gnu, xcc: mips-linux-gnu-gcc-9, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static, os: ubuntu-20.04, },
{ name: 'ARM, gcc-8', xcc_pkg: gcc-8-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc-8, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static, os: ubuntu-20.04, },
# aarch64-linux-gnu-gcc-8 linker has an issue for LDFLAGS="-static"
# { name: 'AARCH64, gcc-8', xcc_pkg: gcc-8-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc-8, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static, os: ubuntu-20.04, },
{ name: 'PPC64LE, gcc-8', xcc_pkg: gcc-8-powerpc64le-linux-gnu, xcc: powerpc64le-linux-gnu-gcc-8, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64le-static, os: ubuntu-20.04, },
{ name: 'PPC64, gcc-8', xcc_pkg: gcc-8-powerpc64-linux-gnu, xcc: powerpc64-linux-gnu-gcc-8, xemu_pkg: qemu-system-ppc, xemu: qemu-ppc64-static, os: ubuntu-20.04, },
{ name: 'S390X, gcc-8', xcc_pkg: gcc-8-s390x-linux-gnu, xcc: s390x-linux-gnu-gcc-8, xemu_pkg: qemu-system-s390x, xemu: qemu-s390x-static, os: ubuntu-20.04, },
# ubuntu-20.04 fails to retrieve gcc-8-mips-linux-gnu for some reason.
# { name: 'MIPS, gcc-8', xcc_pkg: gcc-8-mips-linux-gnu, xcc: mips-linux-gnu-gcc-8, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static, os: ubuntu-20.04, },
]
env: # Set environment variables
XCC: ${{ matrix.xcc }}
XEMU: ${{ matrix.xemu }}
MOREFLAGS: -Werror
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- name: apt update & install
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: apt update & install (1)
run: |
sudo apt-get update
sudo apt-get install gcc-multilib g++-multilib qemu-utils qemu-user-static
- name: Environment info (1)
run: |
echo && apt-cache search "^gcc-" | grep "linux" | sort
- name: apt update & install (2)
run: |
sudo apt-get install ${{ matrix.xcc_pkg }} ${{ matrix.xemu_pkg }}
- name: Environment info
- name: Environment info (2)
run: |
echo && which $XCC
echo && $XCC --version
@ -294,41 +439,54 @@ jobs:
echo && $XEMU --version
- name: ARM (XXH_VECTOR=[ scalar, NEON ])
if: ${{ matrix.name == 'ARM' }}
if: ${{ startsWith(matrix.name, 'ARM') }}
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_NEON" CFLAGS="-O3 -march=armv7-a -fPIC -mfloat-abi=softfp -mfpu=neon-vfpv4" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
- name: ARM64 (XXH_VECTOR=[ scalar, NEON ])
if: ${{ matrix.name == 'ARM64' }}
- name: AARCH64 (XXH_VECTOR=[ scalar, NEON, SVE ])
if: ${{ startsWith(matrix.name, 'AARCH64') }}
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_NEON" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_SVE" LDFLAGS="-static" CC="$XCC -march=armv8.2-a+sve" RUN_ENV="$XEMU -cpu max,sve128=on,sve256=off,sve512=off,sve1024=off,sve2048=off" make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_SVE" LDFLAGS="-static" CC="$XCC -march=armv8.2-a+sve" RUN_ENV="$XEMU -cpu max,sve128=on,sve256=on,sve512=off,sve1024=off,sve2048=off" make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_SVE" LDFLAGS="-static" CC="$XCC -march=armv8.2-a+sve" RUN_ENV="$XEMU -cpu max,sve128=on,sve256=on,sve512=on,sve1024=off,sve2048=off" make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_SVE" LDFLAGS="-static" CC="$XCC -march=armv8.2-a+sve" RUN_ENV="$XEMU -cpu max,sve128=on,sve256=on,sve512=on,sve1024=on,sve2048=off" make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_SVE" LDFLAGS="-static" CC="$XCC -march=armv8.2-a+sve" RUN_ENV="$XEMU -cpu max,sve128=on,sve256=on,sve512=on,sve1024=on,sve2048=on" make clean check
- name: ${{ matrix.name }} (XXH_VECTOR=[ scalar, VSX ])
- name: PPC64(LE) (XXH_VECTOR=[ scalar, VSX ])
if: ${{ startsWith(matrix.name, 'PPC64') }}
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_VSX" CFLAGS="-O3 -maltivec -mvsx -mpower8-vector -mcpu=power8" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
- name: S390X (XXH_VECTOR=[ scalar, VSX ])
if: ${{ matrix.name == 'S390X' }}
if: ${{ startsWith(matrix.name, 'S390X') }}
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-O3 -march=arch11 -mzvector" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
- name: MIPS (XXH_VECTOR=[ scalar ])
if: ${{ matrix.name == 'MIPS' }}
- name: MIPS-M68K-RISCV (XXH_VECTOR=[ scalar ])
if: ${{ startsWith(matrix.name, 'MIPS') || startsWith(matrix.name, 'M68K') || startsWith(matrix.name, 'RISC-V') }}
run: |
LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
# macOS
macos-latest-general:
name: macOS general test
runs-on: macos-latest
# macOS, { 11 }
macos-general:
name: ${{ matrix.system.os }}
runs-on: ${{ matrix.system.os }}
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
system: [
{ os: macos-11 },
{ os: macos-12 },
]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Environment info
run: |
@ -345,30 +503,31 @@ jobs:
make clean test MOREFLAGS='-Werror' | tee
# Windows, { VC++2019, VC++2017 } x { x64, Win32, ARM, ARM64 }
# Windows, { VC++2022, VC++2019, VC++2017 } x { x64, Win32, ARM, ARM64 }
#
# - Default shell for Windows environment is PowerShell Core.
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
#
# - "windows-2022" uses Visual Studio 2022.
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2022-Readme.md#visual-studio-enterprise-2022
#
# - "windows-2019" uses Visual Studio 2019.
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md#visual-studio-enterprise-2019
#
# - "windows-2016" uses Visual Studio 2017.
# https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md#visual-studio-enterprise-2017
windows-visualc-general:
name: ${{ matrix.system.vc }}, ${{ matrix.arch }}
runs-on: ${{ matrix.system.os }} # Runs-on foreach value of strategy.matrix.system.os
strategy:
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
fail-fast: false # 'false' means: Don't stop matrix workflows even if some matrix failed.
matrix:
system: [
{ os: windows-2019, vc: "VC++ 2019" },
{ os: windows-2016, vc: "VC++ 2017" },
{ os: windows-2022, vc: "VC++ 2022", clangcl: 'true', },
{ os: windows-2019, vc: "VC++ 2019", clangcl: 'true', },
]
arch: [ x64, Win32, ARM, ARM64 ]
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Build ${{ matrix.system.os }}, ${{ matrix.arch }}
run: |
@ -384,6 +543,38 @@ jobs:
run: |
.\cmake_unofficial\build\Release\xxhsum.exe -bi1
- name: Build ${{ matrix.system.os }}, clang-cl, ${{ matrix.arch }}
if: ${{ matrix.system.clangcl == 'true' }}
run: |
cd cmake_unofficial
mkdir build-clang-cl
cd build-clang-cl
cmake .. -DCMAKE_BUILD_TYPE=Release -A x64 -DCMAKE_GENERATOR_TOOLSET=ClangCL
cmake --build . --config Release
- name: Test (clang-cl)
# Run benchmark for testing only if target arch is x64 or Win32.
if: ${{ matrix.system.clangcl == 'true' && ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
run: |
.\cmake_unofficial\build-clang-cl\Release\xxhsum.exe -bi1
- name: Build ${{ matrix.system.os }}, clang-cl, ${{ matrix.arch }}, with DISPATCH
# DISPATCH only if target arch is x64 or Win32.
if: ${{ matrix.system.clangcl == 'true' && ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
run: |
cd cmake_unofficial
mkdir build-clang-cl-dispatch
cd build-clang-cl-dispatch
cmake .. -DCMAKE_BUILD_TYPE=Release -DDISPATCH=ON -A x64 -DCMAKE_GENERATOR_TOOLSET=ClangCL
cmake --build . --config Release
- name: Runtime Test (clang-cl + DISPATCH)
# Run benchmark for testing only if target arch is x64 or Win32.
if: ${{ matrix.system.clangcl == 'true' && ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
run: |
.\cmake_unofficial\build-clang-cl-dispatch\Release\xxhsum.exe -V | grep autoVec
.\cmake_unofficial\build-clang-cl-dispatch\Release\xxhsum.exe -bi1
# Windows, { mingw64, mingw32 }
#
@ -406,8 +597,8 @@ jobs:
run:
shell: msys2 {0}
steps:
- uses: actions/checkout@v2 # https://github.com/actions/checkout
- uses: msys2/setup-msys2@v2 # https://github.com/msys2/setup-msys2
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- uses: msys2/setup-msys2@7efe20baefed56359985e327d329042cde2434ff # v2
with:
msystem: MSYS
install: mingw-w64-i686-make ${{ matrix.toolchain }}

View File

@ -18,6 +18,8 @@ xxhsum_privateXXH
xxhsum_inlinedXXH
dispatch
tests/generate_unicode_test
tests/sanity_test
tests/sanity_test_vectors_generator
# local conf
.clang_complete
@ -45,3 +47,6 @@ infer-out
tmp*
tests/*.unicode
tests/unicode_test*
# Doxygen
doxygen/

View File

@ -1,3 +1,27 @@
v0.8.2
- fix : XXH3 S390x vector implementation (@hzhuang1)
- fix : PowerPC vector compilation with IBM XL compiler (@MaxiBoether)
- perf : improved WASM speed by x2/x3 using SIMD128 (@easyaspi314)
- perf : improved speed (+20%) for XXH3 on ARM NEON (@easyaspi314)
- cli : Fix filename contain /LF character (@t-mat)
- cli : Support # comment lines in --check files (@t-mat)
- cli : Support commands --binary and --ignore-missing (@t-mat)
- build: fix -Og compilation (@easyaspi314, @t-mat)
- build: fix pkgconfig generation with cmake (@ilya-fedin)
- build: fix icc compilation
- build: fix cmake install directories
- build: new build options XXH_NO_XXH3, XXH_SIZE_OPT and XXH_NO_STREAM to reduce binary size (@easyaspi314)
- build: dedicated install targets (@ffontaine)
- build: support DISPATCH mode in cmake (@hzhuang1)
- portability: fix x86dispatch when building with Visual + clang-cl (@t-mat)
- portability: SVE vector implementation of XXH3 (@hzhuang1)
- portability: compatibility with freestanding environments, using XXH_NO_STDLIB
- portability: can build on Haiku (@Begasus)
- portability: validated on m68k and risc-v
- doc : XXH3 specification (@Adrien1018)
- doc : improved doxygen documentation (@easyaspi314, @t-mat)
- misc : dedicated sanity test binary (@t-mat)
v0.8.1
- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc
- perf : improved XXH64 speed and latency on small inputs

View File

@ -2,13 +2,13 @@
DOXYFILE_ENCODING = UTF-8
PROJECT_NAME = "xxHash"
PROJECT_NUMBER = "0.8.0"
PROJECT_NUMBER = "0.8.2"
PROJECT_BRIEF = "Extremely fast non-cryptographic hash function"
OUTPUT_DIRECTORY = doxygen
OUTPUT_LANGUAGE = English
# We already separate the internal docs.
INTERNAL_DOCS = YES
INTERNAL_DOCS = NO
# Consistency
SORT_MEMBER_DOCS = NO
BRIEF_MEMBER_DESC = YES
@ -28,8 +28,10 @@ MARKDOWN_SUPPORT = YES
# xxHash is a C library
OPTIMIZE_OUTPUT_FOR_C = YES
# So we can document the internals
EXTRACT_STATIC = YES
# We hide private part from public document
EXTRACT_STATIC = NO
# We hide private part from public document
EXTRACT_PRIVATE = NO
# Document the macros
MACRO_EXPANSION = YES
EXPAND_ONLY_PREDEF = YES
@ -42,6 +44,9 @@ PREDEFINED = "XXH_DOXYGEN=" \
"XSUM_API=" \
"XXH_STATIC_LINKING_ONLY" \
"XXH_IMPLEMENTATION" \
"XXH_PUREF=[[gnu::pure]]" \
"XXH_CONSTF=[[gnu::const]]" \
"XXH_MALLOCF=[[gnu::malloc]]" \
"XXH_ALIGN(N)=alignas(N)" \
"XXH_ALIGN_MEMBER(align,type)=alignas(align) type"

63
third-party/xxhash/Doxyfile-internal vendored Normal file
View File

@ -0,0 +1,63 @@
# Doxygen config for xxHash
DOXYFILE_ENCODING = UTF-8
PROJECT_NAME = "xxHash"
PROJECT_NUMBER = "0.8.2"
PROJECT_BRIEF = "Extremely fast non-cryptographic hash function"
OUTPUT_DIRECTORY = doxygen
OUTPUT_LANGUAGE = English
# We already separate the internal docs.
INTERNAL_DOCS = YES
# Consistency
SORT_MEMBER_DOCS = NO
BRIEF_MEMBER_DESC = YES
REPEAT_BRIEF = YES
# Warnings
QUIET = YES
# Until we document everything
WARN_IF_UNDOCUMENTED = NO
# TODO: Add the other files. It is just xxhash.h for now.
FILE_PATTERNS = xxhash.h xxh_x86dispatch.c
# Note: xxHash's source files are technically ASCII only.
INPUT_ENCODING = UTF-8
TAB_SIZE = 4
MARKDOWN_SUPPORT = YES
# xxHash is a C library
OPTIMIZE_OUTPUT_FOR_C = YES
# So we can document the internals
EXTRACT_STATIC = YES
# We show private part in the internal document
EXTRACT_PRIVATE = YES
# Document the macros
MACRO_EXPANSION = YES
EXPAND_ONLY_PREDEF = YES
# Predefine some macros to clean up the output.
PREDEFINED = "XXH_DOXYGEN=" \
"XXH_PUBLIC_API=" \
"XXH_FORCE_INLINE=static inline" \
"XXH_NO_INLINE=static" \
"XXH_RESTRICT=restrict" \
"XSUM_API=" \
"XXH_STATIC_LINKING_ONLY" \
"XXH_IMPLEMENTATION" \
"XXH_PUREF=[[gnu::pure]]" \
"XXH_CONSTF=[[gnu::const]]" \
"XXH_MALLOCF=[[gnu::malloc]]" \
"XXH_ALIGN(N)=alignas(N)" \
"XXH_ALIGN_MEMBER(align,type)=alignas(align) type"
# We want HTML docs
GENERATE_HTML = YES
HTML_OUTPUT = html
HTML_FILE_EXTENSION = .html
# Tweak the colors a bit
HTML_COLORSTYLE_HUE = 220
HTML_COLORSTYLE_GAMMA = 100
HTML_COLORSTYLE_SAT = 100
# We don't want LaTeX.
GENERATE_LATEX = NO

View File

@ -1,5 +1,5 @@
xxHash Library
Copyright (c) 2012-2020 Yann Collet
Copyright (c) 2012-2021 Yann Collet
All rights reserved.
BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)

View File

@ -1,6 +1,6 @@
# ################################################################
# xxHash Makefile
# Copyright (C) 2012-2020 Yann Collet
# Copyright (C) 2012-2021 Yann Collet
#
# GPL v2 License
#
@ -55,6 +55,13 @@ else
EXT =
endif
ifeq ($(NODE_JS),1)
# Link in unrestricted filesystem support
LDFLAGS += -sNODERAWFS
# Set flag to fix isatty() support
CPPFLAGS += -DXSUM_NODE_JS=1
endif
# OS X linker doesn't support -soname, and use different extension
# see: https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html
ifeq ($(UNAME), Darwin)
@ -174,6 +181,7 @@ clean: ## remove all build artifacts
$(Q)$(RM) -r *.dSYM # Mac OS-X specific
$(Q)$(RM) core *.o *.obj *.$(SHARED_EXT) *.$(SHARED_EXT).* *.a libxxhash.pc
$(Q)$(RM) xxhsum$(EXT) xxhsum32$(EXT) xxhsum_inlinedXXH$(EXT) dispatch$(EXT)
$(Q)$(RM) xxhsum.wasm xxhsum.js xxhsum.html
$(Q)$(RM) xxh32sum$(EXT) xxh64sum$(EXT) xxh128sum$(EXT)
$(Q)$(RM) $(XXHSUM_SRC_DIR)/*.o $(XXHSUM_SRC_DIR)/*.obj
$(MAKE) -C tests clean
@ -189,8 +197,9 @@ clean: ## remove all build artifacts
# make check can be run with cross-compiled binaries on emulated environments (qemu user mode)
# by setting $(RUN_ENV) to the target emulation environment
.PHONY: check
check: xxhsum ## basic tests for xxhsum CLI, set RUN_ENV for emulated environments
check: xxhsum test_sanity ## basic tests for xxhsum CLI, set RUN_ENV for emulated environments
# stdin
# If you get "Wrong parameters" on Emscripten+Node.js, recompile with `NODE_JS=1`
$(RUN_ENV) ./xxhsum$(EXT) < xxhash.c
# multiple files
$(RUN_ENV) ./xxhsum$(EXT) xxhash.*
@ -216,6 +225,10 @@ check: xxhsum ## basic tests for xxhsum CLI, set RUN_ENV for emulated environm
test-unicode:
$(MAKE) -C tests test_unicode
.PHONY: test_sanity
test_sanity:
$(MAKE) -C tests test_sanity
.PHONY: test-mem
VALGRIND = valgrind --leak-check=yes --error-exitcode=1
test-mem: RUN_ENV = $(VALGRIND)
@ -302,6 +315,18 @@ test-xxhsum-c: xxhsum
echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
@$(RM) .test.*
.PHONY: test-filename-escape
test-filename-escape:
$(MAKE) -C tests test_filename_escape
.PHONY: test-cli-comment-line
test-cli-comment-line:
$(MAKE) -C tests test_cli_comment_line
.PHONY: test-cli-ignore-missing
test-cli-ignore-missing:
$(MAKE) -C tests test_cli_ignore_missing
.PHONY: armtest
armtest: clean
@echo ---- test ARM compilation ----
@ -312,6 +337,11 @@ clangtest: clean
@echo ---- test clang compilation ----
CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion" $(MAKE) all
.PHONY: gcc-og-test
gcc-og-test: clean
@echo ---- test gcc -Og compilation ----
CFLAGS="-Og -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -fPIC" MOREFLAGS="-Werror" $(MAKE) all
.PHONY: cxxtest
cxxtest: clean
@echo ---- test C++ compilation ----
@ -332,14 +362,35 @@ c90test: xxhash.c
$(RM) xxhash.o
endif
.PHONY: noxxh3test
noxxh3test: CPPFLAGS += -DXXH_NO_XXH3
noxxh3test: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
noxxh3test: OFILE = xxh_noxxh3.o
noxxh3test: xxhash.c
@echo ---- test compilation without XXH3 ----
$(RM) xxhash.o
$(CC) $(FLAGS) $^ -c
$(NM) xxhash.o | $(GREP) XXH3_ ; test $$? -eq 1
$(RM) xxhash.o
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) XXH3_ ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: nostreamtest
nostreamtest: CPPFLAGS += -DXXH_NO_STREAM
nostreamtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
nostreamtest: OFILE = xxh_nostream.o
nostreamtest: xxhash.c
@echo ---- test compilation without streaming ----
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) update ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: nostdlibtest
nostdlibtest: CPPFLAGS += -DXXH_NO_STDLIB
nostdlibtest: CFLAGS += -Werror -pedantic -Wno-long-long # XXH64 requires long long support
nostdlibtest: OFILE = xxh_nostdlib.o
nostdlibtest: xxhash.c
@echo ---- test compilation without \<stdlib.h\> ----
$(CC) $(FLAGS) -c $^ -o $(OFILE)
$(NM) $(OFILE) | $(GREP) "U _free\|U free" ; test $$? -eq 1
$(RM) $(OFILE)
.PHONY: usan
usan: CC=clang
@ -387,7 +438,7 @@ preview-man: man
.PHONY: test
test: DEBUGFLAGS += -DXXH_DEBUGLEVEL=1
test: all namespaceTest check test-xxhsum-c c90test test-tools
test: all namespaceTest check test-xxhsum-c c90test test-tools noxxh3test nostdlibtest
.PHONY: test-inline
test-inline:
@ -395,20 +446,47 @@ test-inline:
.PHONY: test-all
test-all: CFLAGS += -Werror
test-all: test test32 test-unicode clangtest cxxtest usan test-inline listL120 trailingWhitespace
test-all: test test32 test-unicode clangtest gcc-og-test cxxtest usan test-inline listL120 trailingWhitespace test-xxh-nnn-sums
.PHONY: test-tools
test-tools:
CFLAGS=-Werror $(MAKE) -C tests/bench
CFLAGS=-Werror $(MAKE) -C tests/collisions
.PHONY: test-xxh-nnn-sums
test-xxh-nnn-sums: xxhsum_and_links
./xxhsum README.md > tmp.xxhsum.out # xxhsum outputs xxh64
./xxh32sum README.md > tmp.xxh32sum.out
./xxh64sum README.md > tmp.xxh64sum.out
./xxh128sum README.md > tmp.xxh128sum.out
cat tmp.xxhsum.out
cat tmp.xxh32sum.out
cat tmp.xxh64sum.out
cat tmp.xxh128sum.out
./xxhsum -c tmp.xxhsum.out
./xxhsum -c tmp.xxh32sum.out
./xxhsum -c tmp.xxh64sum.out
./xxhsum -c tmp.xxh128sum.out
./xxh32sum -c tmp.xxhsum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh32sum -c tmp.xxh32sum.out
./xxh32sum -c tmp.xxh64sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh32sum -c tmp.xxh128sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh64sum -c tmp.xxhsum.out
./xxh64sum -c tmp.xxh32sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh64sum -c tmp.xxh64sum.out
./xxh64sum -c tmp.xxh128sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxhsum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh32sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh64sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh128sum.out
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note: $$, for Makefile compatibility)
find . -type f -name '*.c' -o -name '*.h' | while read -r filename; do awk 'length > 120 {print FILENAME "(" FNR "): " $$0}' $$filename; done
.PHONY: trailingWhitespace
trailingWhitespace:
! $(GREP) -E "`printf '[ \\t]$$'`" cli/*.{c,h,1} *.c *.h LICENSE Makefile cmake_unofficial/CMakeLists.txt
! $(GREP) -E "`printf '[ \\t]$$'`" cli/*.c cli/*.h cli/*.1 *.c *.h LICENSE Makefile cmake_unofficial/CMakeLists.txt
.PHONY: lint-unicode
lint-unicode:
@ -457,6 +535,7 @@ endif
INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_DATA ?= $(INSTALL) -m 644
INSTALL_DIR ?= $(INSTALL) -d -m 755
# Escape special symbols by putting each character into its separate class
@ -487,40 +566,55 @@ libxxhash.pc: libxxhash.pc.in
@echo creating pkgconfig
$(Q)$(SED) $(SED_ERE_OPT) -e 's|@PREFIX@|$(PREFIX)|' \
-e 's|@EXECPREFIX@|$(PCEXECDIR)|' \
-e 's|@LIBDIR@|$(PCLIBDIR)|' \
-e 's|@INCLUDEDIR@|$(PCINCDIR)|' \
-e 's|@LIBDIR@|$$\{exec_prefix\}/$(PCLIBDIR)|' \
-e 's|@INCLUDEDIR@|$$\{prefix\}/$(PCINCDIR)|' \
-e 's|@VERSION@|$(LIBVER)|' \
$< > $@
.PHONY: install
install: lib libxxhash.pc xxhsum ## install libraries, CLI, links and man page
@echo Installing libxxhash
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)
install_libxxhash.a: libxxhash.a
@echo Installing libxxhash.a
$(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_DATA) libxxhash.a $(DESTDIR)$(LIBDIR)
install_libxxhash: libxxhash
@echo Installing libxxhash
$(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_PROGRAM) $(LIBXXH) $(DESTDIR)$(LIBDIR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT)
install_libxxhash.includes:
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR) # includes
$(Q)$(INSTALL_DATA) xxhash.h $(DESTDIR)$(INCLUDEDIR)
$(Q)$(INSTALL_DATA) xxh3.h $(DESTDIR)$(INCLUDEDIR) # for compatibility, will be removed in v0.9.0
ifeq ($(DISPATCH),1)
$(Q)$(INSTALL_DATA) xxh_x86dispatch.h $(DESTDIR)$(INCLUDEDIR)
endif
install_libxxhash.pc: libxxhash.pc
@echo Installing pkgconfig
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
$(Q)$(INSTALL_DIR) $(DESTDIR)$(PKGCONFIGDIR)/
$(Q)$(INSTALL_DATA) libxxhash.pc $(DESTDIR)$(PKGCONFIGDIR)/
install_xxhsum: xxhsum
@echo Installing xxhsum
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(BINDIR)/ $(DESTDIR)$(MANDIR)/
$(Q)$(INSTALL_DIR) $(DESTDIR)$(BINDIR)/
$(Q)$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh128sum
install_man:
@echo Installing man pages
$(Q)$(INSTALL_DIR) $(DESTDIR)$(MANDIR)/
$(Q)$(INSTALL_DATA) $(MAN) $(DESTDIR)$(MANDIR)/xxhsum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)ln -sf $(MAN) $(DESTDIR)$(MANDIR)/xxh128sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh128sum.1
.PHONY: install
install: install_libxxhash.a install_libxxhash install_libxxhash.includes install_libxxhash.pc install_xxhsum install_man ## install libraries, CLI, links and man page
@echo xxhash installation completed
.PHONY: uninstall

View File

@ -2,20 +2,28 @@
xxHash - Extremely fast hash algorithm
======================================
xxHash is an Extremely fast Hash algorithm, running at RAM speed limits.
It successfully completes the [SMHasher](https://code.google.com/p/smhasher/wiki/SMHasher) test suite
which evaluates collision, dispersion and randomness qualities of hash functions.
Code is highly portable, and hashes are identical across all platforms (little / big endian).
xxHash is an Extremely fast Hash algorithm, processing at RAM speed limits.
Code is highly portable, and produces hashes identical across all platforms (little / big endian).
The library includes the following algorithms :
- XXH32 : generates 32-bit hashes, using 32-bit arithmetic
- XXH64 : generates 64-bit hashes, using 64-bit arithmetic
- XXH3 (since `v0.8.0`): generates 64 or 128-bit hashes, using vectorized arithmetic.
The 128-bit variant is called XXH128.
All variants successfully complete the [SMHasher](https://code.google.com/p/smhasher/wiki/SMHasher) test suite
which evaluates the quality of hash functions (collision, dispersion and randomness).
Additional tests, which evaluate more thoroughly speed and collision properties of 64-bit hashes, [are also provided](https://github.com/Cyan4973/xxHash/tree/dev/tests).
|Branch |Status |
|------------|---------|
|release | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=release)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Arelease+) |
|dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) |
Benchmarks
-------------------------
The reference system uses an Intel i7-9700K cpu, and runs Ubuntu x64 20.04.
The benchmarked reference system uses an Intel i7-9700K cpu, and runs Ubuntu x64 20.04.
The [open source benchmark program] is compiled with `clang` v10.0 using `-O3` flag.
| Hash Name | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment |
@ -43,7 +51,7 @@ The [open source benchmark program] is compiled with `clang` v10.0 using `-O3` f
note 1: Small data velocity is a _rough_ evaluation of algorithm's efficiency on small data. For more detailed analysis, please refer to next paragraph.
note 2: some algorithms feature _faster than RAM_ speed. In which case, they can only reach their full speed when input data is already in CPU cache (L3 or better). Otherwise, they max out on RAM speed limit.
note 2: some algorithms feature _faster than RAM_ speed. In which case, they can only reach their full speed potential when input is already in CPU cache (L3 or better). Otherwise, they max out on RAM speed limit.
### Small data
@ -59,7 +67,7 @@ which can be observed in the following graph:
![XXH3, latency, random size](https://user-images.githubusercontent.com/750081/61976089-aedeab00-af9f-11e9-9239-e5375d6c080f.png)
For a more detailed analysis, visit the wiki :
For a more detailed analysis, please visit the wiki :
https://github.com/Cyan4973/xxHash/wiki/Performance-comparison#benchmarks-concentrating-on-small-data-
Quality
@ -93,25 +101,29 @@ The following macros can be set at compilation time to modify libxxhash's behavi
with performance improvements observed in the +200% range .
See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details.
- `XXH_PRIVATE_API`: same outcome as `XXH_INLINE_ALL`. Still available for legacy support.
The name underlines that `XXH_*` symbols will not be exported.
The name underlines that `XXH_*` symbol names will not be exported.
- `XXH_NAMESPACE`: Prefixes all symbols with the value of `XXH_NAMESPACE`.
This macro can only use compilable character set.
Useful to evade symbol naming collisions,
in case of multiple inclusions of xxHash's source code.
Client applications still use the regular function names,
as symbols are automatically translated through `xxhash.h`.
- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation.
Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets.
Method `2` forces unaligned reads, which is not standards compliant, but might sometimes be the only way to extract better read performance.
Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction
- `XXH_FORCE_ALIGN_CHECK`: Use a faster direct read path when input is aligned.
This option can result in dramatic performance improvement when input to hash is aligned on 32 or 64-bit boundaries,
when running on architectures unable to load memory from unaligned addresses, or suffering a performance penalty from it.
It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses).
This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms.
- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for AVX2, and `-mavx512f` for AVX512).
- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation.
Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets.
Method `2` forces unaligned reads, which is not standard compliant, but might sometimes be the only way to extract better read performance.
Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction.
- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for `AVX2`, and `-mavx512f` for `AVX512`).
- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only.
- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only.
- `XXH_NO_STREAM`: Disables the streaming API, limiting it to single shot variants only.
- `XXH_SIZE_OPT`: `0`: default, optimize for speed
`1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization
`2`: makes code as small as possible, performance may cry
- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size.
Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not.
This is very useful when optimizing for smallest binary size,
@ -120,11 +132,16 @@ The following macros can be set at compilation time to modify libxxhash's behavi
- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump.
This is generally undesirable for performance, especially when hashing inputs of random sizes.
But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default.
- `XXH_NO_STDLIB`: Disable invocation of `<stdlib.h>` functions, notably `malloc()` and `free()`.
`libxxhash`'s `XXH*_createState()` will always fail and return `NULL`.
But one-shot hashing (like `XXH32()`) or streaming using statically allocated states
still work as expected.
This build flag is useful for embedded environments without dynamic allocation.
- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation.
Incompatible with dynamic linking, due to risks of ABI changes.
- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary.
Useful to reduce binary size, notably for applications which do not use `XXH3`.
- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (XXH3 and XXH64). Only XXH32 will be compiled.
Useful to reduce binary size, notably for applications which do not employ `XXH3`.
- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (`XXH3` and `XXH64`). Only `XXH32` will be compiled.
Useful for targets (architectures and compilers) without 64-bit support.
- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors.
- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time.
@ -134,9 +151,10 @@ The following macros can be set at compilation time to modify libxxhash's behavi
- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements.
This (slightly) slows down execution, but may help finding bugs during debugging sessions.
When compiling the Command Line Interface `xxhsum` with `make`, the following environment variables can also be set :
When compiling the Command Line Interface `xxhsum` using `make`, the following environment variables can also be set :
- `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems.
- `XXH_1ST_SPEED_TARGET` : select an initial speed target, expressed in MB/s, for the first speed test in benchmark mode. Benchmark will adjust the target at subsequent iterations, but the first test is made "blindly" by targeting this speed. Currently conservatively set to 10 MB/s, to support very slow (emulated) platforms.
- `NODE_JS=1` : When compiling `xxhsum` for Node.js with Emscripten, this links the `NODERAWFS` library for unrestricted filesystem access and patches `isatty` to make the command line utility correctly detect the terminal. This does make the binary specific to Node.js.
### Building xxHash - Using vcpkg
@ -150,7 +168,6 @@ You can download and install xxHash using the [vcpkg](https://github.com/Microso
The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
### Example
The simplest example calls xxhash 64-bit variant as a one-shot function

13
third-party/xxhash/SECURITY.md vendored Normal file
View File

@ -0,0 +1,13 @@
# Security Policy
## Supported Versions
Security updates are applied only to the latest release.
## Reporting a Vulnerability
If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
Please disclose it at [security advisory](https://github.com/Cyan4973/xxHash/security/advisories/new).
This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure.

3
third-party/xxhash/cli/.tipi/deps vendored Normal file
View File

@ -0,0 +1,3 @@
{
"Cyan4973/xxHash": { }
}

0
third-party/xxhash/cli/.tipi/opts vendored Normal file
View File

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -79,7 +79,7 @@
#endif
/* makes the next part easier */
#if defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
#if (defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) && !defined(_M_ARM64EC)
# define XSUM_ARCH_X64 1
# define XSUM_ARCH_X86 "x86_64"
#elif defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP)
@ -102,11 +102,21 @@
# else
# define XSUM_ARCH XSUM_ARCH_X86
# endif
#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)
# define XSUM_ARCH "aarch64 + NEON"
#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
# if defined(__ARM_FEATURE_SVE)
# define XSUM_ARCH "aarch64 + SVE"
# else
# define XSUM_ARCH "aarch64 + NEON"
# endif
#elif defined(__arm__) || defined(__thumb__) || defined(__thumb2__) || defined(_M_ARM)
/* ARM has a lot of different features that can change xxHash significantly. */
# if defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7))
# ifdef __ARM_ARCH
# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(__ARM_ARCH)
# else
# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(_M_ARM)
# endif
# if defined(_M_ARM) /* windows arm is always thumb-2 */ \
|| defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7))
# define XSUM_ARCH_THUMB " Thumb-2"
# elif defined(__thumb__)
# define XSUM_ARCH_THUMB " Thumb-1"
@ -114,17 +124,17 @@
# define XSUM_ARCH_THUMB ""
# endif
/* ARMv7 has unaligned by default */
# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARMV7VE)
# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARM)
# define XSUM_ARCH_UNALIGNED " + unaligned"
# else
# define XSUM_ARCH_UNALIGNED ""
# endif
# if defined(__ARM_NEON) || defined(__ARM_NEON__)
# if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM)
# define XSUM_ARCH_NEON " + NEON"
# else
# define XSUM_ARCH_NEON ""
# endif
# define XSUM_ARCH "ARMv" XSUM_EXPAND_AND_QUOTE(__ARM_ARCH) XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED
# define XSUM_ARCH "ARMv" XSUM_ARCH_ARM_VER XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED
#elif defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)
# if defined(__GNUC__) && defined(__POWER9_VECTOR__)
# define XSUM_ARCH "ppc64 + POWER9 vector"
@ -145,6 +155,12 @@
# define XSUM_ARCH "s390x"
#elif defined(__s390__)
# define XSUM_ARCH "s390"
#elif defined(__wasm__) || defined(__asmjs__) || defined(__EMSCRIPTEN__)
# if defined(__wasm_simd128__)
# define XSUM_ARCH "wasm/asmjs + simd128"
# else
# define XSUM_ARCH "wasm/asmjs"
# endif
#else
# define XSUM_ARCH "unknown"
#endif

View File

@ -45,6 +45,18 @@
#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* target timing per iteration */
#define TIMELOOP_MIN (TIMELOOP / 2) /* minimum timing to validate a result */
/* Each benchmark iteration attempts to match TIMELOOP (1 second).
* The nb of loops is adjusted at each iteration to reach that target.
* However, initially, there is no information, so 1st iteration blindly targets an arbitrary speed.
* If it's too small, it will be adjusted, and a new attempt will be made.
* But if it's too large, the first iteration can be very long,
* before being fixed at second attempt.
* So prefer starting with small speed targets.
* XXH_1ST_SPEED_TARGET is defined in MB/s */
#ifndef XXH_1ST_SPEED_TARGET
# define XXH_1ST_SPEED_TARGET 10
#endif
#define MAX_MEM (2 GB - 64 MB)
static clock_t XSUM_clockSpan( clock_t start )
@ -216,7 +228,7 @@ int g_nbIterations = NBLOOPS_DEFAULT;
static void XSUM_benchHash(hashFunction h, const char* hName, int testID,
const void* buffer, size_t bufferSize)
{
XSUM_U32 nbh_perIteration = (XSUM_U32)((300 MB) / (bufferSize+1)) + 1; /* first iteration conservatively aims for 300 MB/s */
XSUM_U32 nbh_perIteration = (XSUM_U32)((XXH_1ST_SPEED_TARGET MB) / (bufferSize+1)) + 1;
int iterationNb, nbIterations = g_nbIterations + !g_nbIterations /* min 1 */;
double fastestH = 100000000.;
assert(HASHNAME_MAX > 2);

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -39,10 +39,36 @@
typedef struct stat XSUM_stat_t;
#endif
#if (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \
#if defined(__EMSCRIPTEN__) && defined(XSUM_NODE_JS)
# include <unistd.h> /* isatty */
# include <emscripten.h> /* EM_ASM_INT */
/* The Emscripten SDK does not properly detect when the standard streams
* are piped to node.js, and there does not seem to be any way to tell in
* plain C. To work around it, inline JavaScript is used to call Node's
* isatty() function. */
static int XSUM_IS_CONSOLE(FILE* stdStream)
{
/* https://github.com/iliakan/detect-node */
int is_node = EM_ASM_INT((
return (Object.prototype.toString.call(
typeof process !== 'undefined' ? process : 0
) == '[object process]') | 0
));
if (is_node) {
return EM_ASM_INT(
return require('node:tty').isatty($0),
fileno(stdStream)
);
} else {
return isatty(fileno(stdStream));
}
}
#elif defined(__EMSCRIPTEN__) || (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \
|| (XSUM_PLATFORM_POSIX_VERSION >= 200112L) \
|| defined(__DJGPP__) \
|| defined(__MSYS__)
|| defined(__MSYS__) \
|| defined(__HAIKU__)
# include <unistd.h> /* isatty */
# define XSUM_IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2)
@ -216,7 +242,6 @@ static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf)
XSUM_ATTRIBUTE((__format__(__printf__, 2, 0)))
static int XSUM_vasprintf(char** strp, const char* format, va_list ap)
{
int ret;
int size;
va_list copy;
/*
@ -234,6 +259,7 @@ static int XSUM_vasprintf(char** strp, const char* format, va_list ap)
*strp = NULL;
return size;
} else {
int ret;
*strp = (char*) malloc((size_t)size + 1);
if (*strp == NULL) {
return -1;

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -23,6 +23,7 @@
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
#include "xsum_output.h"
#include "xsum_os_specific.h" /* XSUM_API */
int XSUM_logLevel = 2;

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -130,32 +130,34 @@ static const XSUM_testdata64_t XSUM_XXH64_testdata[] = {
/* XXH3_64bits, seeded */
static const XSUM_testdata64_t XSUM_XXH3_testdata[] = {
{ 0, 0, 0x2D06800538D394C2ULL }, /* empty string */
{ 0, PRIME64, 0xA8A6B918B2F0364AULL }, /* empty string */
{ 0, PRIME64, 0xA8A6B918B2F0364AULL },
{ 1, 0, 0xC44BDFF4074EECDBULL }, /* 1 - 3 */
{ 1, PRIME64, 0x032BE332DD766EF8ULL }, /* 1 - 3 */
{ 1, PRIME64, 0x032BE332DD766EF8ULL },
{ 6, 0, 0x27B56A84CD2D7325ULL }, /* 4 - 8 */
{ 6, PRIME64, 0x84589C116AB59AB9ULL }, /* 4 - 8 */
{ 6, PRIME64, 0x84589C116AB59AB9ULL },
{ 12, 0, 0xA713DAF0DFBB77E7ULL }, /* 9 - 16 */
{ 12, PRIME64, 0xE7303E1B2336DE0EULL }, /* 9 - 16 */
{ 12, PRIME64, 0xE7303E1B2336DE0EULL },
{ 24, 0, 0xA3FE70BF9D3510EBULL }, /* 17 - 32 */
{ 24, PRIME64, 0x850E80FC35BDD690ULL }, /* 17 - 32 */
{ 24, PRIME64, 0x850E80FC35BDD690ULL },
{ 48, 0, 0x397DA259ECBA1F11ULL }, /* 33 - 64 */
{ 48, PRIME64, 0xADC2CBAA44ACC616ULL }, /* 33 - 64 */
{ 48, PRIME64, 0xADC2CBAA44ACC616ULL },
{ 80, 0, 0xBCDEFBBB2C47C90AULL }, /* 65 - 96 */
{ 80, PRIME64, 0xC6DD0CB699532E73ULL }, /* 65 - 96 */
{ 80, PRIME64, 0xC6DD0CB699532E73ULL },
{ 195, 0, 0xCD94217EE362EC3AULL }, /* 129-240 */
{ 195, PRIME64, 0xBA68003D370CB3D9ULL }, /* 129-240 */
{ 195, PRIME64, 0xBA68003D370CB3D9ULL },
{ 403, 0, 0xCDEB804D65C6DEA4ULL }, /* one block, last stripe is overlapping */
{ 403, PRIME64, 0x6259F6ECFD6443FDULL }, /* one block, last stripe is overlapping */
{ 403, PRIME64, 0x6259F6ECFD6443FDULL },
{ 512, 0, 0x617E49599013CB6BULL }, /* one block, finishing at stripe boundary */
{ 512, PRIME64, 0x3CE457DE14C27708ULL }, /* one block, finishing at stripe boundary */
{ 512, PRIME64, 0x3CE457DE14C27708ULL },
{ 2048, 0, 0xDD59E2C3A5F038E0ULL }, /* 2 blocks, finishing at block boundary */
{ 2048, PRIME64, 0x66F81670669ABABCULL }, /* 2 blocks, finishing at block boundary */
{ 2048, PRIME64, 0x66F81670669ABABCULL },
{ 2099, 0, 0xC6B9D9B3FC9AC765ULL }, /* 2 blocks + 1 partial block, to detect off-by-one scrambling issues, like #816 */
{ 2099, PRIME64, 0x184F316843663974ULL },
{ 2240, 0, 0x6E73A90539CF2948ULL }, /* 3 blocks, finishing at stripe boundary */
{ 2240, PRIME64, 0x757BA8487D1B5247ULL }, /* 3 blocks, finishing at stripe boundary */
{ 2240, PRIME64, 0x757BA8487D1B5247ULL },
{ 2367, 0, 0xCB37AEB9E5D361EDULL }, /* 3 blocks, last stripe is overlapping */
{ 2367, PRIME64, 0xD2DB3415B942B42AULL } /* 3 blocks, last stripe is overlapping */
{ 2367, PRIME64, 0xD2DB3415B942B42AULL }
};
/* XXH3_64bits, custom secret */
static const XSUM_testdata64_t XSUM_XXH3_withSecret_testdata[] = {
@ -178,32 +180,32 @@ static const XSUM_testdata64_t XSUM_XXH3_withSecret_testdata[] = {
/* XXH3_128bits, seeded */
static const XSUM_testdata128_t XSUM_XXH128_testdata[] = {
{ 0, 0, { 0x6001C324468D497FULL, 0x99AA06D3014798D8ULL } }, /* empty string */
{ 0, PRIME32, { 0x5444F7869C671AB0ULL, 0x92220AE55E14AB50ULL } }, /* empty string */
{ 0, PRIME32, { 0x5444F7869C671AB0ULL, 0x92220AE55E14AB50ULL } },
{ 1, 0, { 0xC44BDFF4074EECDBULL, 0xA6CD5E9392000F6AULL } }, /* 1 - 3 */
{ 1, PRIME32, { 0xB53D5557E7F76F8DULL, 0x89B99554BA22467CULL } }, /* 1 - 3 */
{ 1, PRIME32, { 0xB53D5557E7F76F8DULL, 0x89B99554BA22467CULL } },
{ 6, 0, { 0x3E7039BDDA43CFC6ULL, 0x082AFE0B8162D12AULL } }, /* 4 - 8 */
{ 6, PRIME32, { 0x269D8F70BE98856EULL, 0x5A865B5389ABD2B1ULL } }, /* 4 - 8 */
{ 6, PRIME32, { 0x269D8F70BE98856EULL, 0x5A865B5389ABD2B1ULL } },
{ 12, 0, { 0x061A192713F69AD9ULL, 0x6E3EFD8FC7802B18ULL } }, /* 9 - 16 */
{ 12, PRIME32, { 0x9BE9F9A67F3C7DFBULL, 0xD7E09D518A3405D3ULL } }, /* 9 - 16 */
{ 12, PRIME32, { 0x9BE9F9A67F3C7DFBULL, 0xD7E09D518A3405D3ULL } },
{ 24, 0, { 0x1E7044D28B1B901DULL, 0x0CE966E4678D3761ULL } }, /* 17 - 32 */
{ 24, PRIME32, { 0xD7304C54EBAD40A9ULL, 0x3162026714A6A243ULL } }, /* 17 - 32 */
{ 24, PRIME32, { 0xD7304C54EBAD40A9ULL, 0x3162026714A6A243ULL } },
{ 48, 0, { 0xF942219AED80F67BULL, 0xA002AC4E5478227EULL } }, /* 33 - 64 */
{ 48, PRIME32, { 0x7BA3C3E453A1934EULL, 0x163ADDE36C072295ULL } }, /* 33 - 64 */
{ 48, PRIME32, { 0x7BA3C3E453A1934EULL, 0x163ADDE36C072295ULL } },
{ 81, 0, { 0x5E8BAFB9F95FB803ULL, 0x4952F58181AB0042ULL } }, /* 65 - 96 */
{ 81, PRIME32, { 0x703FBB3D7A5F755CULL, 0x2724EC7ADC750FB6ULL } }, /* 65 - 96 */
{ 81, PRIME32, { 0x703FBB3D7A5F755CULL, 0x2724EC7ADC750FB6ULL } },
{ 222, 0, { 0xF1AEBD597CEC6B3AULL, 0x337E09641B948717ULL } }, /* 129-240 */
{ 222, PRIME32, { 0xAE995BB8AF917A8DULL, 0x91820016621E97F1ULL } }, /* 129-240 */
{ 222, PRIME32, { 0xAE995BB8AF917A8DULL, 0x91820016621E97F1ULL } },
{ 403, 0, { 0xCDEB804D65C6DEA4ULL, 0x1B6DE21E332DD73DULL } }, /* one block, last stripe is overlapping */
{ 403, PRIME64, { 0x6259F6ECFD6443FDULL, 0xBED311971E0BE8F2ULL } }, /* one block, last stripe is overlapping */
{ 403, PRIME64, { 0x6259F6ECFD6443FDULL, 0xBED311971E0BE8F2ULL } },
{ 512, 0, { 0x617E49599013CB6BULL, 0x18D2D110DCC9BCA1ULL } }, /* one block, finishing at stripe boundary */
{ 512, PRIME64, { 0x3CE457DE14C27708ULL, 0x925D06B8EC5B8040ULL } }, /* one block, finishing at stripe boundary */
{ 512, PRIME64, { 0x3CE457DE14C27708ULL, 0x925D06B8EC5B8040ULL } },
{ 2048, 0, { 0xDD59E2C3A5F038E0ULL, 0xF736557FD47073A5ULL } }, /* 2 blocks, finishing at block boundary */
{ 2048, PRIME32, { 0x230D43F30206260BULL, 0x7FB03F7E7186C3EAULL } }, /* 2 blocks, finishing at block boundary */
{ 2048, PRIME32, { 0x230D43F30206260BULL, 0x7FB03F7E7186C3EAULL } },
{ 2240, 0, { 0x6E73A90539CF2948ULL, 0xCCB134FBFA7CE49DULL } }, /* 3 blocks, finishing at stripe boundary */
{ 2240, PRIME32, { 0xED385111126FBA6FULL, 0x50A1FE17B338995FULL } }, /* 3 blocks, finishing at stripe boundary */
{ 2240, PRIME32, { 0xED385111126FBA6FULL, 0x50A1FE17B338995FULL } },
{ 2367, 0, { 0xCB37AEB9E5D361EDULL, 0xE89C0F6FF369B427ULL } }, /* 3 blocks, last stripe is overlapping */
{ 2367, PRIME32, { 0x6F5360AE69C2F406ULL, 0xD23AAE4B76C31ECBULL } } /* 3 blocks, last stripe is overlapping */
{ 2367, PRIME32, { 0x6F5360AE69C2F406ULL, 0xD23AAE4B76C31ECBULL } }
};
/* XXH128, custom secret */
@ -568,7 +570,7 @@ static void XSUM_testXXH128_withSecret(const void* data, const void* secret, siz
XXH128_hash_t Nresult = testData->Nresult;
if (len == 0) {
data = NULL;
} else if (len>0) {
} else {
assert(data != NULL);
}
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecret(data, len, secret, secretSize);

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,25 +1,41 @@
.TH "XXHSUM" "1" "November 2021" "xxhsum 0.8.1" "User Commands"
.TH "XXHSUM" "1" "July 2023" "xxhsum 0.8.2" "User Commands"
.SH "NAME"
\fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums
.SH "SYNOPSIS"
\fBxxhsum [<OPTION>] \|\.\|\.\|\. [<FILE>] \|\.\|\.\|\.\fR \fBxxhsum \-b [<OPTION>] \|\.\|\.\|\.\fR
\fBxxhsum\fR [\fIOPTION\fR]\|\.\|\.\|\. [\fIFILE\fR]\|\.\|\.\|\.
.br
\fBxxhsum \-b\fR [\fIOPTION\fR]\|\.\|\.\|\.
.P
\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR
\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR, \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR, \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR\.
.SH "DESCRIPTION"
Print or check xxHash (32, 64 or 128 bits) checksums\. When no \fIFILE\fR, read standard input, except if it\'s the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it\'s the console\.
Print or check xxHash (32, 64 or 128 bits) checksums\.
.br
When no \fIFILE\fR, read standard input, except if it's the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it's the console\.
.P
\fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are: \fBxxhsum\fR doesn\'t have text/binary mode switch (\fB\-b\fR, \fB\-t\fR); \fBxxhsum\fR always treats files as binary file; \fBxxhsum\fR has a hash bit width switch (\fB\-H\fR);
\fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are:
.IP "\[ci]" 4
\fBxxhsum\fR doesn't have text mode switch (\fB\-t\fR)
.IP "\[ci]" 4
\fBxxhsum\fR doesn't have short binary mode switch (\fB\-b\fR)
.IP "\[ci]" 4
\fBxxhsum\fR always treats files as binary file
.IP "\[ci]" 4
\fBxxhsum\fR has a hash selection switch (\fB\-H\fR)
.IP "" 0
.P
As xxHash is a fast non\-cryptographic checksum algorithm, \fBxxhsum\fR should not be used for security related purposes\.
.P
\fBxxhsum \-b\fR invokes benchmark mode\. See \fIOPTIONS\fR and \fIEXAMPLES\fR for details\.
\fBxxhsum \-b\fR invokes benchmark mode\. See OPTIONS and EXAMPLES for details\.
.SH "OPTIONS"
.TP
\fB\-V\fR, \fB\-\-version\fR
Displays xxhsum version and exits
.TP
\fB\-H\fR\fIHASHTYPE\fR
Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (64bits)
Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Note that \fB\-H3\fR triggers \fB\-\-tag\fR, which can't be skipped (this is to reduce risks of confusion with \fB\-H2\fR (\fBXXH64\fR))\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (XXH64)
.TP
\fB\-\-binary\fR
Read in binary mode\.
.TP
\fB\-\-tag\fR
Output in the BSD style\.
@ -29,39 +45,37 @@ Set output hexadecimal checksum value as little endian convention\. By default,
.TP
\fB\-h\fR, \fB\-\-help\fR
Displays help and exits
.P
\fBThe following four options are useful only when verifying checksums (\fB\-c\fR)\fR
.SS "The following options are useful only when verifying checksums (\-c):"
.TP
\fB\-c\fR, \fB\-\-check\fR \fIFILE\fR
Read xxHash sums from \fIFILE\fR and check them
.TP
\fB\-q\fR, \fB\-\-quiet\fR
Don\'t print OK for each successfully verified file
Don't print OK for each successfully verified file
.TP
\fB\-\-strict\fR
Return an error code if any line in the file is invalid, not just if some checksums are wrong\. This policy is disabled by default, though UI will prompt an informational message if any line in the file is detected invalid\.
.TP
\fB\-\-status\fR
Don\'t output anything\. Status code shows success\.
Don't output anything\. Status code shows success\.
.TP
\fB\-w\fR, \fB\-\-warn\fR
Emit a warning message about each improperly formatted checksum line\.
.P
\fBThe following options are useful only benchmark purpose\fR
.SS "The following options are useful only benchmark purpose:"
.TP
\fB\-b\fR
Benchmark mode\. See \fIEXAMPLES\fR for details\.
Benchmark mode\. See EXAMPLES for details\.
.TP
\fB\-b#\fR
Specify ID of variant to be tested\. Multiple variants can be selected, separated by a \',\' comma\.
Specify ID of variant to be tested\. Multiple variants can be selected, separated by a ',' comma\.
.TP
\fB\-B\fR\fIBLOCKSIZE\fR
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode\'s test data block size in bytes\. Default value is 102400
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode's test data block size in bytes\. Default value is 102400
.TP
\fB\-i\fR\fIITERATIONS\fR
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIITERATIONS\fR specifies number of iterations in benchmark\. Single iteration lasts approximately 1000 milliseconds\. Default value is 3
.SH "EXIT STATUS"
\fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn\'t be read or doesn\'t have the same checksum as the \fB\-c\fR option\.
\fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn't be read or doesn't have the same checksum as the \fB\-c\fR option\.
.SH "EXAMPLES"
Output xxHash (64bit) checksum values of specific files to standard output
.IP "" 4

View File

@ -4,31 +4,32 @@ xxhsum(1) -- print or check xxHash non-cryptographic checksums
SYNOPSIS
--------
`xxhsum [<OPTION>] ... [<FILE>] ...`
`xxhsum -b [<OPTION>] ...`
`xxhsum` [*OPTION*]... [*FILE*]...
`xxhsum -b` [*OPTION*]...
`xxh32sum` is equivalent to `xxhsum -H0`
`xxh64sum` is equivalent to `xxhsum -H1`
`xxh128sum` is equivalent to `xxhsum -H2`
`xxh32sum` is equivalent to `xxhsum -H0`,
`xxh64sum` is equivalent to `xxhsum -H1`,
`xxh128sum` is equivalent to `xxhsum -H2`.
DESCRIPTION
-----------
Print or check xxHash (32, 64 or 128 bits) checksums.
When no <FILE>, read standard input, except if it's the console.
When <FILE> is `-`, read standard input even if it's the console.
Print or check xxHash (32, 64 or 128 bits) checksums.
When no *FILE*, read standard input, except if it's the console.
When *FILE* is `-`, read standard input even if it's the console.
`xxhsum` supports a command line syntax similar but not identical to md5sum(1).
Differences are:
`xxhsum` doesn't have text/binary mode switch (`-b`, `-t`);
`xxhsum` always treats files as binary file;
`xxhsum` has a hash bit width switch (`-H`);
`xxhsum` supports a command line syntax similar but not identical to md5sum(1). Differences are:
* `xxhsum` doesn't have text mode switch (`-t`)
* `xxhsum` doesn't have short binary mode switch (`-b`)
* `xxhsum` always treats files as binary file
* `xxhsum` has a hash selection switch (`-H`)
As xxHash is a fast non-cryptographic checksum algorithm,
`xxhsum` should not be used for security related purposes.
`xxhsum -b` invokes benchmark mode. See [OPTIONS](#OPTIONS) and [EXAMPLES](#EXAMPLES) for details.
`xxhsum -b` invokes benchmark mode. See OPTIONS and EXAMPLES for details.
OPTIONS
-------
@ -36,10 +37,15 @@ OPTIONS
* `-V`, `--version`:
Displays xxhsum version and exits
* `-H`<HASHTYPE>:
Hash selection. <HASHTYPE> means `0`=XXH32, `1`=XXH64, `2`=XXH128, `3`=XXH3.
Alternatively, <HASHTYPE> `32`=XXH32, `64`=XXH64, `128`=XXH128.
Default value is `1` (64bits)
* `-H`*HASHTYPE*:
Hash selection. *HASHTYPE* means `0`=XXH32, `1`=XXH64, `2`=XXH128, `3`=XXH3.
Note that `-H3` triggers `--tag`, which can't be skipped
(this is to reduce risks of confusion with `-H2` (`XXH64`)).
Alternatively, *HASHTYPE* `32`=XXH32, `64`=XXH64, `128`=XXH128.
Default value is `1` (XXH64)
* `--binary`:
Read in binary mode.
* `--tag`:
Output in the BSD style.
@ -51,10 +57,10 @@ OPTIONS
* `-h`, `--help`:
Displays help and exits
**The following four options are useful only when verifying checksums (`-c`)**
### The following options are useful only when verifying checksums (-c):
* `-c`, `--check` <FILE>:
Read xxHash sums from <FILE> and check them
* `-c`, `--check` *FILE*:
Read xxHash sums from *FILE* and check them
* `-q`, `--quiet`:
Don't print OK for each successfully verified file
@ -72,22 +78,22 @@ OPTIONS
* `-w`, `--warn`:
Emit a warning message about each improperly formatted checksum line.
**The following options are useful only benchmark purpose**
### The following options are useful only benchmark purpose:
* `-b`:
Benchmark mode. See [EXAMPLES](#EXAMPLES) for details.
Benchmark mode. See EXAMPLES for details.
* `-b#`:
Specify ID of variant to be tested.
Multiple variants can be selected, separated by a ',' comma.
* `-B`<BLOCKSIZE>:
Only useful for benchmark mode (`-b`). See [EXAMPLES](#EXAMPLES) for details.
* `-B`*BLOCKSIZE*:
Only useful for benchmark mode (`-b`). See *EXAMPLES* for details.
<BLOCKSIZE> specifies benchmark mode's test data block size in bytes.
Default value is 102400
* `-i`<ITERATIONS>:
Only useful for benchmark mode (`-b`). See [EXAMPLES](#EXAMPLES) for details.
* `-i`*ITERATIONS*:
Only useful for benchmark mode (`-b`). See *EXAMPLES* for details.
<ITERATIONS> specifies number of iterations in benchmark. Single iteration
lasts approximately 1000 milliseconds. Default value is 3

View File

@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
* Copyright (C) 2013-2020 Yann Collet
* Copyright (C) 2013-2021 Yann Collet
*
* GPL v2 License
*
@ -80,6 +80,18 @@ static const char stdinFileName[] = "stdin";
typedef enum { algo_xxh32=0, algo_xxh64=1, algo_xxh128=2, algo_xxh3=3 } AlgoSelected;
static AlgoSelected g_defaultAlgo = algo_xxh64; /* required within main() & XSUM_usage() */
typedef enum {
algo_bitmask_xxh32 = 1 << algo_xxh32, /* 1 << 0 */
algo_bitmask_xxh64 = 1 << algo_xxh64, /* 1 << 1 */
algo_bitmask_xxh128 = 1 << algo_xxh128, /* 1 << 2 */
algo_bitmask_xxh3 = 1 << algo_xxh3, /* 1 << 3 */
algo_bitmask_all = algo_bitmask_xxh32 /* All algorithms */
| algo_bitmask_xxh64
| algo_bitmask_xxh128
| algo_bitmask_xxh3
} AlgoBitmask;
/* <16 hex char> <SPC> <SPC> <filename> <'\0'>
* '4096' is typical Linux PATH_MAX configuration. */
#define DEFAULT_LINE_LENGTH (sizeof(XXH64_hash_t) * 2 + 2 + 4096 + 1)
@ -90,6 +102,116 @@ static AlgoSelected g_defaultAlgo = algo_xxh64; /* required within main() & X
static size_t XSUM_DEFAULT_SAMPLE_SIZE = 100 KB;
/* ********************************************************
* Filename (un)escaping
**********************************************************/
static int XSUM_filenameNeedsEscape(const char* filename) {
return strchr(filename, '\\')
|| strchr(filename, '\n')
|| strchr(filename, '\r');
}
static int XSUM_lineNeedsUnescape(const char* line) {
/* Skip white-space characters */
while (*line == ' ' || *line == '\t') {
++line;
}
/* Returns true if first non-white-space character is '\\' (0x5c) */
return *line == '\\';
}
static void XSUM_printFilename(const char* filename, int needsEscape) {
if (!needsEscape) {
XSUM_output("%s", filename);
} else {
const char* p;
for (p = filename; *p != '\0'; ++p) {
switch (*p)
{
case '\n':
XSUM_output("\\n");
break;
case '\r':
XSUM_output("\\r");
break;
case '\\':
XSUM_output("\\\\");
break;
default:
XSUM_output("%c", *p);
break;
}
}
}
}
/* Unescape filename in place.
- Replace '\\', 'n' (0x5c, 0x6e) with '\n' (0x0a).
- Replace '\\', 'r' (0x5c, 0x72) with '\r' (0x0d).
- Replace '\\', '\\' (0x5c, 0x5c) with '\\' (0x5c).
- filename may not contain other backslash sequences.
- filename may not ends with backslash.
- filename may not contain NUL (0x00).
Return filename if everything is okay.
Return NULL if something wrong.
*/
static char* XSUM_filenameUnescape(char* filename, size_t filenameLen) {
char *p = filename;
size_t i;
for (i = 0; i < filenameLen; ++i) {
switch (filename[i])
{
case '\\':
++i;
if (i == filenameLen) {
return NULL; /* Don't accept '\\', <EOL> */
}
switch (filename[i])
{
case 'n':
*p++ = '\n';
break;
case 'r':
*p++ = '\r';
break;
case '\\':
*p++ = '\\';
break;
default:
return NULL; /* Don't accept any other backslash sequence */
}
break;
case '\0':
return NULL; /* Don't accept NUL (0x00) */
default:
*p++ = filename[i];
break;
}
}
if (p < filename + filenameLen) {
*p = '\0';
}
return filename;
}
/* ********************************************************
* Algorithm Bitmask
**********************************************************/
/* Compute AlgoBitmask (as a U32) from AlgoSelected */
static XSUM_U32 XSUM_algoBitmask_ComputeAlgoBitmaskFromAlgoSelected(AlgoSelected algoSelected) {
return (XSUM_U32) (1U << algoSelected);
}
/* Returns true (!0) if algoBitmask contains (accepts) parsedLineAlgo */
static int XSUM_algoBitmask_Accepts(XSUM_U32 algoBitmask, AlgoSelected parsedLineAlgo) {
const XSUM_U32 mask = XSUM_algoBitmask_ComputeAlgoBitmaskFromAlgoSelected(parsedLineAlgo);
return (algoBitmask & mask) != 0;
}
/* ********************************************************
* File Hashing
**********************************************************/
@ -204,7 +326,13 @@ static void XSUM_printLine_BSD_internal(const char* filename,
assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
{ const char* const typeString = algoString[hashType];
const size_t hashLength = XSUM_algoLength[hashType];
XSUM_output("%s (%s) = ", typeString, filename);
const int needsEscape = XSUM_filenameNeedsEscape(filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
XSUM_output("%s (", typeString);
XSUM_printFilename(filename, needsEscape);
XSUM_output(") = ");
f_displayHash(canonicalHash, hashLength);
XSUM_output("\n");
} }
@ -225,8 +353,14 @@ static void XSUM_printLine_GNU_internal(const char* filename,
{
assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
{ const size_t hashLength = XSUM_algoLength[hashType];
const int needsEscape = XSUM_filenameNeedsEscape(filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
f_displayHash(canonicalHash, hashLength);
XSUM_output(" %s\n", filename);
XSUM_output(" ");
XSUM_printFilename(filename, needsEscape);
XSUM_output("\n");
} }
static void XSUM_printLine_GNU(const char* filename,
@ -354,6 +488,7 @@ static int XSUM_hashFiles(const char* fnList[], int fnTotal,
typedef enum {
GetLine_ok,
GetLine_comment,
GetLine_eof,
GetLine_exceedMaxLineLength,
GetLine_outOfMemory
@ -391,8 +526,10 @@ typedef struct {
unsigned long nProperlyFormattedLines;
unsigned long nImproperlyFormattedLines;
unsigned long nMismatchedChecksums;
unsigned long nMatchedChecksums;
unsigned long nOpenOrReadFailures;
unsigned long nMixedFormatLines;
unsigned long nMissing;
int quit;
} ParseFileReport;
@ -405,8 +542,10 @@ typedef struct {
char* blockBuf;
XSUM_U32 strictMode;
XSUM_U32 statusOnly;
XSUM_U32 ignoreMissing;
XSUM_U32 warn;
XSUM_U32 quiet;
XSUM_U32 algoBitmask;
ParseFileReport report;
} ParseFileArg;
@ -414,6 +553,7 @@ typedef struct {
/*
* Reads a line from stream `inFile`.
* Returns GetLine_ok, if it reads line successfully.
* Returns GetLine_comment, if the line is beginning with '#'.
* Returns GetLine_eof, if stream reaches EOF.
* Returns GetLine_exceedMaxLineLength, if line length is longer than MAX_LINE_LENGTH.
* Returns GetLine_outOfMemory, if line buffer memory allocation failed.
@ -463,6 +603,12 @@ static GetLineResult XSUM_getLine(char** lineBuf, int* lineMax, FILE* inFile)
}
(*lineBuf)[len] = '\0';
/* Ignore comment lines, which begin with a '#' character. */
if (result == GetLine_ok && len > 0 && ((*lineBuf)[0] == '#')) {
result = GetLine_comment;
}
return result;
}
@ -533,7 +679,7 @@ static CanonicalFromStringResult XSUM_canonicalFromString(unsigned char* dst,
*
* <algorithm> <' ('> <filename> <') = '> <hexstring> <'\0'>
*/
static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int rev)
static ParseLineResult XSUM_parseLine1(ParsedLine* parsedLine, char* line, int rev, int needsUnescape, XSUM_U32 algoBitmask)
{
char* const firstSpace = strchr(line, ' ');
const char* hash_ptr;
@ -564,6 +710,11 @@ static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int re
if (hash_len==32) parsedLine->algo = algo_xxh128;
}
/* Check current CLI accepts the algorithm or not */
if(! XSUM_algoBitmask_Accepts(algoBitmask, parsedLine->algo)) {
return ParseLine_invalidFormat;
}
switch (hash_len)
{
case 8:
@ -603,10 +754,29 @@ static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int re
/* note : skipping second separation character, which can be anything,
* allowing insertion of custom markers such as '*' */
parsedLine->filename = firstSpace + 2;
{
char* const filename = firstSpace + 2;
const size_t filenameLen = strlen(filename);
if (needsUnescape) {
char* const result = XSUM_filenameUnescape(filename, filenameLen);
if (result == NULL) {
return ParseLine_invalidFormat;
}
}
parsedLine->filename = filename;
}
return ParseLine_ok;
}
static ParseLineResult XSUM_parseLine(ParsedLine* parsedLine, char* line, int rev, XSUM_U32 algoBitmask) {
const int needsUnescape = XSUM_lineNeedsUnescape(line);
if (needsUnescape) {
++line;
}
return XSUM_parseLine1(parsedLine, line, rev, needsUnescape, algoBitmask);
}
/*!
* Parse xxHash checksum file.
@ -635,12 +805,19 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
{ GetLineResult const XSUM_getLineResult = XSUM_getLine(&XSUM_parseFileArg->lineBuf,
&XSUM_parseFileArg->lineMax,
XSUM_parseFileArg->inFile);
/* Ignore comment lines */
if (XSUM_getLineResult == GetLine_comment) {
continue;
}
if (XSUM_getLineResult != GetLine_ok) {
if (XSUM_getLineResult == GetLine_eof) break;
switch (XSUM_getLineResult)
{
case GetLine_ok:
case GetLine_comment:
case GetLine_eof:
/* These cases never happen. See above XSUM_getLineResult related "if"s.
They exist just for make gcc's -Wswitch-enum happy. */
@ -663,7 +840,7 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
break;
} }
if (XSUM_parseLine(&parsedLine, XSUM_parseFileArg->lineBuf, rev) != ParseLine_ok) {
if (XSUM_parseLine(&parsedLine, XSUM_parseFileArg->lineBuf, rev, XSUM_parseFileArg->algoBitmask) != ParseLine_ok) {
report->nImproperlyFormattedLines++;
if (XSUM_parseFileArg->warn) {
XSUM_log("%s:%lu: Error: Improperly formatted checksum line.\n",
@ -722,10 +899,14 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
break;
case LineStatus_failedToOpen:
report->nOpenOrReadFailures++;
if (!XSUM_parseFileArg->statusOnly) {
XSUM_output("%s:%lu: Could not open or read '%s': %s.\n",
inFileName, lineNumber, parsedLine.filename, strerror(errno));
if (XSUM_parseFileArg->ignoreMissing) {
report->nMissing++;
} else {
report->nOpenOrReadFailures++;
if (!XSUM_parseFileArg->statusOnly) {
XSUM_output("%s:%lu: Could not open or read '%s': %s.\n",
inFileName, lineNumber, parsedLine.filename, strerror(errno));
}
}
break;
@ -733,6 +914,7 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
case LineStatus_hashFailed:
{ int b = 1;
if (lineStatus == LineStatus_hashOk) {
report->nMatchedChecksums++;
/* If --quiet is specified, don't display "OK" */
if (XSUM_parseFileArg->quiet) b = 0;
} else {
@ -740,8 +922,12 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
}
if (b && !XSUM_parseFileArg->statusOnly) {
XSUM_output("%s: %s\n", parsedLine.filename
, lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
const int needsEscape = XSUM_filenameNeedsEscape(parsedLine.filename);
if (needsEscape) {
XSUM_output("%c", '\\');
}
XSUM_printFilename(parsedLine.filename, needsEscape);
XSUM_output(": %s\n", lineStatus == LineStatus_hashOk ? "OK" : "FAILED");
} }
break;
}
@ -755,6 +941,7 @@ static void XSUM_parseFile1(ParseFileArg* XSUM_parseFileArg, int rev)
*
* If strictMode != 0, return error code if any line is invalid.
* If statusOnly != 0, don't generate any output.
* If ignoreMissing != 0, ignore missing file. But if no file was verified, returns 0 (failed).
* If warn != 0, print a warning message to stderr.
* If quiet != 0, suppress "OK" line.
*
@ -768,8 +955,10 @@ static int XSUM_checkFile(const char* inFileName,
const Display_endianess displayEndianess,
XSUM_U32 strictMode,
XSUM_U32 statusOnly,
XSUM_U32 ignoreMissing,
XSUM_U32 warn,
XSUM_U32 quiet)
XSUM_U32 quiet,
XSUM_U32 algoBitmask)
{
int result = 0;
FILE* inFile = NULL;
@ -802,8 +991,10 @@ static int XSUM_checkFile(const char* inFileName,
XSUM_parseFileArg->blockBuf = (char*) malloc(XSUM_parseFileArg->blockSize);
XSUM_parseFileArg->strictMode = strictMode;
XSUM_parseFileArg->statusOnly = statusOnly;
XSUM_parseFileArg->ignoreMissing = ignoreMissing;
XSUM_parseFileArg->warn = warn;
XSUM_parseFileArg->quiet = quiet;
XSUM_parseFileArg->algoBitmask = algoBitmask;
if ( (XSUM_parseFileArg->lineBuf == NULL)
|| (XSUM_parseFileArg->blockBuf == NULL) ) {
@ -845,6 +1036,14 @@ static int XSUM_checkFile(const char* inFileName,
&& report->nOpenOrReadFailures == 0
&& (!strictMode || report->nImproperlyFormattedLines == 0)
&& report->quit == 0;
/* If "--ignore-missing" is enabled and there's no matched checksum, report it as error.
* See https://github.com/coreutils/coreutils/blob/2f1cffe07ab0f0b4135a52d95f1689d7fc7f26c9/src/digest.c#L1325-L1328 */
if (ignoreMissing && report->nMatchedChecksums == 0) {
XSUM_output("%s: no file was verified\n", inFileName);
result = 0;
}
return result;
}
@ -853,19 +1052,21 @@ static int XSUM_checkFiles(const char* fnList[], int fnTotal,
const Display_endianess displayEndianess,
XSUM_U32 strictMode,
XSUM_U32 statusOnly,
XSUM_U32 ignoreMissing,
XSUM_U32 warn,
XSUM_U32 quiet)
XSUM_U32 quiet,
XSUM_U32 algoBitmask)
{
int ok = 1;
/* Special case for stdinName "-",
* note: stdinName is not a string. It's special pointer. */
if (fnTotal==0) {
ok &= XSUM_checkFile(stdinName, displayEndianess, strictMode, statusOnly, warn, quiet);
ok &= XSUM_checkFile(stdinName, displayEndianess, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
} else {
int fnNb;
for (fnNb=0; fnNb<fnTotal; fnNb++)
ok &= XSUM_checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, warn, quiet);
ok &= XSUM_checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
}
return ok ? 0 : 1;
}
@ -896,16 +1097,18 @@ static int XSUM_usage_advanced(const char* exename)
XSUM_log( " -V, --version Display version information \n");
XSUM_log( " --tag Produce BSD-style checksum lines \n");
XSUM_log( " --little-endian Checksum values use little endian convention (default: big endian) \n");
XSUM_log( " --binary Read in binary mode \n");
XSUM_log( " -b Run benchmark \n");
XSUM_log( " -b# Bench only algorithm variant # \n");
XSUM_log( " -i# Number of times to run the benchmark (default: %u) \n", NBLOOPS_DEFAULT);
XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT);
XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n");
XSUM_log( "\n");
XSUM_log( "The following four options are useful only when verifying checksums (-c): \n");
XSUM_log( "The following five options are useful only when verifying checksums (-c): \n");
XSUM_log( " -q, --quiet Don't print OK for each successfully verified file \n");
XSUM_log( " --status Don't output anything, status code shows success \n");
XSUM_log( " --strict Exit non-zero for improperly formatted checksum lines \n");
XSUM_log( " --warn Warn about improperly formatted checksum lines \n");
XSUM_log( " --ignore-missing Don't fail or report status for missing files \n");
return 0;
}
@ -988,6 +1191,8 @@ XSUM_API int XSUM_main(int argc, const char* argv[])
XSUM_U32 strictMode = 0;
XSUM_U32 statusOnly = 0;
XSUM_U32 warn = 0;
XSUM_U32 ignoreMissing = 0;
XSUM_U32 algoBitmask = algo_bitmask_all;
int explicitStdin = 0;
XSUM_U32 selectBenchIDs= 0; /* 0 == use default k_testIDs_default, kBenchAll == bench all */
static const XSUM_U32 kBenchAll = 99;
@ -998,9 +1203,9 @@ XSUM_API int XSUM_main(int argc, const char* argv[])
int nbIterations = NBLOOPS_DEFAULT;
/* special case: xxhNNsum default to NN bits checksum */
if (strstr(exename, "xxh32sum") != NULL) algo = g_defaultAlgo = algo_xxh32;
if (strstr(exename, "xxh64sum") != NULL) algo = g_defaultAlgo = algo_xxh64;
if (strstr(exename, "xxh128sum") != NULL) algo = g_defaultAlgo = algo_xxh128;
if (strstr(exename, "xxh32sum") != NULL) { algo = g_defaultAlgo = algo_xxh32; algoBitmask = algo_bitmask_xxh32; }
if (strstr(exename, "xxh64sum") != NULL) { algo = g_defaultAlgo = algo_xxh64; algoBitmask = algo_bitmask_xxh64; }
if (strstr(exename, "xxh128sum") != NULL) { algo = g_defaultAlgo = algo_xxh128; algoBitmask = algo_bitmask_xxh128; }
for (i=1; i<argc; i++) {
const char* argument = argv[i];
@ -1014,6 +1219,8 @@ XSUM_API int XSUM_main(int argc, const char* argv[])
if (!strcmp(argument, "--strict")) { strictMode = 1; continue; }
if (!strcmp(argument, "--status")) { statusOnly = 1; continue; }
if (!strcmp(argument, "--warn")) { warn = 1; continue; }
if (!strcmp(argument, "--binary")) { continue; } /* Just ignore it. See https://github.com/Cyan4973/xxHash/issues/812 */
if (!strcmp(argument, "--ignore-missing")) { ignoreMissing = 1; continue; }
if (!strcmp(argument, "--help")) { return XSUM_usage_advanced(exename); }
if (!strcmp(argument, "--version")) { XSUM_log(FULL_WELCOME_MESSAGE(exename)); XSUM_sanityCheck(); return 0; }
if (!strcmp(argument, "--tag")) { convention = display_bsd; continue; }
@ -1131,7 +1338,7 @@ XSUM_API int XSUM_main(int argc, const char* argv[])
if (filenamesStart==0) filenamesStart = argc;
if (fileCheckMode) {
return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart,
displayEndianess, strictMode, statusOnly, warn, (XSUM_logLevel < 2) /*quiet*/);
displayEndianess, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask);
} else {
return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess, convention);
}

View File

@ -42,9 +42,14 @@ endif()
if(NOT CMAKE_CONFIGURATION_TYPES)
message(STATUS "xxHash build type: ${CMAKE_BUILD_TYPE}")
endif()
# Enable assert() statements in debug builds
if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
add_compile_definitions(XXH_DEBUGLEVEL=1)
if("${CMAKE_VERSION}" VERSION_LESS "3.12")
# add_compile_definitions is not available for older cmake => do nothing
else()
add_compile_definitions(XXH_DEBUGLEVEL=1)
endif()
endif()
option(BUILD_SHARED_LIBS "Build shared library" ON)
@ -70,8 +75,29 @@ mark_as_advanced(XXHASH_BUNDLED_MODE)
include(CMakeDependentOption)
CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT XXHASH_BUNDLED_MODE" OFF)
if("${CMAKE_VERSION}" VERSION_LESS "3.10")
# Can not enable DISPATCH mode since it fails to recognize architecture.
else()
CMAKE_HOST_SYSTEM_INFORMATION(RESULT PLATFORM QUERY OS_PLATFORM)
message(STATUS "Architecture: ${PLATFORM}")
endif()
# libxxhash
add_library(xxhash "${XXHASH_DIR}/xxhash.c")
if((DEFINED DISPATCH) AND (DEFINED PLATFORM))
# Only support DISPATCH option on x86_64.
if(("${PLATFORM}" STREQUAL "x86_64") OR ("${PLATFORM}" STREQUAL "AMD64"))
set(XXHSUM_DISPATCH ON)
message(STATUS "Enable xxHash dispatch mode")
add_library(xxhash "${XXHASH_DIR}/xxh_x86dispatch.c"
"${XXHASH_DIR}/xxhash.c"
)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DXXHSUM_DISPATCH=1")
else()
add_library(xxhash "${XXHASH_DIR}/xxhash.c")
endif()
else()
add_library(xxhash "${XXHASH_DIR}/xxhash.c")
endif()
add_library(${PROJECT_NAME}::xxhash ALIAS xxhash)
target_include_directories(xxhash
@ -88,12 +114,17 @@ set_target_properties(xxhash PROPERTIES
if(XXHASH_BUILD_XXHSUM)
set(XXHSUM_DIR "${XXHASH_DIR}/cli")
# xxhsum
add_executable(xxhsum "${XXHSUM_DIR}/xxhsum.c"
"${XXHSUM_DIR}/xsum_os_specific.c"
"${XXHSUM_DIR}/xsum_output.c"
"${XXHSUM_DIR}/xsum_sanity_check.c"
"${XXHSUM_DIR}/xsum_bench.c"
)
set(XXHSUM_SOURCES)
if (XXHSUM_DISPATCH)
list(APPEND XXHSUM_SOURCES "${XXHASH_DIR}/xxh_x86dispatch.c")
endif()
list(APPEND XXHSUM_SOURCES "${XXHSUM_DIR}/xxhsum.c"
"${XXHSUM_DIR}/xsum_os_specific.c"
"${XXHSUM_DIR}/xsum_output.c"
"${XXHSUM_DIR}/xsum_sanity_check.c"
"${XXHSUM_DIR}/xsum_bench.c"
)
add_executable(xxhsum ${XXHSUM_SOURCES})
add_executable(${PROJECT_NAME}::xxhsum ALIAS xxhsum)
target_link_libraries(xxhsum PRIVATE xxhash)
@ -118,11 +149,15 @@ if(NOT XXHASH_BUNDLED_MODE)
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
install(FILES "${XXHASH_DIR}/xxh3.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
if(DISPATCH)
install(FILES "${XXHASH_DIR}/xxh_x86dispatch.h"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
endif()
if(XXHASH_BUILD_XXHSUM)
install(TARGETS xxhsum
EXPORT xxHashTargets
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
install(FILES "${XXHASH_DIR}/xxhsum.1"
install(FILES "${XXHSUM_DIR}/xxhsum.1"
DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
endif(XXHASH_BUILD_XXHSUM)
@ -160,10 +195,11 @@ if(NOT XXHASH_BUNDLED_MODE)
NAMESPACE ${PROJECT_NAME}::)
# configure and install pkg-config
include(JoinPaths.cmake)
set(PREFIX ${CMAKE_INSTALL_PREFIX})
set(EXECPREFIX "\${prefix}")
set(INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}")
set(LIBDIR "${CMAKE_INSTALL_LIBDIR}")
join_paths(INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
join_paths(LIBDIR "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")
set(VERSION "${XXHASH_VERSION_STRING}")
configure_file(${XXHASH_DIR}/libxxhash.pc.in ${CMAKE_BINARY_DIR}/libxxhash.pc @ONLY)
@ -171,3 +207,5 @@ if(NOT XXHASH_BUNDLED_MODE)
DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
endif(NOT XXHASH_BUNDLED_MODE)
include(CPack)

View File

@ -0,0 +1,23 @@
# This module provides function for joining paths
# known from most languages
#
# SPDX-License-Identifier: (MIT OR CC0-1.0)
# Copyright 2020 Jan Tojnar
# https://github.com/jtojnar/cmake-snips
#
# Modelled after Pythons os.path.join
# https://docs.python.org/3.7/library/os.path.html#os.path.join
# Windows not supported
function(join_paths joined_path first_path_segment)
set(temp_path "${first_path_segment}")
foreach(current_segment IN LISTS ARGN)
if(NOT ("${current_segment}" STREQUAL ""))
if(IS_ABSOLUTE "${current_segment}")
set(temp_path "${current_segment}")
else()
set(temp_path "${temp_path}/${current_segment}")
endif()
endif()
endforeach()
set(${joined_path} "${temp_path}" PARENT_SCOPE)
endfunction()

View File

@ -16,6 +16,7 @@ Where possible options are:
- `-DXXHASH_BUILD_XXHSUM=<ON|OFF>`: build the command line binary. ON by default
- `-DBUILD_SHARED_LIBS=<ON|OFF>`: build dynamic library. ON by default.
- `-DCMAKE_INSTALL_PREFIX=<path>`: use custom install prefix path.
- `-DDISPATCH=<ON|OFF>`: enable dispatch mode. OFF by default.
Add lines into downstream CMakeLists.txt:

View File

@ -16,7 +16,7 @@ Distribution of this document is unlimited.
### Version
0.1.1 (10/10/18)
0.2.0 (29/06/23)
Table of Contents
@ -24,6 +24,10 @@ Table of Contents
- [Introduction](#introduction)
- [XXH32 algorithm description](#xxh32-algorithm-description)
- [XXH64 algorithm description](#xxh64-algorithm-description)
- [XXH3 algorithm description](#xxh3-algorithm-overview)
- [Small inputs](#xxh3-algorithm-description-for-small-inputs)
- [Medium inputs](#xxh3-algorithm-description-for-medium-inputs)
- [Large inputs](#xxh3-algorithm-description-for-large-inputs)
- [Performance considerations](#performance-considerations)
- [Reference Implementation](#reference-implementation)
@ -43,13 +47,20 @@ However, a given variant shall produce exactly the same output, irrespective of
### Operation notations
All operations are performed modulo {32,64} bits. Arithmetic overflows are expected.
`XXH32` uses 32-bit modular operations. `XXH64` uses 64-bit modular operations.
`XXH32` uses 32-bit modular operations.
`XXH64` and `XXH3` use 64-bit modular operations.
When an operation ingests input or secret as multi-bytes values, it reads it using little-endian convention.
- `+`: denotes modular addition
- `-`: denotes modular subtraction
- `*`: denotes modular multiplication
- **Exception:** In `XXH3`, if it is in the form `(u128)x * (u128)y`, it denotes 64-bit by 64-bit normal multiplication into a full 128-bit result.
- `X <<< s`: denotes the value obtained by circularly shifting (rotating) `X` left by `s` bit positions.
- `X >> s`: denotes the value obtained by shifting `X` right by s bit positions. Upper `s` bits become `0`.
- `X << s`: denotes the value obtained by shifting `X` left by s bit positions. Lower `s` bits become `0`.
- `X xor Y`: denotes the bit-wise XOR of `X` and `Y` (same width).
- `X | Y`: denotes the bit-wise OR of `X` and `Y` (same width).
- `~X`: denotes the bit-wise negation of `X`.
XXH32 Algorithm Description
@ -232,7 +243,7 @@ Each lane read its associated 64-bit value using __little-endian__ convention.
For each {lane, accumulator}, the update process is called a _round_, and applies the following formula:
```c
round(accN,laneN):
round(accN,laneN):
accN = accN + (laneN * PRIME64_2);
accN = accN <<< 31;
return accN * PRIME64_1;
@ -250,7 +261,7 @@ All 4 lane accumulators from previous steps are merged to produce a single remai
Note that accumulator convergence is more complex than 32-bit variant, and requires to define another function called _mergeAccumulator()_:
```c
mergeAccumulator(acc,accN):
mergeAccumulator(acc,accN):
acc = acc xor round(0, accN);
acc = acc * PRIME64_1;
return acc + PRIME64_4;
@ -324,6 +335,462 @@ The `XXH64()` function produces an unsigned 64-bit value as output.
For systems which require to store and/or display the result in binary or hexadecimal format, the canonical format is defined to reproduce the same value as the natural decimal format, hence follows __big-endian__ convention (most significant byte first).
XXH3 Algorithm Overview
-------------------------------------
XXH3 comes in two different versions: XXH3-64 and XXH3-128 (or XXH128), producing 64 and 128 bits of output, respectively.
XXH3 uses different algorithms for small (0-16 bytes), medium (17-240 bytes), and large (241+ bytes) inputs. The algorithms for small and medium inputs are optimized for performance. The three algorithms are described in the following sections.
Many operations require some 64-bit prime number constants, which are mostly the same constants used in XXH32 and XXH64, all defined below:
```c
static const u64 PRIME32_1 = 0x9E3779B1U; // 0b10011110001101110111100110110001
static const u64 PRIME32_2 = 0x85EBCA77U; // 0b10000101111010111100101001110111
static const u64 PRIME32_3 = 0xC2B2AE3DU; // 0b11000010101100101010111000111101
static const u64 PRIME64_1 = 0x9E3779B185EBCA87ULL; // 0b1001111000110111011110011011000110000101111010111100101010000111
static const u64 PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; // 0b1100001010110010101011100011110100100111110101001110101101001111
static const u64 PRIME64_3 = 0x165667B19E3779F9ULL; // 0b0001011001010110011001111011000110011110001101110111100111111001
static const u64 PRIME64_4 = 0x85EBCA77C2B2AE63ULL; // 0b1000010111101011110010100111011111000010101100101010111001100011
static const u64 PRIME64_5 = 0x27D4EB2F165667C5ULL; // 0b0010011111010100111010110010111100010110010101100110011111000101
static const u64 PRIME_MX1 = 0x165667919E3779F9ULL; // 0b0001011001010110011001111001000110011110001101110111100111111001
static const u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; // 0b1001111110110010000111000110010100011110100110001101111100100101
```
The `XXH3_64bits()` function produces an unsigned 64-bit value.
The `XXH3_128bits()` function produces a `XXH128_hash_t` struct containing `low64` and `high64` - the lower and higher 64-bit half values of the result, respectively.
For systems requiring storing and/or displaying the result in binary or hexadecimal format, the canonical format is defined to reproduce the same value as the natural decimal format, hence following **big-endian** convention (most significant byte first).
### Seed and Secret
XXH3 provides seeded hashing by introducing two configurable constants used in the hashing process: the seed and the secret. The seed is an unsigned 64-bit value, and the secret is an array of bytes that is at least 136 bytes in size. The default seed is 0, and the default secret is the following 192-byte value:
```c
static const u8 defaultSecret[192] = {
0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
};
```
The seed and the secret can be optionally specified using the `*_withSecret` and `*_withSeed` versions of the hash function.
The seed and the secret cannot be specified simultaneously (`*_withSecretAndSeed` is actually `*_withSeed` for short and medium inputs <= 240 bytes, and `*_withSecret` for large inputs). When one is specified, the other one uses the default value.
There is one exception though: when input is large (> 240 bytes) and a seed is given, a secret is derived from the seed value and the default secret using the following procedure:
```c
deriveSecret(u64 seed):
u64 derivedSecret[24] = defaultSecret[0:192];
for (i = 0; i < 12; i++) {
derivedSecret[i*2] += seed;
derivedSecret[i*2+1] -= seed;
}
return derivedSecret; // convert to u8[192] (little-endian)
```
The derivation treats the secrets as 24 64-bit values. In XXH3 algorithms, the secret is always read similarly by treating a contiguous segment of the array as one or more 32-bit or 64-bit values. **The secret values are always read using little-endian convention**.
### Final Mixing Step (avalanche)
To make sure that all input bits have a chance to impact any bit in the output digest (avalanche effect), the final step of the XXH3 algorithm is usually one of the two fixed operations that mix the bits in a 64-bit value. These operations are denoted `avalanche()` and `avalanche_XXH64()` in the following XXH3 description.
```c
avalanche(u64 x):
x = x xor (x >> 37);
x = x * PRIME_MX1;
x = x xor (x >> 32);
return x;
avalanche_XXH64(u64 x):
x = x xor (x >> 33);
x = x * PRIME64_2;
x = x xor (x >> 29);
x = x * PRIME64_3;
x = x xor (x >> 32);
return x;
```
XXH3 Algorithm Description (for small inputs)
-------------------------------------
The algorithm for small inputs (0-16 bytes of input) is further divided into 4 cases: empty, 1-3 bytes, 4-8 bytes, and 9-16 bytes of input.
The algorithm uses byte-swap operations. The byte-swap operation reverses the byte order in a 32-bit or 64-bit value. It is denoted `bswap32` and `bswap64` for its 32-bit and 64-bit versions, respectively.
### Empty input
The hash of empty input is calculated from the seed and a segment of the secret:
```c
XXH3_64_empty():
u64 secretWords[2] = secret[56:72];
return avalanche_XXH64(seed xor secretWords[0] xor secretWords[1]);
XXH3_128_empty():
u64 secretWords[4] = secret[64:96];
return {avalanche_XXH64(seed xor secretWords[0] xor secretWords[1]), // lower half
avalanche_XXH64(seed xor secretWords[2] xor secretWords[3])}; // higher half
```
### 1-3 bytes of input
The algorithm starts from a single 32-bit value combining the input bytes and its length:
```c
u32 combined = (u32)input[inputLength-1] | ((u32)inputLength << 8) |
((u32)input[0] << 16) | ((u32)input[inputLength>>1] << 24);
// LSB 8 16 24 MSB
// | last byte | length | first byte | middle-or-last byte |
```
Then the final output is calculated from the value and the first 8 bytes (XXH3-64) or 16 bytes (XXH3-128) of the secret to produce the final result. The secret here is read as 32-bit values instead of the usual 64-bit values.
```c
XXH3_64_1to3():
u32 secretWords[2] = secret[0:8];
u64 value = ((u64)(secretWords[0] xor secretWords[1]) + seed) xor (u64)combined;
return avalanche_XXH64(value);
XXH3_128_1to3():
u32 secretWords[4] = secret[0:16];
u64 low = ((u64)(secretWords[0] xor secretWords[1]) + seed) xor (u64)combined;
u64 high = ((u64)(secretWords[2] xor secretWords[3]) - seed) xor (u64)(bswap32(combined) <<< 13);
// note that the bswap32(combined) <<< 13 above is 32-bit rotate
return {avalanche_XXH64(low), // lower half
avalanche_XXH64(high)}; // higher half
```
Note that the XXH3-64 result is the lower half of XXH3-128 result.
### 4-8 bytes of input
The algorithm starts from reading the first and last 4 bytes of the input as little-endian 32-bit values, and a modified seed:
```c
u32 inputFirst = input[0:4];
u32 inputLast = input[inputLength-4:inputLength];
u64 modifiedSeed = seed xor ((u64)bswap32((u32)lowerHalf(seed)) << 32);
```
Again, these values are combined with a segment of the secret to produce the final value.
```c
XXH3_64_4to8():
u64 secretWords[2] = secret[8:24];
u64 combined = (u64)inputLast | ((u64)inputFirst << 32);
u64 value = ((secretWords[0] xor secretWords[1]) - modifiedSeed) xor combined;
value = value xor (value <<< 49) xor (value <<< 24);
value = value * PRIME_MX2;
value = value xor ((value >> 35) + inputLength);
value = value * PRIME_MX2;
value = value xor (value >> 28);
return value;
XXH3_128_4to8():
u64 secretWords[2] = secret[16:32];
u64 combined = (u64)inputFirst | ((u64)inputLast << 32);
u64 value = ((secretWords[0] xor secretWords[1]) + modifiedSeed) xor combined;
u128 mulResult = (u128)value * (u128)(PRIME64_1 + (inputLength << 2));
u64 high = higherHalf(mulResult); // mulResult >> 64
u64 low = lowerHalf(mulResult); // mulResult & 0xFFFFFFFFFFFFFFFF
high = high + (low << 1);
low = low xor (high >> 3);
low = low xor (low >> 35);
low = low * PRIME_MX2;
low = low xor (low >> 28);
high = avalanche(high);
return {low, high};
```
### 9-16 bytes of input
The algorithm starts from reading the first and last 8 bytes of the input as little-endian 64-bit values:
```c
u64 inputFirst = input[0:8];
u64 inputLast = input[inputLength-8:inputLength];
```
Once again, these values are combined with a segment of the secret to produce the final value.
```c
XXH3_64_9to16():
u64 secretWords[4] = secret[24:56];
u64 low = ((secretWords[0] xor secretWords[1]) + seed) xor inputFirst;
u64 high = ((secretWords[2] xor secretWords[3]) - seed) xor inputLast;
u128 mulResult = (u128)low * (u128)high;
u64 value = inputLength + bswap64(low) + high + (u64)(lowerHalf(mulResult) xor higherHalf(mulResult));
return avalanche(value);
XXH3_128_9to16():
u64 secretWords[4] = secret[32:64];
u64 val1 = ((secretWords[0] xor secretWords[1]) - seed) xor inputFirst xor inputLast;
u64 val2 = ((secretWords[2] xor secretWords[3]) + seed) xor inputLast;
u128 mulResult = (u128)val1 * (u128)PRIME64_1;
u64 low = lowerHalf(mulResult) + ((u64)(inputLength - 1) << 54);
u64 high = higherHalf(mulResult) + ((u64)higherHalf(val2) << 32) + (u64)lowerHalf(val2) * PRIME32_2;
// the above line can also be simplified to higherHalf(mulResult) + val2 + (u64)lowerHalf(val2) * (PRIME32_2 - 1);
low = low xor bswap64(high);
// the following three lines are in fact a 128x64 -> 128 multiplication ({low,high} = (u128){low,high} * PRIME64_2)
u128 mulResult2 = (u128)low * (u128)PRIME64_2;
low = lowerHalf(mulResult2);
high = higherHalf(mulResult2) + high * PRIME64_2;
return {avalanche(low), // lower half
avalanche(high)}; // higher half
```
XXH3 Algorithm Description (for medium inputs)
-------------------------------------
This algorithm is used for medium inputs (17-240 bytes of input). Its internal hash state is stored inside 1 (XXH3-64) or 2 (XXH3-128) "accumulators", each storing an unsigned 64-bit value.
### Step 1. Initialize internal accumulators
The accumulator(s) are initialized based on the input length.
```c
// For XXH3-64
u64 acc = inputLength * PRIME64_1;
// For XXH3-128
u64 acc[2] = {inputLength * PRIME64_1, 0};
```
### Step 2. Process the input
This step is further divided into two cases: one for 17-128 bytes of input, and one for 129-240 bytes of input.
#### Mixing operation
This step uses a mixing operation that mixes a 16-byte segment of data, a 16-byte segment of secret and the seed into a 64-bit value as a building block. This operation treat the segment of data and secret as little-endian 64-bit values.
```c
mixStep(u8 data[16], size secretOffset, u64 seed):
u64 dataWords[2] = data[0:16];
u64 secretWords[2] = secret[secretOffset:secretOffset+16];
u128 mulResult = (u128)(dataWords[0] xor (secretWords[0] + seed)) *
(u128)(dataWords[1] xor (secretWords[1] - seed));
return lowerHalf(mulResult) xor higherHalf(mulResult);
```
The mixing operation is always invoked in groups of two in XXH3-128, where two 16-byte segments of data are mixed with a 32-byte segment of secret, and the accumulators are updated accordingly.
```c
mixTwoChunks(u8 data1[16], u8 data2[16], size secretOffset, u64 seed):
u64 dataWords1[2] = data1[0:16]; // again, little-endian conversion
u64 dataWords2[2] = data2[0:16];
acc[0] = acc[0] + mixStep(data1, secretOffset, seed);
acc[1] = acc[1] + mixStep(data2, secretOffset + 16, seed);
acc[0] = acc[0] xor (dataWords2[0] + dataWords2[1]);
acc[1] = acc[1] xor (dataWords1[0] + dataWords1[1]);
```
The input is split into several 16-byte chunks and mixed, and the result is added to the accumulator(s).
#### 17-128 bytes of input
The input is read as *N* 16-byte chunks starting from the beginning and *N* chunks starting from the end, where *N* is the smallest number that these 2*N* chunks cover the whole input. These chunks are paired up and mixed, and the results are accumulated to the accumulator(s).
```c
// the loop variable `i` should be signed to avoid underflow in implementation
processInput_XXH3_64_17to128():
u64 numRounds = ((inputLength - 1) >> 5) + 1;
for (i = numRounds - 1; i >= 0; i--) {
size offsetStart = i*16;
size offsetEnd = inputLength - i*16 - 16;
acc += mixStep(input[offsetStart:offsetStart+16], i*32, seed);
acc += mixStep(input[offsetEnd:offsetEnd+16], i*32+16, seed);
}
processInput_XXH3_128_17to128():
u64 numRounds = ((inputLength - 1) >> 5) + 1;
for (i = numRounds - 1; i >= 0; i--) {
size offsetStart = i*16;
size offsetEnd = inputLength - i*16 - 16;
mixTwoChunks(input[offsetStart:offsetStart+16], input[offsetEnd:offsetEnd+16], i*32, seed);
}
```
#### 129-240 bytes of input
The input is split into 16-byte (XXH3-64) or 32-byte (XXH3-128) chunks. The first 128 bytes are first mixed chunk by chunk, followed by an intermediate avalanche operation. Then the remaining full chunks are processed, and finally the last 16/32 bytes are treated as a chunk to process.
```c
processInput_XXH3_64_129to240():
u64 numChunks = inputLength >> 4;
for (i = 0; i < 8; i++) {
acc += mixStep(input[i*16:i*16+16], i*16, seed);
}
acc = avalanche(acc);
for (i = 8; i < numChunks; i++) {
acc += mixStep(input[i*16:i*16+16], (i-8)*16 + 3, seed);
}
acc += mixStep(input[inputLength-16:inputLength], 119, seed);
processInput_XXH3_128_129to240():
u64 numChunks = inputLength >> 5;
for (i = 0; i < 4; i++) {
mixTwoChunks(input[i*32:i*32+16], input[i*32+16:i*32+32], i*32, seed);
}
acc[0] = avalanche(acc[0]);
acc[1] = avalanche(acc[1]);
for (i = 4; i < numChunks; i++) {
mixTwoChunks(input[i*32:i*32+16], input[i*32+16:i*32+32], (i-4)*32 + 3, seed);
}
// note that the half-chunk order and the seed is different here
mixTwoChunks(input[inputLength-16:inputLength], input[inputLength-32:inputLength-16], 103, (u64)0 - seed);
```
### Step 3. Finalization
The final result is extracted from the accumulator(s).
```c
XXH3_64_17to240():
return avalanche(acc);
XXH3_128_17to240():
u64 low = acc[0] + acc[1];
u64 high = (acc[0] * PRIME64_1) + (acc[1] * PRIME64_4) + (((u64)inputLength - seed) * PRIME64_2);
return {avalanche(low), // lower half
(u64)0 - avalanche(high)}; // higher half
```
XXH3 Algorithm Description (for large inputs)
-------------------------------------
This algorithm is used for inputs larger than 240 bytes. The internal hash state is stored inside 8 "accumulators", each one storing an unsigned 64-bit value.
### Step 1. Initialize internal accumulators
The accumulators are initialized to fixed constants:
```c
u64 acc[8] = {
PRIME32_3, PRIME64_1, PRIME64_2, PRIME64_3,
PRIME64_4, PRIME32_2, PRIME64_5, PRIME32_1};
```
### Step 2. Process blocks
The input is consumed and processed one full block at a time. The size of the block depends on the length of the secret. Specifically, a block consists of several 64-byte stripes. The number of stripes per block is `floor((secretLength-64)/8)` . For the default 192-byte secret, there are 16 stripes in a block, and thus the block size is 1024 bytes.
```c
secretLength = lengthInBytes(secret); // default 192; at least 136
stripesPerBlock = (secretLength-64) / 8; // default 16; at least 9
blockSize = 64 * stripesPerBlock; // default 1024; at least 576
```
The process of processing a full block is called a *round*. It consists of the following two sub-steps:
#### Step 2-1. Process stripes in the block
A stripe is evenly divided into 8 lanes, of 8 bytes each. In an accumulation step, one stripe and a 64-byte contiguous segment of the secret are used to update the accumulators. Each lane reads its associated 64-bit value using little-endian convention.
The accumulation step applies the following procedure:
```c
accumulate(u64 stripe[8], size secretOffset):
u64 secretWords[8] = secret[secretOffset:secretOffset+64];
for (i = 0; i < 8; i++) {
u64 value = stripe[i] xor secretWords[i];
acc[i xor 1] = acc[i xor 1] + stripe[i];
acc[i] = acc[i] + (u64)lowerHalf(value) * (u64)higherHalf(value);
// (value and 0xFFFFFFFF) * (value >> 32)
}
```
The accumulation step is repeated for all stripes in a block, using different segments of the secret, starting from the first 64 bytes for the first stripe, and offset by 8 bytes for each following round:
```c
round_accumulate(u8 block[blockSize]):
for (n = 0; n < stripesPerBlock; n++) {
u64 stripe[8] = block[n*64:n*64+64]; // 64 bytes = 8 u64s
accumulate(stripe, n*8);
}
```
#### Step 2-2. Scramble accumulators
After the accumulation steps are finished for all stripes in the block, the accumulators are scrambled using the last 64 bytes of the secret.
```c
round_scramble():
u64 secretWords[8] = secret[secretLength-64:secretLength];
for (i = 0; i < 8; i++) {
acc[i] = acc[i] xor (acc[i] >> 47);
acc[i] = acc[i] xor secretWords[i];
acc[i] = acc[i] * PRIME32_1;
}
```
A round is thus a `round_accumulate` followed by a `round_scramble`:
```c
round(u8 block[blockSize]):
round_accumulate(block);
round_scramble();
```
Step 2 is looped to consume the input until there are less than or equal to `blockSize` bytes of input left. Note that we leave the last block to the next step even if it is a full block.
### Step 3. Process the last block and the last 64 bytes
Accumulation steps are run for the stripes in the last block, except for the last stripe (whether it is full or not). After that, run a final accumulation step by treating the last 64 bytes as a stripe. Note that the last 64 bytes might overlap with the second-to-last block.
```c
// len is the size of the last block (1 <= len <= blockSize)
lastRound(u8 block[], size len, u64 lastStripe[8]):
size nFullStripes = (len-1)/64;
for (n = 0; n < nFullStripes; n++) {
u64 stripe[8] = block[n*64:n*64+64];
accumulate(stripe, n * 8);
}
accumulate(lastStripe, secretLength - 71);
```
### Step 4. Finalization
In the finalization step, a merging procedure is used to extract a single 64-bit value from the accumulators, using an initial seed value and a 64-byte segment of the secret.
```c
finalMerge(u64 initValue, size secretOffset):
u64 secretWords[8] = secret[secretOffset:secretOffset+64];
u64 result = initValue;
for (i = 0; i < 4; i++) {
// 64-bit by 64-bit multiplication to 128-bit full result
u128 mulResult = (u128)(acc[i*2] xor secretWords[i*2]) *
(u128)(acc[i*2+1] xor secretWords[i*2+1]);
result = result + (lowerHalf(mulResult) xor higherHalf(mulResult));
// (mulResult and 0xFFFFFFFFFFFFFFFF) xor (mulResult >> 64)
}
return avalanche(result);
```
XXH3-128 runs the merging procedure twice for the two halves of the result, using different secret segments and different initial values derived from the total input length.
The XXH3-64 result is just the lower half of the XXH3-128 result.
```c
XXH3_64_large():
return finalMerge((u64)inputLength * PRIME64_1, 11);
XXH3_128_large():
return {finalMerge((u64)inputLength * PRIME64_1, 11), // lower half
finalMerge(~((u64)inputLength * PRIME64_2), secretLength - 75)}; // higher half
```
Performance considerations
----------------------------------
@ -335,6 +802,8 @@ On 64-bit systems, the 64-bit variant `XXH64` is generally faster to compute, so
On 32-bit systems though, positions are reversed: `XXH64` performance is reduced, due to its usage of 64-bit arithmetic. `XXH32` becomes a faster variant.
Finally, when vector operations are possible, `XXH3` is likely the faster variant.
Reference Implementation
----------------------------------------
@ -346,6 +815,6 @@ It links to the [github project page](https://github.com/Cyan4973/xxHash) where
Version changes
--------------------
v0.7.3: Minor fixes
v0.2.0: added XXH3 specification, by Adrien Wu
v0.1.1: added a note on rationale for selection of constants
v0.1.0: initial release

View File

@ -1,11 +1,11 @@
# xxHash - Extremely fast hash algorithm
# Copyright (C) 2012-2020, Yann Collet, Facebook
# Copyright (C) 2012-2021, Yann Collet, Facebook
# BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
prefix=@PREFIX@
exec_prefix=@EXECPREFIX@
includedir=${prefix}/@INCLUDEDIR@
libdir=${exec_prefix}/@LIBDIR@
includedir=@INCLUDEDIR@
libdir=@LIBDIR@
Name: xxhash
Description: extremely fast hash algorithm

View File

@ -1,6 +1,6 @@
# ################################################################
# xxHash Makefile
# Copyright (C) 2012-2020 Yann Collet
# Copyright (C) 2012-2021 Yann Collet
#
# GPL v2 License
#
@ -51,7 +51,7 @@ default: all
all: test
.PHONY: test
test: test_multiInclude test_unicode
test: test_multiInclude test_unicode test_sanity
.PHONY: test_multiInclude
test_multiInclude:
@ -100,6 +100,28 @@ test_unicode: $(XXHSUM) generate_unicode_test.c
$(SHELL) ./unicode_test.sh
endif
.PHONY: test_filename_escape
test_filename_escape: $(XXHSUM)
./filename-escape.sh
.PHONY: test_cli_comment_line
test_cli_comment_line: $(XXHSUM)
$(SHELL) ./cli-comment-line.sh
.PHONY: test_cli_ignore_missing
test_cli_ignore_missing: $(XXHSUM)
$(SHELL) ./cli-ignore-missing.sh
.PHONY: test_sanity
test_sanity: sanity_test.c
$(CC) $(CFLAGS) $(LDFLAGS) sanity_test.c -o sanity_test$(EXT)
$(RUN_ENV) ./sanity_test$(EXT)
.PHONY: sanity_test_vectors.h
sanity_test_vectors.h: sanity_test_vectors_generator.c
$(CC) $(CFLAGS) $(LDFLAGS) sanity_test_vectors_generator.c -o sanity_test_vectors_generator$(EXT)
./sanity_test_vectors_generator$(EXT)
xxhash.o: ../xxhash.c ../xxhash.h
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -c -o $@ $<
@ -110,3 +132,4 @@ clean:
@$(RM) *.o
@$(RM) multiInclude multiInclude_withxxhash
@$(RM) *.unicode generate_unicode_test$(EXT) unicode_test.* xxhsum*
@$(RM) sanity_test$(EXT) sanity_test_vectors_generator$(EXT)

View File

@ -1,6 +1,6 @@
# ################################################################
# xxHash benchHash Makefile
# Copyright (C) 2019-2020 Yann Collet
# Copyright (C) 2019-2021 Yann Collet
#
# GPL v2 License
#

View File

@ -1,7 +1,7 @@
/*
* Hash benchmark module
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*
@ -28,6 +28,7 @@
#include <stdlib.h> // malloc
#include <assert.h>
#include <string.h>
#include "benchHash.h"
@ -104,6 +105,7 @@ bench_hash_internal(BMK_benchFn_t hashfn, void* payload,
};
BMK_runOutcome_t result;
memset(&result, 0, sizeof(result));
while (!BMK_isCompleted_TimedFn(txf)) {
result = BMK_benchTimedFn(txf, params);
assert(BMK_isSuccessful_runOutcome(result));

View File

@ -1,7 +1,7 @@
/*
* Hash benchmark module
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
@ -90,6 +90,7 @@ static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
{
BMK_runOutcome_t outcome;
memset(&outcome, 0, sizeof(outcome));
outcome.error_tag_never_ever_use_directly = 0;
outcome.internal_never_ever_use_directly = runTime;
return outcome;

View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

View File

@ -1,7 +1,7 @@
/*
* CSV Display module for the hash benchmark program
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,7 +1,7 @@
/*
* CSV Display module for the hash benchmark program
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,7 +1,7 @@
/*
* List hash algorithms to benchmark
* Part of xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,7 +1,7 @@
/*
* Main program to benchmark hash functions
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2019-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2019-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Exit immediately if any command fails.
# https://stackoverflow.com/a/2871034
set -euxo
# Default
./xxhsum ./Makefile > ./.test.xxh
echo '# Test comment line' | cat - ./.test.xxh > temp && mv temp ./.test.xxh
./xxhsum --check ./.test.xxh
# XXH32
./xxhsum -H32 ./Makefile > ./.test.xxh32
echo '# Test comment line' | cat - ./.test.xxh32 > temp && mv temp ./.test.xxh32
./xxhsum --check ./.test.xxh32
# XXH64
./xxhsum -H64 ./Makefile > ./.test.xxh64
echo '# Test comment line' | cat - ./.test.xxh64 > temp && mv temp ./.test.xxh64
./xxhsum --check ./.test.xxh64
# XXH128
./xxhsum -H128 ./Makefile > ./.test.xxh128
echo '# Test comment line' | cat - ./.test.xxh128 > temp && mv temp ./.test.xxh128
./xxhsum --check ./.test.xxh128
rm ./.test.xxh
rm ./.test.xxh32
rm ./.test.xxh64
rm ./.test.xxh128

View File

@ -0,0 +1,51 @@
#!/bin/bash
# Exit immediately if any command fails.
# https://stackoverflow.com/a/2871034
set -e -u -x
# Normal
./xxhsum ./Makefile > ./.test.xxh
./xxhsum --check ./.test.xxh
# Missing, expect error
# (1) Create checksum file.
# (2) Remove one of them.
# (3) --check it
# (4) Expect NG (missing file)
cp Makefile .test.makefile
./xxhsum ./.test.makefile > ./.test.xxh
rm ./.test.makefile
! ./xxhsum --check ./.test.xxh # Put '!' for expecting error
# Missing, --ignore-missing
# (1) Create checksum file.
# (2) Remove one of them.
# (3) --check it with --ignore-missing.
# (4) Expect OK
cp Makefile .test.makefile
./xxhsum Makefile ./.test.makefile > ./.test.xxh
rm ./.test.makefile
./xxhsum --check --ignore-missing ./.test.xxh
# Missing, --ignore-missing, expect error
# (1) Create checksum file.
# (2) Remove all of them.
# (3) --check it with --ignore-missing.
# (4) Expect NG (no file was verified).
cp Makefile .test.makefile
./xxhsum ./.test.makefile > ./.test.xxh
rm ./.test.makefile
! ./xxhsum --check --ignore-missing ./.test.xxh # Put '!' for expecting error
# Cleanup
( rm ./.test.* ) || true
echo OK

View File

@ -1,6 +1,6 @@
# Brute force collision tester for 64-bit hashes
# Part of xxHash project
# Copyright (C) 2019-2020 Yann Collet
# Copyright (C) 2019-2021 Yann Collet
#
# GPL v2 License
#

View File

@ -1,7 +1,7 @@
/*
* List of hashes for the brute force collision tester
* Part of xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*

View File

@ -1,7 +1,7 @@
/*
* Brute force collision tester for 64-bit hashes
* Part of the xxHash project
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* GPL v2 License
*
@ -206,7 +206,7 @@ static double Cnm(int n, int m)
{
assert(n > 0);
assert(m > 0);
assert(m <= m);
assert(m <= n);
double acc = 1;
for (int i=0; i<m; i++) {
acc *= n - i;

View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2020 Yann Collet, Facebook, Inc.
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the

View File

@ -1,6 +1,6 @@
/*
* sort.cc - C++ sort functions
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

View File

@ -1,6 +1,6 @@
/*
* sort.hh - headers for C++ sort functions
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
* GPL v2 License
*
* This program is free software; you can redistribute it and/or modify

21
third-party/xxhash/tests/filename-escape.sh vendored Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
# Exit immediately if any command fails.
# https://stackoverflow.com/a/2871034
set -euxo pipefail
echo filename-escape-test-string > $'filename-escape-foo\nbar'
./xxhsum $'filename-escape-foo\nbar' | tee filename-escape-xxh64.txt
cat filename-escape-xxh64.txt
./xxhsum -c filename-escape-xxh64.txt
hexdump -C filename-escape-xxh64.txt
./xxhsum --tag $'filename-escape-foo\nbar' | tee filename-escape-xxh64-tag.txt
cat filename-escape-xxh64-tag.txt
./xxhsum -c filename-escape-xxh64-tag.txt
hexdump -C filename-escape-xxh64-tag.txt
rm filename-escape-xxh64-tag.txt
rm filename-escape-xxh64.txt
rm $'filename-escape-foo\nbar'

763
third-party/xxhash/tests/sanity_test.c vendored Normal file
View File

@ -0,0 +1,763 @@
// xxHash/tests/sanity_test.c
// SPDX-License-Identifier: GPL-2.0-only
//
// Building
// ========
//
// cc sanity_test.c && ./a.out
//
/*
notes or changes:
main()
------
- All test methods (XXH32, XXH64, ...) are context free.
- It means that there's no restriction by order of tests and # of test. (Ready for multi-threaded / distributed test)
- To achieve it, some test has dedicated 'randSeed' to decouple dependency between tests.
- Note that for() loop is not ready for distributed test.
- randSeed still needs to be computed step by step in the for() loop so far.
*/
#define XXH_STATIC_LINKING_ONLY
#define XXH_IMPLEMENTATION /* access definitions */
#include "../cli/xsum_arch.h"
#include "../cli/xsum_os_specific.h"
#include "../cli/xsum_output.h"
#include "../cli/xsum_output.c"
#define XSUM_NO_MAIN 1
#include "../cli/xsum_os_specific.h"
#include "../cli/xsum_os_specific.c"
#include "../xxhash.h"
#include "sanity_test_vectors.h"
#include <assert.h> /* assert */
/* use #define to make them constant, required for initialization */
#define PRIME32 2654435761U
#define PRIME64 11400714785074694797ULL
#define SANITY_BUFFER_SIZE (4096 + 64 + 1)
/**/
static int abortByError = 1;
/**/
static void abortSanityTest(void) {
/* ??? : Should we show this message? */
XSUM_log("\rNote: If you modified the hash functions, make sure to either update tests/sanity_test_vectors.h with the following command\n"
"\r make -C tests sanity_test_vectors.h\n");
XSUM_log("\rAbort.\n");
exit(1);
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Fills a test buffer with pseudorandom data.
*
* This is used in the sanity check - its values must not be changed.
*/
static void fillTestBuffer(XSUM_U8* buffer, size_t bufferLenInBytes)
{
XSUM_U64 byteGen = PRIME32;
size_t i;
assert(buffer != NULL);
for (i = 0; i < bufferLenInBytes; ++i) {
buffer[i] = (XSUM_U8)(byteGen>>56);
byteGen *= PRIME64;
}
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Create (malloc) and fill buffer with pseudorandom data for sanity check.
*
* Use releaseSanityBuffer() to delete the buffer.
*/
static XSUM_U8* createSanityBuffer(size_t bufferLenInBytes)
{
XSUM_U8* buffer = (XSUM_U8*) malloc(bufferLenInBytes);
assert(buffer != NULL);
fillTestBuffer(buffer, bufferLenInBytes);
return buffer;
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Delete (free) the buffer which has been genereated by createSanityBuffer()
*/
static void releaseSanityBuffer(XSUM_U8* buffer)
{
assert(buffer != NULL);
free(buffer);
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void checkResult32(XXH32_hash_t r1, XXH32_hash_t r2, const char* testName, size_t testNb, size_t lineNb)
{
if(r1 == r2) {
return;
}
XSUM_log("\rError: %s #%zd, line #%zd: Sanity check failed!\n", testName, testNb, lineNb);
XSUM_log("\rGot 0x%08X, expected 0x%08X.\n", (unsigned)r1, (unsigned)r2);
if(abortByError) {
abortSanityTest();
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void checkResult64(XXH64_hash_t r1, XXH64_hash_t r2, const char* testName, size_t testNb, size_t lineNb)
{
if(r1 == r2) {
return;
}
XSUM_log("\rError: %s #%zd, line #%zd: Sanity check failed!\n", testName, testNb, lineNb);
XSUM_log("\rGot 0x%08X%08XULL, expected 0x%08X%08XULL.\n",
(unsigned)(r1>>32), (unsigned)r1, (unsigned)(r2>>32), (unsigned)r2);
if(abortByError) {
abortSanityTest();
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void checkResult128(XXH128_hash_t r1, XXH128_hash_t r2, const char* testName, size_t testNb, size_t lineNb)
{
if ((r1.low64 == r2.low64) && (r1.high64 == r2.high64)) {
return;
}
XSUM_log("\rError: %s #%zd, line #%zd: Sanity check failed!\n", testName, testNb, lineNb);
XSUM_log("\rGot { 0x%08X%08XULL, 0x%08X%08XULL }, expected { 0x%08X%08XULL, 0x%08X%08XULL } \n",
(unsigned)(r1.low64>>32), (unsigned)r1.low64, (unsigned)(r1.high64>>32), (unsigned)r1.high64,
(unsigned)(r2.low64>>32), (unsigned)r2.low64, (unsigned)(r2.high64>>32), (unsigned)r2.high64 );
if(abortByError) {
abortSanityTest();
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void checkResultTestDataSample(const XSUM_U8* r1, const XSUM_U8* r2, const char* testName, size_t testNb, size_t lineNb)
{
if(memcmp(r1, r2, SECRET_SAMPLE_NBBYTES) == 0) {
return;
}
XSUM_log("\rError: %s #%zd, line #%zd: Sanity check failed!\n", testName, testNb, lineNb);
XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n",
r1[0], r1[1], r1[2], r1[3], r1[4],
r2[0], r2[1], r2[2], r2[3], r2[4] );
if(abortByError) {
abortSanityTest();
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH32(
const void* data,
const XSUM_testdata32_t* testData,
const char* testName,
size_t testNb
) {
size_t const len = testData->len;
XSUM_U32 const seed = testData->seed;
XSUM_U32 const Nresult = testData->Nresult;
XXH32_state_t * const state = XXH32_createState();
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
assert(state != NULL);
checkResult32(XXH32(data, len, seed), Nresult, testName, testNb, __LINE__);
(void)XXH32_reset(state, seed);
(void)XXH32_update(state, data, len);
checkResult32(XXH32_digest(state), Nresult, testName, testNb, __LINE__);
(void)XXH32_reset(state, seed);
{
size_t pos;
for (pos = 0; pos < len; ++pos) {
(void)XXH32_update(state, ((const char*)data)+pos, 1);
}
}
checkResult32(XXH32_digest(state), Nresult, testName, testNb, __LINE__);
XXH32_freeState(state);
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH64(
const void* data,
const XSUM_testdata64_t* testData,
const char* testName,
size_t testNb
)
{
size_t const len = (size_t)testData->len;
XSUM_U64 const seed = testData->seed;
XSUM_U64 const Nresult = testData->Nresult;
XXH64_state_t * const state = XXH64_createState();
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
assert(state != NULL);
checkResult64(XXH64(data, len, seed), Nresult, testName, testNb, __LINE__);
(void)XXH64_reset(state, seed);
(void)XXH64_update(state, data, len);
checkResult64(XXH64_digest(state), Nresult, testName, testNb, __LINE__);
(void)XXH64_reset(state, seed);
{
size_t pos;
for (pos = 0; pos < len; ++pos) {
(void)XXH64_update(state, ((const char*)data)+pos, 1);
}
}
checkResult64(XXH64_digest(state), Nresult, testName, testNb, __LINE__);
XXH64_freeState(state);
}
/* TODO : Share this function with xsum_sanity_check.c */
/*
* Used to get "random" (but actually 100% reproducible) lengths for
* XSUM_XXH3_randomUpdate.
*/
static XSUM_U32 SANITY_TEST_rand(XSUM_U64* pRandSeed)
{
XSUM_U64 seed = *pRandSeed;
seed *= PRIME64;
*pRandSeed = seed;
return (XSUM_U32)(seed >> 40);
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static XSUM_U64 SANITY_TEST_computeRandSeed(size_t step)
{
XSUM_U64 randSeed = PRIME32;
size_t i = 0;
for(i = 0; i < step; ++i) {
SANITY_TEST_rand(&randSeed);
}
return randSeed;
}
/* TODO : Share this definition with xsum_sanity_check.c */
/*
* Technically, XXH3_64bits_update is identical to XXH3_128bits_update as of
* v0.8.0, but we treat them as separate.
*/
typedef XXH_errorcode (*SANITY_TEST_XXH3_update_t)(XXH3_state_t* state, const void* input, size_t length);
/* TODO : Share this function with xsum_sanity_check.c */
/*
* Runs the passed XXH3_update variant on random lengths. This is to test the
* more complex logic of the update function, catching bugs like this one:
* https://github.com/Cyan4973/xxHash/issues/378
*/
static void SANITY_TEST_XXH3_randomUpdate(
XXH3_state_t* state,
const void* data,
size_t len,
XSUM_U64* pRandSeed,
SANITY_TEST_XXH3_update_t update_fn
)
{
size_t p = 0;
while (p < len) {
size_t const modulo = len > 2 ? len : 2;
size_t l = (size_t)(SANITY_TEST_rand(pRandSeed)) % modulo;
if (p + l > len) l = len - p;
(void)update_fn(state, (const char*)data+p, l);
p += l;
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH3(
const void* data,
const XSUM_testdata64_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
size_t testNb
)
{
size_t const len = testData->len;
XSUM_U64 const seed = testData->seed;
XSUM_U64 const Nresult = testData->Nresult;
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
{ XSUM_U64 const Dresult = XXH3_64bits_withSeed(data, len, seed);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that the no-seed variant produces same result as seed==0 */
if (seed == 0) {
XSUM_U64 const Dresult = XXH3_64bits(data, len);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that the combination of
* XXH3_generateSecret_fromSeed() and XXH3_64bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
{ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
} }
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
assert(state != NULL);
/* single ingestion */
(void)XXH3_64bits_reset_withSeed(state, seed);
(void)XXH3_64bits_update(state, data, len);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
/* random ingestion */
(void)XXH3_64bits_reset_withSeed(state, seed);
SANITY_TEST_XXH3_randomUpdate(state, data, len, pRandSeed, &XXH3_64bits_update);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
/* byte by byte ingestion */
{ size_t pos;
(void)XXH3_64bits_reset_withSeed(state, seed);
for (pos=0; pos<len; pos++)
(void)XXH3_64bits_update(state, ((const char*)data)+pos, 1);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
}
/* check that streaming with a combination of
* XXH3_generateSecret_fromSeed() and XXH3_64bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
/* single ingestion */
(void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_64bits_update(state, data, len);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
}
XXH3_freeState(state);
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH3_withSecret(
const void* data,
const void* secret,
size_t secretSize,
const XSUM_testdata64_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
size_t testNb
)
{
size_t const len = (size_t)testData->len;
XSUM_U64 const Nresult = testData->Nresult;
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
{ XSUM_U64 const Dresult = XXH3_64bits_withSecret(data, len, secret, secretSize);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that XXH3_64bits_withSecretandSeed()
* results in exactly the same return value as XXH3_64bits_withSecret() */
if (len > XXH3_MIDSIZE_MAX)
{ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, secretSize, 0);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
}
/* streaming API test */
{ XXH3_state_t * const state = XXH3_createState();
assert(state != NULL);
(void)XXH3_64bits_reset_withSecret(state, secret, secretSize);
(void)XXH3_64bits_update(state, data, len);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
/* random ingestion */
(void)XXH3_64bits_reset_withSecret(state, secret, secretSize);
SANITY_TEST_XXH3_randomUpdate(state, data, len, pRandSeed, &XXH3_64bits_update);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
/* byte by byte ingestion */
{ size_t pos;
(void)XXH3_64bits_reset_withSecret(state, secret, secretSize);
for (pos=0; pos<len; pos++)
(void)XXH3_64bits_update(state, ((const char*)data)+pos, 1);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
}
/* check that XXH3_64bits_reset_withSecretandSeed()
* results in exactly the same return value as XXH3_64bits_reset_withSecret() */
if (len > XXH3_MIDSIZE_MAX) {
/* single ingestion */
(void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 0);
(void)XXH3_64bits_update(state, data, len);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
}
XXH3_freeState(state);
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH128(
const void* data,
const XSUM_testdata128_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
size_t testNb
)
{
size_t const len = (size_t)testData->len;
XSUM_U64 const seed = testData->seed;
XXH128_hash_t const Nresult = testData->Nresult;
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
{ XXH128_hash_t const Dresult = XXH3_128bits_withSeed(data, len, seed);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that XXH128() is identical to XXH3_128bits_withSeed() */
{ XXH128_hash_t const Dresult2 = XXH128(data, len, seed);
checkResult128(Dresult2, Nresult, testName, testNb, __LINE__);
}
/* check that the no-seed variant produces same result as seed==0 */
if (seed == 0) {
XXH128_hash_t const Dresult = XXH3_128bits(data, len);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that the combination of
* XXH3_generateSecret_fromSeed() and XXH3_128bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
} }
/* streaming API test */
{ XXH3_state_t * const state = XXH3_createState();
assert(state != NULL);
/* single ingestion */
(void)XXH3_128bits_reset_withSeed(state, seed);
(void)XXH3_128bits_update(state, data, len);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
/* random ingestion */
(void)XXH3_128bits_reset_withSeed(state, seed);
SANITY_TEST_XXH3_randomUpdate(state, data, len, pRandSeed, &XXH3_128bits_update);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
/* byte by byte ingestion */
{ size_t pos;
(void)XXH3_128bits_reset_withSeed(state, seed);
for (pos=0; pos<len; pos++)
(void)XXH3_128bits_update(state, ((const char*)data)+pos, 1);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
}
/* check that streaming with a combination of
* XXH3_generateSecret_fromSeed() and XXH3_128bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_128bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
char* const secret = secretBuffer + 1; /* intentional unalignment */
XXH3_generateSecret_fromSeed(secret, seed);
/* single ingestion */
(void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_128bits_update(state, data, len);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
}
XXH3_freeState(state);
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testXXH128_withSecret(
const void* data,
const void* secret,
size_t secretSize,
const XSUM_testdata128_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
size_t testNb
)
{
size_t const len = testData->len;
XXH128_hash_t const Nresult = testData->Nresult;
if (len == 0) {
data = NULL;
} else {
assert(data != NULL);
}
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecret(data, len, secret, secretSize);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
}
/* check that XXH3_128bits_withSecretandSeed()
* results in exactly the same return value as XXH3_128bits_withSecret() */
if (len > XXH3_MIDSIZE_MAX)
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, secretSize, 0);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
}
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
assert(state != NULL);
(void)XXH3_128bits_reset_withSecret(state, secret, secretSize);
(void)XXH3_128bits_update(state, data, len);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
/* random ingestion */
(void)XXH3_128bits_reset_withSecret(state, secret, secretSize);
SANITY_TEST_XXH3_randomUpdate(state, data, len, pRandSeed, &XXH3_128bits_update);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
/* byte by byte ingestion */
{ size_t pos;
(void)XXH3_128bits_reset_withSecret(state, secret, secretSize);
for (pos=0; pos<len; pos++)
(void)XXH3_128bits_update(state, ((const char*)data)+pos, 1);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
}
/* check that XXH3_128bits_reset_withSecretandSeed()
* results in exactly the same return value as XXH3_128bits_reset_withSecret() */
if (len > XXH3_MIDSIZE_MAX) {
/* single ingestion */
(void)XXH3_128bits_reset_withSecretandSeed(state, secret, secretSize, 0);
(void)XXH3_128bits_update(state, data, len);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
}
XXH3_freeState(state);
}
}
/* TODO : Share this function with xsum_sanity_check.c */
/**/
static void testSecretGenerator(
const void* customSeed,
const XSUM_testdata_sample_t* testData,
const char* testName,
size_t testNb
)
{
/* TODO : Share this array with sanity_check.c and xsum_sanity_check.c */
static const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191, 241 }; /* position of sampled bytes */
XSUM_U8 secretBuffer[SECRET_SIZE_MAX] = {0};
XSUM_U8 samples[SECRET_SAMPLE_NBBYTES];
int i;
assert(testData->secretLen <= SECRET_SIZE_MAX);
XXH3_generateSecret(secretBuffer, testData->secretLen, customSeed, testData->seedLen);
for (i=0; i<SECRET_SAMPLE_NBBYTES; i++) {
samples[i] = secretBuffer[sampleIndex[i]];
}
checkResultTestDataSample(samples, testData->byte, testName, testNb, __LINE__);
}
/**/
int main(int argc, const char* argv[])
{
size_t testCount = 0;
size_t const sanityBufferSizeInBytes = SANITY_BUFFER_SIZE;
XSUM_U8* const sanityBuffer = createSanityBuffer(sanityBufferSizeInBytes);
const void* const secret = sanityBuffer + 7;
size_t const secretSize = XXH3_SECRET_SIZE_MIN + 11;
assert(sanityBufferSizeInBytes >= 7 + secretSize);
(void) argc;
(void) argv;
{
/* XXH32 */
size_t const n = sizeof(XSUM_XXH32_testdata) / sizeof(XSUM_XXH32_testdata[0]);
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH32(
sanityBuffer,
&XSUM_XXH32_testdata[i],
"XSUM_XXH32_testdata",
i
);
}
}
{
/* XXH64 */
size_t const n = sizeof(XSUM_XXH64_testdata) / sizeof(XSUM_XXH64_testdata[0]);
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH64(
sanityBuffer,
&XSUM_XXH64_testdata[i],
"XSUM_XXH64_testdata",
i
);
}
}
{
/* XXH3_64bits, seeded */
size_t const randCount = 0;
XSUM_U64 randSeed = SANITY_TEST_computeRandSeed(randCount);
size_t const n = sizeof(XSUM_XXH3_testdata) / sizeof(XSUM_XXH3_testdata[0]);
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH3(
sanityBuffer,
&XSUM_XXH3_testdata[i],
&randSeed,
"XSUM_XXH3_testdata",
i
);
}
}
{
/* XXH3_64bits, custom secret */
size_t const randCount = 22730;
XSUM_U64 randSeed = SANITY_TEST_computeRandSeed(randCount);
size_t const n = sizeof(XSUM_XXH3_withSecret_testdata) / sizeof(XSUM_XXH3_withSecret_testdata[0]);
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH3_withSecret(
sanityBuffer,
secret,
secretSize,
&XSUM_XXH3_withSecret_testdata[i],
&randSeed,
"XSUM_XXH3_withSecret_testdata",
i
);
testCount++;
}
}
{
/* XXH128 */
size_t const randCount = 34068;
XSUM_U64 randSeed = SANITY_TEST_computeRandSeed(randCount);
size_t const n = (sizeof(XSUM_XXH128_testdata)/sizeof(XSUM_XXH128_testdata[0]));
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH128(
sanityBuffer,
&XSUM_XXH128_testdata[i],
&randSeed,
"XSUM_XXH128_testdata",
i
);
}
}
{
/* XXH128 with custom Secret */
size_t const randCount = 68019;
XSUM_U64 randSeed = SANITY_TEST_computeRandSeed(randCount);
size_t const n = (sizeof(XSUM_XXH128_withSecret_testdata)/sizeof(XSUM_XXH128_withSecret_testdata[0]));
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
testXXH128_withSecret(
sanityBuffer,
secret,
secretSize,
&XSUM_XXH128_withSecret_testdata[i],
&randSeed,
"XSUM_XXH128_withSecret_testdata",
i
);
}
}
{
/* secret generator */
size_t const n = sizeof(XSUM_XXH3_generateSecret_testdata) / sizeof(XSUM_XXH3_generateSecret_testdata[0]);
size_t i;
for (i = 0; i < n; ++i, ++testCount) {
assert(XSUM_XXH3_generateSecret_testdata[i].seedLen <= SANITY_BUFFER_SIZE);
testSecretGenerator(
sanityBuffer,
&XSUM_XXH3_generateSecret_testdata[i],
"XSUM_XXH3_generateSecret_testdata",
i
);
}
}
releaseSanityBuffer(sanityBuffer);
XSUM_log("\rOK. (passes %zd tests)\n", testCount);
return EXIT_SUCCESS;
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,464 @@
// xxHash/tests/sanity_test_vectors_generator.c
// SPDX-License-Identifier: GPL-2.0-only
//
// So far, this program just generates sanity_test_vectors.h
//
// Building
// ========
//
// cc sanity_test_vectors_generator.c && ./a.out
// less sanity_test_vectors.h
//
#define XXH_STATIC_LINKING_ONLY
#define XXH_IMPLEMENTATION /* access definitions */
#include "../cli/xsum_arch.h"
#include "../cli/xsum_os_specific.h"
#include "../xxhash.h"
#include <assert.h> /* assert */
/* use #define to make them constant, required for initialization */
#define PRIME32 2654435761U
#define PRIME64 11400714785074694797ULL
#define SANITY_BUFFER_SIZE (4096 + 64 + 1)
/* TODO : Share these test vector definitions with sanity_check.c and xsum_sanity_check.c */
/*
* Test data vectors
*/
typedef struct {
XSUM_U32 len;
XSUM_U32 seed;
XSUM_U32 Nresult;
} XSUM_testdata32_t;
typedef struct {
XSUM_U32 len;
XSUM_U64 seed;
XSUM_U64 Nresult;
} XSUM_testdata64_t;
typedef struct {
XSUM_U32 len;
XSUM_U64 seed;
XXH128_hash_t Nresult;
} XSUM_testdata128_t;
#define SECRET_SAMPLE_NBBYTES 5
typedef struct {
XSUM_U32 seedLen;
XSUM_U32 secretLen;
XSUM_U8 byte[SECRET_SAMPLE_NBBYTES];
} XSUM_testdata_sample_t;
#ifndef SECRET_SIZE_MAX
# define SECRET_SIZE_MAX 9867
#endif
/* TODO : Share these generators with sanity_check.c and xsum_sanity_check.c */
/* Test vector generators */
static XSUM_testdata32_t tvgen_XXH32(const XSUM_U8* buf, size_t len, XSUM_U32 seed) {
XSUM_testdata32_t v;
v.len = len;
v.seed = seed;
v.Nresult = XXH32(buf, len, seed);
return v;
}
static XSUM_testdata64_t tvgen_XXH64(const XSUM_U8* buf, size_t len, XSUM_U64 seed) {
XSUM_testdata64_t v;
v.len = len;
v.seed = seed;
v.Nresult = XXH64(buf, len, seed);
return v;
}
static XSUM_testdata64_t tvgen_XXH3_64bits_withSeed(const XSUM_U8* buf, size_t len, XSUM_U64 seed) {
XSUM_testdata64_t v;
v.len = len;
v.seed = seed;
v.Nresult = XXH3_64bits_withSeed(buf, len, seed);
return v;
}
static XSUM_testdata64_t tvgen_XXH3_64bits_withSecret(const XSUM_U8* buf, size_t len, const void* secret, size_t secretSize) {
XSUM_testdata64_t v;
v.len = len;
v.seed = 0;
v.Nresult = XXH3_64bits_withSecret(buf, len, secret, secretSize);
return v;
}
static XSUM_testdata128_t tvgen_XXH3_128bits_withSeed(const XSUM_U8* buf, size_t len, XSUM_U64 seed) {
XSUM_testdata128_t v;
v.len = len;
v.seed = seed;
v.Nresult = XXH3_128bits_withSeed(buf, len, seed);
return v;
}
static XSUM_testdata128_t tvgen_XXH3_128bits_withSecret(const XSUM_U8* buf, size_t len, const void* secret, size_t secretSize) {
XSUM_testdata128_t v;
v.len = len;
v.seed = 0;
v.Nresult = XXH3_128bits_withSecret(buf, len, secret, secretSize);
return v;
}
static XSUM_testdata_sample_t tvgen_XXH3_generateSecret(
void* secretBuffer,
size_t secretSize,
const void* customSeed,
size_t customSeedSize
) {
XXH3_generateSecret(secretBuffer, secretSize, customSeed, customSeedSize);
XSUM_testdata_sample_t v;
v.seedLen = customSeedSize;
v.secretLen = secretSize;
/* TODO : Share this array with sanity_check.c and xsum_sanity_check.c */
/* position of sampled bytes */
static const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191, 241 };
for(int i = 0; i < SECRET_SAMPLE_NBBYTES; ++i) {
const XSUM_U8* const secretBufferAsU8 = (const XSUM_U8*) secretBuffer;
v.byte[i] = secretBufferAsU8[sampleIndex[i]];
}
return v;
}
/* Test vector serializers */
static void fprintf_XSUM_testdata32_t(FILE* fp, XSUM_testdata32_t const v) {
fprintf(fp, "{ %4d, 0x%08XU, 0x%08XU },", v.len, v.seed, v.Nresult);
}
static void fprintf_XSUM_testdata64_t(FILE* fp, XSUM_testdata64_t const v) {
fprintf(fp, "{ %4d, 0x%016zXULL, 0x%016zXULL },", v.len, v.seed, v.Nresult);
}
static void fprintf_XSUM_testdata128_t(FILE* fp, XSUM_testdata128_t const v) {
fprintf(fp, "{ %4d, 0x%016zXULL, { 0x%016zXULL, 0x%016zXULL } },",
v.len, v.seed, v.Nresult.low64, v.Nresult.high64);
}
static void fprintf_XSUM_testdata_sample_t(FILE* fp, XSUM_testdata_sample_t const v) {
fprintf(fp,"{ %4d, %4d, { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X } },",
v.seedLen, v.secretLen, v.byte[0], v.byte[1], v.byte[2], v.byte[3], v.byte[4]);
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Fills a test buffer with pseudorandom data.
*
* This is used in the sanity check - its values must not be changed.
*/
static void fillTestBuffer(XSUM_U8* buffer, size_t bufferLenInBytes)
{
XSUM_U64 byteGen = PRIME32;
size_t i;
assert(buffer != NULL);
for (i = 0; i < bufferLenInBytes; ++i) {
buffer[i] = (XSUM_U8)(byteGen>>56);
byteGen *= PRIME64;
}
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Create (malloc) and fill buffer with pseudorandom data for sanity check.
*
* Use releaseSanityBuffer() to delete the buffer.
*/
static XSUM_U8* createSanityBuffer(size_t bufferLenInBytes)
{
XSUM_U8* const buffer = (XSUM_U8*) malloc(bufferLenInBytes);
assert(buffer != NULL);
fillTestBuffer(buffer, bufferLenInBytes);
return buffer;
}
/* TODO : Share this function with sanity_check.c and xsum_sanity_check.c */
/*
* Delete (free) the buffer which has been genereated by createSanityBuffer()
*/
static void releaseSanityBuffer(XSUM_U8* buffer)
{
assert(buffer != NULL);
free(buffer);
}
/* Generate test vectors for XXH32() */
static void generate_sanity_test_vectors_xxh32(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata32_t";
const char* const arrayName = "XSUM_XXH32_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
static const size_t seeds[] = { 0, PRIME32 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata32_t const v = tvgen_XXH32(sanityBuffer, len, seed);
fprintf(fp, " ");
fprintf_XSUM_testdata32_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
}
releaseSanityBuffer(sanityBuffer);
fprintf(fp, "};\n");
}
/* Generate test vectors for XXH64() */
static void generate_sanity_test_vectors_xxh64(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata64_t";
const char* const arrayName = "XSUM_XXH64_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
static const size_t seeds[] = { 0, PRIME32 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata64_t const v = tvgen_XXH64(sanityBuffer, len, seed);
fprintf(fp, " ");
fprintf_XSUM_testdata64_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
}
releaseSanityBuffer(sanityBuffer);
fprintf(fp, "};\n");
}
/* Generate test vectors for XXH3_64bits_withSeed() */
static void generate_sanity_test_vectors_xxh3(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata64_t";
const char* const arrayName = "XSUM_XXH3_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
static const size_t seeds[] = { 0, PRIME64 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata64_t const v = tvgen_XXH3_64bits_withSeed(sanityBuffer, len, seed);
fprintf(fp, " ");
fprintf_XSUM_testdata64_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
}
releaseSanityBuffer(sanityBuffer);
fprintf(fp, "};\n");
}
/* Generate test vectors for XXH3_64bits_withSecret() */
static void generate_sanity_test_vectors_xxh3_withSecret(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata64_t";
const char* const arrayName = "XSUM_XXH3_withSecret_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
const void* const secret = sanityBuffer + 7;
size_t const secretSize = XXH3_SECRET_SIZE_MIN + 11;
assert(maxLen >= 7 + secretSize);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
XSUM_testdata64_t const v = tvgen_XXH3_64bits_withSecret(sanityBuffer, len, secret, secretSize);
fprintf(fp, " ");
fprintf_XSUM_testdata64_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
releaseSanityBuffer(sanityBuffer);
fprintf(fp, "};\n");
}
/* Generate test vectors for XXH3_128bits_withSeed() */
static void generate_sanity_test_vectors_xxh128(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata128_t";
const char* const arrayName = "XSUM_XXH128_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
static const size_t seeds[] = { 0, PRIME32, PRIME64 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
XSUM_U64 const seed = seeds[iSeed];
XSUM_testdata128_t const v = tvgen_XXH3_128bits_withSeed(sanityBuffer, len, seed);
fprintf(fp, " ");
fprintf_XSUM_testdata128_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
}
fprintf(fp, "};\n");
releaseSanityBuffer(sanityBuffer);
}
/* Generate test vectors for XXH3_128bits_withSecret() */
static void generate_sanity_test_vectors_xxh128_withSecret(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata128_t";
const char* const arrayName = "XSUM_XXH128_withSecret_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
const void* const secret = sanityBuffer + 7;
size_t const secretSize = XXH3_SECRET_SIZE_MIN + 11;
assert(maxLen >= 7 + secretSize);
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
XSUM_testdata128_t const v = tvgen_XXH3_128bits_withSecret(sanityBuffer, len, secret, secretSize);
fprintf(fp, " ");
fprintf_XSUM_testdata128_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
fprintf(fp, "};\n");
releaseSanityBuffer(sanityBuffer);
}
/* Generate test vectors for XXH3_generateSecret() */
static void generate_sanity_test_vectors_xxh3_generateSecret(FILE* fp, size_t maxLen) {
const char* const arrayTypeName = "XSUM_testdata_sample_t";
const char* const arrayName = "XSUM_XXH3_generateSecret_testdata";
fprintf(fp, "static const %s %s[] = {\n", arrayTypeName, arrayName);
XSUM_U8* const sanityBuffer = createSanityBuffer(maxLen);
const void* const customSeed = sanityBuffer;
static const size_t seedLens[] = {
0,
1,
XXH3_SECRET_SIZE_MIN - 1,
XXH3_SECRET_DEFAULT_SIZE + 500
};
static const size_t secretLens[] = {
192,
240,
277,
SECRET_SIZE_MAX
};
size_t index = 0;
for(size_t iSeedLen = 0; iSeedLen < sizeof(seedLens)/sizeof(seedLens[0]); ++iSeedLen) {
for(size_t iSecretLen = 0; iSecretLen < sizeof(secretLens)/sizeof(secretLens[0]); ++iSecretLen) {
size_t const seedLen = seedLens[iSeedLen];
size_t const secretLen = secretLens[iSecretLen];
XSUM_U8 secretBuffer[SECRET_SIZE_MAX] = {0};
assert(seedLen <= maxLen);
assert(secretLen <= SECRET_SIZE_MAX);
XSUM_testdata_sample_t const v = tvgen_XXH3_generateSecret(
secretBuffer,
secretLen,
customSeed,
seedLen
);
fprintf(fp, " ");
fprintf_XSUM_testdata_sample_t(fp, v);
fprintf(fp, " /* %s[%zd] */\n", arrayName, index++);
}
}
fprintf(fp, "};\n");
releaseSanityBuffer(sanityBuffer);
}
/* Generate test vectors */
void generate_sanity_test_vectors(size_t maxLen) {
const char* filename = "sanity_test_vectors.h";
fprintf(stderr, "Generating %s\n", filename);
FILE* fp = fopen(filename, "w");
fprintf(fp,
"typedef struct {\n"
" XSUM_U32 len;\n"
" XSUM_U32 seed;\n"
" XSUM_U32 Nresult;\n"
"} XSUM_testdata32_t;\n"
"\n"
"typedef struct {\n"
" XSUM_U32 len;\n"
" XSUM_U64 seed;\n"
" XSUM_U64 Nresult;\n"
"} XSUM_testdata64_t;\n"
"\n"
"typedef struct {\n"
" XSUM_U32 len;\n"
" XSUM_U64 seed;\n"
" XXH128_hash_t Nresult;\n"
"} XSUM_testdata128_t;\n"
"\n"
"#ifndef SECRET_SAMPLE_NBBYTES\n"
"#define SECRET_SAMPLE_NBBYTES 5\n"
"#endif\n"
"\n"
"typedef struct {\n"
" XSUM_U32 seedLen;\n"
" XSUM_U32 secretLen;\n"
" XSUM_U8 byte[SECRET_SAMPLE_NBBYTES];\n"
"} XSUM_testdata_sample_t;\n"
"\n"
"#ifndef SECRET_SIZE_MAX\n"
"#define SECRET_SIZE_MAX 9867\n"
"#endif\n"
"\n"
);
generate_sanity_test_vectors_xxh32(fp, maxLen);
generate_sanity_test_vectors_xxh64(fp, maxLen);
generate_sanity_test_vectors_xxh3(fp, maxLen);
generate_sanity_test_vectors_xxh3_withSecret(fp, maxLen);
generate_sanity_test_vectors_xxh128(fp, maxLen);
generate_sanity_test_vectors_xxh128_withSecret(fp, maxLen);
generate_sanity_test_vectors_xxh3_generateSecret(fp, maxLen);
fclose(fp);
}
/**/
int main(int argc, const char* argv[])
{
(void) argc;
(void) argv;
const size_t sanityBufferSizeInBytes = SANITY_BUFFER_SIZE;
generate_sanity_test_vectors(sanityBufferSizeInBytes);
return EXIT_SUCCESS;
}

View File

@ -1,7 +1,7 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Development source file for `xxh3`
* Copyright (C) 2019-2020 Yann Collet
* Copyright (C) 2019-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*

View File

@ -1,6 +1,6 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2020 Yann Collet
* Copyright (C) 2020-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@ -36,7 +36,7 @@
/*!
* @file xxh_x86dispatch.c
*
* Automatic dispatcher code for the @ref xxh3_family on x86-based targets.
* Automatic dispatcher code for the @ref XXH3_family on x86-based targets.
*
* Optional add-on.
*
@ -56,39 +56,42 @@ extern "C" {
# error "Dispatching is currently only supported on x86 and x86_64."
#endif
/*! @cond Doxygen ignores this part */
#ifndef XXH_HAS_INCLUDE
# ifdef __has_include
# define XXH_HAS_INCLUDE(x) __has_include(x)
# else
# define XXH_HAS_INCLUDE(x) 0
# endif
#endif
/*! @endcond */
/*!
* @def XXH_X86DISPATCH_ALLOW_AVX
* @brief Disables the AVX sanity check.
*
* Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`,
* or `/arch:AVX*`. It is intended to be compiled for the minimum target, and
* xxh_x86dispatch.c is intended to be compiled for the minimum target, and
* it selectively enables SSE2, AVX2, and AVX512 when it is needed.
*
* Using this option _globally_ allows this feature, and therefore makes it
* Compiling with options like `-mavx*`, `-march=native`, or `/arch:AVX*`
* _globally_ will always enable this feature, and therefore makes it
* undefined behavior to execute on any CPU without said feature.
*
* Even if the source code isn't directly using AVX intrinsics in a function,
* the compiler can still generate AVX code from autovectorization and by
* "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128).
*
* Use the same flags that you use to compile the rest of the program; this
* file will safely generate SSE2, AVX2, and AVX512 without these flags.
*
* Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open
* an issue if there is a target in the future where AVX is a default feature.
* Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check,
* thus accepting that the produced binary will not work correctly
* on any CPU with less features than the ones stated at compilation time.
*/
#ifdef XXH_DOXYGEN
# define XXH_X86DISPATCH_ALLOW_AVX
#endif
#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX)
# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above."
#endif
#ifdef __has_include
# define XXH_HAS_INCLUDE(header) __has_include(header)
#else
# define XXH_HAS_INCLUDE(header) 0
# error "Error: if xxh_x86dispatch.c is compiled with AVX enabled, the resulting binary will crash on sse2-only cpus !! " \
"If you nonetheless want to do that, please enable the XXH_X86DISPATCH_ALLOW_AVX build variable"
#endif
/*!
@ -107,7 +110,7 @@ extern "C" {
#ifndef XXH_DISPATCH_SCALAR
# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \
|| defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \
|| defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */
|| defined(__ANDROID__) || defined(__APPLE__) /* Android or macOS */
# define XXH_DISPATCH_SCALAR 0 /* disable */
# else
# define XXH_DISPATCH_SCALAR 1
@ -174,6 +177,7 @@ extern "C" {
*
* @def XXH_TARGET_AVX512
* @brief Like @ref XXH_TARGET_SSE2, but for AVX512.
*
*/
#if defined(__GNUC__)
# include <emmintrin.h> /* SSE2 */
@ -183,6 +187,18 @@ extern "C" {
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
#elif defined(__clang__) && defined(_MSC_VER) /* clang-cl.exe */
# include <emmintrin.h> /* SSE2 */
# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
# include <immintrin.h> /* AVX2, AVX512F */
# include <smmintrin.h>
# include <avxintrin.h>
# include <avx2intrin.h>
# include <avx512fintrin.h>
# endif
# define XXH_TARGET_SSE2 __attribute__((__target__("sse2")))
# define XXH_TARGET_AVX2 __attribute__((__target__("avx2")))
# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f")))
#elif defined(_MSC_VER)
# include <intrin.h>
# define XXH_TARGET_SSE2
@ -192,6 +208,7 @@ extern "C" {
# error "Dispatching is currently not supported for your compiler."
#endif
/*! @cond Doxygen ignores this part */
#ifdef XXH_DISPATCH_DEBUG
/* debug logging */
# include <stdio.h>
@ -201,12 +218,37 @@ extern "C" {
# undef NDEBUG /* avoid redefinition */
# define NDEBUG
#endif
/*! @endcond */
#include <assert.h>
#ifndef XXH_DOXYGEN
#define XXH_INLINE_ALL
#define XXH_X86DISPATCH
#include "xxhash.h"
#endif
/*! @cond Doxygen ignores this part */
#ifndef XXH_HAS_ATTRIBUTE
# ifdef __has_attribute
# define XXH_HAS_ATTRIBUTE(...) __has_attribute(__VA_ARGS__)
# else
# define XXH_HAS_ATTRIBUTE(...) 0
# endif
#endif
/*! @endcond */
/*! @cond Doxygen ignores this part */
#if XXH_HAS_ATTRIBUTE(constructor)
# define XXH_CONSTRUCTOR __attribute__((constructor))
# define XXH_DISPATCH_MAYBE_NULL 0
#else
# define XXH_CONSTRUCTOR
# define XXH_DISPATCH_MAYBE_NULL 1
#endif
/*! @endcond */
/*! @cond Doxygen ignores this part */
/*
* Support both AT&T and Intel dialects
*
@ -224,9 +266,10 @@ extern "C" {
#else
# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t"
#endif
/*! @endcond */
/*!
* @internal
* @private
* @brief Runs CPUID.
*
* @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively.
@ -235,7 +278,7 @@ extern "C" {
static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
{
#if defined(_MSC_VER)
__cpuidex(abcd, eax, ecx);
__cpuidex((int*)abcd, eax, ecx);
#else
xxh_u32 ebx, edx;
# if defined(__i386__) && defined(__PIC__)
@ -269,7 +312,7 @@ static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd)
#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512
/*!
* @internal
* @private
* @brief Runs `XGETBV`.
*
* While the CPU may support AVX2, the operating system might not properly save
@ -303,20 +346,22 @@ static xxh_u64 XXH_xgetbv(void)
}
#endif
/*! @cond Doxygen ignores this part */
#define XXH_SSE2_CPUID_MASK (1 << 26)
#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27))
#define XXH_AVX2_CPUID_MASK (1 << 5)
#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1))
#define XXH_AVX512F_CPUID_MASK (1 << 16)
#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1))
/*! @endcond */
/*!
* @internal
* @private
* @brief Returns the best XXH3 implementation.
*
* Runs various CPUID/XGETBV tests to try and determine the best implementation.
*
* @ret The best @ref XXH_VECTOR implementation.
* @return The best @ref XXH_VECTOR implementation.
* @see XXH_VECTOR_TYPES
*/
static int XXH_featureTest(void)
@ -441,8 +486,9 @@ static int XXH_featureTest(void)
/* === Vector implementations === */
/*! @cond PRIVATE */
/*!
* @internal
* @private
* @brief Defines the various dispatch functions.
*
* TODO: Consolidate?
@ -450,27 +496,29 @@ static int XXH_featureTest(void)
* @param suffix The suffix for the functions, e.g. sse2 or scalar
* @param target XXH_TARGET_* or empty.
*/
#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \
\
/* === XXH3, default variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
XXHL64_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len) \
{ \
return XXH3_hashLong_64b_internal( \
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH3, Seeded variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
XXHL64_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len, \
XXH64_hash_t seed) \
{ \
return XXH3_hashLong_64b_withSeed_internal( \
input, len, seed, XXH3_accumulate_512_##suffix, \
input, len, seed, XXH3_accumulate_##suffix, \
XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \
); \
} \
@ -478,59 +526,66 @@ XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
/* === XXH3, Secret variants === */ \
\
XXH_NO_INLINE target XXH64_hash_t \
XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
const void* secret, size_t secretLen) \
XXHL64_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len, XXH_NOESCAPE const void* secret, \
size_t secretLen) \
{ \
return XXH3_hashLong_64b_internal( \
input, len, secret, secretLen, \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH3 update variants === */ \
\
XXH_NO_INLINE target XXH_errorcode \
XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \
XXH3_update_##suffix(XXH_NOESCAPE XXH3_state_t* state, \
XXH_NOESCAPE const void* input, size_t len) \
{ \
return XXH3_update(state, (const xxh_u8*)input, len, \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
} \
\
/* === XXH128 default variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \
XXHL128_default_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len) \
{ \
return XXH3_hashLong_128b_internal( \
input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix \
); \
} \
\
/* === XXH128 Secret variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \
const void* XXH_RESTRICT secret, size_t secretLen) \
XXHL128_secret_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, \
size_t len, \
XXH_NOESCAPE const void* XXH_RESTRICT secret, \
size_t secretLen) \
{ \
return XXH3_hashLong_128b_internal( \
input, len, (const xxh_u8*)secret, secretLen, \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix); \
} \
\
/* === XXH128 Seeded variants === */ \
\
XXH_NO_INLINE target XXH128_hash_t \
XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \
XXHL128_seed_##suffix(XXH_NOESCAPE const void* XXH_RESTRICT input, size_t len,\
XXH64_hash_t seed) \
{ \
return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \
XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \
XXH3_accumulate_##suffix, XXH3_scrambleAcc_##suffix, \
XXH3_initCustomSecret_##suffix); \
}
/*! @endcond */
/* End XXH_DEFINE_DISPATCH_FUNCS */
/*! @cond Doxygen ignores this part */
#if XXH_DISPATCH_SCALAR
XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */)
#endif
@ -542,16 +597,18 @@ XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2)
XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512)
#endif
#undef XXH_DEFINE_DISPATCH_FUNCS
/*! @endcond */
/* ==== Dispatchers ==== */
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t);
/*! @cond Doxygen ignores this part */
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t);
typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH_NOESCAPE XXH3_state_t*, XXH_NOESCAPE const void*, size_t);
typedef struct {
XXH3_dispatchx86_hashLong64_default hashLong64_default;
@ -561,9 +618,10 @@ typedef struct {
} XXH_dispatchFunctions_s;
#define XXH_NB_DISPATCHES 4
/*! @endcond */
/*!
* @internal
* @private
* @brief Table of dispatchers for @ref XXH3_64bits().
*
* @pre The indices must match @ref XXH_VECTOR_TYPE.
@ -587,17 +645,18 @@ static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = {
#endif
};
/*!
* @internal
* @private
* @brief The selected dispatch table for @ref XXH3_64bits().
*/
static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL };
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t);
/*! @cond Doxygen ignores this part */
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, XXH64_hash_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(XXH_NOESCAPE const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t);
typedef struct {
XXH3_dispatchx86_hashLong128_default hashLong128_default;
@ -605,10 +664,11 @@ typedef struct {
XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret;
XXH3_dispatchx86_update update;
} XXH_dispatch128Functions_s;
/*! @endcond */
/*!
* @internal
* @private
* @brief Table of dispatchers for @ref XXH3_128bits().
*
* @pre The indices must match @ref XXH_VECTOR_TYPE.
@ -633,16 +693,16 @@ static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = {
};
/*!
* @internal
* @private
* @brief The selected dispatch table for @ref XXH3_64bits().
*/
static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL };
/*!
* @internal
* @private
* @brief Runs a CPUID check and sets the correct dispatch tables.
*/
static void XXH_setDispatch(void)
static XXH_CONSTRUCTOR void XXH_setDispatch(void)
{
int vecID = XXH_featureTest();
XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1);
@ -662,17 +722,19 @@ static void XXH_setDispatch(void)
/* ==== XXH3 public functions ==== */
/*! @cond Doxygen ignores this part */
static XXH64_hash_t
XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
{
(void)seed64; (void)secret; (void)secretLen;
if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_default == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_default(input, len);
}
XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len)
XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
{
return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection);
}
@ -682,11 +744,12 @@ XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len,
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
{
(void)secret; (void)secretLen;
if (XXH_g_dispatch.hashLong64_seed == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_seed == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_seed(input, len, seed64);
}
XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection);
}
@ -696,49 +759,57 @@ XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
{
(void)seed64;
if (XXH_g_dispatch.hashLong64_secret == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_secret == NULL)
XXH_setDispatch();
return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen);
}
XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
{
return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection);
}
XXH_errorcode
XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
if (XXH_g_dispatch.update == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.update == NULL)
XXH_setDispatch();
return XXH_g_dispatch.update(state, (const xxh_u8*)input, len);
}
/*! @endcond */
/* ==== XXH128 public functions ==== */
/*! @cond Doxygen ignores this part */
static XXH128_hash_t
XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)seed64; (void)secret; (void)secretLen;
if (XXH_g_dispatch128.hashLong128_default == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_default == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_default(input, len);
}
XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len)
XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len)
{
return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection);
}
static XXH128_hash_t
XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)secret; (void)secretLen;
if (XXH_g_dispatch128.hashLong128_seed == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_seed == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_seed(input, len, seed64);
}
XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed)
XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
{
return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection);
}
@ -748,22 +819,26 @@ XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len,
XXH64_hash_t seed64, const void* secret, size_t secretLen)
{
(void)seed64;
if (XXH_g_dispatch128.hashLong128_secret == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.hashLong128_secret == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen);
}
XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen)
XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen)
{
return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection);
}
XXH_errorcode
XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len)
XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
{
if (XXH_g_dispatch128.update == NULL) XXH_setDispatch();
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch128.update == NULL)
XXH_setDispatch();
return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len);
}
/*! @endcond */
#if defined (__cplusplus)
}
#endif

View File

@ -1,6 +1,6 @@
/*
* xxHash - XXH3 Dispatcher for x86-based targets
* Copyright (C) 2020 Yann Collet
* Copyright (C) 2020-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@ -41,15 +41,15 @@
extern "C" {
#endif
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
#if defined (__cplusplus)
}
@ -71,7 +71,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, c
# undef XXH128
# define XXH128 XXH3_128bits_withSeed_dispatch
# define XXH3_128bits XXH3_128bits_dispatch
# undef XXH3_128bits
# define XXH3_128bits XXH3_128bits_dispatch
# undef XXH3_128bits_withSeed

View File

@ -1,6 +1,6 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Copyright (C) 2012-2020 Yann Collet
* Copyright (C) 2012-2021 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*

File diff suppressed because it is too large Load Diff