#!/usr/bin/env bash set -euo pipefail export TB_VERSION_WARNING=0 # Get the expected count once, outside of any function ndjson_file="./datasources/fixtures/analytics_events.ndjson" export expected_count=$(wc -l < "$ndjson_file" || echo "0") check_sum() { local file=$1 local expected_count=$2 # Only perform the check if the file starts with "all_" if [[ ! $(basename "$file") =~ ^all_ ]]; then return 0 fi local sum=0 local column_name="" # Determine if the file has a 'pageviews' column if head -n1 "$file" | grep -q 'pageviews'; then column_name="pageviews" else echo "No 'pageviews' column found in $file" return 0 # No relevant column found, skip the check fi # Get the column number local column_num=$(head -n1 "$file" | tr ',' '\n' | grep -n "$column_name" | cut -d: -f1) # Sum the values in the column sum=$(tail -n +2 "$file" | cut -d',' -f"$column_num" | awk '{s+=$1} END {print s}') # Check if the sum equals the number of lines in the NDJSON file if [ "$sum" -eq "$expected_count" ]; then echo "✅ Sanity check passed: Sum of $column_name is $sum (matches NDJSON line count)" return 0 else echo "⚠️ WARNING: Sanity check failed: Sum of $column_name is $sum, expected $expected_count (NDJSON line count)" return 1 # Return 1 to indicate a warning, but not a failure fi } run_test() { t=$1 echo "** Running $t **" echo "** $(cat $t)" tmpfile=$(mktemp) retries=0 TOTAL_RETRIES=3 # When appending fixtures, we need to retry in case of the data is not replicated in time while [ $retries -lt $TOTAL_RETRIES ]; do # Run the test and store the output in a temporary file bash "$t" >"$tmpfile" exit_code=$? if [ "$exit_code" -eq 0 ]; then # If the test passed, break the loop if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then break # If the test failed, increment the retries counter and try again else retries=$((retries + 1)) fi # If the bash command failed, print an error message and break the loop else break fi done if diff -B "${t}.result" "$tmpfile" >/dev/null 2>&1; then echo "✅ Test $t passed" check_sum "${t}.result" "$expected_count" || echo "⚠️ Warning: Sanity check did not pass." rm "$tmpfile" return 0 elif [ $retries -eq $TOTAL_RETRIES ]; then echo "🚨 ERROR: Test $t failed, showing differences:" diff -B -u --color -U3 "${t}.result" "$tmpfile" # Use unified diff format with 3 lines of context rm "$tmpfile" return 1 else echo "🚨 ERROR: Test $t failed with bash command exit code $?" cat "$tmpfile" rm "$tmpfile" return 1 fi echo "" } export -f run_test export -f check_sum fail=0 # Check if a test name was provided as an argument if [ $# -eq 1 ]; then test_name=$1 # Find the test file that matches the provided name test_file=$(find ./tests -name "${test_name}*.test") if [ -n "$test_file" ]; then run_test "$test_file" || fail=1 else echo "🚨 ERROR: No test found matching name: $test_name" fail=1 fi else # If no test name provided, run all tests find ./tests -name "*.test" -print0 | xargs -0 -I {} bash -c 'run_test "$@"' _ {} || fail=1 fi if [ $fail == 1 ]; then echo "🚨 ERROR: Some tests failed" exit 1 fi