2022-10-28 21:52:39 +03:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
|
|
|
|
2022-10-31 23:28:21 +03:00
|
|
|
# TODO: use --fail-with-body to get the response body on failure
|
2023-02-20 18:44:22 +03:00
|
|
|
CURL=${CURL:-curl --silent --show-error --fail --compressed}
|
2022-10-28 21:52:39 +03:00
|
|
|
DATABASE_URL="${DATABASE_URL:-postgres://postgres@localhost/db}"
|
2023-07-04 00:29:44 +03:00
|
|
|
MARTIN_BUILD="${MARTIN_BUILD:-cargo build --features ssl}"
|
2023-01-01 08:03:21 +03:00
|
|
|
MARTIN_PORT="${MARTIN_PORT:-3111}"
|
2022-12-19 05:24:06 +03:00
|
|
|
MARTIN_URL="http://localhost:${MARTIN_PORT}"
|
2023-01-01 08:03:21 +03:00
|
|
|
MARTIN_ARGS="${MARTIN_ARGS:---listen-addresses localhost:${MARTIN_PORT}}"
|
2023-07-04 00:29:44 +03:00
|
|
|
MARTIN_BIN="${MARTIN_BIN:-cargo run --features ssl --} ${MARTIN_ARGS}"
|
2022-10-28 21:52:39 +03:00
|
|
|
|
2023-06-04 01:54:50 +03:00
|
|
|
MBTILES_BUILD="${MBTILES_BUILD:-cargo build -p martin-mbtiles}"
|
|
|
|
MBTILES_BIN="${MBTILES_BIN:-target/debug/mbtiles}"
|
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
function wait_for_martin {
|
|
|
|
# Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
|
2022-12-19 05:24:06 +03:00
|
|
|
# timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS "$MARTIN_URL/health"
|
2022-11-02 21:00:05 +03:00
|
|
|
PROCESS_ID=$1
|
2022-12-27 09:56:27 +03:00
|
|
|
echo "Waiting for Martin ($PROCESS_ID) to start by checking $MARTIN_URL/health to be valid..."
|
|
|
|
for i in {1..60}; do
|
2023-02-20 18:44:22 +03:00
|
|
|
if $CURL "$MARTIN_URL/health" 2>/dev/null >/dev/null; then
|
2022-10-31 23:28:21 +03:00
|
|
|
echo "Martin is up!"
|
2023-02-20 18:44:22 +03:00
|
|
|
$CURL "$MARTIN_URL/health"
|
2022-10-31 23:28:21 +03:00
|
|
|
return
|
|
|
|
fi
|
2022-11-02 21:00:05 +03:00
|
|
|
if ps -p $PROCESS_ID > /dev/null ; then
|
2022-12-27 09:56:27 +03:00
|
|
|
echo "Martin is not up yet, waiting for $MARTIN_URL/health ..."
|
2022-11-02 21:00:05 +03:00
|
|
|
sleep 1
|
|
|
|
else
|
|
|
|
echo "Martin died!"
|
|
|
|
ps au
|
2022-12-12 17:11:10 +03:00
|
|
|
lsof -i || true
|
2022-11-02 21:00:05 +03:00
|
|
|
exit 1
|
|
|
|
fi
|
2022-10-28 21:52:39 +03:00
|
|
|
done
|
2022-10-31 23:28:21 +03:00
|
|
|
echo "Martin did not start in time"
|
|
|
|
ps au
|
2022-12-12 17:11:10 +03:00
|
|
|
lsof -i || true
|
2022-10-31 23:28:21 +03:00
|
|
|
exit 1
|
|
|
|
}
|
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
function kill_process {
|
|
|
|
PROCESS_ID=$1
|
|
|
|
echo "Waiting for Martin ($PROCESS_ID) to stop..."
|
|
|
|
kill $PROCESS_ID
|
|
|
|
for i in {1..50}; do
|
|
|
|
if ps -p $PROCESS_ID > /dev/null ; then
|
|
|
|
sleep 0.1
|
|
|
|
else
|
|
|
|
echo "Martin ($PROCESS_ID) has stopped"
|
|
|
|
return
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
echo "Martin did not stop in time, killing it"
|
|
|
|
kill -9 $PROCESS_ID
|
|
|
|
# wait for it to die using timeout and wait
|
|
|
|
timeout -k 1s 1s wait $PROCESS_ID || true
|
|
|
|
}
|
|
|
|
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn()
|
|
|
|
{
|
|
|
|
FILENAME="$TEST_OUT_DIR/$1.json"
|
|
|
|
URL="$MARTIN_URL/$2"
|
|
|
|
|
|
|
|
echo "Testing $(basename "$FILENAME") from $URL"
|
2023-02-22 19:25:48 +03:00
|
|
|
$CURL "$URL" | jq -e > "$FILENAME"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
}
|
|
|
|
|
2022-10-31 23:28:21 +03:00
|
|
|
test_pbf()
|
|
|
|
{
|
|
|
|
FILENAME="$TEST_OUT_DIR/$1.pbf"
|
2022-12-19 05:24:06 +03:00
|
|
|
URL="$MARTIN_URL/$2"
|
2022-10-31 23:28:21 +03:00
|
|
|
|
|
|
|
echo "Testing $(basename "$FILENAME") from $URL"
|
2023-02-20 18:44:22 +03:00
|
|
|
$CURL "$URL" > "$FILENAME"
|
2022-10-31 23:28:21 +03:00
|
|
|
|
|
|
|
if [[ $OSTYPE == linux* ]]; then
|
2023-01-03 19:09:41 +03:00
|
|
|
./tests/fixtures/vtzero-check "$FILENAME"
|
|
|
|
./tests/fixtures/vtzero-show "$FILENAME" > "$FILENAME.txt"
|
2022-10-31 23:28:21 +03:00
|
|
|
fi
|
2022-10-28 21:52:39 +03:00
|
|
|
}
|
|
|
|
|
2023-01-08 22:19:11 +03:00
|
|
|
test_png()
|
|
|
|
{
|
|
|
|
FILENAME="$TEST_OUT_DIR/$1.png"
|
|
|
|
URL="$MARTIN_URL/$2"
|
|
|
|
|
|
|
|
echo "Testing $(basename "$FILENAME") from $URL"
|
|
|
|
$CURL "$URL" > "$FILENAME"
|
|
|
|
|
|
|
|
if [[ $OSTYPE == linux* ]]; then
|
|
|
|
file "$FILENAME" > "$FILENAME.txt"
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
2023-02-07 09:05:47 +03:00
|
|
|
# Delete a line from a file $1 that matches parameter $2
|
|
|
|
remove_line()
|
2023-01-01 08:03:21 +03:00
|
|
|
{
|
2023-02-07 09:05:47 +03:00
|
|
|
FILE="$1"
|
|
|
|
LINE_TO_REMOVE="$2"
|
|
|
|
>&2 echo "Removing line '$LINE_TO_REMOVE' from $FILE"
|
|
|
|
grep -v "$LINE_TO_REMOVE" "${FILE}" > "${FILE}.tmp"
|
|
|
|
mv "${FILE}.tmp" "${FILE}"
|
|
|
|
}
|
|
|
|
|
|
|
|
test_log_has_str()
|
|
|
|
{
|
|
|
|
LOG_FILE="$1"
|
|
|
|
EXPECTED_TEXT="$2"
|
|
|
|
echo "Checking $LOG_FILE for expected text: '$EXPECTED_TEXT'"
|
|
|
|
grep -q "$EXPECTED_TEXT" "$LOG_FILE"
|
|
|
|
remove_line "$LOG_FILE" "$EXPECTED_TEXT"
|
|
|
|
}
|
|
|
|
|
|
|
|
validate_log()
|
|
|
|
{
|
|
|
|
LOG_FILE="$1"
|
|
|
|
>&2 echo "Validating log file $LOG_FILE"
|
|
|
|
|
|
|
|
# Older versions of PostGIS don't support the margin parameter, so we need to remove it from the log
|
|
|
|
remove_line "$LOG_FILE" 'Margin parameter in ST_TileEnvelope is not supported'
|
2023-08-26 16:53:06 +03:00
|
|
|
remove_line "$LOG_FILE" 'Source IDs must be unique'
|
2023-02-07 09:05:47 +03:00
|
|
|
|
|
|
|
# Make sure the log has just the expected warnings, remove them, and test that there are no other ones
|
|
|
|
test_log_has_str "$LOG_FILE" 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
|
|
|
|
|
|
|
|
echo "Checking for no other warnings or errors in the log"
|
|
|
|
if grep -e ' ERROR ' -e ' WARN ' "$LOG_FILE"; then
|
|
|
|
echo "Log file $LOG_FILE has unexpected warnings or errors"
|
|
|
|
exit 1
|
|
|
|
fi
|
2023-01-01 08:03:21 +03:00
|
|
|
}
|
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
curl --version
|
|
|
|
|
2023-06-04 01:54:50 +03:00
|
|
|
# Make sure martin and mbtiles are built - this way it won't timeout while waiting for it to start
|
|
|
|
# If set to "-", don't build
|
2022-10-31 23:28:21 +03:00
|
|
|
if [[ "$MARTIN_BUILD" != "-" ]]; then
|
|
|
|
$MARTIN_BUILD
|
|
|
|
fi
|
2023-06-04 01:54:50 +03:00
|
|
|
if [[ "$MBTILES_BUILD" != "-" ]]; then
|
|
|
|
$MBTILES_BUILD
|
|
|
|
fi
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
|
|
|
|
echo "------------------------------------------------------------------------------------------------------------------------"
|
|
|
|
echo "Test auto configured Martin"
|
2022-12-12 17:11:10 +03:00
|
|
|
|
2022-12-27 09:56:27 +03:00
|
|
|
TEST_OUT_DIR="$(dirname "$0")/output/auto"
|
|
|
|
mkdir -p "$TEST_OUT_DIR"
|
|
|
|
|
2023-02-20 18:44:22 +03:00
|
|
|
ARG=(--default-srid 900913 --disable-bounds --save-config "$(dirname "$0")/output/generated_config.yaml" tests/fixtures/files)
|
2022-12-27 09:56:27 +03:00
|
|
|
set -x
|
2022-12-12 17:11:10 +03:00
|
|
|
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_1.txt &
|
|
|
|
PROCESS_ID=`jobs -p`
|
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
{ set +x; } 2> /dev/null
|
|
|
|
trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
|
|
|
|
wait_for_martin $PROCESS_ID
|
2022-10-31 23:28:21 +03:00
|
|
|
|
|
|
|
>&2 echo "Test catalog"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn catalog_auto catalog
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-12-27 09:56:27 +03:00
|
|
|
>&2 echo "***** Test server response for table source *****"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn table_source table_source
|
2022-12-19 05:24:06 +03:00
|
|
|
test_pbf tbl_0_0_0 table_source/0/0/0
|
2022-12-27 09:56:27 +03:00
|
|
|
test_pbf tbl_6_57_29 table_source/6/57/29
|
|
|
|
test_pbf tbl_12_3673_1911 table_source/12/3673/1911
|
|
|
|
test_pbf tbl_13_7346_3822 table_source/13/7346/3822
|
|
|
|
test_pbf tbl_14_14692_7645 table_source/14/14692/7645
|
|
|
|
test_pbf tbl_17_117542_61161 table_source/17/117542/61161
|
|
|
|
test_pbf tbl_18_235085_122323 table_source/18/235085/122323
|
|
|
|
|
|
|
|
>&2 echo "***** Test server response for composite source *****"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn cmp table_source,points1,points2
|
2022-12-19 05:24:06 +03:00
|
|
|
test_pbf cmp_0_0_0 table_source,points1,points2/0/0/0
|
2022-12-27 09:56:27 +03:00
|
|
|
test_pbf cmp_6_57_29 table_source,points1,points2/6/57/29
|
|
|
|
test_pbf cmp_12_3673_1911 table_source,points1,points2/12/3673/1911
|
|
|
|
test_pbf cmp_13_7346_3822 table_source,points1,points2/13/7346/3822
|
|
|
|
test_pbf cmp_14_14692_7645 table_source,points1,points2/14/14692/7645
|
|
|
|
test_pbf cmp_17_117542_61161 table_source,points1,points2/17/117542/61161
|
|
|
|
test_pbf cmp_18_235085_122323 table_source,points1,points2/18/235085/122323
|
|
|
|
|
|
|
|
>&2 echo "***** Test server response for function source *****"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn fnc function_zxy_query
|
2022-12-19 05:24:06 +03:00
|
|
|
test_pbf fnc_0_0_0 function_zxy_query/0/0/0
|
2022-12-27 09:56:27 +03:00
|
|
|
test_pbf fnc_6_57_29 function_zxy_query/6/57/29
|
|
|
|
test_pbf fnc_12_3673_1911 function_zxy_query/12/3673/1911
|
|
|
|
test_pbf fnc_13_7346_3822 function_zxy_query/13/7346/3822
|
|
|
|
test_pbf fnc_14_14692_7645 function_zxy_query/14/14692/7645
|
|
|
|
test_pbf fnc_17_117542_61161 function_zxy_query/17/117542/61161
|
|
|
|
test_pbf fnc_18_235085_122323 function_zxy_query/18/235085/122323
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
|
|
|
|
test_jsn fnc_token function_zxy_query_test
|
|
|
|
test_pbf fnc_token_0_0_0 function_zxy_query_test/0/0/0?token=martin
|
|
|
|
|
|
|
|
test_jsn fnc_b function_zxy_query_jsonb
|
2022-12-27 09:56:27 +03:00
|
|
|
test_pbf fnc_b_6_38_20 function_zxy_query_jsonb/6/57/29
|
|
|
|
|
|
|
|
>&2 echo "***** Test server response for different function call types *****"
|
|
|
|
test_pbf fnc_zoom_xy_6_57_29 function_zoom_xy/6/57/29
|
|
|
|
test_pbf fnc_zxy_6_57_29 function_zxy/6/57/29
|
|
|
|
test_pbf fnc_zxy2_6_57_29 function_zxy2/6/57/29
|
|
|
|
test_pbf fnc_zxy_query_6_57_29 function_zxy_query/6/57/29
|
|
|
|
test_pbf fnc_zxy_row_6_57_29 function_zxy_row/6/57/29
|
|
|
|
test_pbf fnc_zxy_row2_6_57_29 function_Mixed_Name/6/57/29
|
|
|
|
test_pbf fnc_zxy_row_key_6_57_29 function_zxy_row_key/6/57/29
|
|
|
|
|
|
|
|
>&2 echo "***** Test server response for table source with different SRID *****"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn points3857_srid points3857
|
2022-12-19 05:24:06 +03:00
|
|
|
test_pbf points3857_srid_0_0_0 points3857/0/0/0
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2023-01-08 22:19:11 +03:00
|
|
|
>&2 echo "***** Test server response for PMTiles source *****"
|
|
|
|
test_jsn pmt stamen_toner__raster_CC-BY-ODbL_z3
|
|
|
|
test_png pmt_3_4_2 stamen_toner__raster_CC-BY-ODbL_z3/3/4/2
|
|
|
|
|
Add .mbtiles support (#549)
Adds a new [.mbtiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md)
backend, without the grid support. Uses extensive tile content
detection, i.e. if the content is gzipped, png, jpeg, gif, webp.
From CLI, can be as easy as adding a path to a directory that contains a
.mbtiles file (works just like pmtiles support)
```bash
# All *.mbtiles files in this dir will be published.
# The filename will be used as the source ID
martin ./tests/fixtures
```
From configuration file, the path can be specified in a number of ways
(same as pmtiles)
```yaml
mbtiles:
paths:
# scan this whole dir, matching all *.mbtiles files
- /dir-path
# specific mbtiles file will be published as mbtiles2 source
- /path/to/mbtiles2.mbtiles
sources:
# named source matching source name to a single file
pm-src1: /tmp/mbtiles.mbtiles
# named source, where the filename is explicitly set. This way we will be able to add more options later
pm-src2:
path: /tmp/mbtiles.mbtiles
```
Fixes #494
2023-01-09 08:10:23 +03:00
|
|
|
>&2 echo "***** Test server response for MbTiles source *****"
|
|
|
|
test_jsn mb_jpg geography-class-jpg
|
|
|
|
test_png mb_jpg_0_0_0 geography-class-jpg/0/0/0
|
|
|
|
test_jsn mb_png geography-class-png
|
|
|
|
test_png mb_png_0_0_0 geography-class-png/0/0/0
|
|
|
|
test_jsn mb_mvt world_cities
|
|
|
|
test_pbf mb_mvt_2_3_1 world_cities/2/3/1
|
|
|
|
|
2022-12-27 09:56:27 +03:00
|
|
|
>&2 echo "***** Test server response for table source with empty SRID *****"
|
Add .mbtiles support (#549)
Adds a new [.mbtiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md)
backend, without the grid support. Uses extensive tile content
detection, i.e. if the content is gzipped, png, jpeg, gif, webp.
From CLI, can be as easy as adding a path to a directory that contains a
.mbtiles file (works just like pmtiles support)
```bash
# All *.mbtiles files in this dir will be published.
# The filename will be used as the source ID
martin ./tests/fixtures
```
From configuration file, the path can be specified in a number of ways
(same as pmtiles)
```yaml
mbtiles:
paths:
# scan this whole dir, matching all *.mbtiles files
- /dir-path
# specific mbtiles file will be published as mbtiles2 source
- /path/to/mbtiles2.mbtiles
sources:
# named source matching source name to a single file
pm-src1: /tmp/mbtiles.mbtiles
# named source, where the filename is explicitly set. This way we will be able to add more options later
pm-src2:
path: /tmp/mbtiles.mbtiles
```
Fixes #494
2023-01-09 08:10:23 +03:00
|
|
|
test_pbf points_empty_srid_0_0_0 points_empty_srid/0/0/0
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
kill_process $PROCESS_ID
|
2023-02-07 09:05:47 +03:00
|
|
|
validate_log test_log_1.txt
|
2022-10-28 21:52:39 +03:00
|
|
|
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
echo "------------------------------------------------------------------------------------------------------------------------"
|
|
|
|
echo "Test pre-configured Martin"
|
2022-12-27 09:56:27 +03:00
|
|
|
TEST_OUT_DIR="$(dirname "$0")/output/configured"
|
|
|
|
mkdir -p "$TEST_OUT_DIR"
|
2022-12-12 17:11:10 +03:00
|
|
|
|
2023-02-17 19:11:16 +03:00
|
|
|
ARG=(--config tests/config.yaml --max-feature-count 1000 --save-config "$(dirname "$0")/output/given_config.yaml" -W 1)
|
2022-12-27 09:56:27 +03:00
|
|
|
set -x
|
2022-12-12 17:11:10 +03:00
|
|
|
$MARTIN_BIN "${ARG[@]}" 2>&1 | tee test_log_2.txt &
|
|
|
|
PROCESS_ID=`jobs -p`
|
2022-11-02 21:00:05 +03:00
|
|
|
{ set +x; } 2> /dev/null
|
|
|
|
trap "kill -9 $PROCESS_ID 2> /dev/null || true" EXIT
|
|
|
|
wait_for_martin $PROCESS_ID
|
2022-10-31 23:28:21 +03:00
|
|
|
|
|
|
|
>&2 echo "Test catalog"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
test_jsn catalog_cfg catalog
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-12-19 05:24:06 +03:00
|
|
|
test_pbf tbl_0_0_0 table_source/0/0/0
|
|
|
|
test_pbf cmp_0_0_0 points1,points2/0/0/0
|
|
|
|
test_pbf fnc_0_0_0 function_zxy_query/0/0/0
|
|
|
|
test_pbf fnc2_0_0_0 function_zxy_query_test/0/0/0?token=martin
|
2023-01-08 22:19:11 +03:00
|
|
|
test_png pmt_0_0_0 pmt/0/0/0
|
2022-10-31 23:28:21 +03:00
|
|
|
|
Add dynamic sprites support (#715)
Dynamically create image sprites for MapLibre rendering, given a
directory with images.
### TODO
* [x] Work with @flother to merge these PRs
* [x] https://github.com/flother/spreet/pull/59 (must have)
* [x] https://github.com/flother/spreet/pull/57
* [x] https://github.com/flother/spreet/pull/56
* [ ] https://github.com/flother/spreet/pull/62 (not required but nice
to have, can upgrade later without any code changes)
* [x] Add docs to the book
* [x] Add CLI param, e.g. `--sprite <dir_path>`
* [x] Don't output `.sprites` in auto-genned config when not in use
### API
Per [MapLibre sprites
API](https://maplibre.org/maplibre-style-spec/sprite/), we need to
support the following:
* `/sprite/<sprite_id>.json` metadata about the sprite file - all coming
from a single directory
* `/sprite/<sprite_id>.png` all images combined into a single PNG
* `/sprite/<sprite_id>@2x.json` same but for high DPI devices
* `/sprite/<sprite_id>@2x.png`
Multiple sprite_id values can be combined into one sprite with the same
pattern as for tile joining:
`/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>[.json|.png|@2x.json|@2x.png]`.
No ID renaming is done, so identical names will override one another.
### Configuration
[Config file](https://maplibre.org/martin/config-file.html) and possibly
CLI should have a simple option to serve sprites. The configuration may
look similar to how mbtiles and pmtiles are configured:
```yaml
# Publish sprite images
sprites:
paths:
# scan this whole dir, matching all image files, and publishing it as "my_images" sprite source
- /path/to/my_images
sources:
# named source matching source name to a directory
my_sprites: /path/to/some_dir
```
Implement #705
2023-06-16 15:19:47 +03:00
|
|
|
test_jsn spr_src1 sprite/src1.json
|
|
|
|
test_png spr_src1 sprite/src1.png
|
|
|
|
test_jsn spr_src1_2x sprite/src1@2x.json
|
|
|
|
test_png spr_src1_2x sprite/src1@2x.png
|
|
|
|
test_jsn spr_mysrc sprite/mysrc.json
|
|
|
|
test_png spr_mysrc sprite/mysrc.png
|
|
|
|
test_jsn spr_mysrc_2x sprite/mysrc@2x.json
|
|
|
|
test_png spr_mysrc_2x sprite/mysrc@2x.png
|
|
|
|
test_jsn spr_cmp sprite/src1,mysrc.json
|
|
|
|
test_png spr_cmp sprite/src1,mysrc.png
|
|
|
|
test_jsn spr_cmp_2x sprite/src1,mysrc@2x.json
|
|
|
|
test_png spr_cmp_2x sprite/src1,mysrc@2x.png
|
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
kill_process $PROCESS_ID
|
2023-02-07 09:05:47 +03:00
|
|
|
validate_log test_log_2.txt
|
2023-01-01 08:03:21 +03:00
|
|
|
|
2023-02-07 09:05:47 +03:00
|
|
|
remove_line "$(dirname "$0")/output/given_config.yaml" " connection_string: "
|
|
|
|
remove_line "$(dirname "$0")/output/generated_config.yaml" " connection_string: "
|
2022-11-30 19:57:27 +03:00
|
|
|
|
2023-06-04 01:54:50 +03:00
|
|
|
|
|
|
|
echo "------------------------------------------------------------------------------------------------------------------------"
|
|
|
|
echo "Test mbtiles utility"
|
|
|
|
if [[ "$MBTILES_BIN" != "-" ]]; then
|
|
|
|
TEST_OUT_DIR="$(dirname "$0")/output/mbtiles"
|
|
|
|
mkdir -p "$TEST_OUT_DIR"
|
2023-07-05 19:55:39 +03:00
|
|
|
TEST_TEMP_DIR="$(dirname "$0")/temp"
|
|
|
|
if [[ -d "$TEST_TEMP_DIR" ]]; then
|
|
|
|
echo "ERROR: ./$TEST_TEMP_DIR already exists. Please remove it first."
|
|
|
|
exit 1
|
|
|
|
else
|
|
|
|
mkdir -p "$TEST_TEMP_DIR"
|
|
|
|
fi
|
2023-06-04 01:54:50 +03:00
|
|
|
|
|
|
|
set -x
|
|
|
|
|
|
|
|
$MBTILES_BIN --help 2>&1 | tee "$TEST_OUT_DIR/help.txt"
|
|
|
|
$MBTILES_BIN meta-get --help 2>&1 | tee "$TEST_OUT_DIR/meta-get_help.txt"
|
|
|
|
$MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles name 2>&1 | tee "$TEST_OUT_DIR/meta-get_name.txt"
|
|
|
|
$MBTILES_BIN meta-get ./tests/fixtures/files/world_cities.mbtiles missing_value 2>&1 | tee "$TEST_OUT_DIR/meta-get_missing_value.txt"
|
|
|
|
|
2023-07-05 19:55:39 +03:00
|
|
|
# Create diff file
|
|
|
|
$MBTILES_BIN copy \
|
|
|
|
./tests/fixtures/files/world_cities.mbtiles \
|
|
|
|
"$TEST_TEMP_DIR/world_cities_diff.mbtiles" \
|
|
|
|
--diff-with-file ./tests/fixtures/files/world_cities_modified.mbtiles \
|
|
|
|
2>&1 | tee "$TEST_OUT_DIR/copy_diff.txt"
|
|
|
|
|
|
|
|
if command -v sqlite3 > /dev/null; then
|
|
|
|
# Apply this diff to the original version of the file
|
|
|
|
cp ./tests/fixtures/files/world_cities.mbtiles "$TEST_TEMP_DIR/world_cities_copy.mbtiles"
|
|
|
|
|
|
|
|
sqlite3 "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \
|
|
|
|
-bail \
|
|
|
|
-cmd ".parameter set @diffDbFilename $TEST_TEMP_DIR/world_cities_diff.mbtiles" \
|
|
|
|
"ATTACH DATABASE @diffDbFilename AS diffDb;" \
|
|
|
|
"DELETE FROM tiles WHERE (zoom_level, tile_column, tile_row) IN (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);" \
|
|
|
|
"INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;"
|
|
|
|
|
|
|
|
# Ensure that applying the diff resulted in the modified version of the file
|
|
|
|
$MBTILES_BIN copy \
|
|
|
|
--diff-with-file "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \
|
|
|
|
./tests/fixtures/files/world_cities_modified.mbtiles \
|
|
|
|
"$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \
|
|
|
|
2>&1 | tee "$TEST_OUT_DIR/copy_diff2.txt"
|
|
|
|
|
|
|
|
sqlite3 "$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \
|
|
|
|
"SELECT COUNT(*) FROM tiles;" \
|
|
|
|
2>&1 | tee "$TEST_OUT_DIR/copy_apply.txt"
|
|
|
|
|
|
|
|
else
|
|
|
|
echo "---------------------------------------------------------"
|
|
|
|
echo "##### sqlite3 is not installed, skipping apply test #####"
|
|
|
|
# Copy expected output files as if they were generated by the test
|
|
|
|
EXPECTED_DIR="$(dirname "$0")/expected/mbtiles"
|
|
|
|
cp "$EXPECTED_DIR/copy_diff2.txt" "$TEST_OUT_DIR/copy_diff2.txt"
|
|
|
|
cp "$EXPECTED_DIR/copy_apply.txt" "$TEST_OUT_DIR/copy_apply.txt"
|
|
|
|
fi
|
|
|
|
|
|
|
|
rm -rf "$TEST_TEMP_DIR"
|
|
|
|
|
2023-06-04 01:54:50 +03:00
|
|
|
{ set +x; } 2> /dev/null
|
|
|
|
else
|
|
|
|
echo "Skipping mbtiles utility tests"
|
|
|
|
fi
|
|
|
|
|
2022-11-30 19:57:27 +03:00
|
|
|
>&2 echo "All integration tests have passed"
|