2022-10-28 21:52:39 +03:00
#!/usr/bin/env bash
set -euo pipefail
2023-11-13 07:40:34 +03:00
MARTIN_DATABASE_URL = " ${ DATABASE_URL :- postgres : //postgres@localhost/db } "
unset DATABASE_URL
2022-10-31 23:28:21 +03:00
# TODO: use --fail-with-body to get the response body on failure
2023-02-20 18:44:22 +03:00
CURL = ${ CURL :- curl --silent --show-error --fail --compressed }
2023-11-13 07:40:34 +03:00
2023-11-19 13:06:37 +03:00
MARTIN_BUILD_ALL = " ${ MARTIN_BUILD_ALL :- cargo build } "
2023-01-01 08:03:21 +03:00
MARTIN_PORT = " ${ MARTIN_PORT :- 3111 } "
2022-12-19 05:24:06 +03:00
MARTIN_URL = " http://localhost: ${ MARTIN_PORT } "
2023-01-01 08:03:21 +03:00
MARTIN_ARGS = " ${ MARTIN_ARGS :- --listen-addresses localhost : ${ MARTIN_PORT } } "
2022-10-28 21:52:39 +03:00
2023-11-19 13:06:37 +03:00
# Using direct compiler output paths to avoid extra log entries
MARTIN_BIN = " ${ MARTIN_BIN :- target /debug/martin } ${ MARTIN_ARGS } "
2023-11-20 09:27:51 +03:00
MARTIN_CP_BIN = " ${ MARTIN_CP_BIN :- target /debug/martin-cp } "
2023-06-04 01:54:50 +03:00
MBTILES_BIN = " ${ MBTILES_BIN :- target /debug/mbtiles } "
2023-11-19 13:06:37 +03:00
TEST_OUT_BASE_DIR = " $( dirname " $0 " ) /output "
2023-10-28 09:06:37 +03:00
LOG_DIR = " ${ LOG_DIR :- target /test_logs } "
mkdir -p " $LOG_DIR "
2023-10-01 05:49:56 +03:00
2023-11-19 13:06:37 +03:00
TEST_TEMP_DIR = " $( dirname " $0 " ) /mbtiles_temp_files "
rm -rf " $TEST_TEMP_DIR "
mkdir -p " $TEST_TEMP_DIR "
2023-11-06 05:19:29 +03:00
function wait_for {
2022-10-28 21:52:39 +03:00
# Seems the --retry-all-errors option is not available on older curl versions, but maybe in the future we can just use this:
2022-12-19 05:24:06 +03:00
# timeout -k 20s 20s curl --retry 10 --retry-all-errors --retry-delay 1 -sS "$MARTIN_URL/health"
2022-11-02 21:00:05 +03:00
PROCESS_ID = $1
2023-11-06 05:19:29 +03:00
PROC_NAME = $2
TEST_URL = $3
echo " Waiting for $PROC_NAME ( $PROCESS_ID ) to start by checking $TEST_URL to be valid... "
2022-12-27 09:56:27 +03:00
for i in { 1..60} ; do
2023-11-06 05:19:29 +03:00
if $CURL " $TEST_URL " 2>/dev/null >/dev/null; then
echo " $PROC_NAME is up! "
if [ [ " $PROC_NAME " = = "Martin" ] ] ; then
$CURL " $TEST_URL "
fi
2022-10-31 23:28:21 +03:00
return
fi
2022-11-02 21:00:05 +03:00
if ps -p $PROCESS_ID > /dev/null ; then
2023-11-06 05:19:29 +03:00
echo " $PROC_NAME is not up yet, waiting for $TEST_URL ... "
2022-11-02 21:00:05 +03:00
sleep 1
else
2023-11-06 05:19:29 +03:00
echo " $PROC_NAME died! "
2022-11-02 21:00:05 +03:00
ps au
2023-11-06 05:19:29 +03:00
lsof -i || true;
2022-11-02 21:00:05 +03:00
exit 1
fi
2022-10-28 21:52:39 +03:00
done
2023-11-06 05:19:29 +03:00
echo " $PROC_NAME did not start in time "
2022-10-31 23:28:21 +03:00
ps au
2023-11-06 05:19:29 +03:00
lsof -i || true;
2022-10-31 23:28:21 +03:00
exit 1
}
2022-11-02 21:00:05 +03:00
function kill_process {
PROCESS_ID = $1
2023-11-06 05:19:29 +03:00
PROC_NAME = $2
echo " Waiting for $PROC_NAME ( $PROCESS_ID ) to stop... "
2022-11-02 21:00:05 +03:00
kill $PROCESS_ID
for i in { 1..50} ; do
if ps -p $PROCESS_ID > /dev/null ; then
sleep 0.1
else
2023-11-06 05:19:29 +03:00
echo " $PROC_NAME ( $PROCESS_ID ) has stopped "
2022-11-02 21:00:05 +03:00
return
fi
done
2023-11-06 05:19:29 +03:00
echo " $PROC_NAME did not stop in time, killing it "
2022-11-02 21:00:05 +03:00
kill -9 $PROCESS_ID
# wait for it to die using timeout and wait
2023-11-06 05:19:29 +03:00
timeout -k 1s 1s wait $PROCESS_ID || true;
2022-11-02 21:00:05 +03:00
}
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn( )
{
FILENAME = " $TEST_OUT_DIR / $1 .json "
URL = " $MARTIN_URL / $2 "
echo " Testing $( basename " $FILENAME " ) from $URL "
2023-02-22 19:25:48 +03:00
$CURL " $URL " | jq -e > " $FILENAME "
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
}
2022-10-31 23:28:21 +03:00
test_pbf( )
{
FILENAME = " $TEST_OUT_DIR / $1 .pbf "
2022-12-19 05:24:06 +03:00
URL = " $MARTIN_URL / $2 "
2022-10-31 23:28:21 +03:00
echo " Testing $( basename " $FILENAME " ) from $URL "
2023-02-20 18:44:22 +03:00
$CURL " $URL " > " $FILENAME "
2022-10-31 23:28:21 +03:00
if [ [ $OSTYPE = = linux* ] ] ; then
2023-01-03 19:09:41 +03:00
./tests/fixtures/vtzero-check " $FILENAME "
./tests/fixtures/vtzero-show " $FILENAME " > " $FILENAME .txt "
2022-10-31 23:28:21 +03:00
fi
2022-10-28 21:52:39 +03:00
}
2023-01-08 22:19:11 +03:00
test_png( )
{
2023-10-31 04:52:11 +03:00
# 3rd argument is optional, .png by default
FILENAME = " $TEST_OUT_DIR / $1 . ${ 3 :- png } "
2023-01-08 22:19:11 +03:00
URL = " $MARTIN_URL / $2 "
echo " Testing $( basename " $FILENAME " ) from $URL "
$CURL " $URL " > " $FILENAME "
if [ [ $OSTYPE = = linux* ] ] ; then
file " $FILENAME " > " $FILENAME .txt "
fi
}
2023-10-31 04:52:11 +03:00
test_jpg( )
{
2023-11-19 13:06:37 +03:00
# test_png can test any image format, but this is a separate function to make it easier to find all the jpeg tests
2023-10-31 04:52:11 +03:00
test_png $1 $2 jpg
}
Implement dynamic font support `/font/<name>/<start>-<end>` (#755)
This implements dynamic font protobuf generation, allowing users to
request font ranges on the fly, and combining them in any order, e.g.
`Font1,Font2,Font3`, same as with sprites and tiles
This is a first iteration, without any multithreading support. In
theory, this could be done far faster by generating SDFs with multiple
threads.
### Current process
* during init, figure out all glyphs available in each font, and store
them as a bitset
* during request:
* combine requested bitsets to figure out which glyph should come from
which font file
* load those glyphs from files (using a single instance of the freetype
lib)
* convert them to SDFs and package them into a protobuf
---------
Co-authored-by: Lucas <zhangyijunmetro@hotmail.com>
2023-10-28 08:10:48 +03:00
test_font( )
{
FILENAME = " $TEST_OUT_DIR / $1 .pbf "
URL = " $MARTIN_URL / $2 "
echo " Testing $( basename " $FILENAME " ) from $URL "
2023-10-31 04:52:11 +03:00
$CURL " $URL " > " $FILENAME "
Implement dynamic font support `/font/<name>/<start>-<end>` (#755)
This implements dynamic font protobuf generation, allowing users to
request font ranges on the fly, and combining them in any order, e.g.
`Font1,Font2,Font3`, same as with sprites and tiles
This is a first iteration, without any multithreading support. In
theory, this could be done far faster by generating SDFs with multiple
threads.
### Current process
* during init, figure out all glyphs available in each font, and store
them as a bitset
* during request:
* combine requested bitsets to figure out which glyph should come from
which font file
* load those glyphs from files (using a single instance of the freetype
lib)
* convert them to SDFs and package them into a protobuf
---------
Co-authored-by: Lucas <zhangyijunmetro@hotmail.com>
2023-10-28 08:10:48 +03:00
}
2023-02-07 09:05:47 +03:00
# Delete a line from a file $1 that matches parameter $2
remove_line( )
2023-01-01 08:03:21 +03:00
{
2023-02-07 09:05:47 +03:00
FILE = " $1 "
LINE_TO_REMOVE = " $2 "
>& 2 echo " Removing line ' $LINE_TO_REMOVE ' from $FILE "
grep -v " $LINE_TO_REMOVE " " ${ FILE } " > " ${ FILE } .tmp "
mv " ${ FILE } .tmp " " ${ FILE } "
}
test_log_has_str( )
{
LOG_FILE = " $1 "
EXPECTED_TEXT = " $2 "
echo " Checking $LOG_FILE for expected text: ' $EXPECTED_TEXT ' "
grep -q " $EXPECTED_TEXT " " $LOG_FILE "
remove_line " $LOG_FILE " " $EXPECTED_TEXT "
}
2023-11-20 09:27:51 +03:00
test_martin_cp( )
{
TEST_NAME = " $1 "
ARG = ( " ${ @ : 2 } " )
LOG_FILE = " ${ LOG_DIR } / ${ TEST_NAME } .txt "
SAVE_CONFIG_FILE = " ${ TEST_OUT_DIR } / ${ TEST_NAME } _save_config.yaml "
SUMMARY_FILE = " $TEST_OUT_DIR / ${ TEST_NAME } _summary.txt "
TEST_FILE = " ${ TEST_TEMP_DIR } /cp_ ${ TEST_NAME } .mbtiles "
ARG_EXTRAS = ( --output-file " $TEST_FILE " --save-config " $SAVE_CONFIG_FILE " )
set -x
$MARTIN_CP_BIN " ${ ARG [@] } " " ${ ARG_EXTRAS [@] } " 2>& 1 | tee " $LOG_FILE "
$MBTILES_BIN validate --agg-hash off " $TEST_FILE " 2>& 1 | tee " $TEST_OUT_DIR / ${ TEST_NAME } _validate.txt "
$MBTILES_BIN summary " $TEST_FILE " 2>& 1 | tee " $SUMMARY_FILE "
$MBTILES_BIN meta-all " $TEST_FILE " 2>& 1 | tee " $TEST_OUT_DIR / ${ TEST_NAME } _metadata.txt "
{ set +x; } 2> /dev/null
remove_line " $SAVE_CONFIG_FILE " " connection_string: "
# These tend to vary between runs. In theory, vacuuming might make it the same.
remove_line " $SUMMARY_FILE " "File size: "
remove_line " $SUMMARY_FILE " "Page count: "
}
2023-02-07 09:05:47 +03:00
validate_log( )
{
LOG_FILE = " $1 "
>& 2 echo " Validating log file $LOG_FILE "
# Older versions of PostGIS don't support the margin parameter, so we need to remove it from the log
remove_line " $LOG_FILE " 'Margin parameter in ST_TileEnvelope is not supported'
2023-08-26 16:53:06 +03:00
remove_line " $LOG_FILE " 'Source IDs must be unique'
2023-02-07 09:05:47 +03:00
echo "Checking for no other warnings or errors in the log"
if grep -e ' ERROR ' -e ' WARN ' " $LOG_FILE " ; then
echo " Log file $LOG_FILE has unexpected warnings or errors "
exit 1
fi
2023-01-01 08:03:21 +03:00
}
2022-10-28 21:52:39 +03:00
curl --version
2023-11-19 13:06:37 +03:00
# Make sure all targets are built - this way it won't timeout while waiting for it to start
# If set to "-", skip this step (e.g. when testing a pre-built binary)
if [ [ " $MARTIN_BUILD_ALL " != "-" ] ] ; then
2023-11-20 09:27:51 +03:00
rm -rf " $MARTIN_BIN " " $MARTIN_CP_BIN " " $MBTILES_BIN "
2023-11-19 13:06:37 +03:00
$MARTIN_BUILD_ALL
2023-06-04 01:54:50 +03:00
fi
2022-10-31 23:28:21 +03:00
2022-11-02 21:00:05 +03:00
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test auto configured Martin"
2022-12-12 17:11:10 +03:00
2023-11-19 13:06:37 +03:00
TEST_NAME = "auto"
LOG_FILE = " ${ LOG_DIR } / ${ TEST_NAME } .txt "
TEST_OUT_DIR = " ${ TEST_OUT_BASE_DIR } / ${ TEST_NAME } "
2022-12-27 09:56:27 +03:00
mkdir -p " $TEST_OUT_DIR "
2023-11-19 13:06:37 +03:00
ARG = ( --default-srid 900913 --auto-bounds calc --save-config " ${ TEST_OUT_DIR } /save_config.yaml " tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2 --sprite tests/fixtures/sprites/src1 --font tests/fixtures/fonts/overpass-mono-regular.ttf --font tests/fixtures/fonts)
2023-11-13 07:40:34 +03:00
export DATABASE_URL = " $MARTIN_DATABASE_URL "
2022-12-27 09:56:27 +03:00
set -x
2023-11-13 07:40:34 +03:00
$MARTIN_BIN " ${ ARG [@] } " 2>& 1 | tee " $LOG_FILE " &
2023-11-06 05:19:29 +03:00
MARTIN_PROC_ID = ` jobs -p | tail -n 1`
2022-11-02 21:00:05 +03:00
{ set +x; } 2> /dev/null
2023-11-06 05:19:29 +03:00
trap " echo 'Stopping Martin server $MARTIN_PROC_ID ...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID '; " EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin " $MARTIN_URL /health "
2023-11-13 07:40:34 +03:00
unset DATABASE_URL
2022-10-31 23:28:21 +03:00
>& 2 echo "Test catalog"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn catalog_auto catalog
2022-10-31 23:28:21 +03:00
2022-12-27 09:56:27 +03:00
>& 2 echo "***** Test server response for table source *****"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn table_source table_source
2022-12-19 05:24:06 +03:00
test_pbf tbl_0_0_0 table_source/0/0/0
2022-12-27 09:56:27 +03:00
test_pbf tbl_6_57_29 table_source/6/57/29
test_pbf tbl_12_3673_1911 table_source/12/3673/1911
test_pbf tbl_13_7346_3822 table_source/13/7346/3822
test_pbf tbl_14_14692_7645 table_source/14/14692/7645
test_pbf tbl_17_117542_61161 table_source/17/117542/61161
test_pbf tbl_18_235085_122323 table_source/18/235085/122323
>& 2 echo "***** Test server response for composite source *****"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn cmp table_source,points1,points2
2022-12-19 05:24:06 +03:00
test_pbf cmp_0_0_0 table_source,points1,points2/0/0/0
2022-12-27 09:56:27 +03:00
test_pbf cmp_6_57_29 table_source,points1,points2/6/57/29
test_pbf cmp_12_3673_1911 table_source,points1,points2/12/3673/1911
test_pbf cmp_13_7346_3822 table_source,points1,points2/13/7346/3822
test_pbf cmp_14_14692_7645 table_source,points1,points2/14/14692/7645
test_pbf cmp_17_117542_61161 table_source,points1,points2/17/117542/61161
test_pbf cmp_18_235085_122323 table_source,points1,points2/18/235085/122323
>& 2 echo "***** Test server response for function source *****"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn fnc function_zxy_query
2022-12-19 05:24:06 +03:00
test_pbf fnc_0_0_0 function_zxy_query/0/0/0
2022-12-27 09:56:27 +03:00
test_pbf fnc_6_57_29 function_zxy_query/6/57/29
test_pbf fnc_12_3673_1911 function_zxy_query/12/3673/1911
test_pbf fnc_13_7346_3822 function_zxy_query/13/7346/3822
test_pbf fnc_14_14692_7645 function_zxy_query/14/14692/7645
test_pbf fnc_17_117542_61161 function_zxy_query/17/117542/61161
test_pbf fnc_18_235085_122323 function_zxy_query/18/235085/122323
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn fnc_token function_zxy_query_test
test_pbf fnc_token_0_0_0 function_zxy_query_test/0/0/0?token= martin
test_jsn fnc_b function_zxy_query_jsonb
2022-12-27 09:56:27 +03:00
test_pbf fnc_b_6_38_20 function_zxy_query_jsonb/6/57/29
>& 2 echo "***** Test server response for different function call types *****"
test_pbf fnc_zoom_xy_6_57_29 function_zoom_xy/6/57/29
test_pbf fnc_zxy_6_57_29 function_zxy/6/57/29
test_pbf fnc_zxy2_6_57_29 function_zxy2/6/57/29
test_pbf fnc_zxy_query_6_57_29 function_zxy_query/6/57/29
test_pbf fnc_zxy_row_6_57_29 function_zxy_row/6/57/29
test_pbf fnc_zxy_row2_6_57_29 function_Mixed_Name/6/57/29
test_pbf fnc_zxy_row_key_6_57_29 function_zxy_row_key/6/57/29
>& 2 echo "***** Test server response for table source with different SRID *****"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn points3857_srid points3857
2022-12-19 05:24:06 +03:00
test_pbf points3857_srid_0_0_0 points3857/0/0/0
2022-10-31 23:28:21 +03:00
2023-01-08 22:19:11 +03:00
>& 2 echo "***** Test server response for PMTiles source *****"
2023-11-09 03:46:23 +03:00
test_jsn pmt stamen_toner__raster_CC-BY-ODbL_z3
test_png pmt_3_4_2 stamen_toner__raster_CC-BY-ODbL_z3/3/4/2
test_png webp2_1_0_0 webp2/1/0/0 # HTTP pmtiles
2023-01-08 22:19:11 +03:00
Add .mbtiles support (#549)
Adds a new [.mbtiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md)
backend, without the grid support. Uses extensive tile content
detection, i.e. if the content is gzipped, png, jpeg, gif, webp.
From CLI, can be as easy as adding a path to a directory that contains a
.mbtiles file (works just like pmtiles support)
```bash
# All *.mbtiles files in this dir will be published.
# The filename will be used as the source ID
martin ./tests/fixtures
```
From configuration file, the path can be specified in a number of ways
(same as pmtiles)
```yaml
mbtiles:
paths:
# scan this whole dir, matching all *.mbtiles files
- /dir-path
# specific mbtiles file will be published as mbtiles2 source
- /path/to/mbtiles2.mbtiles
sources:
# named source matching source name to a single file
pm-src1: /tmp/mbtiles.mbtiles
# named source, where the filename is explicitly set. This way we will be able to add more options later
pm-src2:
path: /tmp/mbtiles.mbtiles
```
Fixes #494
2023-01-09 08:10:23 +03:00
>& 2 echo "***** Test server response for MbTiles source *****"
test_jsn mb_jpg geography-class-jpg
2023-10-31 04:52:11 +03:00
test_jpg mb_jpg_0_0_0 geography-class-jpg/0/0/0
Add .mbtiles support (#549)
Adds a new [.mbtiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md)
backend, without the grid support. Uses extensive tile content
detection, i.e. if the content is gzipped, png, jpeg, gif, webp.
From CLI, can be as easy as adding a path to a directory that contains a
.mbtiles file (works just like pmtiles support)
```bash
# All *.mbtiles files in this dir will be published.
# The filename will be used as the source ID
martin ./tests/fixtures
```
From configuration file, the path can be specified in a number of ways
(same as pmtiles)
```yaml
mbtiles:
paths:
# scan this whole dir, matching all *.mbtiles files
- /dir-path
# specific mbtiles file will be published as mbtiles2 source
- /path/to/mbtiles2.mbtiles
sources:
# named source matching source name to a single file
pm-src1: /tmp/mbtiles.mbtiles
# named source, where the filename is explicitly set. This way we will be able to add more options later
pm-src2:
path: /tmp/mbtiles.mbtiles
```
Fixes #494
2023-01-09 08:10:23 +03:00
test_jsn mb_png geography-class-png
test_png mb_png_0_0_0 geography-class-png/0/0/0
test_jsn mb_mvt world_cities
test_pbf mb_mvt_2_3_1 world_cities/2/3/1
2022-12-27 09:56:27 +03:00
>& 2 echo "***** Test server response for table source with empty SRID *****"
Add .mbtiles support (#549)
Adds a new [.mbtiles](https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md)
backend, without the grid support. Uses extensive tile content
detection, i.e. if the content is gzipped, png, jpeg, gif, webp.
From CLI, can be as easy as adding a path to a directory that contains a
.mbtiles file (works just like pmtiles support)
```bash
# All *.mbtiles files in this dir will be published.
# The filename will be used as the source ID
martin ./tests/fixtures
```
From configuration file, the path can be specified in a number of ways
(same as pmtiles)
```yaml
mbtiles:
paths:
# scan this whole dir, matching all *.mbtiles files
- /dir-path
# specific mbtiles file will be published as mbtiles2 source
- /path/to/mbtiles2.mbtiles
sources:
# named source matching source name to a single file
pm-src1: /tmp/mbtiles.mbtiles
# named source, where the filename is explicitly set. This way we will be able to add more options later
pm-src2:
path: /tmp/mbtiles.mbtiles
```
Fixes #494
2023-01-09 08:10:23 +03:00
test_pbf points_empty_srid_0_0_0 points_empty_srid/0/0/0
2022-10-31 23:28:21 +03:00
2023-12-07 10:24:51 +03:00
>& 2 echo "***** Test server response for comments *****"
test_jsn tbl_comment MixPoints
test_jsn fnc_comment function_Mixed_Name
2023-11-06 05:19:29 +03:00
kill_process $MARTIN_PROC_ID Martin
2023-11-13 07:40:34 +03:00
test_log_has_str " $LOG_FILE " 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
test_log_has_str " $LOG_FILE " 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
validate_log " $LOG_FILE "
2023-11-19 13:06:37 +03:00
remove_line " ${ TEST_OUT_DIR } /save_config.yaml " " connection_string: "
2023-11-13 07:40:34 +03:00
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test minimum auto configured Martin"
2023-11-19 13:06:37 +03:00
TEST_NAME = "auto_mini"
LOG_FILE = " ${ LOG_DIR } / ${ TEST_NAME } .txt "
TEST_OUT_DIR = " ${ TEST_OUT_BASE_DIR } / ${ TEST_NAME } "
2023-11-13 07:40:34 +03:00
mkdir -p " $TEST_OUT_DIR "
2023-11-19 13:06:37 +03:00
ARG = ( --save-config " ${ TEST_OUT_DIR } /save_config.yaml " tests/fixtures/pmtiles2)
2023-11-13 07:40:34 +03:00
set -x
$MARTIN_BIN " ${ ARG [@] } " 2>& 1 | tee " $LOG_FILE " &
MARTIN_PROC_ID = ` jobs -p | tail -n 1`
{ set +x; } 2> /dev/null
trap " echo 'Stopping Martin server $MARTIN_PROC_ID ...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID '; " EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin " $MARTIN_URL /health "
>& 2 echo "Test catalog"
test_jsn catalog_auto catalog
kill_process $MARTIN_PROC_ID Martin
validate_log " $LOG_FILE "
2022-10-28 21:52:39 +03:00
2022-10-31 23:28:21 +03:00
2022-11-02 21:00:05 +03:00
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test pre-configured Martin"
2023-11-13 07:40:34 +03:00
2023-11-19 13:06:37 +03:00
TEST_NAME = "configured"
LOG_FILE = " ${ LOG_DIR } / ${ TEST_NAME } .txt "
TEST_OUT_DIR = " ${ TEST_OUT_BASE_DIR } / ${ TEST_NAME } "
2022-12-27 09:56:27 +03:00
mkdir -p " $TEST_OUT_DIR "
2022-12-12 17:11:10 +03:00
2023-11-19 13:06:37 +03:00
ARG = ( --config tests/config.yaml --max-feature-count 1000 --save-config " ${ TEST_OUT_DIR } /save_config.yaml " -W 1)
2023-11-13 07:40:34 +03:00
export DATABASE_URL = " $MARTIN_DATABASE_URL "
2022-12-27 09:56:27 +03:00
set -x
2023-11-13 07:40:34 +03:00
$MARTIN_BIN " ${ ARG [@] } " 2>& 1 | tee " $LOG_FILE " &
2023-11-06 05:19:29 +03:00
MARTIN_PROC_ID = ` jobs -p | tail -n 1`
2022-11-02 21:00:05 +03:00
{ set +x; } 2> /dev/null
2023-11-06 05:19:29 +03:00
trap " echo 'Stopping Martin server $MARTIN_PROC_ID ...'; kill -9 $MARTIN_PROC_ID 2> /dev/null || true; echo 'Stopped Martin server $MARTIN_PROC_ID '; " EXIT HUP INT TERM
wait_for $MARTIN_PROC_ID Martin " $MARTIN_URL /health "
2023-11-13 07:40:34 +03:00
unset DATABASE_URL
2022-10-31 23:28:21 +03:00
>& 2 echo "Test catalog"
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
test_jsn catalog_cfg catalog
2022-10-31 23:28:21 +03:00
2023-11-19 13:06:37 +03:00
# Test tile sources
test_pbf tbl_0_0_0 table_source/0/0/0
test_pbf cmp_0_0_0 points1,points2/0/0/0
test_pbf fnc_0_0_0 function_zxy_query/0/0/0
test_pbf fnc2_0_0_0 function_zxy_query_test/0/0/0?token= martin
test_png pmt_0_0_0 pmt/0/0/0
test_png pmt2_0_0_0 pmt2/0/0/0 # HTTP pmtiles
2022-10-31 23:28:21 +03:00
2023-11-19 13:06:37 +03:00
# Test sprites
Add dynamic sprites support (#715)
Dynamically create image sprites for MapLibre rendering, given a
directory with images.
### TODO
* [x] Work with @flother to merge these PRs
* [x] https://github.com/flother/spreet/pull/59 (must have)
* [x] https://github.com/flother/spreet/pull/57
* [x] https://github.com/flother/spreet/pull/56
* [ ] https://github.com/flother/spreet/pull/62 (not required but nice
to have, can upgrade later without any code changes)
* [x] Add docs to the book
* [x] Add CLI param, e.g. `--sprite <dir_path>`
* [x] Don't output `.sprites` in auto-genned config when not in use
### API
Per [MapLibre sprites
API](https://maplibre.org/maplibre-style-spec/sprite/), we need to
support the following:
* `/sprite/<sprite_id>.json` metadata about the sprite file - all coming
from a single directory
* `/sprite/<sprite_id>.png` all images combined into a single PNG
* `/sprite/<sprite_id>@2x.json` same but for high DPI devices
* `/sprite/<sprite_id>@2x.png`
Multiple sprite_id values can be combined into one sprite with the same
pattern as for tile joining:
`/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>[.json|.png|@2x.json|@2x.png]`.
No ID renaming is done, so identical names will override one another.
### Configuration
[Config file](https://maplibre.org/martin/config-file.html) and possibly
CLI should have a simple option to serve sprites. The configuration may
look similar to how mbtiles and pmtiles are configured:
```yaml
# Publish sprite images
sprites:
paths:
# scan this whole dir, matching all image files, and publishing it as "my_images" sprite source
- /path/to/my_images
sources:
# named source matching source name to a directory
my_sprites: /path/to/some_dir
```
Implement #705
2023-06-16 15:19:47 +03:00
test_jsn spr_src1 sprite/src1.json
test_png spr_src1 sprite/src1.png
test_jsn spr_src1_2x sprite/src1@2x.json
test_png spr_src1_2x sprite/src1@2x.png
test_jsn spr_mysrc sprite/mysrc.json
test_png spr_mysrc sprite/mysrc.png
test_jsn spr_mysrc_2x sprite/mysrc@2x.json
test_png spr_mysrc_2x sprite/mysrc@2x.png
test_jsn spr_cmp sprite/src1,mysrc.json
test_png spr_cmp sprite/src1,mysrc.png
test_jsn spr_cmp_2x sprite/src1,mysrc@2x.json
test_png spr_cmp_2x sprite/src1,mysrc@2x.png
2023-11-19 13:06:37 +03:00
# Test fonts
Implement dynamic font support `/font/<name>/<start>-<end>` (#755)
This implements dynamic font protobuf generation, allowing users to
request font ranges on the fly, and combining them in any order, e.g.
`Font1,Font2,Font3`, same as with sprites and tiles
This is a first iteration, without any multithreading support. In
theory, this could be done far faster by generating SDFs with multiple
threads.
### Current process
* during init, figure out all glyphs available in each font, and store
them as a bitset
* during request:
* combine requested bitsets to figure out which glyph should come from
which font file
* load those glyphs from files (using a single instance of the freetype
lib)
* convert them to SDFs and package them into a protobuf
---------
Co-authored-by: Lucas <zhangyijunmetro@hotmail.com>
2023-10-28 08:10:48 +03:00
test_font font_1 font/Overpass%20Mono%20Light/0-255
test_font font_2 font/Overpass%20Mono%20Regular/0-255
test_font font_3 font/Overpass%20Mono%20Regular,Overpass%20Mono%20Light/0-255
2023-12-07 10:24:51 +03:00
# Test comments override
test_jsn tbl_comment_cfg MixPoints
test_jsn fnc_comment_cfg fnc_Mixed_Name
2023-11-06 05:19:29 +03:00
kill_process $MARTIN_PROC_ID Martin
2023-11-13 07:40:34 +03:00
test_log_has_str " $LOG_FILE " 'WARN martin::pg::table_source] Table public.table_source has no spatial index on column geom'
test_log_has_str " $LOG_FILE " 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests'
validate_log " $LOG_FILE "
2023-11-19 13:06:37 +03:00
remove_line " ${ TEST_OUT_DIR } /save_config.yaml " " connection_string: "
2023-01-01 08:03:21 +03:00
2022-11-30 19:57:27 +03:00
2023-11-20 09:27:51 +03:00
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test martin-cp"
if [ [ " $MARTIN_CP_BIN " != "-" ] ] ; then
TEST_NAME = "martin-cp"
TEST_OUT_DIR = " ${ TEST_OUT_BASE_DIR } / ${ TEST_NAME } "
mkdir -p " $TEST_OUT_DIR "
export DATABASE_URL = " $MARTIN_DATABASE_URL "
CFG = ( --default-srid 900913 --auto-bounds calc tests/fixtures/mbtiles tests/fixtures/pmtiles tests/fixtures/pmtiles2)
test_martin_cp "flat" " ${ CFG [@] } " \
2023-11-21 01:38:28 +03:00
--source table_source --mbtiles-type flat --concurrency 3 \
2023-12-19 05:20:36 +03:00
--min-zoom 0 --max-zoom 6 "--bbox=-2,-1,142.84,45" \
--set-meta "generator=martin-cp v0.0.0"
2023-11-20 09:27:51 +03:00
test_martin_cp "flat-with-hash" " ${ CFG [@] } " \
2023-11-22 08:26:53 +03:00
--source function_zxy_query_test --url-query 'foo=bar&token=martin' --encoding 'identity' --mbtiles-type flat-with-hash --concurrency 3 \
2023-12-19 05:20:36 +03:00
--min-zoom 0 --max-zoom 6 "--bbox=-2,-1,142.84,45" \
--set-meta "generator=martin-cp v0.0.0"
2023-11-20 09:27:51 +03:00
test_martin_cp "normalized" " ${ CFG [@] } " \
2023-11-22 08:26:53 +03:00
--source geography-class-png --mbtiles-type normalized --concurrency 3 \
2023-12-06 08:41:32 +03:00
--min-zoom 0 --max-zoom 6 "--bbox=-2,-1,142.84,45" \
2023-12-19 05:20:36 +03:00
--set-meta "generator=martin-cp v0.0.0" --set-meta "name=normalized" --set-meta= center = 0,0,0
2023-11-20 09:27:51 +03:00
unset DATABASE_URL
else
echo "Skipping martin-cp tests"
fi
2023-06-04 01:54:50 +03:00
echo "------------------------------------------------------------------------------------------------------------------------"
echo "Test mbtiles utility"
if [ [ " $MBTILES_BIN " != "-" ] ] ; then
2023-11-19 13:06:37 +03:00
TEST_NAME = "mbtiles"
TEST_OUT_DIR = " ${ TEST_OUT_BASE_DIR } / ${ TEST_NAME } "
2023-06-04 01:54:50 +03:00
mkdir -p " $TEST_OUT_DIR "
set -x
2023-11-13 10:50:10 +03:00
$MBTILES_BIN summary ./tests/fixtures/mbtiles/world_cities.mbtiles 2>& 1 | tee " $TEST_OUT_DIR /summary.txt "
2023-09-06 06:12:45 +03:00
$MBTILES_BIN meta-all --help 2>& 1 | tee " $TEST_OUT_DIR /meta-all_help.txt "
2023-09-29 21:37:18 +03:00
$MBTILES_BIN meta-all ./tests/fixtures/mbtiles/world_cities.mbtiles 2>& 1 | tee " $TEST_OUT_DIR /meta-all.txt "
2023-06-04 01:54:50 +03:00
$MBTILES_BIN meta-get --help 2>& 1 | tee " $TEST_OUT_DIR /meta-get_help.txt "
2023-09-29 21:37:18 +03:00
$MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles name 2>& 1 | tee " $TEST_OUT_DIR /meta-get_name.txt "
$MBTILES_BIN meta-get ./tests/fixtures/mbtiles/world_cities.mbtiles missing_value 2>& 1 | tee " $TEST_OUT_DIR /meta-get_missing_value.txt "
$MBTILES_BIN validate ./tests/fixtures/mbtiles/zoomed_world_cities.mbtiles 2>& 1 | tee " $TEST_OUT_DIR /validate-ok.txt "
set +e
2023-12-10 05:59:14 +03:00
$MBTILES_BIN validate ./tests/fixtures/files/invalid-tile-idx.mbtiles 2>& 1 | tee " $TEST_OUT_DIR /validate-bad-tiles.txt "
2023-09-29 21:37:18 +03:00
if [ [ $? -eq 0 ] ] ; then
2023-12-10 05:59:14 +03:00
echo "ERROR: validate with invalid-tile-idx.mbtiles should have failed"
exit 1
fi
$MBTILES_BIN validate ./tests/fixtures/files/bad_hash.mbtiles 2>& 1 | tee " $TEST_OUT_DIR /validate-bad-hash.txt "
if [ [ $? -eq 0 ] ] ; then
echo "ERROR: validate with bad_hash.mbtiles should have failed"
2023-09-29 21:37:18 +03:00
exit 1
fi
set -e
cp ./tests/fixtures/files/bad_hash.mbtiles " $TEST_TEMP_DIR /fix_bad_hash.mbtiles "
2023-11-19 13:06:37 +03:00
$MBTILES_BIN validate --agg-hash update " $TEST_TEMP_DIR /fix_bad_hash.mbtiles " 2>& 1 | tee " $TEST_OUT_DIR /validate-fix.txt "
2023-09-29 21:37:18 +03:00
$MBTILES_BIN validate " $TEST_TEMP_DIR /fix_bad_hash.mbtiles " 2>& 1 | tee " $TEST_OUT_DIR /validate-fix2.txt "
2023-06-04 01:54:50 +03:00
2023-07-05 19:55:39 +03:00
# Create diff file
$MBTILES_BIN copy \
2023-09-29 21:37:18 +03:00
./tests/fixtures/mbtiles/world_cities.mbtiles \
2023-07-05 19:55:39 +03:00
" $TEST_TEMP_DIR /world_cities_diff.mbtiles " \
2023-09-29 21:37:18 +03:00
--diff-with-file ./tests/fixtures/mbtiles/world_cities_modified.mbtiles \
2023-07-05 19:55:39 +03:00
2>& 1 | tee " $TEST_OUT_DIR /copy_diff.txt "
if command -v sqlite3 > /dev/null; then
# Apply this diff to the original version of the file
2023-09-29 21:37:18 +03:00
cp ./tests/fixtures/mbtiles/world_cities.mbtiles " $TEST_TEMP_DIR /world_cities_copy.mbtiles "
2023-07-05 19:55:39 +03:00
sqlite3 " $TEST_TEMP_DIR /world_cities_copy.mbtiles " \
-bail \
-cmd " .parameter set @diffDbFilename $TEST_TEMP_DIR /world_cities_diff.mbtiles " \
"ATTACH DATABASE @diffDbFilename AS diffDb;" \
"DELETE FROM tiles WHERE (zoom_level, tile_column, tile_row) IN (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);" \
"INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;"
# Ensure that applying the diff resulted in the modified version of the file
$MBTILES_BIN copy \
--diff-with-file " $TEST_TEMP_DIR /world_cities_copy.mbtiles " \
2023-09-29 21:37:18 +03:00
./tests/fixtures/mbtiles/world_cities_modified.mbtiles \
2023-07-05 19:55:39 +03:00
" $TEST_TEMP_DIR /world_cities_diff_modified.mbtiles " \
2>& 1 | tee " $TEST_OUT_DIR /copy_diff2.txt "
sqlite3 " $TEST_TEMP_DIR /world_cities_diff_modified.mbtiles " \
"SELECT COUNT(*) FROM tiles;" \
2>& 1 | tee " $TEST_OUT_DIR /copy_apply.txt "
else
echo "---------------------------------------------------------"
echo "##### sqlite3 is not installed, skipping apply test #####"
# Copy expected output files as if they were generated by the test
EXPECTED_DIR = " $( dirname " $0 " ) /expected/mbtiles "
cp " $EXPECTED_DIR /copy_diff2.txt " " $TEST_OUT_DIR /copy_diff2.txt "
cp " $EXPECTED_DIR /copy_apply.txt " " $TEST_OUT_DIR /copy_apply.txt "
fi
2023-06-04 01:54:50 +03:00
{ set +x; } 2> /dev/null
else
echo "Skipping mbtiles utility tests"
fi
2023-11-19 13:06:37 +03:00
rm -rf " $TEST_TEMP_DIR "
2022-11-30 19:57:27 +03:00
>& 2 echo "All integration tests have passed"