2022-10-28 21:52:39 +03:00
|
|
|
#!/usr/bin/env just --justfile
|
2023-05-24 06:47:55 +03:00
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
set shell := ["bash", "-c"]
|
|
|
|
|
2023-02-06 22:32:28 +03:00
|
|
|
export PGPORT := "5411"
|
|
|
|
export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db"
|
2022-10-28 21:52:39 +03:00
|
|
|
export CARGO_TERM_COLOR := "always"
|
2023-05-24 06:47:55 +03:00
|
|
|
|
Support z,x,y and record-returning funcs, table rework (#380)
Can now handle several additional Postgres functions to get a tile, plus
tons of small fixes
### Multiple result variants
* `getmvt(z,x,y) -> [bytea,md5]` (single row with two columns)
* `getmvt(z,x,y) -> [bytea]` (single row with a single column)
* `getmvt(z,x,y) -> bytea` (value)
### Multiple input parameter variants
* `getmvt(z, x, y)` or `getmvt(zoom, x, y)` (all 3 vars must be
integers)
* `getmvt(z, x, y, url_query)`, where instead of `url_query` it could be
any other name, but must be of type JSON
### Breaking
* srid is now the same type as PG -- `i32`
* renamed config vals `table_sources` and `function_sources` into
`tables` and `functions`
### Features and fixes
* if postgis is v3.1+, uses margin parameter to extend the search box by
the size of the buffer. I think we should make 3.1 minimal required.
* fixes feature ID issue from #466
* fixes mixed case names for schemas, tables and columns, functions and
parameter names per #389
### Notes
* More dynamic SQL generation in code instead of using external SQL
files. Those should only be used when they are not parametrized.
* The new function/table discovery mechanism: query for all functions in
the database, and match up those functions with the ones configured (if
any), plus adds all the rest of the un-declared ones if discovery mode
is on.
* During table and function discovery, the code generates a map of
`(PgSqlInfo, FunctionInfo)` (or table) tupples containing SQL needed to
get the tile.
* Auto-discovery mode is currently hidden - the discovery is on only
when no tables or functions are configured. TBD - how to configure it in
the future
* The new system allows for an easy way to auto-discover for the
specific schemas only, solving #47
* predictable order of table/function instantiation
* bounding boxes computed in parallel for all tables (when not
configured)
* proper identifier escaping
* test cleanup
fixes #378
fixes #466
fixes #65
fixes #389
2022-12-10 17:20:42 +03:00
|
|
|
# export RUST_LOG := "debug"
|
2022-10-28 21:52:39 +03:00
|
|
|
# export RUST_BACKTRACE := "1"
|
|
|
|
|
|
|
|
@_default:
|
2023-05-24 06:47:55 +03:00
|
|
|
just --list --unsorted
|
2022-10-28 21:52:39 +03:00
|
|
|
|
2023-08-29 19:55:52 +03:00
|
|
|
# Start Martin server
|
|
|
|
run *ARGS:
|
2023-05-24 06:47:55 +03:00
|
|
|
cargo run -- {{ ARGS }}
|
2022-10-28 21:52:39 +03:00
|
|
|
|
2023-06-04 22:02:00 +03:00
|
|
|
# Start release-compiled Martin server and a test database
|
|
|
|
run-release *ARGS: start
|
|
|
|
cargo run -- {{ ARGS }}
|
|
|
|
|
2022-12-10 10:40:01 +03:00
|
|
|
# Start Martin server and open a test page
|
2022-12-27 09:56:27 +03:00
|
|
|
debug-page *ARGS: start
|
2022-12-10 10:40:01 +03:00
|
|
|
open tests/debug.html # run will not exit, so open debug page first
|
2023-05-24 06:47:55 +03:00
|
|
|
just run {{ ARGS }}
|
2022-12-10 10:40:01 +03:00
|
|
|
|
2022-11-30 19:57:27 +03:00
|
|
|
# Run PSQL utility against the test database
|
2023-01-01 08:03:21 +03:00
|
|
|
psql *ARGS:
|
2023-05-24 06:47:55 +03:00
|
|
|
psql {{ ARGS }} {{ DATABASE_URL }}
|
2022-11-30 19:57:27 +03:00
|
|
|
|
2023-09-06 18:12:53 +03:00
|
|
|
# Run pg_dump utility against the test database
|
2023-08-13 05:06:23 +03:00
|
|
|
pg_dump *ARGS:
|
|
|
|
pg_dump {{ ARGS }} {{ DATABASE_URL }}
|
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
# Perform cargo clean to delete all build files
|
2022-11-30 19:57:27 +03:00
|
|
|
clean: clean-test stop
|
2022-10-28 21:52:39 +03:00
|
|
|
cargo clean
|
|
|
|
|
|
|
|
# Delete test output files
|
2023-01-01 08:03:21 +03:00
|
|
|
[private]
|
2022-10-28 21:52:39 +03:00
|
|
|
clean-test:
|
|
|
|
rm -rf tests/output
|
|
|
|
|
|
|
|
# Start a test database
|
2022-12-27 09:56:27 +03:00
|
|
|
start: (docker-up "db")
|
2022-12-04 08:34:44 +03:00
|
|
|
|
2023-02-06 22:32:28 +03:00
|
|
|
# Start an ssl-enabled test database
|
|
|
|
start-ssl: (docker-up "db-ssl")
|
|
|
|
|
2022-12-04 08:34:44 +03:00
|
|
|
# Start a legacy test database
|
|
|
|
start-legacy: (docker-up "db-legacy")
|
|
|
|
|
|
|
|
# Start a specific test database, e.g. db or db-legacy
|
2023-01-01 08:03:21 +03:00
|
|
|
[private]
|
|
|
|
docker-up name:
|
2023-05-24 06:47:55 +03:00
|
|
|
docker-compose up -d {{ name }}
|
2023-06-03 03:40:22 +03:00
|
|
|
docker-compose run -T --rm db-is-ready
|
2022-10-28 21:52:39 +03:00
|
|
|
|
|
|
|
alias _down := stop
|
|
|
|
alias _stop-db := stop
|
|
|
|
|
2023-08-04 00:51:10 +03:00
|
|
|
# Restart the test database
|
|
|
|
restart:
|
|
|
|
just stop
|
|
|
|
just start
|
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
# Stop the test database
|
|
|
|
stop:
|
|
|
|
docker-compose down
|
|
|
|
|
|
|
|
# Run benchmark tests
|
2022-12-27 09:56:27 +03:00
|
|
|
bench: start
|
2022-10-28 21:52:39 +03:00
|
|
|
cargo bench
|
|
|
|
|
2023-06-04 22:02:00 +03:00
|
|
|
# Run HTTP requests benchmark using OHA tool. Use with `just run-release`
|
2023-07-07 23:44:30 +03:00
|
|
|
bench-http: (cargo-install "oha")
|
2023-06-04 22:02:00 +03:00
|
|
|
@echo "Make sure Martin was started with 'just run-release'"
|
|
|
|
@echo "Warming up..."
|
|
|
|
oha -z 5s --no-tui http://localhost:3000/function_zxy_query/18/235085/122323 > /dev/null
|
|
|
|
oha -z 120s http://localhost:3000/function_zxy_query/18/235085/122323
|
|
|
|
|
2022-10-28 21:52:39 +03:00
|
|
|
# Run all tests using a test database
|
2023-01-01 08:03:21 +03:00
|
|
|
test: (docker-up "db") test-unit test-int
|
|
|
|
|
2023-02-06 22:32:28 +03:00
|
|
|
# Run all tests using an SSL connection to a test database. Expected output won't match.
|
|
|
|
test-ssl: (docker-up "ssl") test-unit clean-test
|
|
|
|
tests/test.sh
|
|
|
|
|
|
|
|
# Run all tests using the oldest supported version of the database
|
2023-01-01 08:03:21 +03:00
|
|
|
test-legacy: (docker-up "db-legacy") test-unit test-int
|
2022-10-28 21:52:39 +03:00
|
|
|
|
2022-11-19 18:52:58 +03:00
|
|
|
# Run Rust unit and doc tests (cargo test)
|
2023-01-01 08:03:21 +03:00
|
|
|
test-unit *ARGS:
|
2023-05-24 06:47:55 +03:00
|
|
|
cargo test --all-targets {{ ARGS }}
|
2022-11-19 18:52:58 +03:00
|
|
|
cargo test --doc
|
2022-10-28 21:52:39 +03:00
|
|
|
|
|
|
|
# Run integration tests
|
2023-07-04 15:05:23 +03:00
|
|
|
test-int: clean-test install-sqlx
|
2023-02-07 09:05:47 +03:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
2022-10-28 21:52:39 +03:00
|
|
|
tests/test.sh
|
2023-06-13 06:05:54 +03:00
|
|
|
if ! diff --brief --recursive --new-file tests/output tests/expected; then
|
2023-01-01 08:03:21 +03:00
|
|
|
echo "** Expected output does not match actual output"
|
|
|
|
echo "** If this is expected, run 'just bless' to update expected output"
|
Refactorings, content-type/enc, cli parsing, tests, minor fixes (#548)
* introduce a new Connections object to track all positional strings
passed as the CLI arguments
* each tile provider can now indicate if it can take a positional CLI
arg, and if the value can be shared between multiple providers, i.e. if
its a directory that could contain files for multiple providers
* make xyz use better types - u8 for zoom, u32 for x&y. Postgres casts
those to INT2 and INT8
* minor bug in pre-push git hook to abort in case of a testingerror
* added GIF detection/type
* combine MVT and compression concepts into one enum more explicitly. It
is not ideal (technically they are separate concerns), but it keeps it a
bit simpler for now for multiple providers.
* set content encoding and content type on HTTP responses if known, and
also include them in the `/catalog` response (json)
* raise an error if the user attempts to merge non-concatenatable tiles
from multiple sources. We may want to implement it in the future, e.g.
combine multiple semi-transparent PNGs. Or even combine GIF & PNG & JPEG
* do not set content-type on empty responses (http 204)
* add tilejson outputs to testing
2023-01-08 17:31:58 +03:00
|
|
|
exit 1
|
2023-01-01 08:03:21 +03:00
|
|
|
else
|
|
|
|
echo "Expected output matches actual output"
|
|
|
|
fi
|
|
|
|
|
|
|
|
# Run integration tests and save its output as the new expected output
|
|
|
|
bless: start clean-test
|
Add dynamic sprites support (#715)
Dynamically create image sprites for MapLibre rendering, given a
directory with images.
### TODO
* [x] Work with @flother to merge these PRs
* [x] https://github.com/flother/spreet/pull/59 (must have)
* [x] https://github.com/flother/spreet/pull/57
* [x] https://github.com/flother/spreet/pull/56
* [ ] https://github.com/flother/spreet/pull/62 (not required but nice
to have, can upgrade later without any code changes)
* [x] Add docs to the book
* [x] Add CLI param, e.g. `--sprite <dir_path>`
* [x] Don't output `.sprites` in auto-genned config when not in use
### API
Per [MapLibre sprites
API](https://maplibre.org/maplibre-style-spec/sprite/), we need to
support the following:
* `/sprite/<sprite_id>.json` metadata about the sprite file - all coming
from a single directory
* `/sprite/<sprite_id>.png` all images combined into a single PNG
* `/sprite/<sprite_id>@2x.json` same but for high DPI devices
* `/sprite/<sprite_id>@2x.png`
Multiple sprite_id values can be combined into one sprite with the same
pattern as for tile joining:
`/sprite/<sprite_id1>,<sprite_id2>,...,<sprite_idN>[.json|.png|@2x.json|@2x.png]`.
No ID renaming is done, so identical names will override one another.
### Configuration
[Config file](https://maplibre.org/martin/config-file.html) and possibly
CLI should have a simple option to serve sprites. The configuration may
look similar to how mbtiles and pmtiles are configured:
```yaml
# Publish sprite images
sprites:
paths:
# scan this whole dir, matching all image files, and publishing it as "my_images" sprite source
- /path/to/my_images
sources:
# named source matching source name to a directory
my_sprites: /path/to/some_dir
```
Implement #705
2023-06-16 15:19:47 +03:00
|
|
|
cargo test --features bless-tests
|
2023-01-01 08:03:21 +03:00
|
|
|
tests/test.sh
|
|
|
|
rm -rf tests/expected
|
|
|
|
mv tests/output tests/expected
|
2022-12-27 09:56:27 +03:00
|
|
|
|
2023-05-29 05:06:35 +03:00
|
|
|
# Build and open mdbook documentation
|
2023-07-07 23:44:30 +03:00
|
|
|
book: (cargo-install "mdbook")
|
2023-06-04 21:11:44 +03:00
|
|
|
mdbook serve docs --open --port 8321
|
2023-05-29 05:06:35 +03:00
|
|
|
|
2023-07-10 21:38:48 +03:00
|
|
|
# Build debian package
|
|
|
|
package-deb: (cargo-install "cargo-deb")
|
|
|
|
cargo deb -v -p martin --output target/debian/martin.deb
|
|
|
|
|
2023-05-29 05:06:35 +03:00
|
|
|
# Build and open code documentation
|
|
|
|
docs:
|
|
|
|
cargo doc --no-deps --open
|
|
|
|
|
2022-12-27 09:56:27 +03:00
|
|
|
# Run code coverage on tests and save its output in the coverage directory. Parameter could be html or lcov.
|
2023-07-07 23:44:30 +03:00
|
|
|
coverage FORMAT='html': (cargo-install "grcov")
|
2022-12-27 09:56:27 +03:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
set -euo pipefail
|
2023-06-13 06:05:54 +03:00
|
|
|
if ! rustup component list | grep llvm-tools-preview &> /dev/null; then \
|
2022-12-27 09:56:27 +03:00
|
|
|
echo "llvm-tools-preview could not be found. Installing..." ;\
|
|
|
|
rustup component add llvm-tools-preview ;\
|
|
|
|
fi
|
|
|
|
|
|
|
|
just clean
|
|
|
|
just start
|
|
|
|
|
|
|
|
PROF_DIR=target/prof
|
|
|
|
mkdir -p "$PROF_DIR"
|
|
|
|
PROF_DIR=$(realpath "$PROF_DIR")
|
|
|
|
|
2023-05-24 06:47:55 +03:00
|
|
|
OUTPUT_RESULTS_DIR=target/coverage/{{ FORMAT }}
|
2022-12-27 09:56:27 +03:00
|
|
|
mkdir -p "$OUTPUT_RESULTS_DIR"
|
|
|
|
|
|
|
|
export CARGO_INCREMENTAL=0
|
|
|
|
export RUSTFLAGS=-Cinstrument-coverage
|
|
|
|
# Avoid problems with relative paths
|
|
|
|
export LLVM_PROFILE_FILE=$PROF_DIR/cargo-test-%p-%m.profraw
|
|
|
|
export MARTIN_PORT=3111
|
|
|
|
|
|
|
|
cargo test --all-targets
|
|
|
|
tests/test.sh
|
|
|
|
|
|
|
|
set -x
|
2023-05-24 06:47:55 +03:00
|
|
|
grcov --binary-path ./target/debug \
|
|
|
|
-s . \
|
|
|
|
-t {{ FORMAT }} \
|
|
|
|
--branch \
|
|
|
|
--ignore 'benches/*' \
|
|
|
|
--ignore 'tests/*' \
|
|
|
|
--ignore-not-existing \
|
|
|
|
-o target/coverage/{{ FORMAT }} \
|
|
|
|
--llvm \
|
2022-12-27 09:56:27 +03:00
|
|
|
"$PROF_DIR"
|
|
|
|
{ set +x; } 2>/dev/null
|
|
|
|
|
|
|
|
# if this is html, open it in the browser
|
2023-05-24 06:47:55 +03:00
|
|
|
if [ "{{ FORMAT }}" = "html" ]; then
|
2022-12-27 09:56:27 +03:00
|
|
|
open "$OUTPUT_RESULTS_DIR/index.html"
|
|
|
|
fi
|
2022-10-31 23:28:21 +03:00
|
|
|
|
2022-11-02 21:00:05 +03:00
|
|
|
# Build martin docker image
|
|
|
|
docker-build:
|
2023-03-16 23:01:33 +03:00
|
|
|
docker build -t ghcr.io/maplibre/martin .
|
2022-11-02 21:00:05 +03:00
|
|
|
|
|
|
|
# Build and run martin docker image
|
|
|
|
docker-run *ARGS:
|
2023-05-24 06:47:55 +03:00
|
|
|
docker run -it --rm --net host -e DATABASE_URL -v $PWD/tests:/tests ghcr.io/maplibre/martin {{ ARGS }}
|
2022-11-02 21:00:05 +03:00
|
|
|
|
2022-10-31 23:28:21 +03:00
|
|
|
# Do any git command, ensuring that the testing environment is set up. Accepts the same arguments as git.
|
2022-11-30 19:57:27 +03:00
|
|
|
[no-exit-message]
|
2022-12-27 09:56:27 +03:00
|
|
|
git *ARGS: start
|
2023-05-24 06:47:55 +03:00
|
|
|
git {{ ARGS }}
|
2022-12-10 10:40:01 +03:00
|
|
|
|
2023-05-16 21:12:24 +03:00
|
|
|
# Print the connection string for the test database
|
|
|
|
print-conn-str:
|
2023-05-24 06:47:55 +03:00
|
|
|
@echo {{ DATABASE_URL }}
|
2023-05-16 21:12:24 +03:00
|
|
|
|
2022-12-22 09:35:29 +03:00
|
|
|
# Run cargo fmt and cargo clippy
|
2023-06-04 01:54:50 +03:00
|
|
|
lint: fmt clippy
|
|
|
|
|
|
|
|
# Run cargo fmt
|
|
|
|
fmt:
|
2022-12-22 09:35:29 +03:00
|
|
|
cargo fmt --all -- --check
|
2023-06-04 01:54:50 +03:00
|
|
|
|
2023-06-16 01:36:41 +03:00
|
|
|
# Run Nightly cargo fmt, ordering imports
|
|
|
|
fmt2:
|
|
|
|
cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate
|
|
|
|
|
2023-06-04 01:54:50 +03:00
|
|
|
# Run cargo clippy
|
|
|
|
clippy:
|
2023-07-04 00:29:44 +03:00
|
|
|
cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings
|
2022-12-22 09:35:29 +03:00
|
|
|
|
2022-12-10 10:40:01 +03:00
|
|
|
# These steps automatically run before git push via a git hook
|
2023-01-01 08:03:21 +03:00
|
|
|
[private]
|
2022-12-27 09:56:27 +03:00
|
|
|
git-pre-push: stop start
|
2022-12-19 05:24:06 +03:00
|
|
|
rustc --version
|
|
|
|
cargo --version
|
2022-12-22 09:35:29 +03:00
|
|
|
just lint
|
2022-12-10 10:40:01 +03:00
|
|
|
just test
|
2023-06-01 20:07:13 +03:00
|
|
|
|
2023-07-04 15:05:23 +03:00
|
|
|
# Update sqlite database schema.
|
|
|
|
prepare-sqlite: install-sqlx
|
|
|
|
mkdir -p martin-mbtiles/.sqlx
|
2023-07-27 21:54:49 +03:00
|
|
|
cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests
|
2023-07-04 15:05:23 +03:00
|
|
|
|
|
|
|
# Install SQLX cli if not already installed.
|
|
|
|
[private]
|
2023-07-07 23:44:30 +03:00
|
|
|
install-sqlx: (cargo-install "cargo-sqlx" "sqlx-cli" "--no-default-features" "--features" "sqlite,native-tls")
|
|
|
|
|
|
|
|
# Check if a certain Cargo command is installed, and install it if needed
|
|
|
|
[private]
|
|
|
|
cargo-install $COMMAND $INSTALL_CMD="" *ARGS="":
|
|
|
|
@if ! command -v $COMMAND &> /dev/null; then \
|
|
|
|
echo "$COMMAND could not be found. Installing it with cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }}" ;\
|
|
|
|
cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }} ;\
|
2023-06-01 20:07:13 +03:00
|
|
|
fi
|