on main help sign in
40 files changed+5259−2131
since splitsnapshots
unifiedsplit
.github/workflows/lint.yaml+3−3 .github/workflows/publish-crate.yaml+20 .github/workflows/publish-docker.yaml+25 Cargo.toml+6−3 Dockerfile+1−1 README.md+377−82 src/db.rs+139−6 src/helpers.rs+7−16 src/lib.rs+414−427 src/main.rs+95−20 src/migrations/add_column.rs+214−60 src/migrations/add_foreign_key.rs+122 src/migrations/add_index.rs+24−5 src/migrations/alter_column.rs+25−19 src/migrations/common.rs+42−26 src/migrations/create_enum.rs+83 src/migrations/create_table.rs+128−24 src/migrations/custom.rs+61 src/migrations/mod.rs+25−4 src/migrations/remove_column.rs+353−35 src/migrations/remove_enum.rs+51 src/migrations/remove_foreign_key.rs+91 src/schema.rs+18−1 src/state.rs+31−40 tests/add_column.rs+318−254 tests/add_foreign_key.rs+166 tests/add_index.rs+215−61 tests/alter_column.rs+637−501 tests/common.rs+236−9 tests/complex.rs+379 tests/create_enum.rs+41 tests/create_table.rs+185−193 tests/custom.rs+128 tests/failure.rs+35−63 tests/remove_column.rs+209−73 tests/remove_enum.rs+69 tests/remove_foreign_key.rs+124 tests/remove_index.rs+68−91 tests/remove_table.rs+38−48 tests/rename_table.rs+56−66
.github/workflows/lint.yaml+3−3
@@ -15,7 +15,7 @@ jobs:
1515 runs-on: ubuntu-latest
1616 steps:
1717 - name: Checkout
18− uses: actions/checkout@v2
18+ uses: actions/checkout@v4
1919 - name: Select Rust toolchain with Clippy
2020 uses: actions-rs/toolchain@v1
2121 with:
@@ -23,6 +23,6 @@ jobs:
2323 components: clippy
2424 override: true
2525 - name: Use cache for Rust dependencies
26− uses: Swatinem/rust-cache@v1
26+ uses: Swatinem/rust-cache@v2
2727 - name: Lint using Clippy
28− run: cargo clippy
28+ run: cargo clippy
.github/workflows/publish-crate.yamlnew file+20
@@ -0,0 +1,20 @@
1+name: Publish crate
2+
3+on: workflow_dispatch
4+
5+jobs:
6+ publish-crate:
7+ runs-on: ubuntu-latest
8+ steps:
9+ - name: Checkout
10+ uses: actions/checkout@v2
11+ - name: Select Rust toolchain
12+ uses: actions-rs/toolchain@v1
13+ with:
14+ toolchain: stable
15+ - name: Publish to crates.io
16+ uses: actions-rs/cargo@v1
17+ with:
18+ command: publish
19+ env:
20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/publish-docker.yamlnew file+25
@@ -0,0 +1,25 @@
1+name: Publish Docker image
2+
3+on:
4+ workflow_dispatch:
5+ inputs:
6+ version:
7+ description: "Version (without 'v' prefix)"
8+ required: true
9+ type: string
10+
11+jobs:
12+ publish-docker:
13+ runs-on: ubuntu-latest
14+ steps:
15+ - name: Checkout
16+ uses: actions/checkout@v2
17+ - name: Log in to Docker
18+ env:
19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
20+ run: |
21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD
22+ - name: Build Docker image
23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
24+ - name: Push Docker image
25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
Cargo.toml+6−3
@@ -1,6 +1,6 @@
11[package]
22name = "reshape"
3−version = "0.2.0"
3+version = "0.7.0"
44description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
55homepage = "https://github.com/fabianlindfors/reshape"
66documentation = "https://github.com/fabianlindfors/reshape"
@@ -9,6 +9,7 @@ license = "MIT"
99keywords = ["postgres", "migrations"]
1010edition = "2021"
1111authors = ["Fabian Lindfors"]
12+rust-version = "1.70"
1213
1314[dependencies]
1415postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
@@ -16,8 +17,10 @@ serde = { version = "1.0", features = ["derive"] }
1617serde_json = "1.0"
1718typetag = "0.1.7"
1819anyhow = { version = "1.0.44", features = ["backtrace"] }
19−clap = { version = "3.0.0", features = ["derive"] }
20+clap = { version = "3.1.9", features = ["derive"] }
2021toml = "0.5"
2122version = "3.0.0"
2223colored = "2"
23−"derive_builder" = "0.10.2"
24+rand = "0.8"
25+dotenv = "0.15.0"
26+lexical-sort = "0.3.1"
Dockerfile+1−1
@@ -1,4 +1,4 @@
1−FROM rust:1.57.0 AS builder
1+FROM rust:1.70.0 AS builder
22WORKDIR /usr/src/reshape
33COPY . .
44RUN cargo build --release
README.md+377−82
@@ -2,50 +2,56 @@
22
33[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)
44
5−Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application.
5+> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!
66
7−Reshape is designed for Postgres 12 and later.
7+Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).
88
9−*Note: Reshape is **experimental** and should not be used in production. It can (and probably will) destroy your data and break your application.*
9+Designed for Postgres 12 and later.
1010
1111- [How it works](#how-it-works)
1212- [Getting started](#getting-started)
13− - [Installation](#installation)
14− - [Creating your first migration](#creating-your-first-migration)
15− - [Preparing your application](#preparing-your-application)
16− - [Running your migration](#running-your-migration)
17− - [Using during development](#using-during-development)
13+ - [Installation](#installation)
14+ - [Creating your first migration](#creating-your-first-migration)
15+ - [Preparing your application](#preparing-your-application)
16+ - [Running your migration](#running-your-migration)
17+ - [Using during development](#using-during-development)
1818- [Writing migrations](#writing-migrations)
19− - [Basics](#basics)
20− - [Tables](#tables)
21− - [Create table](#create-table)
22− - [Rename table](#rename-table)
23− - [Remove table](#remove-table)
24− - [Columns](#columns)
25− - [Add column](#add-column)
26− - [Alter column](#alter-column)
27− - [Remove column](#remove-column)
28− - [Indices](#indices)
29− - [Add index](#add-index)
30− - [Remove index](#remove-index)
19+ - [Basics](#basics)
20+ - [Tables](#tables)
21+ - [Create table](#create-table)
22+ - [Rename table](#rename-table)
23+ - [Remove table](#remove-table)
24+ - [Add foreign key](#add-foreign-key)
25+ - [Remove foreign key](#remove-foreign-key)
26+ - [Columns](#columns)
27+ - [Add column](#add-column)
28+ - [Alter column](#alter-column)
29+ - [Remove column](#remove-column)
30+ - [Indices](#indices)
31+ - [Add index](#add-index)
32+ - [Remove index](#remove-index)
33+ - [Enums](#enums)
34+ - [Create enum](#create-enum)
35+ - [Remove enum](#remove-enum)
36+ - [Custom](#custom)
37+ - [Complex changes across tables](#complex-changes-across-tables)
3138- [Commands and options](#commands-and-options)
32− - [`reshape migrate`](#reshape-migrate)
33− - [`reshape complete`](#reshape-complete)
34− - [`reshape abort`](#reshape-abort)
35− - [`reshape generate-schema-query`](#reshape-generate-schema-query)
36− - [Connection options](#connection-options)
39+ - [`reshape migration start`](#reshape-migration-start)
40+ - [`reshape migration complete`](#reshape-migration-complete)
41+ - [`reshape migration abort`](#reshape-migration-abort)
42+ - [`reshape schema-query`](#reshape-schema-query)
43+ - [Connection options](#connection-options)
3744- [License](#license)
3845
39−
4046## How it works
4147
4248Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every deployment is a three-phase process:
4349
44−1. **Start migration** (`reshape migrate`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
50+1. **Start migration** (`reshape migration start`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
45512. **Roll out application**: Your application can be gradually rolled out without downtime. The existing deployment will continue using the old schema whilst the new deployment uses the new schema.
46−3. **Complete migration** (`reshape complete`): Removes the old schema and any intermediate data and triggers.
52+3. **Complete migration** (`reshape migration complete`): Removes the old schema and any intermediate data and triggers.
4753
48−If the application deployment fails, you should run `reshape abort` which will roll back any changes made by `reshape migrate` without losing data.
54+If the application deployment fails, you should run `reshape migration abort` which will roll back any changes made by `reshape migration start` without losing data.
4955
5056## Getting started
5157
@@ -57,7 +63,7 @@ Binaries are available for macOS and Linux under [Releases](https://github.com/f
5763
5864#### Cargo
5965
60−Reshape can be installed using [Cargo](https://doc.rust-lang.org/cargo/):
66+Reshape can be installed using [Cargo](https://doc.rust-lang.org/cargo/) (requires Rust 1.58 or later):
6167
6268```shell
6369cargo install reshape
@@ -65,10 +71,10 @@ cargo install reshape
6571
6672#### Docker
6773
68−Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
74+Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
6975
7076```shell
71−docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migrate
77+docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migration start
7278```
7379
7480### Creating your first migration
@@ -97,9 +103,14 @@ This is the equivalent of running `CREATE TABLE users (id INTEGER GENERATED ALWA
97103
98104### Preparing your application
99105
100−Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. This query can be generated using: `reshape generate-schema-query`.
106+Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. The simplest way to do this is to use one of the helper libraries:
107+
108+- [Rust](https://github.com/fabianlindfors/reshape-helper)
109+- [Ruby (and Rails)](https://github.com/fabianlindfors/reshape-ruby)
110+- [Python (and Django)](https://github.com/fabianlindfors/reshape-python)
111+- [Go](https://github.com/leourbina/reshape-helper)
101112
102−To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape generate-schema-query)`. Then in your application:
113+If your application is not using one of the languages with an available helper library, you can instead generate the query with the command: `reshape schema-query`. To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape schema-query)`. Then in your application:
103114
104115```python
105116# Example for Python
@@ -112,16 +123,16 @@ db.execute(reshape_schema_query)
112123To create your new `users` table, run:
113124
114125```bash
115−reshape migrate
126+reshape migration start --complete
116127```
117128
118−As this is the first migration, Reshape will automatically complete it. For subsequent migrations, you will need to first run `reshape migrate`, roll out your application and then complete the migration using `reshape complete`.
129+We use the `--complete` flag to automatically complete the migration. During a production deployment, you should first run `reshape migration start` followed by `reshape migration complete` once your application has been fully rolled out.
119130
120131If nothing else is specified, Reshape will try to connect to a Postgres database running on `localhost` using `postgres` as both username and password. See [Connection options](#connection-options) for details on how to change the connection settings.
121132
122133### Using during development
123134
124−When adding new migrations during development, we recommend running `reshape migrate` but skipping `reshape complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape abort` followed by `reshape migrate`.
135+When adding new migrations during development, we recommend running `reshape migration start` but skipping `reshape migration complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape migration abort` followed by `reshape migration start`.
125136
126137## Writing migrations
127138
@@ -156,9 +167,9 @@ Every action has a `type`. The supported types are detailed below.
156167
157168#### Create table
158169
159−The `create_table` action will create a new table with the specified columns, indices and constraints.
170+The `create_table` action will create a new table with the specified columns, indices and constraints. You can optionally provide an `up` option to backfill values from an existing table.
160171
161−*Example: create a `customers` table with a few columns and a primary key*
172+_Example: create a `customers` table with a few columns and a primary key_
162173
163174```toml
164175[[actions]]
@@ -176,13 +187,13 @@ primary_key = ["id"]
176187 type = "TEXT"
177188
178189 # Columns default to nullable
179− nullable = false
190+ nullable = false
180191
181192 # default can be any valid SQL value, in this case a string literal
182− default = "'PLACEHOLDER'"
193+ default = "'PLACEHOLDER'"
183194```
184195
185−*Example: create `users` and `items` tables with a foreign key between them*
196+_Example: create `users` and `items` tables with a foreign key between them_
186197
187198```toml
188199[[actions]]
@@ -215,11 +226,34 @@ primary_key = ["id"]
215226 referenced_columns = ["id"]
216227```
217228
229+_Example: create `profiles` table based on existing `users` table_
230+
231+```toml
232+[[actions]]
233+type = "create_table"
234+name = "profiles"
235+primary_key = ["user_id"]
236+
237+ [[actions.columns]]
238+ name = "user_id"
239+ type = "INTEGER"
240+
241+ [[actions.columns]]
242+ name = "user_email"
243+ type = "TEXT"
244+
245+ # Backfill from `users` table and copy `users.email` to `user_email` column
246+ # This will perform an upsert based on the primary key to avoid duplicate rows
247+ [actions.up]
248+ table = "users"
249+ values = { user_id = "id", user_email = "email" }
250+```
251+
218252#### Rename table
219253
220254The `rename_table` action will change the name of an existing table.
221255
222−*Example: change name of `users` table to `customers`*
256+_Example: change name of `users` table to `customers`_
223257
224258```toml
225259[[actions]]
@@ -232,7 +266,7 @@ new_name = "customers"
232266
233267The `remove_table` action will remove an existing table.
234268
235−*Example: remove `users` table*
269+_Example: remove `users` table_
236270
237271```toml
238272[[actions]]
@@ -240,13 +274,43 @@ type = "remove_table"
240274table = "users"
241275```
242276
277+#### Add foreign key
278+
279+The `add_foreign_key` action will add a foreign key between two existing tables. The migration will fail if the existing column values aren't valid references.
280+
281+_Example: create foreign key from `items` to `users` table_
282+
283+```toml
284+[[actions]]
285+type = "add_foreign_key"
286+table = "items"
287+
288+ [actions.foreign_key]
289+ columns = ["user_id"]
290+ referenced_table = "users"
291+ referenced_columns = ["id"]
292+```
293+
294+#### Remove foreign key
295+
296+The `remove_foreign_key` action will remove an existing foreign key. The foreign key will only be removed once the migration is completed, which means that your new application must continue to adhere to the foreign key constraint.
297+
298+_Example: remove foreign key `items_user_id_fkey` from `users` table_
299+
300+```toml
301+[[actions]]
302+type = "remove_foreign_key"
303+table = "items"
304+foreign_key = "items_user_id_fkey"
305+```
306+
243307### Columns
244308
245309#### Add column
246310
247−The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column.
311+The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column. `up` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)).
248312
249−*Example: add a new column `reference` to table `products`*
313+_Example: add a new column `reference` to table `products`_
250314
251315```toml
252316[[actions]]
@@ -260,7 +324,7 @@ table = "products"
260324 default = "10"
261325```
262326
263−*Example: replace an existing `name` column with two new columns, `first_name` and `last_name`*
327+_Example: replace an existing `name` column with two new columns, `first_name` and `last_name`_
264328
265329```toml
266330[[actions]]
@@ -296,14 +360,48 @@ column = "name"
296360down = "first_name || ' ' || last_name"
297361```
298362
363+_Example: extract nested value from unstructured JSON `data` column to new `name` column_
364+
365+```toml
366+[[actions]]
367+type = "add_column"
368+table = "users"
369+
370+# #>> '{}' converts the JSON string value to TEXT
371+up = "data['path']['to']['value'] #>> '{}'"
372+
373+ [actions.column]
374+ name = "name"
375+ type = "TEXT"
376+```
377+
378+_Example: duplicate `email` column from `users` to `profiles` table_
379+
380+```toml
381+# `profiles` has `user_id` column which maps to `users.id`
382+[[actions]]
383+type = "add_column"
384+table = "profiles"
385+
386+ [actions.column]
387+ name = "email"
388+ type = "TEXT"
389+ nullable = false
390+
391+ # When `users` is updated in the old schema, we write the email value to `profiles`
392+ [actions.up]
393+ table = "users"
394+ value = "email"
395+ where = "user_id = id"
396+```
299397
300398#### Alter column
301399
302400The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.
303401
304−When performing more complex changes than a rename, `up` and `down` must be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
402+When performing more complex changes than a rename, `up` and `down` should be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
305403
306−*Example: rename `last_name` column on `users` table to `family_name`*
404+_Example: rename `last_name` column on `users` table to `family_name`_
307405
308406```toml
309407[[actions]]
@@ -315,7 +413,7 @@ column = "last_name"
315413 name = "family_name"
316414```
317415
318−*Example: change the type of `reference` column from `INTEGER` to `TEXT`*
416+_Example: change the type of `reference` column from `INTEGER` to `TEXT`_
319417
320418```toml
321419[[actions]]
@@ -330,7 +428,7 @@ down = "CAST(reference AS INTEGER)" # Converts from text value to integer
330428 type = "TEXT" # Previous type was 'INTEGER'
331429```
332430
333−*Example: increment all values of an `index` column by one*
431+_Example: increment all values of an `index` column by one_
334432
335433```toml
336434[[actions]]
@@ -345,7 +443,22 @@ down = "index - 1" # Decrement to revert for old schema
345443 name = "index"
346444```
347445
348−*Example: change default value of `created_at` column to current time*
446+_Example: make `name` column not nullable_
447+
448+```toml
449+[[actions]]
450+type = "alter_column"
451+table = "users"
452+column = "name"
453+
454+# Use "N/A" for any rows that currently have a NULL name
455+up = "COALESCE(name, 'N/A')"
456+
457+ [actions.changes]
458+ nullable = false
459+```
460+
461+_Example: change default value of `created_at` column to current time_
349462
350463```toml
351464[[actions]]
@@ -359,9 +472,11 @@ column = "created_at"
359472
360473#### Remove column
361474
362−The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
475+The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. `down` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)) . The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
476+
477+Any indices that cover the column will be removed.
363478
364−*Example: remove column `name` from table `users`*
479+_Example: remove column `name` from table `users`_
365480
366481```toml
367482[[actions]]
@@ -373,18 +488,34 @@ column = "name"
373488down = "'N/A'"
374489```
375490
491+_Example: remove `email` column from `users` table and use column from `profiles` table instead_
492+
493+```toml
494+[[actions]]
495+type = "remove_column"
496+table = "users"
497+column = "email"
498+
499+ # Our application will use the `profiles.email` column instead
500+ # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed
501+ [actions.down]
502+ table = "profiles"
503+ value = "profiles.email"
504+ where = "users.id = profiles.user_id"
505+```
506+
376507### Indices
377508
378509#### Add index
379510
380511The `add_index` action will add a new index to an existing table.
381512
382−*Example: create a `users` table with an index on the `name` column*
513+_Example: create a `users` table with a unique index on the `name` column_
383514
384515```toml
385516[[actions]]
386517type = "create_table"
387−table = "users"
518+name = "users"
388519primary_key = "id"
389520
390521 [[actions.columns]]
@@ -399,15 +530,35 @@ primary_key = "id"
399530[[actions]]
400531type = "add_index"
401532table = "users"
402−name = "name_idx"
403−columns = ["name"]
533+
534+ [actions.index]
535+ name = "name_idx"
536+ columns = ["name"]
537+
538+ # Defaults to false
539+ unique = true
540+```
541+
542+_Example: add GIN index to `data` column on `products` table_
543+
544+```toml
545+[[actions]]
546+type = "add_index"
547+table = "products"
548+
549+ [actions.index]
550+ name = "data_idx"
551+ columns = ["data"]
552+
553+ # One of: btree (default), hash, gist, spgist, gin, brin
554+ type = "gin"
404555```
405556
406557#### Remove index
407558
408559The `remove_index` action will remove an existing index. The index won't actually be removed until the migration is completed.
409560
410−*Example: remove the `name_idx` index*
561+_Example: remove the `name_idx` index_
411562
412563```toml
413564[[actions]]
@@ -415,61 +566,205 @@ type = "remove_index"
415566index = "name_idx"
416567```
417568
569+### Enums
570+
571+#### Create enum
572+
573+The `create_enum` action will create a new [enum type](https://www.postgresql.org/docs/current/datatype-enum.html) with the specified values.
574+
575+_Example: add a new `mood` enum type with three possible values_
576+
577+```toml
578+[[actions]]
579+type = "create_enum"
580+name = "mood"
581+values = ["happy", "ok", "sad"]
582+```
583+
584+#### Remove enum
585+
586+The `remove_enum` action will remove an existing [enum type](https://www.postgresql.org/docs/current/datatype-enum.html). Make sure all usages of the enum has been removed before running the migration. The enum will only be removed once the migration is completed.
587+
588+_Example: remove the `mood` enum type_
589+
590+```toml
591+[[actions]]
592+type = "remove_enum"
593+enum = "mood"
594+```
595+
596+### Custom
597+
598+The `custom` action lets you create a migration which runs custom SQL. It should be used with great care as it provides no guarantees of zero-downtime and will simply run whatever SQL is provided. Use other actions whenever possible as they are explicitly designed for zero downtime.
599+
600+There are three optional settings available which all accept SQL queries. All queries need to be idempotent, for example by using `IF NOT EXISTS` wherever available.
601+
602+- `start`: run when a migration is started using `reshape migration start`
603+- `complete`: run when a migration is completed using `reshape migration complete`
604+- `abort`: run when a migration is aborted using `reshape migration abort`
605+
606+_Example: enable PostGIS and pg_stat_statements extensions_
607+
608+```toml
609+[[actions]]
610+type = "custom"
611+
612+start = """
613+ CREATE EXTENSION IF NOT EXISTS postgis;
614+ CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
615+"""
616+
617+abort = """
618+ DROP EXTENSION IF EXISTS postgis;
619+ DROP EXTENSION IF EXISTS pg_stat_statements;
620+"""
621+```
622+
623+### Complex changes across tables
624+
625+The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.
626+
627+_Example: move `email` column from `users` to `profiles` table_
628+
629+```toml
630+[[actions]]
631+type = "add_column"
632+table = "profiles"
633+
634+ [actions.column]
635+ name = "email"
636+ type = "TEXT"
637+ nullable = false
638+
639+ # When `users` is updated in the old schema, we write the email value to `profiles`
640+ [actions.up]
641+ table = "users"
642+ value = "users.email"
643+ where = "profiles.user_id = users.id"
644+
645+[[actions]]
646+type = "remove_column"
647+table = "users"
648+column = "email"
649+
650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
651+ [actions.down]
652+ table = "profiles"
653+ value = "profiles.email"
654+ where = "users.id = profiles.user_id"
655+```
656+
657+_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_
658+
659+```toml
660+# Add `user_account_connections` as a junction table
661+[[actions]]
662+type = "create_table"
663+name = "user_account_connections"
664+primary_key = ["account_id", "user_id"]
665+
666+ [[actions.columns]]
667+ name = "account_id"
668+ type = "INTEGER"
669+
670+ [[actions.columns]]
671+ name = "user_id"
672+ type = "INTEGER"
673+
674+ # `role` is currently stored directly on the `users` table but is part of the relationship
675+ [[actions.columns]]
676+ name = "role"
677+ type = "TEXT"
678+ nullable = false
679+
680+ # Backfill the new table from `users` and uppercase the `role`
681+ [actions.up]
682+ table = "users"
683+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
684+ where = "user_account_connections.user_id = users.id"
685+
686+[[actions]]
687+type = "remove_column"
688+table = "users"
689+column = "account_id"
690+
691+ # When `user_account_connections` is updated, we write the `account_id` back to `users`
692+ [actions.down]
693+ table = "user_account_connections"
694+ value = "user_account_connections.account_id"
695+ where = "users.id = user_account_connections.user_id"
696+
697+[[actions]]
698+type = "remove_column"
699+table = "users"
700+column = "account_role"
701+
702+ # When `user_account_connections` is updated, we write the lowercase role back to `users`
703+ [actions.down]
704+ table = "user_account_connections"
705+ value = "LOWER(user_account_connections.role)"
706+ where = "users.id = user_account_connections.user_id"
707+```
708+
418709## Commands and options
419710
420−### `reshape migrate`
711+### `reshape migration start`
421712
422−Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape complete`.
713+Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape migration complete`.
423714
424715#### Options
425716
426−*See also [Connection options](#connection-options)*
717+_See also [Connection options](#connection-options)_
427718
428−| Option | Default | Description |
429−| ------ | ------- | ----------- |
430−| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
431−| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
719+| Option | Default | Description |
720+| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------------- |
721+| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
722+| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
432723
433−### `reshape complete`
724+### `reshape migration complete`
434725
435−Completes migrations previously started with `reshape complete`.
726+Completes migrations previously started with `reshape migration complete`.
436727
437728#### Options
438729
439730See [Connection options](#connection-options)
440731
441−### `reshape abort`
732+### `reshape migration abort`
442733
443−Aborts any migrations which haven't yet been completed.
734+Aborts any migrations which haven't yet been completed.
444735
445736#### Options
446737
447738See [Connection options](#connection-options)
448739
449−### `reshape generate-schema-query`
740+### `reshape schema-query`
450741
451742Generates the SQL query you need to run in your application before using the database. This command does not require a database connection. Instead it will generate the query based on the latest migration in the `migrations/` directory (or the directories specified by `--dirs`).
452743
744+If your application is written in Rust, we recommend using the [Rust helper library](https://github.com/fabianlindfors/reshape-helper/) instead.
745+
453746The query should look something like `SET search_path TO migration_1_initial_migration`.
454747
455748#### Options
456749
457−| Option | Default | Description |
458−| ------ | ------- | ----------- |
750+| Option | Default | Description |
751+| -------- | ------------- | --------------------------------------------------------------------------------------------------------------- |
459752| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
460753
461754### Connection options
462755
463−The options below can be used with all commands that communicate with Postgres. Use either a connection URL or specify each connection option individually.
756+The options below can be used with all commands that communicate with Postgres. Use either a [connection URL](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) or specify each connection option individually.
757+
758+All options can also be set using environment variables instead of flags. If a `.env` file exists, then variables will be automatically loaded from there.
464759
465−| Option | Default | Description |
466−| ------ | ------- | ----------- |
467−| `--url` | | URI to connect to your Postgres database.<br>Can also be provided with the environment variable `DATABASE_URL`. |
468−| `--host` | `localhost` | Hostname to use when connecting to Postgres |
469−| `--port` | `5432` | Port which Postgres is listening on |
470−| `--database` | `postgres` | Database name |
471−| `--username` | `postgres` | Postgres username |
472−| `--password` | `postgres` | Postgres password |
760+| Option | Default | Environment variable | Description |
761+| ------------ | ----------- | -------------------- | ------------------------------------------- |
762+| `--url` | | `DB_URL` | URL to your Postgres database |
763+| `--host` | `localhost` | `DB_HOST` | Hostname to use when connecting to Postgres |
764+| `--port` | `5432` | `DB_PORT` | Port which Postgres is listening on |
765+| `--database` | `postgres` | `DB_NAME` | Database name |
766+| `--username` | `postgres` | `DB_USERNAME` | Postgres username |
767+| `--password` | `postgres` | `DB_PASSWORD` | Postgres password |
473768
474769## License
475770
src/db.rs+139−6
@@ -1,4 +1,87 @@
1+use std::{cmp::min, time::Duration};
2+
3+use anyhow::{anyhow, Context};
14use postgres::{types::ToSql, NoTls, Row};
5+use rand::prelude::*;
6+
7+// DbLocker wraps a regular DbConn, only allowing access using the
8+// `lock` method. This method will acquire the advisory lock before
9+// allowing access to the database, and then release it afterwards.
10+//
11+// We use advisory locks to avoid multiple Reshape instances working
12+// on the same database as the same time. DbLocker is the only way to
13+// get a DbConn which ensures that all database access is protected by
14+// a lock.
15+//
16+// Postgres docs on advisory locks:
17+// https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS
18+pub struct DbLocker {
19+ client: DbConn,
20+}
21+
22+impl DbLocker {
23+ // Advisory lock keys in Postgres are 64-bit integers.
24+ // The key we use was chosen randomly.
25+ const LOCK_KEY: i64 = 4036779288569897133;
26+
27+ pub fn connect(config: &postgres::Config) -> anyhow::Result<Self> {
28+ let mut pg = config.connect(NoTls)?;
29+
30+ // When running DDL queries that acquire locks, we risk causing a "lock queue".
31+ // When attempting to acquire a lock, Postgres will wait for any long running queries to complete.
32+ // At the same time, it will block other queries until the lock has been acquired and released.
33+ // This has the bad effect of the long-running query blocking other queries because of us, forming
34+ // a queue of other queries until we release our lock.
35+ //
36+ // We set the lock_timeout setting to avoid this. This puts an upper bound for how long Postgres will
37+ // wait to acquire locks and also the maximum amount of time a long-running query can block other queries.
38+ // We should also add automatic retries to handle these timeouts gracefully.
39+ //
40+ // Reference: https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680
41+ //
42+ // TODO: Make lock_timeout configurable
43+ pg.simple_query("SET lock_timeout = '1s'")
44+ .context("failed to set lock_timeout")?;
45+
46+ Ok(Self {
47+ client: DbConn::new(pg),
48+ })
49+ }
50+
51+ pub fn lock(
52+ &mut self,
53+ f: impl FnOnce(&mut DbConn) -> anyhow::Result<()>,
54+ ) -> anyhow::Result<()> {
55+ self.acquire_lock()?;
56+ let result = f(&mut self.client);
57+ self.release_lock()?;
58+
59+ result
60+ }
61+
62+ fn acquire_lock(&mut self) -> anyhow::Result<()> {
63+ let success = self
64+ .client
65+ .query(&format!("SELECT pg_try_advisory_lock({})", Self::LOCK_KEY))?
66+ .first()
67+ .ok_or_else(|| anyhow!("unexpectedly failed when acquiring advisory lock"))
68+ .map(|row| row.get::<'_, _, bool>(0))?;
69+
70+ if success {
71+ Ok(())
72+ } else {
73+ Err(anyhow!("another instance of Reshape is already running"))
74+ }
75+ }
76+
77+ fn release_lock(&mut self) -> anyhow::Result<()> {
78+ self.client
79+ .query(&format!("SELECT pg_advisory_unlock({})", Self::LOCK_KEY))?
80+ .first()
81+ .ok_or_else(|| anyhow!("unexpectedly failed when releasing advisory lock"))?;
82+ Ok(())
83+ }
84+}
285
386pub trait Conn {
487 fn run(&mut self, query: &str) -> anyhow::Result<()>;
@@ -16,20 +99,19 @@ pub struct DbConn {
1699}
17100
18101impl DbConn {
19− pub fn connect(config: &postgres::Config) -> anyhow::Result<DbConn> {
20− let client = config.connect(NoTls)?;
21− Ok(DbConn { client })
102+ fn new(client: postgres::Client) -> Self {
103+ DbConn { client }
22104 }
23105}
24106
25107impl Conn for DbConn {
26108 fn run(&mut self, query: &str) -> anyhow::Result<()> {
27− self.client.batch_execute(query)?;
109+ retry_automatically(|| self.client.batch_execute(query))?;
28110 Ok(())
29111 }
30112
31113 fn query(&mut self, query: &str) -> anyhow::Result<Vec<Row>> {
32− let rows = self.client.query(query, &[])?;
114+ let rows = retry_automatically(|| self.client.query(query, &[]))?;
33115 Ok(rows)
34116 }
35117
@@ -38,7 +120,7 @@ impl Conn for DbConn {
38120 query: &str,
39121 params: &[&(dyn ToSql + Sync)],
40122 ) -> anyhow::Result<Vec<Row>> {
41− let rows = self.client.query(query, params)?;
123+ let rows = retry_automatically(|| self.client.query(query, params))?;
42124 Ok(rows)
43125 }
44126
@@ -89,3 +171,54 @@ impl Conn for Transaction<'_> {
89171 Ok(Transaction { transaction })
90172 }
91173}
174+
175+// Retry a database operation with exponential backoff and jitter
176+fn retry_automatically<T>(
177+ mut f: impl FnMut() -> Result<T, postgres::Error>,
178+) -> Result<T, postgres::Error> {
179+ const STARTING_WAIT_TIME: u64 = 100;
180+ const MAX_WAIT_TIME: u64 = 3_200;
181+ const MAX_ATTEMPTS: u32 = 10;
182+
183+ let mut rng = rand::thread_rng();
184+ let mut attempts = 0;
185+ loop {
186+ let result = f();
187+
188+ let error = match result {
189+ Ok(_) => return result,
190+ Err(err) => err,
191+ };
192+
193+ // If we got a database error, we check if it's retryable.
194+ // If we didn't get a database error, then it's most likely some kind of connection
195+ // error which should also be retried.
196+ if let Some(db_error) = error.as_db_error() {
197+ if !error_retryable(db_error) {
198+ return Err(error);
199+ }
200+ }
201+
202+ attempts += 1;
203+ if attempts >= MAX_ATTEMPTS {
204+ return Err(error);
205+ }
206+
207+ // The wait time increases exponentially, starting at 100ms and doubling up to a max of 3.2s.
208+ let wait_time = min(
209+ MAX_WAIT_TIME,
210+ STARTING_WAIT_TIME * u64::pow(2, attempts - 1),
211+ );
212+
213+ // The jitter is up to half the wait time
214+ let jitter: u64 = rng.gen_range(0..wait_time / 2);
215+
216+ std::thread::sleep(Duration::from_millis(wait_time + jitter));
217+ }
218+}
219+
220+// Check if a database error can be retried
221+fn error_retryable(error: &postgres::error::DbError) -> bool {
222+ // LOCK_NOT_AVAILABLE is caused by lock_timeout being exceeded
223+ matches!(error.code(), &postgres::error::SqlState::LOCK_NOT_AVAILABLE)
224+}
src/helpers.rs+7−16
@@ -2,37 +2,28 @@ use anyhow::Context;
22
33use crate::db::Conn;
44
5−pub fn set_up_helpers(db: &mut dyn Conn, current_migration: &Option<String>) -> anyhow::Result<()> {
6− let predicate = if let Some(current_migration) = current_migration {
7− format!(
8− "current_setting('search_path') = 'migration_{}' OR setting_bool",
9− current_migration
10− )
11− } else {
12− "setting_bool".to_string()
13− };
14−
5+pub fn set_up_helpers(db: &mut dyn Conn, target_migration: &str) -> anyhow::Result<()> {
156 let query = format!(
167 "
17− CREATE OR REPLACE FUNCTION reshape.is_old_schema()
8+ CREATE OR REPLACE FUNCTION reshape.is_new_schema()
189 RETURNS BOOLEAN AS $$
1910 DECLARE
20− setting TEXT := current_setting('reshape.is_old_schema', TRUE);
11+ setting TEXT := current_setting('reshape.is_new_schema', TRUE);
2112 setting_bool BOOLEAN := setting IS NOT NULL AND setting = 'YES';
2213 BEGIN
23− RETURN {};
14+ RETURN current_setting('search_path') = 'migration_{}' OR setting_bool;
2415 END
2516 $$ language 'plpgsql';
2617 ",
27− predicate
18+ target_migration,
2819 );
2920 db.query(&query)
30− .context("failed creating helper function reshape.is_old_schema()")?;
21+ .context("failed creating helper function reshape.is_new_schema()")?;
3122
3223 Ok(())
3324}
3425
3526pub fn tear_down_helpers(db: &mut dyn Conn) -> anyhow::Result<()> {
36− db.query("DROP FUNCTION IF EXISTS reshape.is_old_schema;")?;
27+ db.query("DROP FUNCTION IF EXISTS reshape.is_new_schema;")?;
3728 Ok(())
3829}
src/lib.rs+414−427
@@ -5,7 +5,7 @@ use crate::{
55
66use anyhow::{anyhow, Context};
77use colored::*;
8−use db::{Conn, DbConn};
8+use db::{Conn, DbConn, DbLocker};
99use postgres::Config;
1010use schema::Table;
1111
@@ -15,11 +15,10 @@ pub mod migrations;
1515mod schema;
1616mod state;
1717
18−pub use crate::state::{State, Status};
18+pub use crate::state::State;
1919
2020pub struct Reshape {
21− pub state: State,
22− db: DbConn,
21+ db: DbLocker,
2322}
2423
2524impl Reshape {
@@ -31,6 +30,7 @@ impl Reshape {
3130 pub fn new_with_options(
3231 host: &str,
3332 port: u16,
33+ database: &str,
3434 username: &str,
3535 password: &str,
3636 ) -> anyhow::Result<Reshape> {
@@ -39,501 +39,488 @@ impl Reshape {
3939 .host(host)
4040 .port(port)
4141 .user(username)
42+ .dbname(database)
4243 .password(password);
4344
4445 Self::new_with_config(&config)
4546 }
4647
4748 fn new_with_config(config: &Config) -> anyhow::Result<Reshape> {
48− let mut db = DbConn::connect(config)?;
49− let state = State::load(&mut db);
50−
51− Ok(Reshape { db, state })
49+ let db = DbLocker::connect(config)?;
50+ Ok(Reshape { db })
5251 }
5352
54− pub fn migrate<T>(&mut self, migrations: T) -> anyhow::Result<()>
55− where
56− T: IntoIterator<Item = Migration>,
57− {
58− self.state = State::load(&mut self.db);
53+ pub fn migrate(
54+ &mut self,
55+ migrations: impl IntoIterator<Item = Migration>,
56+ ) -> anyhow::Result<()> {
57+ self.db.lock(|db| {
58+ let mut state = State::load(db)?;
59+ migrate(db, &mut state, migrations)
60+ })
61+ }
5962
60− // Make sure no migration is in progress
61− if let state::Status::InProgress { .. } = &self.state.status {
62− println!("Migration already in progress, please complete using 'reshape complete'");
63− return Ok(());
64− }
63+ pub fn complete(&mut self) -> anyhow::Result<()> {
64+ self.db.lock(|db| {
65+ let mut state = State::load(db)?;
66+ complete(db, &mut state)
67+ })
68+ }
6569
66− if let state::Status::Completing { .. } = &self.state.status {
67− println!(
68− "Migration already in progress and has started completion, please finish using 'reshape complete'"
69− );
70− return Ok(());
71− }
70+ pub fn abort(&mut self) -> anyhow::Result<()> {
71+ self.db.lock(|db| {
72+ let mut state = State::load(db)?;
73+ abort(db, &mut state)
74+ })
75+ }
7276
73− // Determine which migrations need to be applied by comparing the provided migrations
74− // with the already applied ones stored in the state. This will throw an error if the
75− // two sets of migrations don't agree, for example if a new migration has been added
76− // in between two existing ones.
77− let current_migration = state::current_migration(&mut self.db)?;
78− let remaining_migrations = state::remaining_migrations(&mut self.db, migrations)?;
79− if remaining_migrations.is_empty() {
80− println!("No migrations left to apply");
81− return Ok(());
82− }
77+ pub fn remove(&mut self) -> anyhow::Result<()> {
78+ self.db.lock(|db| {
79+ let mut state = State::load(db)?;
8380
84− // If we have already started applying some migrations we need to ensure that
85− // they are the same ones we want to apply now
86− if let state::Status::Applying {
87− migrations: existing_migrations,
88− } = &self.state.status
89− {
90− if existing_migrations != &remaining_migrations {
91− return Err(anyhow!(
92− "a previous migration seems to have failed without cleaning up. Please run `reshape abort` and then run migrate again."
93− ));
81+ // Remove migration schemas and views
82+ if let Some(current_migration) = &state::current_migration(db)? {
83+ db.run(&format!(
84+ "DROP SCHEMA IF EXISTS {} CASCADE",
85+ schema_name_for_migration(current_migration)
86+ ))?;
9487 }
95− }
96−
97− // Move to the "Applying" state which is necessary as we can't run the migrations
98− // and state update as a single transaction. If a migration unexpectedly fails without
99− // automatically aborting, this state saves us from dangling migrations. It forces the user
100− // to either run migrate again (which works as all migrations are idempotent) or abort.
101− self.state.applying(remaining_migrations.clone());
102− self.state.save(&mut self.db)?;
103−
104− println!("Applying {} migrations\n", remaining_migrations.len());
10588
106− helpers::set_up_helpers(&mut self.db, &current_migration)
107− .context("failed to set up helpers")?;
89+ if let State::InProgress { migrations } = &state {
90+ let target_migration = migrations.last().unwrap().name.to_string();
91+ db.run(&format!(
92+ "DROP SCHEMA IF EXISTS {} CASCADE",
93+ schema_name_for_migration(&target_migration)
94+ ))?;
95+ }
10896
109− let mut new_schema = Schema::new();
110− let mut last_migration_index = usize::MAX;
111− let mut last_action_index = usize::MAX;
112− let mut result: anyhow::Result<()> = Ok(());
97+ // Remove all tables
98+ let schema = Schema::new();
99+ for table in schema.get_tables(db)? {
100+ db.run(&format!(
101+ r#"
102+ DROP TABLE IF EXISTS "{}" CASCADE
103+ "#,
104+ table.real_name
105+ ))?;
106+ }
113107
114− for (migration_index, migration) in remaining_migrations.iter().enumerate() {
115− println!("Migrating '{}':", migration.name);
116− last_migration_index = migration_index;
108+ // Remove all enums
109+ let enums: Vec<String> = db
110+ .query("SELECT typname FROM pg_type WHERE typcategory = 'E'")?
111+ .iter()
112+ .map(|row| row.get("typname"))
113+ .collect();
114+ for enum_type in enums {
115+ db.run(&format!("DROP TYPE {}", enum_type))?;
116+ }
117117
118− for (action_index, action) in migration.actions.iter().enumerate() {
119− last_action_index = action_index;
118+ // Reset state
119+ state.clear(db)?;
120120
121− let description = action.describe();
122− print!(" + {} ", description);
121+ println!("Reshape and all data has been removed");
123122
124− let ctx = MigrationContext::new(migration_index, action_index);
125− result = action
126− .run(&ctx, &mut self.db, &new_schema)
127− .with_context(|| format!("failed to {}", description));
123+ Ok(())
124+ })
125+ }
126+}
128127
129− if result.is_ok() {
130− action.update_schema(&ctx, &mut new_schema);
131− println!("{}", "done".green());
132− } else {
133− println!("{}", "failed".red());
134− break;
135− }
136− }
128+pub fn latest_schema_from_migrations(migrations: &[Migration]) -> Option<String> {
129+ migrations
130+ .last()
131+ .map(|migration| schema_name_for_migration(&migration.name))
132+}
137133
138− println!();
139− }
134+pub fn schema_query_for_migration(migration_name: &str) -> String {
135+ let schema_name = schema_name_for_migration(migration_name);
136+ format!("SET search_path TO {}", schema_name)
137+}
140138
141− // If a migration failed, we abort all the migrations that were applied
142− if let Err(err) = result {
143− println!("A migration failed, aborting migrations that have already been applied");
139+fn schema_name_for_migration(migration_name: &str) -> String {
140+ format!("migration_{}", migration_name)
141+}
144142
145− // Set to the Aborting state. This is to ensure that the failed
146− // migration is fully aborted and nothing is left dangling.
147− // If the abort is interrupted for any reason, the user can try again
148− // by running `reshape abort`.
149− self.state.aborting(
150− remaining_migrations.clone(),
151− last_migration_index + 1,
152− last_action_index + 1,
153− );
143+fn migrate(
144+ db: &mut DbConn,
145+ state: &mut State,
146+ migrations: impl IntoIterator<Item = Migration>,
147+) -> anyhow::Result<()> {
148+ // Make sure no migration is in progress
149+ if let State::InProgress { .. } = &state {
150+ println!(
151+ "Migration already in progress, please complete using 'reshape migration complete'"
152+ );
153+ return Ok(());
154+ }
154155
155− // Abort will only
156− self.abort()?;
156+ if let State::Completing { .. } = &state {
157+ println!(
158+ "Migration already in progress and has started completion, please finish using 'reshape migration complete'"
159+ );
160+ return Ok(());
161+ }
157162
158− return Err(err);
159− }
163+ // Determine which migrations need to be applied by comparing the provided migrations
164+ // with the already applied ones stored in the state. This will throw an error if the
165+ // two sets of migrations don't agree, for example if a new migration has been added
166+ // in between two existing ones.
167+ let remaining_migrations = state::remaining_migrations(db, migrations)?;
168+ if remaining_migrations.is_empty() {
169+ println!("No migrations left to apply");
170+ return Ok(());
171+ }
160172
161− // Create schema and views for migration
162− let target_migration = remaining_migrations.last().unwrap().name.to_string();
163− self.create_schema_for_migration(&target_migration, &new_schema)
164− .with_context(|| {
165− format!("failed to create schema for migration {}", target_migration)
166− })?;
167−
168− // Update state once migrations have been performed
169− self.state.in_progress(remaining_migrations);
170− self.state
171− .save(&mut self.db)
172− .context("failed to save in-progress state")?;
173−
174− // If we started from a blank slate, we can finish the migration immediately
175− if current_migration.is_none() {
176− println!("Automatically completing migrations\n");
177− self.complete_migration()
178− .context("failed to automatically complete migrations")?;
179−
180− println!("Migrations complete:");
181− println!(
182− " - Run '{}' from your application to use the latest schema",
183− schema_query_for_migration(&target_migration)
184− );
185− } else {
186− println!("Migrations have been applied and the new schema is ready for use:");
187− println!(
188− " - Run '{}' from your application to use the latest schema",
189− schema_query_for_migration(&target_migration)
190− );
191− println!(
192− " - Run 'reshape complete' once your application has been updated and the previous schema is no longer in use"
193− );
173+ // If we have already started applying some migrations we need to ensure that
174+ // they are the same ones we want to apply now
175+ if let State::Applying {
176+ migrations: existing_migrations,
177+ } = &state
178+ {
179+ if existing_migrations != &remaining_migrations {
180+ return Err(anyhow!(
181+ "a previous migration seems to have failed without cleaning up. Please run `reshape migration abort` and then run migrate again."
182+ ));
194183 }
195−
196− Ok(())
197184 }
198185
199− pub fn complete_migration(&mut self) -> anyhow::Result<()> {
200− // Make sure a migration is in progress
201− let (remaining_migrations, starting_migration_index, starting_action_index) = match self.state.status.clone() {
202− state::Status::InProgress { migrations } => {
203− // Move into the Completing state. Once in this state,
204− // the migration can't be aborted and must be completed.
205− self.state.completing(migrations.clone(), 0, 0);
206− self.state.save(&mut self.db).context("failed to save state")?;
207−
208− (migrations, 0, 0)
209− },
210− state::Status::Completing {
211− migrations,
212− current_migration_index,
213− current_action_index
214− } => (migrations, current_migration_index, current_action_index),
215− state::Status::Aborting { .. } => {
216− return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape abort`."))
217− }
218− state::Status::Applying { .. } => {
219− return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
220− }
221− state::Status::Idle => {
222− println!("No migration in progress");
223− return Ok(());
186+ // Move to the "Applying" state which is necessary as we can't run the migrations
187+ // and state update as a single transaction. If a migration unexpectedly fails without
188+ // automatically aborting, this state saves us from dangling migrations. It forces the user
189+ // to either run migrate again (which works as all migrations are idempotent) or abort.
190+ state.applying(remaining_migrations.clone());
191+ state.save(db)?;
192+
193+ println!("Applying {} migrations\n", remaining_migrations.len());
194+
195+ let target_migration = remaining_migrations.last().unwrap().name.to_string();
196+ helpers::set_up_helpers(db, &target_migration).context("failed to set up helpers")?;
197+
198+ let mut new_schema = Schema::new();
199+ let mut last_migration_index = usize::MAX;
200+ let mut last_action_index = usize::MAX;
201+ let mut result: anyhow::Result<()> = Ok(());
202+
203+ 'outer: for (migration_index, migration) in remaining_migrations.iter().enumerate() {
204+ println!("Migrating '{}':", migration.name);
205+ last_migration_index = migration_index;
206+
207+ for (action_index, action) in migration.actions.iter().enumerate() {
208+ last_action_index = action_index;
209+
210+ let description = action.describe();
211+ print!(" + {} ", description);
212+
213+ let ctx =
214+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
215+ result = action
216+ .run(&ctx, db, &new_schema)
217+ .with_context(|| format!("failed to {}", description));
218+
219+ if result.is_ok() {
220+ action.update_schema(&ctx, &mut new_schema);
221+ println!("{}", "done".green());
222+ } else {
223+ println!("{}", "failed".red());
224+ break 'outer;
224225 }
225− };
226−
227− // Remove previous migration's schema
228− if let Some(current_migration) = &state::current_migration(&mut self.db)? {
229− self.db
230− .run(&format!(
231− "DROP SCHEMA IF EXISTS {} CASCADE",
232− schema_name_for_migration(current_migration)
233− ))
234− .context("failed to remove previous migration's schema")?;
235226 }
236227
237− for (migration_index, migration) in remaining_migrations.iter().enumerate() {
238− // Skip all the migrations which have already been completed
239− if migration_index < starting_migration_index {
240− continue;
241− }
242−
243− println!("Completing '{}':", migration.name);
228+ println!();
229+ }
244230
245− for (action_index, action) in migration.actions.iter().enumerate() {
246− // Skip all actions which have already been completed
247− if migration_index == starting_migration_index
248− && action_index < starting_action_index
249− {
250− continue;
251− }
231+ // If a migration failed, we abort all the migrations that were applied
232+ if let Err(err) = result {
233+ println!("A migration failed, aborting migrations that have already been applied");
252234
253− let description = action.describe();
254− print!(" + {} ", description);
235+ // Set to the Aborting state. This is to ensure that the failed
236+ // migration is fully aborted and nothing is left dangling.
237+ // If the abort is interrupted for any reason, the user can try again
238+ // by running `reshape migration abort`.
239+ state.aborting(
240+ remaining_migrations.clone(),
241+ last_migration_index + 1,
242+ last_action_index + 1,
243+ );
255244
256− let ctx = MigrationContext::new(migration_index, action_index);
245+ // Abort will only
246+ abort(db, state)?;
257247
258− // Update state to indicate that this action has been completed.
259− // We won't save this new state until after the action has completed.
260− self.state.completing(
261− remaining_migrations.clone(),
262− migration_index + 1,
263− action_index + 1,
264− );
248+ return Err(err);
249+ }
265250
266− // This did_save check is necessary because of the borrow checker.
267− // The Transaction which might be returned from action.complete
268− // contains a mutable reference to self.db. We need the Transaction
269− // to be dropped before we can save the state using self.db instead,
270− // which we achieve here by limiting the lifetime of the Transaction
271− // with a new block.
272− let did_save = {
273− let result = action
274− .complete(&ctx, &mut self.db)
275− .with_context(|| format!("failed to complete migration {}", migration.name))
276− .with_context(|| format!("failed to complete action: {}", description));
277−
278− let maybe_transaction = match result {
279− Ok(maybe_transaction) => {
280− println!("{}", "done".green());
281− maybe_transaction
282− }
283− Err(e) => {
284− println!("{}", "failed".red());
285− return Err(e);
286− }
287− };
288−
289− // Update state with which migrations and actions have been completed.
290− // Each action can create and return a transaction if they need atomicity.
291− // We use this transaction to update the state to ensure the action only completes.
292− // once.
293− // We want to use a single transaction for each action to keep the length of
294− // the transaction as short as possible. Wherever possible, we don't want to
295− // use a transaction at all.
296− if let Some(mut transaction) = maybe_transaction {
297− self.state
298− .save(&mut transaction)
299− .context("failed to save state after completing action")?;
300− transaction
301− .commit()
302− .context("failed to commit transaction")?;
303−
304− true
305− } else {
306− false
307− }
308− };
251+ // Create schema and views for migration
252+ create_schema_for_migration(db, &target_migration, &new_schema)
253+ .with_context(|| format!("failed to create schema for migration {}", target_migration))?;
254+
255+ // Update state once migrations have been performed
256+ state.in_progress(remaining_migrations);
257+ state.save(db).context("failed to save in-progress state")?;
258+
259+ println!("Migrations have been applied and the new schema is ready for use:");
260+ println!(
261+ " - Run '{}' from your application to use the latest schema",
262+ schema_query_for_migration(&target_migration)
263+ );
264+ println!(
265+ " - Run 'reshape migration complete' once your application has been updated and the previous schema is no longer in use"
266+ );
267+ Ok(())
268+}
309269
310− // If the action didn't return a transaction we save the state normally instead
311− if !did_save {
312− self.state
313− .save(&mut self.db)
314− .context("failed to save state after completing action")?;
270+fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
271+ // Make sure a migration is in progress
272+ let (remaining_migrations, starting_migration_index, starting_action_index) = match state.clone() {
273+ State::InProgress { migrations } => {
274+ // Move into the Completing state. Once in this state,
275+ // the migration can't be aborted and must be completed.
276+ state.completing(migrations.clone(), 0, 0);
277+ state.save(db).context("failed to save state")?;
278+
279+ (migrations, 0, 0)
280+ },
281+ State::Completing {
282+ migrations,
283+ current_migration_index,
284+ current_action_index
285+ } => (migrations, current_migration_index, current_action_index),
286+ State::Aborting { .. } => {
287+ return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape migration abort`."))
315288 }
316− }
289+ State::Applying { .. } => {
290+ return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
291+ }
292+ State::Idle => {
293+ println!("No migration in progress");
294+ return Ok(());
295+ }
296+ };
317297
318− println!();
298+ // Remove previous migration's schema
299+ if let Some(current_migration) = &state::current_migration(db)? {
300+ db.run(&format!(
301+ "DROP SCHEMA IF EXISTS {} CASCADE",
302+ schema_name_for_migration(current_migration)
303+ ))
304+ .context("failed to remove previous migration's schema")?;
305+ }
306+
307+ for (migration_index, migration) in remaining_migrations.iter().enumerate() {
308+ // Skip all the migrations which have already been completed
309+ if migration_index < starting_migration_index {
310+ continue;
319311 }
320312
321− // Remove helpers which are no longer in use
322− helpers::tear_down_helpers(&mut self.db).context("failed to tear down helpers")?;
313+ println!("Completing '{}':", migration.name);
323314
324− self.state
325− .complete(&mut self.db)
326− .context("failed to update state as completed")?;
315+ for (action_index, action) in migration.actions.iter().enumerate() {
316+ // Skip all actions which have already been completed
317+ if migration_index == starting_migration_index && action_index < starting_action_index {
318+ continue;
319+ }
327320
328− Ok(())
329− }
321+ let description = action.describe();
322+ print!(" + {} ", description);
330323
331− fn create_schema_for_migration(
332− &mut self,
333− migration_name: &str,
334− schema: &Schema,
335− ) -> anyhow::Result<()> {
336− // Create schema for migration
337− let schema_name = schema_name_for_migration(migration_name);
338− self.db
339− .run(&format!("CREATE SCHEMA IF NOT EXISTS {}", schema_name))
340− .with_context(|| {
341− format!(
342− "failed to create schema {} for migration {}",
343− schema_name, migration_name
344− )
345− })?;
346−
347− // Create views inside schema
348− for table in schema.get_tables(&mut self.db)? {
349− Self::create_view_for_table(&mut self.db, &table, &schema_name)?;
350− }
324+ let ctx =
325+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
351326
352− Ok(())
353− }
327+ // Update state to indicate that this action has been completed.
328+ // We won't save this new state until after the action has completed.
329+ state.completing(
330+ remaining_migrations.clone(),
331+ migration_index + 1,
332+ action_index + 1,
333+ );
354334
355− fn create_view_for_table(
356− db: &mut impl Conn,
357− table: &Table,
358− schema: &str,
359− ) -> anyhow::Result<()> {
360− let select_columns: Vec<String> = table
361− .columns
362− .iter()
363− .map(|column| {
364− format!(
365− r#"
366− "{real_name}" AS "{alias}"
367− "#,
368− real_name = column.real_name,
369− alias = column.name,
370− )
371− })
372− .collect();
335+ // This did_save check is necessary because of the borrow checker.
336+ // The Transaction which might be returned from action.complete
337+ // contains a mutable reference to self.db. We need the Transaction
338+ // to be dropped before we can save the state using self.db instead,
339+ // which we achieve here by limiting the lifetime of the Transaction
340+ // with a new block.
341+ let did_save = {
342+ let result = action
343+ .complete(&ctx, db)
344+ .with_context(|| format!("failed to complete migration {}", migration.name))
345+ .with_context(|| format!("failed to complete action: {}", description));
346+
347+ let maybe_transaction = match result {
348+ Ok(maybe_transaction) => {
349+ println!("{}", "done".green());
350+ maybe_transaction
351+ }
352+ Err(e) => {
353+ println!("{}", "failed".red());
354+ return Err(e);
355+ }
356+ };
373357
374− db.run(&format!(
375− r#"
376− CREATE OR REPLACE VIEW {schema}."{view_name}" AS
377− SELECT {columns}
378− FROM "{table_name}"
379− "#,
380− schema = schema,
381− table_name = table.real_name,
382− view_name = table.name,
383− columns = select_columns.join(","),
384− ))
385− .with_context(|| format!("failed to create view for table {}", table.name))?;
358+ // Update state with which migrations and actions have been completed.
359+ // Each action can create and return a transaction if they need atomicity.
360+ // We use this transaction to update the state to ensure the action only completes.
361+ // once.
362+ // We want to use a single transaction for each action to keep the length of
363+ // the transaction as short as possible. Wherever possible, we don't want to
364+ // use a transaction at all.
365+ if let Some(mut transaction) = maybe_transaction {
366+ state
367+ .save(&mut transaction)
368+ .context("failed to save state after completing action")?;
369+ transaction
370+ .commit()
371+ .context("failed to commit transaction")?;
386372
387− Ok(())
388− }
373+ true
374+ } else {
375+ false
376+ }
377+ };
389378
390− pub fn remove(&mut self) -> anyhow::Result<()> {
391− // Remove migration schemas and views
392− if let Some(current_migration) = &state::current_migration(&mut self.db)? {
393− self.db.run(&format!(
394− "DROP SCHEMA IF EXISTS {} CASCADE",
395− schema_name_for_migration(current_migration)
396− ))?;
379+ // If the action didn't return a transaction we save the state normally instead
380+ if !did_save {
381+ state
382+ .save(db)
383+ .context("failed to save state after completing action")?;
384+ }
397385 }
398386
399− if let Status::InProgress { migrations } = &self.state.status {
400− let target_migration = migrations.last().unwrap().name.to_string();
401− self.db.run(&format!(
402− "DROP SCHEMA IF EXISTS {} CASCADE",
403− schema_name_for_migration(&target_migration)
404− ))?;
405− }
387+ println!();
388+ }
406389
407− // Remove all tables
408− let schema = Schema::new();
409− for table in schema.get_tables(&mut self.db)? {
410− self.db.run(&format!(
411− r#"
412− DROP TABLE IF EXISTS "{}" CASCADE
413− "#,
414− table.real_name
415− ))?;
416− }
390+ // Remove helpers which are no longer in use
391+ helpers::tear_down_helpers(db).context("failed to tear down helpers")?;
417392
418− // Reset state
419− self.state.clear(&mut self.db)?;
393+ state
394+ .complete(db)
395+ .context("failed to update state as completed")?;
420396
421− println!("Reshape and all data has been removed");
397+ Ok(())
398+}
422399
423− Ok(())
424− }
400+fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
401+ let (remaining_migrations, last_migration_index, last_action_index) = match state.clone() {
402+ State::InProgress { migrations } | State::Applying { migrations } => {
403+ // Set to the Aborting state. Once this is done, the migration has to
404+ // be fully aborted and can't be completed.
405+ state.aborting(migrations.clone(), 0, 0);
406+ state.save(db)?;
425407
426− pub fn abort(&mut self) -> anyhow::Result<()> {
427− let (remaining_migrations, last_migration_index, last_action_index) = match self
428− .state
429− .status
430− .clone()
431− {
432− Status::InProgress { migrations } | Status::Applying { migrations } => {
433− // Set to the Aborting state. Once this is done, the migration has to
434− // be fully aborted and can't be completed.
435− self.state.aborting(migrations.clone(), 0, 0);
436− self.state.save(&mut self.db)?;
437−
438− (migrations, usize::MAX, usize::MAX)
439− }
440− Status::Aborting {
441− migrations,
442− last_migration_index,
443− last_action_index,
444− } => (migrations, last_migration_index, last_action_index),
445− Status::Completing { .. } => {
446− return Err(anyhow!("Migration completion has already been started. Please run `reshape complete` again to finish it."));
447− }
448− Status::Idle => {
449− println!("No migration is in progress");
450− return Ok(());
451− }
452− };
453−
454− // Remove new migration's schema
455− let target_migration = remaining_migrations.last().unwrap().name.to_string();
456− let schema_name = schema_name_for_migration(&target_migration);
457− self.db
458− .run(&format!("DROP SCHEMA IF EXISTS {} CASCADE", schema_name,))
459− .with_context(|| format!("failed to drop schema {}", schema_name))?;
460−
461− // Abort all pending migrations
462− self.abort_migrations(
463− &remaining_migrations,
408+ (migrations, usize::MAX, usize::MAX)
409+ }
410+ State::Aborting {
411+ migrations,
464412 last_migration_index,
465413 last_action_index,
466− )?;
467−
468− helpers::tear_down_helpers(&mut self.db).context("failed to tear down helpers")?;
469−
470− self.state.status = state::Status::Idle;
471− self.state
472− .save(&mut self.db)
473− .context("failed to save state")?;
414+ } => (migrations, last_migration_index, last_action_index),
415+ State::Completing { .. } => {
416+ return Err(anyhow!("migration completion has already been started. Please run `reshape migration complete` again to finish it."));
417+ }
418+ State::Idle => {
419+ println!("No migration is in progress");
420+ return Ok(());
421+ }
422+ };
423+
424+ // Remove new migration's schema
425+ let target_migration = remaining_migrations.last().unwrap().name.to_string();
426+ let schema_name = schema_name_for_migration(&target_migration);
427+ db.run(&format!("DROP SCHEMA IF EXISTS {} CASCADE", schema_name,))
428+ .with_context(|| format!("failed to drop schema {}", schema_name))?;
429+
430+ // Abort all pending migrations
431+ // Abort all migrations in reverse order
432+ for (migration_index, migration) in remaining_migrations.iter().enumerate().rev() {
433+ // Skip migrations which shouldn't be aborted
434+ // The reason can be that they have already been aborted or that
435+ // the migration was never applied in the first place.
436+ if migration_index >= last_migration_index {
437+ continue;
438+ }
474439
475− Ok(())
476− }
440+ print!("Aborting '{}' ", migration.name);
477441
478− fn abort_migrations(
479− &mut self,
480− migrations: &[Migration],
481− upper_migration_index: usize,
482− upper_action_index: usize,
483− ) -> anyhow::Result<()> {
484− // Abort all migrations in reverse order
485− for (migration_index, migration) in migrations.iter().enumerate().rev() {
486− // Skip migrations which shouldn't be aborted
442+ for (action_index, action) in migration.actions.iter().enumerate().rev() {
443+ // Skip actions which shouldn't be aborted
487444 // The reason can be that they have already been aborted or that
488− // the migration was never applied in the first place.
489− if migration_index >= upper_migration_index {
445+ // the action was never applied in the first place.
446+ if migration_index == last_migration_index - 1 && action_index >= last_action_index {
490447 continue;
491448 }
492449
493− print!("Aborting '{}' ", migration.name);
450+ let ctx =
451+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
452+ action
453+ .abort(&ctx, db)
454+ .with_context(|| format!("failed to abort migration {}", migration.name))
455+ .with_context(|| format!("failed to abort action: {}", action.describe()))?;
456+
457+ // Update state with which migrations and actions have been aborted.
458+ // We don't need to run this in a transaction as aborts are idempotent.
459+ state.aborting(remaining_migrations.to_vec(), migration_index, action_index);
460+ state.save(db).context("failed to save state")?;
461+ }
494462
495− for (action_index, action) in migration.actions.iter().enumerate().rev() {
496− // Skip actions which shouldn't be aborted
497− // The reason can be that they have already been aborted or that
498− // the action was never applied in the first place.
499− if migration_index == upper_migration_index - 1
500− && action_index >= upper_action_index
501− {
502− continue;
503− }
463+ println!("{}", "done".green());
464+ }
504465
505− let ctx = MigrationContext::new(migration_index, action_index);
506− action
507− .abort(&ctx, &mut self.db)
508− .with_context(|| format!("failed to abort migration {}", migration.name))
509− .with_context(|| format!("failed to abort action: {}", action.describe()))?;
510−
511− // Update state with which migrations and actions have been aborted.
512− // We don't need to run this in a transaction as aborts are idempotent.
513− self.state
514− .aborting(migrations.to_vec(), migration_index, action_index);
515− self.state
516− .save(&mut self.db)
517− .context("failed to save state")?;
518− }
466+ helpers::tear_down_helpers(db).context("failed to tear down helpers")?;
519467
520− println!("{}", "done".green());
521− }
522− Ok(())
523− }
524−}
468+ *state = State::Idle;
469+ state.save(db).context("failed to save state")?;
525470
526−pub fn latest_schema_from_migrations(migrations: &[Migration]) -> Option<String> {
527− migrations
528− .last()
529− .map(|migration| schema_name_for_migration(&migration.name))
471+ Ok(())
530472}
531473
532−pub fn schema_query_for_migration(migration_name: &str) -> String {
474+fn create_schema_for_migration(
475+ db: &mut DbConn,
476+ migration_name: &str,
477+ schema: &Schema,
478+) -> anyhow::Result<()> {
479+ // Create schema for migration
533480 let schema_name = schema_name_for_migration(migration_name);
534− format!("SET search_path TO {}", schema_name)
481+ db.run(&format!("CREATE SCHEMA IF NOT EXISTS {}", schema_name))
482+ .with_context(|| {
483+ format!(
484+ "failed to create schema {} for migration {}",
485+ schema_name, migration_name
486+ )
487+ })?;
488+
489+ // Create views inside schema
490+ for table in schema.get_tables(db)? {
491+ create_view_for_table(db, &table, &schema_name)?;
492+ }
493+
494+ Ok(())
535495}
536496
537−fn schema_name_for_migration(migration_name: &str) -> String {
538− format!("migration_{}", migration_name)
497+fn create_view_for_table(db: &mut impl Conn, table: &Table, schema: &str) -> anyhow::Result<()> {
498+ let select_columns: Vec<String> = table
499+ .columns
500+ .iter()
501+ .map(|column| {
502+ format!(
503+ r#"
504+ "{real_name}" AS "{alias}"
505+ "#,
506+ real_name = column.real_name,
507+ alias = column.name,
508+ )
509+ })
510+ .collect();
511+
512+ db.run(&format!(
513+ r#"
514+ CREATE OR REPLACE VIEW {schema}."{view_name}" AS
515+ SELECT {columns}
516+ FROM "{table_name}"
517+ "#,
518+ schema = schema,
519+ table_name = table.real_name,
520+ view_name = table.name,
521+ columns = select_columns.join(","),
522+ ))
523+ .with_context(|| format!("failed to create view for table {}", table.name))?;
524+
525+ Ok(())
539526}
src/main.rs+95−20
@@ -4,6 +4,7 @@ use std::{
44 path::Path,
55};
66
7+use anyhow::Context;
78use clap::{Args, Parser};
89use reshape::{
910 migrations::{Action, Migration},
@@ -12,22 +13,69 @@ use reshape::{
1213use serde::{Deserialize, Serialize};
1314
1415#[derive(Parser)]
16+#[clap(name = "Reshape", version, about)]
1517struct Opts {
1618 #[clap(subcommand)]
1719 cmd: Command,
1820}
1921
2022#[derive(Parser)]
23+#[clap(about)]
2124enum Command {
25+ #[clap(subcommand)]
26+ Migration(MigrationCommand),
27+
28+ #[clap(
29+ about = "Output the query your application should use to select the right schema",
30+ display_order = 2
31+ )]
32+ SchemaQuery(FindMigrationsOptions),
33+
34+ #[clap(
35+ about = "Deprecated. Use `reshape schema-query` instead",
36+ display_order = 3
37+ )]
38+ GenerateSchemaQuery(FindMigrationsOptions),
39+
40+ #[clap(
41+ about = "Deprecated. Use `reshape migration start` instead",
42+ display_order = 4
43+ )]
2244 Migrate(MigrateOptions),
45+ #[clap(
46+ about = "Deprecated. Use `reshape migration complete` instead",
47+ display_order = 5
48+ )]
2349 Complete(ConnectionOptions),
24− Remove(ConnectionOptions),
50+ #[clap(
51+ about = "Deprecated. Use `reshape migration abort` instead",
52+ display_order = 6
53+ )]
54+ Abort(ConnectionOptions),
55+}
56+
57+#[derive(Parser)]
58+#[clap(about = "Commands for managing migrations", display_order = 1)]
59+enum MigrationCommand {
60+ #[clap(
61+ about = "Starts a new migration, applying any migrations which haven't yet been applied",
62+ display_order = 1
63+ )]
64+ Start(MigrateOptions),
65+
66+ #[clap(about = "Completes an in-progress migration", display_order = 2)]
67+ Complete(ConnectionOptions),
68+
69+ #[clap(
70+ about = "Aborts an in-progress migration without losing any data",
71+ display_order = 3
72+ )]
2573 Abort(ConnectionOptions),
26− GenerateSchemaQuery(FindMigrationsOptions),
2774}
2875
2976#[derive(Args)]
3077struct MigrateOptions {
78+ // Some comment
3179 #[clap(long, short)]
3280 complete: bool,
3381 #[clap(flatten)]
@@ -65,32 +113,28 @@ fn main() -> anyhow::Result<()> {
65113
66114fn run(opts: Opts) -> anyhow::Result<()> {
67115 match opts.cmd {
68− Command::Migrate(opts) => {
116+ Command::Migration(MigrationCommand::Start(opts)) | Command::Migrate(opts) => {
69117 let mut reshape = reshape_from_connection_options(&opts.connection_options)?;
70118 let migrations = find_migrations(&opts.find_migrations_options)?;
71119 reshape.migrate(migrations)?;
72120
73121 // Automatically complete migration if --complete flag is set
74122 if opts.complete {
75− reshape.complete_migration()?;
123+ reshape.complete()?;
76124 }
77125
78126 Ok(())
79127 }
80− Command::Complete(opts) => {
81− let mut reshape = reshape_from_connection_options(&opts)?;
82− reshape.complete_migration()
83− }
84− Command::Remove(opts) => {
128+ Command::Migration(MigrationCommand::Complete(opts)) | Command::Complete(opts) => {
85129 let mut reshape = reshape_from_connection_options(&opts)?;
86− reshape.remove()
130+ reshape.complete()
87131 }
88− Command::Abort(opts) => {
132+ Command::Migration(MigrationCommand::Abort(opts)) | Command::Abort(opts) => {
89133 let mut reshape = reshape_from_connection_options(&opts)?;
90134 reshape.abort()
91135 }
92− Command::GenerateSchemaQuery(find_migrations_options) => {
93− let migrations = find_migrations(&find_migrations_options)?;
136+ Command::SchemaQuery(opts) | Command::GenerateSchemaQuery(opts) => {
137+ let migrations = find_migrations(&opts)?;
94138 let query = migrations
95139 .last()
96140 .map(|migration| reshape::schema_query_for_migration(&migration.name));
@@ -102,13 +146,35 @@ fn run(opts: Opts) -> anyhow::Result<()> {
102146}
103147
104148fn reshape_from_connection_options(opts: &ConnectionOptions) -> anyhow::Result<Reshape> {
105− let env_url = std::env::var("POSTGRES_URL").ok();
106− let url = env_url.as_ref().or_else(|| opts.url.as_ref());
149+ // Load environment variables from .env file if it exists
150+ dotenv::dotenv().ok();
107151
108− match url {
109− Some(url) => Reshape::new(url),
110− None => Reshape::new_with_options(&opts.host, opts.port, &opts.username, &opts.password),
152+ let url_env = std::env::var("DB_URL").ok();
153+ let url = url_env.as_ref().or_else(|| opts.url.as_ref());
154+
155+ // Use the connection URL if it has been set
156+ if let Some(url) = url {
157+ return Reshape::new(url);
111158 }
159+
160+ let host_env = std::env::var("DB_HOST").ok();
161+ let host = host_env.as_ref().unwrap_or_else(|| &opts.host);
162+
163+ let port = std::env::var("DB_PORT")
164+ .ok()
165+ .and_then(|port| port.parse::<u16>().ok())
166+ .unwrap_or(opts.port);
167+
168+ let username_env = std::env::var("DB_USERNAME").ok();
169+ let username = username_env.as_ref().unwrap_or_else(|| &opts.username);
170+
171+ let password_env = std::env::var("DB_PASSWORD").ok();
172+ let password = password_env.as_ref().unwrap_or_else(|| &opts.password);
173+
174+ let database_env = std::env::var("DB_NAME").ok();
175+ let database = database_env.as_ref().unwrap_or_else(|| &opts.database);
176+
177+ Reshape::new_with_options(host, port, database, username, password)
112178}
113179
114180fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration>> {
@@ -130,7 +196,13 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
130196 }
131197
132198 // Sort all files by their file names (without extension)
133− file_paths.sort_unstable_by_key(|path| path.as_path().file_stem().unwrap().to_os_string());
199+ // The files are sorted naturally, e.g. "1_test_migration" < "10_test_migration"
200+ file_paths.sort_unstable_by(|path1, path2| {
201+ let file1 = path1.as_path().file_stem().unwrap().to_str().unwrap();
202+ let file2 = path2.as_path().file_stem().unwrap().to_str().unwrap();
203+
204+ lexical_sort::natural_cmp(file1, file2)
205+ });
134206
135207 file_paths
136208 .iter()
@@ -146,7 +218,10 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
146218 .map(|result| {
147219 result.and_then(|(path, data)| {
148220 let extension = path.extension().and_then(|ext| ext.to_str()).unwrap();
149− let file_migration = decode_migration_file(&data, extension)?;
221+ let file_migration =
222+ decode_migration_file(&data, extension).with_context(|| {
223+ format!("failed to parse migration file {}", path.display())
224+ })?;
150225
151226 let file_name = path.file_stem().and_then(|name| name.to_str()).unwrap();
152227 Ok(Migration {
src/migrations/add_column.rs+214−60
@@ -3,14 +3,25 @@ use crate::{
33 db::{Conn, Transaction},
44 schema::Schema,
55};
6−use anyhow::Context;
6+use anyhow::{bail, Context};
77use serde::{Deserialize, Serialize};
88
99#[derive(Serialize, Deserialize, Debug)]
1010pub struct AddColumn {
1111 pub table: String,
12− pub up: Option<String>,
1312 pub column: Column,
13+ pub up: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
1425}
1526
1627impl AddColumn {
@@ -32,6 +43,15 @@ impl AddColumn {
3243 )
3344 }
3445
46+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_add_column_{}_{}_rev",
49+ ctx.prefix(),
50+ self.table,
51+ self.column.name
52+ )
53+ }
54+
3555 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
3656 format!(
3757 "{}_add_column_not_null_{}_{}",
@@ -61,7 +81,7 @@ impl Action for AddColumn {
6181 let temp_column_name = self.temp_column_name(ctx);
6282
6383 let mut definition_parts = vec![
64− temp_column_name.to_string(),
84+ format!("\"{}\"", temp_column_name.to_string()),
6585 self.column.data_type.to_string(),
6686 ];
6787
@@ -86,55 +106,189 @@ impl Action for AddColumn {
86106 );
87107 db.run(&query).context("failed to add column")?;
88108
109+ let declarations: Vec<String> = table
110+ .columns
111+ .iter()
112+ .map(|column| {
113+ format!(
114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",
115+ table = table.real_name,
116+ alias = column.name,
117+ real_name = column.real_name,
118+ )
119+ })
120+ .collect();
121+
89122 if let Some(up) = &self.up {
90− let table = schema.get_table(db, &self.table)?;
91−
92− let declarations: Vec<String> = table
93− .columns
94− .iter()
95− .map(|column| {
96− format!(
97− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
98− table = table.real_name,
99− alias = column.name,
100− real_name = column.real_name,
101− )
102− })
103− .collect();
104−
105− // Add triggers to fill in values as they are inserted/updated
106− let query = format!(
107− r#"
108− CREATE OR REPLACE FUNCTION {trigger_name}()
109− RETURNS TRIGGER AS $$
110− BEGIN
111− IF reshape.is_old_schema() THEN
112− DECLARE
113− {declarations}
114− BEGIN
115− NEW.{temp_column_name} = {up};
116− END;
117− END IF;
118− RETURN NEW;
119− END
120− $$ language 'plpgsql';
121−
122− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
123− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
124− "#,
125− temp_column_name = temp_column_name,
126− trigger_name = self.trigger_name(ctx),
127− up = up,
128− table = self.table,
129− declarations = declarations.join("\n"),
130− );
131− db.run(&query).context("failed to create up trigger")?;
132− }
123+ if let Transformation::Simple(up) = up {
124+ // Add triggers to fill in values as they are inserted/updated
125+ let query = format!(
126+ r#"
127+ CREATE OR REPLACE FUNCTION {trigger_name}()
128+ RETURNS TRIGGER AS $$
129+ BEGIN
130+ IF NOT reshape.is_new_schema() THEN
131+ DECLARE
132+ {declarations}
133+ BEGIN
134+ NEW."{temp_column_name}" = {up};
135+ END;
136+ END IF;
137+ RETURN NEW;
138+ END
139+ $$ language 'plpgsql';
140+
141+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
142+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
143+ "#,
144+ temp_column_name = temp_column_name,
145+ trigger_name = self.trigger_name(ctx),
146+ up = up,
147+ table = self.table,
148+ declarations = declarations.join("\n"),
149+ );
150+ db.run(&query).context("failed to create up trigger")?;
133151
134− // Backfill values in batches
135− if self.up.is_some() {
136− common::batch_touch_rows(db, &table.real_name, &temp_column_name)
137− .context("failed to batch update existing rows")?;
152+ // Backfill values in batches
153+ common::batch_touch_rows(db, &table.real_name, Some(&temp_column_name))
154+ .context("failed to batch update existing rows")?;
155+ }
156+
157+ if let Transformation::Update {
158+ table: from_table,
159+ value,
160+ r#where,
161+ } = up
162+ {
163+ let existing_schema_name = match &ctx.existing_schema_name {
164+ Some(name) => name,
165+ None => bail!("can't use update without previous migration"),
166+ };
167+
168+ let from_table = schema.get_table(db, &from_table)?;
169+
170+ let from_table_assignments: Vec<String> = from_table
171+ .columns
172+ .iter()
173+ .map(|column| {
174+ format!(
175+ "{table}.{alias} = NEW.{real_name};",
176+ table = from_table.name,
177+ alias = column.name,
178+ real_name = column.real_name,
179+ )
180+ })
181+ .collect();
182+
183+ // Add triggers to fill in values as they are inserted/updated
184+ let query = format!(
185+ r#"
186+ CREATE OR REPLACE FUNCTION {trigger_name}()
187+ RETURNS TRIGGER AS $$
188+ #variable_conflict use_variable
189+ BEGIN
190+ IF NOT reshape.is_new_schema() THEN
191+ DECLARE
192+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
193+ BEGIN
194+ {assignments}
195+
196+ -- Don't trigger reverse trigger when making this update
197+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
198+
199+ UPDATE public."{changed_table_real}"
200+ SET "{temp_column_name}" = {value}
201+ WHERE {where};
202+
203+ perform set_config('reshape.disable_triggers', '', TRUE);
204+ END;
205+ END IF;
206+ RETURN NEW;
207+ END
208+ $$ language 'plpgsql';
209+
210+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
211+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
212+ "#,
213+ assignments = from_table_assignments.join("\n"),
214+ changed_table_real = table.real_name,
215+ from_table = from_table.name,
216+ from_table_real = from_table.real_name,
217+ trigger_name = self.trigger_name(ctx),
218+ // declarations = from_table_declarations.join("\n"),
219+ temp_column_name = temp_column_name,
220+ );
221+ db.run(&query).context("failed to create up trigger")?;
222+
223+ let from_table_columns = from_table
224+ .columns
225+ .iter()
226+ .map(|column| format!("{} as {}", column.real_name, column.name))
227+ .collect::<Vec<String>>()
228+ .join(", ");
229+
230+ let changed_table_assignments: Vec<String> = table
231+ .columns
232+ .iter()
233+ .map(|column| {
234+ format!(
235+ "{table}.{alias} := NEW.{real_name};",
236+ table = table.name,
237+ alias = column.name,
238+ real_name = column.real_name,
239+ )
240+ })
241+ .collect();
242+
243+ // Add triggers to fill in values as they are inserted/updated
244+ let query = format!(
245+ r#"
246+ CREATE OR REPLACE FUNCTION {trigger_name}()
247+ RETURNS TRIGGER AS $$
248+ #variable_conflict use_variable
249+ BEGIN
250+ IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
251+ DECLARE
252+ {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;
253+ __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;
254+ BEGIN
255+ {changed_table_assignments}
256+
257+ SELECT {from_table_columns}
258+ INTO __temp_row
259+ FROM migration_{existing_schema_name}.{from_table} {from_table}
260+ WHERE {where};
261+
262+ DECLARE
263+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
264+ BEGIN
265+ {from_table} = __temp_row;
266+ NEW.{temp_column_name} = {value};
267+ END;
268+ END;
269+ END IF;
270+ RETURN NEW;
271+ END
272+ $$ language 'plpgsql';
273+
274+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
275+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
276+ "#,
277+ changed_table_assignments = changed_table_assignments.join("\n"),
278+ changed_table_real = table.real_name,
279+ changed_table = table.name,
280+ from_table = from_table.name,
281+ trigger_name = self.reverse_trigger_name(ctx),
282+ temp_column_name = temp_column_name,
283+ // declarations = declarations.join("\n"),
284+ );
285+ db.run(&query)
286+ .context("failed to create reverse up trigger")?;
287+
288+ // Backfill values in batches by touching the from table
289+ common::batch_touch_rows(db, &from_table.real_name, None)
290+ .context("failed to batch update existing rows")?;
291+ }
138292 }
139293
140294 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
@@ -144,10 +298,10 @@ impl Action for AddColumn {
144298 if !self.column.nullable {
145299 let query = format!(
146300 r#"
147− ALTER TABLE "{table}"
148− ADD CONSTRAINT "{constraint_name}"
149− CHECK ("{column}" IS NOT NULL) NOT VALID
150− "#,
301+ ALTER TABLE "{table}"
302+ ADD CONSTRAINT "{constraint_name}"
303+ CHECK ("{column}" IS NOT NULL) NOT VALID
304+ "#,
151305 table = self.table,
152306 constraint_name = self.not_null_constraint_name(ctx),
153307 column = temp_column_name,
@@ -169,11 +323,11 @@ impl Action for AddColumn {
169323 // Remove triggers and procedures
170324 let query = format!(
171325 r#"
172− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
173− DROP FUNCTION IF EXISTS "{trigger_name}";
326+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
327+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
174328 "#,
175− table = self.table,
176329 trigger_name = self.trigger_name(ctx),
330+ reverse_trigger_name = self.reverse_trigger_name(ctx),
177331 );
178332 transaction
179333 .run(&query)
@@ -264,11 +418,11 @@ impl Action for AddColumn {
264418 // Remove triggers and procedures
265419 let query = format!(
266420 r#"
267− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
268− DROP FUNCTION IF EXISTS "{trigger_name}";
421+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
422+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
269423 "#,
270− table = self.table,
271424 trigger_name = self.trigger_name(ctx),
425+ reverse_trigger_name = self.reverse_trigger_name(ctx),
272426 );
273427 db.run(&query).context("failed to drop up trigger")?;
274428
src/migrations/add_foreign_key.rsnew file+122
@@ -0,0 +1,122 @@
1+use super::{common::ForeignKey, Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct AddForeignKey {
11+ pub table: String,
12+ foreign_key: ForeignKey,
13+}
14+
15+#[typetag::serde(name = "add_foreign_key")]
16+impl Action for AddForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Adding foreign key from table \"{}\" to \"{}\"",
20+ self.table, self.foreign_key.referenced_table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ let table = schema.get_table(db, &self.table)?;
31+ let referenced_table = schema.get_table(db, &self.foreign_key.referenced_table)?;
32+
33+ // Add quotes around all column names
34+ let columns: Vec<String> = table
35+ .real_column_names(&self.foreign_key.columns)
36+ .map(|col| format!("\"{}\"", col))
37+ .collect();
38+ let referenced_columns: Vec<String> = referenced_table
39+ .real_column_names(&self.foreign_key.referenced_columns)
40+ .map(|col| format!("\"{}\"", col))
41+ .collect();
42+
43+ // Create foreign key but set is as NOT VALID.
44+ // This means the foreign key will be enforced for inserts and updates
45+ // but the existing data won't be checked, that would cause a long-lived lock.
46+ db.run(&format!(
47+ r#"
48+ ALTER TABLE "{table}"
49+ ADD CONSTRAINT {constraint_name}
50+ FOREIGN KEY ({columns})
51+ REFERENCES "{referenced_table}" ({referenced_columns})
52+ NOT VALID
53+ "#,
54+ table = table.real_name,
55+ constraint_name = self.temp_constraint_name(ctx),
56+ columns = columns.join(", "),
57+ referenced_table = referenced_table.real_name,
58+ referenced_columns = referenced_columns.join(", "),
59+ ))
60+ .context("failed to create foreign key")?;
61+
62+ db.run(&format!(
63+ r#"
64+ ALTER TABLE "{table}"
65+ VALIDATE CONSTRAINT "{constraint_name}"
66+ "#,
67+ table = table.real_name,
68+ constraint_name = self.temp_constraint_name(ctx),
69+ ))
70+ .context("failed to validate foreign key")?;
71+
72+ Ok(())
73+ }
74+
75+ fn complete<'a>(
76+ &self,
77+ ctx: &MigrationContext,
78+ db: &'a mut dyn Conn,
79+ ) -> anyhow::Result<Option<Transaction<'a>>> {
80+ db.run(&format!(
81+ r#"
82+ ALTER TABLE {table}
83+ RENAME CONSTRAINT {temp_constraint_name} TO {constraint_name}
84+ "#,
85+ table = self.table,
86+ temp_constraint_name = self.temp_constraint_name(ctx),
87+ constraint_name = self.final_constraint_name(),
88+ ))
89+ .context("failed to rename temporary constraint")?;
90+ Ok(None)
91+ }
92+
93+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
94+
95+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
96+ db.run(&format!(
97+ r#"
98+ ALTER TABLE "{table}"
99+ DROP CONSTRAINT IF EXISTS "{constraint_name}"
100+ "#,
101+ table = self.table,
102+ constraint_name = self.temp_constraint_name(ctx),
103+ ))
104+ .context("failed to validate foreign key")?;
105+
106+ Ok(())
107+ }
108+}
109+
110+impl AddForeignKey {
111+ fn temp_constraint_name(&self, ctx: &MigrationContext) -> String {
112+ format!("{}_temp_fkey", ctx.prefix())
113+ }
114+
115+ fn final_constraint_name(&self) -> String {
116+ format!(
117+ "{table}_{columns}_fkey",
118+ table = self.table,
119+ columns = self.foreign_key.columns.join("_")
120+ )
121+ }
122+}
src/migrations/add_index.rs+24−5
@@ -9,14 +9,26 @@ use serde::{Deserialize, Serialize};
99#[derive(Serialize, Deserialize, Debug)]
1010pub struct AddIndex {
1111 pub table: String,
12+ pub index: Index,
13+}
14+
15+#[derive(Serialize, Deserialize, Clone, Debug)]
16+pub struct Index {
1217 pub name: String,
1318 pub columns: Vec<String>,
19+ #[serde(default)]
20+ pub unique: bool,
21+ #[serde(rename = "type")]
22+ pub index_type: Option<String>,
1423}
1524
1625#[typetag::serde(name = "add_index")]
1726impl Action for AddIndex {
1827 fn describe(&self) -> String {
19− format!("Adding index \"{}\" to table \"{}\"", self.name, self.table)
28+ format!(
29+ "Adding index \"{}\" to table \"{}\"",
30+ self.index.name, self.table
31+ )
2032 }
2133
2234 fn run(
@@ -30,15 +42,22 @@ impl Action for AddIndex {
3042 let column_real_names: Vec<String> = table
3143 .columns
3244 .iter()
33− .filter(|column| self.columns.contains(&column.name))
45+ .filter(|column| self.index.columns.contains(&column.name))
3446 .map(|column| format!("\"{}\"", column.real_name))
3547 .collect();
3648
49+ let unique = if self.index.unique { "UNIQUE" } else { "" };
50+ let index_type_def = if let Some(index_type) = &self.index.index_type {
51+ format!("USING {index_type}")
52+ } else {
53+ "".to_string()
54+ };
55+
3756 db.run(&format!(
3857 r#"
39− CREATE INDEX CONCURRENTLY "{name}" ON "{table}" ({columns})
58+ CREATE {unique} INDEX CONCURRENTLY "{name}" ON "{table}" {index_type_def} ({columns})
4059 "#,
41− name = self.name,
60+ name = self.index.name,
4261 table = self.table,
4362 columns = column_real_names.join(", "),
4463 ))
@@ -61,7 +80,7 @@ impl Action for AddIndex {
6180 r#"
6281 DROP INDEX CONCURRENTLY IF EXISTS "{name}"
6382 "#,
64− name = self.name,
83+ name = self.index.name,
6584 ))
6685 .context("failed to drop index")?;
6786 Ok(())
src/migrations/alter_column.rs+25−19
@@ -13,10 +13,11 @@ pub struct AlterColumn {
1313 pub column: String,
1414 pub up: Option<String>,
1515 pub down: Option<String>,
16+ #[serde(default)]
1617 pub changes: ColumnChanges,
1718}
1819
19−#[derive(Serialize, Deserialize, Debug)]
20+#[derive(Serialize, Deserialize, Default, Debug)]
2021pub struct ColumnChanges {
2122 pub name: Option<String>,
2223 #[serde(rename = "type")]
@@ -47,9 +48,7 @@ impl Action for AlterColumn {
4748 let table = schema.get_table(db, &self.table)?;
4849
4950 let column = table
50− .columns
51− .iter()
52− .find(|column| column.name == self.column)
51+ .get_column(&self.column)
5352 .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
5453
5554 let temporary_column_name = self.temporary_column_name(ctx);
@@ -57,10 +56,14 @@ impl Action for AlterColumn {
5756
5857 // Add temporary, nullable column
5958 let mut temp_column_definition_parts: Vec<&str> =
60− vec![&temporary_column_name, &temporary_column_type];
59+ vec![&temporary_column_name, temporary_column_type];
6160
6261 // Use either new default value or existing one if one exists
63− let default_value = self.changes.default.as_ref().or(column.default.as_ref());
62+ let default_value = self
63+ .changes
64+ .default
65+ .as_ref()
66+ .or_else(|| column.default.as_ref());
6467 if let Some(default) = default_value {
6568 temp_column_definition_parts.push("DEFAULT");
6669 temp_column_definition_parts.push(default);
@@ -100,7 +103,7 @@ impl Action for AlterColumn {
100103 CREATE OR REPLACE FUNCTION {up_trigger}()
101104 RETURNS TRIGGER AS $$
102105 BEGIN
103− IF reshape.is_old_schema() THEN
106+ IF NOT reshape.is_new_schema() THEN
104107 DECLARE
105108 {declarations}
106109 {existing_column} public.{table}.{existing_column_real}%TYPE := NEW.{existing_column_real};
@@ -118,7 +121,7 @@ impl Action for AlterColumn {
118121 CREATE OR REPLACE FUNCTION {down_trigger}()
119122 RETURNS TRIGGER AS $$
120123 BEGIN
121− IF NOT reshape.is_old_schema() THEN
124+ IF reshape.is_new_schema() THEN
122125 DECLARE
123126 {declarations}
124127 {existing_column} public.{table}.{temp_column}%TYPE := NEW.{temp_column};
@@ -147,13 +150,13 @@ impl Action for AlterColumn {
147150 .context("failed to create up and down triggers")?;
148151
149152 // Backfill values in batches by touching the previous column
150− common::batch_touch_rows(db, &table.real_name, &column.real_name)
153+ common::batch_touch_rows(db, &table.real_name, Some(&column.real_name))
151154 .context("failed to batch update existing rows")?;
152155
153156 // Duplicate any indices to the temporary column
154157 let indices = common::get_indices_for_column(db, &table.real_name, &column.real_name)?;
155− for (index_name, index_oid) in indices {
156− let index_columns: Vec<String> = common::get_index_columns(db, &index_name)?
158+ for index in indices {
159+ let index_columns: Vec<String> = common::get_index_columns(db, &index.name)?
157160 .into_iter()
158161 .map(|idx_column| {
159162 // Replace column with temporary column for new index
@@ -164,15 +167,18 @@ impl Action for AlterColumn {
164167 }
165168 })
166169 .collect();
167− let temp_index_name = self.temp_index_name(ctx, index_oid);
170+ let temp_index_name = self.temp_index_name(ctx, index.oid);
171+
172+ let unique_def = if index.unique { "UNIQUE" } else { "" };
168173
169174 db.query(&format!(
170175 r#"
171− CREATE INDEX CONCURRENTLY IF NOT EXISTS "{new_index_name}" ON "{table}" ({columns})
176+ CREATE {unique_def} INDEX CONCURRENTLY IF NOT EXISTS "{new_index_name}" ON "{table}" USING {index_type} ({columns})
172177 "#,
173178 new_index_name = temp_index_name,
174179 table = table.real_name,
175180 columns = index_columns.join(", "),
181+ index_type = index.index_type,
176182 ))
177183 .context("failed to create temporary index")?;
178184 }
@@ -181,7 +187,7 @@ impl Action for AlterColumn {
181187 // This constraint is set as NOT VALID so it doesn't apply to existing rows and
182188 // the existing rows don't need to be scanned under an exclusive lock.
183189 // Thanks to this, we can set the full column as NOT NULL later with minimal locking.
184− if !column.nullable {
190+ if !self.changes.nullable.unwrap_or(column.nullable) {
185191 let query = format!(
186192 r#"
187193 ALTER TABLE "{table}"
@@ -275,7 +281,7 @@ impl Action for AlterColumn {
275281
276282 // Replace old indices with the new temporary ones created for the temporary column
277283 let indices = common::get_indices_for_column(db, &self.table, &self.column)?;
278− for (current_index_name, index_oid) in indices {
284+ for current_index in indices {
279285 // To keep the index handling idempotent, we need to do the following:
280286 // 1. Add a prefix to the existing index
281287 // 2. Rename temporary index to its final name
@@ -283,7 +289,7 @@ impl Action for AlterColumn {
283289
284290 // Add prefix (if not already added) to existing index
285291 let prefix = "__reshape_old";
286− let target_index_name = current_index_name.trim_start_matches(prefix);
292+ let target_index_name = current_index.name.trim_start_matches(prefix);
287293 let old_index_name = format!("{}_{}", prefix, target_index_name);
288294 db.query(&format!(
289295 r#"
@@ -295,7 +301,7 @@ impl Action for AlterColumn {
295301 .context("failed to rename old index")?;
296302
297303 // Rename temporary index to real name
298− let temp_index_name = self.temp_index_name(ctx, index_oid);
304+ let temp_index_name = self.temp_index_name(ctx, current_index.oid);
299305 db.query(&format!(
300306 r#"
301307 ALTER INDEX IF EXISTS "{temp_index_name}" RENAME TO "{target_index_name}"
@@ -383,8 +389,8 @@ impl Action for AlterColumn {
383389 // Safely remove any indices created for the temporary column
384390 let temp_column_name = self.temporary_column_name(ctx);
385391 let indices = common::get_indices_for_column(db, &self.table, &temp_column_name)?;
386− for (_, index_oid) in indices {
387− let temp_index_name = self.temp_index_name(ctx, index_oid);
392+ for index in indices {
393+ let temp_index_name = self.temp_index_name(ctx, index.oid);
388394 db.query(&format!(
389395 r#"
390396 DROP INDEX CONCURRENTLY IF EXISTS "{index_name}"
src/migrations/common.rs+42−26
@@ -1,26 +1,17 @@
11use anyhow::anyhow;
2−use derive_builder::Builder;
32use postgres::types::{FromSql, ToSql};
43use serde::{Deserialize, Serialize};
54
65use crate::db::Conn;
76
8−#[derive(Serialize, Deserialize, Builder, Clone, Debug)]
9−#[builder(setter(into))]
7+#[derive(Serialize, Deserialize, Clone, Debug)]
108pub struct Column {
119 pub name: String,
12−
1310 #[serde(rename = "type")]
1411 pub data_type: String,
15−
1612 #[serde(default = "nullable_default")]
17− #[builder(default = "true")]
1813 pub nullable: bool,
19−
20− #[builder(setter(name = "default_value", strip_option), default)]
2114 pub default: Option<String>,
22−
23− #[builder(setter(strip_option), default)]
2415 pub generated: Option<String>,
2516}
2617
@@ -28,6 +19,13 @@ fn nullable_default() -> bool {
2819 true
2920}
3021
22+#[derive(Serialize, Deserialize, Clone, Debug)]
23+pub struct ForeignKey {
24+ pub columns: Vec<String>,
25+ pub referenced_table: String,
26+ pub referenced_columns: Vec<String>,
27+}
28+
3129#[derive(Debug)]
3230struct PostgresRawValue {
3331 bytes: Vec<u8>,
@@ -71,17 +69,26 @@ impl ToSql for PostgresRawValue {
7169 postgres::types::to_sql_checked!();
7270}
7371
74−pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow::Result<()> {
72+pub fn batch_touch_rows(
73+ db: &mut dyn Conn,
74+ table: &str,
75+ column: Option<&str>,
76+) -> anyhow::Result<()> {
7577 const BATCH_SIZE: u16 = 1000;
7678
77− db.query("SET reshape.is_old_schema = 'YES'")?;
78−
7979 let mut cursor: Option<PostgresRawValue> = None;
8080
8181 loop {
8282 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
8383
8484 let primary_key = get_primary_key_columns_for_table(db, table)?;
85+
86+ // If no column to touch is passed, we default to the first primary key column (just to make some "update")
87+ let touched_column = match column {
88+ Some(column) => column,
89+ None => primary_key.first().unwrap(),
90+ };
91+
8592 let primary_key_columns = primary_key.join(", ");
8693
8794 let primary_key_where = primary_key
@@ -124,8 +131,8 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
124131 ORDER BY {primary_key_columns}
125132 LIMIT {batch_size}
126133 ), update AS (
127− UPDATE public."{table}"
128− SET "{column}" = "{column}"
134+ UPDATE public."{table}" "{table}"
135+ SET "{touched_column}" = "{table}"."{touched_column}"
129136 FROM rows
130137 WHERE {primary_key_where}
131138 RETURNING {returning_columns}
@@ -134,13 +141,7 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
134141 FROM update
135142 LIMIT 1
136143 "#,
137− table = table,
138− primary_key_columns = primary_key_columns,
139− cursor_where = cursor_where,
140144 batch_size = BATCH_SIZE,
141− column = column,
142− primary_key_where = primary_key_where,
143− returning_columns = returning_columns,
144145 );
145146 let last_value = db
146147 .query_with_params(&query, &params)?
@@ -154,8 +155,6 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
154155 cursor = last_value
155156 }
156157
157− db.query("SET reshape.is_old_schema = ''")?;
158−
159158 Ok(())
160159}
161160
@@ -182,18 +181,30 @@ fn get_primary_key_columns_for_table(
182181 Ok(primary_key_columns)
183182}
184183
184+pub struct Index {
185+ pub name: String,
186+ pub oid: u32,
187+ pub unique: bool,
188+ pub index_type: String,
189+}
190+
185191pub fn get_indices_for_column(
186192 db: &mut dyn Conn,
187193 table: &str,
188194 column: &str,
189−) -> anyhow::Result<Vec<(String, u32)>> {
195+) -> anyhow::Result<Vec<Index>> {
190196 let indices = db
191197 .query(&format!(
192198 "
193− SELECT i.relname AS index_name, i.oid AS index_oid
199+ SELECT
200+ i.relname AS name,
201+ i.oid AS oid,
202+ ix.indisunique AS unique,
203+ am.amname AS type
194204 FROM pg_index ix
195205 JOIN pg_class t ON t.oid = ix.indrelid
196206 JOIN pg_class i ON i.oid = ix.indexrelid
207+ JOIN pg_am am ON i.relam = am.oid
197208 JOIN pg_attribute a ON
198209 a.attrelid = t.oid AND
199210 a.attnum = ANY(ix.indkey)
@@ -205,7 +216,12 @@ pub fn get_indices_for_column(
205216 column = column,
206217 ))?
207218 .iter()
208− .map(|row| (row.get("index_name"), row.get("index_oid")))
219+ .map(|row| Index {
220+ name: row.get("name"),
221+ oid: row.get("oid"),
222+ unique: row.get("unique"),
223+ index_type: row.get("type"),
224+ })
209225 .collect();
210226
211227 Ok(indices)
src/migrations/create_enum.rsnew file+83
@@ -0,0 +1,83 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct CreateEnum {
11+ pub name: String,
12+ pub values: Vec<String>,
13+}
14+
15+#[typetag::serde(name = "create_enum")]
16+impl Action for CreateEnum {
17+ fn describe(&self) -> String {
18+ format!("Creating enum \"{}\"", self.name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ // Check if enum already exists. CREATE TYPE doesn't have
28+ // a IF NOT EXISTS option so we have to do it manually.
29+ let enum_exists = !db
30+ .query(&format!(
31+ "
32+ SELECT typname
33+ FROM pg_catalog.pg_type
34+ WHERE typcategory = 'E'
35+ AND typname = '{name}'
36+ ",
37+ name = self.name,
38+ ))?
39+ .is_empty();
40+ if enum_exists {
41+ return Ok(());
42+ }
43+
44+ let values_def: Vec<String> = self
45+ .values
46+ .iter()
47+ .map(|value| format!("'{}'", value))
48+ .collect();
49+
50+ db.run(&format!(
51+ r#"
52+ CREATE TYPE "{name}" AS ENUM ({values})
53+ "#,
54+ name = self.name,
55+ values = values_def.join(", "),
56+ ))
57+ .context("failed to create enum")?;
58+
59+ Ok(())
60+ }
61+
62+ fn complete<'a>(
63+ &self,
64+ _ctx: &MigrationContext,
65+ _db: &'a mut dyn Conn,
66+ ) -> anyhow::Result<Option<Transaction<'a>>> {
67+ Ok(None)
68+ }
69+
70+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
71+
72+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
73+ db.run(&format!(
74+ r#"
75+ DROP TYPE IF EXISTS {name}
76+ "#,
77+ name = self.name,
78+ ))
79+ .context("failed to drop enum")?;
80+
81+ Ok(())
82+ }
83+}
src/migrations/create_table.rs+128−24
@@ -1,29 +1,37 @@
1−use super::{Action, Column, MigrationContext};
1+use std::collections::HashMap;
2+
3+use super::{common::ForeignKey, Action, Column, MigrationContext};
24use crate::{
35 db::{Conn, Transaction},
6+ migrations::common,
47 schema::Schema,
58};
69use anyhow::Context;
7−use derive_builder::Builder;
810use serde::{Deserialize, Serialize};
911
10−#[derive(Serialize, Deserialize, Builder, Debug)]
11−#[builder(setter(into))]
12+#[derive(Serialize, Deserialize, Debug)]
1213pub struct CreateTable {
1314 pub name: String,
1415 pub columns: Vec<Column>,
1516 pub primary_key: Vec<String>,
1617
1718 #[serde(default)]
18− #[builder(default)]
1919 pub foreign_keys: Vec<ForeignKey>,
20+
21+ pub up: Option<Transformation>,
2022}
2123
22−#[derive(Serialize, Deserialize, Clone, Debug)]
23−pub struct ForeignKey {
24− pub columns: Vec<String>,
25− pub referenced_table: String,
26− pub referenced_columns: Vec<String>,
24+#[derive(Serialize, Deserialize, Debug)]
25+pub struct Transformation {
26+ table: String,
27+ values: HashMap<String, String>,
28+ upsert_constraint: Option<String>,
29+}
30+
31+impl CreateTable {
32+ fn trigger_name(&self, ctx: &MigrationContext) -> String {
33+ format!("{}_create_table_{}", ctx.prefix(), self.name)
34+ }
2735}
2836
2937#[typetag::serde(name = "create_table")]
@@ -34,9 +42,9 @@ impl Action for CreateTable {
3442
3543 fn run(
3644 &self,
37− _ctx: &MigrationContext,
45+ ctx: &MigrationContext,
3846 db: &mut dyn Conn,
39− _schema: &Schema,
47+ schema: &Schema,
4048 ) -> anyhow::Result<()> {
4149 let mut definition_rows: Vec<String> = self
4250 .columns
@@ -78,9 +86,10 @@ impl Action for CreateTable {
7886 .iter()
7987 .map(|col| format!("\"{}\"", col))
8088 .collect();
81− let referenced_columns: Vec<String> = foreign_key
82− .referenced_columns
83− .iter()
89+
90+ let referenced_table = schema.get_table(db, &foreign_key.referenced_table)?;
91+ let referenced_columns: Vec<String> = referenced_table
92+ .real_column_names(&foreign_key.referenced_columns)
8493 .map(|col| format!("\"{}\"", col))
8594 .collect();
8695
@@ -89,12 +98,12 @@ impl Action for CreateTable {
8998 FOREIGN KEY ({columns}) REFERENCES "{table}" ({referenced_columns})
9099 "#,
91100 columns = columns.join(", "),
92− table = foreign_key.referenced_table,
101+ table = referenced_table.real_name,
93102 referenced_columns = referenced_columns.join(", "),
94103 ));
95104 }
96105
97− db.run(&format!(
106+ let query = &format!(
98107 r#"
99108 CREATE TABLE "{name}" (
100109 {definition}
@@ -102,26 +111,121 @@ impl Action for CreateTable {
102111 "#,
103112 name = self.name,
104113 definition = definition_rows.join(",\n"),
105− ))
106− .context("failed to create table")?;
114+ );
115+ db.run(query).context("failed to create table")?;
116+
117+ if let Some(Transformation {
118+ table: from_table,
119+ values,
120+ upsert_constraint,
121+ }) = &self.up
122+ {
123+ let from_table = schema.get_table(db, &from_table)?;
124+
125+ let declarations: Vec<String> = from_table
126+ .columns
127+ .iter()
128+ .map(|column| {
129+ format!(
130+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
131+ table = from_table.real_name,
132+ alias = column.name,
133+ real_name = column.real_name,
134+ )
135+ })
136+ .collect();
137+
138+ let (insert_columns, insert_values): (Vec<&str>, Vec<&str>) = values
139+ .iter()
140+ .map(|(k, v)| -> (&str, &str) { (k, v) }) // Force &String to &str
141+ .unzip();
142+
143+ let update_set: Vec<String> = values
144+ .iter()
145+ .map(|(field, value)| format!("\"{field}\" = {value}"))
146+ .collect();
147+
148+ // Constraint to check for conflicts. Defaults to the primary key constraint.
149+ let conflict_constraint_name = match upsert_constraint {
150+ Some(custom_constraint) => custom_constraint.clone(),
151+ _ => format!("{table}_pkey", table = self.name),
152+ };
153+
154+ // Add triggers to fill in values as they are inserted/updated
155+ let query = format!(
156+ r#"
157+ CREATE OR REPLACE FUNCTION {trigger_name}()
158+ RETURNS TRIGGER AS $$
159+ #variable_conflict use_variable
160+ BEGIN
161+ IF NOT reshape.is_new_schema() THEN
162+ DECLARE
163+ {declarations}
164+ BEGIN
165+ INSERT INTO public."{changed_table_real}" ({columns})
166+ VALUES ({values})
167+ ON CONFLICT ON CONSTRAINT "{conflict_constraint_name}"
168+ DO UPDATE SET
169+ {updates};
170+ END;
171+ END IF;
172+ RETURN NEW;
173+ END
174+ $$ language 'plpgsql';
175+
176+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
177+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
178+ "#,
179+ changed_table_real = self.name,
180+ from_table_real = from_table.real_name,
181+ trigger_name = self.trigger_name(ctx),
182+ declarations = declarations.join("\n"),
183+ columns = insert_columns.join(", "),
184+ values = insert_values.join(", "),
185+ updates = update_set.join(",\n"),
186+ );
187+ db.run(&query).context("failed to create up trigger")?;
188+
189+ // Backfill values in batches by touching the from table
190+ common::batch_touch_rows(db, &from_table.real_name, None)
191+ .context("failed to batch update existing rows")?;
192+ }
193+
107194 Ok(())
108195 }
109196
110197 fn complete<'a>(
111198 &self,
112− _ctx: &MigrationContext,
113− _db: &'a mut dyn Conn,
199+ ctx: &MigrationContext,
200+ db: &'a mut dyn Conn,
114201 ) -> anyhow::Result<Option<Transaction<'a>>> {
115− // Do nothing
202+ // Remove triggers and procedures
203+ let query = format!(
204+ r#"
205+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
206+ "#,
207+ trigger_name = self.trigger_name(ctx),
208+ );
209+ db.run(&query).context("failed to drop up trigger")?;
210+
116211 Ok(None)
117212 }
118213
119214 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
120215
121− fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
216+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
217+ // Remove triggers and procedures
218+ let query = format!(
219+ r#"
220+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
221+ "#,
222+ trigger_name = self.trigger_name(ctx),
223+ );
224+ db.run(&query).context("failed to drop up trigger")?;
225+
122226 db.run(&format!(
123227 r#"
124− DROP TABLE IF EXISTS {name}
228+ DROP TABLE IF EXISTS "{name}"
125229 "#,
126230 name = self.name,
127231 ))
src/migrations/custom.rsnew file+61
@@ -0,0 +1,61 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use serde::{Deserialize, Serialize};
7+
8+#[derive(Serialize, Deserialize, Debug)]
9+pub struct Custom {
10+ #[serde(default)]
11+ pub start: Option<String>,
12+
13+ #[serde(default)]
14+ pub complete: Option<String>,
15+
16+ #[serde(default)]
17+ pub abort: Option<String>,
18+}
19+
20+#[typetag::serde(name = "custom")]
21+impl Action for Custom {
22+ fn describe(&self) -> String {
23+ "Running custom migration".to_string()
24+ }
25+
26+ fn run(
27+ &self,
28+ _ctx: &MigrationContext,
29+ db: &mut dyn Conn,
30+ _schema: &Schema,
31+ ) -> anyhow::Result<()> {
32+ if let Some(start_query) = &self.start {
33+ println!("Running query: {}", start_query);
34+ db.run(start_query)?;
35+ }
36+
37+ Ok(())
38+ }
39+
40+ fn complete<'a>(
41+ &self,
42+ _ctx: &MigrationContext,
43+ db: &'a mut dyn Conn,
44+ ) -> anyhow::Result<Option<Transaction<'a>>> {
45+ if let Some(complete_query) = &self.complete {
46+ db.run(complete_query)?;
47+ }
48+
49+ Ok(None)
50+ }
51+
52+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
53+
54+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
55+ if let Some(abort_query) = &self.abort {
56+ db.run(abort_query)?;
57+ }
58+
59+ Ok(())
60+ }
61+}
src/migrations/mod.rs+25−4
@@ -7,10 +7,10 @@ use serde::{Deserialize, Serialize};
77
88// Re-export migration types
99mod common;
10−pub use common::{Column, ColumnBuilder};
10+pub use common::Column;
1111
1212mod create_table;
13−pub use create_table::{CreateTable, CreateTableBuilder, ForeignKey};
13+pub use create_table::CreateTable;
1414
1515mod alter_column;
1616pub use alter_column::{AlterColumn, ColumnChanges};
@@ -22,7 +22,7 @@ mod remove_column;
2222pub use remove_column::RemoveColumn;
2323
2424mod add_index;
25−pub use add_index::AddIndex;
25+pub use add_index::{AddIndex, Index};
2626
2727mod remove_index;
2828pub use remove_index::RemoveIndex;
@@ -33,6 +33,21 @@ pub use remove_table::RemoveTable;
3333mod rename_table;
3434pub use rename_table::RenameTable;
3535
36+mod create_enum;
37+pub use create_enum::CreateEnum;
38+
39+mod remove_enum;
40+pub use remove_enum::RemoveEnum;
41+
42+mod custom;
43+pub use custom::Custom;
44+
45+mod add_foreign_key;
46+pub use add_foreign_key::AddForeignKey;
47+
48+mod remove_foreign_key;
49+pub use remove_foreign_key::RemoveForeignKey;
50+
3651#[derive(Serialize, Deserialize, Debug)]
3752pub struct Migration {
3853 pub name: String,
@@ -73,13 +88,19 @@ impl Clone for Migration {
7388pub struct MigrationContext {
7489 migration_index: usize,
7590 action_index: usize,
91+ existing_schema_name: Option<String>,
7692}
7793
7894impl MigrationContext {
79− pub fn new(migration_index: usize, action_index: usize) -> Self {
95+ pub fn new(
96+ migration_index: usize,
97+ action_index: usize,
98+ existing_schema_name: Option<String>,
99+ ) -> Self {
80100 MigrationContext {
81101 migration_index,
82102 action_index,
103+ existing_schema_name,
83104 }
84105 }
85106
src/migrations/remove_column.rs+353−35
@@ -1,16 +1,27 @@
1−use super::{Action, MigrationContext};
1+use super::{common, Action, MigrationContext};
22use crate::{
33 db::{Conn, Transaction},
44 schema::Schema,
55};
6−use anyhow::Context;
6+use anyhow::{anyhow, bail, Context};
77use serde::{Deserialize, Serialize};
88
99#[derive(Serialize, Deserialize, Debug)]
1010pub struct RemoveColumn {
1111 pub table: String,
1212 pub column: String,
13− pub down: Option<String>,
13+ pub down: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
1425}
1526
1627impl RemoveColumn {
@@ -22,6 +33,33 @@ impl RemoveColumn {
2233 self.column
2334 )
2435 }
36+
37+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
38+ format!(
39+ "{}_remove_column_{}_{}_rev",
40+ ctx.prefix(),
41+ self.table,
42+ self.column
43+ )
44+ }
45+
46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_remove_column_{}_{}_nn",
49+ ctx.prefix(),
50+ self.table,
51+ self.column
52+ )
53+ }
54+
55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
56+ format!(
57+ "{}_add_column_not_null_{}_{}",
58+ ctx.prefix(),
59+ self.table,
60+ self.column
61+ )
62+ }
2563}
2664
2765#[typetag::serde(name = "remove_column")]
@@ -39,10 +77,13 @@ impl Action for RemoveColumn {
3977 db: &mut dyn Conn,
4078 schema: &Schema,
4179 ) -> anyhow::Result<()> {
80+ let table = schema.get_table(db, &self.table)?;
81+ let column = table
82+ .get_column(&self.column)
83+ .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
84+
4285 // Add down trigger
4386 if let Some(down) = &self.down {
44− let table = schema.get_table(db, &self.table)?;
45−
4687 let declarations: Vec<String> = table
4788 .columns
4889 .iter()
@@ -56,32 +97,226 @@ impl Action for RemoveColumn {
5697 })
5798 .collect();
5899
59− let query = format!(
60− r#"
61− CREATE OR REPLACE FUNCTION {trigger_name}()
62− RETURNS TRIGGER AS $$
63− BEGIN
64− IF NOT reshape.is_old_schema() IS NULL THEN
65− DECLARE
66− {declarations}
100+ if let Transformation::Simple(down) = down {
101+ let query = format!(
102+ r#"
103+ CREATE OR REPLACE FUNCTION {trigger_name}()
104+ RETURNS TRIGGER AS $$
105+ BEGIN
106+ IF reshape.is_new_schema() THEN
107+ DECLARE
108+ {declarations}
109+ BEGIN
110+ NEW.{column_name} = {down};
111+ END;
112+ END IF;
113+ RETURN NEW;
114+ END
115+ $$ language 'plpgsql';
116+
117+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
118+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
119+ "#,
120+ column_name = self.column,
121+ trigger_name = self.trigger_name(ctx),
122+ down = down,
123+ table = self.table,
124+ declarations = declarations.join("\n"),
125+ );
126+ db.run(&query).context("failed to create down trigger")?;
127+ }
128+
129+ if let Transformation::Update {
130+ table: from_table,
131+ value,
132+ r#where,
133+ } = down
134+ {
135+ let existing_schema_name = match &ctx.existing_schema_name {
136+ Some(name) => name,
137+ None => bail!("can't use update without previous migration"),
138+ };
139+
140+ let from_table = schema.get_table(db, &from_table)?;
141+
142+ let maybe_null_check = if !column.nullable {
143+ // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.
144+ // We will add a null check to the down function on the new schema below as well to cover both cases.
145+ // As we are using a complex down function, we must remove the NOT NULL check for the new schema.
146+ // NOT NULL is not checked at the end of a transaction, but immediately upon update.
147+ let query = format!(
148+ r#"
149+ CREATE OR REPLACE FUNCTION {trigger_name}()
150+ RETURNS TRIGGER AS $$
67151 BEGIN
68− NEW.{column_name} = {down};
69− END;
70− END IF;
71− RETURN NEW;
72− END
73− $$ language 'plpgsql';
74−
75− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
76− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
77− "#,
78− column_name = self.column,
79− trigger_name = self.trigger_name(ctx),
80− down = down,
81− table = self.table,
82− declarations = declarations.join("\n"),
83− );
84− db.run(&query).context("failed to create down trigger")?;
152+ IF NOT reshape.is_new_schema() THEN
153+ IF NEW.{column} IS NULL THEN
154+ RAISE EXCEPTION '{column} can not be null';
155+ END IF;
156+ END IF;
157+ RETURN NEW;
158+ END
159+ $$ language 'plpgsql';
160+
161+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
162+
163+ CREATE CONSTRAINT TRIGGER "{trigger_name}"
164+ AFTER INSERT OR UPDATE
165+ ON "{table}"
166+ FOR EACH ROW
167+ EXECUTE PROCEDURE {trigger_name}();
168+ "#,
169+ table = self.table,
170+ trigger_name = self.not_null_constraint_trigger_name(ctx),
171+ column = self.column,
172+ );
173+ db.run(&query)
174+ .context("failed to create null constraint trigger")?;
175+
176+ db.run(&format!(
177+ r#"
178+ ALTER TABLE {table}
179+ ALTER COLUMN {column}
180+ DROP NOT NULL
181+ "#,
182+ table = self.table,
183+ column = self.column
184+ ))
185+ .context("failed to remove column not null constraint")?;
186+
187+ format!(
188+ r#"
189+ IF {value} IS NULL THEN
190+ RAISE EXCEPTION '{column_name} can not be null';
191+ END IF;
192+ "#,
193+ column_name = self.column,
194+ )
195+ } else {
196+ "".to_string()
197+ };
198+
199+ let into_variables = from_table
200+ .columns
201+ .iter()
202+ .map(|column| {
203+ format!(
204+ "NEW.{real_name} AS {alias}",
205+ alias = column.name,
206+ real_name = column.real_name,
207+ )
208+ })
209+ .collect::<Vec<String>>()
210+ .join(", ");
211+
212+ let query = format!(
213+ r#"
214+ CREATE OR REPLACE FUNCTION {trigger_name}()
215+ RETURNS TRIGGER AS $$
216+ #variable_conflict use_variable
217+ BEGIN
218+ IF reshape.is_new_schema() THEN
219+ DECLARE
220+ {from_table} record;
221+ BEGIN
222+ SELECT {into_variables}
223+ INTO {from_table};
224+
225+ {maybe_null_check}
226+
227+ -- Don't trigger reverse trigger when making this update
228+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
229+
230+ UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"
231+ SET "{column_name}" = {value}
232+ WHERE {where};
233+
234+ perform set_config('reshape.disable_triggers', '', TRUE);
235+ END;
236+ END IF;
237+ RETURN NEW;
238+ END
239+ $$ language 'plpgsql';
240+
241+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
242+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
243+ "#,
244+ changed_table = self.table,
245+ from_table = from_table.name,
246+ from_table_real = from_table.real_name,
247+ column_name = self.column,
248+ trigger_name = self.trigger_name(ctx),
249+ );
250+ db.run(&query).context("failed to create down trigger")?;
251+
252+ let changed_into_variables = table
253+ .columns
254+ .iter()
255+ .map(|column| {
256+ format!(
257+ "NEW.{real_name} AS {alias}",
258+ alias = column.name,
259+ real_name = column.real_name,
260+ )
261+ })
262+ .collect::<Vec<String>>()
263+ .join(", ");
264+
265+ let from_table_columns = from_table
266+ .columns
267+ .iter()
268+ .map(|column| format!("{} as {}", column.real_name, column.name))
269+ .collect::<Vec<String>>()
270+ .join(", ");
271+
272+ let query = format!(
273+ r#"
274+ CREATE OR REPLACE FUNCTION {trigger_name}()
275+ RETURNS TRIGGER AS $$
276+ #variable_conflict use_variable
277+ BEGIN
278+ IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
279+ DECLARE
280+ {changed_table} record;
281+ __temp_row record;
282+ BEGIN
283+ SELECT {changed_into_variables}
284+ INTO {changed_table};
285+
286+ SELECT *
287+ INTO __temp_row
288+ FROM (
289+ SELECT {from_table_columns}
290+ FROM public.{from_table_real}
291+ ) {from_table}
292+ WHERE {where};
293+
294+ DECLARE
295+ {from_table} record;
296+ BEGIN
297+ {from_table} := __temp_row;
298+ NEW.{column_name_real} = {value};
299+ END;
300+ END;
301+ END IF;
302+ RETURN NEW;
303+ END
304+ $$ language 'plpgsql';
305+
306+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
307+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
308+ "#,
309+ changed_table = table.name,
310+ changed_table_real = table.real_name,
311+ from_table = from_table.name,
312+ from_table_real = from_table.real_name,
313+ column_name_real = column.real_name,
314+ trigger_name = self.reverse_trigger_name(ctx),
315+ // declarations = declarations.join("\n"),
316+ );
317+ db.run(&query)
318+ .context("failed to create reverse down trigger")?;
319+ }
85320 }
86321
87322 Ok(())
@@ -92,18 +327,34 @@ impl Action for RemoveColumn {
92327 ctx: &MigrationContext,
93328 db: &'a mut dyn Conn,
94329 ) -> anyhow::Result<Option<Transaction<'a>>> {
330+ let indices = common::get_indices_for_column(db, &self.table, &self.column)
331+ .context("failed getting column indices")?;
332+
333+ for index in indices {
334+ db.run(&format!(
335+ "
336+ DROP INDEX CONCURRENTLY IF EXISTS {name}
337+ ",
338+ name = index.name,
339+ ))
340+ .context("failed to drop index")?;
341+ }
342+
95343 // Remove column, function and trigger
96344 let query = format!(
97345 r#"
98346 ALTER TABLE "{table}"
99347 DROP COLUMN IF EXISTS "{column}";
100348
101− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
102− DROP FUNCTION IF EXISTS "{trigger_name}";
349+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
350+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
351+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
103352 "#,
104353 table = self.table,
105354 column = self.column,
106355 trigger_name = self.trigger_name(ctx),
356+ reverse_trigger_name = self.reverse_trigger_name(ctx),
357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
107358 );
108359 db.run(&query)
109360 .context("failed to drop column and down trigger")?;
@@ -120,14 +371,81 @@ impl Action for RemoveColumn {
120371 }
121372
122373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
374+ // We might have temporaily removed the NOT NULL check and have to reinstate it
375+ let has_not_null_function = !db
376+ .query_with_params(
377+ "
378+ SELECT routine_name
379+ FROM information_schema.routines
380+ WHERE routine_schema = 'public'
381+ AND routine_name = $1
382+ ",
383+ &[&self.not_null_constraint_trigger_name(ctx)],
384+ )
385+ .context("failed to get any NOT NULL function")?
386+ .is_empty();
387+
388+ if has_not_null_function {
389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint
390+ let query = format!(
391+ r#"
392+ ALTER TABLE "{table}"
393+ ADD CONSTRAINT "{constraint_name}"
394+ CHECK ("{column}" IS NOT NULL) NOT VALID
395+ "#,
396+ table = self.table,
397+ constraint_name = self.not_null_constraint_name(ctx),
398+ column = self.column,
399+ );
400+ db.run(&query)
401+ .context("failed to add NOT NULL constraint")?;
402+
403+ let query = format!(
404+ r#"
405+ ALTER TABLE "{table}"
406+ VALIDATE CONSTRAINT "{constraint_name}"
407+ "#,
408+ table = self.table,
409+ constraint_name = self.not_null_constraint_name(ctx),
410+ );
411+ db.run(&query)
412+ .context("failed to validate NOT NULL constraint")?;
413+
414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above
415+ db.run(&format!(
416+ r#"
417+ ALTER TABLE {table}
418+ ALTER COLUMN {column}
419+ SET NOT NULL
420+ "#,
421+ table = self.table,
422+ column = self.column
423+ ))
424+ .context("failed to reinstate column NOT NULL")?;
425+
426+ // Drop the temporary constraint
427+ let query = format!(
428+ r#"
429+ ALTER TABLE "{table}"
430+ DROP CONSTRAINT "{constraint_name}"
431+ "#,
432+ table = self.table,
433+ constraint_name = self.not_null_constraint_name(ctx),
434+ );
435+ db.run(&query)
436+ .context("failed to drop NOT NULL constraint")?;
437+ }
438+
123439 // Remove function and trigger
124440 db.run(&format!(
125441 r#"
126− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
127− DROP FUNCTION IF EXISTS "{trigger_name}";
442+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
443+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
444+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
128445 "#,
129− table = self.table,
130446 trigger_name = self.trigger_name(ctx),
447+ reverse_trigger_name = self.reverse_trigger_name(ctx),
448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
131449 ))
132450 .context("failed to drop down trigger")?;
133451
src/migrations/remove_enum.rsnew file+51
@@ -0,0 +1,51 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveEnum {
11+ #[serde(rename = "enum")]
12+ pub enum_name: String,
13+}
14+
15+#[typetag::serde(name = "remove_enum")]
16+impl Action for RemoveEnum {
17+ fn describe(&self) -> String {
18+ format!("Removing enum \"{}\"", self.enum_name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ _db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ Ok(())
28+ }
29+
30+ fn complete<'a>(
31+ &self,
32+ _ctx: &MigrationContext,
33+ db: &'a mut dyn Conn,
34+ ) -> anyhow::Result<Option<Transaction<'a>>> {
35+ db.run(&format!(
36+ r#"
37+ DROP TYPE IF EXISTS {name}
38+ "#,
39+ name = self.enum_name,
40+ ))
41+ .context("failed to drop enum")?;
42+
43+ Ok(None)
44+ }
45+
46+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
47+
48+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
49+ Ok(())
50+ }
51+}
src/migrations/remove_foreign_key.rsnew file+91
@@ -0,0 +1,91 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::{anyhow, Context};
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveForeignKey {
11+ table: String,
12+ foreign_key: String,
13+}
14+
15+#[typetag::serde(name = "remove_foreign_key")]
16+impl Action for RemoveForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Removing foreign key \"{}\" from table \"{}\"",
20+ self.foreign_key, self.table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ _ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ // The foreign key is only removed once the migration is completed.
31+ // Removing it earlier would be hard/undesirable for several reasons:
32+ // - Postgres doesn't have an easy way to temporarily disable a foreign key check.
33+ // If it did, we could disable the FK for the new schema.
34+ // - Even if we could, it probably wouldn't be a good idea as it would cause temporary
35+ // inconsistencies for the old schema which still expects the FK to hold.
36+ // - For the same reason, we can't remove the FK when the migration is first applied.
37+ // If the migration was to be aborted, then the FK would have to be recreated with
38+ // the risk that it would no longer be valid.
39+
40+ // Ensure foreign key exists
41+ let table = schema.get_table(db, &self.table)?;
42+ let fk_exists = !db
43+ .query(&format!(
44+ r#"
45+ SELECT constraint_name
46+ FROM information_schema.table_constraints
47+ WHERE
48+ constraint_type = 'FOREIGN KEY' AND
49+ table_name = '{table_name}' AND
50+ constraint_name = '{foreign_key}'
51+ "#,
52+ table_name = table.real_name,
53+ foreign_key = self.foreign_key,
54+ ))
55+ .context("failed to check for foreign key")?
56+ .is_empty();
57+
58+ if !fk_exists {
59+ return Err(anyhow!(
60+ "no foreign key \"{}\" exists on table \"{}\"",
61+ self.foreign_key,
62+ self.table
63+ ));
64+ }
65+
66+ Ok(())
67+ }
68+
69+ fn complete<'a>(
70+ &self,
71+ _ctx: &MigrationContext,
72+ db: &'a mut dyn Conn,
73+ ) -> anyhow::Result<Option<Transaction<'a>>> {
74+ db.run(&format!(
75+ r#"
76+ ALTER TABLE {table}
77+ DROP CONSTRAINT IF EXISTS {foreign_key}
78+ "#,
79+ table = self.table,
80+ foreign_key = self.foreign_key,
81+ ))
82+ .context("failed to remove foreign key")?;
83+ Ok(None)
84+ }
85+
86+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
87+
88+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
89+ Ok(())
90+ }
91+}
src/schema.rs+18−1
@@ -209,7 +209,7 @@ impl Schema {
209209 let real_columns: Vec<(String, String, bool, Option<String>)> = db
210210 .query(&format!(
211211 "
212− SELECT column_name, data_type, is_nullable, column_default
212+ SELECT column_name, CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END, is_nullable, column_default
213213 FROM information_schema.columns
214214 WHERE table_name = '{table}' AND table_schema = 'public'
215215 ORDER BY ordinal_position
@@ -286,3 +286,20 @@ impl Schema {
286286 Ok(table)
287287 }
288288}
289+
290+impl Table {
291+ pub fn real_column_names<'a>(
292+ &'a self,
293+ columns: &'a [String],
294+ ) -> impl Iterator<Item = &'a String> {
295+ columns.iter().map(|name| {
296+ self.get_column(name)
297+ .map(|col| &col.real_name)
298+ .unwrap_or(name)
299+ })
300+ }
301+
302+ pub fn get_column(&self, name: &str) -> Option<&Column> {
303+ self.columns.iter().find(|column| column.name == name)
304+ }
305+}
src/state.rs+31−40
@@ -4,14 +4,9 @@ use anyhow::anyhow;
44use serde::{Deserialize, Serialize};
55use version::version;
66
7−#[derive(Serialize, Deserialize, Debug)]
8−pub struct State {
9− pub status: Status,
10−}
11−
127#[derive(Serialize, Deserialize, Clone, Debug)]
13−#[serde(tag = "status")]
14−pub enum Status {
8+#[serde(tag = "state")]
9+pub enum State {
1510 #[serde(rename = "idle")]
1611 Idle,
1712
@@ -37,24 +32,23 @@ pub enum Status {
3732}
3833
3934impl State {
40− pub fn load(db: &mut impl Conn) -> State {
41− Self::ensure_schema_and_table(db);
35+ pub fn load(db: &mut impl Conn) -> anyhow::Result<State> {
36+ Self::ensure_schema_and_table(db)?;
4237
43− let results = db
44− .query("SELECT value FROM reshape.data WHERE key = 'state'")
45− .unwrap();
38+ let results = db.query("SELECT value FROM reshape.data WHERE key = 'state'")?;
4639
47− match results.first() {
40+ let state = match results.first() {
4841 Some(row) => {
4942 let json: serde_json::Value = row.get(0);
50− serde_json::from_value(json).unwrap()
43+ serde_json::from_value(json)?
5144 }
5245 None => Default::default(),
53− }
46+ };
47+ Ok(state)
5448 }
5549
5650 pub fn save(&self, db: &mut impl Conn) -> anyhow::Result<()> {
57− Self::ensure_schema_and_table(db);
51+ Self::ensure_schema_and_table(db)?;
5852
5953 let json = serde_json::to_value(self)?;
6054 db.query_with_params(
@@ -67,17 +61,17 @@ impl State {
6761 pub fn clear(&mut self, db: &mut impl Conn) -> anyhow::Result<()> {
6862 db.run("DROP SCHEMA reshape CASCADE")?;
6963
70− let default = Self::default();
71− self.status = default.status;
64+ *self = Self::default();
7265
7366 Ok(())
7467 }
7568
76− // Complete will change the status from Completing to Idle
69+ // Complete will change the state from Completing to Idle
7770 pub fn complete(&mut self, db: &mut impl Conn) -> anyhow::Result<()> {
78− let current_status = std::mem::replace(&mut self.status, Status::Idle);
79− match current_status {
80− Status::Completing { migrations, .. } => {
71+ let current_state = std::mem::replace(self, Self::Idle);
72+
73+ match current_state {
74+ Self::Completing { migrations, .. } => {
8175 // Add migrations and update state in a transaction to ensure atomicity
8276 let mut transaction = db.transaction()?;
8377 save_migrations(&mut transaction, &migrations)?;
@@ -85,8 +79,8 @@ impl State {
8579 transaction.commit()?;
8680 }
8781 _ => {
88− // Move old status back
89− self.status = current_status;
82+ // Move old state back
83+ *self = current_state;
9084
9185 return Err(anyhow!(
9286 "couldn't update state to be completed, not in Completing state"
@@ -98,13 +92,13 @@ impl State {
9892 }
9993
10094 pub fn applying(&mut self, new_migrations: Vec<Migration>) {
101− self.status = Status::Applying {
95+ *self = Self::Applying {
10296 migrations: new_migrations,
10397 };
10498 }
10599
106100 pub fn in_progress(&mut self, new_migrations: Vec<Migration>) {
107− self.status = Status::InProgress {
101+ *self = Self::InProgress {
108102 migrations: new_migrations,
109103 };
110104 }
@@ -115,7 +109,7 @@ impl State {
115109 current_migration_index: usize,
116110 current_action_index: usize,
117111 ) {
118− self.status = Status::Completing {
112+ *self = Self::Completing {
119113 migrations,
120114 current_migration_index,
121115 current_action_index,
@@ -128,20 +122,19 @@ impl State {
128122 last_migration_index: usize,
129123 last_action_index: usize,
130124 ) {
131− self.status = Status::Aborting {
125+ *self = Self::Aborting {
132126 migrations,
133127 last_migration_index,
134128 last_action_index,
135129 }
136130 }
137131
138− fn ensure_schema_and_table(db: &mut impl Conn) {
139− db.run("CREATE SCHEMA IF NOT EXISTS reshape").unwrap();
132+ fn ensure_schema_and_table(db: &mut impl Conn) -> anyhow::Result<()> {
133+ db.run("CREATE SCHEMA IF NOT EXISTS reshape")?;
140134
141135 // Create data table which will be a key-value table containing
142136 // the version and current state.
143− db.run("CREATE TABLE IF NOT EXISTS reshape.data (key TEXT PRIMARY KEY, value JSONB)")
144− .unwrap();
137+ db.run("CREATE TABLE IF NOT EXISTS reshape.data (key TEXT PRIMARY KEY, value JSONB)")?;
145138
146139 // Create migrations table which will store all completed migrations
147140 db.run(
@@ -154,11 +147,10 @@ impl State {
154147 completed_at TIMESTAMP DEFAULT NOW()
155148 )
156149 ",
157− )
158− .unwrap();
150+ )?;
159151
160152 // Update the current version
161− let encoded_version = serde_json::to_value(version!().to_string()).unwrap();
153+ let encoded_version = serde_json::to_value(version!().to_string())?;
162154 db.query_with_params(
163155 "
164156 INSERT INTO reshape.data (key, value)
@@ -166,16 +158,15 @@ impl State {
166158 ON CONFLICT (key) DO UPDATE SET value = $1
167159 ",
168160 &[&encoded_version],
169− )
170− .unwrap();
161+ )?;
162+
163+ Ok(())
171164 }
172165}
173166
174167impl Default for State {
175168 fn default() -> Self {
176− State {
177− status: Status::Idle,
178− }
169+ Self::Idle
179170 }
180171}
181172
tests/add_column.rs+318−254
@@ -1,81 +1,60 @@
1−use reshape::migrations::{AddColumn, Column, ColumnBuilder, CreateTableBuilder, Migration};
2−use reshape::Status;
3−
41mod common;
2+use common::Test;
53
64#[test]
75fn add_column() {
8− let (mut reshape, mut old_db, mut new_db) = common::setup();
9−
10− let create_users_table = Migration::new("create_user_table", None).with_action(
11− CreateTableBuilder::default()
12− .name("users")
13− .primary_key(vec!["id".to_string()])
14− .columns(vec![
15− ColumnBuilder::default()
16− .name("id")
17− .data_type("INTEGER")
18− .build()
19− .unwrap(),
20− ColumnBuilder::default()
21− .name("name")
22− .data_type("TEXT")
23− .build()
24− .unwrap(),
25− ])
26− .build()
27− .unwrap(),
6+ let mut test = Test::new("Add column");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
2825 );
2926
30− let add_first_last_name_columns = Migration::new("add_first_and_last_name_columns", None)
31− .with_action(AddColumn {
32− table: "users".to_string(),
33− column: Column {
34− name: "first".to_string(),
35− data_type: "TEXT".to_string(),
36− nullable: false,
37− default: None,
38− generated: None,
39− },
40− up: Some("(STRING_TO_ARRAY(name, ' '))[1]".to_string()),
41− })
42− .with_action(AddColumn {
43− table: "users".to_string(),
44− column: Column {
45− name: "last".to_string(),
46− data_type: "TEXT".to_string(),
47− nullable: false,
48− default: None,
49− generated: None,
50− },
51− up: Some("(STRING_TO_ARRAY(name, ' '))[2]".to_string()),
52− });
53−
54− let first_migrations = vec![create_users_table.clone()];
55− let second_migrations = vec![
56− create_users_table.clone(),
57− add_first_last_name_columns.clone(),
58− ];
59−
60− // Run first migration, should automatically finish
61− reshape.migrate(first_migrations.clone()).unwrap();
62− assert!(matches!(reshape.state.status, Status::Idle));
63−
64− // Update search paths
65− old_db
66− .simple_query(&reshape::schema_query_for_migration(
67− &first_migrations.last().unwrap().name,
68− ))
69− .unwrap();
70− new_db
71− .simple_query(&reshape::schema_query_for_migration(
72− &first_migrations.last().unwrap().name,
73− ))
74− .unwrap();
27+ test.second_migration(
28+ r#"
29+ name = "add_first_and_last_name_columns"
30+
31+ [[actions]]
32+ type = "add_column"
33+ table = "users"
34+
35+ up = "(STRING_TO_ARRAY(name, ' '))[1]"
7536
76− // Insert some test users
77− new_db
78− .simple_query(
37+ [actions.column]
38+ name = "first"
39+ type = "TEXT"
40+ nullable = false
41+
42+ [[actions]]
43+ type = "add_column"
44+ table = "users"
45+
46+ up = "(STRING_TO_ARRAY(name, ' '))[2]"
47+
48+ [actions.column]
49+ name = "last"
50+ type = "TEXT"
51+ nullable = false
52+ "#,
53+ );
54+
55+ test.after_first(|db| {
56+ // Insert some test users
57+ db.simple_query(
7958 "
8059 INSERT INTO users (id, name) VALUES
8160 (1, 'John Doe'),
@@ -83,215 +62,300 @@ fn add_column() {
8362 ",
8463 )
8564 .unwrap();
65+ });
8666
87− // Run second migration
88− reshape.migrate(second_migrations.clone()).unwrap();
89− new_db
90− .simple_query(&reshape::schema_query_for_migration(
91− &second_migrations.last().unwrap().name,
92− ))
93− .unwrap();
67+ test.intermediate(|old_db, new_db| {
68+ // Check that the existing users have the new columns populated
69+ let expected = vec![("John", "Doe"), ("Jane", "Doe")];
70+ assert!(new_db
71+ .query("SELECT first, last FROM users ORDER BY id", &[],)
72+ .unwrap()
73+ .iter()
74+ .map(|row| (row.get("first"), row.get("last")))
75+ .eq(expected));
76+
77+ // Insert data using old schema and make sure the new columns are populated
78+ old_db
79+ .simple_query("INSERT INTO users (id, name) VALUES (3, 'Test Testsson')")
80+ .unwrap();
81+ let (first_name, last_name): (String, String) = new_db
82+ .query_one("SELECT first, last from users WHERE id = 3", &[])
83+ .map(|row| (row.get("first"), row.get("last")))
84+ .unwrap();
85+ assert_eq!(
86+ ("Test", "Testsson"),
87+ (first_name.as_ref(), last_name.as_ref())
88+ );
89+ });
9490
95− // Check that the existing users have the new columns populated
96− let expected = vec![("John", "Doe"), ("Jane", "Doe")];
97− assert!(new_db
98− .query("SELECT first, last FROM users ORDER BY id", &[],)
99− .unwrap()
100− .iter()
101− .map(|row| (row.get("first"), row.get("last")))
102− .eq(expected));
103−
104− // Insert data using old schema and make sure the new columns are populated
105− old_db
106− .simple_query("INSERT INTO users (id, name) VALUES (3, 'Test Testsson')")
107− .unwrap();
108− let (first_name, last_name): (String, String) = new_db
109− .query_one("SELECT first, last from users WHERE id = 3", &[])
110− .map(|row| (row.get("first"), row.get("last")))
111− .unwrap();
112− assert_eq!(
113− ("Test", "Testsson"),
114− (first_name.as_ref(), last_name.as_ref())
115− );
91+ test.after_completion(|db| {
92+ let expected = vec![("John", "Doe"), ("Jane", "Doe"), ("Test", "Testsson")];
93+ assert!(db
94+ .query("SELECT first, last FROM users ORDER BY id", &[],)
95+ .unwrap()
96+ .iter()
97+ .map(|row| (row.get("first"), row.get("last")))
98+ .eq(expected));
99+ });
116100
117− reshape.complete_migration().unwrap();
118− common::assert_cleaned_up(&mut new_db);
101+ test.after_abort(|db| {
102+ let expected = vec![("John Doe"), ("Jane Doe"), ("Test Testsson")];
103+ assert!(db
104+ .query("SELECT name FROM users ORDER BY id", &[],)
105+ .unwrap()
106+ .iter()
107+ .map(|row| row.get::<'_, _, String>("name"))
108+ .eq(expected));
109+ });
110+
111+ test.run()
119112}
120113
121114#[test]
122115fn add_column_nullable() {
123− let (mut reshape, mut old_db, mut new_db) = common::setup();
124−
125− let create_users_table = Migration::new("create_user_table", None).with_action(
126− CreateTableBuilder::default()
127− .name("users")
128− .primary_key(vec!["id".to_string()])
129− .columns(vec![ColumnBuilder::default()
130− .name("id")
131− .data_type("INTEGER")
132− .build()
133− .unwrap()])
134− .build()
135− .unwrap(),
116+ let mut test = Test::new("Add nullable column");
117+
118+ test.first_migration(
119+ r#"
120+ name = "create_users_table"
121+
122+ [[actions]]
123+ type = "create_table"
124+ name = "users"
125+ primary_key = ["id"]
126+
127+ [[actions.columns]]
128+ name = "id"
129+ type = "INTEGER"
130+ "#,
136131 );
137− let add_name_column = Migration::new("add_nullable_name_column", None).with_action(AddColumn {
138− table: "users".to_string(),
139− column: Column {
140− name: "name".to_string(),
141− data_type: "TEXT".to_string(),
142− nullable: true,
143− default: None,
144− generated: None,
145− },
146− up: None,
147− });
148132
149− let first_migrations = vec![create_users_table.clone()];
150− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
133+ test.second_migration(
134+ r#"
135+ name = "add_nullable_name_column"
151136
152− // Run first migration, should automatically finish
153− reshape.migrate(first_migrations.clone()).unwrap();
154− assert!(matches!(reshape.state.status, Status::Idle));
137+ [[actions]]
138+ type = "add_column"
139+ table = "users"
155140
156− // Update search paths
157− old_db
158− .simple_query(&reshape::schema_query_for_migration(
159− &first_migrations.last().unwrap().name,
160− ))
161− .unwrap();
162− new_db
163− .simple_query(&reshape::schema_query_for_migration(
164− &first_migrations.last().unwrap().name,
165− ))
166− .unwrap();
141+ [actions.column]
142+ name = "name"
143+ type = "TEXT"
144+ "#,
145+ );
167146
168− // Insert some test values
169− new_db
170− .simple_query(
147+ test.after_first(|db| {
148+ // Insert some test values
149+ db.simple_query(
171150 "
172− INSERT INTO users (id) VALUES
173− (1),
174− (2);
151+ INSERT INTO users (id) VALUES (1), (2);
175152 ",
176153 )
177154 .unwrap();
155+ });
178156
179− // Run second migration
180− reshape.migrate(second_migrations.clone()).unwrap();
181− new_db
182− .simple_query(&reshape::schema_query_for_migration(
183− &second_migrations.last().unwrap().name,
184− ))
185− .unwrap();
186−
187− // Ensure existing data got updated
188− let expected: Vec<Option<String>> = vec![None, None];
189− assert!(new_db
190− .query("SELECT name FROM users ORDER BY id", &[],)
191− .unwrap()
192− .iter()
193− .map(|row| row.get::<_, Option<String>>("name"))
194− .eq(expected));
195−
196− // Insert data using old schema and ensure new column is NULL
197− old_db
198− .simple_query("INSERT INTO users (id) VALUES (3)")
199− .unwrap();
200− let name: Option<String> = new_db
201− .query_one("SELECT name from users WHERE id = 3", &[])
202− .map(|row| (row.get("name")))
203− .unwrap();
204− assert_eq!(None, name);
157+ test.intermediate(|old_db, new_db| {
158+ // Ensure existing data got updated
159+ let expected: Vec<Option<String>> = vec![None, None];
160+ assert!(new_db
161+ .query("SELECT name FROM users ORDER BY id", &[],)
162+ .unwrap()
163+ .iter()
164+ .map(|row| row.get::<_, Option<String>>("name"))
165+ .eq(expected));
166+
167+ // Insert data using old schema and ensure new column is NULL
168+ old_db
169+ .simple_query("INSERT INTO users (id) VALUES (3)")
170+ .unwrap();
171+ let name: Option<String> = new_db
172+ .query_one("SELECT name from users WHERE id = 3", &[])
173+ .map(|row| (row.get("name")))
174+ .unwrap();
175+ assert_eq!(None, name);
176+
177+ // Ensure data can be inserted against new schema
178+ new_db
179+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
180+ .unwrap();
181+ });
205182
206− // Ensure data can be inserted against new schema
207− new_db
208− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
209− .unwrap();
183+ test.after_completion(|db| {
184+ let expected: Vec<Option<String>> =
185+ vec![None, None, None, Some("Test Testsson".to_string()), None];
186+ let result: Vec<Option<String>> = db
187+ .query("SELECT id, name FROM users ORDER BY id", &[])
188+ .unwrap()
189+ .iter()
190+ .map(|row| row.get("name"))
191+ .collect();
192+
193+ assert_eq!(result, expected);
194+ });
210195
211− reshape.complete_migration().unwrap();
212− common::assert_cleaned_up(&mut new_db);
196+ test.run();
213197}
214198
215199#[test]
216200fn add_column_with_default() {
217− let (mut reshape, mut old_db, mut new_db) = common::setup();
218−
219− let create_users_table = Migration::new("create_user_table", None).with_action(
220− CreateTableBuilder::default()
221− .name("users")
222− .primary_key(vec!["id".to_string()])
223− .columns(vec![ColumnBuilder::default()
224− .name("id")
225− .data_type("INTEGER")
226− .build()
227− .unwrap()])
228− .build()
229− .unwrap(),
201+ let mut test = Test::new("Add column with default value");
202+
203+ test.first_migration(
204+ r#"
205+ name = "create_users_table"
206+
207+ [[actions]]
208+ type = "create_table"
209+ name = "users"
210+ primary_key = ["id"]
211+
212+ [[actions.columns]]
213+ name = "id"
214+ type = "INTEGER"
215+ "#,
230216 );
231− let add_name_column =
232− Migration::new("add_name_column_with_default", None).with_action(AddColumn {
233− table: "users".to_string(),
234− column: Column {
235− name: "name".to_string(),
236− data_type: "TEXT".to_string(),
237− nullable: false,
238− default: Some("'DEFAULT'".to_string()),
239− generated: None,
240− },
241− up: None,
242− });
243−
244− let first_migrations = vec![create_users_table.clone()];
245− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
246−
247− // Run first migration, should automatically finish
248− reshape.migrate(first_migrations.clone()).unwrap();
249− assert!(matches!(reshape.state.status, Status::Idle));
250−
251− // Update search paths
252− old_db
253− .simple_query(&reshape::schema_query_for_migration(
254− &first_migrations.last().unwrap().name,
255− ))
256− .unwrap();
257− new_db
258− .simple_query(&reshape::schema_query_for_migration(
259− &first_migrations.last().unwrap().name,
260− ))
261− .unwrap();
262217
263− // Insert some test values
264− new_db
265− .simple_query("INSERT INTO users (id) VALUES (1), (2)")
266− .unwrap();
218+ test.second_migration(
219+ r#"
220+ name = "add_name_column_with_default"
267221
268− // Run second migration
269− reshape.migrate(second_migrations.clone()).unwrap();
270− new_db
271− .simple_query(&reshape::schema_query_for_migration(
272− &second_migrations.last().unwrap().name,
273− ))
274− .unwrap();
222+ [[actions]]
223+ type = "add_column"
224+ table = "users"
275225
276− // Ensure existing data got updated with defaults
277− let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
278− assert!(new_db
279− .query("SELECT name FROM users ORDER BY id", &[],)
280− .unwrap()
281− .iter()
282− .map(|row| row.get::<_, String>("name"))
283− .eq(expected));
284−
285− // Insert data using old schema and ensure new column gets the default value
286− old_db
287− .simple_query("INSERT INTO users (id) VALUES (3)")
288− .unwrap();
289− let name: String = new_db
290− .query_one("SELECT name from users WHERE id = 3", &[])
291− .map(|row| row.get("name"))
292− .unwrap();
293− assert_eq!("DEFAULT", name);
226+ [actions.column]
227+ name = "name"
228+ type = "TEXT"
229+ nullable = false
230+ default = "'DEFAULT'"
231+ "#,
232+ );
233+
234+ test.after_first(|db| {
235+ // Insert some test values
236+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
237+ .unwrap();
238+ });
239+
240+ test.intermediate(|old_db, new_db| {
241+ // Ensure existing data got updated with defaults
242+ let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
243+ assert!(new_db
244+ .query("SELECT name FROM users ORDER BY id", &[],)
245+ .unwrap()
246+ .iter()
247+ .map(|row| row.get::<_, String>("name"))
248+ .eq(expected));
249+
250+ // Insert data using old schema and ensure new column gets the default value
251+ old_db
252+ .simple_query("INSERT INTO users (id) VALUES (3)")
253+ .unwrap();
254+ let name: String = new_db
255+ .query_one("SELECT name from users WHERE id = 3", &[])
256+ .map(|row| row.get("name"))
257+ .unwrap();
258+ assert_eq!("DEFAULT", name);
259+ });
260+
261+ test.run();
262+}
263+
264+#[test]
265+fn add_column_with_complex_up() {
266+ let mut test = Test::new("Add column complex");
267+
268+ test.first_migration(
269+ r#"
270+ name = "create_tables"
271+
272+ [[actions]]
273+ type = "create_table"
274+ name = "users"
275+ primary_key = ["id"]
276+
277+ [[actions.columns]]
278+ name = "id"
279+ type = "INTEGER"
280+
281+ [[actions.columns]]
282+ name = "email"
283+ type = "TEXT"
284+
285+ [[actions]]
286+ type = "create_table"
287+ name = "profiles"
288+ primary_key = ["user_id"]
289+
290+ [[actions.columns]]
291+ name = "user_id"
292+ type = "INTEGER"
293+ "#,
294+ );
295+
296+ test.second_migration(
297+ r#"
298+ name = "add_profiles_email_column"
299+
300+ [[actions]]
301+ type = "add_column"
302+ table = "profiles"
303+
304+ [actions.column]
305+ name = "email"
306+ type = "TEXT"
307+ nullable = false
308+
309+ [actions.up]
310+ table = "users"
311+ value = "users.email"
312+ where = "profiles.user_id = users.id"
313+ "#,
314+ );
315+
316+ test.after_first(|db| {
317+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
318+ .unwrap();
319+ db.simple_query("INSERT INTO profiles (user_id) VALUES (1)")
320+ .unwrap();
321+ });
322+
323+ test.intermediate(|old_db, new_db| {
324+ // Ensure email was backfilled on profiles
325+ let email: String = new_db
326+ .query(
327+ "
328+ SELECT email
329+ FROM profiles
330+ WHERE user_id = 1
331+ ",
332+ &[],
333+ )
334+ .unwrap()
335+ .first()
336+ .map(|row| row.get("email"))
337+ .unwrap();
338+ assert_eq!("test@example.com", email);
339+
340+ // Ensure email change in old schema is propagated to profiles table in new schema
341+ old_db
342+ .simple_query("UPDATE users SET email = 'test2@example.com' WHERE id = 1")
343+ .unwrap();
344+ let email: String = new_db
345+ .query(
346+ "
347+ SELECT email
348+ FROM profiles
349+ WHERE user_id = 1
350+ ",
351+ &[],
352+ )
353+ .unwrap()
354+ .first()
355+ .map(|row| row.get("email"))
356+ .unwrap();
357+ assert_eq!("test2@example.com", email);
358+ });
294359
295− reshape.complete_migration().unwrap();
296− common::assert_cleaned_up(&mut new_db);
360+ test.run();
297361}
tests/add_foreign_key.rsnew file+166
@@ -0,0 +1,166 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn add_foreign_key() {
6+ let mut test = Test::new("Add foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+ "#,
34+ );
35+
36+ test.second_migration(
37+ r#"
38+ name = "add_foreign_key"
39+
40+ [[actions]]
41+ type = "add_foreign_key"
42+ table = "items"
43+
44+ [actions.foreign_key]
45+ columns = ["user_id"]
46+ referenced_table = "users"
47+ referenced_columns = ["id"]
48+ "#,
49+ );
50+
51+ test.after_first(|db| {
52+ // Insert some test users
53+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
54+ .unwrap();
55+ });
56+
57+ test.intermediate(|db, _| {
58+ // Ensure items can be inserted if they reference valid users
59+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
60+ .unwrap();
61+
62+ // Ensure items can't be inserted if they don't reference valid users
63+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
64+ assert!(result.is_err(), "expected insert to fail");
65+ });
66+
67+ test.after_completion(|db| {
68+ // Ensure items can be inserted if they reference valid users
69+ db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 1), (4, 2)")
70+ .unwrap();
71+
72+ // Ensure items can't be inserted if they don't reference valid users
73+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)");
74+ assert!(result.is_err(), "expected insert to fail");
75+
76+ // Ensure foreign key exists with the right name
77+ let foreign_key_name: Option<String> = db
78+ .query(
79+ "
80+ SELECT tc.constraint_name
81+ FROM information_schema.table_constraints AS tc
82+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
83+ ",
84+ &[],
85+ )
86+ .unwrap()
87+ .first()
88+ .map(|row| row.get(0));
89+ assert_eq!(Some("items_user_id_fkey".to_string()), foreign_key_name);
90+ });
91+
92+ test.after_abort(|db| {
93+ // Ensure foreign key doesn't exist
94+ let fk_does_not_exist = db
95+ .query(
96+ "
97+ SELECT tc.constraint_name
98+ FROM information_schema.table_constraints AS tc
99+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
100+ ",
101+ &[],
102+ )
103+ .unwrap()
104+ .is_empty();
105+ assert!(fk_does_not_exist);
106+ });
107+
108+ test.run()
109+}
110+
111+#[test]
112+fn add_invalid_foreign_key() {
113+ let mut test = Test::new("Add invalid foreign key");
114+
115+ test.first_migration(
116+ r#"
117+ name = "create_user_table"
118+
119+ [[actions]]
120+ type = "create_table"
121+ name = "users"
122+ primary_key = ["id"]
123+
124+ [[actions.columns]]
125+ name = "id"
126+ type = "INTEGER"
127+
128+ [[actions]]
129+ type = "create_table"
130+ name = "items"
131+ primary_key = ["id"]
132+
133+ [[actions.columns]]
134+ name = "id"
135+ type = "INTEGER"
136+
137+ [[actions.columns]]
138+ name = "user_id"
139+ type = "INTEGER"
140+ "#,
141+ );
142+
143+ test.second_migration(
144+ r#"
145+ name = "add_foreign_key"
146+
147+ [[actions]]
148+ type = "add_foreign_key"
149+ table = "items"
150+
151+ [actions.foreign_key]
152+ columns = ["user_id"]
153+ referenced_table = "users"
154+ referenced_columns = ["id"]
155+ "#,
156+ );
157+
158+ test.after_first(|db| {
159+ // Insert some items which don't reference a valid user
160+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
161+ .unwrap();
162+ });
163+
164+ test.expect_failure();
165+ test.run()
166+}
tests/add_index.rs+215−61
@@ -1,69 +1,223 @@
1−use reshape::migrations::{AddIndex, ColumnBuilder, CreateTableBuilder, Migration};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn add_index() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
10− CreateTableBuilder::default()
11− .name("users")
12− .primary_key(vec!["id".to_string()])
13− .columns(vec![
14− ColumnBuilder::default()
15− .name("id")
16− .data_type("INTEGER")
17− .build()
18− .unwrap(),
19− ColumnBuilder::default()
20− .name("name")
21− .data_type("TEXT")
22− .build()
23− .unwrap(),
24− ])
25− .build()
26− .unwrap(),
6+ let mut test = Test::new("Add index");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "add_users_name_index"
30+
31+ [[actions]]
32+ type = "add_index"
33+ table = "users"
34+
35+ [actions.index]
36+ name = "name_idx"
37+ columns = ["name"]
38+ "#,
39+ );
40+
41+ test.intermediate(|db, _| {
42+ // Ensure index is valid and ready
43+ let (is_ready, is_valid): (bool, bool) = db
44+ .query(
45+ "
46+ SELECT pg_index.indisready, pg_index.indisvalid
47+ FROM pg_catalog.pg_index
48+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
49+ WHERE pg_class.relname = 'name_idx'
50+ ",
51+ &[],
52+ )
53+ .unwrap()
54+ .first()
55+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
56+ .unwrap();
57+
58+ assert!(is_ready, "expected index to be ready");
59+ assert!(is_valid, "expected index to be valid");
60+ });
61+
62+ test.after_completion(|db| {
63+ // Ensure index is valid and ready
64+ let (is_ready, is_valid): (bool, bool) = db
65+ .query(
66+ "
67+ SELECT pg_index.indisready, pg_index.indisvalid
68+ FROM pg_catalog.pg_index
69+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
70+ WHERE pg_class.relname = 'name_idx'
71+ ",
72+ &[],
73+ )
74+ .unwrap()
75+ .first()
76+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
77+ .unwrap();
78+
79+ assert!(is_ready, "expected index to be ready");
80+ assert!(is_valid, "expected index to be valid");
81+ });
82+
83+ test.run();
84+}
85+
86+#[test]
87+fn add_index_unique() {
88+ let mut test = Test::new("Add unique index");
89+
90+ test.first_migration(
91+ r#"
92+ name = "create_users_table"
93+
94+ [[actions]]
95+ type = "create_table"
96+ name = "users"
97+ primary_key = ["id"]
98+
99+ [[actions.columns]]
100+ name = "id"
101+ type = "INTEGER"
102+
103+ [[actions.columns]]
104+ name = "name"
105+ type = "TEXT"
106+ "#,
107+ );
108+
109+ test.second_migration(
110+ r#"
111+ name = "add_name_index"
112+
113+ [[actions]]
114+ type = "add_index"
115+ table = "users"
116+
117+ [actions.index]
118+ name = "name_idx"
119+ columns = ["name"]
120+ unique = true
121+ "#,
27122 );
28− let add_index_migration = Migration::new("add_name_index", None).with_action(AddIndex {
29− table: "users".to_string(),
30− name: "name_idx".to_string(),
31− columns: vec!["name".to_string()],
123+
124+ test.intermediate(|db, _| {
125+ // Ensure index is valid, ready and unique
126+ let (is_ready, is_valid, is_unique): (bool, bool, bool) = db
127+ .query(
128+ "
129+ SELECT pg_index.indisready, pg_index.indisvalid, pg_index.indisunique
130+ FROM pg_catalog.pg_index
131+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
132+ WHERE pg_class.relname = 'name_idx'
133+ ",
134+ &[],
135+ )
136+ .unwrap()
137+ .first()
138+ .map(|row| {
139+ (
140+ row.get("indisready"),
141+ row.get("indisvalid"),
142+ row.get("indisunique"),
143+ )
144+ })
145+ .unwrap();
146+
147+ assert!(is_ready, "expected index to be ready");
148+ assert!(is_valid, "expected index to be valid");
149+ assert!(is_unique, "expected index to be unique");
150+ });
151+
152+ test.run();
153+}
154+
155+#[test]
156+fn add_index_with_type() {
157+ let mut test = Test::new("Add GIN index");
158+
159+ test.first_migration(
160+ r#"
161+ name = "create_users_table"
162+
163+ [[actions]]
164+ type = "create_table"
165+ name = "users"
166+ primary_key = ["id"]
167+
168+ [[actions.columns]]
169+ name = "id"
170+ type = "INTEGER"
171+
172+ [[actions.columns]]
173+ name = "data"
174+ type = "JSONB"
175+ "#,
176+ );
177+
178+ test.second_migration(
179+ r#"
180+ name = "add_data_index"
181+
182+ [[actions]]
183+ type = "add_index"
184+ table = "users"
185+
186+ [actions.index]
187+ name = "data_idx"
188+ columns = ["data"]
189+ type = "gin"
190+ "#,
191+ );
192+
193+ test.intermediate(|db, _| {
194+ // Ensure index is valid, ready and has the right type
195+ let (is_ready, is_valid, index_type): (bool, bool, String) = db
196+ .query(
197+ "
198+ SELECT pg_index.indisready, pg_index.indisvalid, pg_am.amname
199+ FROM pg_catalog.pg_index
200+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
201+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
202+ WHERE pg_class.relname = 'data_idx'
203+ ",
204+ &[],
205+ )
206+ .unwrap()
207+ .first()
208+ .map(|row| {
209+ (
210+ row.get("indisready"),
211+ row.get("indisvalid"),
212+ row.get("amname"),
213+ )
214+ })
215+ .unwrap();
216+
217+ assert!(is_ready, "expected index to be ready");
218+ assert!(is_valid, "expected index to be valid");
219+ assert_eq!("gin", index_type, "expected index type to be GIN");
32220 });
33221
34− let first_migrations = vec![create_table_migration.clone()];
35− let second_migrations = vec![create_table_migration.clone(), add_index_migration.clone()];
36−
37− // Run migrations
38− reshape.migrate(first_migrations.clone()).unwrap();
39− reshape.migrate(second_migrations.clone()).unwrap();
40−
41− // Update schemas of Postgres connections
42− let old_schema_query =
43− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
44− let new_schema_query =
45− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
46− old_db.simple_query(&old_schema_query).unwrap();
47− new_db.simple_query(&new_schema_query).unwrap();
48−
49− // Ensure index is valid and ready
50− let result: Vec<(bool, bool)> = new_db
51− .query(
52− "
53− SELECT pg_index.indisready, pg_index.indisvalid
54− FROM pg_catalog.pg_index
55− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
56− WHERE pg_class.relname = 'name_idx'
57− ",
58− &[],
59− )
60− .unwrap()
61− .iter()
62− .map(|row| (row.get("indisready"), row.get("indisvalid")))
63− .collect();
64−
65− assert_eq!(vec![(true, true)], result);
66−
67− reshape.complete_migration().unwrap();
68− common::assert_cleaned_up(&mut new_db);
222+ test.run();
69223}
tests/alter_column.rs+637−501
@@ -1,63 +1,45 @@
1−use reshape::migrations::{
2− AddIndex, AlterColumn, ColumnBuilder, ColumnChanges, CreateTableBuilder, Migration,
3−};
4−use reshape::Status;
5−
61mod common;
2+use common::Test;
73
84#[test]
95fn alter_column_data() {
10− let (mut reshape, mut old_db, mut new_db) = common::setup();
11−
12− let create_users_table = Migration::new("create_user_table", None).with_action(
13− CreateTableBuilder::default()
14− .name("users")
15− .primary_key(vec!["id".to_string()])
16− .columns(vec![
17− ColumnBuilder::default()
18− .name("id")
19− .data_type("INTEGER")
20− .build()
21− .unwrap(),
22− ColumnBuilder::default()
23− .name("name")
24− .data_type("TEXT")
25− .build()
26− .unwrap(),
27− ])
28− .build()
29− .unwrap(),
30− );
31− let uppercase_name = Migration::new("uppercase_name", None).with_action(AlterColumn {
32− table: "users".to_string(),
33− column: "name".to_string(),
34− up: Some("UPPER(name)".to_string()),
35− down: Some("LOWER(name)".to_string()),
36− changes: ColumnChanges {
37− data_type: None,
38− nullable: None,
39− name: None,
40− default: None,
41− },
42− });
6+ let mut test = Test::new("Alter column");
437
44− let first_migrations = vec![create_users_table.clone()];
45− let second_migrations = vec![create_users_table.clone(), uppercase_name.clone()];
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
4611
47− // Run first migration, should automatically finish
48− reshape.migrate(first_migrations.clone()).unwrap();
49− assert!(matches!(reshape.state.status, Status::Idle));
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
5016
51− // Update search paths
52− old_db
53− .simple_query(&reshape::schema_query_for_migration(
54− &first_migrations.last().unwrap().name,
55− ))
56− .unwrap();
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "uppercase_name"
30+
31+ [[actions]]
32+ type = "alter_column"
33+ table = "users"
34+ column = "name"
35+ up = "UPPER(name)"
36+ down = "LOWER(name)"
37+ "#,
38+ );
5739
58− // Insert some test users
59− old_db
60− .simple_query(
40+ test.after_first(|db| {
41+ // Insert some test users
42+ db.simple_query(
6143 "
6244 INSERT INTO users (id, name) VALUES
6345 (1, 'john Doe'),
@@ -65,99 +47,81 @@ fn alter_column_data() {
6547 ",
6648 )
6749 .unwrap();
50+ });
6851
69− // Run second migration
70− reshape.migrate(second_migrations.clone()).unwrap();
71− new_db
72− .simple_query(&reshape::schema_query_for_migration(
73− &second_migrations.last().unwrap().name,
74− ))
75− .unwrap();
76−
77− // Check that the existing users has the altered data
78− let expected = vec!["JOHN DOE", "JANE DOE"];
79− assert!(new_db
80− .query("SELECT name FROM users ORDER BY id", &[],)
81− .unwrap()
82− .iter()
83− .map(|row| row.get::<_, String>("name"))
84− .eq(expected));
85−
86− // Insert data using old schema and make sure the new schema gets correct values
87− old_db
88− .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
89− .unwrap();
90− let result = new_db
91− .query_one("SELECT name from users WHERE id = 3", &[])
92− .unwrap();
93− assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
94−
95− // Insert data using new schema and make sure the old schema gets correct values
96− new_db
97− .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
98− .unwrap();
99− let result = old_db
100− .query_one("SELECT name from users WHERE id = 4", &[])
101− .unwrap();
102− assert_eq!("test testsson", result.get::<_, &str>("name"));
52+ test.intermediate(|old_db, new_db| {
53+ // Check that the existing users has the altered data
54+ let expected = vec!["JOHN DOE", "JANE DOE"];
55+ assert!(new_db
56+ .query("SELECT name FROM users ORDER BY id", &[],)
57+ .unwrap()
58+ .iter()
59+ .map(|row| row.get::<_, String>("name"))
60+ .eq(expected));
61+
62+ // Insert data using old schema and make sure the new schema gets correct values
63+ old_db
64+ .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
65+ .unwrap();
66+ let result = new_db
67+ .query_one("SELECT name from users WHERE id = 3", &[])
68+ .unwrap();
69+ assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
70+
71+ // Insert data using new schema and make sure the old schema gets correct values
72+ new_db
73+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
74+ .unwrap();
75+ let result = old_db
76+ .query_one("SELECT name from users WHERE id = 4", &[])
77+ .unwrap();
78+ assert_eq!("test testsson", result.get::<_, &str>("name"));
79+ });
10380
104− reshape.complete_migration().unwrap();
105− common::assert_cleaned_up(&mut new_db);
81+ test.run();
10682}
10783
10884#[test]
10985fn alter_column_set_not_null() {
110− let (mut reshape, mut old_db, mut new_db) = common::setup();
111−
112− let create_users_table = Migration::new("create_user_table", None).with_action(
113− CreateTableBuilder::default()
114− .name("users")
115− .primary_key(vec!["id".to_string()])
116− .columns(vec![
117− ColumnBuilder::default()
118− .name("id")
119− .data_type("INTEGER")
120− .build()
121− .unwrap(),
122− ColumnBuilder::default()
123− .name("name")
124− .data_type("TEXT")
125− .build()
126− .unwrap(),
127− ])
128− .build()
129− .unwrap(),
86+ let mut test = Test::new("Set column not null");
87+
88+ test.first_migration(
89+ r#"
90+ name = "create_user_table"
91+
92+ [[actions]]
93+ type = "create_table"
94+ name = "users"
95+ primary_key = ["id"]
96+
97+ [[actions.columns]]
98+ name = "id"
99+ type = "INTEGER"
100+
101+ [[actions.columns]]
102+ name = "name"
103+ type = "TEXT"
104+ "#,
130105 );
131− let set_name_not_null = Migration::new("set_name_not_null", None).with_action(AlterColumn {
132− table: "users".to_string(),
133− column: "name".to_string(),
134− up: Some("COALESCE(name, 'TEST_DEFAULT_VALUE')".to_string()),
135− down: Some("name".to_string()),
136− changes: ColumnChanges {
137− data_type: None,
138− nullable: Some(false),
139− name: None,
140− default: None,
141− },
142− });
143106
144− let first_migrations = vec![create_users_table.clone()];
145− let second_migrations = vec![create_users_table.clone(), set_name_not_null.clone()];
107+ test.second_migration(
108+ r#"
109+ name = "set_name_not_null"
146110
147− // Run first migration, should automatically finish
148− reshape.migrate(first_migrations.clone()).unwrap();
149− assert!(matches!(reshape.state.status, Status::Idle));
111+ [[actions]]
112+ type = "alter_column"
113+ table = "users"
114+ column = "name"
115+ up = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
150116
151− // Update search paths
152− old_db
153− .simple_query(&reshape::schema_query_for_migration(
154− &first_migrations.last().unwrap().name,
155− ))
156− .unwrap();
117+ [actions.changes]
118+ nullable = false
119+ "#,
120+ );
157121
158− // Insert some test users
159− old_db
160− .simple_query(
122+ test.after_first(|db| {
123+ // Insert some test users
124+ db.simple_query(
161125 "
162126 INSERT INTO users (id, name) VALUES
163127 (1, 'John Doe'),
@@ -165,100 +129,180 @@ fn alter_column_set_not_null() {
165129 ",
166130 )
167131 .unwrap();
132+ });
168133
169− // Run second migration
170− reshape.migrate(second_migrations.clone()).unwrap();
171− new_db
172− .simple_query(&reshape::schema_query_for_migration(
173− &second_migrations.last().unwrap().name,
174− ))
175− .unwrap();
134+ test.intermediate(|old_db, new_db| {
135+ // Check that existing users got the correct values
136+ let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
137+ assert!(new_db
138+ .query("SELECT name FROM users ORDER BY id", &[],)
139+ .unwrap()
140+ .iter()
141+ .map(|row| row.get::<_, String>("name"))
142+ .eq(expected));
143+
144+ // Insert data using old schema and make sure the new schema gets correct values
145+ old_db
146+ .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
147+ .unwrap();
148+ let result = new_db
149+ .query_one("SELECT name from users WHERE id = 3", &[])
150+ .unwrap();
151+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
152+
153+ // Insert data using new schema and make sure the old schema gets correct values
154+ new_db
155+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
156+ .unwrap();
157+ let result = old_db
158+ .query_one("SELECT name from users WHERE id = 4", &[])
159+ .unwrap();
160+ assert_eq!("Jane Doe", result.get::<_, &str>("name"));
161+
162+ // Ensure NULL can't be inserted using the new schema
163+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
164+ assert!(result.is_err(), "expected insert to fail");
165+ });
176166
177− // Check that existing users got the correct values
178− let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
179− assert!(new_db
180− .query("SELECT name FROM users ORDER BY id", &[],)
181− .unwrap()
182− .iter()
183− .map(|row| row.get::<_, String>("name"))
184− .eq(expected));
185−
186− // Insert data using old schema and make sure the new schema gets correct values
187− old_db
188− .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
189− .unwrap();
190− let result = new_db
191− .query_one("SELECT name from users WHERE id = 3", &[])
192− .unwrap();
193− assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
167+ test.after_completion(|db| {
168+ // Ensure NULL can't be inserted
169+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
170+ assert!(result.is_err(), "expected insert to fail");
171+ });
194172
195− // Insert data using new schema and make sure the old schema gets correct values
196− new_db
197− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
198− .unwrap();
199− let result = old_db
200− .query_one("SELECT name from users WHERE id = 4", &[])
173+ test.after_abort(|db| {
174+ // Ensure NULL can be inserted
175+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
176+ assert!(result.is_ok(), "expected insert to succeed");
177+ });
178+
179+ test.run();
180+}
181+
182+#[test]
183+fn alter_column_set_nullable() {
184+ let mut test = Test::new("Set column nullable");
185+
186+ test.first_migration(
187+ r#"
188+ name = "create_user_table"
189+
190+ [[actions]]
191+ type = "create_table"
192+ name = "users"
193+ primary_key = ["id"]
194+
195+ [[actions.columns]]
196+ name = "id"
197+ type = "INTEGER"
198+
199+ [[actions.columns]]
200+ name = "name"
201+ type = "TEXT"
202+ nullable = false
203+ "#,
204+ );
205+
206+ test.second_migration(
207+ r#"
208+ name = "set_name_nullable"
209+
210+ [[actions]]
211+ type = "alter_column"
212+ table = "users"
213+ column = "name"
214+ down = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
215+
216+ [actions.changes]
217+ nullable = true
218+ "#,
219+ );
220+
221+ test.after_first(|db| {
222+ // Insert a test user
223+ db.simple_query(
224+ "
225+ INSERT INTO users (id, name) VALUES
226+ (1, 'John Doe')
227+ ",
228+ )
201229 .unwrap();
202− assert_eq!("Jane Doe", result.get::<_, &str>("name"));
230+ });
203231
204− reshape.complete_migration().unwrap();
205− common::assert_cleaned_up(&mut new_db);
232+ test.intermediate(|old_db, new_db| {
233+ // Insert data using new schema and make sure the old schema gets correct values
234+ new_db
235+ .simple_query("INSERT INTO users (id, name) VALUES (2, NULL)")
236+ .unwrap();
237+ let result = old_db
238+ .query_one("SELECT name from users WHERE id = 2", &[])
239+ .unwrap();
240+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
241+
242+ // Ensure NULL can't be inserted using the old schema
243+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, NULL)");
244+ assert!(result.is_err(), "expected insert to fail");
245+
246+ // Ensure NULL can be inserted using the new schema
247+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (4, NULL)");
248+ assert!(result.is_ok(), "expected insert to succeed");
249+ });
250+
251+ test.after_completion(|db| {
252+ // Ensure NULL can be inserted
253+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
254+ assert!(result.is_ok(), "expected insert to succeed");
255+ });
256+
257+ test.after_abort(|db| {
258+ // Ensure NULL can't be inserted
259+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
260+ assert!(result.is_err(), "expected insert to fail");
261+ });
262+
263+ test.run();
206264}
207265
208266#[test]
209267fn alter_column_rename() {
210− let (mut reshape, mut old_db, mut new_db) = common::setup();
211−
212− let create_users_table = Migration::new("create_user_table", None).with_action(
213− CreateTableBuilder::default()
214− .name("users")
215− .primary_key(vec!["id".to_string()])
216− .columns(vec![
217− ColumnBuilder::default()
218− .name("id")
219− .data_type("INTEGER")
220− .build()
221− .unwrap(),
222− ColumnBuilder::default()
223− .name("name")
224− .data_type("TEXT")
225− .build()
226− .unwrap(),
227− ])
228− .build()
229− .unwrap(),
268+ let mut test = Test::new("Rename column");
269+
270+ test.first_migration(
271+ r#"
272+ name = "create_user_table"
273+
274+ [[actions]]
275+ type = "create_table"
276+ name = "users"
277+ primary_key = ["id"]
278+
279+ [[actions.columns]]
280+ name = "id"
281+ type = "INTEGER"
282+
283+ [[actions.columns]]
284+ name = "name"
285+ type = "TEXT"
286+ "#,
230287 );
231− let rename_to_full_name =
232− Migration::new("rename_to_full_name", None).with_action(AlterColumn {
233− table: "users".to_string(),
234− column: "name".to_string(),
235− up: None, // up and down are not required when only renaming a column
236− down: None,
237− changes: ColumnChanges {
238− data_type: None,
239− nullable: None,
240− name: Some("full_name".to_string()),
241− default: None,
242− },
243− });
244−
245− let first_migrations = vec![create_users_table.clone()];
246− let second_migrations = vec![create_users_table.clone(), rename_to_full_name.clone()];
247−
248− // Run first migration, should automatically finish
249− reshape.migrate(first_migrations.clone()).unwrap();
250− assert!(matches!(reshape.state.status, Status::Idle));
251−
252− // Update search paths
253− old_db
254− .simple_query(&reshape::schema_query_for_migration(
255− &first_migrations.last().unwrap().name,
256− ))
257− .unwrap();
258288
259− // Insert some test data
260− old_db
261− .simple_query(
289+ test.second_migration(
290+ r#"
291+ name = "set_name_not_null"
292+
293+ [[actions]]
294+ type = "alter_column"
295+ table = "users"
296+ column = "name"
297+
298+ [actions.changes]
299+ name = "full_name"
300+ "#,
301+ );
302+
303+ test.after_first(|db| {
304+ // Insert some test data
305+ db.simple_query(
262306 "
263307 INSERT INTO users (id, name) VALUES
264308 (1, 'John Doe'),
@@ -266,95 +310,69 @@ fn alter_column_rename() {
266310 ",
267311 )
268312 .unwrap();
313+ });
269314
270− // Run second migration
271− reshape.migrate(second_migrations.clone()).unwrap();
272− new_db
273− .simple_query(&reshape::schema_query_for_migration(
274− &second_migrations.last().unwrap().name,
275− ))
276− .unwrap();
315+ test.intermediate(|_, new_db| {
316+ // Check that existing values can be queried using new column name
317+ let expected = vec!["John Doe", "Jane Doe"];
318+ assert!(new_db
319+ .query("SELECT full_name FROM users ORDER BY id", &[],)
320+ .unwrap()
321+ .iter()
322+ .map(|row| row.get::<_, String>("full_name"))
323+ .eq(expected));
324+ });
277325
278− // Check that existing values can be queried using new column name
279− let expected = vec!["John Doe", "Jane Doe"];
280− assert!(new_db
281− .query("SELECT full_name FROM users ORDER BY id", &[],)
282− .unwrap()
283− .iter()
284− .map(|row| row.get::<_, String>("full_name"))
285− .eq(expected));
286−
287− reshape.complete_migration().unwrap();
288− common::assert_cleaned_up(&mut new_db);
326+ test.run();
289327}
290328
291329#[test]
292330fn alter_column_multiple() {
293− let (mut reshape, mut old_db, mut new_db) = common::setup();
294−
295− let create_users_table = Migration::new("create_user_table", None).with_action(
296− CreateTableBuilder::default()
297− .name("users")
298− .primary_key(vec!["id".to_string()])
299− .columns(vec![
300− ColumnBuilder::default()
301− .name("id")
302− .data_type("INTEGER")
303− .build()
304− .unwrap(),
305− ColumnBuilder::default()
306− .name("counter")
307− .data_type("INTEGER")
308− .nullable(false)
309− .build()
310− .unwrap(),
311− ])
312− .build()
313− .unwrap(),
331+ let mut test = Test::new("Alter column value multiple times");
332+
333+ test.first_migration(
334+ r#"
335+ name = "create_user_table"
336+
337+ [[actions]]
338+ type = "create_table"
339+ name = "users"
340+ primary_key = ["id"]
341+
342+ [[actions.columns]]
343+ name = "id"
344+ type = "INTEGER"
345+
346+ [[actions.columns]]
347+ name = "counter"
348+ type = "INTEGER"
349+ nullable = false
350+ "#,
351+ );
352+
353+ test.second_migration(
354+ r#"
355+ name = "increment_counter_twice"
356+
357+ [[actions]]
358+ type = "alter_column"
359+ table = "users"
360+ column = "counter"
361+ up = "counter + 1"
362+ down = "counter - 1"
363+
364+ [[actions]]
365+ type = "alter_column"
366+ table = "users"
367+ column = "counter"
368+ up = "counter + 1"
369+ down = "counter - 1"
370+ "#,
314371 );
315− let increment_counter_twice = Migration::new("increment_counter_twice", None)
316− .with_action(AlterColumn {
317− table: "users".to_string(),
318− column: "counter".to_string(),
319− up: Some("counter + 1".to_string()),
320− down: Some("counter - 1".to_string()),
321− changes: ColumnChanges {
322− data_type: None,
323− nullable: None,
324− name: None,
325− default: None,
326− },
327− })
328− .with_action(AlterColumn {
329− table: "users".to_string(),
330− column: "counter".to_string(),
331− up: Some("counter + 1".to_string()),
332− down: Some("counter - 1".to_string()),
333− changes: ColumnChanges {
334− data_type: None,
335− nullable: None,
336− name: None,
337− default: None,
338− },
339− });
340−
341− let first_migrations = vec![create_users_table.clone()];
342− let second_migrations = vec![create_users_table.clone(), increment_counter_twice.clone()];
343−
344− // Run first migration, should automatically finish
345− reshape.migrate(first_migrations.clone()).unwrap();
346− assert!(matches!(reshape.state.status, Status::Idle));
347−
348− // Update search paths
349− old_db
350− .simple_query(&reshape::schema_query_for_migration(
351− &first_migrations.last().unwrap().name,
352− ))
353− .unwrap();
354372
355− // Insert some test data
356− old_db
357− .simple_query(
373+ test.after_first(|db| {
374+ // Insert some test data
375+ db.simple_query(
358376 "
359377 INSERT INTO users (id, counter) VALUES
360378 (1, 0),
@@ -362,237 +380,355 @@ fn alter_column_multiple() {
362380 ",
363381 )
364382 .unwrap();
383+ });
365384
366− // Run second migration
367− reshape.migrate(second_migrations.clone()).unwrap();
368− new_db
369− .simple_query(&reshape::schema_query_for_migration(
370− &second_migrations.last().unwrap().name,
371− ))
372− .unwrap();
373−
374− // Check that the existing data has been updated
375− let expected = vec![2, 102];
376− let results: Vec<i32> = new_db
377− .query("SELECT counter FROM users ORDER BY id", &[])
378− .unwrap()
379− .iter()
380− .map(|row| row.get::<_, i32>("counter"))
381− .collect();
382− assert_eq!(expected, results);
383−
384− // Update data using old schema and make sure it was updated for the new schema
385− old_db
386− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
387− .unwrap();
388− let result: i32 = new_db
389− .query("SELECT counter FROM users WHERE id = 1", &[])
390− .unwrap()
391− .iter()
392− .map(|row| row.get("counter"))
393− .nth(0)
394− .unwrap();
395− assert_eq!(52, result);
396−
397− // Update data using new schema and make sure it was updated for the old schema
398− new_db
399− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
400− .unwrap();
401− let result: i32 = old_db
402− .query("SELECT counter FROM users WHERE id = 1", &[])
403− .unwrap()
404− .iter()
405− .map(|row| row.get("counter"))
406− .nth(0)
407− .unwrap();
408− assert_eq!(48, result);
385+ test.intermediate(|old_db, new_db| {
386+ // Check that the existing data has been updated
387+ let expected = vec![2, 102];
388+ let results: Vec<i32> = new_db
389+ .query("SELECT counter FROM users ORDER BY id", &[])
390+ .unwrap()
391+ .iter()
392+ .map(|row| row.get::<_, i32>("counter"))
393+ .collect();
394+ assert_eq!(expected, results);
395+
396+ // Update data using old schema and make sure it was updated for the new schema
397+ old_db
398+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
399+ .unwrap();
400+ let result: i32 = new_db
401+ .query("SELECT counter FROM users WHERE id = 1", &[])
402+ .unwrap()
403+ .iter()
404+ .map(|row| row.get("counter"))
405+ .nth(0)
406+ .unwrap();
407+ assert_eq!(52, result);
408+
409+ // Update data using new schema and make sure it was updated for the old schema
410+ new_db
411+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
412+ .unwrap();
413+ let result: i32 = old_db
414+ .query("SELECT counter FROM users WHERE id = 1", &[])
415+ .unwrap()
416+ .iter()
417+ .map(|row| row.get("counter"))
418+ .nth(0)
419+ .unwrap();
420+ assert_eq!(48, result);
421+ });
409422
410− reshape.complete_migration().unwrap();
411− common::assert_cleaned_up(&mut new_db);
423+ test.run();
412424}
413425
414426#[test]
415427fn alter_column_default() {
416− let (mut reshape, mut old_db, mut new_db) = common::setup();
417−
418− let create_users_table = Migration::new("create_user_table", None).with_action(
419− CreateTableBuilder::default()
420− .name("users")
421− .primary_key(vec!["id".to_string()])
422− .columns(vec![
423− ColumnBuilder::default()
424− .name("id")
425− .data_type("INTEGER")
426− .build()
427− .unwrap(),
428− ColumnBuilder::default()
429− .name("name")
430− .data_type("TEXT")
431− .nullable(false)
432− .default_value("'DEFAULT'")
433− .build()
434− .unwrap(),
435− ])
436− .build()
437− .unwrap(),
428+ let mut test = Test::new("Change default value for column");
429+
430+ test.first_migration(
431+ r#"
432+ name = "create_user_table"
433+
434+ [[actions]]
435+ type = "create_table"
436+ name = "users"
437+ primary_key = ["id"]
438+
439+ [[actions.columns]]
440+ name = "id"
441+ type = "INTEGER"
442+
443+ [[actions.columns]]
444+ name = "name"
445+ type = "TEXT"
446+ nullable = false
447+ default = "'DEFAULT'"
448+ "#,
438449 );
439− let change_name_default =
440− Migration::new("change_name_default", None).with_action(AlterColumn {
441− table: "users".to_string(),
442− column: "name".to_string(),
443− up: None,
444− down: None,
445− changes: ColumnChanges {
446− data_type: None,
447− nullable: None,
448− name: None,
449− default: Some("'NEW DEFAULT'".to_string()),
450− },
451− });
452−
453− let first_migrations = vec![create_users_table.clone()];
454− let second_migrations = vec![create_users_table.clone(), change_name_default.clone()];
455−
456− // Run first migration, should automatically finish
457− reshape.migrate(first_migrations.clone()).unwrap();
458− assert!(matches!(reshape.state.status, Status::Idle));
459−
460− // Update search paths
461− old_db
462− .simple_query(&reshape::schema_query_for_migration(
463− &first_migrations.last().unwrap().name,
464− ))
465− .unwrap();
466450
467− // Insert a test user
468− old_db
469− .simple_query(
451+ test.second_migration(
452+ r#"
453+ name = "change_name_default"
454+
455+ [[actions]]
456+ type = "alter_column"
457+ table = "users"
458+ column = "name"
459+
460+ [actions.changes]
461+ default = "'NEW DEFAULT'"
462+ "#,
463+ );
464+
465+ test.after_first(|db| {
466+ // Insert a test user
467+ db.simple_query(
470468 "
471469 INSERT INTO users (id) VALUES (1)
472470 ",
473471 )
474472 .unwrap();
473+ });
475474
476− // Run second migration
477− reshape.migrate(second_migrations.clone()).unwrap();
478− new_db
479− .simple_query(&reshape::schema_query_for_migration(
480− &second_migrations.last().unwrap().name,
481− ))
482− .unwrap();
483−
484− // Check that the existing users has the old default value
485− let expected = vec!["DEFAULT"];
486− assert!(new_db
487− .query("SELECT name FROM users", &[],)
488− .unwrap()
489− .iter()
490− .map(|row| row.get::<_, String>("name"))
491− .eq(expected));
492−
493− // Insert data using old schema and make those get the old default value
494− old_db
495− .simple_query("INSERT INTO users (id) VALUES (2)")
496− .unwrap();
497− let result = new_db
498− .query_one("SELECT name from users WHERE id = 2", &[])
499− .unwrap();
500− assert_eq!("DEFAULT", result.get::<_, &str>("name"));
501−
502− // Insert data using new schema and make sure it gets the new default value
503− new_db
504− .simple_query("INSERT INTO users (id) VALUES (3)")
505− .unwrap();
506− let result = old_db
507− .query_one("SELECT name from users WHERE id = 3", &[])
508− .unwrap();
509− assert_eq!("NEW DEFAULT", result.get::<_, &str>("name"));
475+ test.intermediate(|old_db, new_db| {
476+ // Check that the existing users has the old default value
477+ let expected = vec!["DEFAULT"];
478+ assert!(new_db
479+ .query("SELECT name FROM users", &[],)
480+ .unwrap()
481+ .iter()
482+ .map(|row| row.get::<_, String>("name"))
483+ .eq(expected));
484+
485+ // Insert data using old schema and make those get the old default value
486+ old_db
487+ .simple_query("INSERT INTO users (id) VALUES (2)")
488+ .unwrap();
489+ let result = new_db
490+ .query_one("SELECT name from users WHERE id = 2", &[])
491+ .unwrap();
492+ assert_eq!("DEFAULT", result.get::<_, &str>("name"));
493+
494+ // Insert data using new schema and make sure it gets the new default value
495+ new_db
496+ .simple_query("INSERT INTO users (id) VALUES (3)")
497+ .unwrap();
498+ let result = old_db
499+ .query_one("SELECT name from users WHERE id = 3", &[])
500+ .unwrap();
501+ assert_eq!("NEW DEFAULT", result.get::<_, &str>("name"));
502+ });
510503
511− reshape.complete_migration().unwrap();
512− common::assert_cleaned_up(&mut new_db);
504+ test.run();
513505}
514506
515507#[test]
516508fn alter_column_with_index() {
517− let (mut reshape, mut db, _) = common::setup();
518−
519− let create_users_table = Migration::new("create_user_table", None)
520− .with_action(
521− CreateTableBuilder::default()
522− .name("users")
523− .primary_key(vec!["id".to_string()])
524− .columns(vec![
525− ColumnBuilder::default()
526− .name("id")
527− .data_type("INTEGER")
528− .build()
529− .unwrap(),
530− ColumnBuilder::default()
531− .name("first_name")
532− .data_type("TEXT")
533− .build()
534− .unwrap(),
535− ColumnBuilder::default()
536− .name("last_name")
537− .data_type("TEXT")
538− .build()
539− .unwrap(),
540− ])
541− .build()
542− .unwrap(),
543− )
544− .with_action(AddIndex {
545− table: "users".to_string(),
546− name: "users_name_idx".to_string(),
547− columns: vec!["first_name".to_string(), "last_name".to_string()],
548− });
549− let uppercase_name = Migration::new("uppercase_name", None).with_action(AlterColumn {
550− table: "users".to_string(),
551− column: "last_name".to_string(),
552− up: Some("UPPER(last_name)".to_string()),
553− down: Some("LOWER(last_name)".to_string()),
554− changes: ColumnChanges {
555− data_type: None,
556− nullable: None,
557− name: None,
558− default: None,
559− },
560− });
509+ let mut test = Test::new("Alter column with index");
561510
562− let first_migrations = vec![create_users_table.clone()];
563− let second_migrations = vec![create_users_table.clone(), uppercase_name.clone()];
511+ test.first_migration(
512+ r#"
513+ name = "create_user_table"
564514
565− // Run first migration, should automatically finish
566− reshape.migrate(first_migrations.clone()).unwrap();
567− assert!(matches!(reshape.state.status, Status::Idle));
515+ [[actions]]
516+ type = "create_table"
517+ name = "users"
518+ primary_key = ["id"]
568519
569− // Run second migration
570− reshape.migrate(second_migrations.clone()).unwrap();
571− db.simple_query(&reshape::schema_query_for_migration(
572− &second_migrations.last().unwrap().name,
573− ))
574− .unwrap();
520+ [[actions.columns]]
521+ name = "id"
522+ type = "INTEGER"
575523
576− // Complete the second migration which should replace the existing column
577− // with the temporary one
578− reshape.complete_migration().unwrap();
524+ [[actions.columns]]
525+ name = "first_name"
526+ type = "TEXT"
579527
580− // Make sure index still exists
581− let result: i64 = db
582− .query(
583− "
528+ [[actions.columns]]
529+ name = "last_name"
530+ type = "TEXT"
531+
532+ [[actions]]
533+ type = "add_index"
534+ table = "users"
535+
536+ [actions.index]
537+ name = "users_name_idx"
538+ columns = ["first_name", "last_name"]
539+ "#,
540+ );
541+
542+ test.second_migration(
543+ r#"
544+ name = "uppercase_last_name"
545+
546+ [[actions]]
547+ type = "alter_column"
548+ table = "users"
549+ column = "last_name"
550+ up = "UPPER(last_name)"
551+ down = "LOWER(last_name)"
552+ "#,
553+ );
554+
555+ test.after_completion(|db| {
556+ // Make sure index still exists
557+ let result: i64 = db
558+ .query(
559+ "
584560 SELECT COUNT(*)
585561 FROM pg_catalog.pg_index
586562 JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
587563 WHERE pg_class.relname = 'users_name_idx'
588564 ",
589− &[],
590− )
591− .unwrap()
592− .first()
593− .map(|row| row.get(0))
594− .unwrap();
595− assert_eq!(1, result, "expected index to still exist");
565+ &[],
566+ )
567+ .unwrap()
568+ .first()
569+ .map(|row| row.get(0))
570+ .unwrap();
571+ assert_eq!(1, result, "expected index to still exist");
572+ });
573+
574+ test.run();
575+}
576+
577+#[test]
578+fn alter_column_with_unique_index() {
579+ let mut test = Test::new("Alter column with unique index");
580+
581+ test.first_migration(
582+ r#"
583+ name = "create_user_table"
584+
585+ [[actions]]
586+ type = "create_table"
587+ name = "users"
588+ primary_key = ["id"]
589+
590+ [[actions.columns]]
591+ name = "id"
592+ type = "INTEGER"
593+
594+ [[actions.columns]]
595+ name = "name"
596+ type = "TEXT"
597+
598+ [[actions]]
599+ type = "add_index"
600+ table = "users"
601+
602+ [actions.index]
603+ name = "name_idx"
604+ columns = ["name"]
605+ unique = true
606+ "#,
607+ );
608+
609+ test.second_migration(
610+ r#"
611+ name = "uppercase_name"
612+
613+ [[actions]]
614+ type = "alter_column"
615+ table = "users"
616+ column = "name"
617+ up = "UPPER(name)"
618+ down = "LOWER(name)"
619+ "#,
620+ );
621+
622+ test.after_first(|db| {
623+ db.simple_query("INSERT INTO users (id, name) VALUES (1, 'Test')")
624+ .unwrap();
625+ });
626+
627+ test.intermediate(|old_db, new_db| {
628+ // Try inserting a value which duplicates the uppercase value of an existing row
629+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (2, 'TEST')");
630+ assert!(
631+ result.is_err(),
632+ "expected duplicate insert to new schema to fail"
633+ );
634+
635+ // Try inserting a value which duplicates the lowercase value of an existing row
636+ new_db
637+ .simple_query("INSERT INTO users (id, name) VALUES (2, 'JOHN')")
638+ .unwrap();
639+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, 'john')");
640+ assert!(
641+ result.is_err(),
642+ "expected duplicate insert to old schema to fail"
643+ );
644+ });
645+
646+ test.after_completion(|db| {
647+ // Make sure index still exists
648+ let is_unique: bool = db
649+ .query(
650+ "
651+ SELECT pg_index.indisunique
652+ FROM pg_catalog.pg_index
653+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
654+ WHERE pg_class.relname = 'name_idx'
655+ ",
656+ &[],
657+ )
658+ .unwrap()
659+ .first()
660+ .map(|row| row.get("indisunique"))
661+ .unwrap();
662+ assert!(is_unique, "expected index to still be unique");
663+ });
664+
665+ test.run();
666+}
667+
668+#[test]
669+fn alter_column_with_hash_index() {
670+ let mut test = Test::new("Alter column with custom index type");
671+
672+ test.first_migration(
673+ r#"
674+ name = "create_user_table"
675+
676+ [[actions]]
677+ type = "create_table"
678+ name = "users"
679+ primary_key = ["id"]
680+
681+ [[actions.columns]]
682+ name = "id"
683+ type = "INTEGER"
684+
685+ [[actions.columns]]
686+ name = "name"
687+ type = "TEXT"
688+
689+ [[actions]]
690+ type = "add_index"
691+ table = "users"
692+
693+ [actions.index]
694+ name = "name_idx"
695+ columns = ["name"]
696+ type = "hash"
697+ "#,
698+ );
699+
700+ test.second_migration(
701+ r#"
702+ name = "uppercase_name"
703+
704+ [[actions]]
705+ type = "alter_column"
706+ table = "users"
707+ column = "name"
708+ up = "UPPER(name)"
709+ down = "LOWER(name)"
710+ "#,
711+ );
712+
713+ test.after_completion(|db| {
714+ // Make sure index still has type GIN
715+ let index_type: String = db
716+ .query(
717+ "
718+ SELECT pg_am.amname
719+ FROM pg_catalog.pg_index
720+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
721+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
722+ WHERE pg_class.relname = 'name_idx'
723+ ",
724+ &[],
725+ )
726+ .unwrap()
727+ .first()
728+ .map(|row| row.get("amname"))
729+ .unwrap();
730+ assert_eq!("hash", index_type);
731+ });
596732
597− common::assert_cleaned_up(&mut db);
733+ test.run();
598734}
tests/common.rs+236−9
@@ -1,17 +1,244 @@
1+use colored::Colorize;
12use postgres::{Client, NoTls};
2−use reshape::Reshape;
3+use reshape::{migrations::Migration, Reshape};
34
4−pub fn setup() -> (Reshape, Client, Client) {
5− let connection_string = std::env::var("POSTGRES_CONNECTION_STRING")
6− .unwrap_or("postgres://postgres:postgres@localhost/reshape_test".to_string());
5+pub struct Test<'a> {
6+ name: &'a str,
7+ reshape: Reshape,
8+ old_db: Client,
9+ new_db: Client,
710
8− let old_db = Client::connect(&connection_string, NoTls).unwrap();
9− let new_db = Client::connect(&connection_string, NoTls).unwrap();
11+ first_migration: Option<Migration>,
12+ second_migration: Option<Migration>,
13+ expect_failure: bool,
1014
11− let mut reshape = Reshape::new(&connection_string).unwrap();
12− reshape.remove().unwrap();
15+ clear_fn: Option<fn(&mut Client) -> ()>,
16+ after_first_fn: Option<fn(&mut Client) -> ()>,
17+ intermediate_fn: Option<fn(&mut Client, &mut Client) -> ()>,
18+ after_completion_fn: Option<fn(&mut Client) -> ()>,
19+ after_abort_fn: Option<fn(&mut Client) -> ()>,
20+}
21+
22+impl Test<'_> {
23+ pub fn new<'a>(name: &'a str) -> Test<'a> {
24+ let connection_string = std::env::var("POSTGRES_CONNECTION_STRING")
25+ .unwrap_or("postgres://postgres:postgres@localhost/reshape_test".to_string());
26+
27+ let old_db = Client::connect(&connection_string, NoTls).unwrap();
28+ let new_db = Client::connect(&connection_string, NoTls).unwrap();
29+
30+ let reshape = Reshape::new(&connection_string).unwrap();
31+
32+ Test {
33+ name,
34+ reshape,
35+ old_db,
36+ new_db,
37+ first_migration: None,
38+ second_migration: None,
39+ expect_failure: false,
40+ clear_fn: None,
41+ after_first_fn: None,
42+ intermediate_fn: None,
43+ after_completion_fn: None,
44+ after_abort_fn: None,
45+ }
46+ }
47+
48+ pub fn first_migration(&mut self, migration: &str) -> &mut Self {
49+ self.first_migration = Some(Self::parse_migration(migration));
50+ self
51+ }
52+
53+ #[allow(dead_code)]
54+ pub fn second_migration(&mut self, migration: &str) -> &mut Self {
55+ self.second_migration = Some(Self::parse_migration(migration));
56+ self
57+ }
58+
59+ #[allow(dead_code)]
60+ pub fn clear(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
61+ self.clear_fn = Some(f);
62+ self
63+ }
64+
65+ #[allow(dead_code)]
66+ pub fn after_first(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
67+ self.after_first_fn = Some(f);
68+ self
69+ }
70+
71+ #[allow(dead_code)]
72+ pub fn intermediate(&mut self, f: fn(&mut Client, &mut Client) -> ()) -> &mut Self {
73+ self.intermediate_fn = Some(f);
74+ self
75+ }
76+
77+ #[allow(dead_code)]
78+ pub fn after_completion(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
79+ self.after_completion_fn = Some(f);
80+ self
81+ }
82+
83+ #[allow(dead_code)]
84+ pub fn after_abort(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
85+ self.after_abort_fn = Some(f);
86+ self
87+ }
88+
89+ #[allow(dead_code)]
90+ pub fn expect_failure(&mut self) {
91+ self.expect_failure = true;
92+ }
93+
94+ fn parse_migration(encoded: &str) -> Migration {
95+ toml::from_str(encoded).unwrap()
96+ }
97+}
98+
99+enum RunType {
100+ Simple,
101+ Completion,
102+ Abort,
103+}
104+
105+impl Test<'_> {
106+ #[allow(dead_code)]
107+ pub fn run(&mut self) {
108+ if self.second_migration.is_some() {
109+ // Run to completion
110+ print_heading(&format!("Test completion: {}", self.name));
111+ self.run_internal(RunType::Completion);
112+
113+ // Run and abort
114+ print_heading(&format!("Test abort: {}", self.name));
115+ self.run_internal(RunType::Abort);
116+ } else {
117+ print_heading(&format!("Test: {}", self.name));
118+ self.run_internal(RunType::Simple);
119+ }
120+ }
121+
122+ fn run_internal(&mut self, run_type: RunType) {
123+ print_subheading("Clearing database");
124+ self.reshape.remove().unwrap();
125+
126+ if let Some(clear_fn) = self.clear_fn {
127+ clear_fn(&mut self.old_db);
128+ }
129+
130+ // Apply first migration, will automatically complete
131+ print_subheading("Applying first migration");
132+ let first_migration = self
133+ .first_migration
134+ .as_ref()
135+ .expect("no starting migration set");
136+ self.reshape.migrate(vec![first_migration.clone()]).unwrap();
137+
138+ // Update search path
139+ self.old_db
140+ .simple_query(&reshape::schema_query_for_migration(&first_migration.name))
141+ .unwrap();
142+
143+ // Automatically complete first migration
144+ self.reshape.complete().unwrap();
145+
146+ // Run setup function
147+ if let Some(after_first_fn) = self.after_first_fn {
148+ print_subheading("Running setup and first checks");
149+ after_first_fn(&mut self.old_db);
150+ print_success();
151+ }
152+
153+ // Apply second migration
154+ if let Some(second_migration) = &self.second_migration {
155+ if self.expect_failure {
156+ print_subheading("Applying second migration (expecting failure)");
157+ let result = self
158+ .reshape
159+ .migrate(vec![first_migration.clone(), second_migration.clone()]);
160+
161+ if result.is_ok() {
162+ panic!("expected second migration to fail");
163+ }
164+ } else {
165+ print_subheading("Applying second migration");
166+ self.reshape
167+ .migrate(vec![first_migration.clone(), second_migration.clone()])
168+ .unwrap();
169+ }
170+
171+ // Update search path
172+ self.new_db
173+ .simple_query(&reshape::schema_query_for_migration(&second_migration.name))
174+ .unwrap();
175+
176+ if let Some(intermediate_fn) = self.intermediate_fn {
177+ print_subheading("Running intermediate checks");
178+ intermediate_fn(&mut self.old_db, &mut self.new_db);
179+ print_success();
180+ }
181+
182+ match run_type {
183+ RunType::Completion => {
184+ print_subheading("Completing");
185+ self.reshape.complete().unwrap();
186+
187+ if let Some(after_completion_fn) = self.after_completion_fn {
188+ print_subheading("Running post-completion checks");
189+ after_completion_fn(&mut self.new_db);
190+ print_success();
191+ }
192+ }
193+ RunType::Abort => {
194+ print_subheading("Aborting");
195+ self.reshape.abort().unwrap();
196+
197+ if let Some(after_abort_fn) = self.after_abort_fn {
198+ print_subheading("Running post-abort checks");
199+ after_abort_fn(&mut self.old_db);
200+ print_success();
201+ }
202+ }
203+ _ => {}
204+ }
205+ }
206+
207+ print_subheading("Checking cleanup");
208+ assert_cleaned_up(&mut self.new_db);
209+ print_success();
210+ }
211+}
212+
213+fn print_heading(text: &str) {
214+ let delimiter = std::iter::repeat("=").take(80).collect::<String>();
215+
216+ println!();
217+ println!();
218+ println!("{}", delimiter.blue().bold());
219+ println!("{}", add_spacer(text, "=").blue().bold());
220+ println!("{}", delimiter.blue().bold());
221+}
222+
223+fn print_subheading(text: &str) {
224+ println!();
225+ println!("{}", add_spacer(text, "=").blue());
226+}
227+
228+fn print_success() {
229+ println!("{}", add_spacer("Success", "=").green());
230+}
231+
232+fn add_spacer(text: &str, char: &str) -> String {
233+ const TARGET_WIDTH: usize = 80;
234+ let num_of_chars = (TARGET_WIDTH - text.len() - 2) / 2;
235+ let spacer = std::iter::repeat(char)
236+ .take(num_of_chars)
237+ .collect::<String>();
238+
239+ let extra = if text.len() % 2 == 0 { "" } else { char };
13240
14− (reshape, old_db, new_db)
241+ format!("{spacer} {text} {spacer}{extra}", spacer = spacer)
15242}
16243
17244pub fn assert_cleaned_up(db: &mut Client) {
tests/complex.rsnew file+379
@@ -0,0 +1,379 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn move_column_between_tables() {
6+ let mut test = Test::new("Move column between tables");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "email"
23+ type = "TEXT"
24+
25+ [[actions]]
26+ type = "create_table"
27+ name = "profiles"
28+ primary_key = ["id"]
29+
30+ [[actions.columns]]
31+ name = "id"
32+ type = "INTEGER"
33+
34+ [[actions.columns]]
35+ name = "user_id"
36+ type = "INTEGER"
37+ nullable = false
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "move_email_column"
44+
45+ [[actions]]
46+ type = "add_column"
47+ table = "profiles"
48+
49+ [actions.column]
50+ name = "email"
51+ type = "TEXT"
52+ nullable = false
53+
54+ # When `users` is updated in the old schema, we write the email value to `profiles`
55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated
56+ [actions.up]
57+ table = "users"
58+ value = "users.email"
59+ where = "profiles.user_id = users.id"
60+
61+ [[actions]]
62+ type = "remove_column"
63+ table = "users"
64+ column = "email"
65+
66+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
67+ [actions.down]
68+ table = "profiles"
69+ value = "profiles.email"
70+ where = "users.id = profiles.user_id"
71+ "#,
72+ );
73+
74+ test.after_first(|db| {
75+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")
76+ .unwrap();
77+ db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")
78+ .unwrap();
79+
80+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")
81+ .unwrap();
82+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")
83+ .unwrap();
84+ });
85+
86+ test.intermediate(|old_db, new_db| {
87+ // Ensure emails were backfilled into profiles
88+ let profiles_emails: Vec<String> = new_db
89+ .query(
90+ r#"
91+ SELECT email
92+ FROM profiles
93+ ORDER BY id
94+ "#,
95+ &[],
96+ )
97+ .unwrap()
98+ .iter()
99+ .map(|row| row.get("email"))
100+ .collect();
101+ assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);
102+
103+ // Ensure insert in old schema updates new
104+ old_db
105+ .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")
106+ .unwrap();
107+ old_db
108+ .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")
109+ .unwrap();
110+ let new_email: String = new_db
111+ .query("SELECT email FROM profiles WHERE id = 3", &[])
112+ .unwrap()
113+ .first()
114+ .map(|row| row.get("email"))
115+ .unwrap();
116+ assert_eq!("test3@test.com", new_email);
117+
118+ // Ensure updates in old schema updates new
119+ old_db
120+ .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")
121+ .unwrap();
122+ let new_email: String = new_db
123+ .query("SELECT email FROM profiles WHERE id = 3", &[])
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get("email"))
127+ .unwrap();
128+ assert_eq!("test3+updated@test.com", new_email);
129+
130+ // Ensure insert in new schema updates old
131+ new_db
132+ .simple_query(
133+ "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",
134+ )
135+ .unwrap();
136+ new_db
137+ .simple_query("INSERT INTO users (id) VALUES (4)")
138+ .unwrap();
139+ let old_email: String = old_db
140+ .query("SELECT email FROM users WHERE id = 4", &[])
141+ .unwrap()
142+ .first()
143+ .map(|row| row.get("email"))
144+ .unwrap();
145+ assert_eq!("test4@test.com", old_email);
146+
147+ // Ensure update in new schema updates old
148+ new_db
149+ .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")
150+ .unwrap();
151+ let old_email: String = old_db
152+ .query("SELECT email FROM users WHERE id = 4", &[])
153+ .unwrap()
154+ .first()
155+ .map(|row| row.get("email"))
156+ .unwrap();
157+ assert_eq!("test4+updated@test.com", old_email);
158+ });
159+
160+ test.run();
161+}
162+
163+#[test]
164+fn extract_relation_into_new_table() {
165+ let mut test = Test::new("Extract relation into new table");
166+
167+ test.first_migration(
168+ r#"
169+ name = "create_tables"
170+
171+ [[actions]]
172+ type = "create_table"
173+ name = "accounts"
174+ primary_key = ["id"]
175+
176+ [[actions.columns]]
177+ name = "id"
178+ type = "INTEGER"
179+
180+ [[actions]]
181+ type = "create_table"
182+ name = "users"
183+ primary_key = ["id"]
184+
185+ [[actions.columns]]
186+ name = "id"
187+ type = "INTEGER"
188+
189+ [[actions.columns]]
190+ name = "account_id"
191+ type = "INTEGER"
192+ nullable = false
193+
194+ [[actions.columns]]
195+ name = "account_role"
196+ type = "TEXT"
197+ nullable = false
198+ "#,
199+ );
200+
201+ test.second_migration(
202+ r#"
203+ name = "add_account_user_connection"
204+
205+ [[actions]]
206+ type = "create_table"
207+ name = "user_account_connections"
208+ primary_key = ["account_id", "user_id"]
209+
210+ [[actions.columns]]
211+ name = "account_id"
212+ type = "INTEGER"
213+
214+ [[actions.columns]]
215+ name = "user_id"
216+ type = "INTEGER"
217+
218+ [[actions.columns]]
219+ name = "role"
220+ type = "TEXT"
221+ nullable = false
222+
223+ [actions.up]
224+ table = "users"
225+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
226+ where = "user_account_connections.user_id = users.id"
227+
228+ [[actions]]
229+ type = "remove_column"
230+ table = "users"
231+ column = "account_id"
232+
233+ [actions.down]
234+ table = "user_account_connections"
235+ value = "user_account_connections.account_id"
236+ where = "users.id = user_account_connections.user_id"
237+
238+ [[actions]]
239+ type = "remove_column"
240+ table = "users"
241+ column = "account_role"
242+
243+ [actions.down]
244+ table = "user_account_connections"
245+ value = "LOWER(user_account_connections.role)"
246+ where = "users.id = user_account_connections.user_id"
247+ "#,
248+ );
249+
250+ test.after_first(|db| {
251+ db.simple_query("INSERT INTO accounts (id) VALUES (1)")
252+ .unwrap();
253+ db.simple_query("INSERT INTO users (id, account_id, account_role) VALUES (1, 1, 'admin')")
254+ .unwrap();
255+ });
256+
257+ test.intermediate(|old_db, new_db| {
258+ // Ensure connections was backfilled
259+ let rows: Vec<(i32, i32, String)> = new_db
260+ .query(
261+ "
262+ SELECT account_id, user_id, role
263+ FROM user_account_connections
264+ ",
265+ &[],
266+ )
267+ .unwrap()
268+ .iter()
269+ .map(|row| (row.get("account_id"), row.get("user_id"), row.get("role")))
270+ .collect();
271+ assert_eq!(1, rows.len());
272+
273+ let row = rows.first().unwrap();
274+ assert_eq!(1, row.0);
275+ assert_eq!(1, row.1);
276+ assert_eq!("ADMIN", row.2);
277+
278+ // Ensure inserted user in old schema creates a new connection
279+ old_db
280+ .simple_query(
281+ "INSERT INTO users (id, account_id, account_role) VALUES (2, 1, 'developer')",
282+ )
283+ .unwrap();
284+ assert!(
285+ new_db
286+ .query(
287+ "
288+ SELECT account_id, user_id, role
289+ FROM user_account_connections
290+ WHERE account_id = 1 AND user_id = 2 AND role = 'DEVELOPER'
291+ ",
292+ &[],
293+ )
294+ .unwrap()
295+ .len()
296+ == 1
297+ );
298+
299+ // Ensure NOT NULL constraint still applies to old schema
300+ let result = old_db
301+ .simple_query(
302+ "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",
303+ );
304+ assert!(result.is_err());
305+
306+ // Ensure updated user role in old schema updates connection in new schema
307+ old_db
308+ .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")
309+ .unwrap();
310+ assert!(
311+ new_db
312+ .query(
313+ "
314+ SELECT account_id, user_id, role
315+ FROM user_account_connections
316+ WHERE account_id = 1 AND user_id = 2 AND role = 'ADMIN'
317+ ",
318+ &[],
319+ )
320+ .unwrap()
321+ .len()
322+ == 1
323+ );
324+
325+ // Ensure updated connection in new schema updates old schema user
326+ new_db
327+ .simple_query(
328+ "UPDATE user_account_connections SET role = 'DEVELOPER' WHERE account_id = 1 AND user_id = 2",
329+ )
330+ .unwrap();
331+ assert!(
332+ old_db
333+ .query(
334+ "
335+ SELECT id
336+ FROM users
337+ WHERE id = 2 AND account_id = 1 AND account_role = 'developer'
338+ ",
339+ &[],
340+ )
341+ .unwrap()
342+ .len()
343+ == 1
344+ );
345+
346+ // Ensure insert of user with connection through new schema updates user in old schema
347+ new_db
348+ .simple_query(
349+ r#"
350+ BEGIN;
351+ INSERT INTO users (id) VALUES (3);
352+ INSERT INTO user_account_connections (user_id, account_id, role) VALUES (3, 1, 'DEVELOPER');
353+ COMMIT;
354+ "#,
355+ )
356+ .unwrap();
357+ new_db
358+ .simple_query(
359+ "",
360+ )
361+ .unwrap();
362+ assert!(
363+ old_db
364+ .query(
365+ "
366+ SELECT id
367+ FROM users
368+ WHERE id = 3 AND account_id = 1 AND account_role = 'developer'
369+ ",
370+ &[],
371+ )
372+ .unwrap()
373+ .len()
374+ == 1
375+ );
376+ });
377+
378+ test.run();
379+}
tests/create_enum.rsnew file+41
@@ -0,0 +1,41 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn create_enum() {
6+ let mut test = Test::new("Create enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum_and_table"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+
17+ [[actions]]
18+ type = "create_table"
19+ name = "updates"
20+ primary_key = ["id"]
21+
22+ [[actions.columns]]
23+ name = "id"
24+ type = "INTEGER"
25+
26+ [[actions.columns]]
27+ name = "status"
28+ type = "mood"
29+ "#,
30+ );
31+
32+ test.after_first(|db| {
33+ // Valid enum values should succeed
34+ db.simple_query(
35+ "INSERT INTO updates (id, status) VALUES (1, 'happy'), (2, 'ok'), (3, 'sad')",
36+ )
37+ .unwrap();
38+ });
39+
40+ test.run();
41+}
tests/create_table.rs+185−193
@@ -1,208 +1,200 @@
1−use reshape::{
2− migrations::{ColumnBuilder, CreateTableBuilder, ForeignKey, Migration},
3− Status,
4−};
5−
61mod common;
2+use common::Test;
73
84#[test]
95fn create_table() {
10− let (mut reshape, mut db, _) = common::setup();
11−
12− let create_table_migration = Migration::new("create_users_table", None).with_action(
13− CreateTableBuilder::default()
14− .name("users")
15− .primary_key(vec!["id".to_string()])
16− .columns(vec![
17− ColumnBuilder::default()
18− .name("id")
19− .data_type("INTEGER")
20− .generated("ALWAYS AS IDENTITY")
21− .build()
22− .unwrap(),
23− ColumnBuilder::default()
24− .name("name")
25− .data_type("TEXT")
26− .build()
27− .unwrap(),
28− ColumnBuilder::default()
29− .name("created_at")
30− .data_type("TIMESTAMP")
31− .nullable(false)
32− .default_value("NOW()")
33− .build()
34− .unwrap(),
35− ])
36− .build()
37− .unwrap(),
6+ let mut test = Test::new("Create table");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ generated = "ALWAYS AS IDENTITY"
21+
22+ [[actions.columns]]
23+ name = "name"
24+ type = "TEXT"
25+
26+ [[actions.columns]]
27+ name = "created_at"
28+ type = "TIMESTAMP"
29+ nullable = false
30+ default = "NOW()"
31+ "#,
3832 );
3933
40− reshape
41− .migrate(vec![create_table_migration.clone()])
42− .unwrap();
43− assert!(matches!(reshape.state.status, Status::Idle));
44−
45− // Ensure table was created
46− let result = db
47− .query_opt(
48− "
49− SELECT table_name
50− FROM information_schema.tables
51− WHERE table_name = 'users' AND table_schema = 'public'",
52− &[],
53− )
54− .unwrap();
55− assert!(result.is_some());
56−
57− // Ensure table has the right columns
58− let result = db
59− .query(
60− "
61− SELECT column_name, column_default, is_nullable, data_type
62− FROM information_schema.columns
63− WHERE table_name = 'users' AND table_schema = 'public'
64− ORDER BY ordinal_position",
65− &[],
66− )
67− .unwrap();
68−
69− // id column
70− let id_row = &result[0];
71− assert_eq!("id", id_row.get::<_, String>("column_name"));
72− assert!(id_row.get::<_, Option<String>>("column_default").is_none());
73− assert_eq!("NO", id_row.get::<_, String>("is_nullable"));
74− assert_eq!("integer", id_row.get::<_, String>("data_type"));
75−
76− // name column
77− let name_row = &result[1];
78− assert_eq!("name", name_row.get::<_, String>("column_name"));
79− assert!(name_row
80− .get::<_, Option<String>>("column_default")
81− .is_none());
82− assert_eq!("YES", name_row.get::<_, String>("is_nullable"));
83− assert_eq!("text", name_row.get::<_, String>("data_type"));
84−
85− // created_at column
86− let created_at_column = &result[2];
87− assert_eq!(
88− "created_at",
89− created_at_column.get::<_, String>("column_name")
90− );
91− assert!(created_at_column
92− .get::<_, Option<String>>("column_default")
93− .is_some());
94− assert_eq!("NO", created_at_column.get::<_, String>("is_nullable"));
95− assert_eq!(
96− "timestamp without time zone",
97− created_at_column.get::<_, String>("data_type")
98− );
34+ test.after_first(|db| {
35+ // Ensure table was created
36+ let result = db
37+ .query_opt(
38+ "
39+ SELECT table_name
40+ FROM information_schema.tables
41+ WHERE table_name = 'users' AND table_schema = 'public'",
42+ &[],
43+ )
44+ .unwrap();
45+ assert!(result.is_some());
46+
47+ // Ensure table has the right columns
48+ let result = db
49+ .query(
50+ "
51+ SELECT column_name, column_default, is_nullable, data_type
52+ FROM information_schema.columns
53+ WHERE table_name = 'users' AND table_schema = 'public'
54+ ORDER BY ordinal_position",
55+ &[],
56+ )
57+ .unwrap();
58+
59+ // id column
60+ let id_row = &result[0];
61+ assert_eq!("id", id_row.get::<_, String>("column_name"));
62+ assert!(id_row.get::<_, Option<String>>("column_default").is_none());
63+ assert_eq!("NO", id_row.get::<_, String>("is_nullable"));
64+ assert_eq!("integer", id_row.get::<_, String>("data_type"));
65+
66+ // name column
67+ let name_row = &result[1];
68+ assert_eq!("name", name_row.get::<_, String>("column_name"));
69+ assert!(name_row
70+ .get::<_, Option<String>>("column_default")
71+ .is_none());
72+ assert_eq!("YES", name_row.get::<_, String>("is_nullable"));
73+ assert_eq!("text", name_row.get::<_, String>("data_type"));
74+
75+ // created_at column
76+ let created_at_column = &result[2];
77+ assert_eq!(
78+ "created_at",
79+ created_at_column.get::<_, String>("column_name")
80+ );
81+ assert!(created_at_column
82+ .get::<_, Option<String>>("column_default")
83+ .is_some());
84+ assert_eq!("NO", created_at_column.get::<_, String>("is_nullable"));
85+ assert_eq!(
86+ "timestamp without time zone",
87+ created_at_column.get::<_, String>("data_type")
88+ );
89+
90+ // Ensure the primary key has the right columns
91+ let primary_key_columns: Vec<String> = db
92+ .query(
93+ "
94+ SELECT a.attname AS column
95+ FROM pg_index i
96+ JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
97+ JOIN pg_class t ON t.oid = i.indrelid
98+ WHERE t.relname = 'users' AND i.indisprimary
99+ ",
100+ &[],
101+ )
102+ .unwrap()
103+ .iter()
104+ .map(|row| row.get("column"))
105+ .collect();
99106
100− // Ensure the primary key has the right columns
101− let primary_key_columns: Vec<String> = db
102− .query(
103− "
104− SELECT a.attname AS column
105− FROM pg_index i
106− JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
107− WHERE i.indrelid = 'users'::regclass AND i.indisprimary
108− ",
109− &[],
110− )
111− .unwrap()
112− .iter()
113− .map(|row| row.get("column"))
114− .collect();
115−
116− assert_eq!(vec!["id"], primary_key_columns);
117− common::assert_cleaned_up(&mut db);
107+ assert_eq!(vec!["id"], primary_key_columns);
108+ });
109+
110+ test.run();
118111}
119112
120113#[test]
121114fn create_table_with_foreign_keys() {
122− let (mut reshape, mut db, _) = common::setup();
123−
124− let create_table_migration = Migration::new("create_users_table", None).with_action(
125− CreateTableBuilder::default()
126− .name("users")
127− .primary_key(vec!["id".to_string()])
128− .columns(vec![ColumnBuilder::default()
129− .name("id")
130− .data_type("INTEGER")
131− .generated("ALWAYS AS IDENTITY")
132− .build()
133− .unwrap()])
134− .build()
135− .unwrap(),
136− );
137−
138− let create_second_table_migration = Migration::new("create_items_table", None).with_action(
139− CreateTableBuilder::default()
140− .name("items")
141− .primary_key(vec!["id".to_string()])
142− .foreign_keys(vec![ForeignKey {
143− columns: vec!["user_id".to_string()],
144− referenced_table: "users".to_string(),
145− referenced_columns: vec!["id".to_string()],
146− }])
147− .columns(vec![
148− ColumnBuilder::default()
149− .name("id")
150− .data_type("INTEGER")
151− .generated("ALWAYS AS IDENTITY")
152− .build()
153− .unwrap(),
154− ColumnBuilder::default()
155− .name("user_id")
156− .data_type("INTEGER")
157− .nullable(false)
158− .build()
159− .unwrap(),
160− ])
161− .build()
162− .unwrap(),
115+ let mut test = Test::new("Create table");
116+
117+ test.first_migration(
118+ r#"
119+ name = "create_users_table"
120+
121+ [[actions]]
122+ type = "create_table"
123+ name = "users"
124+ primary_key = ["id"]
125+
126+ [[actions.columns]]
127+ name = "id"
128+ type = "INTEGER"
129+ generated = "ALWAYS AS IDENTITY"
130+
131+ [[actions.columns]]
132+ name = "name"
133+ type = "TEXT"
134+
135+ [[actions.columns]]
136+ name = "created_at"
137+ type = "TIMESTAMP"
138+ nullable = false
139+ default = "NOW()"
140+
141+ [[actions]]
142+ type = "create_table"
143+ name = "items"
144+ primary_key = ["id"]
145+
146+ [[actions.columns]]
147+ name = "id"
148+ type = "INTEGER"
149+
150+ [[actions.columns]]
151+ name = "user_id"
152+ type = "INTEGER"
153+ nullable = false
154+
155+ [[actions.foreign_keys]]
156+ columns = ["user_id"]
157+ referenced_table = "users"
158+ referenced_columns = ["id"]
159+ "#,
163160 );
164161
165− reshape
166− .migrate(vec![
167− create_table_migration.clone(),
168− create_second_table_migration.clone(),
169− ])
170− .unwrap();
171−
172− let foreign_key_columns: Vec<(String, String, String)> = db
173− .query(
174− "
175− SELECT
176− kcu.column_name,
177− ccu.table_name AS foreign_table_name,
178− ccu.column_name AS foreign_column_name
179− FROM
180− information_schema.table_constraints AS tc
181− JOIN information_schema.key_column_usage AS kcu
182− ON tc.constraint_name = kcu.constraint_name
183− AND tc.table_schema = kcu.table_schema
184− JOIN information_schema.constraint_column_usage AS ccu
185− ON ccu.constraint_name = tc.constraint_name
186− AND ccu.table_schema = tc.table_schema
187− WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
188− ",
189− &[],
190− )
191− .unwrap()
192− .iter()
193− .map(|row| {
194− (
195− row.get("column_name"),
196− row.get("foreign_table_name"),
197− row.get("foreign_column_name"),
162+ test.after_first(|db| {
163+ let foreign_key_columns: Vec<(String, String, String)> = db
164+ .query(
165+ "
166+ SELECT
167+ kcu.column_name,
168+ ccu.table_name AS foreign_table_name,
169+ ccu.column_name AS foreign_column_name
170+ FROM
171+ information_schema.table_constraints AS tc
172+ JOIN information_schema.key_column_usage AS kcu
173+ ON tc.constraint_name = kcu.constraint_name
174+ AND tc.table_schema = kcu.table_schema
175+ JOIN information_schema.constraint_column_usage AS ccu
176+ ON ccu.constraint_name = tc.constraint_name
177+ AND ccu.table_schema = tc.table_schema
178+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
179+ ",
180+ &[],
198181 )
199− })
200− .collect();
201−
202− assert_eq!(
203− vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
204− foreign_key_columns
205− );
206−
207− common::assert_cleaned_up(&mut db);
182+ .unwrap()
183+ .iter()
184+ .map(|row| {
185+ (
186+ row.get("column_name"),
187+ row.get("foreign_table_name"),
188+ row.get("foreign_column_name"),
189+ )
190+ })
191+ .collect();
192+
193+ assert_eq!(
194+ vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
195+ foreign_key_columns
196+ );
197+ });
198+
199+ test.run();
208200}
tests/custom.rsnew file+128
@@ -0,0 +1,128 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn custom_enable_extension() {
6+ let mut test = Test::new("Custom migration");
7+
8+ test.clear(|db| {
9+ db.simple_query(
10+ "
11+ DROP EXTENSION IF EXISTS bloom;
12+ DROP EXTENSION IF EXISTS btree_gin;
13+ DROP EXTENSION IF EXISTS btree_gist;
14+ ",
15+ )
16+ .unwrap();
17+ });
18+
19+ test.first_migration(
20+ r#"
21+ name = "empty_migration"
22+
23+ [[actions]]
24+ type = "custom"
25+ "#,
26+ );
27+
28+ test.second_migration(
29+ r#"
30+ name = "enable_extensions"
31+
32+ [[actions]]
33+ type = "custom"
34+
35+ start = """
36+ CREATE EXTENSION IF NOT EXISTS bloom;
37+ CREATE EXTENSION IF NOT EXISTS btree_gin;
38+ """
39+
40+ complete = "CREATE EXTENSION IF NOT EXISTS btree_gist"
41+
42+ abort = """
43+ DROP EXTENSION IF EXISTS bloom;
44+ DROP EXTENSION IF EXISTS btree_gin;
45+ """
46+ "#,
47+ );
48+
49+ test.intermediate(|db, _| {
50+ let bloom_activated = !db
51+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
52+ .unwrap()
53+ .is_empty();
54+ assert!(bloom_activated);
55+
56+ let btree_gin_activated = !db
57+ .query(
58+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
59+ &[],
60+ )
61+ .unwrap()
62+ .is_empty();
63+ assert!(btree_gin_activated);
64+
65+ let btree_gist_activated = !db
66+ .query(
67+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
68+ &[],
69+ )
70+ .unwrap()
71+ .is_empty();
72+ assert!(!btree_gist_activated);
73+ });
74+
75+ test.after_completion(|db| {
76+ let bloom_activated = !db
77+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
78+ .unwrap()
79+ .is_empty();
80+ assert!(bloom_activated);
81+
82+ let btree_gin_activated = !db
83+ .query(
84+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
85+ &[],
86+ )
87+ .unwrap()
88+ .is_empty();
89+ assert!(btree_gin_activated);
90+
91+ let btree_gist_activated = !db
92+ .query(
93+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
94+ &[],
95+ )
96+ .unwrap()
97+ .is_empty();
98+ assert!(btree_gist_activated);
99+ });
100+
101+ test.after_abort(|db| {
102+ let bloom_activated = !db
103+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
104+ .unwrap()
105+ .is_empty();
106+ assert!(!bloom_activated);
107+
108+ let btree_gin_activated = !db
109+ .query(
110+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
111+ &[],
112+ )
113+ .unwrap()
114+ .is_empty();
115+ assert!(!btree_gin_activated);
116+
117+ let btree_gist_activated = !db
118+ .query(
119+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
120+ &[],
121+ )
122+ .unwrap()
123+ .is_empty();
124+ assert!(!btree_gist_activated);
125+ });
126+
127+ test.run();
128+}
tests/failure.rs+35−63
@@ -1,79 +1,51 @@
1−use reshape::migrations::{AddColumn, Column, CreateTable, Migration};
2−use reshape::Status;
3−
41mod common;
2+use common::Test;
53
64#[test]
75fn invalid_migration() {
8− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Invalid migration");
97
10− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![
15− Column {
16− name: "id".to_string(),
17− data_type: "SERIAL".to_string(),
18− nullable: true,
19− default: None,
20− generated: None,
21− },
22− Column {
23− name: "name".to_string(),
24− data_type: "TEXT".to_string(),
25− nullable: false,
26− default: None,
27− generated: None,
28− },
29− ],
30− });
31− let add_first_column = Migration::new("add_first_column", None).with_action(AddColumn {
32− table: "users".to_string(),
33− column: Column {
34− name: "first".to_string(),
35− data_type: "TEXT".to_string(),
36− nullable: false,
37− default: None,
38− generated: None,
39− },
40− up: Some("INVALID SQL".to_string()),
41− });
8+ test.first_migration(
9+ r#"
10+ name = "invalid_migration"
4211
43− let first_migrations = vec![create_users_table.clone()];
44− let second_migrations = vec![create_users_table.clone(), add_first_column.clone()];
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
4516
46− // Run first migration, should automatically finish
47− reshape.migrate(first_migrations.clone()).unwrap();
48− assert!(matches!(reshape.state.status, Status::Idle));
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
4922
50− // Update search paths
51− old_db
52− .simple_query(&reshape::schema_query_for_migration(
53− &first_migrations.last().unwrap().name,
54− ))
55− .unwrap();
56− new_db
57− .simple_query(&reshape::schema_query_for_migration(
58− &first_migrations.last().unwrap().name,
59− ))
60− .unwrap();
23+ test.second_migration(
24+ r#"
25+ name = "add_invalid_column"
26+
27+ [[actions]]
28+ type = "add_column"
29+ table = "users"
30+
31+ up = "INVALID SQL"
32+
33+ [actions.column]
34+ name = "first"
35+ type = "TEXT"
36+ "#,
37+ );
6138
6239 // Insert a test user
63− new_db
64− .simple_query(
40+ test.after_first(|db| {
41+ db.simple_query(
6542 "
66− INSERT INTO users (id, name) VALUES
67− (1, 'John Doe')
43+ INSERT INTO users (id) VALUES (1)
6844 ",
6945 )
7046 .unwrap();
47+ });
7148
72− // Run second migration and ensure it fails
73− assert!(
74− reshape.migrate(second_migrations.clone()).is_err(),
75− "invalid migration succeeded"
76− );
77−
78− common::assert_cleaned_up(&mut new_db);
49+ test.expect_failure();
50+ test.run();
7951}
tests/remove_column.rs+209−73
@@ -1,81 +1,217 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RemoveColumn};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn remove_column() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
10− CreateTableBuilder::default()
11− .name("users")
12− .primary_key(vec!["id".to_string()])
13− .columns(vec![
14− ColumnBuilder::default()
15− .name("id")
16− .data_type("INTEGER")
17− .build()
18− .unwrap(),
19− ColumnBuilder::default()
20− .name("name")
21− .data_type("TEXT")
22− .nullable(false)
23− .build()
24− .unwrap(),
25− ])
26− .build()
27− .unwrap(),
6+ let mut test = Test::new("Remove column");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "remove_name_column"
30+
31+ [[actions]]
32+ type = "remove_column"
33+ table = "users"
34+ column = "name"
35+ down = "'TEST_DOWN_VALUE'"
36+ "#,
37+ );
38+
39+ test.intermediate(|old_db, new_db| {
40+ // Insert using old schema and ensure it can be retrieved through new schema
41+ old_db
42+ .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
43+ .unwrap();
44+ let results = new_db
45+ .query("SELECT id FROM users WHERE id = 1", &[])
46+ .unwrap();
47+ assert_eq!(1, results.len());
48+ assert_eq!(1, results[0].get::<_, i32>("id"));
49+
50+ // Ensure the name column is not accesible through the new schema
51+ assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
52+
53+ // Insert using new schema and ensure the down function is correctly applied
54+ new_db
55+ .simple_query("INSERT INTO users(id) VALUES (2)")
56+ .unwrap();
57+ let result = old_db
58+ .query_opt("SELECT name FROM users WHERE id = 2", &[])
59+ .unwrap();
60+ assert_eq!(
61+ Some("TEST_DOWN_VALUE"),
62+ result.as_ref().map(|row| row.get("name"))
63+ );
64+ });
65+
66+ test.run();
67+}
68+
69+#[test]
70+fn remove_column_with_index() {
71+ let mut test = Test::new("Remove column");
72+
73+ test.first_migration(
74+ r#"
75+ name = "create_user_table"
76+
77+ [[actions]]
78+ type = "create_table"
79+ name = "users"
80+ primary_key = ["id"]
81+
82+ [[actions.columns]]
83+ name = "id"
84+ type = "INTEGER"
85+
86+ [[actions.columns]]
87+ name = "name"
88+ type = "TEXT"
89+
90+ [[actions]]
91+ type = "add_index"
92+ table = "users"
93+
94+ [actions.index]
95+ name = "name_idx"
96+ columns = ["name"]
97+ "#,
98+ );
99+
100+ test.second_migration(
101+ r#"
102+ name = "remove_name_column"
103+
104+ [[actions]]
105+ type = "remove_column"
106+ table = "users"
107+ column = "name"
108+ down = "'TEST_DOWN_VALUE'"
109+ "#,
110+ );
111+
112+ test.after_completion(|db| {
113+ // Ensure index has been removed after the migration is complete
114+ let count: i64 = db
115+ .query(
116+ "
117+ SELECT COUNT(*)
118+ FROM pg_catalog.pg_index
119+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
120+ WHERE pg_class.relname = 'name_idx'
121+ ",
122+ &[],
123+ )
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get(0))
127+ .unwrap();
128+
129+ assert_eq!(0, count, "expected index to not exist");
130+ });
131+
132+ test.run();
133+}
134+
135+#[test]
136+fn remove_column_with_complex_down() {
137+ let mut test = Test::new("Remove column complex");
138+
139+ test.first_migration(
140+ r#"
141+ name = "create_tables"
142+
143+ [[actions]]
144+ type = "create_table"
145+ name = "users"
146+ primary_key = ["id"]
147+
148+ [[actions.columns]]
149+ name = "id"
150+ type = "INTEGER"
151+
152+ [[actions.columns]]
153+ name = "email"
154+ type = "TEXT"
155+
156+ [[actions]]
157+ type = "create_table"
158+ name = "profiles"
159+ primary_key = ["user_id"]
160+
161+ [[actions.columns]]
162+ name = "user_id"
163+ type = "INTEGER"
164+
165+ [[actions.columns]]
166+ name = "email"
167+ type = "TEXT"
168+ "#,
28169 );
29− let remove_column_migration =
30− Migration::new("remove_name_column", None).with_action(RemoveColumn {
31− table: "users".to_string(),
32− column: "name".to_string(),
33− down: Some("'TEST_DOWN_VALUE'".to_string()),
34− });
35−
36− let first_migrations = vec![create_table_migration.clone()];
37− let second_migrations = vec![
38− create_table_migration.clone(),
39− remove_column_migration.clone(),
40− ];
41−
42− // Run migrations
43− reshape.migrate(first_migrations.clone()).unwrap();
44− reshape.migrate(second_migrations.clone()).unwrap();
45−
46− // Update schemas of Postgres connections
47− let old_schema_query =
48− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
49− let new_schema_query =
50− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
51− old_db.simple_query(&old_schema_query).unwrap();
52− new_db.simple_query(&new_schema_query).unwrap();
53−
54− // Insert using old schema and ensure it can be retrieved through new schema
55− old_db
56− .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
57− .unwrap();
58− let results = new_db
59− .query("SELECT id FROM users WHERE id = 1", &[])
60− .unwrap();
61− assert_eq!(1, results.len());
62− assert_eq!(1, results[0].get::<_, i32>("id"));
63−
64− // Ensure the name column is not accesible through the new schema
65− assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
66−
67− // Insert using new schema and ensure the down function is correctly applied
68− new_db
69− .simple_query("INSERT INTO users(id) VALUES (2)")
70− .unwrap();
71− let result = old_db
72− .query_opt("SELECT name FROM users WHERE id = 2", &[])
73− .unwrap();
74− assert_eq!(
75− Some("TEST_DOWN_VALUE"),
76− result.as_ref().map(|row| row.get("name"))
170+
171+ test.second_migration(
172+ r#"
173+ name = "remove_users_email_column"
174+
175+ [[actions]]
176+ type = "remove_column"
177+ table = "users"
178+ column = "email"
179+
180+ [actions.down]
181+ table = "profiles"
182+ value = "profiles.email"
183+ where = "users.id = profiles.user_id"
184+ "#,
77185 );
78186
79− reshape.complete_migration().unwrap();
80− common::assert_cleaned_up(&mut new_db);
187+ test.after_first(|db| {
188+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
189+ .unwrap();
190+ db.simple_query("INSERT INTO profiles (user_id, email) VALUES (1, 'test@example.com')")
191+ .unwrap();
192+ });
193+
194+ test.intermediate(|old_db, new_db| {
195+ new_db
196+ .simple_query("UPDATE profiles SET email = 'test2@example.com' WHERE user_id = 1")
197+ .unwrap();
198+
199+ // Ensure new email was propagated to users table in old schema
200+ let email: String = old_db
201+ .query(
202+ "
203+ SELECT email
204+ FROM users
205+ WHERE id = 1
206+ ",
207+ &[],
208+ )
209+ .unwrap()
210+ .first()
211+ .map(|row| row.get("email"))
212+ .unwrap();
213+ assert_eq!("test2@example.com", email);
214+ });
215+
216+ test.run();
81217}
tests/remove_enum.rsnew file+69
@@ -0,0 +1,69 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_enum() {
6+ let mut test = Test::new("Remove enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+ "#,
17+ );
18+
19+ test.second_migration(
20+ r#"
21+ name = "remove_enum"
22+
23+ [[actions]]
24+ type = "remove_enum"
25+ enum = "mood"
26+ "#,
27+ );
28+
29+ test.after_first(|db| {
30+ // Ensure enum was created
31+ let enum_exists = !db
32+ .query(
33+ "
34+ SELECT typname
35+ FROM pg_catalog.pg_type
36+ WHERE typcategory = 'E'
37+ AND typname = 'mood'
38+ ",
39+ &[],
40+ )
41+ .unwrap()
42+ .is_empty();
43+
44+ assert!(enum_exists, "expected mood enum to have been created");
45+ });
46+
47+ test.after_completion(|db| {
48+ // Ensure enum was removed after completion
49+ let enum_does_not_exist = db
50+ .query(
51+ "
52+ SELECT typname
53+ FROM pg_catalog.pg_type
54+ WHERE typcategory = 'E'
55+ AND typname = 'mood'
56+ ",
57+ &[],
58+ )
59+ .unwrap()
60+ .is_empty();
61+
62+ assert!(
63+ enum_does_not_exist,
64+ "expected mood enum to have been removed"
65+ );
66+ });
67+
68+ test.run();
69+}
tests/remove_foreign_key.rsnew file+124
@@ -0,0 +1,124 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_foreign_key() {
6+ let mut test = Test::new("Remove foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+
34+ [[actions.foreign_keys]]
35+ columns = ["user_id"]
36+ referenced_table = "users"
37+ referenced_columns = ["id"]
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "remove_foreign_key"
44+
45+ [[actions]]
46+ type = "remove_foreign_key"
47+ table = "items"
48+ foreign_key = "items_user_id_fkey"
49+ "#,
50+ );
51+
52+ test.after_first(|db| {
53+ // Insert some test users
54+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
55+ .unwrap();
56+ });
57+
58+ test.intermediate(|old_db, new_db| {
59+ // Ensure items can't be inserted if they don't reference valid users
60+ // The foreign key is only removed when the migration is completed so
61+ // it should still be enforced for the new and old schema.
62+ let result = old_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
63+ assert!(
64+ result.is_err(),
65+ "expected insert against old schema to fail"
66+ );
67+
68+ let result = new_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
69+ assert!(
70+ result.is_err(),
71+ "expected insert against new schema to fail"
72+ );
73+ });
74+
75+ test.after_completion(|db| {
76+ // Ensure items can be inserted even if they don't reference valid users
77+ let result = db
78+ .simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)")
79+ .map(|_| ());
80+ assert!(
81+ result.is_ok(),
82+ "expected insert to not fail, got {:?}",
83+ result
84+ );
85+
86+ // Ensure foreign key doesn't exist
87+ let foreign_keys = db
88+ .query(
89+ "
90+ SELECT tc.constraint_name
91+ FROM information_schema.table_constraints AS tc
92+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
93+ ",
94+ &[],
95+ )
96+ .unwrap();
97+ assert!(
98+ foreign_keys.is_empty(),
99+ "expected no foreign keys to exist on items table"
100+ );
101+ });
102+
103+ test.after_abort(|db| {
104+ // Ensure items can't be inserted if they don't reference valid users
105+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
106+ assert!(result.is_err(), "expected insert to fail");
107+
108+ // Ensure foreign key still exists
109+ let fk_exists = !db
110+ .query(
111+ "
112+ SELECT tc.constraint_name
113+ FROM information_schema.table_constraints AS tc
114+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
115+ ",
116+ &[],
117+ )
118+ .unwrap()
119+ .is_empty();
120+ assert!(fk_exists);
121+ });
122+
123+ test.run()
124+}
tests/remove_index.rs+68−91
@@ -1,109 +1,86 @@
1−use reshape::migrations::{AddIndex, ColumnBuilder, CreateTableBuilder, Migration, RemoveIndex};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn remove_index() {
7− let (mut reshape, _, mut db) = common::setup();
6+ let mut test = Test::new("Remove index");
87
9− let create_table_migration = Migration::new("create_user_table", None)
10− .with_action(
11− CreateTableBuilder::default()
12− .name("users")
13− .primary_key(vec!["id".to_string()])
14− .columns(vec![
15− ColumnBuilder::default()
16− .name("id")
17− .data_type("INTEGER")
18− .build()
19− .unwrap(),
20− ColumnBuilder::default()
21− .name("name")
22− .data_type("TEXT")
23− .build()
24− .unwrap(),
25− ])
26− .build()
27− .unwrap(),
28− )
29− .with_action(AddIndex {
30− table: "users".to_string(),
31− name: "name_idx".to_string(),
32− columns: vec!["name".to_string()],
33− });
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
3411
35− let remove_index_migration =
36− Migration::new("remove_name_index", None).with_action(RemoveIndex {
37− index: "name_idx".to_string(),
38− });
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
3916
40− let first_migrations = vec![create_table_migration.clone()];
41− let second_migrations = vec![
42− create_table_migration.clone(),
43− remove_index_migration.clone(),
44− ];
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
4520
46− // Run migrations
47− reshape.migrate(first_migrations.clone()).unwrap();
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
4824
49− // Ensure index is valid and ready
50− let result: Vec<(bool, bool)> = db
51− .query(
52− "
53− SELECT pg_index.indisready, pg_index.indisvalid
54− FROM pg_catalog.pg_index
55− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
56− WHERE pg_class.relname = 'name_idx'
57− ",
58− &[],
59− )
60− .unwrap()
61− .iter()
62− .map(|row| (row.get("indisready"), row.get("indisvalid")))
63− .collect();
25+ [[actions]]
26+ type = "add_index"
27+ table = "users"
6428
65− assert_eq!(vec![(true, true)], result);
29+ [actions.index]
30+ name = "name_idx"
31+ columns = ["name"]
32+ "#,
33+ );
6634
67− // Run migration to remove index
68− reshape.migrate(second_migrations.clone()).unwrap();
35+ test.second_migration(
36+ r#"
37+ name = "remove_name_index"
6938
70− // Ensure index is still valid and ready during the migration
71− let result: Vec<(bool, bool)> = db
72− .query(
73− "
74− SELECT pg_index.indisready, pg_index.indisvalid
75− FROM pg_catalog.pg_index
76− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
77− WHERE pg_class.relname = 'name_idx'
78− ",
79− &[],
80− )
81− .unwrap()
82− .iter()
83− .map(|row| (row.get("indisready"), row.get("indisvalid")))
84− .collect();
39+ [[actions]]
40+ type = "remove_index"
41+ index = "name_idx"
42+ "#,
43+ );
8544
86− assert_eq!(vec![(true, true)], result);
45+ test.intermediate(|db, _| {
46+ // Ensure index is still valid and ready during the migration
47+ let result: Vec<(bool, bool)> = db
48+ .query(
49+ "
50+ SELECT pg_index.indisready, pg_index.indisvalid
51+ FROM pg_catalog.pg_index
52+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
53+ WHERE pg_class.relname = 'name_idx'
54+ ",
55+ &[],
56+ )
57+ .unwrap()
58+ .iter()
59+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
60+ .collect();
8761
88− reshape.complete_migration().unwrap();
62+ assert_eq!(vec![(true, true)], result);
63+ });
8964
90− // Ensure index has been removed after the migration is complete
91− let count: i64 = db
92− .query(
93− "
94− SELECT COUNT(*)
95− FROM pg_catalog.pg_index
96− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
97− WHERE pg_class.relname = 'name_idx'
98− ",
99− &[],
100− )
101− .unwrap()
102− .first()
103− .map(|row| row.get(0))
104− .unwrap();
65+ test.after_completion(|db| {
66+ // Ensure index has been removed after the migration is complete
67+ let count: i64 = db
68+ .query(
69+ "
70+ SELECT COUNT(*)
71+ FROM pg_catalog.pg_index
72+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
73+ WHERE pg_class.relname = 'name_idx'
74+ ",
75+ &[],
76+ )
77+ .unwrap()
78+ .first()
79+ .map(|row| row.get(0))
80+ .unwrap();
10581
106− assert_eq!(0, count, "expected index to not exist");
82+ assert_eq!(0, count, "expected index to not exist");
83+ });
10784
108− common::assert_cleaned_up(&mut db);
85+ test.run();
10986}
tests/remove_table.rs+38−48
@@ -1,54 +1,44 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RemoveTable};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn remove_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
10− CreateTableBuilder::default()
11− .name("users")
12− .primary_key(vec!["id".to_string()])
13− .columns(vec![ColumnBuilder::default()
14− .name("id")
15− .data_type("INTEGER")
16− .build()
17− .unwrap()])
18− .build()
19− .unwrap(),
6+ let mut test = Test::new("Remove table");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
22+
23+ test.second_migration(
24+ r#"
25+ name = "remove_users_table"
26+
27+ [[actions]]
28+ type = "remove_table"
29+ table = "users"
30+ "#,
2031 );
21− let remove_table_migration =
22− Migration::new("remove_users_table", None).with_action(RemoveTable {
23− table: "users".to_string(),
24− });
25−
26− let first_migrations = vec![create_table_migration.clone()];
27− let second_migrations = vec![
28− create_table_migration.clone(),
29− remove_table_migration.clone(),
30− ];
31−
32− // Run migrations
33− reshape.migrate(first_migrations.clone()).unwrap();
34− reshape.migrate(second_migrations.clone()).unwrap();
35−
36− // Update schemas of Postgres connections
37− let old_schema_query =
38− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
39− let new_schema_query =
40− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
41− old_db.simple_query(&old_schema_query).unwrap();
42− new_db.simple_query(&new_schema_query).unwrap();
43−
44− // Make sure inserts work against the old schema
45− old_db
46− .simple_query("INSERT INTO users(id) VALUES (1)")
47− .unwrap();
48−
49− // Ensure the table is not accessible through the new schema
50− assert!(new_db.query("SELECT id FROM users", &[]).is_err());
51−
52− reshape.complete_migration().unwrap();
53− common::assert_cleaned_up(&mut new_db);
32+
33+ test.intermediate(|old_db, new_db| {
34+ // Make sure inserts work against the old schema
35+ old_db
36+ .simple_query("INSERT INTO users(id) VALUES (1)")
37+ .unwrap();
38+
39+ // Ensure the table is not accessible through the new schema
40+ assert!(new_db.query("SELECT id FROM users", &[]).is_err());
41+ });
42+
43+ test.run();
5444}
tests/rename_table.rs+56−66
@@ -1,80 +1,70 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RenameTable};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn rename_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
10− CreateTableBuilder::default()
11− .name("users")
12− .primary_key(vec!["id".to_string()])
13− .columns(vec![ColumnBuilder::default()
14− .name("id")
15− .data_type("INTEGER")
16− .build()
17− .unwrap()])
18− .build()
19− .unwrap(),
20− );
21− let rename_table_migration = Migration::new("rename_users_table_to_customers", None)
22− .with_action(RenameTable {
23− table: "users".to_string(),
24− new_name: "customers".to_string(),
25− });
6+ let mut test = Test::new("Rename table");
267
27− let first_migrations = vec![create_table_migration.clone()];
28− let second_migrations = vec![
29− create_table_migration.clone(),
30− rename_table_migration.clone(),
31− ];
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
3216
33− // Run migrations
34− reshape.migrate(first_migrations.clone()).unwrap();
35− reshape.migrate(second_migrations.clone()).unwrap();
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
3622
37− // Update schemas of Postgres connections
38− let old_schema_query =
39− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
40− let new_schema_query =
41− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
42− old_db.simple_query(&old_schema_query).unwrap();
43− new_db.simple_query(&new_schema_query).unwrap();
23+ test.second_migration(
24+ r#"
25+ name = "rename_users_table_to_customers"
4426
45− // Make sure inserts work using both the old and new name
46− old_db
47− .simple_query("INSERT INTO users(id) VALUES (1)")
48− .unwrap();
49− new_db
50− .simple_query("INSERT INTO customers(id) VALUES (2)")
51− .unwrap();
27+ [[actions]]
28+ type = "rename_table"
29+ table = "users"
30+ new_name = "customers"
31+ "#,
32+ );
5233
53− // Ensure the table can be queried using both the old and new name
54− let expected: Vec<i32> = vec![1, 2];
55− assert_eq!(
56− expected,
34+ test.intermediate(|old_db, new_db| {
35+ // Make sure inserts work using both the old and new name
5736 old_db
58− .query("SELECT id FROM users ORDER BY id", &[])
59− .unwrap()
60− .iter()
61− .map(|row| row.get::<_, i32>("id"))
62− .collect::<Vec<i32>>()
63− );
64− assert_eq!(
65− expected,
37+ .simple_query("INSERT INTO users(id) VALUES (1)")
38+ .unwrap();
6639 new_db
67− .query("SELECT id FROM customers ORDER BY id", &[])
68− .unwrap()
69− .iter()
70− .map(|row| row.get::<_, i32>("id"))
71− .collect::<Vec<i32>>()
72− );
40+ .simple_query("INSERT INTO customers(id) VALUES (2)")
41+ .unwrap();
42+
43+ // Ensure the table can be queried using both the old and new name
44+ let expected: Vec<i32> = vec![1, 2];
45+ assert_eq!(
46+ expected,
47+ old_db
48+ .query("SELECT id FROM users ORDER BY id", &[])
49+ .unwrap()
50+ .iter()
51+ .map(|row| row.get::<_, i32>("id"))
52+ .collect::<Vec<i32>>()
53+ );
54+ assert_eq!(
55+ expected,
56+ new_db
57+ .query("SELECT id FROM customers ORDER BY id", &[])
58+ .unwrap()
59+ .iter()
60+ .map(|row| row.get::<_, i32>("id"))
61+ .collect::<Vec<i32>>()
62+ );
7363
74− // Ensure the table can't be queried using the wrong name for the schema
75− assert!(old_db.simple_query("SELECT id FROM customers").is_err());
76− assert!(new_db.simple_query("SELECT id FROM users").is_err());
64+ // Ensure the table can't be queried using the wrong name for the schema
65+ assert!(old_db.simple_query("SELECT id FROM customers").is_err());
66+ assert!(new_db.simple_query("SELECT id FROM users").is_err());
67+ });
7768
78− reshape.complete_migration().unwrap();
79− common::assert_cleaned_up(&mut new_db);
69+ test.run();
8070}