on main help sign in
.github/workflows/lint.yaml+3−3
since splitsnapshots
unifiedsplit
@@ -15,7 +15,7 @@ jobs:
15 runs-on: ubuntu-latest
15 runs-on: ubuntu-latest
16 steps:
16 steps:
17 - name: Checkout
17 - name: Checkout
18− uses: actions/checkout@v2
18+ uses: actions/checkout@v4
19 - name: Select Rust toolchain with Clippy
19 - name: Select Rust toolchain with Clippy
20 uses: actions-rs/toolchain@v1
20 uses: actions-rs/toolchain@v1
21 with:
21 with:
@@ -23,6 +23,6 @@ jobs:
23 components: clippy
23 components: clippy
24 override: true
24 override: true
25 - name: Use cache for Rust dependencies
25 - name: Use cache for Rust dependencies
26− uses: Swatinem/rust-cache@v1
26+ uses: Swatinem/rust-cache@v2
27 - name: Lint using Clippy
27 - name: Lint using Clippy
28− run: cargo clippy
28+ run: cargo clippy
.github/workflows/publish-crate.yamlnew file+20
@@ -0,0 +1,20 @@
1+name: Publish crate
2+
3+on: workflow_dispatch
4+
5+jobs:
6+ publish-crate:
7+ runs-on: ubuntu-latest
8+ steps:
9+ - name: Checkout
10+ uses: actions/checkout@v2
11+ - name: Select Rust toolchain
12+ uses: actions-rs/toolchain@v1
13+ with:
14+ toolchain: stable
15+ - name: Publish to crates.io
16+ uses: actions-rs/cargo@v1
17+ with:
18+ command: publish
19+ env:
20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/publish-docker.yamlnew file+25
@@ -0,0 +1,25 @@
1+name: Publish Docker image
2+
3+on:
4+ workflow_dispatch:
5+ inputs:
6+ version:
7+ description: "Version (without 'v' prefix)"
8+ required: true
9+ type: string
10+
11+jobs:
12+ publish-docker:
13+ runs-on: ubuntu-latest
14+ steps:
15+ - name: Checkout
16+ uses: actions/checkout@v2
17+ - name: Log in to Docker
18+ env:
19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
20+ run: |
21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD
22+ - name: Build Docker image
23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
24+ - name: Push Docker image
25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
Cargo.toml+6−3
@@ -1,6 +1,6 @@
1[package]
1[package]
2name = "reshape"
2name = "reshape"
3−version = "0.3.0"
3+version = "0.7.0"
4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
5homepage = "https://github.com/fabianlindfors/reshape"
5homepage = "https://github.com/fabianlindfors/reshape"
6documentation = "https://github.com/fabianlindfors/reshape"
6documentation = "https://github.com/fabianlindfors/reshape"
@@ -9,6 +9,7 @@ license = "MIT"
9keywords = ["postgres", "migrations"]
9keywords = ["postgres", "migrations"]
10edition = "2021"
10edition = "2021"
11authors = ["Fabian Lindfors"]
11authors = ["Fabian Lindfors"]
12+rust-version = "1.70"
12
13
13[dependencies]
14[dependencies]
14postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
15postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
@@ -16,8 +17,10 @@ serde = { version = "1.0", features = ["derive"] }
16serde_json = "1.0"
17serde_json = "1.0"
17typetag = "0.1.7"
18typetag = "0.1.7"
18anyhow = { version = "1.0.44", features = ["backtrace"] }
19anyhow = { version = "1.0.44", features = ["backtrace"] }
19−clap = { version = "3.0.0", features = ["derive"] }
20+clap = { version = "3.1.9", features = ["derive"] }
20toml = "0.5"
21toml = "0.5"
21version = "3.0.0"
22version = "3.0.0"
22colored = "2"
23colored = "2"
23−"derive_builder" = "0.10.2"
24+rand = "0.8"
25+dotenv = "0.15.0"
26+lexical-sort = "0.3.1"
Dockerfile+1−1
@@ -1,4 +1,4 @@
1−FROM rust:1.58.0 AS builder
1+FROM rust:1.70.0 AS builder
2WORKDIR /usr/src/reshape
2WORKDIR /usr/src/reshape
3COPY . .
3COPY . .
4RUN cargo build --release
4RUN cargo build --release
README.md+356−81
@@ -2,50 +2,56 @@
2
2
3[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)
3[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)
4
4
5−Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries.
5+> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!
6
6
7−Reshape is designed for Postgres 12 and later.
7+Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).
8
8
9−*Note: Reshape is **experimental** and should not be used in production. It can (and probably will) break your application.*
9+Designed for Postgres 12 and later.
10
10
11- [How it works](#how-it-works)
11- [How it works](#how-it-works)
12- [Getting started](#getting-started)
12- [Getting started](#getting-started)
13− - [Installation](#installation)
13+ - [Installation](#installation)
14− - [Creating your first migration](#creating-your-first-migration)
14+ - [Creating your first migration](#creating-your-first-migration)
15− - [Preparing your application](#preparing-your-application)
15+ - [Preparing your application](#preparing-your-application)
16− - [Running your migration](#running-your-migration)
16+ - [Running your migration](#running-your-migration)
17− - [Using during development](#using-during-development)
17+ - [Using during development](#using-during-development)
18- [Writing migrations](#writing-migrations)
18- [Writing migrations](#writing-migrations)
19− - [Basics](#basics)
19+ - [Basics](#basics)
20− - [Tables](#tables)
20+ - [Tables](#tables)
21− - [Create table](#create-table)
21+ - [Create table](#create-table)
22− - [Rename table](#rename-table)
22+ - [Rename table](#rename-table)
23− - [Remove table](#remove-table)
23+ - [Remove table](#remove-table)
24+ - [Add foreign key](#add-foreign-key)
25+ - [Remove foreign key](#remove-foreign-key)
24− - [Columns](#columns)
26+ - [Columns](#columns)
25− - [Add column](#add-column)
27+ - [Add column](#add-column)
26− - [Alter column](#alter-column)
28+ - [Alter column](#alter-column)
27− - [Remove column](#remove-column)
29+ - [Remove column](#remove-column)
28− - [Indices](#indices)
30+ - [Indices](#indices)
29− - [Add index](#add-index)
31+ - [Add index](#add-index)
30− - [Remove index](#remove-index)
32+ - [Remove index](#remove-index)
33+ - [Enums](#enums)
34+ - [Create enum](#create-enum)
35+ - [Remove enum](#remove-enum)
36+ - [Custom](#custom)
37+ - [Complex changes across tables](#complex-changes-across-tables)
31- [Commands and options](#commands-and-options)
38- [Commands and options](#commands-and-options)
32− - [`reshape migrate`](#reshape-migrate)
39+ - [`reshape migration start`](#reshape-migration-start)
33− - [`reshape complete`](#reshape-complete)
40+ - [`reshape migration complete`](#reshape-migration-complete)
34− - [`reshape abort`](#reshape-abort)
41+ - [`reshape migration abort`](#reshape-migration-abort)
35− - [`reshape generate-schema-query`](#reshape-generate-schema-query)
42+ - [`reshape schema-query`](#reshape-schema-query)
36− - [Connection options](#connection-options)
43+ - [Connection options](#connection-options)
37- [License](#license)
44- [License](#license)
38
45
39−
40## How it works
46## How it works
41
47
42Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every deployment is a three-phase process:
48Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every deployment is a three-phase process:
43
49
44−1. **Start migration** (`reshape migrate`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
50+1. **Start migration** (`reshape migration start`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
452. **Roll out application**: Your application can be gradually rolled out without downtime. The existing deployment will continue using the old schema whilst the new deployment uses the new schema.
512. **Roll out application**: Your application can be gradually rolled out without downtime. The existing deployment will continue using the old schema whilst the new deployment uses the new schema.
46−3. **Complete migration** (`reshape complete`): Removes the old schema and any intermediate data and triggers.
52+3. **Complete migration** (`reshape migration complete`): Removes the old schema and any intermediate data and triggers.
47
53
48−If the application deployment fails, you should run `reshape abort` which will roll back any changes made by `reshape migrate` without losing data.
54+If the application deployment fails, you should run `reshape migration abort` which will roll back any changes made by `reshape migration start` without losing data.
49
55
50## Getting started
56## Getting started
51
57
@@ -57,7 +63,7 @@ Binaries are available for macOS and Linux under [Releases](https://github.com/f
57
63
58#### Cargo
64#### Cargo
59
65
60−Reshape can be installed using [Cargo](https://doc.rust-lang.org/cargo/):
66+Reshape can be installed using [Cargo](https://doc.rust-lang.org/cargo/) (requires Rust 1.58 or later):
61
67
62```shell
68```shell
63cargo install reshape
69cargo install reshape
@@ -65,10 +71,10 @@ cargo install reshape
65
71
66#### Docker
72#### Docker
67
73
68−Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
74+Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
69
75
70```shell
76```shell
71−docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migrate
77+docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migration start
72```
78```
73
79
74### Creating your first migration
80### Creating your first migration
@@ -97,9 +103,14 @@ This is the equivalent of running `CREATE TABLE users (id INTEGER GENERATED ALWA
97
103
98### Preparing your application
104### Preparing your application
99
105
100−Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. This query can be generated using: `reshape generate-schema-query`.
106+Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. The simplest way to do this is to use one of the helper libraries:
107+
108+- [Rust](https://github.com/fabianlindfors/reshape-helper)
109+- [Ruby (and Rails)](https://github.com/fabianlindfors/reshape-ruby)
110+- [Python (and Django)](https://github.com/fabianlindfors/reshape-python)
111+- [Go](https://github.com/leourbina/reshape-helper)
101
112
102−To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape generate-schema-query)`. Then in your application:
113+If your application is not using one of the languages with an available helper library, you can instead generate the query with the command: `reshape schema-query`. To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape schema-query)`. Then in your application:
103
114
104```python
115```python
105# Example for Python
116# Example for Python
@@ -112,16 +123,16 @@ db.execute(reshape_schema_query)
112To create your new `users` table, run:
123To create your new `users` table, run:
113
124
114```bash
125```bash
115−reshape migrate
126+reshape migration start --complete
116```
127```
117
128
118−As this is the first migration, Reshape will automatically complete it. For subsequent migrations, you will need to first run `reshape migrate`, roll out your application and then complete the migration using `reshape complete`.
129+We use the `--complete` flag to automatically complete the migration. During a production deployment, you should first run `reshape migration start` followed by `reshape migration complete` once your application has been fully rolled out.
119
130
120If nothing else is specified, Reshape will try to connect to a Postgres database running on `localhost` using `postgres` as both username and password. See [Connection options](#connection-options) for details on how to change the connection settings.
131If nothing else is specified, Reshape will try to connect to a Postgres database running on `localhost` using `postgres` as both username and password. See [Connection options](#connection-options) for details on how to change the connection settings.
121
132
122### Using during development
133### Using during development
123
134
124−When adding new migrations during development, we recommend running `reshape migrate` but skipping `reshape complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape abort` followed by `reshape migrate`.
135+When adding new migrations during development, we recommend running `reshape migration start` but skipping `reshape migration complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape migration abort` followed by `reshape migration start`.
125
136
126## Writing migrations
137## Writing migrations
127
138
@@ -156,9 +167,9 @@ Every action has a `type`. The supported types are detailed below.
156
167
157#### Create table
168#### Create table
158
169
159−The `create_table` action will create a new table with the specified columns, indices and constraints.
170+The `create_table` action will create a new table with the specified columns, indices and constraints. You can optionally provide an `up` option to backfill values from an existing table.
160
171
161−*Example: create a `customers` table with a few columns and a primary key*
172+_Example: create a `customers` table with a few columns and a primary key_
162
173
163```toml
174```toml
164[[actions]]
175[[actions]]
@@ -176,13 +187,13 @@ primary_key = ["id"]
176 type = "TEXT"
187 type = "TEXT"
177
188
178 # Columns default to nullable
189 # Columns default to nullable
179− nullable = false
190+ nullable = false
180
191
181 # default can be any valid SQL value, in this case a string literal
192 # default can be any valid SQL value, in this case a string literal
182− default = "'PLACEHOLDER'"
193+ default = "'PLACEHOLDER'"
183```
194```
184
195
185−*Example: create `users` and `items` tables with a foreign key between them*
196+_Example: create `users` and `items` tables with a foreign key between them_
186
197
187```toml
198```toml
188[[actions]]
199[[actions]]
@@ -215,11 +226,34 @@ primary_key = ["id"]
215 referenced_columns = ["id"]
226 referenced_columns = ["id"]
216```
227```
217
228
229+_Example: create `profiles` table based on existing `users` table_
230+
231+```toml
232+[[actions]]
233+type = "create_table"
234+name = "profiles"
235+primary_key = ["user_id"]
236+
237+ [[actions.columns]]
238+ name = "user_id"
239+ type = "INTEGER"
240+
241+ [[actions.columns]]
242+ name = "user_email"
243+ type = "TEXT"
244+
245+ # Backfill from `users` table and copy `users.email` to `user_email` column
246+ # This will perform an upsert based on the primary key to avoid duplicate rows
247+ [actions.up]
248+ table = "users"
249+ values = { user_id = "id", user_email = "email" }
250+```
251+
218#### Rename table
252#### Rename table
219
253
220The `rename_table` action will change the name of an existing table.
254The `rename_table` action will change the name of an existing table.
221
255
222−*Example: change name of `users` table to `customers`*
256+_Example: change name of `users` table to `customers`_
223
257
224```toml
258```toml
225[[actions]]
259[[actions]]
@@ -232,7 +266,7 @@ new_name = "customers"
232
266
233The `remove_table` action will remove an existing table.
267The `remove_table` action will remove an existing table.
234
268
235−*Example: remove `users` table*
269+_Example: remove `users` table_
236
270
237```toml
271```toml
238[[actions]]
272[[actions]]
@@ -240,13 +274,43 @@ type = "remove_table"
240table = "users"
274table = "users"
241```
275```
242
276
277+#### Add foreign key
278+
279+The `add_foreign_key` action will add a foreign key between two existing tables. The migration will fail if the existing column values aren't valid references.
280+
281+_Example: create foreign key from `items` to `users` table_
282+
283+```toml
284+[[actions]]
285+type = "add_foreign_key"
286+table = "items"
287+
288+ [actions.foreign_key]
289+ columns = ["user_id"]
290+ referenced_table = "users"
291+ referenced_columns = ["id"]
292+```
293+
294+#### Remove foreign key
295+
296+The `remove_foreign_key` action will remove an existing foreign key. The foreign key will only be removed once the migration is completed, which means that your new application must continue to adhere to the foreign key constraint.
297+
298+_Example: remove foreign key `items_user_id_fkey` from `users` table_
299+
300+```toml
301+[[actions]]
302+type = "remove_foreign_key"
303+table = "items"
304+foreign_key = "items_user_id_fkey"
305+```
306+
243### Columns
307### Columns
244
308
245#### Add column
309#### Add column
246
310
247−The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column.
311+The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column. `up` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)).
248
312
249−*Example: add a new column `reference` to table `products`*
313+_Example: add a new column `reference` to table `products`_
250
314
251```toml
315```toml
252[[actions]]
316[[actions]]
@@ -260,7 +324,7 @@ table = "products"
260 default = "10"
324 default = "10"
261```
325```
262
326
263−*Example: replace an existing `name` column with two new columns, `first_name` and `last_name`*
327+_Example: replace an existing `name` column with two new columns, `first_name` and `last_name`_
264
328
265```toml
329```toml
266[[actions]]
330[[actions]]
@@ -296,14 +360,48 @@ column = "name"
296down = "first_name || ' ' || last_name"
360down = "first_name || ' ' || last_name"
297```
361```
298
362
363+_Example: extract nested value from unstructured JSON `data` column to new `name` column_
364+
365+```toml
366+[[actions]]
367+type = "add_column"
368+table = "users"
369+
370+# #>> '{}' converts the JSON string value to TEXT
371+up = "data['path']['to']['value'] #>> '{}'"
372+
373+ [actions.column]
374+ name = "name"
375+ type = "TEXT"
376+```
377+
378+_Example: duplicate `email` column from `users` to `profiles` table_
379+
380+```toml
381+# `profiles` has `user_id` column which maps to `users.id`
382+[[actions]]
383+type = "add_column"
384+table = "profiles"
385+
386+ [actions.column]
387+ name = "email"
388+ type = "TEXT"
389+ nullable = false
390+
391+ # When `users` is updated in the old schema, we write the email value to `profiles`
392+ [actions.up]
393+ table = "users"
394+ value = "email"
395+ where = "user_id = id"
396+```
299
397
300#### Alter column
398#### Alter column
301
399
302The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.
400The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.
303
401
304−When performing more complex changes than a rename, `up` and `down` must be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
402+When performing more complex changes than a rename, `up` and `down` should be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
305
403
306−*Example: rename `last_name` column on `users` table to `family_name`*
404+_Example: rename `last_name` column on `users` table to `family_name`_
307
405
308```toml
406```toml
309[[actions]]
407[[actions]]
@@ -315,7 +413,7 @@ column = "last_name"
315 name = "family_name"
413 name = "family_name"
316```
414```
317
415
318−*Example: change the type of `reference` column from `INTEGER` to `TEXT`*
416+_Example: change the type of `reference` column from `INTEGER` to `TEXT`_
319
417
320```toml
418```toml
321[[actions]]
419[[actions]]
@@ -330,7 +428,7 @@ down = "CAST(reference AS INTEGER)" # Converts from text value to integer
330 type = "TEXT" # Previous type was 'INTEGER'
428 type = "TEXT" # Previous type was 'INTEGER'
331```
429```
332
430
333−*Example: increment all values of an `index` column by one*
431+_Example: increment all values of an `index` column by one_
334
432
335```toml
433```toml
336[[actions]]
434[[actions]]
@@ -345,7 +443,22 @@ down = "index - 1" # Decrement to revert for old schema
345 name = "index"
443 name = "index"
346```
444```
347
445
446+_Example: make `name` column not nullable_
447+
448+```toml
449+[[actions]]
450+type = "alter_column"
451+table = "users"
452+column = "name"
453+
454+# Use "N/A" for any rows that currently have a NULL name
455+up = "COALESCE(name, 'N/A')"
456+
457+ [actions.changes]
458+ nullable = false
459+```
460+
348−*Example: change default value of `created_at` column to current time*
461+_Example: change default value of `created_at` column to current time_
349
462
350```toml
463```toml
351[[actions]]
464[[actions]]
@@ -359,9 +472,11 @@ column = "created_at"
359
472
360#### Remove column
473#### Remove column
361
474
362−The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
475+The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. `down` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)) . The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
476+
477+Any indices that cover the column will be removed.
363
478
364−*Example: remove column `name` from table `users`*
479+_Example: remove column `name` from table `users`_
365
480
366```toml
481```toml
367[[actions]]
482[[actions]]
@@ -373,18 +488,34 @@ column = "name"
373down = "'N/A'"
488down = "'N/A'"
374```
489```
375
490
491+_Example: remove `email` column from `users` table and use column from `profiles` table instead_
492+
493+```toml
494+[[actions]]
495+type = "remove_column"
496+table = "users"
497+column = "email"
498+
499+ # Our application will use the `profiles.email` column instead
500+ # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed
501+ [actions.down]
502+ table = "profiles"
503+ value = "profiles.email"
504+ where = "users.id = profiles.user_id"
505+```
506+
376### Indices
507### Indices
377
508
378#### Add index
509#### Add index
379
510
380The `add_index` action will add a new index to an existing table.
511The `add_index` action will add a new index to an existing table.
381
512
382−*Example: create a `users` table with a unique index on the `name` column*
513+_Example: create a `users` table with a unique index on the `name` column_
383
514
384```toml
515```toml
385[[actions]]
516[[actions]]
386type = "create_table"
517type = "create_table"
387−table = "users"
518+name = "users"
388primary_key = "id"
519primary_key = "id"
389
520
390 [[actions.columns]]
521 [[actions.columns]]
@@ -408,7 +539,7 @@ table = "users"
408 unique = true
539 unique = true
409```
540```
410
541
411−*Example: add GIN index to `data` column on `products` table*
542+_Example: add GIN index to `data` column on `products` table_
412
543
413```toml
544```toml
414[[actions]]
545[[actions]]
@@ -427,7 +558,7 @@ table = "products"
427
558
428The `remove_index` action will remove an existing index. The index won't actually be removed until the migration is completed.
559The `remove_index` action will remove an existing index. The index won't actually be removed until the migration is completed.
429
560
430−*Example: remove the `name_idx` index*
561+_Example: remove the `name_idx` index_
431
562
432```toml
563```toml
433[[actions]]
564[[actions]]
@@ -435,61 +566,205 @@ type = "remove_index"
435index = "name_idx"
566index = "name_idx"
436```
567```
437
568
569+### Enums
570+
571+#### Create enum
572+
573+The `create_enum` action will create a new [enum type](https://www.postgresql.org/docs/current/datatype-enum.html) with the specified values.
574+
575+_Example: add a new `mood` enum type with three possible values_
576+
577+```toml
578+[[actions]]
579+type = "create_enum"
580+name = "mood"
581+values = ["happy", "ok", "sad"]
582+```
583+
584+#### Remove enum
585+
586+The `remove_enum` action will remove an existing [enum type](https://www.postgresql.org/docs/current/datatype-enum.html). Make sure all usages of the enum has been removed before running the migration. The enum will only be removed once the migration is completed.
587+
588+_Example: remove the `mood` enum type_
589+
590+```toml
591+[[actions]]
592+type = "remove_enum"
593+enum = "mood"
594+```
595+
596+### Custom
597+
598+The `custom` action lets you create a migration which runs custom SQL. It should be used with great care as it provides no guarantees of zero-downtime and will simply run whatever SQL is provided. Use other actions whenever possible as they are explicitly designed for zero downtime.
599+
600+There are three optional settings available which all accept SQL queries. All queries need to be idempotent, for example by using `IF NOT EXISTS` wherever available.
601+
602+- `start`: run when a migration is started using `reshape migration start`
603+- `complete`: run when a migration is completed using `reshape migration complete`
604+- `abort`: run when a migration is aborted using `reshape migration abort`
605+
606+_Example: enable PostGIS and pg_stat_statements extensions_
607+
608+```toml
609+[[actions]]
610+type = "custom"
611+
612+start = """
613+ CREATE EXTENSION IF NOT EXISTS postgis;
614+ CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
615+"""
616+
617+abort = """
618+ DROP EXTENSION IF EXISTS postgis;
619+ DROP EXTENSION IF EXISTS pg_stat_statements;
620+"""
621+```
622+
623+### Complex changes across tables
624+
625+The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.
626+
627+_Example: move `email` column from `users` to `profiles` table_
628+
629+```toml
630+[[actions]]
631+type = "add_column"
632+table = "profiles"
633+
634+ [actions.column]
635+ name = "email"
636+ type = "TEXT"
637+ nullable = false
638+
639+ # When `users` is updated in the old schema, we write the email value to `profiles`
640+ [actions.up]
641+ table = "users"
642+ value = "users.email"
643+ where = "profiles.user_id = users.id"
644+
645+[[actions]]
646+type = "remove_column"
647+table = "users"
648+column = "email"
649+
650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
651+ [actions.down]
652+ table = "profiles"
653+ value = "profiles.email"
654+ where = "users.id = profiles.user_id"
655+```
656+
657+_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_
658+
659+```toml
660+# Add `user_account_connections` as a junction table
661+[[actions]]
662+type = "create_table"
663+name = "user_account_connections"
664+primary_key = ["account_id", "user_id"]
665+
666+ [[actions.columns]]
667+ name = "account_id"
668+ type = "INTEGER"
669+
670+ [[actions.columns]]
671+ name = "user_id"
672+ type = "INTEGER"
673+
674+ # `role` is currently stored directly on the `users` table but is part of the relationship
675+ [[actions.columns]]
676+ name = "role"
677+ type = "TEXT"
678+ nullable = false
679+
680+ # Backfill the new table from `users` and uppercase the `role`
681+ [actions.up]
682+ table = "users"
683+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
684+ where = "user_account_connections.user_id = users.id"
685+
686+[[actions]]
687+type = "remove_column"
688+table = "users"
689+column = "account_id"
690+
691+ # When `user_account_connections` is updated, we write the `account_id` back to `users`
692+ [actions.down]
693+ table = "user_account_connections"
694+ value = "user_account_connections.account_id"
695+ where = "users.id = user_account_connections.user_id"
696+
697+[[actions]]
698+type = "remove_column"
699+table = "users"
700+column = "account_role"
701+
702+ # When `user_account_connections` is updated, we write the lowercase role back to `users`
703+ [actions.down]
704+ table = "user_account_connections"
705+ value = "LOWER(user_account_connections.role)"
706+ where = "users.id = user_account_connections.user_id"
707+```
708+
438## Commands and options
709## Commands and options
439
710
440−### `reshape migrate`
711+### `reshape migration start`
441
712
442−Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape complete`.
713+Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape migration complete`.
443
714
444#### Options
715#### Options
445
716
446−*See also [Connection options](#connection-options)*
717+_See also [Connection options](#connection-options)_
447
718
448−| Option | Default | Description |
719+| Option | Default | Description |
449−| ------ | ------- | ----------- |
720+| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------------- |
450−| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
721+| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
451−| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
722+| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
452
723
453−### `reshape complete`
724+### `reshape migration complete`
454
725
455−Completes migrations previously started with `reshape complete`.
726+Completes migrations previously started with `reshape migration complete`.
456
727
457#### Options
728#### Options
458
729
459See [Connection options](#connection-options)
730See [Connection options](#connection-options)
460
731
461−### `reshape abort`
732+### `reshape migration abort`
462
733
463−Aborts any migrations which haven't yet been completed.
734+Aborts any migrations which haven't yet been completed.
464
735
465#### Options
736#### Options
466
737
467See [Connection options](#connection-options)
738See [Connection options](#connection-options)
468
739
469−### `reshape generate-schema-query`
740+### `reshape schema-query`
470
741
471Generates the SQL query you need to run in your application before using the database. This command does not require a database connection. Instead it will generate the query based on the latest migration in the `migrations/` directory (or the directories specified by `--dirs`).
742Generates the SQL query you need to run in your application before using the database. This command does not require a database connection. Instead it will generate the query based on the latest migration in the `migrations/` directory (or the directories specified by `--dirs`).
472
743
744+If your application is written in Rust, we recommend using the [Rust helper library](https://github.com/fabianlindfors/reshape-helper/) instead.
745+
473The query should look something like `SET search_path TO migration_1_initial_migration`.
746The query should look something like `SET search_path TO migration_1_initial_migration`.
474
747
475#### Options
748#### Options
476
749
477−| Option | Default | Description |
750+| Option | Default | Description |
478−| ------ | ------- | ----------- |
751+| -------- | ------------- | --------------------------------------------------------------------------------------------------------------- |
479| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
752| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
480
753
481### Connection options
754### Connection options
482
755
483−The options below can be used with all commands that communicate with Postgres. Use either a connection URL or specify each connection option individually.
756+The options below can be used with all commands that communicate with Postgres. Use either a [connection URL](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) or specify each connection option individually.
757+
758+All options can also be set using environment variables instead of flags. If a `.env` file exists, then variables will be automatically loaded from there.
484
759
485−| Option | Default | Description |
760+| Option | Default | Environment variable | Description |
486−| ------ | ------- | ----------- |
761+| ------------ | ----------- | -------------------- | ------------------------------------------- |
487−| `--url` | | URI to connect to your Postgres database.<br>Can also be provided with the environment variable `DATABASE_URL`. |
762+| `--url` | | `DB_URL` | URL to your Postgres database |
488−| `--host` | `localhost` | Hostname to use when connecting to Postgres |
763+| `--host` | `localhost` | `DB_HOST` | Hostname to use when connecting to Postgres |
489−| `--port` | `5432` | Port which Postgres is listening on |
764+| `--port` | `5432` | `DB_PORT` | Port which Postgres is listening on |
490−| `--database` | `postgres` | Database name |
765+| `--database` | `postgres` | `DB_NAME` | Database name |
491−| `--username` | `postgres` | Postgres username |
766+| `--username` | `postgres` | `DB_USERNAME` | Postgres username |
492−| `--password` | `postgres` | Postgres password |
767+| `--password` | `postgres` | `DB_PASSWORD` | Postgres password |
493
768
494## License
769## License
495
770
src/db.rs+76−5
@@ -1,5 +1,8 @@
1+use std::{cmp::min, time::Duration};
2+
1−use anyhow::anyhow;
3+use anyhow::{anyhow, Context};
2use postgres::{types::ToSql, NoTls, Row};
4use postgres::{types::ToSql, NoTls, Row};
5+use rand::prelude::*;
3
6
4// DbLocker wraps a regular DbConn, only allowing access using the
7// DbLocker wraps a regular DbConn, only allowing access using the
5// `lock` method. This method will acquire the advisory lock before
8// `lock` method. This method will acquire the advisory lock before
@@ -22,7 +25,24 @@ impl DbLocker {
22 const LOCK_KEY: i64 = 4036779288569897133;
25 const LOCK_KEY: i64 = 4036779288569897133;
23
26
24 pub fn connect(config: &postgres::Config) -> anyhow::Result<Self> {
27 pub fn connect(config: &postgres::Config) -> anyhow::Result<Self> {
25− let pg = config.connect(NoTls)?;
28+ let mut pg = config.connect(NoTls)?;
29+
30+ // When running DDL queries that acquire locks, we risk causing a "lock queue".
31+ // When attempting to acquire a lock, Postgres will wait for any long running queries to complete.
32+ // At the same time, it will block other queries until the lock has been acquired and released.
33+ // This has the bad effect of the long-running query blocking other queries because of us, forming
34+ // a queue of other queries until we release our lock.
35+ //
36+ // We set the lock_timeout setting to avoid this. This puts an upper bound for how long Postgres will
37+ // wait to acquire locks and also the maximum amount of time a long-running query can block other queries.
38+ // We should also add automatic retries to handle these timeouts gracefully.
39+ //
40+ // Reference: https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680
41+ //
42+ // TODO: Make lock_timeout configurable
43+ pg.simple_query("SET lock_timeout = '1s'")
44+ .context("failed to set lock_timeout")?;
45+
26 Ok(Self {
46 Ok(Self {
27 client: DbConn::new(pg),
47 client: DbConn::new(pg),
28 })
48 })
@@ -86,12 +106,12 @@ impl DbConn {
86
106
87impl Conn for DbConn {
107impl Conn for DbConn {
88 fn run(&mut self, query: &str) -> anyhow::Result<()> {
108 fn run(&mut self, query: &str) -> anyhow::Result<()> {
89− self.client.batch_execute(query)?;
109+ retry_automatically(|| self.client.batch_execute(query))?;
90 Ok(())
110 Ok(())
91 }
111 }
92
112
93 fn query(&mut self, query: &str) -> anyhow::Result<Vec<Row>> {
113 fn query(&mut self, query: &str) -> anyhow::Result<Vec<Row>> {
94− let rows = self.client.query(query, &[])?;
114+ let rows = retry_automatically(|| self.client.query(query, &[]))?;
95 Ok(rows)
115 Ok(rows)
96 }
116 }
97
117
@@ -100,7 +120,7 @@ impl Conn for DbConn {
100 query: &str,
120 query: &str,
101 params: &[&(dyn ToSql + Sync)],
121 params: &[&(dyn ToSql + Sync)],
102 ) -> anyhow::Result<Vec<Row>> {
122 ) -> anyhow::Result<Vec<Row>> {
103− let rows = self.client.query(query, params)?;
123+ let rows = retry_automatically(|| self.client.query(query, params))?;
104 Ok(rows)
124 Ok(rows)
105 }
125 }
106
126
@@ -151,3 +171,54 @@ impl Conn for Transaction<'_> {
151 Ok(Transaction { transaction })
171 Ok(Transaction { transaction })
152 }
172 }
153}
173}
174+
175+// Retry a database operation with exponential backoff and jitter
176+fn retry_automatically<T>(
177+ mut f: impl FnMut() -> Result<T, postgres::Error>,
178+) -> Result<T, postgres::Error> {
179+ const STARTING_WAIT_TIME: u64 = 100;
180+ const MAX_WAIT_TIME: u64 = 3_200;
181+ const MAX_ATTEMPTS: u32 = 10;
182+
183+ let mut rng = rand::thread_rng();
184+ let mut attempts = 0;
185+ loop {
186+ let result = f();
187+
188+ let error = match result {
189+ Ok(_) => return result,
190+ Err(err) => err,
191+ };
192+
193+ // If we got a database error, we check if it's retryable.
194+ // If we didn't get a database error, then it's most likely some kind of connection
195+ // error which should also be retried.
196+ if let Some(db_error) = error.as_db_error() {
197+ if !error_retryable(db_error) {
198+ return Err(error);
199+ }
200+ }
201+
202+ attempts += 1;
203+ if attempts >= MAX_ATTEMPTS {
204+ return Err(error);
205+ }
206+
207+ // The wait time increases exponentially, starting at 100ms and doubling up to a max of 3.2s.
208+ let wait_time = min(
209+ MAX_WAIT_TIME,
210+ STARTING_WAIT_TIME * u64::pow(2, attempts - 1),
211+ );
212+
213+ // The jitter is up to half the wait time
214+ let jitter: u64 = rng.gen_range(0..wait_time / 2);
215+
216+ std::thread::sleep(Duration::from_millis(wait_time + jitter));
217+ }
218+}
219+
220+// Check if a database error can be retried
221+fn error_retryable(error: &postgres::error::DbError) -> bool {
222+ // LOCK_NOT_AVAILABLE is caused by lock_timeout being exceeded
223+ matches!(error.code(), &postgres::error::SqlState::LOCK_NOT_AVAILABLE)
224+}
src/helpers.rs+7−16
@@ -2,37 +2,28 @@ use anyhow::Context;
2
2
3use crate::db::Conn;
3use crate::db::Conn;
4
4
5−pub fn set_up_helpers(db: &mut dyn Conn, current_migration: &Option<String>) -> anyhow::Result<()> {
5+pub fn set_up_helpers(db: &mut dyn Conn, target_migration: &str) -> anyhow::Result<()> {
6− let predicate = if let Some(current_migration) = current_migration {
7− format!(
8− "current_setting('search_path') = 'migration_{}' OR setting_bool",
9− current_migration
10− )
11− } else {
12− "setting_bool".to_string()
13− };
14−
15 let query = format!(
6 let query = format!(
16 "
7 "
17− CREATE OR REPLACE FUNCTION reshape.is_old_schema()
8+ CREATE OR REPLACE FUNCTION reshape.is_new_schema()
18 RETURNS BOOLEAN AS $$
9 RETURNS BOOLEAN AS $$
19 DECLARE
10 DECLARE
20− setting TEXT := current_setting('reshape.is_old_schema', TRUE);
11+ setting TEXT := current_setting('reshape.is_new_schema', TRUE);
21 setting_bool BOOLEAN := setting IS NOT NULL AND setting = 'YES';
12 setting_bool BOOLEAN := setting IS NOT NULL AND setting = 'YES';
22 BEGIN
13 BEGIN
23− RETURN {};
14+ RETURN current_setting('search_path') = 'migration_{}' OR setting_bool;
24 END
15 END
25 $$ language 'plpgsql';
16 $$ language 'plpgsql';
26 ",
17 ",
27− predicate
18+ target_migration,
28 );
19 );
29 db.query(&query)
20 db.query(&query)
30− .context("failed creating helper function reshape.is_old_schema()")?;
21+ .context("failed creating helper function reshape.is_new_schema()")?;
31
22
32 Ok(())
23 Ok(())
33}
24}
34
25
35pub fn tear_down_helpers(db: &mut dyn Conn) -> anyhow::Result<()> {
26pub fn tear_down_helpers(db: &mut dyn Conn) -> anyhow::Result<()> {
36− db.query("DROP FUNCTION IF EXISTS reshape.is_old_schema;")?;
27+ db.query("DROP FUNCTION IF EXISTS reshape.is_new_schema;")?;
37 Ok(())
28 Ok(())
38}
29}
src/lib.rs+39−36
@@ -30,6 +30,7 @@ impl Reshape {
30 pub fn new_with_options(
30 pub fn new_with_options(
31 host: &str,
31 host: &str,
32 port: u16,
32 port: u16,
33+ database: &str,
33 username: &str,
34 username: &str,
34 password: &str,
35 password: &str,
35 ) -> anyhow::Result<Reshape> {
36 ) -> anyhow::Result<Reshape> {
@@ -38,6 +39,7 @@ impl Reshape {
38 .host(host)
39 .host(host)
39 .port(port)
40 .port(port)
40 .user(username)
41 .user(username)
42+ .dbname(database)
41 .password(password);
43 .password(password);
42
44
43 Self::new_with_config(&config)
45 Self::new_with_config(&config)
@@ -103,6 +105,16 @@ impl Reshape {
103 ))?;
105 ))?;
104 }
106 }
105
107
108+ // Remove all enums
109+ let enums: Vec<String> = db
110+ .query("SELECT typname FROM pg_type WHERE typcategory = 'E'")?
111+ .iter()
112+ .map(|row| row.get("typname"))
113+ .collect();
114+ for enum_type in enums {
115+ db.run(&format!("DROP TYPE {}", enum_type))?;
116+ }
117+
106 // Reset state
118 // Reset state
107 state.clear(db)?;
119 state.clear(db)?;
108
120
@@ -135,13 +147,15 @@ fn migrate(
135) -> anyhow::Result<()> {
147) -> anyhow::Result<()> {
136 // Make sure no migration is in progress
148 // Make sure no migration is in progress
137 if let State::InProgress { .. } = &state {
149 if let State::InProgress { .. } = &state {
150+ println!(
138− println!("Migration already in progress, please complete using 'reshape complete'");
151+ "Migration already in progress, please complete using 'reshape migration complete'"
152+ );
139 return Ok(());
153 return Ok(());
140 }
154 }
141
155
142 if let State::Completing { .. } = &state {
156 if let State::Completing { .. } = &state {
143 println!(
157 println!(
144− "Migration already in progress and has started completion, please finish using 'reshape complete'"
158+ "Migration already in progress and has started completion, please finish using 'reshape migration complete'"
145 );
159 );
146 return Ok(());
160 return Ok(());
147 }
161 }
@@ -150,7 +164,6 @@ fn migrate(
150 // with the already applied ones stored in the state. This will throw an error if the
164 // with the already applied ones stored in the state. This will throw an error if the
151 // two sets of migrations don't agree, for example if a new migration has been added
165 // two sets of migrations don't agree, for example if a new migration has been added
152 // in between two existing ones.
166 // in between two existing ones.
153− let current_migration = state::current_migration(db)?;
154 let remaining_migrations = state::remaining_migrations(db, migrations)?;
167 let remaining_migrations = state::remaining_migrations(db, migrations)?;
155 if remaining_migrations.is_empty() {
168 if remaining_migrations.is_empty() {
156 println!("No migrations left to apply");
169 println!("No migrations left to apply");
@@ -165,8 +178,8 @@ fn migrate(
165 {
178 {
166 if existing_migrations != &remaining_migrations {
179 if existing_migrations != &remaining_migrations {
167 return Err(anyhow!(
180 return Err(anyhow!(
168− "a previous migration seems to have failed without cleaning up. Please run `reshape abort` and then run migrate again."
181+ "a previous migration seems to have failed without cleaning up. Please run `reshape migration abort` and then run migrate again."
169− ));
182+ ));
170 }
183 }
171 }
184 }
172
185
@@ -179,14 +192,15 @@ fn migrate(
179
192
180 println!("Applying {} migrations\n", remaining_migrations.len());
193 println!("Applying {} migrations\n", remaining_migrations.len());
181
194
195+ let target_migration = remaining_migrations.last().unwrap().name.to_string();
182− helpers::set_up_helpers(db, &current_migration).context("failed to set up helpers")?;
196+ helpers::set_up_helpers(db, &target_migration).context("failed to set up helpers")?;
183
197
184 let mut new_schema = Schema::new();
198 let mut new_schema = Schema::new();
185 let mut last_migration_index = usize::MAX;
199 let mut last_migration_index = usize::MAX;
186 let mut last_action_index = usize::MAX;
200 let mut last_action_index = usize::MAX;
187 let mut result: anyhow::Result<()> = Ok(());
201 let mut result: anyhow::Result<()> = Ok(());
188
202
189− for (migration_index, migration) in remaining_migrations.iter().enumerate() {
203+ 'outer: for (migration_index, migration) in remaining_migrations.iter().enumerate() {
190 println!("Migrating '{}':", migration.name);
204 println!("Migrating '{}':", migration.name);
191 last_migration_index = migration_index;
205 last_migration_index = migration_index;
192
206
@@ -196,7 +210,8 @@ fn migrate(
196 let description = action.describe();
210 let description = action.describe();
197 print!(" + {} ", description);
211 print!(" + {} ", description);
198
212
213+ let ctx =
199− let ctx = MigrationContext::new(migration_index, action_index);
214+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
200 result = action
215 result = action
201 .run(&ctx, db, &new_schema)
216 .run(&ctx, db, &new_schema)
202 .with_context(|| format!("failed to {}", description));
217 .with_context(|| format!("failed to {}", description));
@@ -206,7 +221,7 @@ fn migrate(
206 println!("{}", "done".green());
221 println!("{}", "done".green());
207 } else {
222 } else {
208 println!("{}", "failed".red());
223 println!("{}", "failed".red());
209− break;
224+ break 'outer;
210 }
225 }
211 }
226 }
212
227
@@ -220,7 +235,7 @@ fn migrate(
220 // Set to the Aborting state. This is to ensure that the failed
235 // Set to the Aborting state. This is to ensure that the failed
221 // migration is fully aborted and nothing is left dangling.
236 // migration is fully aborted and nothing is left dangling.
222 // If the abort is interrupted for any reason, the user can try again
237 // If the abort is interrupted for any reason, the user can try again
223− // by running `reshape abort`.
238+ // by running `reshape migration abort`.
224 state.aborting(
239 state.aborting(
225 remaining_migrations.clone(),
240 remaining_migrations.clone(),
226 last_migration_index + 1,
241 last_migration_index + 1,
@@ -234,7 +249,6 @@ fn migrate(
234 }
249 }
235
250
236 // Create schema and views for migration
251 // Create schema and views for migration
237− let target_migration = remaining_migrations.last().unwrap().name.to_string();
238 create_schema_for_migration(db, &target_migration, &new_schema)
252 create_schema_for_migration(db, &target_migration, &new_schema)
239 .with_context(|| format!("failed to create schema for migration {}", target_migration))?;
253 .with_context(|| format!("failed to create schema for migration {}", target_migration))?;
240
254
@@ -242,27 +256,14 @@ fn migrate(
242 state.in_progress(remaining_migrations);
256 state.in_progress(remaining_migrations);
243 state.save(db).context("failed to save in-progress state")?;
257 state.save(db).context("failed to save in-progress state")?;
244
258
245− // If we started from a blank slate, we can finish the migration immediately
246− if current_migration.is_none() {
247− println!("Automatically completing migrations\n");
248− complete(db, state).context("failed to automatically complete migrations")?;
249−
250− println!("Migrations complete:");
251− println!(
252− " - Run '{}' from your application to use the latest schema",
253− schema_query_for_migration(&target_migration)
254− );
255− } else {
256− println!("Migrations have been applied and the new schema is ready for use:");
259+ println!("Migrations have been applied and the new schema is ready for use:");
257− println!(
260+ println!(
258− " - Run '{}' from your application to use the latest schema",
261+ " - Run '{}' from your application to use the latest schema",
259− schema_query_for_migration(&target_migration)
262+ schema_query_for_migration(&target_migration)
260− );
263+ );
261− println!(
264+ println!(
262− " - Run 'reshape complete' once your application has been updated and the previous schema is no longer in use"
265+ " - Run 'reshape migration complete' once your application has been updated and the previous schema is no longer in use"
263− );
266+ );
264− }
265−
266 Ok(())
267 Ok(())
267}
268}
268
269
@@ -283,7 +284,7 @@ fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
283 current_action_index
284 current_action_index
284 } => (migrations, current_migration_index, current_action_index),
285 } => (migrations, current_migration_index, current_action_index),
285 State::Aborting { .. } => {
286 State::Aborting { .. } => {
286− return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape abort`."))
287+ return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape migration abort`."))
287 }
288 }
288 State::Applying { .. } => {
289 State::Applying { .. } => {
289 return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
290 return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
@@ -320,7 +321,8 @@ fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
320 let description = action.describe();
321 let description = action.describe();
321 print!(" + {} ", description);
322 print!(" + {} ", description);
322
323
324+ let ctx =
323− let ctx = MigrationContext::new(migration_index, action_index);
325+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
324
326
325 // Update state to indicate that this action has been completed.
327 // Update state to indicate that this action has been completed.
326 // We won't save this new state until after the action has completed.
328 // We won't save this new state until after the action has completed.
@@ -411,7 +413,7 @@ fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
411 last_action_index,
413 last_action_index,
412 } => (migrations, last_migration_index, last_action_index),
414 } => (migrations, last_migration_index, last_action_index),
413 State::Completing { .. } => {
415 State::Completing { .. } => {
414− return Err(anyhow!("Migration completion has already been started. Please run `reshape complete` again to finish it."));
416+ return Err(anyhow!("migration completion has already been started. Please run `reshape migration complete` again to finish it."));
415 }
417 }
416 State::Idle => {
418 State::Idle => {
417 println!("No migration is in progress");
419 println!("No migration is in progress");
@@ -445,7 +447,8 @@ fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
445 continue;
447 continue;
446 }
448 }
447
449
450+ let ctx =
448− let ctx = MigrationContext::new(migration_index, action_index);
451+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
449 action
452 action
450 .abort(&ctx, db)
453 .abort(&ctx, db)
451 .with_context(|| format!("failed to abort migration {}", migration.name))
454 .with_context(|| format!("failed to abort migration {}", migration.name))
src/main.rs+93−18
@@ -4,6 +4,7 @@ use std::{
4 path::Path,
4 path::Path,
5};
5};
6
6
7+use anyhow::Context;
7use clap::{Args, Parser};
8use clap::{Args, Parser};
8use reshape::{
9use reshape::{
9 migrations::{Action, Migration},
10 migrations::{Action, Migration},
@@ -12,22 +13,69 @@ use reshape::{
12use serde::{Deserialize, Serialize};
13use serde::{Deserialize, Serialize};
13
14
14#[derive(Parser)]
15#[derive(Parser)]
16+#[clap(name = "Reshape", version, about)]
15struct Opts {
17struct Opts {
16 #[clap(subcommand)]
18 #[clap(subcommand)]
17 cmd: Command,
19 cmd: Command,
18}
20}
19
21
20#[derive(Parser)]
22#[derive(Parser)]
23+#[clap(about)]
21enum Command {
24enum Command {
25+ #[clap(subcommand)]
26+ Migration(MigrationCommand),
27+
28+ #[clap(
29+ about = "Output the query your application should use to select the right schema",
30+ display_order = 2
31+ )]
32+ SchemaQuery(FindMigrationsOptions),
33+
34+ #[clap(
35+ about = "Deprecated. Use `reshape schema-query` instead",
36+ display_order = 3
37+ )]
38+ GenerateSchemaQuery(FindMigrationsOptions),
39+
40+ #[clap(
41+ about = "Deprecated. Use `reshape migration start` instead",
42+ display_order = 4
43+ )]
22 Migrate(MigrateOptions),
44 Migrate(MigrateOptions),
45+ #[clap(
46+ about = "Deprecated. Use `reshape migration complete` instead",
47+ display_order = 5
48+ )]
23 Complete(ConnectionOptions),
49 Complete(ConnectionOptions),
50+ #[clap(
51+ about = "Deprecated. Use `reshape migration abort` instead",
52+ display_order = 6
53+ )]
24− Remove(ConnectionOptions),
54+ Abort(ConnectionOptions),
55+}
56+
57+#[derive(Parser)]
58+#[clap(about = "Commands for managing migrations", display_order = 1)]
59+enum MigrationCommand {
60+ #[clap(
61+ about = "Starts a new migration, applying any migrations which haven't yet been applied",
62+ display_order = 1
63+ )]
64+ Start(MigrateOptions),
65+
66+ #[clap(about = "Completes an in-progress migration", display_order = 2)]
67+ Complete(ConnectionOptions),
68+
69+ #[clap(
70+ about = "Aborts an in-progress migration without losing any data",
71+ display_order = 3
72+ )]
25 Abort(ConnectionOptions),
73 Abort(ConnectionOptions),
26− GenerateSchemaQuery(FindMigrationsOptions),
27}
74}
28
75
29#[derive(Args)]
76#[derive(Args)]
30struct MigrateOptions {
77struct MigrateOptions {
78+ // Some comment
31 #[clap(long, short)]
79 #[clap(long, short)]
32 complete: bool,
80 complete: bool,
33 #[clap(flatten)]
81 #[clap(flatten)]
@@ -65,7 +113,7 @@ fn main() -> anyhow::Result<()> {
65
113
66fn run(opts: Opts) -> anyhow::Result<()> {
114fn run(opts: Opts) -> anyhow::Result<()> {
67 match opts.cmd {
115 match opts.cmd {
68− Command::Migrate(opts) => {
116+ Command::Migration(MigrationCommand::Start(opts)) | Command::Migrate(opts) => {
69 let mut reshape = reshape_from_connection_options(&opts.connection_options)?;
117 let mut reshape = reshape_from_connection_options(&opts.connection_options)?;
70 let migrations = find_migrations(&opts.find_migrations_options)?;
118 let migrations = find_migrations(&opts.find_migrations_options)?;
71 reshape.migrate(migrations)?;
119 reshape.migrate(migrations)?;
@@ -77,20 +125,16 @@ fn run(opts: Opts) -> anyhow::Result<()> {
77
125
78 Ok(())
126 Ok(())
79 }
127 }
80− Command::Complete(opts) => {
128+ Command::Migration(MigrationCommand::Complete(opts)) | Command::Complete(opts) => {
81 let mut reshape = reshape_from_connection_options(&opts)?;
129 let mut reshape = reshape_from_connection_options(&opts)?;
82 reshape.complete()
130 reshape.complete()
83 }
131 }
84− Command::Remove(opts) => {
85− let mut reshape = reshape_from_connection_options(&opts)?;
86− reshape.remove()
87− }
88− Command::Abort(opts) => {
132+ Command::Migration(MigrationCommand::Abort(opts)) | Command::Abort(opts) => {
89 let mut reshape = reshape_from_connection_options(&opts)?;
133 let mut reshape = reshape_from_connection_options(&opts)?;
90 reshape.abort()
134 reshape.abort()
91 }
135 }
92− Command::GenerateSchemaQuery(find_migrations_options) => {
136+ Command::SchemaQuery(opts) | Command::GenerateSchemaQuery(opts) => {
93− let migrations = find_migrations(&find_migrations_options)?;
137+ let migrations = find_migrations(&opts)?;
94 let query = migrations
138 let query = migrations
95 .last()
139 .last()
96 .map(|migration| reshape::schema_query_for_migration(&migration.name));
140 .map(|migration| reshape::schema_query_for_migration(&migration.name));
@@ -102,13 +146,35 @@ fn run(opts: Opts) -> anyhow::Result<()> {
102}
146}
103
147
104fn reshape_from_connection_options(opts: &ConnectionOptions) -> anyhow::Result<Reshape> {
148fn reshape_from_connection_options(opts: &ConnectionOptions) -> anyhow::Result<Reshape> {
105− let env_url = std::env::var("POSTGRES_URL").ok();
149+ // Load environment variables from .env file if it exists
106− let url = env_url.as_ref().or_else(|| opts.url.as_ref());
150+ dotenv::dotenv().ok();
107
151
108− match url {
152+ let url_env = std::env::var("DB_URL").ok();
153+ let url = url_env.as_ref().or_else(|| opts.url.as_ref());
154+
155+ // Use the connection URL if it has been set
156+ if let Some(url) = url {
109− Some(url) => Reshape::new(url),
157+ return Reshape::new(url);
110− None => Reshape::new_with_options(&opts.host, opts.port, &opts.username, &opts.password),
111 }
158 }
159+
160+ let host_env = std::env::var("DB_HOST").ok();
161+ let host = host_env.as_ref().unwrap_or_else(|| &opts.host);
162+
163+ let port = std::env::var("DB_PORT")
164+ .ok()
165+ .and_then(|port| port.parse::<u16>().ok())
166+ .unwrap_or(opts.port);
167+
168+ let username_env = std::env::var("DB_USERNAME").ok();
169+ let username = username_env.as_ref().unwrap_or_else(|| &opts.username);
170+
171+ let password_env = std::env::var("DB_PASSWORD").ok();
172+ let password = password_env.as_ref().unwrap_or_else(|| &opts.password);
173+
174+ let database_env = std::env::var("DB_NAME").ok();
175+ let database = database_env.as_ref().unwrap_or_else(|| &opts.database);
176+
177+ Reshape::new_with_options(host, port, database, username, password)
112}
178}
113
179
114fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration>> {
180fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration>> {
@@ -130,7 +196,13 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
130 }
196 }
131
197
132 // Sort all files by their file names (without extension)
198 // Sort all files by their file names (without extension)
133− file_paths.sort_unstable_by_key(|path| path.as_path().file_stem().unwrap().to_os_string());
199+ // The files are sorted naturally, e.g. "1_test_migration" < "10_test_migration"
200+ file_paths.sort_unstable_by(|path1, path2| {
201+ let file1 = path1.as_path().file_stem().unwrap().to_str().unwrap();
202+ let file2 = path2.as_path().file_stem().unwrap().to_str().unwrap();
203+
204+ lexical_sort::natural_cmp(file1, file2)
205+ });
134
206
135 file_paths
207 file_paths
136 .iter()
208 .iter()
@@ -146,7 +218,10 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
146 .map(|result| {
218 .map(|result| {
147 result.and_then(|(path, data)| {
219 result.and_then(|(path, data)| {
148 let extension = path.extension().and_then(|ext| ext.to_str()).unwrap();
220 let extension = path.extension().and_then(|ext| ext.to_str()).unwrap();
221+ let file_migration =
149− let file_migration = decode_migration_file(&data, extension)?;
222+ decode_migration_file(&data, extension).with_context(|| {
223+ format!("failed to parse migration file {}", path.display())
224+ })?;
150
225
151 let file_name = path.file_stem().and_then(|name| name.to_str()).unwrap();
226 let file_name = path.file_stem().and_then(|name| name.to_str()).unwrap();
152 Ok(Migration {
227 Ok(Migration {
src/migrations/add_column.rs+214−60
@@ -3,14 +3,25 @@ use crate::{
3 db::{Conn, Transaction},
3 db::{Conn, Transaction},
4 schema::Schema,
4 schema::Schema,
5};
5};
6−use anyhow::Context;
6+use anyhow::{bail, Context};
7use serde::{Deserialize, Serialize};
7use serde::{Deserialize, Serialize};
8
8
9#[derive(Serialize, Deserialize, Debug)]
9#[derive(Serialize, Deserialize, Debug)]
10pub struct AddColumn {
10pub struct AddColumn {
11 pub table: String,
11 pub table: String,
12− pub up: Option<String>,
13 pub column: Column,
12 pub column: Column,
13+ pub up: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
14}
25}
15
26
16impl AddColumn {
27impl AddColumn {
@@ -32,6 +43,15 @@ impl AddColumn {
32 )
43 )
33 }
44 }
34
45
46+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_add_column_{}_{}_rev",
49+ ctx.prefix(),
50+ self.table,
51+ self.column.name
52+ )
53+ }
54+
35 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
55 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
36 format!(
56 format!(
37 "{}_add_column_not_null_{}_{}",
57 "{}_add_column_not_null_{}_{}",
@@ -61,7 +81,7 @@ impl Action for AddColumn {
61 let temp_column_name = self.temp_column_name(ctx);
81 let temp_column_name = self.temp_column_name(ctx);
62
82
63 let mut definition_parts = vec![
83 let mut definition_parts = vec![
64− temp_column_name.to_string(),
84+ format!("\"{}\"", temp_column_name.to_string()),
65 self.column.data_type.to_string(),
85 self.column.data_type.to_string(),
66 ];
86 ];
67
87
@@ -86,55 +106,189 @@ impl Action for AddColumn {
86 );
106 );
87 db.run(&query).context("failed to add column")?;
107 db.run(&query).context("failed to add column")?;
88
108
109+ let declarations: Vec<String> = table
110+ .columns
111+ .iter()
112+ .map(|column| {
113+ format!(
114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",
115+ table = table.real_name,
116+ alias = column.name,
117+ real_name = column.real_name,
118+ )
119+ })
120+ .collect();
121+
89 if let Some(up) = &self.up {
122 if let Some(up) = &self.up {
90− let table = schema.get_table(db, &self.table)?;
123+ if let Transformation::Simple(up) = up {
91−
92− let declarations: Vec<String> = table
93− .columns
94− .iter()
95− .map(|column| {
96− format!(
97− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
98− table = table.real_name,
99− alias = column.name,
100− real_name = column.real_name,
101− )
102− })
103− .collect();
104−
105− // Add triggers to fill in values as they are inserted/updated
124+ // Add triggers to fill in values as they are inserted/updated
106− let query = format!(
125+ let query = format!(
107− r#"
126+ r#"
108− CREATE OR REPLACE FUNCTION {trigger_name}()
127+ CREATE OR REPLACE FUNCTION {trigger_name}()
109− RETURNS TRIGGER AS $$
128+ RETURNS TRIGGER AS $$
110− BEGIN
129+ BEGIN
111− IF reshape.is_old_schema() THEN
130+ IF NOT reshape.is_new_schema() THEN
112− DECLARE
131+ DECLARE
113− {declarations}
132+ {declarations}
114− BEGIN
133+ BEGIN
115− NEW.{temp_column_name} = {up};
134+ NEW."{temp_column_name}" = {up};
116− END;
135+ END;
117− END IF;
136+ END IF;
118− RETURN NEW;
137+ RETURN NEW;
119− END
138+ END
120− $$ language 'plpgsql';
139+ $$ language 'plpgsql';
121−
140+
122− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
141+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
123− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
142+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
124− "#,
143+ "#,
125− temp_column_name = temp_column_name,
144+ temp_column_name = temp_column_name,
126− trigger_name = self.trigger_name(ctx),
145+ trigger_name = self.trigger_name(ctx),
127− up = up,
146+ up = up,
128− table = self.table,
147+ table = self.table,
129− declarations = declarations.join("\n"),
148+ declarations = declarations.join("\n"),
130− );
149+ );
131− db.run(&query).context("failed to create up trigger")?;
150+ db.run(&query).context("failed to create up trigger")?;
132− }
133
151
134− // Backfill values in batches
152+ // Backfill values in batches
135− if self.up.is_some() {
153+ common::batch_touch_rows(db, &table.real_name, Some(&temp_column_name))
154+ .context("failed to batch update existing rows")?;
155+ }
156+
157+ if let Transformation::Update {
158+ table: from_table,
159+ value,
160+ r#where,
161+ } = up
162+ {
163+ let existing_schema_name = match &ctx.existing_schema_name {
164+ Some(name) => name,
165+ None => bail!("can't use update without previous migration"),
166+ };
167+
168+ let from_table = schema.get_table(db, &from_table)?;
169+
170+ let from_table_assignments: Vec<String> = from_table
171+ .columns
172+ .iter()
173+ .map(|column| {
174+ format!(
175+ "{table}.{alias} = NEW.{real_name};",
176+ table = from_table.name,
177+ alias = column.name,
178+ real_name = column.real_name,
179+ )
180+ })
181+ .collect();
182+
183+ // Add triggers to fill in values as they are inserted/updated
184+ let query = format!(
185+ r#"
186+ CREATE OR REPLACE FUNCTION {trigger_name}()
187+ RETURNS TRIGGER AS $$
188+ #variable_conflict use_variable
189+ BEGIN
190+ IF NOT reshape.is_new_schema() THEN
191+ DECLARE
192+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
193+ BEGIN
194+ {assignments}
195+
196+ -- Don't trigger reverse trigger when making this update
197+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
198+
199+ UPDATE public."{changed_table_real}"
200+ SET "{temp_column_name}" = {value}
201+ WHERE {where};
202+
203+ perform set_config('reshape.disable_triggers', '', TRUE);
204+ END;
205+ END IF;
206+ RETURN NEW;
207+ END
208+ $$ language 'plpgsql';
209+
210+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
211+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
212+ "#,
213+ assignments = from_table_assignments.join("\n"),
214+ changed_table_real = table.real_name,
215+ from_table = from_table.name,
216+ from_table_real = from_table.real_name,
217+ trigger_name = self.trigger_name(ctx),
218+ // declarations = from_table_declarations.join("\n"),
219+ temp_column_name = temp_column_name,
220+ );
221+ db.run(&query).context("failed to create up trigger")?;
222+
223+ let from_table_columns = from_table
224+ .columns
225+ .iter()
226+ .map(|column| format!("{} as {}", column.real_name, column.name))
227+ .collect::<Vec<String>>()
228+ .join(", ");
229+
230+ let changed_table_assignments: Vec<String> = table
231+ .columns
232+ .iter()
233+ .map(|column| {
234+ format!(
235+ "{table}.{alias} := NEW.{real_name};",
236+ table = table.name,
237+ alias = column.name,
238+ real_name = column.real_name,
239+ )
240+ })
241+ .collect();
242+
243+ // Add triggers to fill in values as they are inserted/updated
244+ let query = format!(
245+ r#"
246+ CREATE OR REPLACE FUNCTION {trigger_name}()
247+ RETURNS TRIGGER AS $$
248+ #variable_conflict use_variable
249+ BEGIN
250+ IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
251+ DECLARE
252+ {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;
253+ __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;
254+ BEGIN
255+ {changed_table_assignments}
256+
257+ SELECT {from_table_columns}
258+ INTO __temp_row
259+ FROM migration_{existing_schema_name}.{from_table} {from_table}
260+ WHERE {where};
261+
262+ DECLARE
263+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
264+ BEGIN
265+ {from_table} = __temp_row;
266+ NEW.{temp_column_name} = {value};
267+ END;
268+ END;
269+ END IF;
270+ RETURN NEW;
271+ END
272+ $$ language 'plpgsql';
273+
274+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
275+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
276+ "#,
277+ changed_table_assignments = changed_table_assignments.join("\n"),
278+ changed_table_real = table.real_name,
279+ changed_table = table.name,
280+ from_table = from_table.name,
281+ trigger_name = self.reverse_trigger_name(ctx),
282+ temp_column_name = temp_column_name,
283+ // declarations = declarations.join("\n"),
284+ );
285+ db.run(&query)
286+ .context("failed to create reverse up trigger")?;
287+
288+ // Backfill values in batches by touching the from table
136− common::batch_touch_rows(db, &table.real_name, &temp_column_name)
289+ common::batch_touch_rows(db, &from_table.real_name, None)
137− .context("failed to batch update existing rows")?;
290+ .context("failed to batch update existing rows")?;
291+ }
138 }
292 }
139
293
140 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
294 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
@@ -144,10 +298,10 @@ impl Action for AddColumn {
144 if !self.column.nullable {
298 if !self.column.nullable {
145 let query = format!(
299 let query = format!(
146 r#"
300 r#"
147− ALTER TABLE "{table}"
301+ ALTER TABLE "{table}"
148− ADD CONSTRAINT "{constraint_name}"
302+ ADD CONSTRAINT "{constraint_name}"
149− CHECK ("{column}" IS NOT NULL) NOT VALID
303+ CHECK ("{column}" IS NOT NULL) NOT VALID
150− "#,
304+ "#,
151 table = self.table,
305 table = self.table,
152 constraint_name = self.not_null_constraint_name(ctx),
306 constraint_name = self.not_null_constraint_name(ctx),
153 column = temp_column_name,
307 column = temp_column_name,
@@ -169,11 +323,11 @@ impl Action for AddColumn {
169 // Remove triggers and procedures
323 // Remove triggers and procedures
170 let query = format!(
324 let query = format!(
171 r#"
325 r#"
172− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
326+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
173− DROP FUNCTION IF EXISTS "{trigger_name}";
327+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
174 "#,
328 "#,
175− table = self.table,
176 trigger_name = self.trigger_name(ctx),
329 trigger_name = self.trigger_name(ctx),
330+ reverse_trigger_name = self.reverse_trigger_name(ctx),
177 );
331 );
178 transaction
332 transaction
179 .run(&query)
333 .run(&query)
@@ -264,11 +418,11 @@ impl Action for AddColumn {
264 // Remove triggers and procedures
418 // Remove triggers and procedures
265 let query = format!(
419 let query = format!(
266 r#"
420 r#"
267− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
421+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
268− DROP FUNCTION IF EXISTS "{trigger_name}";
422+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
269 "#,
423 "#,
270− table = self.table,
271 trigger_name = self.trigger_name(ctx),
424 trigger_name = self.trigger_name(ctx),
425+ reverse_trigger_name = self.reverse_trigger_name(ctx),
272 );
426 );
273 db.run(&query).context("failed to drop up trigger")?;
427 db.run(&query).context("failed to drop up trigger")?;
274
428
src/migrations/add_foreign_key.rsnew file+122
@@ -0,0 +1,122 @@
1+use super::{common::ForeignKey, Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct AddForeignKey {
11+ pub table: String,
12+ foreign_key: ForeignKey,
13+}
14+
15+#[typetag::serde(name = "add_foreign_key")]
16+impl Action for AddForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Adding foreign key from table \"{}\" to \"{}\"",
20+ self.table, self.foreign_key.referenced_table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ let table = schema.get_table(db, &self.table)?;
31+ let referenced_table = schema.get_table(db, &self.foreign_key.referenced_table)?;
32+
33+ // Add quotes around all column names
34+ let columns: Vec<String> = table
35+ .real_column_names(&self.foreign_key.columns)
36+ .map(|col| format!("\"{}\"", col))
37+ .collect();
38+ let referenced_columns: Vec<String> = referenced_table
39+ .real_column_names(&self.foreign_key.referenced_columns)
40+ .map(|col| format!("\"{}\"", col))
41+ .collect();
42+
43+ // Create foreign key but set is as NOT VALID.
44+ // This means the foreign key will be enforced for inserts and updates
45+ // but the existing data won't be checked, that would cause a long-lived lock.
46+ db.run(&format!(
47+ r#"
48+ ALTER TABLE "{table}"
49+ ADD CONSTRAINT {constraint_name}
50+ FOREIGN KEY ({columns})
51+ REFERENCES "{referenced_table}" ({referenced_columns})
52+ NOT VALID
53+ "#,
54+ table = table.real_name,
55+ constraint_name = self.temp_constraint_name(ctx),
56+ columns = columns.join(", "),
57+ referenced_table = referenced_table.real_name,
58+ referenced_columns = referenced_columns.join(", "),
59+ ))
60+ .context("failed to create foreign key")?;
61+
62+ db.run(&format!(
63+ r#"
64+ ALTER TABLE "{table}"
65+ VALIDATE CONSTRAINT "{constraint_name}"
66+ "#,
67+ table = table.real_name,
68+ constraint_name = self.temp_constraint_name(ctx),
69+ ))
70+ .context("failed to validate foreign key")?;
71+
72+ Ok(())
73+ }
74+
75+ fn complete<'a>(
76+ &self,
77+ ctx: &MigrationContext,
78+ db: &'a mut dyn Conn,
79+ ) -> anyhow::Result<Option<Transaction<'a>>> {
80+ db.run(&format!(
81+ r#"
82+ ALTER TABLE {table}
83+ RENAME CONSTRAINT {temp_constraint_name} TO {constraint_name}
84+ "#,
85+ table = self.table,
86+ temp_constraint_name = self.temp_constraint_name(ctx),
87+ constraint_name = self.final_constraint_name(),
88+ ))
89+ .context("failed to rename temporary constraint")?;
90+ Ok(None)
91+ }
92+
93+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
94+
95+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
96+ db.run(&format!(
97+ r#"
98+ ALTER TABLE "{table}"
99+ DROP CONSTRAINT IF EXISTS "{constraint_name}"
100+ "#,
101+ table = self.table,
102+ constraint_name = self.temp_constraint_name(ctx),
103+ ))
104+ .context("failed to validate foreign key")?;
105+
106+ Ok(())
107+ }
108+}
109+
110+impl AddForeignKey {
111+ fn temp_constraint_name(&self, ctx: &MigrationContext) -> String {
112+ format!("{}_temp_fkey", ctx.prefix())
113+ }
114+
115+ fn final_constraint_name(&self) -> String {
116+ format!(
117+ "{table}_{columns}_fkey",
118+ table = self.table,
119+ columns = self.foreign_key.columns.join("_")
120+ )
121+ }
122+}
src/migrations/alter_column.rs+17−16
@@ -48,9 +48,7 @@ impl Action for AlterColumn {
48 let table = schema.get_table(db, &self.table)?;
48 let table = schema.get_table(db, &self.table)?;
49
49
50 let column = table
50 let column = table
51− .columns
52− .iter()
53− .find(|column| column.name == self.column)
51+ .get_column(&self.column)
54 .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
52 .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
55
53
56 let temporary_column_name = self.temporary_column_name(ctx);
54 let temporary_column_name = self.temporary_column_name(ctx);
@@ -105,7 +103,7 @@ impl Action for AlterColumn {
105 CREATE OR REPLACE FUNCTION {up_trigger}()
103 CREATE OR REPLACE FUNCTION {up_trigger}()
106 RETURNS TRIGGER AS $$
104 RETURNS TRIGGER AS $$
107 BEGIN
105 BEGIN
108− IF reshape.is_old_schema() THEN
106+ IF NOT reshape.is_new_schema() THEN
109 DECLARE
107 DECLARE
110 {declarations}
108 {declarations}
111 {existing_column} public.{table}.{existing_column_real}%TYPE := NEW.{existing_column_real};
109 {existing_column} public.{table}.{existing_column_real}%TYPE := NEW.{existing_column_real};
@@ -123,7 +121,7 @@ impl Action for AlterColumn {
123 CREATE OR REPLACE FUNCTION {down_trigger}()
121 CREATE OR REPLACE FUNCTION {down_trigger}()
124 RETURNS TRIGGER AS $$
122 RETURNS TRIGGER AS $$
125 BEGIN
123 BEGIN
126− IF NOT reshape.is_old_schema() THEN
124+ IF reshape.is_new_schema() THEN
127 DECLARE
125 DECLARE
128 {declarations}
126 {declarations}
129 {existing_column} public.{table}.{temp_column}%TYPE := NEW.{temp_column};
127 {existing_column} public.{table}.{temp_column}%TYPE := NEW.{temp_column};
@@ -152,13 +150,13 @@ impl Action for AlterColumn {
152 .context("failed to create up and down triggers")?;
150 .context("failed to create up and down triggers")?;
153
151
154 // Backfill values in batches by touching the previous column
152 // Backfill values in batches by touching the previous column
155− common::batch_touch_rows(db, &table.real_name, &column.real_name)
153+ common::batch_touch_rows(db, &table.real_name, Some(&column.real_name))
156 .context("failed to batch update existing rows")?;
154 .context("failed to batch update existing rows")?;
157
155
158 // Duplicate any indices to the temporary column
156 // Duplicate any indices to the temporary column
159 let indices = common::get_indices_for_column(db, &table.real_name, &column.real_name)?;
157 let indices = common::get_indices_for_column(db, &table.real_name, &column.real_name)?;
160− for (index_name, index_oid) in indices {
158+ for index in indices {
161− let index_columns: Vec<String> = common::get_index_columns(db, &index_name)?
159+ let index_columns: Vec<String> = common::get_index_columns(db, &index.name)?
162 .into_iter()
160 .into_iter()
163 .map(|idx_column| {
161 .map(|idx_column| {
164 // Replace column with temporary column for new index
162 // Replace column with temporary column for new index
@@ -169,15 +167,18 @@ impl Action for AlterColumn {
169 }
167 }
170 })
168 })
171 .collect();
169 .collect();
172− let temp_index_name = self.temp_index_name(ctx, index_oid);
170+ let temp_index_name = self.temp_index_name(ctx, index.oid);
171+
172+ let unique_def = if index.unique { "UNIQUE" } else { "" };
173
173
174 db.query(&format!(
174 db.query(&format!(
175 r#"
175 r#"
176− CREATE INDEX CONCURRENTLY IF NOT EXISTS "{new_index_name}" ON "{table}" ({columns})
176+ CREATE {unique_def} INDEX CONCURRENTLY IF NOT EXISTS "{new_index_name}" ON "{table}" USING {index_type} ({columns})
177 "#,
177 "#,
178 new_index_name = temp_index_name,
178 new_index_name = temp_index_name,
179 table = table.real_name,
179 table = table.real_name,
180 columns = index_columns.join(", "),
180 columns = index_columns.join(", "),
181+ index_type = index.index_type,
181 ))
182 ))
182 .context("failed to create temporary index")?;
183 .context("failed to create temporary index")?;
183 }
184 }
@@ -186,7 +187,7 @@ impl Action for AlterColumn {
186 // This constraint is set as NOT VALID so it doesn't apply to existing rows and
187 // This constraint is set as NOT VALID so it doesn't apply to existing rows and
187 // the existing rows don't need to be scanned under an exclusive lock.
188 // the existing rows don't need to be scanned under an exclusive lock.
188 // Thanks to this, we can set the full column as NOT NULL later with minimal locking.
189 // Thanks to this, we can set the full column as NOT NULL later with minimal locking.
189− if !column.nullable {
190+ if !self.changes.nullable.unwrap_or(column.nullable) {
190 let query = format!(
191 let query = format!(
191 r#"
192 r#"
192 ALTER TABLE "{table}"
193 ALTER TABLE "{table}"
@@ -280,7 +281,7 @@ impl Action for AlterColumn {
280
281
281 // Replace old indices with the new temporary ones created for the temporary column
282 // Replace old indices with the new temporary ones created for the temporary column
282 let indices = common::get_indices_for_column(db, &self.table, &self.column)?;
283 let indices = common::get_indices_for_column(db, &self.table, &self.column)?;
283− for (current_index_name, index_oid) in indices {
284+ for current_index in indices {
284 // To keep the index handling idempotent, we need to do the following:
285 // To keep the index handling idempotent, we need to do the following:
285 // 1. Add a prefix to the existing index
286 // 1. Add a prefix to the existing index
286 // 2. Rename temporary index to its final name
287 // 2. Rename temporary index to its final name
@@ -288,7 +289,7 @@ impl Action for AlterColumn {
288
289
289 // Add prefix (if not already added) to existing index
290 // Add prefix (if not already added) to existing index
290 let prefix = "__reshape_old";
291 let prefix = "__reshape_old";
291− let target_index_name = current_index_name.trim_start_matches(prefix);
292+ let target_index_name = current_index.name.trim_start_matches(prefix);
292 let old_index_name = format!("{}_{}", prefix, target_index_name);
293 let old_index_name = format!("{}_{}", prefix, target_index_name);
293 db.query(&format!(
294 db.query(&format!(
294 r#"
295 r#"
@@ -300,7 +301,7 @@ impl Action for AlterColumn {
300 .context("failed to rename old index")?;
301 .context("failed to rename old index")?;
301
302
302 // Rename temporary index to real name
303 // Rename temporary index to real name
303− let temp_index_name = self.temp_index_name(ctx, index_oid);
304+ let temp_index_name = self.temp_index_name(ctx, current_index.oid);
304 db.query(&format!(
305 db.query(&format!(
305 r#"
306 r#"
306 ALTER INDEX IF EXISTS "{temp_index_name}" RENAME TO "{target_index_name}"
307 ALTER INDEX IF EXISTS "{temp_index_name}" RENAME TO "{target_index_name}"
@@ -388,8 +389,8 @@ impl Action for AlterColumn {
388 // Safely remove any indices created for the temporary column
389 // Safely remove any indices created for the temporary column
389 let temp_column_name = self.temporary_column_name(ctx);
390 let temp_column_name = self.temporary_column_name(ctx);
390 let indices = common::get_indices_for_column(db, &self.table, &temp_column_name)?;
391 let indices = common::get_indices_for_column(db, &self.table, &temp_column_name)?;
391− for (_, index_oid) in indices {
392+ for index in indices {
392− let temp_index_name = self.temp_index_name(ctx, index_oid);
393+ let temp_index_name = self.temp_index_name(ctx, index.oid);
393 db.query(&format!(
394 db.query(&format!(
394 r#"
395 r#"
395 DROP INDEX CONCURRENTLY IF EXISTS "{index_name}"
396 DROP INDEX CONCURRENTLY IF EXISTS "{index_name}"
src/migrations/common.rs+42−26
@@ -1,26 +1,17 @@
1use anyhow::anyhow;
1use anyhow::anyhow;
2−use derive_builder::Builder;
3use postgres::types::{FromSql, ToSql};
2use postgres::types::{FromSql, ToSql};
4use serde::{Deserialize, Serialize};
3use serde::{Deserialize, Serialize};
5
4
6use crate::db::Conn;
5use crate::db::Conn;
7
6
8−#[derive(Serialize, Deserialize, Builder, Clone, Debug)]
7+#[derive(Serialize, Deserialize, Clone, Debug)]
9−#[builder(setter(into))]
10pub struct Column {
8pub struct Column {
11 pub name: String,
9 pub name: String,
12−
13 #[serde(rename = "type")]
10 #[serde(rename = "type")]
14 pub data_type: String,
11 pub data_type: String,
15−
16 #[serde(default = "nullable_default")]
12 #[serde(default = "nullable_default")]
17− #[builder(default = "true")]
18 pub nullable: bool,
13 pub nullable: bool,
19−
20− #[builder(setter(name = "default_value", strip_option), default)]
21 pub default: Option<String>,
14 pub default: Option<String>,
22−
23− #[builder(setter(strip_option), default)]
24 pub generated: Option<String>,
15 pub generated: Option<String>,
25}
16}
26
17
@@ -28,6 +19,13 @@ fn nullable_default() -> bool {
28 true
19 true
29}
20}
30
21
22+#[derive(Serialize, Deserialize, Clone, Debug)]
23+pub struct ForeignKey {
24+ pub columns: Vec<String>,
25+ pub referenced_table: String,
26+ pub referenced_columns: Vec<String>,
27+}
28+
31#[derive(Debug)]
29#[derive(Debug)]
32struct PostgresRawValue {
30struct PostgresRawValue {
33 bytes: Vec<u8>,
31 bytes: Vec<u8>,
@@ -71,17 +69,26 @@ impl ToSql for PostgresRawValue {
71 postgres::types::to_sql_checked!();
69 postgres::types::to_sql_checked!();
72}
70}
73
71
74−pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow::Result<()> {
72+pub fn batch_touch_rows(
73+ db: &mut dyn Conn,
74+ table: &str,
75+ column: Option<&str>,
76+) -> anyhow::Result<()> {
75 const BATCH_SIZE: u16 = 1000;
77 const BATCH_SIZE: u16 = 1000;
76
78
77− db.query("SET reshape.is_old_schema = 'YES'")?;
78−
79 let mut cursor: Option<PostgresRawValue> = None;
79 let mut cursor: Option<PostgresRawValue> = None;
80
80
81 loop {
81 loop {
82 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
82 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
83
83
84 let primary_key = get_primary_key_columns_for_table(db, table)?;
84 let primary_key = get_primary_key_columns_for_table(db, table)?;
85+
86+ // If no column to touch is passed, we default to the first primary key column (just to make some "update")
87+ let touched_column = match column {
88+ Some(column) => column,
89+ None => primary_key.first().unwrap(),
90+ };
91+
85 let primary_key_columns = primary_key.join(", ");
92 let primary_key_columns = primary_key.join(", ");
86
93
87 let primary_key_where = primary_key
94 let primary_key_where = primary_key
@@ -124,8 +131,8 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
124 ORDER BY {primary_key_columns}
131 ORDER BY {primary_key_columns}
125 LIMIT {batch_size}
132 LIMIT {batch_size}
126 ), update AS (
133 ), update AS (
127− UPDATE public."{table}"
134+ UPDATE public."{table}" "{table}"
128− SET "{column}" = "{column}"
135+ SET "{touched_column}" = "{table}"."{touched_column}"
129 FROM rows
136 FROM rows
130 WHERE {primary_key_where}
137 WHERE {primary_key_where}
131 RETURNING {returning_columns}
138 RETURNING {returning_columns}
@@ -134,13 +141,7 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
134 FROM update
141 FROM update
135 LIMIT 1
142 LIMIT 1
136 "#,
143 "#,
137− table = table,
138− primary_key_columns = primary_key_columns,
139− cursor_where = cursor_where,
140 batch_size = BATCH_SIZE,
144 batch_size = BATCH_SIZE,
141− column = column,
142− primary_key_where = primary_key_where,
143− returning_columns = returning_columns,
144 );
145 );
145 let last_value = db
146 let last_value = db
146 .query_with_params(&query, &params)?
147 .query_with_params(&query, &params)?
@@ -154,8 +155,6 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
154 cursor = last_value
155 cursor = last_value
155 }
156 }
156
157
157− db.query("SET reshape.is_old_schema = ''")?;
158−
159 Ok(())
158 Ok(())
160}
159}
161
160
@@ -182,18 +181,30 @@ fn get_primary_key_columns_for_table(
182 Ok(primary_key_columns)
181 Ok(primary_key_columns)
183}
182}
184
183
184+pub struct Index {
185+ pub name: String,
186+ pub oid: u32,
187+ pub unique: bool,
188+ pub index_type: String,
189+}
190+
185pub fn get_indices_for_column(
191pub fn get_indices_for_column(
186 db: &mut dyn Conn,
192 db: &mut dyn Conn,
187 table: &str,
193 table: &str,
188 column: &str,
194 column: &str,
189−) -> anyhow::Result<Vec<(String, u32)>> {
195+) -> anyhow::Result<Vec<Index>> {
190 let indices = db
196 let indices = db
191 .query(&format!(
197 .query(&format!(
192 "
198 "
193− SELECT i.relname AS index_name, i.oid AS index_oid
199+ SELECT
200+ i.relname AS name,
201+ i.oid AS oid,
202+ ix.indisunique AS unique,
203+ am.amname AS type
194 FROM pg_index ix
204 FROM pg_index ix
195 JOIN pg_class t ON t.oid = ix.indrelid
205 JOIN pg_class t ON t.oid = ix.indrelid
196 JOIN pg_class i ON i.oid = ix.indexrelid
206 JOIN pg_class i ON i.oid = ix.indexrelid
207+ JOIN pg_am am ON i.relam = am.oid
197 JOIN pg_attribute a ON
208 JOIN pg_attribute a ON
198 a.attrelid = t.oid AND
209 a.attrelid = t.oid AND
199 a.attnum = ANY(ix.indkey)
210 a.attnum = ANY(ix.indkey)
@@ -205,7 +216,12 @@ pub fn get_indices_for_column(
205 column = column,
216 column = column,
206 ))?
217 ))?
207 .iter()
218 .iter()
208− .map(|row| (row.get("index_name"), row.get("index_oid")))
219+ .map(|row| Index {
220+ name: row.get("name"),
221+ oid: row.get("oid"),
222+ unique: row.get("unique"),
223+ index_type: row.get("type"),
224+ })
209 .collect();
225 .collect();
210
226
211 Ok(indices)
227 Ok(indices)
src/migrations/create_enum.rsnew file+83
@@ -0,0 +1,83 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct CreateEnum {
11+ pub name: String,
12+ pub values: Vec<String>,
13+}
14+
15+#[typetag::serde(name = "create_enum")]
16+impl Action for CreateEnum {
17+ fn describe(&self) -> String {
18+ format!("Creating enum \"{}\"", self.name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ // Check if enum already exists. CREATE TYPE doesn't have
28+ // a IF NOT EXISTS option so we have to do it manually.
29+ let enum_exists = !db
30+ .query(&format!(
31+ "
32+ SELECT typname
33+ FROM pg_catalog.pg_type
34+ WHERE typcategory = 'E'
35+ AND typname = '{name}'
36+ ",
37+ name = self.name,
38+ ))?
39+ .is_empty();
40+ if enum_exists {
41+ return Ok(());
42+ }
43+
44+ let values_def: Vec<String> = self
45+ .values
46+ .iter()
47+ .map(|value| format!("'{}'", value))
48+ .collect();
49+
50+ db.run(&format!(
51+ r#"
52+ CREATE TYPE "{name}" AS ENUM ({values})
53+ "#,
54+ name = self.name,
55+ values = values_def.join(", "),
56+ ))
57+ .context("failed to create enum")?;
58+
59+ Ok(())
60+ }
61+
62+ fn complete<'a>(
63+ &self,
64+ _ctx: &MigrationContext,
65+ _db: &'a mut dyn Conn,
66+ ) -> anyhow::Result<Option<Transaction<'a>>> {
67+ Ok(None)
68+ }
69+
70+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
71+
72+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
73+ db.run(&format!(
74+ r#"
75+ DROP TYPE IF EXISTS {name}
76+ "#,
77+ name = self.name,
78+ ))
79+ .context("failed to drop enum")?;
80+
81+ Ok(())
82+ }
83+}
src/migrations/create_table.rs+128−24
@@ -1,29 +1,37 @@
1+use std::collections::HashMap;
2+
1−use super::{Action, Column, MigrationContext};
3+use super::{common::ForeignKey, Action, Column, MigrationContext};
2use crate::{
4use crate::{
3 db::{Conn, Transaction},
5 db::{Conn, Transaction},
6+ migrations::common,
4 schema::Schema,
7 schema::Schema,
5};
8};
6use anyhow::Context;
9use anyhow::Context;
7−use derive_builder::Builder;
8use serde::{Deserialize, Serialize};
10use serde::{Deserialize, Serialize};
9
11
10−#[derive(Serialize, Deserialize, Builder, Debug)]
12+#[derive(Serialize, Deserialize, Debug)]
11−#[builder(setter(into))]
12pub struct CreateTable {
13pub struct CreateTable {
13 pub name: String,
14 pub name: String,
14 pub columns: Vec<Column>,
15 pub columns: Vec<Column>,
15 pub primary_key: Vec<String>,
16 pub primary_key: Vec<String>,
16
17
17 #[serde(default)]
18 #[serde(default)]
18− #[builder(default)]
19 pub foreign_keys: Vec<ForeignKey>,
19 pub foreign_keys: Vec<ForeignKey>,
20+
21+ pub up: Option<Transformation>,
20}
22}
21
23
22−#[derive(Serialize, Deserialize, Clone, Debug)]
24+#[derive(Serialize, Deserialize, Debug)]
23−pub struct ForeignKey {
25+pub struct Transformation {
24− pub columns: Vec<String>,
26+ table: String,
25− pub referenced_table: String,
27+ values: HashMap<String, String>,
26− pub referenced_columns: Vec<String>,
28+ upsert_constraint: Option<String>,
29+}
30+
31+impl CreateTable {
32+ fn trigger_name(&self, ctx: &MigrationContext) -> String {
33+ format!("{}_create_table_{}", ctx.prefix(), self.name)
34+ }
27}
35}
28
36
29#[typetag::serde(name = "create_table")]
37#[typetag::serde(name = "create_table")]
@@ -34,9 +42,9 @@ impl Action for CreateTable {
34
42
35 fn run(
43 fn run(
36 &self,
44 &self,
37− _ctx: &MigrationContext,
45+ ctx: &MigrationContext,
38 db: &mut dyn Conn,
46 db: &mut dyn Conn,
39− _schema: &Schema,
47+ schema: &Schema,
40 ) -> anyhow::Result<()> {
48 ) -> anyhow::Result<()> {
41 let mut definition_rows: Vec<String> = self
49 let mut definition_rows: Vec<String> = self
42 .columns
50 .columns
@@ -78,9 +86,10 @@ impl Action for CreateTable {
78 .iter()
86 .iter()
79 .map(|col| format!("\"{}\"", col))
87 .map(|col| format!("\"{}\"", col))
80 .collect();
88 .collect();
89+
90+ let referenced_table = schema.get_table(db, &foreign_key.referenced_table)?;
81− let referenced_columns: Vec<String> = foreign_key
91+ let referenced_columns: Vec<String> = referenced_table
82− .referenced_columns
92+ .real_column_names(&foreign_key.referenced_columns)
83− .iter()
84 .map(|col| format!("\"{}\"", col))
93 .map(|col| format!("\"{}\"", col))
85 .collect();
94 .collect();
86
95
@@ -89,12 +98,12 @@ impl Action for CreateTable {
89 FOREIGN KEY ({columns}) REFERENCES "{table}" ({referenced_columns})
98 FOREIGN KEY ({columns}) REFERENCES "{table}" ({referenced_columns})
90 "#,
99 "#,
91 columns = columns.join(", "),
100 columns = columns.join(", "),
92− table = foreign_key.referenced_table,
101+ table = referenced_table.real_name,
93 referenced_columns = referenced_columns.join(", "),
102 referenced_columns = referenced_columns.join(", "),
94 ));
103 ));
95 }
104 }
96
105
97− db.run(&format!(
106+ let query = &format!(
98 r#"
107 r#"
99 CREATE TABLE "{name}" (
108 CREATE TABLE "{name}" (
100 {definition}
109 {definition}
@@ -102,26 +111,121 @@ impl Action for CreateTable {
102 "#,
111 "#,
103 name = self.name,
112 name = self.name,
104 definition = definition_rows.join(",\n"),
113 definition = definition_rows.join(",\n"),
105− ))
114+ );
115+ db.run(query).context("failed to create table")?;
116+
117+ if let Some(Transformation {
118+ table: from_table,
119+ values,
120+ upsert_constraint,
121+ }) = &self.up
122+ {
123+ let from_table = schema.get_table(db, &from_table)?;
124+
125+ let declarations: Vec<String> = from_table
126+ .columns
127+ .iter()
128+ .map(|column| {
129+ format!(
130+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
131+ table = from_table.real_name,
132+ alias = column.name,
133+ real_name = column.real_name,
134+ )
135+ })
136+ .collect();
137+
138+ let (insert_columns, insert_values): (Vec<&str>, Vec<&str>) = values
139+ .iter()
140+ .map(|(k, v)| -> (&str, &str) { (k, v) }) // Force &String to &str
141+ .unzip();
142+
143+ let update_set: Vec<String> = values
144+ .iter()
145+ .map(|(field, value)| format!("\"{field}\" = {value}"))
146+ .collect();
147+
148+ // Constraint to check for conflicts. Defaults to the primary key constraint.
149+ let conflict_constraint_name = match upsert_constraint {
150+ Some(custom_constraint) => custom_constraint.clone(),
151+ _ => format!("{table}_pkey", table = self.name),
152+ };
153+
154+ // Add triggers to fill in values as they are inserted/updated
155+ let query = format!(
156+ r#"
157+ CREATE OR REPLACE FUNCTION {trigger_name}()
158+ RETURNS TRIGGER AS $$
159+ #variable_conflict use_variable
160+ BEGIN
161+ IF NOT reshape.is_new_schema() THEN
162+ DECLARE
163+ {declarations}
164+ BEGIN
165+ INSERT INTO public."{changed_table_real}" ({columns})
166+ VALUES ({values})
167+ ON CONFLICT ON CONSTRAINT "{conflict_constraint_name}"
168+ DO UPDATE SET
169+ {updates};
170+ END;
171+ END IF;
172+ RETURN NEW;
173+ END
174+ $$ language 'plpgsql';
175+
176+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
177+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
178+ "#,
179+ changed_table_real = self.name,
180+ from_table_real = from_table.real_name,
181+ trigger_name = self.trigger_name(ctx),
182+ declarations = declarations.join("\n"),
183+ columns = insert_columns.join(", "),
184+ values = insert_values.join(", "),
185+ updates = update_set.join(",\n"),
186+ );
106− .context("failed to create table")?;
187+ db.run(&query).context("failed to create up trigger")?;
188+
189+ // Backfill values in batches by touching the from table
190+ common::batch_touch_rows(db, &from_table.real_name, None)
191+ .context("failed to batch update existing rows")?;
192+ }
193+
107 Ok(())
194 Ok(())
108 }
195 }
109
196
110 fn complete<'a>(
197 fn complete<'a>(
111 &self,
198 &self,
112− _ctx: &MigrationContext,
199+ ctx: &MigrationContext,
113− _db: &'a mut dyn Conn,
200+ db: &'a mut dyn Conn,
114 ) -> anyhow::Result<Option<Transaction<'a>>> {
201 ) -> anyhow::Result<Option<Transaction<'a>>> {
115− // Do nothing
202+ // Remove triggers and procedures
203+ let query = format!(
204+ r#"
205+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
206+ "#,
207+ trigger_name = self.trigger_name(ctx),
208+ );
209+ db.run(&query).context("failed to drop up trigger")?;
210+
116 Ok(None)
211 Ok(None)
117 }
212 }
118
213
119 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
214 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
120
215
121− fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
216+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
217+ // Remove triggers and procedures
218+ let query = format!(
219+ r#"
220+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
221+ "#,
222+ trigger_name = self.trigger_name(ctx),
223+ );
224+ db.run(&query).context("failed to drop up trigger")?;
225+
122 db.run(&format!(
226 db.run(&format!(
123 r#"
227 r#"
124− DROP TABLE IF EXISTS {name}
228+ DROP TABLE IF EXISTS "{name}"
125 "#,
229 "#,
126 name = self.name,
230 name = self.name,
127 ))
231 ))
src/migrations/custom.rsnew file+61
@@ -0,0 +1,61 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use serde::{Deserialize, Serialize};
7+
8+#[derive(Serialize, Deserialize, Debug)]
9+pub struct Custom {
10+ #[serde(default)]
11+ pub start: Option<String>,
12+
13+ #[serde(default)]
14+ pub complete: Option<String>,
15+
16+ #[serde(default)]
17+ pub abort: Option<String>,
18+}
19+
20+#[typetag::serde(name = "custom")]
21+impl Action for Custom {
22+ fn describe(&self) -> String {
23+ "Running custom migration".to_string()
24+ }
25+
26+ fn run(
27+ &self,
28+ _ctx: &MigrationContext,
29+ db: &mut dyn Conn,
30+ _schema: &Schema,
31+ ) -> anyhow::Result<()> {
32+ if let Some(start_query) = &self.start {
33+ println!("Running query: {}", start_query);
34+ db.run(start_query)?;
35+ }
36+
37+ Ok(())
38+ }
39+
40+ fn complete<'a>(
41+ &self,
42+ _ctx: &MigrationContext,
43+ db: &'a mut dyn Conn,
44+ ) -> anyhow::Result<Option<Transaction<'a>>> {
45+ if let Some(complete_query) = &self.complete {
46+ db.run(complete_query)?;
47+ }
48+
49+ Ok(None)
50+ }
51+
52+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
53+
54+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
55+ if let Some(abort_query) = &self.abort {
56+ db.run(abort_query)?;
57+ }
58+
59+ Ok(())
60+ }
61+}
src/migrations/mod.rs+24−3
@@ -7,10 +7,10 @@ use serde::{Deserialize, Serialize};
7
7
8// Re-export migration types
8// Re-export migration types
9mod common;
9mod common;
10−pub use common::{Column, ColumnBuilder};
10+pub use common::Column;
11
11
12mod create_table;
12mod create_table;
13−pub use create_table::{CreateTable, CreateTableBuilder, ForeignKey};
13+pub use create_table::CreateTable;
14
14
15mod alter_column;
15mod alter_column;
16pub use alter_column::{AlterColumn, ColumnChanges};
16pub use alter_column::{AlterColumn, ColumnChanges};
@@ -33,6 +33,21 @@ pub use remove_table::RemoveTable;
33mod rename_table;
33mod rename_table;
34pub use rename_table::RenameTable;
34pub use rename_table::RenameTable;
35
35
36+mod create_enum;
37+pub use create_enum::CreateEnum;
38+
39+mod remove_enum;
40+pub use remove_enum::RemoveEnum;
41+
42+mod custom;
43+pub use custom::Custom;
44+
45+mod add_foreign_key;
46+pub use add_foreign_key::AddForeignKey;
47+
48+mod remove_foreign_key;
49+pub use remove_foreign_key::RemoveForeignKey;
50+
36#[derive(Serialize, Deserialize, Debug)]
51#[derive(Serialize, Deserialize, Debug)]
37pub struct Migration {
52pub struct Migration {
38 pub name: String,
53 pub name: String,
@@ -73,13 +88,19 @@ impl Clone for Migration {
73pub struct MigrationContext {
88pub struct MigrationContext {
74 migration_index: usize,
89 migration_index: usize,
75 action_index: usize,
90 action_index: usize,
91+ existing_schema_name: Option<String>,
76}
92}
77
93
78impl MigrationContext {
94impl MigrationContext {
95+ pub fn new(
79− pub fn new(migration_index: usize, action_index: usize) -> Self {
96+ migration_index: usize,
97+ action_index: usize,
98+ existing_schema_name: Option<String>,
99+ ) -> Self {
80 MigrationContext {
100 MigrationContext {
81 migration_index,
101 migration_index,
82 action_index,
102 action_index,
103+ existing_schema_name,
83 }
104 }
84 }
105 }
85
106
src/migrations/remove_column.rs+353−35
@@ -1,16 +1,27 @@
1−use super::{Action, MigrationContext};
1+use super::{common, Action, MigrationContext};
2use crate::{
2use crate::{
3 db::{Conn, Transaction},
3 db::{Conn, Transaction},
4 schema::Schema,
4 schema::Schema,
5};
5};
6−use anyhow::Context;
6+use anyhow::{anyhow, bail, Context};
7use serde::{Deserialize, Serialize};
7use serde::{Deserialize, Serialize};
8
8
9#[derive(Serialize, Deserialize, Debug)]
9#[derive(Serialize, Deserialize, Debug)]
10pub struct RemoveColumn {
10pub struct RemoveColumn {
11 pub table: String,
11 pub table: String,
12 pub column: String,
12 pub column: String,
13− pub down: Option<String>,
13+ pub down: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
14}
25}
15
26
16impl RemoveColumn {
27impl RemoveColumn {
@@ -22,6 +33,33 @@ impl RemoveColumn {
22 self.column
33 self.column
23 )
34 )
24 }
35 }
36+
37+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
38+ format!(
39+ "{}_remove_column_{}_{}_rev",
40+ ctx.prefix(),
41+ self.table,
42+ self.column
43+ )
44+ }
45+
46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_remove_column_{}_{}_nn",
49+ ctx.prefix(),
50+ self.table,
51+ self.column
52+ )
53+ }
54+
55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
56+ format!(
57+ "{}_add_column_not_null_{}_{}",
58+ ctx.prefix(),
59+ self.table,
60+ self.column
61+ )
62+ }
25}
63}
26
64
27#[typetag::serde(name = "remove_column")]
65#[typetag::serde(name = "remove_column")]
@@ -39,10 +77,13 @@ impl Action for RemoveColumn {
39 db: &mut dyn Conn,
77 db: &mut dyn Conn,
40 schema: &Schema,
78 schema: &Schema,
41 ) -> anyhow::Result<()> {
79 ) -> anyhow::Result<()> {
80+ let table = schema.get_table(db, &self.table)?;
81+ let column = table
82+ .get_column(&self.column)
83+ .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
84+
42 // Add down trigger
85 // Add down trigger
43 if let Some(down) = &self.down {
86 if let Some(down) = &self.down {
44− let table = schema.get_table(db, &self.table)?;
45−
46 let declarations: Vec<String> = table
87 let declarations: Vec<String> = table
47 .columns
88 .columns
48 .iter()
89 .iter()
@@ -56,32 +97,226 @@ impl Action for RemoveColumn {
56 })
97 })
57 .collect();
98 .collect();
58
99
100+ if let Transformation::Simple(down) = down {
59− let query = format!(
101+ let query = format!(
60− r#"
102+ r#"
61− CREATE OR REPLACE FUNCTION {trigger_name}()
103+ CREATE OR REPLACE FUNCTION {trigger_name}()
62− RETURNS TRIGGER AS $$
104+ RETURNS TRIGGER AS $$
63− BEGIN
105+ BEGIN
64− IF NOT reshape.is_old_schema() IS NULL THEN
106+ IF reshape.is_new_schema() THEN
65− DECLARE
107+ DECLARE
108+ {declarations}
109+ BEGIN
110+ NEW.{column_name} = {down};
111+ END;
112+ END IF;
113+ RETURN NEW;
114+ END
115+ $$ language 'plpgsql';
116+
117+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
118+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
119+ "#,
120+ column_name = self.column,
121+ trigger_name = self.trigger_name(ctx),
122+ down = down,
123+ table = self.table,
66− {declarations}
124+ declarations = declarations.join("\n"),
125+ );
126+ db.run(&query).context("failed to create down trigger")?;
127+ }
128+
129+ if let Transformation::Update {
130+ table: from_table,
131+ value,
132+ r#where,
133+ } = down
134+ {
135+ let existing_schema_name = match &ctx.existing_schema_name {
136+ Some(name) => name,
137+ None => bail!("can't use update without previous migration"),
138+ };
139+
140+ let from_table = schema.get_table(db, &from_table)?;
141+
142+ let maybe_null_check = if !column.nullable {
143+ // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.
144+ // We will add a null check to the down function on the new schema below as well to cover both cases.
145+ // As we are using a complex down function, we must remove the NOT NULL check for the new schema.
146+ // NOT NULL is not checked at the end of a transaction, but immediately upon update.
147+ let query = format!(
148+ r#"
149+ CREATE OR REPLACE FUNCTION {trigger_name}()
150+ RETURNS TRIGGER AS $$
67 BEGIN
151 BEGIN
68− NEW.{column_name} = {down};
152+ IF NOT reshape.is_new_schema() THEN
153+ IF NEW.{column} IS NULL THEN
154+ RAISE EXCEPTION '{column} can not be null';
69− END;
155+ END IF;
70− END IF;
156+ END IF;
71− RETURN NEW;
157+ RETURN NEW;
72− END
158+ END
73− $$ language 'plpgsql';
159+ $$ language 'plpgsql';
74−
160+
75− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
161+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
162+
163+ CREATE CONSTRAINT TRIGGER "{trigger_name}"
164+ AFTER INSERT OR UPDATE
165+ ON "{table}"
166+ FOR EACH ROW
76− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
167+ EXECUTE PROCEDURE {trigger_name}();
77− "#,
168+ "#,
78− column_name = self.column,
169+ table = self.table,
79− trigger_name = self.trigger_name(ctx),
170+ trigger_name = self.not_null_constraint_trigger_name(ctx),
80− down = down,
171+ column = self.column,
172+ );
173+ db.run(&query)
174+ .context("failed to create null constraint trigger")?;
175+
176+ db.run(&format!(
177+ r#"
178+ ALTER TABLE {table}
179+ ALTER COLUMN {column}
180+ DROP NOT NULL
181+ "#,
81− table = self.table,
182+ table = self.table,
183+ column = self.column
184+ ))
185+ .context("failed to remove column not null constraint")?;
186+
187+ format!(
188+ r#"
189+ IF {value} IS NULL THEN
190+ RAISE EXCEPTION '{column_name} can not be null';
191+ END IF;
192+ "#,
193+ column_name = self.column,
194+ )
195+ } else {
196+ "".to_string()
197+ };
198+
199+ let into_variables = from_table
200+ .columns
201+ .iter()
202+ .map(|column| {
203+ format!(
204+ "NEW.{real_name} AS {alias}",
205+ alias = column.name,
206+ real_name = column.real_name,
207+ )
208+ })
209+ .collect::<Vec<String>>()
210+ .join(", ");
211+
212+ let query = format!(
213+ r#"
214+ CREATE OR REPLACE FUNCTION {trigger_name}()
215+ RETURNS TRIGGER AS $$
216+ #variable_conflict use_variable
217+ BEGIN
218+ IF reshape.is_new_schema() THEN
219+ DECLARE
220+ {from_table} record;
221+ BEGIN
222+ SELECT {into_variables}
223+ INTO {from_table};
224+
225+ {maybe_null_check}
226+
227+ -- Don't trigger reverse trigger when making this update
228+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
229+
230+ UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"
231+ SET "{column_name}" = {value}
232+ WHERE {where};
233+
234+ perform set_config('reshape.disable_triggers', '', TRUE);
235+ END;
236+ END IF;
237+ RETURN NEW;
238+ END
239+ $$ language 'plpgsql';
240+
241+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
242+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
243+ "#,
244+ changed_table = self.table,
245+ from_table = from_table.name,
246+ from_table_real = from_table.real_name,
247+ column_name = self.column,
248+ trigger_name = self.trigger_name(ctx),
249+ );
250+ db.run(&query).context("failed to create down trigger")?;
251+
252+ let changed_into_variables = table
253+ .columns
254+ .iter()
255+ .map(|column| {
256+ format!(
257+ "NEW.{real_name} AS {alias}",
258+ alias = column.name,
259+ real_name = column.real_name,
260+ )
261+ })
262+ .collect::<Vec<String>>()
263+ .join(", ");
264+
265+ let from_table_columns = from_table
266+ .columns
267+ .iter()
268+ .map(|column| format!("{} as {}", column.real_name, column.name))
269+ .collect::<Vec<String>>()
270+ .join(", ");
271+
272+ let query = format!(
273+ r#"
274+ CREATE OR REPLACE FUNCTION {trigger_name}()
275+ RETURNS TRIGGER AS $$
276+ #variable_conflict use_variable
277+ BEGIN
278+ IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
279+ DECLARE
280+ {changed_table} record;
281+ __temp_row record;
282+ BEGIN
283+ SELECT {changed_into_variables}
284+ INTO {changed_table};
285+
286+ SELECT *
287+ INTO __temp_row
288+ FROM (
289+ SELECT {from_table_columns}
290+ FROM public.{from_table_real}
291+ ) {from_table}
292+ WHERE {where};
293+
294+ DECLARE
295+ {from_table} record;
296+ BEGIN
297+ {from_table} := __temp_row;
298+ NEW.{column_name_real} = {value};
299+ END;
300+ END;
301+ END IF;
302+ RETURN NEW;
303+ END
304+ $$ language 'plpgsql';
305+
306+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
307+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
308+ "#,
309+ changed_table = table.name,
310+ changed_table_real = table.real_name,
311+ from_table = from_table.name,
312+ from_table_real = from_table.real_name,
313+ column_name_real = column.real_name,
314+ trigger_name = self.reverse_trigger_name(ctx),
82− declarations = declarations.join("\n"),
315+ // declarations = declarations.join("\n"),
83− );
316+ );
317+ db.run(&query)
84− db.run(&query).context("failed to create down trigger")?;
318+ .context("failed to create reverse down trigger")?;
319+ }
85 }
320 }
86
321
87 Ok(())
322 Ok(())
@@ -92,18 +327,34 @@ impl Action for RemoveColumn {
92 ctx: &MigrationContext,
327 ctx: &MigrationContext,
93 db: &'a mut dyn Conn,
328 db: &'a mut dyn Conn,
94 ) -> anyhow::Result<Option<Transaction<'a>>> {
329 ) -> anyhow::Result<Option<Transaction<'a>>> {
330+ let indices = common::get_indices_for_column(db, &self.table, &self.column)
331+ .context("failed getting column indices")?;
332+
333+ for index in indices {
334+ db.run(&format!(
335+ "
336+ DROP INDEX CONCURRENTLY IF EXISTS {name}
337+ ",
338+ name = index.name,
339+ ))
340+ .context("failed to drop index")?;
341+ }
342+
95 // Remove column, function and trigger
343 // Remove column, function and trigger
96 let query = format!(
344 let query = format!(
97 r#"
345 r#"
98 ALTER TABLE "{table}"
346 ALTER TABLE "{table}"
99 DROP COLUMN IF EXISTS "{column}";
347 DROP COLUMN IF EXISTS "{column}";
100
348
101− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
349+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
350+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
102− DROP FUNCTION IF EXISTS "{trigger_name}";
351+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
103 "#,
352 "#,
104 table = self.table,
353 table = self.table,
105 column = self.column,
354 column = self.column,
106 trigger_name = self.trigger_name(ctx),
355 trigger_name = self.trigger_name(ctx),
356+ reverse_trigger_name = self.reverse_trigger_name(ctx),
357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
107 );
358 );
108 db.run(&query)
359 db.run(&query)
109 .context("failed to drop column and down trigger")?;
360 .context("failed to drop column and down trigger")?;
@@ -120,14 +371,81 @@ impl Action for RemoveColumn {
120 }
371 }
121
372
122 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
374+ // We might have temporaily removed the NOT NULL check and have to reinstate it
375+ let has_not_null_function = !db
376+ .query_with_params(
377+ "
378+ SELECT routine_name
379+ FROM information_schema.routines
380+ WHERE routine_schema = 'public'
381+ AND routine_name = $1
382+ ",
383+ &[&self.not_null_constraint_trigger_name(ctx)],
384+ )
385+ .context("failed to get any NOT NULL function")?
386+ .is_empty();
387+
388+ if has_not_null_function {
389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint
390+ let query = format!(
391+ r#"
392+ ALTER TABLE "{table}"
393+ ADD CONSTRAINT "{constraint_name}"
394+ CHECK ("{column}" IS NOT NULL) NOT VALID
395+ "#,
396+ table = self.table,
397+ constraint_name = self.not_null_constraint_name(ctx),
398+ column = self.column,
399+ );
400+ db.run(&query)
401+ .context("failed to add NOT NULL constraint")?;
402+
403+ let query = format!(
404+ r#"
405+ ALTER TABLE "{table}"
406+ VALIDATE CONSTRAINT "{constraint_name}"
407+ "#,
408+ table = self.table,
409+ constraint_name = self.not_null_constraint_name(ctx),
410+ );
411+ db.run(&query)
412+ .context("failed to validate NOT NULL constraint")?;
413+
414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above
415+ db.run(&format!(
416+ r#"
417+ ALTER TABLE {table}
418+ ALTER COLUMN {column}
419+ SET NOT NULL
420+ "#,
421+ table = self.table,
422+ column = self.column
423+ ))
424+ .context("failed to reinstate column NOT NULL")?;
425+
426+ // Drop the temporary constraint
427+ let query = format!(
428+ r#"
429+ ALTER TABLE "{table}"
430+ DROP CONSTRAINT "{constraint_name}"
431+ "#,
432+ table = self.table,
433+ constraint_name = self.not_null_constraint_name(ctx),
434+ );
435+ db.run(&query)
436+ .context("failed to drop NOT NULL constraint")?;
437+ }
438+
123 // Remove function and trigger
439 // Remove function and trigger
124 db.run(&format!(
440 db.run(&format!(
125 r#"
441 r#"
126− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
442+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
443+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
127− DROP FUNCTION IF EXISTS "{trigger_name}";
444+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
128 "#,
445 "#,
129− table = self.table,
130 trigger_name = self.trigger_name(ctx),
446 trigger_name = self.trigger_name(ctx),
447+ reverse_trigger_name = self.reverse_trigger_name(ctx),
448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
131 ))
449 ))
132 .context("failed to drop down trigger")?;
450 .context("failed to drop down trigger")?;
133
451
src/migrations/remove_enum.rsnew file+51
@@ -0,0 +1,51 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveEnum {
11+ #[serde(rename = "enum")]
12+ pub enum_name: String,
13+}
14+
15+#[typetag::serde(name = "remove_enum")]
16+impl Action for RemoveEnum {
17+ fn describe(&self) -> String {
18+ format!("Removing enum \"{}\"", self.enum_name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ _db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ Ok(())
28+ }
29+
30+ fn complete<'a>(
31+ &self,
32+ _ctx: &MigrationContext,
33+ db: &'a mut dyn Conn,
34+ ) -> anyhow::Result<Option<Transaction<'a>>> {
35+ db.run(&format!(
36+ r#"
37+ DROP TYPE IF EXISTS {name}
38+ "#,
39+ name = self.enum_name,
40+ ))
41+ .context("failed to drop enum")?;
42+
43+ Ok(None)
44+ }
45+
46+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
47+
48+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
49+ Ok(())
50+ }
51+}
src/migrations/remove_foreign_key.rsnew file+91
@@ -0,0 +1,91 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::{anyhow, Context};
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveForeignKey {
11+ table: String,
12+ foreign_key: String,
13+}
14+
15+#[typetag::serde(name = "remove_foreign_key")]
16+impl Action for RemoveForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Removing foreign key \"{}\" from table \"{}\"",
20+ self.foreign_key, self.table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ _ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ // The foreign key is only removed once the migration is completed.
31+ // Removing it earlier would be hard/undesirable for several reasons:
32+ // - Postgres doesn't have an easy way to temporarily disable a foreign key check.
33+ // If it did, we could disable the FK for the new schema.
34+ // - Even if we could, it probably wouldn't be a good idea as it would cause temporary
35+ // inconsistencies for the old schema which still expects the FK to hold.
36+ // - For the same reason, we can't remove the FK when the migration is first applied.
37+ // If the migration was to be aborted, then the FK would have to be recreated with
38+ // the risk that it would no longer be valid.
39+
40+ // Ensure foreign key exists
41+ let table = schema.get_table(db, &self.table)?;
42+ let fk_exists = !db
43+ .query(&format!(
44+ r#"
45+ SELECT constraint_name
46+ FROM information_schema.table_constraints
47+ WHERE
48+ constraint_type = 'FOREIGN KEY' AND
49+ table_name = '{table_name}' AND
50+ constraint_name = '{foreign_key}'
51+ "#,
52+ table_name = table.real_name,
53+ foreign_key = self.foreign_key,
54+ ))
55+ .context("failed to check for foreign key")?
56+ .is_empty();
57+
58+ if !fk_exists {
59+ return Err(anyhow!(
60+ "no foreign key \"{}\" exists on table \"{}\"",
61+ self.foreign_key,
62+ self.table
63+ ));
64+ }
65+
66+ Ok(())
67+ }
68+
69+ fn complete<'a>(
70+ &self,
71+ _ctx: &MigrationContext,
72+ db: &'a mut dyn Conn,
73+ ) -> anyhow::Result<Option<Transaction<'a>>> {
74+ db.run(&format!(
75+ r#"
76+ ALTER TABLE {table}
77+ DROP CONSTRAINT IF EXISTS {foreign_key}
78+ "#,
79+ table = self.table,
80+ foreign_key = self.foreign_key,
81+ ))
82+ .context("failed to remove foreign key")?;
83+ Ok(None)
84+ }
85+
86+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
87+
88+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
89+ Ok(())
90+ }
91+}
src/schema.rs+18−1
@@ -209,7 +209,7 @@ impl Schema {
209 let real_columns: Vec<(String, String, bool, Option<String>)> = db
209 let real_columns: Vec<(String, String, bool, Option<String>)> = db
210 .query(&format!(
210 .query(&format!(
211 "
211 "
212− SELECT column_name, data_type, is_nullable, column_default
212+ SELECT column_name, CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END, is_nullable, column_default
213 FROM information_schema.columns
213 FROM information_schema.columns
214 WHERE table_name = '{table}' AND table_schema = 'public'
214 WHERE table_name = '{table}' AND table_schema = 'public'
215 ORDER BY ordinal_position
215 ORDER BY ordinal_position
@@ -286,3 +286,20 @@ impl Schema {
286 Ok(table)
286 Ok(table)
287 }
287 }
288}
288}
289+
290+impl Table {
291+ pub fn real_column_names<'a>(
292+ &'a self,
293+ columns: &'a [String],
294+ ) -> impl Iterator<Item = &'a String> {
295+ columns.iter().map(|name| {
296+ self.get_column(name)
297+ .map(|col| &col.real_name)
298+ .unwrap_or(name)
299+ })
300+ }
301+
302+ pub fn get_column(&self, name: &str) -> Option<&Column> {
303+ self.columns.iter().find(|column| column.name == name)
304+ }
305+}
tests/add_column.rs+225−153
@@ -1,10 +1,9 @@
1−use reshape::migrations::{AddColumn, Column, ColumnBuilder, CreateTableBuilder, Migration};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn add_column() {
5fn add_column() {
7− let mut test = common::Test::new("Add column");
6+ let mut test = Test::new("Add column");
8
7
9 test.first_migration(
8 test.first_migration(
10 r#"
9 r#"
@@ -114,176 +113,249 @@ fn add_column() {
114
113
115#[test]
114#[test]
116fn add_column_nullable() {
115fn add_column_nullable() {
117− let (mut reshape, mut old_db, mut new_db) = common::setup();
116+ let mut test = Test::new("Add nullable column");
118−
117+
118+ test.first_migration(
119+ r#"
119− let create_users_table = Migration::new("create_user_table", None).with_action(
120+ name = "create_users_table"
120− CreateTableBuilder::default()
121+
122+ [[actions]]
123+ type = "create_table"
121− .name("users")
124+ name = "users"
122− .primary_key(vec!["id".to_string()])
125+ primary_key = ["id"]
123− .columns(vec![ColumnBuilder::default()
126+
127+ [[actions.columns]]
124− .name("id")
128+ name = "id"
125− .data_type("INTEGER")
129+ type = "INTEGER"
126− .build()
130+ "#,
127− .unwrap()])
128− .build()
129− .unwrap(),
130 );
131 );
131− let add_name_column = Migration::new("add_nullable_name_column", None).with_action(AddColumn {
132− table: "users".to_string(),
133− column: Column {
134− name: "name".to_string(),
135− data_type: "TEXT".to_string(),
136− nullable: true,
137− default: None,
138− generated: None,
139− },
140− up: None,
141− });
142
132
143− let first_migrations = vec![create_users_table.clone()];
133+ test.second_migration(
144− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
134+ r#"
135+ name = "add_nullable_name_column"
145
136
146− // Run first migration, should automatically finish
137+ [[actions]]
147− reshape.migrate(first_migrations.clone()).unwrap();
138+ type = "add_column"
139+ table = "users"
148
140
149− // Update search paths
141+ [actions.column]
150− old_db
142+ name = "name"
151− .simple_query(&reshape::schema_query_for_migration(
143+ type = "TEXT"
152− &first_migrations.last().unwrap().name,
144+ "#,
153− ))
145+ );
154− .unwrap();
155− new_db
156− .simple_query(&reshape::schema_query_for_migration(
157− &first_migrations.last().unwrap().name,
158− ))
159− .unwrap();
160
146
147+ test.after_first(|db| {
161− // Insert some test values
148+ // Insert some test values
162− new_db
163− .simple_query(
149+ db.simple_query(
164 "
150 "
165− INSERT INTO users (id) VALUES
151+ INSERT INTO users (id) VALUES (1), (2);
166− (1),
167− (2);
168 ",
152 ",
169 )
153 )
170 .unwrap();
154 .unwrap();
155+ });
171
156
172− // Run second migration
157+ test.intermediate(|old_db, new_db| {
173− reshape.migrate(second_migrations.clone()).unwrap();
158+ // Ensure existing data got updated
174− new_db
159+ let expected: Vec<Option<String>> = vec![None, None];
175− .simple_query(&reshape::schema_query_for_migration(
160+ assert!(new_db
176− &second_migrations.last().unwrap().name,
161+ .query("SELECT name FROM users ORDER BY id", &[],)
177− ))
162+ .unwrap()
178− .unwrap();
163+ .iter()
164+ .map(|row| row.get::<_, Option<String>>("name"))
165+ .eq(expected));
179
166
180− // Ensure existing data got updated
181− let expected: Vec<Option<String>> = vec![None, None];
182− assert!(new_db
183− .query("SELECT name FROM users ORDER BY id", &[],)
184− .unwrap()
185− .iter()
186− .map(|row| row.get::<_, Option<String>>("name"))
187− .eq(expected));
188−
189− // Insert data using old schema and ensure new column is NULL
167+ // Insert data using old schema and ensure new column is NULL
190− old_db
168+ old_db
191− .simple_query("INSERT INTO users (id) VALUES (3)")
169+ .simple_query("INSERT INTO users (id) VALUES (3)")
192− .unwrap();
170+ .unwrap();
193− let name: Option<String> = new_db
171+ let name: Option<String> = new_db
194− .query_one("SELECT name from users WHERE id = 3", &[])
172+ .query_one("SELECT name from users WHERE id = 3", &[])
195− .map(|row| (row.get("name")))
173+ .map(|row| (row.get("name")))
196− .unwrap();
174+ .unwrap();
197− assert_eq!(None, name);
175+ assert_eq!(None, name);
198
176
199− // Ensure data can be inserted against new schema
177+ // Ensure data can be inserted against new schema
200− new_db
178+ new_db
201− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
179+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
202− .unwrap();
180+ .unwrap();
181+ });
203
182
204− reshape.complete().unwrap();
183+ test.after_completion(|db| {
205− common::assert_cleaned_up(&mut new_db);
184+ let expected: Vec<Option<String>> =
185+ vec![None, None, None, Some("Test Testsson".to_string()), None];
186+ let result: Vec<Option<String>> = db
187+ .query("SELECT id, name FROM users ORDER BY id", &[])
188+ .unwrap()
189+ .iter()
190+ .map(|row| row.get("name"))
191+ .collect();
192+
193+ assert_eq!(result, expected);
194+ });
195+
196+ test.run();
206}
197}
207
198
208#[test]
199#[test]
209fn add_column_with_default() {
200fn add_column_with_default() {
210− let (mut reshape, mut old_db, mut new_db) = common::setup();
201+ let mut test = Test::new("Add column with default value");
211−
202+
203+ test.first_migration(
204+ r#"
212− let create_users_table = Migration::new("create_user_table", None).with_action(
205+ name = "create_users_table"
213− CreateTableBuilder::default()
206+
207+ [[actions]]
208+ type = "create_table"
214− .name("users")
209+ name = "users"
215− .primary_key(vec!["id".to_string()])
210+ primary_key = ["id"]
216− .columns(vec![ColumnBuilder::default()
211+
212+ [[actions.columns]]
217− .name("id")
213+ name = "id"
218− .data_type("INTEGER")
214+ type = "INTEGER"
219− .build()
215+ "#,
220− .unwrap()])
221− .build()
222− .unwrap(),
223 );
216 );
224− let add_name_column =
225− Migration::new("add_name_column_with_default", None).with_action(AddColumn {
226− table: "users".to_string(),
227− column: Column {
228− name: "name".to_string(),
229− data_type: "TEXT".to_string(),
230− nullable: false,
231− default: Some("'DEFAULT'".to_string()),
232− generated: None,
233− },
234− up: None,
235− });
236−
237− let first_migrations = vec![create_users_table.clone()];
238− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
239−
240− // Run first migration, should automatically finish
241− reshape.migrate(first_migrations.clone()).unwrap();
242−
243− // Update search paths
244− old_db
245− .simple_query(&reshape::schema_query_for_migration(
246− &first_migrations.last().unwrap().name,
247− ))
248− .unwrap();
249− new_db
250− .simple_query(&reshape::schema_query_for_migration(
251− &first_migrations.last().unwrap().name,
252− ))
253− .unwrap();
254
217
255− // Insert some test values
218+ test.second_migration(
256− new_db
219+ r#"
257− .simple_query("INSERT INTO users (id) VALUES (1), (2)")
220+ name = "add_name_column_with_default"
258− .unwrap();
259
221
260− // Run second migration
222+ [[actions]]
261− reshape.migrate(second_migrations.clone()).unwrap();
223+ type = "add_column"
262− new_db
224+ table = "users"
263− .simple_query(&reshape::schema_query_for_migration(
264− &second_migrations.last().unwrap().name,
265− ))
266− .unwrap();
267
225
226+ [actions.column]
227+ name = "name"
228+ type = "TEXT"
229+ nullable = false
230+ default = "'DEFAULT'"
231+ "#,
232+ );
233+
234+ test.after_first(|db| {
235+ // Insert some test values
236+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
237+ .unwrap();
238+ });
239+
240+ test.intermediate(|old_db, new_db| {
268− // Ensure existing data got updated with defaults
241+ // Ensure existing data got updated with defaults
269− let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
242+ let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
270− assert!(new_db
243+ assert!(new_db
271− .query("SELECT name FROM users ORDER BY id", &[],)
244+ .query("SELECT name FROM users ORDER BY id", &[],)
272− .unwrap()
245+ .unwrap()
273− .iter()
246+ .iter()
274− .map(|row| row.get::<_, String>("name"))
247+ .map(|row| row.get::<_, String>("name"))
275− .eq(expected));
248+ .eq(expected));
276−
249+
277− // Insert data using old schema and ensure new column gets the default value
250+ // Insert data using old schema and ensure new column gets the default value
278− old_db
251+ old_db
279− .simple_query("INSERT INTO users (id) VALUES (3)")
252+ .simple_query("INSERT INTO users (id) VALUES (3)")
280− .unwrap();
253+ .unwrap();
281− let name: String = new_db
254+ let name: String = new_db
282− .query_one("SELECT name from users WHERE id = 3", &[])
255+ .query_one("SELECT name from users WHERE id = 3", &[])
283− .map(|row| row.get("name"))
256+ .map(|row| row.get("name"))
284− .unwrap();
257+ .unwrap();
285− assert_eq!("DEFAULT", name);
258+ assert_eq!("DEFAULT", name);
259+ });
260+
261+ test.run();
262+}
263+
264+#[test]
265+fn add_column_with_complex_up() {
266+ let mut test = Test::new("Add column complex");
267+
268+ test.first_migration(
269+ r#"
270+ name = "create_tables"
271+
272+ [[actions]]
273+ type = "create_table"
274+ name = "users"
275+ primary_key = ["id"]
276+
277+ [[actions.columns]]
278+ name = "id"
279+ type = "INTEGER"
280+
281+ [[actions.columns]]
282+ name = "email"
283+ type = "TEXT"
284+
285+ [[actions]]
286+ type = "create_table"
287+ name = "profiles"
288+ primary_key = ["user_id"]
289+
290+ [[actions.columns]]
291+ name = "user_id"
292+ type = "INTEGER"
293+ "#,
294+ );
295+
296+ test.second_migration(
297+ r#"
298+ name = "add_profiles_email_column"
299+
300+ [[actions]]
301+ type = "add_column"
302+ table = "profiles"
303+
304+ [actions.column]
305+ name = "email"
306+ type = "TEXT"
307+ nullable = false
308+
309+ [actions.up]
310+ table = "users"
311+ value = "users.email"
312+ where = "profiles.user_id = users.id"
313+ "#,
314+ );
315+
316+ test.after_first(|db| {
317+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
318+ .unwrap();
319+ db.simple_query("INSERT INTO profiles (user_id) VALUES (1)")
320+ .unwrap();
321+ });
322+
323+ test.intermediate(|old_db, new_db| {
324+ // Ensure email was backfilled on profiles
325+ let email: String = new_db
326+ .query(
327+ "
328+ SELECT email
329+ FROM profiles
330+ WHERE user_id = 1
331+ ",
332+ &[],
333+ )
334+ .unwrap()
335+ .first()
336+ .map(|row| row.get("email"))
337+ .unwrap();
338+ assert_eq!("test@example.com", email);
339+
340+ // Ensure email change in old schema is propagated to profiles table in new schema
341+ old_db
342+ .simple_query("UPDATE users SET email = 'test2@example.com' WHERE id = 1")
343+ .unwrap();
344+ let email: String = new_db
345+ .query(
346+ "
347+ SELECT email
348+ FROM profiles
349+ WHERE user_id = 1
350+ ",
351+ &[],
352+ )
353+ .unwrap()
354+ .first()
355+ .map(|row| row.get("email"))
356+ .unwrap();
357+ assert_eq!("test2@example.com", email);
358+ });
286
359
287− reshape.complete().unwrap();
360+ test.run();
288− common::assert_cleaned_up(&mut new_db);
289}
361}
tests/add_foreign_key.rsnew file+166
@@ -0,0 +1,166 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn add_foreign_key() {
6+ let mut test = Test::new("Add foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+ "#,
34+ );
35+
36+ test.second_migration(
37+ r#"
38+ name = "add_foreign_key"
39+
40+ [[actions]]
41+ type = "add_foreign_key"
42+ table = "items"
43+
44+ [actions.foreign_key]
45+ columns = ["user_id"]
46+ referenced_table = "users"
47+ referenced_columns = ["id"]
48+ "#,
49+ );
50+
51+ test.after_first(|db| {
52+ // Insert some test users
53+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
54+ .unwrap();
55+ });
56+
57+ test.intermediate(|db, _| {
58+ // Ensure items can be inserted if they reference valid users
59+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
60+ .unwrap();
61+
62+ // Ensure items can't be inserted if they don't reference valid users
63+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
64+ assert!(result.is_err(), "expected insert to fail");
65+ });
66+
67+ test.after_completion(|db| {
68+ // Ensure items can be inserted if they reference valid users
69+ db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 1), (4, 2)")
70+ .unwrap();
71+
72+ // Ensure items can't be inserted if they don't reference valid users
73+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)");
74+ assert!(result.is_err(), "expected insert to fail");
75+
76+ // Ensure foreign key exists with the right name
77+ let foreign_key_name: Option<String> = db
78+ .query(
79+ "
80+ SELECT tc.constraint_name
81+ FROM information_schema.table_constraints AS tc
82+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
83+ ",
84+ &[],
85+ )
86+ .unwrap()
87+ .first()
88+ .map(|row| row.get(0));
89+ assert_eq!(Some("items_user_id_fkey".to_string()), foreign_key_name);
90+ });
91+
92+ test.after_abort(|db| {
93+ // Ensure foreign key doesn't exist
94+ let fk_does_not_exist = db
95+ .query(
96+ "
97+ SELECT tc.constraint_name
98+ FROM information_schema.table_constraints AS tc
99+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
100+ ",
101+ &[],
102+ )
103+ .unwrap()
104+ .is_empty();
105+ assert!(fk_does_not_exist);
106+ });
107+
108+ test.run()
109+}
110+
111+#[test]
112+fn add_invalid_foreign_key() {
113+ let mut test = Test::new("Add invalid foreign key");
114+
115+ test.first_migration(
116+ r#"
117+ name = "create_user_table"
118+
119+ [[actions]]
120+ type = "create_table"
121+ name = "users"
122+ primary_key = ["id"]
123+
124+ [[actions.columns]]
125+ name = "id"
126+ type = "INTEGER"
127+
128+ [[actions]]
129+ type = "create_table"
130+ name = "items"
131+ primary_key = ["id"]
132+
133+ [[actions.columns]]
134+ name = "id"
135+ type = "INTEGER"
136+
137+ [[actions.columns]]
138+ name = "user_id"
139+ type = "INTEGER"
140+ "#,
141+ );
142+
143+ test.second_migration(
144+ r#"
145+ name = "add_foreign_key"
146+
147+ [[actions]]
148+ type = "add_foreign_key"
149+ table = "items"
150+
151+ [actions.foreign_key]
152+ columns = ["user_id"]
153+ referenced_table = "users"
154+ referenced_columns = ["id"]
155+ "#,
156+ );
157+
158+ test.after_first(|db| {
159+ // Insert some items which don't reference a valid user
160+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
161+ .unwrap();
162+ });
163+
164+ test.expect_failure();
165+ test.run()
166+}
tests/add_index.rs+199−207
@@ -1,231 +1,223 @@
1−use reshape::migrations::{AddIndex, ColumnBuilder, CreateTableBuilder, Index, Migration};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn add_index() {
5fn add_index() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Add index");
8−
7+
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
8+ test.first_migration(
10− CreateTableBuilder::default()
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
11− .name("users")
14+ name = "users"
12− .primary_key(vec!["id".to_string()])
15+ primary_key = ["id"]
16+
13− .columns(vec![
17+ [[actions.columns]]
14− ColumnBuilder::default()
15− .name("id")
18+ name = "id"
16− .data_type("INTEGER")
19+ type = "INTEGER"
17− .build()
20+
18− .unwrap(),
21+ [[actions.columns]]
19− ColumnBuilder::default()
20− .name("name")
22+ name = "name"
21− .data_type("TEXT")
23+ type = "TEXT"
22− .build()
24+ "#,
23− .unwrap(),
24− ])
25− .build()
26− .unwrap(),
27 );
25 );
28− let add_index_migration = Migration::new("add_name_index", None).with_action(AddIndex {
26+
27+ test.second_migration(
28+ r#"
29+ name = "add_users_name_index"
30+
31+ [[actions]]
32+ type = "add_index"
29− table: "users".to_string(),
33+ table = "users"
34+
30− index: Index {
35+ [actions.index]
31− name: "name_idx".to_string(),
36+ name = "name_idx"
32− columns: vec!["name".to_string()],
37+ columns = ["name"]
33− unique: true,
38+ "#,
34− index_type: None,
39+ );
35− },
40+
41+ test.intermediate(|db, _| {
42+ // Ensure index is valid and ready
43+ let (is_ready, is_valid): (bool, bool) = db
44+ .query(
45+ "
46+ SELECT pg_index.indisready, pg_index.indisvalid
47+ FROM pg_catalog.pg_index
48+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
49+ WHERE pg_class.relname = 'name_idx'
50+ ",
51+ &[],
52+ )
53+ .unwrap()
54+ .first()
55+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
56+ .unwrap();
57+
58+ assert!(is_ready, "expected index to be ready");
59+ assert!(is_valid, "expected index to be valid");
36 });
60 });
37
61
38− let first_migrations = vec![create_table_migration.clone()];
62+ test.after_completion(|db| {
39− let second_migrations = vec![create_table_migration.clone(), add_index_migration.clone()];
40−
41− // Run migrations
42− reshape.migrate(first_migrations.clone()).unwrap();
43− reshape.migrate(second_migrations.clone()).unwrap();
44−
45− // Update schemas of Postgres connections
46− let old_schema_query =
47− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
48− let new_schema_query =
49− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
50− old_db.simple_query(&old_schema_query).unwrap();
51− new_db.simple_query(&new_schema_query).unwrap();
52−
53− // Ensure index is valid and ready
63+ // Ensure index is valid and ready
54− let (is_ready, is_valid): (bool, bool) = new_db
64+ let (is_ready, is_valid): (bool, bool) = db
55− .query(
65+ .query(
56− "
66+ "
57− SELECT pg_index.indisready, pg_index.indisvalid
67+ SELECT pg_index.indisready, pg_index.indisvalid
58− FROM pg_catalog.pg_index
68+ FROM pg_catalog.pg_index
59− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
69+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
60− WHERE pg_class.relname = 'name_idx'
70+ WHERE pg_class.relname = 'name_idx'
61− ",
71+ ",
62− &[],
72+ &[],
63− )
73+ )
64− .unwrap()
74+ .unwrap()
65− .first()
75+ .first()
66− .map(|row| (row.get("indisready"), row.get("indisvalid")))
76+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
67− .unwrap();
77+ .unwrap();
68−
78+
69− assert!(is_ready, "expected index to be ready");
79+ assert!(is_ready, "expected index to be ready");
70− assert!(is_valid, "expected index to be valid");
80+ assert!(is_valid, "expected index to be valid");
71−
81+ });
72− reshape.complete().unwrap();
82+
73− common::assert_cleaned_up(&mut new_db);
83+ test.run();
74}
84}
75
85
76#[test]
86#[test]
77fn add_index_unique() {
87fn add_index_unique() {
78− let (mut reshape, mut old_db, mut new_db) = common::setup();
88+ let mut test = Test::new("Add unique index");
79−
89+
80− let create_table_migration = Migration::new("create_user_table", None).with_action(
90+ test.first_migration(
81− CreateTableBuilder::default()
91+ r#"
92+ name = "create_users_table"
93+
94+ [[actions]]
95+ type = "create_table"
82− .name("users")
96+ name = "users"
83− .primary_key(vec!["id".to_string()])
97+ primary_key = ["id"]
98+
84− .columns(vec![
99+ [[actions.columns]]
85− ColumnBuilder::default()
86− .name("id")
100+ name = "id"
87− .data_type("INTEGER")
101+ type = "INTEGER"
88− .build()
102+
89− .unwrap(),
103+ [[actions.columns]]
90− ColumnBuilder::default()
91− .name("name")
104+ name = "name"
92− .data_type("TEXT")
105+ type = "TEXT"
93− .build()
106+ "#,
94− .unwrap(),
95− ])
96− .build()
97− .unwrap(),
98 );
107 );
99− let add_index_migration = Migration::new("add_name_index", None).with_action(AddIndex {
100− table: "users".to_string(),
101− index: Index {
102− name: "name_idx".to_string(),
103− columns: vec!["name".to_string()],
104− unique: true,
105− index_type: None,
106− },
107− });
108
108
109− let first_migrations = vec![create_table_migration.clone()];
109+ test.second_migration(
110− let second_migrations = vec![create_table_migration.clone(), add_index_migration.clone()];
110+ r#"
111−
111+ name = "add_name_index"
112− // Run migrations
112+
113− reshape.migrate(first_migrations.clone()).unwrap();
113+ [[actions]]
114− reshape.migrate(second_migrations.clone()).unwrap();
114+ type = "add_index"
115−
115+ table = "users"
116− // Update schemas of Postgres connections
116+
117− let old_schema_query =
117+ [actions.index]
118− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
119− let new_schema_query =
120− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
121− old_db.simple_query(&old_schema_query).unwrap();
122− new_db.simple_query(&new_schema_query).unwrap();
123−
124− // Ensure index is valid, ready and unique
125− let (is_ready, is_valid, is_unique): (bool, bool, bool) = new_db
126− .query(
127− "
128− SELECT pg_index.indisready, pg_index.indisvalid, pg_index.indisunique
129− FROM pg_catalog.pg_index
130− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
131− WHERE pg_class.relname = 'name_idx'
118+ name = "name_idx"
132− ",
119+ columns = ["name"]
133− &[],
120+ unique = true
134− )
121+ "#,
135− .unwrap()
122+ );
136− .first()
137− .map(|row| {
138− (
139− row.get("indisready"),
140− row.get("indisvalid"),
141− row.get("indisunique"),
142− )
143− })
144− .unwrap();
145
123
124+ test.intermediate(|db, _| {
125+ // Ensure index is valid, ready and unique
126+ let (is_ready, is_valid, is_unique): (bool, bool, bool) = db
127+ .query(
128+ "
129+ SELECT pg_index.indisready, pg_index.indisvalid, pg_index.indisunique
130+ FROM pg_catalog.pg_index
131+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
132+ WHERE pg_class.relname = 'name_idx'
133+ ",
134+ &[],
135+ )
136+ .unwrap()
137+ .first()
138+ .map(|row| {
139+ (
140+ row.get("indisready"),
141+ row.get("indisvalid"),
142+ row.get("indisunique"),
143+ )
144+ })
145+ .unwrap();
146+
146− assert!(is_ready, "expected index to be ready");
147+ assert!(is_ready, "expected index to be ready");
147− assert!(is_valid, "expected index to be valid");
148+ assert!(is_valid, "expected index to be valid");
148− assert!(is_unique, "expected index to be unique");
149+ assert!(is_unique, "expected index to be unique");
150+ });
149
151
150− reshape.complete().unwrap();
152+ test.run();
151− common::assert_cleaned_up(&mut new_db);
152}
153}
153
154
154#[test]
155#[test]
155fn add_index_with_type() {
156fn add_index_with_type() {
156− let (mut reshape, mut old_db, mut new_db) = common::setup();
157+ let mut test = Test::new("Add GIN index");
157−
158+
158− let create_table_migration = Migration::new("create_user_table", None).with_action(
159+ test.first_migration(
159− CreateTableBuilder::default()
160+ r#"
161+ name = "create_users_table"
162+
163+ [[actions]]
164+ type = "create_table"
160− .name("users")
165+ name = "users"
161− .primary_key(vec!["id".to_string()])
166+ primary_key = ["id"]
167+
162− .columns(vec![
168+ [[actions.columns]]
163− ColumnBuilder::default()
164− .name("id")
169+ name = "id"
165− .data_type("INTEGER")
170+ type = "INTEGER"
166− .build()
171+
167− .unwrap(),
172+ [[actions.columns]]
168− ColumnBuilder::default()
169− .name("data")
173+ name = "data"
170− .data_type("JSONB")
174+ type = "JSONB"
171− .build()
175+ "#,
172− .unwrap(),
173− ])
174− .build()
175− .unwrap(),
176 );
176 );
177− let add_index_migration = Migration::new("add_name_index", None).with_action(AddIndex {
178− table: "users".to_string(),
179− index: Index {
180− name: "name_idx".to_string(),
181− columns: vec!["data".to_string()],
182− unique: false,
183− index_type: Some("gin".to_string()),
184− },
185− });
186
177
187− let first_migrations = vec![create_table_migration.clone()];
178+ test.second_migration(
188− let second_migrations = vec![create_table_migration.clone(), add_index_migration.clone()];
179+ r#"
189−
180+ name = "add_data_index"
190− // Run migrations
181+
191− reshape.migrate(first_migrations.clone()).unwrap();
182+ [[actions]]
192− reshape.migrate(second_migrations.clone()).unwrap();
183+ type = "add_index"
193−
184+ table = "users"
194− // Update schemas of Postgres connections
195− let old_schema_query =
196− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
197− let new_schema_query =
198− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
199− old_db.simple_query(&old_schema_query).unwrap();
200− new_db.simple_query(&new_schema_query).unwrap();
201−
202− // Ensure index is valid, ready and unique
203− let (is_ready, is_valid, index_type): (bool, bool, String) = new_db
204− .query(
205− "
206− SELECT pg_index.indisready, pg_index.indisvalid, pg_am.amname
207− FROM pg_catalog.pg_index
208− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
209− JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
210− WHERE pg_class.relname = 'name_idx'
211− ",
212− &[],
213− )
214− .unwrap()
215− .first()
216− .map(|row| {
217− (
218− row.get("indisready"),
219− row.get("indisvalid"),
220− row.get("amname"),
221− )
222− })
223− .unwrap();
224
185
186+ [actions.index]
187+ name = "data_idx"
188+ columns = ["data"]
189+ type = "gin"
190+ "#,
191+ );
192+
193+ test.intermediate(|db, _| {
194+ // Ensure index is valid, ready and has the right type
195+ let (is_ready, is_valid, index_type): (bool, bool, String) = db
196+ .query(
197+ "
198+ SELECT pg_index.indisready, pg_index.indisvalid, pg_am.amname
199+ FROM pg_catalog.pg_index
200+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
201+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
202+ WHERE pg_class.relname = 'data_idx'
203+ ",
204+ &[],
205+ )
206+ .unwrap()
207+ .first()
208+ .map(|row| {
209+ (
210+ row.get("indisready"),
211+ row.get("indisvalid"),
212+ row.get("amname"),
213+ )
214+ })
215+ .unwrap();
216+
225− assert!(is_ready, "expected index to be ready");
217+ assert!(is_ready, "expected index to be ready");
226− assert!(is_valid, "expected index to be valid");
218+ assert!(is_valid, "expected index to be valid");
227− assert_eq!("gin", index_type, "expected index type to be GIN");
219+ assert_eq!("gin", index_type, "expected index type to be GIN");
220+ });
228
221
229− reshape.complete().unwrap();
222+ test.run();
230− common::assert_cleaned_up(&mut new_db);
231}
223}
tests/alter_column.rs+637−498
@@ -1,61 +1,45 @@
1−use reshape::migrations::{
2− AddIndex, AlterColumn, ColumnBuilder, ColumnChanges, CreateTableBuilder, Index, Migration,
3−};
4−
5mod common;
1mod common;
2+use common::Test;
6
3
7#[test]
4#[test]
8fn alter_column_data() {
5fn alter_column_data() {
9− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Alter column");
10−
11− let create_users_table = Migration::new("create_user_table", None).with_action(
12− CreateTableBuilder::default()
13− .name("users")
14− .primary_key(vec!["id".to_string()])
15− .columns(vec![
16− ColumnBuilder::default()
17− .name("id")
18− .data_type("INTEGER")
19− .build()
20− .unwrap(),
21− ColumnBuilder::default()
22− .name("name")
23− .data_type("TEXT")
24− .build()
25− .unwrap(),
26− ])
27− .build()
28− .unwrap(),
29− );
30− let uppercase_name = Migration::new("uppercase_name", None).with_action(AlterColumn {
31− table: "users".to_string(),
32− column: "name".to_string(),
33− up: Some("UPPER(name)".to_string()),
34− down: Some("LOWER(name)".to_string()),
35− changes: ColumnChanges {
36− data_type: None,
37− nullable: None,
38− name: None,
39− default: None,
40− },
41− });
42
7
43− let first_migrations = vec![create_users_table.clone()];
8+ test.first_migration(
44− let second_migrations = vec![create_users_table.clone(), uppercase_name.clone()];
9+ r#"
10+ name = "create_user_table"
45
11
46− // Run first migration, should automatically finish
12+ [[actions]]
47− reshape.migrate(first_migrations.clone()).unwrap();
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
48
16
49− // Update search paths
17+ [[actions.columns]]
50− old_db
18+ name = "id"
51− .simple_query(&reshape::schema_query_for_migration(
19+ type = "INTEGER"
52− &first_migrations.last().unwrap().name,
20+
53− ))
21+ [[actions.columns]]
54− .unwrap();
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "uppercase_name"
30+
31+ [[actions]]
32+ type = "alter_column"
33+ table = "users"
34+ column = "name"
35+ up = "UPPER(name)"
36+ down = "LOWER(name)"
37+ "#,
38+ );
55
39
40+ test.after_first(|db| {
56− // Insert some test users
41+ // Insert some test users
57− old_db
58− .simple_query(
42+ db.simple_query(
59 "
43 "
60 INSERT INTO users (id, name) VALUES
44 INSERT INTO users (id, name) VALUES
61 (1, 'john Doe'),
45 (1, 'john Doe'),
@@ -63,98 +47,81 @@ fn alter_column_data() {
63 ",
47 ",
64 )
48 )
65 .unwrap();
49 .unwrap();
50+ });
66
51
67− // Run second migration
52+ test.intermediate(|old_db, new_db| {
68− reshape.migrate(second_migrations.clone()).unwrap();
69− new_db
70− .simple_query(&reshape::schema_query_for_migration(
71− &second_migrations.last().unwrap().name,
72− ))
73− .unwrap();
74−
75− // Check that the existing users has the altered data
53+ // Check that the existing users has the altered data
76− let expected = vec!["JOHN DOE", "JANE DOE"];
54+ let expected = vec!["JOHN DOE", "JANE DOE"];
77− assert!(new_db
55+ assert!(new_db
78− .query("SELECT name FROM users ORDER BY id", &[],)
56+ .query("SELECT name FROM users ORDER BY id", &[],)
79− .unwrap()
57+ .unwrap()
80− .iter()
58+ .iter()
81− .map(|row| row.get::<_, String>("name"))
59+ .map(|row| row.get::<_, String>("name"))
82− .eq(expected));
60+ .eq(expected));
83−
61+
84− // Insert data using old schema and make sure the new schema gets correct values
62+ // Insert data using old schema and make sure the new schema gets correct values
85− old_db
63+ old_db
86− .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
64+ .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
87− .unwrap();
65+ .unwrap();
88− let result = new_db
66+ let result = new_db
89− .query_one("SELECT name from users WHERE id = 3", &[])
67+ .query_one("SELECT name from users WHERE id = 3", &[])
90− .unwrap();
68+ .unwrap();
91− assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
69+ assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
92−
70+
93− // Insert data using new schema and make sure the old schema gets correct values
71+ // Insert data using new schema and make sure the old schema gets correct values
94− new_db
72+ new_db
95− .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
73+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
96− .unwrap();
74+ .unwrap();
97− let result = old_db
75+ let result = old_db
98− .query_one("SELECT name from users WHERE id = 4", &[])
76+ .query_one("SELECT name from users WHERE id = 4", &[])
99− .unwrap();
77+ .unwrap();
100− assert_eq!("test testsson", result.get::<_, &str>("name"));
78+ assert_eq!("test testsson", result.get::<_, &str>("name"));
79+ });
101
80
102− reshape.complete().unwrap();
81+ test.run();
103− common::assert_cleaned_up(&mut new_db);
104}
82}
105
83
106#[test]
84#[test]
107fn alter_column_set_not_null() {
85fn alter_column_set_not_null() {
108− let (mut reshape, mut old_db, mut new_db) = common::setup();
86+ let mut test = Test::new("Set column not null");
109−
87+
110− let create_users_table = Migration::new("create_user_table", None).with_action(
88+ test.first_migration(
111− CreateTableBuilder::default()
89+ r#"
90+ name = "create_user_table"
91+
92+ [[actions]]
93+ type = "create_table"
112− .name("users")
94+ name = "users"
113− .primary_key(vec!["id".to_string()])
95+ primary_key = ["id"]
96+
114− .columns(vec![
97+ [[actions.columns]]
115− ColumnBuilder::default()
116− .name("id")
98+ name = "id"
117− .data_type("INTEGER")
99+ type = "INTEGER"
118− .build()
100+
119− .unwrap(),
101+ [[actions.columns]]
120− ColumnBuilder::default()
121− .name("name")
102+ name = "name"
122− .data_type("TEXT")
103+ type = "TEXT"
123− .build()
104+ "#,
124− .unwrap(),
125− ])
126− .build()
127− .unwrap(),
128 );
105 );
129− let set_name_not_null = Migration::new("set_name_not_null", None).with_action(AlterColumn {
130− table: "users".to_string(),
131− column: "name".to_string(),
132− up: Some("COALESCE(name, 'TEST_DEFAULT_VALUE')".to_string()),
133− down: Some("name".to_string()),
134− changes: ColumnChanges {
135− data_type: None,
136− nullable: Some(false),
137− name: None,
138− default: None,
139− },
140− });
141
106
142− let first_migrations = vec![create_users_table.clone()];
107+ test.second_migration(
143− let second_migrations = vec![create_users_table.clone(), set_name_not_null.clone()];
108+ r#"
109+ name = "set_name_not_null"
144
110
145− // Run first migration, should automatically finish
111+ [[actions]]
146− reshape.migrate(first_migrations.clone()).unwrap();
112+ type = "alter_column"
113+ table = "users"
114+ column = "name"
115+ up = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
147
116
148− // Update search paths
117+ [actions.changes]
149− old_db
118+ nullable = false
150− .simple_query(&reshape::schema_query_for_migration(
119+ "#,
151− &first_migrations.last().unwrap().name,
120+ );
152− ))
153− .unwrap();
154
121
122+ test.after_first(|db| {
155− // Insert some test users
123+ // Insert some test users
156− old_db
157− .simple_query(
124+ db.simple_query(
158 "
125 "
159 INSERT INTO users (id, name) VALUES
126 INSERT INTO users (id, name) VALUES
160 (1, 'John Doe'),
127 (1, 'John Doe'),
@@ -162,99 +129,180 @@ fn alter_column_set_not_null() {
162 ",
129 ",
163 )
130 )
164 .unwrap();
131 .unwrap();
132+ });
165
133
166− // Run second migration
134+ test.intermediate(|old_db, new_db| {
167− reshape.migrate(second_migrations.clone()).unwrap();
135+ // Check that existing users got the correct values
136+ let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
137+ assert!(new_db
138+ .query("SELECT name FROM users ORDER BY id", &[],)
139+ .unwrap()
140+ .iter()
141+ .map(|row| row.get::<_, String>("name"))
142+ .eq(expected));
143+
144+ // Insert data using old schema and make sure the new schema gets correct values
145+ old_db
146+ .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
147+ .unwrap();
168− new_db
148+ let result = new_db
169− .simple_query(&reshape::schema_query_for_migration(
149+ .query_one("SELECT name from users WHERE id = 3", &[])
170− &second_migrations.last().unwrap().name,
150+ .unwrap();
171− ))
151+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
172− .unwrap();
152+
153+ // Insert data using new schema and make sure the old schema gets correct values
154+ new_db
155+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
156+ .unwrap();
157+ let result = old_db
158+ .query_one("SELECT name from users WHERE id = 4", &[])
159+ .unwrap();
160+ assert_eq!("Jane Doe", result.get::<_, &str>("name"));
161+
162+ // Ensure NULL can't be inserted using the new schema
163+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
164+ assert!(result.is_err(), "expected insert to fail");
165+ });
173
166
174− // Check that existing users got the correct values
167+ test.after_completion(|db| {
175− let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
168+ // Ensure NULL can't be inserted
176− assert!(new_db
177− .query("SELECT name FROM users ORDER BY id", &[],)
178− .unwrap()
179− .iter()
180− .map(|row| row.get::<_, String>("name"))
181− .eq(expected));
182−
183− // Insert data using old schema and make sure the new schema gets correct values
184− old_db
185− .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
169+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
186− .unwrap();
170+ assert!(result.is_err(), "expected insert to fail");
187− let result = new_db
171+ });
188− .query_one("SELECT name from users WHERE id = 3", &[])
189− .unwrap();
190− assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
191
172
192− // Insert data using new schema and make sure the old schema gets correct values
173+ test.after_abort(|db| {
193− new_db
174+ // Ensure NULL can be inserted
175+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
176+ assert!(result.is_ok(), "expected insert to succeed");
177+ });
178+
179+ test.run();
180+}
181+
182+#[test]
183+fn alter_column_set_nullable() {
184+ let mut test = Test::new("Set column nullable");
185+
186+ test.first_migration(
187+ r#"
188+ name = "create_user_table"
189+
190+ [[actions]]
191+ type = "create_table"
192+ name = "users"
193+ primary_key = ["id"]
194+
195+ [[actions.columns]]
196+ name = "id"
197+ type = "INTEGER"
198+
199+ [[actions.columns]]
200+ name = "name"
201+ type = "TEXT"
202+ nullable = false
203+ "#,
204+ );
205+
206+ test.second_migration(
207+ r#"
208+ name = "set_name_nullable"
209+
210+ [[actions]]
211+ type = "alter_column"
212+ table = "users"
213+ column = "name"
214+ down = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
215+
216+ [actions.changes]
217+ nullable = true
218+ "#,
219+ );
220+
221+ test.after_first(|db| {
222+ // Insert a test user
223+ db.simple_query(
224+ "
194− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
225+ INSERT INTO users (id, name) VALUES
195− .unwrap();
226+ (1, 'John Doe')
196− let result = old_db
227+ ",
197− .query_one("SELECT name from users WHERE id = 4", &[])
228+ )
198 .unwrap();
229 .unwrap();
199− assert_eq!("Jane Doe", result.get::<_, &str>("name"));
230+ });
200
231
232+ test.intermediate(|old_db, new_db| {
233+ // Insert data using new schema and make sure the old schema gets correct values
234+ new_db
235+ .simple_query("INSERT INTO users (id, name) VALUES (2, NULL)")
201− reshape.complete().unwrap();
236+ .unwrap();
202− common::assert_cleaned_up(&mut new_db);
237+ let result = old_db
238+ .query_one("SELECT name from users WHERE id = 2", &[])
239+ .unwrap();
240+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
241+
242+ // Ensure NULL can't be inserted using the old schema
243+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, NULL)");
244+ assert!(result.is_err(), "expected insert to fail");
245+
246+ // Ensure NULL can be inserted using the new schema
247+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (4, NULL)");
248+ assert!(result.is_ok(), "expected insert to succeed");
249+ });
250+
251+ test.after_completion(|db| {
252+ // Ensure NULL can be inserted
253+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
254+ assert!(result.is_ok(), "expected insert to succeed");
255+ });
256+
257+ test.after_abort(|db| {
258+ // Ensure NULL can't be inserted
259+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
260+ assert!(result.is_err(), "expected insert to fail");
261+ });
262+
263+ test.run();
203}
264}
204
265
205#[test]
266#[test]
206fn alter_column_rename() {
267fn alter_column_rename() {
207− let (mut reshape, mut old_db, mut new_db) = common::setup();
268+ let mut test = Test::new("Rename column");
208−
269+
209− let create_users_table = Migration::new("create_user_table", None).with_action(
270+ test.first_migration(
210− CreateTableBuilder::default()
271+ r#"
272+ name = "create_user_table"
273+
274+ [[actions]]
275+ type = "create_table"
211− .name("users")
276+ name = "users"
212− .primary_key(vec!["id".to_string()])
277+ primary_key = ["id"]
278+
213− .columns(vec![
279+ [[actions.columns]]
214− ColumnBuilder::default()
215− .name("id")
280+ name = "id"
216− .data_type("INTEGER")
281+ type = "INTEGER"
217− .build()
282+
218− .unwrap(),
283+ [[actions.columns]]
219− ColumnBuilder::default()
220− .name("name")
284+ name = "name"
221− .data_type("TEXT")
285+ type = "TEXT"
222− .build()
286+ "#,
223− .unwrap(),
224− ])
225− .build()
226− .unwrap(),
227 );
287 );
228− let rename_to_full_name =
229− Migration::new("rename_to_full_name", None).with_action(AlterColumn {
230− table: "users".to_string(),
231− column: "name".to_string(),
232− up: None, // up and down are not required when only renaming a column
233− down: None,
234− changes: ColumnChanges {
235− data_type: None,
236− nullable: None,
237− name: Some("full_name".to_string()),
238− default: None,
239− },
240− });
241−
242− let first_migrations = vec![create_users_table.clone()];
243− let second_migrations = vec![create_users_table.clone(), rename_to_full_name.clone()];
244−
245− // Run first migration, should automatically finish
246− reshape.migrate(first_migrations.clone()).unwrap();
247−
248− // Update search paths
249− old_db
250− .simple_query(&reshape::schema_query_for_migration(
251− &first_migrations.last().unwrap().name,
252− ))
253− .unwrap();
254
288
289+ test.second_migration(
290+ r#"
291+ name = "set_name_not_null"
292+
293+ [[actions]]
294+ type = "alter_column"
295+ table = "users"
296+ column = "name"
297+
298+ [actions.changes]
299+ name = "full_name"
300+ "#,
301+ );
302+
303+ test.after_first(|db| {
255− // Insert some test data
304+ // Insert some test data
256− old_db
257− .simple_query(
305+ db.simple_query(
258 "
306 "
259 INSERT INTO users (id, name) VALUES
307 INSERT INTO users (id, name) VALUES
260 (1, 'John Doe'),
308 (1, 'John Doe'),
@@ -262,94 +310,69 @@ fn alter_column_rename() {
262 ",
310 ",
263 )
311 )
264 .unwrap();
312 .unwrap();
313+ });
265
314
266− // Run second migration
315+ test.intermediate(|_, new_db| {
267− reshape.migrate(second_migrations.clone()).unwrap();
316+ // Check that existing values can be queried using new column name
268− new_db
317+ let expected = vec!["John Doe", "Jane Doe"];
269− .simple_query(&reshape::schema_query_for_migration(
318+ assert!(new_db
270− &second_migrations.last().unwrap().name,
319+ .query("SELECT full_name FROM users ORDER BY id", &[],)
271− ))
320+ .unwrap()
272− .unwrap();
321+ .iter()
322+ .map(|row| row.get::<_, String>("full_name"))
323+ .eq(expected));
324+ });
273
325
274− // Check that existing values can be queried using new column name
326+ test.run();
275− let expected = vec!["John Doe", "Jane Doe"];
276− assert!(new_db
277− .query("SELECT full_name FROM users ORDER BY id", &[],)
278− .unwrap()
279− .iter()
280− .map(|row| row.get::<_, String>("full_name"))
281− .eq(expected));
282−
283− reshape.complete().unwrap();
284− common::assert_cleaned_up(&mut new_db);
285}
327}
286
328
287#[test]
329#[test]
288fn alter_column_multiple() {
330fn alter_column_multiple() {
289− let (mut reshape, mut old_db, mut new_db) = common::setup();
331+ let mut test = Test::new("Alter column value multiple times");
290−
332+
291− let create_users_table = Migration::new("create_user_table", None).with_action(
333+ test.first_migration(
292− CreateTableBuilder::default()
334+ r#"
335+ name = "create_user_table"
336+
337+ [[actions]]
338+ type = "create_table"
293− .name("users")
339+ name = "users"
294− .primary_key(vec!["id".to_string()])
340+ primary_key = ["id"]
341+
295− .columns(vec![
342+ [[actions.columns]]
296− ColumnBuilder::default()
297− .name("id")
343+ name = "id"
298− .data_type("INTEGER")
344+ type = "INTEGER"
299− .build()
345+
300− .unwrap(),
346+ [[actions.columns]]
301− ColumnBuilder::default()
302− .name("counter")
347+ name = "counter"
303− .data_type("INTEGER")
348+ type = "INTEGER"
304− .nullable(false)
349+ nullable = false
305− .build()
350+ "#,
306− .unwrap(),
351+ );
307− ])
352+
308− .build()
353+ test.second_migration(
309− .unwrap(),
354+ r#"
355+ name = "increment_counter_twice"
356+
357+ [[actions]]
358+ type = "alter_column"
359+ table = "users"
360+ column = "counter"
361+ up = "counter + 1"
362+ down = "counter - 1"
363+
364+ [[actions]]
365+ type = "alter_column"
366+ table = "users"
367+ column = "counter"
368+ up = "counter + 1"
369+ down = "counter - 1"
370+ "#,
310 );
371 );
311− let increment_counter_twice = Migration::new("increment_counter_twice", None)
312− .with_action(AlterColumn {
313− table: "users".to_string(),
314− column: "counter".to_string(),
315− up: Some("counter + 1".to_string()),
316− down: Some("counter - 1".to_string()),
317− changes: ColumnChanges {
318− data_type: None,
319− nullable: None,
320− name: None,
321− default: None,
322− },
323− })
324− .with_action(AlterColumn {
325− table: "users".to_string(),
326− column: "counter".to_string(),
327− up: Some("counter + 1".to_string()),
328− down: Some("counter - 1".to_string()),
329− changes: ColumnChanges {
330− data_type: None,
331− nullable: None,
332− name: None,
333− default: None,
334− },
335− });
336−
337− let first_migrations = vec![create_users_table.clone()];
338− let second_migrations = vec![create_users_table.clone(), increment_counter_twice.clone()];
339−
340− // Run first migration, should automatically finish
341− reshape.migrate(first_migrations.clone()).unwrap();
342−
343− // Update search paths
344− old_db
345− .simple_query(&reshape::schema_query_for_migration(
346− &first_migrations.last().unwrap().name,
347− ))
348− .unwrap();
349
372
373+ test.after_first(|db| {
350− // Insert some test data
374+ // Insert some test data
351− old_db
352− .simple_query(
375+ db.simple_query(
353 "
376 "
354 INSERT INTO users (id, counter) VALUES
377 INSERT INTO users (id, counter) VALUES
355 (1, 0),
378 (1, 0),
@@ -357,239 +380,355 @@ fn alter_column_multiple() {
357 ",
380 ",
358 )
381 )
359 .unwrap();
382 .unwrap();
383+ });
360
384
361− // Run second migration
385+ test.intermediate(|old_db, new_db| {
362− reshape.migrate(second_migrations.clone()).unwrap();
363− new_db
364− .simple_query(&reshape::schema_query_for_migration(
365− &second_migrations.last().unwrap().name,
366− ))
367− .unwrap();
368−
369− // Check that the existing data has been updated
386+ // Check that the existing data has been updated
370− let expected = vec![2, 102];
387+ let expected = vec![2, 102];
371− let results: Vec<i32> = new_db
388+ let results: Vec<i32> = new_db
372− .query("SELECT counter FROM users ORDER BY id", &[])
389+ .query("SELECT counter FROM users ORDER BY id", &[])
373− .unwrap()
390+ .unwrap()
374− .iter()
391+ .iter()
375− .map(|row| row.get::<_, i32>("counter"))
392+ .map(|row| row.get::<_, i32>("counter"))
376− .collect();
393+ .collect();
377− assert_eq!(expected, results);
394+ assert_eq!(expected, results);
378−
395+
379− // Update data using old schema and make sure it was updated for the new schema
396+ // Update data using old schema and make sure it was updated for the new schema
380− old_db
397+ old_db
381− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
398+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
382− .unwrap();
399+ .unwrap();
383− let result: i32 = new_db
400+ let result: i32 = new_db
384− .query("SELECT counter FROM users WHERE id = 1", &[])
401+ .query("SELECT counter FROM users WHERE id = 1", &[])
385− .unwrap()
402+ .unwrap()
386− .iter()
403+ .iter()
387− .map(|row| row.get("counter"))
404+ .map(|row| row.get("counter"))
388− .nth(0)
405+ .nth(0)
389− .unwrap();
406+ .unwrap();
390− assert_eq!(52, result);
407+ assert_eq!(52, result);
391−
408+
392− // Update data using new schema and make sure it was updated for the old schema
409+ // Update data using new schema and make sure it was updated for the old schema
393− new_db
410+ new_db
394− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
411+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
395− .unwrap();
412+ .unwrap();
396− let result: i32 = old_db
413+ let result: i32 = old_db
397− .query("SELECT counter FROM users WHERE id = 1", &[])
414+ .query("SELECT counter FROM users WHERE id = 1", &[])
398− .unwrap()
415+ .unwrap()
399− .iter()
416+ .iter()
400− .map(|row| row.get("counter"))
417+ .map(|row| row.get("counter"))
401− .nth(0)
418+ .nth(0)
402− .unwrap();
419+ .unwrap();
403− assert_eq!(48, result);
420+ assert_eq!(48, result);
421+ });
404
422
405− reshape.complete().unwrap();
423+ test.run();
406− common::assert_cleaned_up(&mut new_db);
407}
424}
408
425
409#[test]
426#[test]
410fn alter_column_default() {
427fn alter_column_default() {
411− let (mut reshape, mut old_db, mut new_db) = common::setup();
428+ let mut test = Test::new("Change default value for column");
412−
429+
413− let create_users_table = Migration::new("create_user_table", None).with_action(
430+ test.first_migration(
414− CreateTableBuilder::default()
431+ r#"
432+ name = "create_user_table"
433+
434+ [[actions]]
435+ type = "create_table"
415− .name("users")
436+ name = "users"
416− .primary_key(vec!["id".to_string()])
437+ primary_key = ["id"]
438+
417− .columns(vec![
439+ [[actions.columns]]
418− ColumnBuilder::default()
419− .name("id")
440+ name = "id"
420− .data_type("INTEGER")
441+ type = "INTEGER"
421− .build()
442+
422− .unwrap(),
443+ [[actions.columns]]
423− ColumnBuilder::default()
424− .name("name")
444+ name = "name"
425− .data_type("TEXT")
445+ type = "TEXT"
426− .nullable(false)
446+ nullable = false
427− .default_value("'DEFAULT'")
447+ default = "'DEFAULT'"
428− .build()
448+ "#,
429− .unwrap(),
430− ])
431− .build()
432− .unwrap(),
433 );
449 );
434− let change_name_default =
435− Migration::new("change_name_default", None).with_action(AlterColumn {
436− table: "users".to_string(),
437− column: "name".to_string(),
438− up: None,
439− down: None,
440− changes: ColumnChanges {
441− data_type: None,
442− nullable: None,
443− name: None,
444− default: Some("'NEW DEFAULT'".to_string()),
445− },
446− });
447−
448− let first_migrations = vec![create_users_table.clone()];
449− let second_migrations = vec![create_users_table.clone(), change_name_default.clone()];
450−
451− // Run first migration, should automatically finish
452− reshape.migrate(first_migrations.clone()).unwrap();
453−
454− // Update search paths
455− old_db
456− .simple_query(&reshape::schema_query_for_migration(
457− &first_migrations.last().unwrap().name,
458− ))
459− .unwrap();
460
450
451+ test.second_migration(
452+ r#"
453+ name = "change_name_default"
454+
455+ [[actions]]
456+ type = "alter_column"
457+ table = "users"
458+ column = "name"
459+
460+ [actions.changes]
461+ default = "'NEW DEFAULT'"
462+ "#,
463+ );
464+
465+ test.after_first(|db| {
461− // Insert a test user
466+ // Insert a test user
462− old_db
463− .simple_query(
467+ db.simple_query(
464 "
468 "
465 INSERT INTO users (id) VALUES (1)
469 INSERT INTO users (id) VALUES (1)
466 ",
470 ",
467 )
471 )
468 .unwrap();
472 .unwrap();
473+ });
469
474
470− // Run second migration
475+ test.intermediate(|old_db, new_db| {
471− reshape.migrate(second_migrations.clone()).unwrap();
472− new_db
473− .simple_query(&reshape::schema_query_for_migration(
474− &second_migrations.last().unwrap().name,
475− ))
476− .unwrap();
477−
478− // Check that the existing users has the old default value
476+ // Check that the existing users has the old default value
479− let expected = vec!["DEFAULT"];
477+ let expected = vec!["DEFAULT"];
480− assert!(new_db
478+ assert!(new_db
481− .query("SELECT name FROM users", &[],)
479+ .query("SELECT name FROM users", &[],)
482− .unwrap()
480+ .unwrap()
483− .iter()
481+ .iter()
484− .map(|row| row.get::<_, String>("name"))
482+ .map(|row| row.get::<_, String>("name"))
485− .eq(expected));
483+ .eq(expected));
486−
484+
487− // Insert data using old schema and make those get the old default value
485+ // Insert data using old schema and make those get the old default value
488− old_db
486+ old_db
489− .simple_query("INSERT INTO users (id) VALUES (2)")
487+ .simple_query("INSERT INTO users (id) VALUES (2)")
490− .unwrap();
488+ .unwrap();
491− let result = new_db
489+ let result = new_db
492− .query_one("SELECT name from users WHERE id = 2", &[])
490+ .query_one("SELECT name from users WHERE id = 2", &[])
493− .unwrap();
491+ .unwrap();
494− assert_eq!("DEFAULT", result.get::<_, &str>("name"));
492+ assert_eq!("DEFAULT", result.get::<_, &str>("name"));
495−
493+
496− // Insert data using new schema and make sure it gets the new default value
494+ // Insert data using new schema and make sure it gets the new default value
497− new_db
495+ new_db
498− .simple_query("INSERT INTO users (id) VALUES (3)")
496+ .simple_query("INSERT INTO users (id) VALUES (3)")
499− .unwrap();
497+ .unwrap();
500− let result = old_db
498+ let result = old_db
501− .query_one("SELECT name from users WHERE id = 3", &[])
499+ .query_one("SELECT name from users WHERE id = 3", &[])
502− .unwrap();
500+ .unwrap();
503− assert_eq!("NEW DEFAULT", result.get::<_, &str>("name"));
501+ assert_eq!("NEW DEFAULT", result.get::<_, &str>("name"));
502+ });
504
503
505− reshape.complete().unwrap();
504+ test.run();
506− common::assert_cleaned_up(&mut new_db);
507}
505}
508
506
509#[test]
507#[test]
510fn alter_column_with_index() {
508fn alter_column_with_index() {
511− let (mut reshape, mut db, _) = common::setup();
509+ let mut test = Test::new("Alter column with index");
512−
513− let create_users_table = Migration::new("create_user_table", None)
514− .with_action(
515− CreateTableBuilder::default()
516− .name("users")
517− .primary_key(vec!["id".to_string()])
518− .columns(vec![
519− ColumnBuilder::default()
520− .name("id")
521− .data_type("INTEGER")
522− .build()
523− .unwrap(),
524− ColumnBuilder::default()
525− .name("first_name")
526− .data_type("TEXT")
527− .build()
528− .unwrap(),
529− ColumnBuilder::default()
530− .name("last_name")
531− .data_type("TEXT")
532− .build()
533− .unwrap(),
534− ])
535− .build()
536− .unwrap(),
537− )
538− .with_action(AddIndex {
539− table: "users".to_string(),
540− index: Index {
541− name: "users_name_idx".to_string(),
542− columns: vec!["first_name".to_string(), "last_name".to_string()],
543− unique: false,
544− index_type: None,
545− },
546− });
547− let uppercase_name = Migration::new("uppercase_name", None).with_action(AlterColumn {
548− table: "users".to_string(),
549− column: "last_name".to_string(),
550− up: Some("UPPER(last_name)".to_string()),
551− down: Some("LOWER(last_name)".to_string()),
552− changes: ColumnChanges {
553− data_type: None,
554− nullable: None,
555− name: None,
556− default: None,
557− },
558− });
559
510
560− let first_migrations = vec![create_users_table.clone()];
511+ test.first_migration(
561− let second_migrations = vec![create_users_table.clone(), uppercase_name.clone()];
512+ r#"
513+ name = "create_user_table"
562
514
563− // Run first migration, should automatically finish
515+ [[actions]]
564− reshape.migrate(first_migrations.clone()).unwrap();
516+ type = "create_table"
517+ name = "users"
518+ primary_key = ["id"]
565
519
566− // Run second migration
520+ [[actions.columns]]
567− reshape.migrate(second_migrations.clone()).unwrap();
521+ name = "id"
568− db.simple_query(&reshape::schema_query_for_migration(
522+ type = "INTEGER"
569− &second_migrations.last().unwrap().name,
570− ))
571− .unwrap();
572
523
573− // Complete the second migration which should replace the existing column
524+ [[actions.columns]]
574− // with the temporary one
525+ name = "first_name"
575− reshape.complete().unwrap();
526+ type = "TEXT"
576
527
528+ [[actions.columns]]
529+ name = "last_name"
530+ type = "TEXT"
531+
532+ [[actions]]
533+ type = "add_index"
534+ table = "users"
535+
536+ [actions.index]
537+ name = "users_name_idx"
538+ columns = ["first_name", "last_name"]
539+ "#,
540+ );
541+
542+ test.second_migration(
543+ r#"
544+ name = "uppercase_last_name"
545+
546+ [[actions]]
547+ type = "alter_column"
548+ table = "users"
549+ column = "last_name"
550+ up = "UPPER(last_name)"
551+ down = "LOWER(last_name)"
552+ "#,
553+ );
554+
555+ test.after_completion(|db| {
577− // Make sure index still exists
556+ // Make sure index still exists
578− let result: i64 = db
557+ let result: i64 = db
579− .query(
558+ .query(
580− "
559+ "
581 SELECT COUNT(*)
560 SELECT COUNT(*)
582 FROM pg_catalog.pg_index
561 FROM pg_catalog.pg_index
583 JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
562 JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
584 WHERE pg_class.relname = 'users_name_idx'
563 WHERE pg_class.relname = 'users_name_idx'
585 ",
564 ",
586− &[],
565+ &[],
587− )
566+ )
588− .unwrap()
567+ .unwrap()
589− .first()
568+ .first()
590− .map(|row| row.get(0))
569+ .map(|row| row.get(0))
591− .unwrap();
570+ .unwrap();
592− assert_eq!(1, result, "expected index to still exist");
571+ assert_eq!(1, result, "expected index to still exist");
572+ });
573+
574+ test.run();
575+}
576+
577+#[test]
578+fn alter_column_with_unique_index() {
579+ let mut test = Test::new("Alter column with unique index");
580+
581+ test.first_migration(
582+ r#"
583+ name = "create_user_table"
584+
585+ [[actions]]
586+ type = "create_table"
587+ name = "users"
588+ primary_key = ["id"]
589+
590+ [[actions.columns]]
591+ name = "id"
592+ type = "INTEGER"
593+
594+ [[actions.columns]]
595+ name = "name"
596+ type = "TEXT"
597+
598+ [[actions]]
599+ type = "add_index"
600+ table = "users"
601+
602+ [actions.index]
603+ name = "name_idx"
604+ columns = ["name"]
605+ unique = true
606+ "#,
607+ );
608+
609+ test.second_migration(
610+ r#"
611+ name = "uppercase_name"
612+
613+ [[actions]]
614+ type = "alter_column"
615+ table = "users"
616+ column = "name"
617+ up = "UPPER(name)"
618+ down = "LOWER(name)"
619+ "#,
620+ );
621+
622+ test.after_first(|db| {
623+ db.simple_query("INSERT INTO users (id, name) VALUES (1, 'Test')")
624+ .unwrap();
625+ });
626+
627+ test.intermediate(|old_db, new_db| {
628+ // Try inserting a value which duplicates the uppercase value of an existing row
629+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (2, 'TEST')");
630+ assert!(
631+ result.is_err(),
632+ "expected duplicate insert to new schema to fail"
633+ );
634+
635+ // Try inserting a value which duplicates the lowercase value of an existing row
636+ new_db
637+ .simple_query("INSERT INTO users (id, name) VALUES (2, 'JOHN')")
638+ .unwrap();
639+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, 'john')");
640+ assert!(
641+ result.is_err(),
642+ "expected duplicate insert to old schema to fail"
643+ );
644+ });
645+
646+ test.after_completion(|db| {
647+ // Make sure index still exists
648+ let is_unique: bool = db
649+ .query(
650+ "
651+ SELECT pg_index.indisunique
652+ FROM pg_catalog.pg_index
653+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
654+ WHERE pg_class.relname = 'name_idx'
655+ ",
656+ &[],
657+ )
658+ .unwrap()
659+ .first()
660+ .map(|row| row.get("indisunique"))
661+ .unwrap();
662+ assert!(is_unique, "expected index to still be unique");
663+ });
664+
665+ test.run();
666+}
667+
668+#[test]
669+fn alter_column_with_hash_index() {
670+ let mut test = Test::new("Alter column with custom index type");
671+
672+ test.first_migration(
673+ r#"
674+ name = "create_user_table"
675+
676+ [[actions]]
677+ type = "create_table"
678+ name = "users"
679+ primary_key = ["id"]
680+
681+ [[actions.columns]]
682+ name = "id"
683+ type = "INTEGER"
684+
685+ [[actions.columns]]
686+ name = "name"
687+ type = "TEXT"
688+
689+ [[actions]]
690+ type = "add_index"
691+ table = "users"
692+
693+ [actions.index]
694+ name = "name_idx"
695+ columns = ["name"]
696+ type = "hash"
697+ "#,
698+ );
699+
700+ test.second_migration(
701+ r#"
702+ name = "uppercase_name"
703+
704+ [[actions]]
705+ type = "alter_column"
706+ table = "users"
707+ column = "name"
708+ up = "UPPER(name)"
709+ down = "LOWER(name)"
710+ "#,
711+ );
712+
713+ test.after_completion(|db| {
714+ // Make sure index still has type GIN
715+ let index_type: String = db
716+ .query(
717+ "
718+ SELECT pg_am.amname
719+ FROM pg_catalog.pg_index
720+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
721+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
722+ WHERE pg_class.relname = 'name_idx'
723+ ",
724+ &[],
725+ )
726+ .unwrap()
727+ .first()
728+ .map(|row| row.get("amname"))
729+ .unwrap();
730+ assert_eq!("hash", index_type);
731+ });
593
732
594− common::assert_cleaned_up(&mut db);
733+ test.run();
595}
734}
tests/common.rs+45−19
@@ -10,7 +10,9 @@ pub struct Test<'a> {
10
10
11 first_migration: Option<Migration>,
11 first_migration: Option<Migration>,
12 second_migration: Option<Migration>,
12 second_migration: Option<Migration>,
13+ expect_failure: bool,
13
14
15+ clear_fn: Option<fn(&mut Client) -> ()>,
14 after_first_fn: Option<fn(&mut Client) -> ()>,
16 after_first_fn: Option<fn(&mut Client) -> ()>,
15 intermediate_fn: Option<fn(&mut Client, &mut Client) -> ()>,
17 intermediate_fn: Option<fn(&mut Client, &mut Client) -> ()>,
16 after_completion_fn: Option<fn(&mut Client) -> ()>,
18 after_completion_fn: Option<fn(&mut Client) -> ()>,
@@ -34,6 +36,8 @@ impl Test<'_> {
34 new_db,
36 new_db,
35 first_migration: None,
37 first_migration: None,
36 second_migration: None,
38 second_migration: None,
39+ expect_failure: false,
40+ clear_fn: None,
37 after_first_fn: None,
41 after_first_fn: None,
38 intermediate_fn: None,
42 intermediate_fn: None,
39 after_completion_fn: None,
43 after_completion_fn: None,
@@ -46,31 +50,47 @@ impl Test<'_> {
46 self
50 self
47 }
51 }
48
52
53+ #[allow(dead_code)]
49 pub fn second_migration(&mut self, migration: &str) -> &mut Self {
54 pub fn second_migration(&mut self, migration: &str) -> &mut Self {
50 self.second_migration = Some(Self::parse_migration(migration));
55 self.second_migration = Some(Self::parse_migration(migration));
51 self
56 self
52 }
57 }
53
58
59+ #[allow(dead_code)]
60+ pub fn clear(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
61+ self.clear_fn = Some(f);
62+ self
63+ }
64+
65+ #[allow(dead_code)]
54 pub fn after_first(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
66 pub fn after_first(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
55 self.after_first_fn = Some(f);
67 self.after_first_fn = Some(f);
56 self
68 self
57 }
69 }
58
70
71+ #[allow(dead_code)]
59 pub fn intermediate(&mut self, f: fn(&mut Client, &mut Client) -> ()) -> &mut Self {
72 pub fn intermediate(&mut self, f: fn(&mut Client, &mut Client) -> ()) -> &mut Self {
60 self.intermediate_fn = Some(f);
73 self.intermediate_fn = Some(f);
61 self
74 self
62 }
75 }
63
76
77+ #[allow(dead_code)]
64 pub fn after_completion(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
78 pub fn after_completion(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
65 self.after_completion_fn = Some(f);
79 self.after_completion_fn = Some(f);
66 self
80 self
67 }
81 }
68
82
83+ #[allow(dead_code)]
69 pub fn after_abort(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
84 pub fn after_abort(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
70 self.after_abort_fn = Some(f);
85 self.after_abort_fn = Some(f);
71 self
86 self
72 }
87 }
73
88
89+ #[allow(dead_code)]
90+ pub fn expect_failure(&mut self) {
91+ self.expect_failure = true;
92+ }
93+
74 fn parse_migration(encoded: &str) -> Migration {
94 fn parse_migration(encoded: &str) -> Migration {
75 toml::from_str(encoded).unwrap()
95 toml::from_str(encoded).unwrap()
76 }
96 }
@@ -83,6 +103,7 @@ enum RunType {
83}
103}
84
104
85impl Test<'_> {
105impl Test<'_> {
106+ #[allow(dead_code)]
86 pub fn run(&mut self) {
107 pub fn run(&mut self) {
87 if self.second_migration.is_some() {
108 if self.second_migration.is_some() {
88 // Run to completion
109 // Run to completion
@@ -102,6 +123,10 @@ impl Test<'_> {
102 print_subheading("Clearing database");
123 print_subheading("Clearing database");
103 self.reshape.remove().unwrap();
124 self.reshape.remove().unwrap();
104
125
126+ if let Some(clear_fn) = self.clear_fn {
127+ clear_fn(&mut self.old_db);
128+ }
129+
105 // Apply first migration, will automatically complete
130 // Apply first migration, will automatically complete
106 print_subheading("Applying first migration");
131 print_subheading("Applying first migration");
107 let first_migration = self
132 let first_migration = self
@@ -115,6 +140,9 @@ impl Test<'_> {
115 .simple_query(&reshape::schema_query_for_migration(&first_migration.name))
140 .simple_query(&reshape::schema_query_for_migration(&first_migration.name))
116 .unwrap();
141 .unwrap();
117
142
143+ // Automatically complete first migration
144+ self.reshape.complete().unwrap();
145+
118 // Run setup function
146 // Run setup function
119 if let Some(after_first_fn) = self.after_first_fn {
147 if let Some(after_first_fn) = self.after_first_fn {
120 print_subheading("Running setup and first checks");
148 print_subheading("Running setup and first checks");
@@ -124,10 +152,21 @@ impl Test<'_> {
124
152
125 // Apply second migration
153 // Apply second migration
126 if let Some(second_migration) = &self.second_migration {
154 if let Some(second_migration) = &self.second_migration {
155+ if self.expect_failure {
127− print_subheading("Applying second migration");
156+ print_subheading("Applying second migration (expecting failure)");
157+ let result = self
158+ .reshape
159+ .migrate(vec![first_migration.clone(), second_migration.clone()]);
160+
161+ if result.is_ok() {
162+ panic!("expected second migration to fail");
163+ }
164+ } else {
165+ print_subheading("Applying second migration");
128− self.reshape
166+ self.reshape
129− .migrate(vec![first_migration.clone(), second_migration.clone()])
167+ .migrate(vec![first_migration.clone(), second_migration.clone()])
130− .unwrap();
168+ .unwrap();
169+ }
131
170
132 // Update search path
171 // Update search path
133 self.new_db
172 self.new_db
@@ -172,7 +211,7 @@ impl Test<'_> {
172}
211}
173
212
174fn print_heading(text: &str) {
213fn print_heading(text: &str) {
175− let delimiter = std::iter::repeat("=").take(50).collect::<String>();
214+ let delimiter = std::iter::repeat("=").take(80).collect::<String>();
176
215
177 println!();
216 println!();
178 println!();
217 println!();
@@ -191,7 +230,7 @@ fn print_success() {
191}
230}
192
231
193fn add_spacer(text: &str, char: &str) -> String {
232fn add_spacer(text: &str, char: &str) -> String {
194− const TARGET_WIDTH: usize = 50;
233+ const TARGET_WIDTH: usize = 80;
195 let num_of_chars = (TARGET_WIDTH - text.len() - 2) / 2;
234 let num_of_chars = (TARGET_WIDTH - text.len() - 2) / 2;
196 let spacer = std::iter::repeat(char)
235 let spacer = std::iter::repeat(char)
197 .take(num_of_chars)
236 .take(num_of_chars)
@@ -202,19 +241,6 @@ fn add_spacer(text: &str, char: &str) -> String {
202 format!("{spacer} {text} {spacer}{extra}", spacer = spacer)
241 format!("{spacer} {text} {spacer}{extra}", spacer = spacer)
203}
242}
204
243
205−pub fn setup() -> (Reshape, Client, Client) {
206− let connection_string = std::env::var("POSTGRES_CONNECTION_STRING")
207− .unwrap_or("postgres://postgres:postgres@localhost/reshape_test".to_string());
208−
209− let old_db = Client::connect(&connection_string, NoTls).unwrap();
210− let new_db = Client::connect(&connection_string, NoTls).unwrap();
211−
212− let mut reshape = Reshape::new(&connection_string).unwrap();
213− reshape.remove().unwrap();
214−
215− (reshape, old_db, new_db)
216−}
217−
218pub fn assert_cleaned_up(db: &mut Client) {
244pub fn assert_cleaned_up(db: &mut Client) {
219 // Make sure no temporary columns remain
245 // Make sure no temporary columns remain
220 let temp_columns: Vec<String> = db
246 let temp_columns: Vec<String> = db
tests/complex.rsnew file+379
@@ -0,0 +1,379 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn move_column_between_tables() {
6+ let mut test = Test::new("Move column between tables");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "email"
23+ type = "TEXT"
24+
25+ [[actions]]
26+ type = "create_table"
27+ name = "profiles"
28+ primary_key = ["id"]
29+
30+ [[actions.columns]]
31+ name = "id"
32+ type = "INTEGER"
33+
34+ [[actions.columns]]
35+ name = "user_id"
36+ type = "INTEGER"
37+ nullable = false
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "move_email_column"
44+
45+ [[actions]]
46+ type = "add_column"
47+ table = "profiles"
48+
49+ [actions.column]
50+ name = "email"
51+ type = "TEXT"
52+ nullable = false
53+
54+ # When `users` is updated in the old schema, we write the email value to `profiles`
55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated
56+ [actions.up]
57+ table = "users"
58+ value = "users.email"
59+ where = "profiles.user_id = users.id"
60+
61+ [[actions]]
62+ type = "remove_column"
63+ table = "users"
64+ column = "email"
65+
66+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
67+ [actions.down]
68+ table = "profiles"
69+ value = "profiles.email"
70+ where = "users.id = profiles.user_id"
71+ "#,
72+ );
73+
74+ test.after_first(|db| {
75+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")
76+ .unwrap();
77+ db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")
78+ .unwrap();
79+
80+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")
81+ .unwrap();
82+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")
83+ .unwrap();
84+ });
85+
86+ test.intermediate(|old_db, new_db| {
87+ // Ensure emails were backfilled into profiles
88+ let profiles_emails: Vec<String> = new_db
89+ .query(
90+ r#"
91+ SELECT email
92+ FROM profiles
93+ ORDER BY id
94+ "#,
95+ &[],
96+ )
97+ .unwrap()
98+ .iter()
99+ .map(|row| row.get("email"))
100+ .collect();
101+ assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);
102+
103+ // Ensure insert in old schema updates new
104+ old_db
105+ .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")
106+ .unwrap();
107+ old_db
108+ .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")
109+ .unwrap();
110+ let new_email: String = new_db
111+ .query("SELECT email FROM profiles WHERE id = 3", &[])
112+ .unwrap()
113+ .first()
114+ .map(|row| row.get("email"))
115+ .unwrap();
116+ assert_eq!("test3@test.com", new_email);
117+
118+ // Ensure updates in old schema updates new
119+ old_db
120+ .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")
121+ .unwrap();
122+ let new_email: String = new_db
123+ .query("SELECT email FROM profiles WHERE id = 3", &[])
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get("email"))
127+ .unwrap();
128+ assert_eq!("test3+updated@test.com", new_email);
129+
130+ // Ensure insert in new schema updates old
131+ new_db
132+ .simple_query(
133+ "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",
134+ )
135+ .unwrap();
136+ new_db
137+ .simple_query("INSERT INTO users (id) VALUES (4)")
138+ .unwrap();
139+ let old_email: String = old_db
140+ .query("SELECT email FROM users WHERE id = 4", &[])
141+ .unwrap()
142+ .first()
143+ .map(|row| row.get("email"))
144+ .unwrap();
145+ assert_eq!("test4@test.com", old_email);
146+
147+ // Ensure update in new schema updates old
148+ new_db
149+ .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")
150+ .unwrap();
151+ let old_email: String = old_db
152+ .query("SELECT email FROM users WHERE id = 4", &[])
153+ .unwrap()
154+ .first()
155+ .map(|row| row.get("email"))
156+ .unwrap();
157+ assert_eq!("test4+updated@test.com", old_email);
158+ });
159+
160+ test.run();
161+}
162+
163+#[test]
164+fn extract_relation_into_new_table() {
165+ let mut test = Test::new("Extract relation into new table");
166+
167+ test.first_migration(
168+ r#"
169+ name = "create_tables"
170+
171+ [[actions]]
172+ type = "create_table"
173+ name = "accounts"
174+ primary_key = ["id"]
175+
176+ [[actions.columns]]
177+ name = "id"
178+ type = "INTEGER"
179+
180+ [[actions]]
181+ type = "create_table"
182+ name = "users"
183+ primary_key = ["id"]
184+
185+ [[actions.columns]]
186+ name = "id"
187+ type = "INTEGER"
188+
189+ [[actions.columns]]
190+ name = "account_id"
191+ type = "INTEGER"
192+ nullable = false
193+
194+ [[actions.columns]]
195+ name = "account_role"
196+ type = "TEXT"
197+ nullable = false
198+ "#,
199+ );
200+
201+ test.second_migration(
202+ r#"
203+ name = "add_account_user_connection"
204+
205+ [[actions]]
206+ type = "create_table"
207+ name = "user_account_connections"
208+ primary_key = ["account_id", "user_id"]
209+
210+ [[actions.columns]]
211+ name = "account_id"
212+ type = "INTEGER"
213+
214+ [[actions.columns]]
215+ name = "user_id"
216+ type = "INTEGER"
217+
218+ [[actions.columns]]
219+ name = "role"
220+ type = "TEXT"
221+ nullable = false
222+
223+ [actions.up]
224+ table = "users"
225+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
226+ where = "user_account_connections.user_id = users.id"
227+
228+ [[actions]]
229+ type = "remove_column"
230+ table = "users"
231+ column = "account_id"
232+
233+ [actions.down]
234+ table = "user_account_connections"
235+ value = "user_account_connections.account_id"
236+ where = "users.id = user_account_connections.user_id"
237+
238+ [[actions]]
239+ type = "remove_column"
240+ table = "users"
241+ column = "account_role"
242+
243+ [actions.down]
244+ table = "user_account_connections"
245+ value = "LOWER(user_account_connections.role)"
246+ where = "users.id = user_account_connections.user_id"
247+ "#,
248+ );
249+
250+ test.after_first(|db| {
251+ db.simple_query("INSERT INTO accounts (id) VALUES (1)")
252+ .unwrap();
253+ db.simple_query("INSERT INTO users (id, account_id, account_role) VALUES (1, 1, 'admin')")
254+ .unwrap();
255+ });
256+
257+ test.intermediate(|old_db, new_db| {
258+ // Ensure connections was backfilled
259+ let rows: Vec<(i32, i32, String)> = new_db
260+ .query(
261+ "
262+ SELECT account_id, user_id, role
263+ FROM user_account_connections
264+ ",
265+ &[],
266+ )
267+ .unwrap()
268+ .iter()
269+ .map(|row| (row.get("account_id"), row.get("user_id"), row.get("role")))
270+ .collect();
271+ assert_eq!(1, rows.len());
272+
273+ let row = rows.first().unwrap();
274+ assert_eq!(1, row.0);
275+ assert_eq!(1, row.1);
276+ assert_eq!("ADMIN", row.2);
277+
278+ // Ensure inserted user in old schema creates a new connection
279+ old_db
280+ .simple_query(
281+ "INSERT INTO users (id, account_id, account_role) VALUES (2, 1, 'developer')",
282+ )
283+ .unwrap();
284+ assert!(
285+ new_db
286+ .query(
287+ "
288+ SELECT account_id, user_id, role
289+ FROM user_account_connections
290+ WHERE account_id = 1 AND user_id = 2 AND role = 'DEVELOPER'
291+ ",
292+ &[],
293+ )
294+ .unwrap()
295+ .len()
296+ == 1
297+ );
298+
299+ // Ensure NOT NULL constraint still applies to old schema
300+ let result = old_db
301+ .simple_query(
302+ "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",
303+ );
304+ assert!(result.is_err());
305+
306+ // Ensure updated user role in old schema updates connection in new schema
307+ old_db
308+ .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")
309+ .unwrap();
310+ assert!(
311+ new_db
312+ .query(
313+ "
314+ SELECT account_id, user_id, role
315+ FROM user_account_connections
316+ WHERE account_id = 1 AND user_id = 2 AND role = 'ADMIN'
317+ ",
318+ &[],
319+ )
320+ .unwrap()
321+ .len()
322+ == 1
323+ );
324+
325+ // Ensure updated connection in new schema updates old schema user
326+ new_db
327+ .simple_query(
328+ "UPDATE user_account_connections SET role = 'DEVELOPER' WHERE account_id = 1 AND user_id = 2",
329+ )
330+ .unwrap();
331+ assert!(
332+ old_db
333+ .query(
334+ "
335+ SELECT id
336+ FROM users
337+ WHERE id = 2 AND account_id = 1 AND account_role = 'developer'
338+ ",
339+ &[],
340+ )
341+ .unwrap()
342+ .len()
343+ == 1
344+ );
345+
346+ // Ensure insert of user with connection through new schema updates user in old schema
347+ new_db
348+ .simple_query(
349+ r#"
350+ BEGIN;
351+ INSERT INTO users (id) VALUES (3);
352+ INSERT INTO user_account_connections (user_id, account_id, role) VALUES (3, 1, 'DEVELOPER');
353+ COMMIT;
354+ "#,
355+ )
356+ .unwrap();
357+ new_db
358+ .simple_query(
359+ "",
360+ )
361+ .unwrap();
362+ assert!(
363+ old_db
364+ .query(
365+ "
366+ SELECT id
367+ FROM users
368+ WHERE id = 3 AND account_id = 1 AND account_role = 'developer'
369+ ",
370+ &[],
371+ )
372+ .unwrap()
373+ .len()
374+ == 1
375+ );
376+ });
377+
378+ test.run();
379+}
tests/create_enum.rsnew file+41
@@ -0,0 +1,41 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn create_enum() {
6+ let mut test = Test::new("Create enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum_and_table"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+
17+ [[actions]]
18+ type = "create_table"
19+ name = "updates"
20+ primary_key = ["id"]
21+
22+ [[actions.columns]]
23+ name = "id"
24+ type = "INTEGER"
25+
26+ [[actions.columns]]
27+ name = "status"
28+ type = "mood"
29+ "#,
30+ );
31+
32+ test.after_first(|db| {
33+ // Valid enum values should succeed
34+ db.simple_query(
35+ "INSERT INTO updates (id, status) VALUES (1, 'happy'), (2, 'ok'), (3, 'sad')",
36+ )
37+ .unwrap();
38+ });
39+
40+ test.run();
41+}
tests/create_table.rs+81−83
@@ -1,10 +1,9 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, ForeignKey, Migration};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn create_table() {
5fn create_table() {
7− let mut test = common::Test::new("Create table");
6+ let mut test = Test::new("Create table");
8
7
9 test.first_migration(
8 test.first_migration(
10 r#"
9 r#"
@@ -113,90 +112,89 @@ fn create_table() {
113
112
114#[test]
113#[test]
115fn create_table_with_foreign_keys() {
114fn create_table_with_foreign_keys() {
116− let (mut reshape, mut db, _) = common::setup();
115+ let mut test = Test::new("Create table");
117−
116+
118− let create_table_migration = Migration::new("create_users_table", None).with_action(
117+ test.first_migration(
119− CreateTableBuilder::default()
118+ r#"
119+ name = "create_users_table"
120+
121+ [[actions]]
122+ type = "create_table"
120− .name("users")
123+ name = "users"
121− .primary_key(vec!["id".to_string()])
124+ primary_key = ["id"]
122− .columns(vec![ColumnBuilder::default()
125+
126+ [[actions.columns]]
123− .name("id")
127+ name = "id"
124− .data_type("INTEGER")
128+ type = "INTEGER"
125− .generated("ALWAYS AS IDENTITY")
129+ generated = "ALWAYS AS IDENTITY"
126− .build()
130+
127− .unwrap()])
131+ [[actions.columns]]
128− .build()
132+ name = "name"
129− .unwrap(),
133+ type = "TEXT"
130− );
134+
135+ [[actions.columns]]
136+ name = "created_at"
137+ type = "TIMESTAMP"
138+ nullable = false
139+ default = "NOW()"
131
140
132− let create_second_table_migration = Migration::new("create_items_table", None).with_action(
141+ [[actions]]
133− CreateTableBuilder::default()
142+ type = "create_table"
134− .name("items")
143+ name = "items"
135− .primary_key(vec!["id".to_string()])
144+ primary_key = ["id"]
145+
146+ [[actions.columns]]
147+ name = "id"
148+ type = "INTEGER"
149+
150+ [[actions.columns]]
151+ name = "user_id"
152+ type = "INTEGER"
153+ nullable = false
154+
136− .foreign_keys(vec![ForeignKey {
155+ [[actions.foreign_keys]]
137− columns: vec!["user_id".to_string()],
156+ columns = ["user_id"]
138− referenced_table: "users".to_string(),
157+ referenced_table = "users"
139− referenced_columns: vec!["id".to_string()],
158+ referenced_columns = ["id"]
140− }])
159+ "#,
141− .columns(vec![
142− ColumnBuilder::default()
143− .name("id")
144− .data_type("INTEGER")
145− .generated("ALWAYS AS IDENTITY")
146− .build()
147− .unwrap(),
148− ColumnBuilder::default()
149− .name("user_id")
150− .data_type("INTEGER")
151− .nullable(false)
152− .build()
153− .unwrap(),
154− ])
155− .build()
156− .unwrap(),
157 );
160 );
158
161
159− reshape
162+ test.after_first(|db| {
160− .migrate(vec![
161− create_table_migration.clone(),
162− create_second_table_migration.clone(),
163− ])
164− .unwrap();
165−
166− let foreign_key_columns: Vec<(String, String, String)> = db
163+ let foreign_key_columns: Vec<(String, String, String)> = db
167− .query(
164+ .query(
168− "
165+ "
169− SELECT
166+ SELECT
170− kcu.column_name,
167+ kcu.column_name,
171− ccu.table_name AS foreign_table_name,
168+ ccu.table_name AS foreign_table_name,
172− ccu.column_name AS foreign_column_name
169+ ccu.column_name AS foreign_column_name
173− FROM
170+ FROM
174− information_schema.table_constraints AS tc
171+ information_schema.table_constraints AS tc
175− JOIN information_schema.key_column_usage AS kcu
172+ JOIN information_schema.key_column_usage AS kcu
176− ON tc.constraint_name = kcu.constraint_name
173+ ON tc.constraint_name = kcu.constraint_name
177− AND tc.table_schema = kcu.table_schema
174+ AND tc.table_schema = kcu.table_schema
178− JOIN information_schema.constraint_column_usage AS ccu
175+ JOIN information_schema.constraint_column_usage AS ccu
179− ON ccu.constraint_name = tc.constraint_name
176+ ON ccu.constraint_name = tc.constraint_name
180− AND ccu.table_schema = tc.table_schema
177+ AND ccu.table_schema = tc.table_schema
181− WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
178+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
182− ",
179+ ",
183− &[],
180+ &[],
184− )
185− .unwrap()
186− .iter()
187− .map(|row| {
188− (
189− row.get("column_name"),
190− row.get("foreign_table_name"),
191− row.get("foreign_column_name"),
192 )
181 )
182+ .unwrap()
183+ .iter()
184+ .map(|row| {
185+ (
186+ row.get("column_name"),
187+ row.get("foreign_table_name"),
188+ row.get("foreign_column_name"),
189+ )
193− })
190+ })
194− .collect();
191+ .collect();
195
192
196− assert_eq!(
193+ assert_eq!(
197− vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
194+ vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
198− foreign_key_columns
195+ foreign_key_columns
196+ );
199− );
197+ });
200
198
201− common::assert_cleaned_up(&mut db);
199+ test.run();
202}
200}
tests/custom.rsnew file+128
@@ -0,0 +1,128 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn custom_enable_extension() {
6+ let mut test = Test::new("Custom migration");
7+
8+ test.clear(|db| {
9+ db.simple_query(
10+ "
11+ DROP EXTENSION IF EXISTS bloom;
12+ DROP EXTENSION IF EXISTS btree_gin;
13+ DROP EXTENSION IF EXISTS btree_gist;
14+ ",
15+ )
16+ .unwrap();
17+ });
18+
19+ test.first_migration(
20+ r#"
21+ name = "empty_migration"
22+
23+ [[actions]]
24+ type = "custom"
25+ "#,
26+ );
27+
28+ test.second_migration(
29+ r#"
30+ name = "enable_extensions"
31+
32+ [[actions]]
33+ type = "custom"
34+
35+ start = """
36+ CREATE EXTENSION IF NOT EXISTS bloom;
37+ CREATE EXTENSION IF NOT EXISTS btree_gin;
38+ """
39+
40+ complete = "CREATE EXTENSION IF NOT EXISTS btree_gist"
41+
42+ abort = """
43+ DROP EXTENSION IF EXISTS bloom;
44+ DROP EXTENSION IF EXISTS btree_gin;
45+ """
46+ "#,
47+ );
48+
49+ test.intermediate(|db, _| {
50+ let bloom_activated = !db
51+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
52+ .unwrap()
53+ .is_empty();
54+ assert!(bloom_activated);
55+
56+ let btree_gin_activated = !db
57+ .query(
58+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
59+ &[],
60+ )
61+ .unwrap()
62+ .is_empty();
63+ assert!(btree_gin_activated);
64+
65+ let btree_gist_activated = !db
66+ .query(
67+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
68+ &[],
69+ )
70+ .unwrap()
71+ .is_empty();
72+ assert!(!btree_gist_activated);
73+ });
74+
75+ test.after_completion(|db| {
76+ let bloom_activated = !db
77+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
78+ .unwrap()
79+ .is_empty();
80+ assert!(bloom_activated);
81+
82+ let btree_gin_activated = !db
83+ .query(
84+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
85+ &[],
86+ )
87+ .unwrap()
88+ .is_empty();
89+ assert!(btree_gin_activated);
90+
91+ let btree_gist_activated = !db
92+ .query(
93+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
94+ &[],
95+ )
96+ .unwrap()
97+ .is_empty();
98+ assert!(btree_gist_activated);
99+ });
100+
101+ test.after_abort(|db| {
102+ let bloom_activated = !db
103+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
104+ .unwrap()
105+ .is_empty();
106+ assert!(!bloom_activated);
107+
108+ let btree_gin_activated = !db
109+ .query(
110+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
111+ &[],
112+ )
113+ .unwrap()
114+ .is_empty();
115+ assert!(!btree_gin_activated);
116+
117+ let btree_gist_activated = !db
118+ .query(
119+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
120+ &[],
121+ )
122+ .unwrap()
123+ .is_empty();
124+ assert!(!btree_gist_activated);
125+ });
126+
127+ test.run();
128+}
tests/failure.rs+34−58
@@ -1,75 +1,51 @@
1−use reshape::migrations::{AddColumn, Column, ColumnBuilder, CreateTableBuilder, Migration};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn invalid_migration() {
5fn invalid_migration() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Invalid migration");
7+
8+ test.first_migration(
9+ r#"
10+ name = "invalid_migration"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
8
16
9− let create_users_table = Migration::new("create_users_table", None).with_action(
17+ [[actions.columns]]
10− CreateTableBuilder::default()
18+ name = "id"
11− .name("users")
19+ type = "INTEGER"
12− .primary_key(vec!["id".to_string()])
20+ "#,
13− .columns(vec![
14− ColumnBuilder::default()
15− .name("id")
16− .data_type("INTEGER")
17− .build()
18− .unwrap(),
19− ColumnBuilder::default()
20− .name("name")
21− .data_type("TEXT")
22− .build()
23− .unwrap(),
24− ])
25− .build()
26− .unwrap(),
27 );
21 );
28− let add_first_column = Migration::new("add_first_column", None).with_action(AddColumn {
29− table: "users".to_string(),
30− column: Column {
31− name: "first".to_string(),
32− data_type: "TEXT".to_string(),
33− nullable: false,
34− default: None,
35− generated: None,
36− },
37− up: Some("INVALID SQL".to_string()),
38− });
39
22
40− let first_migrations = vec![create_users_table.clone()];
23+ test.second_migration(
41− let second_migrations = vec![create_users_table.clone(), add_first_column.clone()];
24+ r#"
25+ name = "add_invalid_column"
42
26
43− // Run first migration, should automatically finish
27+ [[actions]]
44− reshape.migrate(first_migrations.clone()).unwrap();
28+ type = "add_column"
29+ table = "users"
45
30
46− // Update search paths
31+ up = "INVALID SQL"
47− old_db
32+
48− .simple_query(&reshape::schema_query_for_migration(
33+ [actions.column]
49− &first_migrations.last().unwrap().name,
34+ name = "first"
50− ))
35+ type = "TEXT"
51− .unwrap();
36+ "#,
52− new_db
37+ );
53− .simple_query(&reshape::schema_query_for_migration(
54− &first_migrations.last().unwrap().name,
55− ))
56− .unwrap();
57
38
58 // Insert a test user
39 // Insert a test user
59− new_db
40+ test.after_first(|db| {
60− .simple_query(
41+ db.simple_query(
61 "
42 "
62− INSERT INTO users (id, name) VALUES
43+ INSERT INTO users (id) VALUES (1)
63− (1, 'John Doe')
64 ",
44 ",
65 )
45 )
66 .unwrap();
46 .unwrap();
47+ });
67
48
68− // Run second migration and ensure it fails
49+ test.expect_failure();
69− assert!(
50+ test.run();
70− reshape.migrate(second_migrations.clone()).is_err(),
71− "invalid migration succeeded"
72− );
73−
74− common::assert_cleaned_up(&mut new_db);
75}
51}
tests/remove_column.rs+209−73
@@ -1,81 +1,217 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RemoveColumn};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn remove_column() {
5fn remove_column() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Remove column");
8−
7+
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
8+ test.first_migration(
10− CreateTableBuilder::default()
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
11− .name("users")
14+ name = "users"
12− .primary_key(vec!["id".to_string()])
15+ primary_key = ["id"]
16+
13− .columns(vec![
17+ [[actions.columns]]
14− ColumnBuilder::default()
15− .name("id")
18+ name = "id"
16− .data_type("INTEGER")
19+ type = "INTEGER"
17− .build()
20+
18− .unwrap(),
21+ [[actions.columns]]
19− ColumnBuilder::default()
20− .name("name")
22+ name = "name"
21− .data_type("TEXT")
23+ type = "TEXT"
22− .nullable(false)
24+ "#,
23− .build()
25+ );
24− .unwrap(),
26+
25− ])
27+ test.second_migration(
26− .build()
28+ r#"
27− .unwrap(),
29+ name = "remove_name_column"
30+
31+ [[actions]]
32+ type = "remove_column"
33+ table = "users"
34+ column = "name"
35+ down = "'TEST_DOWN_VALUE'"
36+ "#,
37+ );
38+
39+ test.intermediate(|old_db, new_db| {
40+ // Insert using old schema and ensure it can be retrieved through new schema
41+ old_db
42+ .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
43+ .unwrap();
44+ let results = new_db
45+ .query("SELECT id FROM users WHERE id = 1", &[])
46+ .unwrap();
47+ assert_eq!(1, results.len());
48+ assert_eq!(1, results[0].get::<_, i32>("id"));
49+
50+ // Ensure the name column is not accesible through the new schema
51+ assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
52+
53+ // Insert using new schema and ensure the down function is correctly applied
54+ new_db
55+ .simple_query("INSERT INTO users(id) VALUES (2)")
56+ .unwrap();
57+ let result = old_db
58+ .query_opt("SELECT name FROM users WHERE id = 2", &[])
59+ .unwrap();
60+ assert_eq!(
61+ Some("TEST_DOWN_VALUE"),
62+ result.as_ref().map(|row| row.get("name"))
63+ );
64+ });
65+
66+ test.run();
67+}
68+
69+#[test]
70+fn remove_column_with_index() {
71+ let mut test = Test::new("Remove column");
72+
73+ test.first_migration(
74+ r#"
75+ name = "create_user_table"
76+
77+ [[actions]]
78+ type = "create_table"
79+ name = "users"
80+ primary_key = ["id"]
81+
82+ [[actions.columns]]
83+ name = "id"
84+ type = "INTEGER"
85+
86+ [[actions.columns]]
87+ name = "name"
88+ type = "TEXT"
89+
90+ [[actions]]
91+ type = "add_index"
92+ table = "users"
93+
94+ [actions.index]
95+ name = "name_idx"
96+ columns = ["name"]
97+ "#,
98+ );
99+
100+ test.second_migration(
101+ r#"
102+ name = "remove_name_column"
103+
104+ [[actions]]
105+ type = "remove_column"
106+ table = "users"
107+ column = "name"
108+ down = "'TEST_DOWN_VALUE'"
109+ "#,
110+ );
111+
112+ test.after_completion(|db| {
113+ // Ensure index has been removed after the migration is complete
114+ let count: i64 = db
115+ .query(
116+ "
117+ SELECT COUNT(*)
118+ FROM pg_catalog.pg_index
119+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
120+ WHERE pg_class.relname = 'name_idx'
121+ ",
122+ &[],
123+ )
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get(0))
127+ .unwrap();
128+
129+ assert_eq!(0, count, "expected index to not exist");
130+ });
131+
132+ test.run();
133+}
134+
135+#[test]
136+fn remove_column_with_complex_down() {
137+ let mut test = Test::new("Remove column complex");
138+
139+ test.first_migration(
140+ r#"
141+ name = "create_tables"
142+
143+ [[actions]]
144+ type = "create_table"
145+ name = "users"
146+ primary_key = ["id"]
147+
148+ [[actions.columns]]
149+ name = "id"
150+ type = "INTEGER"
151+
152+ [[actions.columns]]
153+ name = "email"
154+ type = "TEXT"
155+
156+ [[actions]]
157+ type = "create_table"
158+ name = "profiles"
159+ primary_key = ["user_id"]
160+
161+ [[actions.columns]]
162+ name = "user_id"
163+ type = "INTEGER"
164+
165+ [[actions.columns]]
166+ name = "email"
167+ type = "TEXT"
168+ "#,
28 );
169 );
29− let remove_column_migration =
170+
30− Migration::new("remove_name_column", None).with_action(RemoveColumn {
171+ test.second_migration(
31− table: "users".to_string(),
172+ r#"
32− column: "name".to_string(),
173+ name = "remove_users_email_column"
33− down: Some("'TEST_DOWN_VALUE'".to_string()),
174+
34− });
175+ [[actions]]
35−
176+ type = "remove_column"
36− let first_migrations = vec![create_table_migration.clone()];
177+ table = "users"
37− let second_migrations = vec![
178+ column = "email"
38− create_table_migration.clone(),
179+
39− remove_column_migration.clone(),
180+ [actions.down]
40− ];
181+ table = "profiles"
41−
182+ value = "profiles.email"
42− // Run migrations
183+ where = "users.id = profiles.user_id"
43− reshape.migrate(first_migrations.clone()).unwrap();
184+ "#,
44− reshape.migrate(second_migrations.clone()).unwrap();
45−
46− // Update schemas of Postgres connections
47− let old_schema_query =
48− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
49− let new_schema_query =
50− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
51− old_db.simple_query(&old_schema_query).unwrap();
52− new_db.simple_query(&new_schema_query).unwrap();
53−
54− // Insert using old schema and ensure it can be retrieved through new schema
55− old_db
56− .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
57− .unwrap();
58− let results = new_db
59− .query("SELECT id FROM users WHERE id = 1", &[])
60− .unwrap();
61− assert_eq!(1, results.len());
62− assert_eq!(1, results[0].get::<_, i32>("id"));
63−
64− // Ensure the name column is not accesible through the new schema
65− assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
66−
67− // Insert using new schema and ensure the down function is correctly applied
68− new_db
69− .simple_query("INSERT INTO users(id) VALUES (2)")
70− .unwrap();
71− let result = old_db
72− .query_opt("SELECT name FROM users WHERE id = 2", &[])
73− .unwrap();
74− assert_eq!(
75− Some("TEST_DOWN_VALUE"),
76− result.as_ref().map(|row| row.get("name"))
77 );
185 );
78
186
187+ test.after_first(|db| {
188+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
189+ .unwrap();
190+ db.simple_query("INSERT INTO profiles (user_id, email) VALUES (1, 'test@example.com')")
79− reshape.complete().unwrap();
191+ .unwrap();
80− common::assert_cleaned_up(&mut new_db);
192+ });
193+
194+ test.intermediate(|old_db, new_db| {
195+ new_db
196+ .simple_query("UPDATE profiles SET email = 'test2@example.com' WHERE user_id = 1")
197+ .unwrap();
198+
199+ // Ensure new email was propagated to users table in old schema
200+ let email: String = old_db
201+ .query(
202+ "
203+ SELECT email
204+ FROM users
205+ WHERE id = 1
206+ ",
207+ &[],
208+ )
209+ .unwrap()
210+ .first()
211+ .map(|row| row.get("email"))
212+ .unwrap();
213+ assert_eq!("test2@example.com", email);
214+ });
215+
216+ test.run();
81}
217}
tests/remove_enum.rsnew file+69
@@ -0,0 +1,69 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_enum() {
6+ let mut test = Test::new("Remove enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+ "#,
17+ );
18+
19+ test.second_migration(
20+ r#"
21+ name = "remove_enum"
22+
23+ [[actions]]
24+ type = "remove_enum"
25+ enum = "mood"
26+ "#,
27+ );
28+
29+ test.after_first(|db| {
30+ // Ensure enum was created
31+ let enum_exists = !db
32+ .query(
33+ "
34+ SELECT typname
35+ FROM pg_catalog.pg_type
36+ WHERE typcategory = 'E'
37+ AND typname = 'mood'
38+ ",
39+ &[],
40+ )
41+ .unwrap()
42+ .is_empty();
43+
44+ assert!(enum_exists, "expected mood enum to have been created");
45+ });
46+
47+ test.after_completion(|db| {
48+ // Ensure enum was removed after completion
49+ let enum_does_not_exist = db
50+ .query(
51+ "
52+ SELECT typname
53+ FROM pg_catalog.pg_type
54+ WHERE typcategory = 'E'
55+ AND typname = 'mood'
56+ ",
57+ &[],
58+ )
59+ .unwrap()
60+ .is_empty();
61+
62+ assert!(
63+ enum_does_not_exist,
64+ "expected mood enum to have been removed"
65+ );
66+ });
67+
68+ test.run();
69+}
tests/remove_foreign_key.rsnew file+124
@@ -0,0 +1,124 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_foreign_key() {
6+ let mut test = Test::new("Remove foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+
34+ [[actions.foreign_keys]]
35+ columns = ["user_id"]
36+ referenced_table = "users"
37+ referenced_columns = ["id"]
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "remove_foreign_key"
44+
45+ [[actions]]
46+ type = "remove_foreign_key"
47+ table = "items"
48+ foreign_key = "items_user_id_fkey"
49+ "#,
50+ );
51+
52+ test.after_first(|db| {
53+ // Insert some test users
54+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
55+ .unwrap();
56+ });
57+
58+ test.intermediate(|old_db, new_db| {
59+ // Ensure items can't be inserted if they don't reference valid users
60+ // The foreign key is only removed when the migration is completed so
61+ // it should still be enforced for the new and old schema.
62+ let result = old_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
63+ assert!(
64+ result.is_err(),
65+ "expected insert against old schema to fail"
66+ );
67+
68+ let result = new_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
69+ assert!(
70+ result.is_err(),
71+ "expected insert against new schema to fail"
72+ );
73+ });
74+
75+ test.after_completion(|db| {
76+ // Ensure items can be inserted even if they don't reference valid users
77+ let result = db
78+ .simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)")
79+ .map(|_| ());
80+ assert!(
81+ result.is_ok(),
82+ "expected insert to not fail, got {:?}",
83+ result
84+ );
85+
86+ // Ensure foreign key doesn't exist
87+ let foreign_keys = db
88+ .query(
89+ "
90+ SELECT tc.constraint_name
91+ FROM information_schema.table_constraints AS tc
92+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
93+ ",
94+ &[],
95+ )
96+ .unwrap();
97+ assert!(
98+ foreign_keys.is_empty(),
99+ "expected no foreign keys to exist on items table"
100+ );
101+ });
102+
103+ test.after_abort(|db| {
104+ // Ensure items can't be inserted if they don't reference valid users
105+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
106+ assert!(result.is_err(), "expected insert to fail");
107+
108+ // Ensure foreign key still exists
109+ let fk_exists = !db
110+ .query(
111+ "
112+ SELECT tc.constraint_name
113+ FROM information_schema.table_constraints AS tc
114+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
115+ ",
116+ &[],
117+ )
118+ .unwrap()
119+ .is_empty();
120+ assert!(fk_exists);
121+ });
122+
123+ test.run()
124+}
tests/remove_index.rs+68−97
@@ -1,115 +1,86 @@
1−use reshape::migrations::{
2− AddIndex, ColumnBuilder, CreateTableBuilder, Index, Migration, RemoveIndex,
3−};
4−
5mod common;
1mod common;
2+use common::Test;
6
3
7#[test]
4#[test]
8fn remove_index() {
5fn remove_index() {
9− let (mut reshape, _, mut db) = common::setup();
6+ let mut test = Test::new("Remove index");
10
7
11− let create_table_migration = Migration::new("create_user_table", None)
8+ test.first_migration(
12− .with_action(
9+ r#"
13− CreateTableBuilder::default()
10+ name = "create_users_table"
14− .name("users")
15− .primary_key(vec!["id".to_string()])
16− .columns(vec![
17− ColumnBuilder::default()
18− .name("id")
19− .data_type("INTEGER")
20− .build()
21− .unwrap(),
22− ColumnBuilder::default()
23− .name("name")
24− .data_type("TEXT")
25− .build()
26− .unwrap(),
27− ])
28− .build()
29− .unwrap(),
30− )
31− .with_action(AddIndex {
32− table: "users".to_string(),
33− index: Index {
34− name: "name_idx".to_string(),
35− columns: vec!["name".to_string()],
36− unique: false,
37− index_type: None,
38− },
39− });
40
11
41− let remove_index_migration =
12+ [[actions]]
42− Migration::new("remove_name_index", None).with_action(RemoveIndex {
13+ type = "create_table"
43− index: "name_idx".to_string(),
14+ name = "users"
44− });
15+ primary_key = ["id"]
45
16
46− let first_migrations = vec![create_table_migration.clone()];
17+ [[actions.columns]]
47− let second_migrations = vec![
18+ name = "id"
48− create_table_migration.clone(),
19+ type = "INTEGER"
49− remove_index_migration.clone(),
50− ];
51
20
52− // Run migrations
21+ [[actions.columns]]
53− reshape.migrate(first_migrations.clone()).unwrap();
22+ name = "name"
23+ type = "TEXT"
54
24
55− // Ensure index is valid and ready
25+ [[actions]]
56− let result: Vec<(bool, bool)> = db
26+ type = "add_index"
57− .query(
27+ table = "users"
58− "
59− SELECT pg_index.indisready, pg_index.indisvalid
60− FROM pg_catalog.pg_index
61− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
62− WHERE pg_class.relname = 'name_idx'
63− ",
64− &[],
65− )
66− .unwrap()
67− .iter()
68− .map(|row| (row.get("indisready"), row.get("indisvalid")))
69− .collect();
70
28
71− assert_eq!(vec![(true, true)], result);
29+ [actions.index]
30+ name = "name_idx"
31+ columns = ["name"]
32+ "#,
33+ );
72
34
73− // Run migration to remove index
35+ test.second_migration(
74− reshape.migrate(second_migrations.clone()).unwrap();
36+ r#"
37+ name = "remove_name_index"
75
38
76− // Ensure index is still valid and ready during the migration
39+ [[actions]]
77− let result: Vec<(bool, bool)> = db
40+ type = "remove_index"
78− .query(
41+ index = "name_idx"
79− "
42+ "#,
80− SELECT pg_index.indisready, pg_index.indisvalid
43+ );
81− FROM pg_catalog.pg_index
82− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
83− WHERE pg_class.relname = 'name_idx'
84− ",
85− &[],
86− )
87− .unwrap()
88− .iter()
89− .map(|row| (row.get("indisready"), row.get("indisvalid")))
90− .collect();
91
44
92− assert_eq!(vec![(true, true)], result);
45+ test.intermediate(|db, _| {
46+ // Ensure index is still valid and ready during the migration
47+ let result: Vec<(bool, bool)> = db
48+ .query(
49+ "
50+ SELECT pg_index.indisready, pg_index.indisvalid
51+ FROM pg_catalog.pg_index
52+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
53+ WHERE pg_class.relname = 'name_idx'
54+ ",
55+ &[],
56+ )
57+ .unwrap()
58+ .iter()
59+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
60+ .collect();
93
61
94− reshape.complete().unwrap();
62+ assert_eq!(vec![(true, true)], result);
63+ });
95
64
65+ test.after_completion(|db| {
96− // Ensure index has been removed after the migration is complete
66+ // Ensure index has been removed after the migration is complete
97− let count: i64 = db
67+ let count: i64 = db
98− .query(
68+ .query(
99− "
69+ "
100− SELECT COUNT(*)
70+ SELECT COUNT(*)
101− FROM pg_catalog.pg_index
71+ FROM pg_catalog.pg_index
102− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
72+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
103− WHERE pg_class.relname = 'name_idx'
73+ WHERE pg_class.relname = 'name_idx'
104− ",
74+ ",
105− &[],
75+ &[],
106− )
76+ )
107− .unwrap()
77+ .unwrap()
108− .first()
78+ .first()
109− .map(|row| row.get(0))
79+ .map(|row| row.get(0))
110− .unwrap();
80+ .unwrap();
111
81
112− assert_eq!(0, count, "expected index to not exist");
82+ assert_eq!(0, count, "expected index to not exist");
83+ });
113
84
114− common::assert_cleaned_up(&mut db);
85+ test.run();
115}
86}
tests/remove_table.rs+38−48
@@ -1,54 +1,44 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RemoveTable};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn remove_table() {
5fn remove_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Remove table");
8−
7+
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
8+ test.first_migration(
10− CreateTableBuilder::default()
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
11− .name("users")
14+ name = "users"
12− .primary_key(vec!["id".to_string()])
15+ primary_key = ["id"]
13− .columns(vec![ColumnBuilder::default()
16+
17+ [[actions.columns]]
14− .name("id")
18+ name = "id"
15− .data_type("INTEGER")
19+ type = "INTEGER"
16− .build()
20+ "#,
17− .unwrap()])
21+ );
18− .build()
22+
19− .unwrap(),
23+ test.second_migration(
24+ r#"
25+ name = "remove_users_table"
26+
27+ [[actions]]
28+ type = "remove_table"
29+ table = "users"
30+ "#,
20 );
31 );
21− let remove_table_migration =
32+
22− Migration::new("remove_users_table", None).with_action(RemoveTable {
33+ test.intermediate(|old_db, new_db| {
23− table: "users".to_string(),
24− });
25−
26− let first_migrations = vec![create_table_migration.clone()];
27− let second_migrations = vec![
28− create_table_migration.clone(),
29− remove_table_migration.clone(),
30− ];
31−
32− // Run migrations
33− reshape.migrate(first_migrations.clone()).unwrap();
34− reshape.migrate(second_migrations.clone()).unwrap();
35−
36− // Update schemas of Postgres connections
37− let old_schema_query =
38− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
39− let new_schema_query =
40− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
41− old_db.simple_query(&old_schema_query).unwrap();
42− new_db.simple_query(&new_schema_query).unwrap();
43−
44− // Make sure inserts work against the old schema
34+ // Make sure inserts work against the old schema
45− old_db
35+ old_db
46− .simple_query("INSERT INTO users(id) VALUES (1)")
36+ .simple_query("INSERT INTO users(id) VALUES (1)")
47− .unwrap();
37+ .unwrap();
48−
38+
49− // Ensure the table is not accessible through the new schema
39+ // Ensure the table is not accessible through the new schema
50− assert!(new_db.query("SELECT id FROM users", &[]).is_err());
40+ assert!(new_db.query("SELECT id FROM users", &[]).is_err());
51−
41+ });
52− reshape.complete().unwrap();
42+
53− common::assert_cleaned_up(&mut new_db);
43+ test.run();
54}
44}
tests/rename_table.rs+56−66
@@ -1,80 +1,70 @@
1−use reshape::migrations::{ColumnBuilder, CreateTableBuilder, Migration, RenameTable};
2−
3mod common;
1mod common;
2+use common::Test;
4
3
5#[test]
4#[test]
6fn rename_table() {
5fn rename_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
6+ let mut test = Test::new("Rename table");
8−
9− let create_table_migration = Migration::new("create_user_table", None).with_action(
10− CreateTableBuilder::default()
11− .name("users")
12− .primary_key(vec!["id".to_string()])
13− .columns(vec![ColumnBuilder::default()
14− .name("id")
15− .data_type("INTEGER")
16− .build()
17− .unwrap()])
18− .build()
19− .unwrap(),
20− );
21− let rename_table_migration = Migration::new("rename_users_table_to_customers", None)
22− .with_action(RenameTable {
23− table: "users".to_string(),
24− new_name: "customers".to_string(),
25− });
26
7
27− let first_migrations = vec![create_table_migration.clone()];
8+ test.first_migration(
28− let second_migrations = vec![
9+ r#"
29− create_table_migration.clone(),
10+ name = "create_users_table"
30− rename_table_migration.clone(),
11+
31− ];
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
32
16
33− // Run migrations
17+ [[actions.columns]]
34− reshape.migrate(first_migrations.clone()).unwrap();
18+ name = "id"
35− reshape.migrate(second_migrations.clone()).unwrap();
19+ type = "INTEGER"
20+ "#,
21+ );
36
22
37− // Update schemas of Postgres connections
23+ test.second_migration(
38− let old_schema_query =
24+ r#"
39− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
25+ name = "rename_users_table_to_customers"
40− let new_schema_query =
41− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
42− old_db.simple_query(&old_schema_query).unwrap();
43− new_db.simple_query(&new_schema_query).unwrap();
44
26
45− // Make sure inserts work using both the old and new name
27+ [[actions]]
46− old_db
28+ type = "rename_table"
47− .simple_query("INSERT INTO users(id) VALUES (1)")
29+ table = "users"
48− .unwrap();
30+ new_name = "customers"
49− new_db
31+ "#,
50− .simple_query("INSERT INTO customers(id) VALUES (2)")
32+ );
51− .unwrap();
52
33
34+ test.intermediate(|old_db, new_db| {
53− // Ensure the table can be queried using both the old and new name
35+ // Make sure inserts work using both the old and new name
54− let expected: Vec<i32> = vec![1, 2];
55− assert_eq!(
56− expected,
57 old_db
36 old_db
58− .query("SELECT id FROM users ORDER BY id", &[])
37+ .simple_query("INSERT INTO users(id) VALUES (1)")
59− .unwrap()
38+ .unwrap();
60− .iter()
61− .map(|row| row.get::<_, i32>("id"))
62− .collect::<Vec<i32>>()
63− );
64− assert_eq!(
65− expected,
66 new_db
39 new_db
40+ .simple_query("INSERT INTO customers(id) VALUES (2)")
41+ .unwrap();
42+
43+ // Ensure the table can be queried using both the old and new name
44+ let expected: Vec<i32> = vec![1, 2];
45+ assert_eq!(
46+ expected,
47+ old_db
67− .query("SELECT id FROM customers ORDER BY id", &[])
48+ .query("SELECT id FROM users ORDER BY id", &[])
68− .unwrap()
49+ .unwrap()
50+ .iter()
51+ .map(|row| row.get::<_, i32>("id"))
52+ .collect::<Vec<i32>>()
53+ );
54+ assert_eq!(
55+ expected,
56+ new_db
57+ .query("SELECT id FROM customers ORDER BY id", &[])
58+ .unwrap()
69− .iter()
59+ .iter()
70− .map(|row| row.get::<_, i32>("id"))
60+ .map(|row| row.get::<_, i32>("id"))
71− .collect::<Vec<i32>>()
61+ .collect::<Vec<i32>>()
72− );
62+ );
73
63
74− // Ensure the table can't be queried using the wrong name for the schema
64+ // Ensure the table can't be queried using the wrong name for the schema
75− assert!(old_db.simple_query("SELECT id FROM customers").is_err());
65+ assert!(old_db.simple_query("SELECT id FROM customers").is_err());
76− assert!(new_db.simple_query("SELECT id FROM users").is_err());
66+ assert!(new_db.simple_query("SELECT id FROM users").is_err());
67+ });
77
68
78− reshape.complete().unwrap();
69+ test.run();
79− common::assert_cleaned_up(&mut new_db);
80}
70}