on main help sign in
46 files changed+6410−2034
since splitsnapshots
unifiedsplit
.github/workflows/lint.yaml+6−3 .github/workflows/publish-crate.yaml+20 .github/workflows/publish-docker.yaml+25 .github/workflows/release.yaml+21−3 .github/workflows/test.yaml+3 Cargo.toml+14−4 Dockerfile+1−1 LICENSE.md+21 README.md+436−89 src/db.rs+155−11 src/helpers.rs+8−17 src/lib.rs+419−198 src/main.rs+98−27 src/migrations/add_column.rs+294−116 src/migrations/add_foreign_key.rs+122 src/migrations/add_index.rs+54−23 src/migrations/alter_column.rs+311−205 src/migrations/common.rs+137−37 src/migrations/create_enum.rs+83 src/migrations/create_table.rs+177−29 src/migrations/custom.rs+61 src/migrations/mod.rs+43−11 src/migrations/remove_column.rs+385−54 src/migrations/remove_enum.rs+51 src/migrations/remove_foreign_key.rs+91 src/migrations/remove_index.rs+51 src/migrations/remove_table.rs+24−13 src/migrations/rename_table.rs+25−14 src/schema.rs+42−19 src/state.rs+200−88 tests/add_column.rs+316−262 tests/add_foreign_key.rs+166 tests/add_index.rs+216−65 tests/alter_column.rs+660−348 tests/common.rs+304−9 tests/complex.rs+379 tests/create_enum.rs+41 tests/create_table.rs+187−198 tests/custom.rs+128 tests/failure.rs+51 tests/remove_column.rs+210−75 tests/remove_enum.rs+69 tests/remove_foreign_key.rs+124 tests/remove_index.rs+86 tests/remove_table.rs+39−49 tests/rename_table.rs+56−66
.github/workflows/lint.yaml+6−3
@@ -3,6 +3,9 @@ name: Lint
33on:
44 push:
55 branches: [ main ]
6+ pull_request:
7+ branches:
8+ - main
69
710env:
811 CARGO_TERM_COLOR: always
@@ -12,7 +15,7 @@ jobs:
1215 runs-on: ubuntu-latest
1316 steps:
1417 - name: Checkout
15− uses: actions/checkout@v2
18+ uses: actions/checkout@v4
1619 - name: Select Rust toolchain with Clippy
1720 uses: actions-rs/toolchain@v1
1821 with:
@@ -20,6 +23,6 @@ jobs:
2023 components: clippy
2124 override: true
2225 - name: Use cache for Rust dependencies
23− uses: Swatinem/rust-cache@v1
26+ uses: Swatinem/rust-cache@v2
2427 - name: Lint using Clippy
25− run: cargo clippy
28+ run: cargo clippy
.github/workflows/publish-crate.yamlnew file+20
@@ -0,0 +1,20 @@
1+name: Publish crate
2+
3+on: workflow_dispatch
4+
5+jobs:
6+ publish-crate:
7+ runs-on: ubuntu-latest
8+ steps:
9+ - name: Checkout
10+ uses: actions/checkout@v2
11+ - name: Select Rust toolchain
12+ uses: actions-rs/toolchain@v1
13+ with:
14+ toolchain: stable
15+ - name: Publish to crates.io
16+ uses: actions-rs/cargo@v1
17+ with:
18+ command: publish
19+ env:
20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/publish-docker.yamlnew file+25
@@ -0,0 +1,25 @@
1+name: Publish Docker image
2+
3+on:
4+ workflow_dispatch:
5+ inputs:
6+ version:
7+ description: "Version (without 'v' prefix)"
8+ required: true
9+ type: string
10+
11+jobs:
12+ publish-docker:
13+ runs-on: ubuntu-latest
14+ steps:
15+ - name: Checkout
16+ uses: actions/checkout@v2
17+ - name: Log in to Docker
18+ env:
19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
20+ run: |
21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD
22+ - name: Build Docker image
23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
24+ - name: Push Docker image
25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
.github/workflows/release.yaml+21−3
@@ -47,6 +47,7 @@ jobs:
4747 uses: actions-rs/toolchain@v1
4848 with:
4949 toolchain: stable
50+ override: true
5051 target: ${{ matrix.target }}
5152 - name: Use cache for Rust dependencies
5253 uses: Swatinem/rust-cache@v1
@@ -54,10 +55,10 @@ jobs:
5455 uses: actions-rs/cargo@v1
5556 with:
5657 command: build
57− args: --release
58+ args: --release --target ${{ matrix.target }}
5859 use-cross: ${{ matrix.use-cross }}
5960 - name: Rename binary
60− run: mv target/release/reshape ./reshape-${{ matrix.file-name }}
61+ run: mv target/${{ matrix.target }}/release/reshape ./reshape-${{ matrix.file-name }}
6162 - name: Upload binary to release
6263 env:
6364 GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -78,4 +79,21 @@ jobs:
7879 # The shell expansion used below will remove everything up to the version number, leaving 0.0.1
7980 run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${GITHUB_REF##*/v} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
8081 - name: Push Docker image
81− run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
82+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
83+
84+ publish-crate:
85+ runs-on: ubuntu-latest
86+ needs: ["build-binary", "publish-docker"]
87+ steps:
88+ - name: Checkout
89+ uses: actions/checkout@v2
90+ - name: Select Rust toolchain
91+ uses: actions-rs/toolchain@v1
92+ with:
93+ toolchain: stable
94+ - name: Publish to crates.io
95+ uses: actions-rs/cargo@v1
96+ with:
97+ command: publish
98+ env:
99+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/test.yaml+3
@@ -3,6 +3,9 @@ name: Tests
33on:
44 push:
55 branches: [ main ]
6+ pull_request:
7+ branches:
8+ - main
69
710env:
811 CARGO_TERM_COLOR: always
Cargo.toml+14−4
@@ -1,16 +1,26 @@
11[package]
22name = "reshape"
3−version = "0.1.0"
3+version = "0.7.0"
4+description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
5+homepage = "https://github.com/fabianlindfors/reshape"
6+documentation = "https://github.com/fabianlindfors/reshape"
7+repository = "https://github.com/fabianlindfors/reshape"
8+license = "MIT"
9+keywords = ["postgres", "migrations"]
410edition = "2021"
11+authors = ["Fabian Lindfors"]
12+rust-version = "1.70"
513
614[dependencies]
715postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
816serde = { version = "1.0", features = ["derive"] }
917serde_json = "1.0"
1018typetag = "0.1.7"
11−anyhow = "1.0.44"
12−clap = { version = "3.0.0-rc.8", features = ["derive"] }
19+anyhow = { version = "1.0.44", features = ["backtrace"] }
20+clap = { version = "3.1.9", features = ["derive"] }
1321toml = "0.5"
1422version = "3.0.0"
1523colored = "2"
16−bimap = "0.6.1"
24+rand = "0.8"
25+dotenv = "0.15.0"
26+lexical-sort = "0.3.1"
Dockerfile+1−1
@@ -1,4 +1,4 @@
1−FROM rust:1.57.0 AS builder
1+FROM rust:1.70.0 AS builder
22WORKDIR /usr/src/reshape
33COPY . .
44RUN cargo build --release
LICENSE.mdnew file+21
@@ -0,0 +1,21 @@
1+MIT License
2+
3+Copyright (c) 2022 Fabian Lindfors
4+
5+Permission is hereby granted, free of charge, to any person obtaining a copy
6+of this software and associated documentation files (the "Software"), to deal
7+in the Software without restriction, including without limitation the rights
8+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+copies of the Software, and to permit persons to whom the Software is
10+furnished to do so, subject to the following conditions:
11+
12+The above copyright notice and this permission notice shall be included in all
13+copies or substantial portions of the Software.
14+
15+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+SOFTWARE.
README.md+436−89
@@ -1,35 +1,57 @@
11# Reshape
22
3−[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml)
3+[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)
44
5−Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application.
5+> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!
66
7−*Note: Reshape is **experimental** and should not be used in production. It can (and probably will) destroy your data and break your application.*
7+Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).
88
9+Designed for Postgres 12 and later.
10+
11+- [How it works](#how-it-works)
912- [Getting started](#getting-started)
10− - [Installation](#installation)
11− - [Creating your first migration](#creating-your-first-migration)
12− - [Preparing your application](#preparing-your-application)
13− - [Running your migration](#running-your-migration)
13+ - [Installation](#installation)
14+ - [Creating your first migration](#creating-your-first-migration)
15+ - [Preparing your application](#preparing-your-application)
16+ - [Running your migration](#running-your-migration)
17+ - [Using during development](#using-during-development)
1418- [Writing migrations](#writing-migrations)
15− - [Basics](#basics)
16− - [Tables](#tables)
17− - [Create table](#create-table)
18− - [Rename table](#rename-table)
19− - [Remove table](#remove-table)
20− - [Columns](#columns)
21− - [Add column](#add-column)
22− - [Alter column](#alter-column)
23− - [Remove column](#remove-column)
24− - [Indices](#indices)
25− - [Add index](#add-index)
19+ - [Basics](#basics)
20+ - [Tables](#tables)
21+ - [Create table](#create-table)
22+ - [Rename table](#rename-table)
23+ - [Remove table](#remove-table)
24+ - [Add foreign key](#add-foreign-key)
25+ - [Remove foreign key](#remove-foreign-key)
26+ - [Columns](#columns)
27+ - [Add column](#add-column)
28+ - [Alter column](#alter-column)
29+ - [Remove column](#remove-column)
30+ - [Indices](#indices)
31+ - [Add index](#add-index)
32+ - [Remove index](#remove-index)
33+ - [Enums](#enums)
34+ - [Create enum](#create-enum)
35+ - [Remove enum](#remove-enum)
36+ - [Custom](#custom)
37+ - [Complex changes across tables](#complex-changes-across-tables)
2638- [Commands and options](#commands-and-options)
27− - [`reshape migrate`](#reshape-migrate)
28− - [`reshape complete`](#reshape-complete)
29− - [`reshape abort`](#reshape-abort)
30− - [`reshape generate-schema-query`](#reshape-generate-schema-query)
31− - [Connection options](#connection-options)
32−- [How it works](#how-it-works)
39+ - [`reshape migration start`](#reshape-migration-start)
40+ - [`reshape migration complete`](#reshape-migration-complete)
41+ - [`reshape migration abort`](#reshape-migration-abort)
42+ - [`reshape schema-query`](#reshape-schema-query)
43+ - [Connection options](#connection-options)
44+- [License](#license)
45+
46+## How it works
47+
48+Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every deployment is a three-phase process:
49+
50+1. **Start migration** (`reshape migration start`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
51+2. **Roll out application**: Your application can be gradually rolled out without downtime. The existing deployment will continue using the old schema whilst the new deployment uses the new schema.
52+3. **Complete migration** (`reshape migration complete`): Removes the old schema and any intermediate data and triggers.
53+
54+If the application deployment fails, you should run `reshape migration abort` which will roll back any changes made by `reshape migration start` without losing data.
3355
3456## Getting started
3557
@@ -39,24 +61,33 @@ Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It
3961
4062Binaries are available for macOS and Linux under [Releases](https://github.com/fabianlindfors/reshape/releases).
4163
64+#### Cargo
65+
66+Reshape can be installed using [Cargo](https://doc.rust-lang.org/cargo/) (requires Rust 1.58 or later):
67+
68+```shell
69+cargo install reshape
70+```
71+
4272#### Docker
4373
44−Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
74+Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
4575
4676```shell
47−docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migrate
77+docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migration start
4878```
4979
5080### Creating your first migration
5181
52−Each migration should be stored as a separate file under `migrations/`. The files can be in either JSON or TOML format. The name of the file will become the name of your migration and they will be sorted by file name. We recommend prefixing every migration with an incrementing number.
82+Each migration should be stored as a separate file in a `migrations/` directory. The files can be in either JSON or TOML format and the name of the file will become the name of your migration. We recommend prefixing every migration with an incrementing number as migrations are sorted by file name.
5383
54−Let's create a simple migration to set up a new table `users` with two fields, `id` and `name`. We'll create a file called `migration/1_create_users_table.toml`:
84+Let's create a simple migration to set up a new table `users` with two fields, `id` and `name`. We'll create a file called `migrations/1_create_users_table.toml`:
5585
5686```toml
5787[[actions]]
5888type = "create_table"
59−table = "users"
89+name = "users"
90+primary_key = ["id"]
6091
6192 [[actions.columns]]
6293 name = "id"
@@ -72,9 +103,14 @@ This is the equivalent of running `CREATE TABLE users (id INTEGER GENERATED ALWA
72103
73104### Preparing your application
74105
75−Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. This query can be generated using: `reshape generate-schema-query`.
106+Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. The simplest way to do this is to use one of the helper libraries:
107+
108+- [Rust](https://github.com/fabianlindfors/reshape-helper)
109+- [Ruby (and Rails)](https://github.com/fabianlindfors/reshape-ruby)
110+- [Python (and Django)](https://github.com/fabianlindfors/reshape-python)
111+- [Go](https://github.com/leourbina/reshape-helper)
76112
77−To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape generate-schema-query)`. Then in your application:
113+If your application is not using one of the languages with an available helper library, you can instead generate the query with the command: `reshape schema-query`. To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape schema-query)`. Then in your application:
78114
79115```python
80116# Example for Python
@@ -87,13 +123,17 @@ db.execute(reshape_schema_query)
87123To create your new `users` table, run:
88124
89125```bash
90−reshape migrate
126+reshape migration start --complete
91127```
92128
93−As this is the first migration, Reshape will automatically complete it. For subsequent migrations, you will need to first run `reshape migrate`, roll out your application and then complete the migration using `reshape complete`.
129+We use the `--complete` flag to automatically complete the migration. During a production deployment, you should first run `reshape migration start` followed by `reshape migration complete` once your application has been fully rolled out.
94130
95131If nothing else is specified, Reshape will try to connect to a Postgres database running on `localhost` using `postgres` as both username and password. See [Connection options](#connection-options) for details on how to change the connection settings.
96132
133+### Using during development
134+
135+When adding new migrations during development, we recommend running `reshape migration start` but skipping `reshape migration complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape migration abort` followed by `reshape migration start`.
136+
97137## Writing migrations
98138
99139### Basics
@@ -103,7 +143,8 @@ Every migration consists of one or more actions. The actions will be run sequent
103143```toml
104144[[actions]]
105145type = "create_table"
106−table = "customers"
146+name = "customers"
147+primary_key = ["id"]
107148
108149 [[actions.columns]]
109150 name = "id"
@@ -112,7 +153,8 @@ table = "customers"
112153
113154[[actions]]
114155type = "create_table"
115−table = "products"
156+name = "products"
157+primary_key = ["sku"]
116158
117159 [[actions.columns]]
118160 name = "sku"
@@ -125,15 +167,15 @@ Every action has a `type`. The supported types are detailed below.
125167
126168#### Create table
127169
128−The `create_table` action will create a new table with the specified columns, indices and constraints.
170+The `create_table` action will create a new table with the specified columns, indices and constraints. You can optionally provide an `up` option to backfill values from an existing table.
129171
130−*Example: create a `customers` table with a few columns and a primary key*
172+_Example: create a `customers` table with a few columns and a primary key_
131173
132174```toml
133175[[actions]]
134176type = "create_table"
135−table = "customers"
136−primary_key = "id"
177+name = "customers"
178+primary_key = ["id"]
137179
138180 [[actions.columns]]
139181 name = "id"
@@ -145,19 +187,19 @@ primary_key = "id"
145187 type = "TEXT"
146188
147189 # Columns default to nullable
148− nullable = false
190+ nullable = false
149191
150192 # default can be any valid SQL value, in this case a string literal
151− default = "'PLACEHOLDER'"
193+ default = "'PLACEHOLDER'"
152194```
153195
154−*Example: create `users` and `items` tables with a foreign key between them*
196+_Example: create `users` and `items` tables with a foreign key between them_
155197
156198```toml
157199[[actions]]
158200type = "create_table"
159−table = "users"
160−primary_key = "id"
201+name = "users"
202+primary_key = ["id"]
161203
162204 [[actions.columns]]
163205 name = "id"
@@ -166,8 +208,8 @@ primary_key = "id"
166208
167209[[actions]]
168210type = "create_table"
169−table = "items"
170−primary_key = "id"
211+name = "items"
212+primary_key = ["id"]
171213
172214 [[actions.columns]]
173215 name = "id"
@@ -184,11 +226,34 @@ primary_key = "id"
184226 referenced_columns = ["id"]
185227```
186228
229+_Example: create `profiles` table based on existing `users` table_
230+
231+```toml
232+[[actions]]
233+type = "create_table"
234+name = "profiles"
235+primary_key = ["user_id"]
236+
237+ [[actions.columns]]
238+ name = "user_id"
239+ type = "INTEGER"
240+
241+ [[actions.columns]]
242+ name = "user_email"
243+ type = "TEXT"
244+
245+ # Backfill from `users` table and copy `users.email` to `user_email` column
246+ # This will perform an upsert based on the primary key to avoid duplicate rows
247+ [actions.up]
248+ table = "users"
249+ values = { user_id = "id", user_email = "email" }
250+```
251+
187252#### Rename table
188253
189254The `rename_table` action will change the name of an existing table.
190255
191−*Example: change name of `users` table to `customers`*
256+_Example: change name of `users` table to `customers`_
192257
193258```toml
194259[[actions]]
@@ -201,7 +266,7 @@ new_name = "customers"
201266
202267The `remove_table` action will remove an existing table.
203268
204−*Example: remove `users` table*
269+_Example: remove `users` table_
205270
206271```toml
207272[[actions]]
@@ -209,13 +274,43 @@ type = "remove_table"
209274table = "users"
210275```
211276
277+#### Add foreign key
278+
279+The `add_foreign_key` action will add a foreign key between two existing tables. The migration will fail if the existing column values aren't valid references.
280+
281+_Example: create foreign key from `items` to `users` table_
282+
283+```toml
284+[[actions]]
285+type = "add_foreign_key"
286+table = "items"
287+
288+ [actions.foreign_key]
289+ columns = ["user_id"]
290+ referenced_table = "users"
291+ referenced_columns = ["id"]
292+```
293+
294+#### Remove foreign key
295+
296+The `remove_foreign_key` action will remove an existing foreign key. The foreign key will only be removed once the migration is completed, which means that your new application must continue to adhere to the foreign key constraint.
297+
298+_Example: remove foreign key `items_user_id_fkey` from `users` table_
299+
300+```toml
301+[[actions]]
302+type = "remove_foreign_key"
303+table = "items"
304+foreign_key = "items_user_id_fkey"
305+```
306+
212307### Columns
213308
214309#### Add column
215310
216−The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column.
311+The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column. `up` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)).
217312
218−*Example: add a new column `reference` to table `products`*
313+_Example: add a new column `reference` to table `products`_
219314
220315```toml
221316[[actions]]
@@ -229,7 +324,7 @@ table = "products"
229324 default = "10"
230325```
231326
232−*Example: replace an existing `name` column with two new columns, `first_name` and `last_name`*
327+_Example: replace an existing `name` column with two new columns, `first_name` and `last_name`_
233328
234329```toml
235330[[actions]]
@@ -265,14 +360,48 @@ column = "name"
265360down = "first_name || ' ' || last_name"
266361```
267362
363+_Example: extract nested value from unstructured JSON `data` column to new `name` column_
364+
365+```toml
366+[[actions]]
367+type = "add_column"
368+table = "users"
369+
370+# #>> '{}' converts the JSON string value to TEXT
371+up = "data['path']['to']['value'] #>> '{}'"
372+
373+ [actions.column]
374+ name = "name"
375+ type = "TEXT"
376+```
377+
378+_Example: duplicate `email` column from `users` to `profiles` table_
379+
380+```toml
381+# `profiles` has `user_id` column which maps to `users.id`
382+[[actions]]
383+type = "add_column"
384+table = "profiles"
385+
386+ [actions.column]
387+ name = "email"
388+ type = "TEXT"
389+ nullable = false
390+
391+ # When `users` is updated in the old schema, we write the email value to `profiles`
392+ [actions.up]
393+ table = "users"
394+ value = "email"
395+ where = "user_id = id"
396+```
268397
269398#### Alter column
270399
271400The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.
272401
273−When performing more complex changes than a rename, `up` and `down` must be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
402+When performing more complex changes than a rename, `up` and `down` should be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
274403
275−*Example: rename `last_name` column on `users` table to `family_name`*
404+_Example: rename `last_name` column on `users` table to `family_name`_
276405
277406```toml
278407[[actions]]
@@ -284,7 +413,7 @@ column = "last_name"
284413 name = "family_name"
285414```
286415
287−*Example: change the type of `reference` column from `INTEGER` to `TEXT`*
416+_Example: change the type of `reference` column from `INTEGER` to `TEXT`_
288417
289418```toml
290419[[actions]]
@@ -299,12 +428,13 @@ down = "CAST(reference AS INTEGER)" # Converts from text value to integer
299428 type = "TEXT" # Previous type was 'INTEGER'
300429```
301430
302−*Example: increment all values of an `index` column by one*
431+_Example: increment all values of an `index` column by one_
303432
304433```toml
305434[[actions]]
306435type = "alter_column"
307436table = "users"
437+column = "index"
308438
309439up = "index + 1" # Increment for new schema
310440down = "index - 1" # Decrement to revert for old schema
@@ -313,11 +443,40 @@ down = "index - 1" # Decrement to revert for old schema
313443 name = "index"
314444```
315445
446+_Example: make `name` column not nullable_
447+
448+```toml
449+[[actions]]
450+type = "alter_column"
451+table = "users"
452+column = "name"
453+
454+# Use "N/A" for any rows that currently have a NULL name
455+up = "COALESCE(name, 'N/A')"
456+
457+ [actions.changes]
458+ nullable = false
459+```
460+
461+_Example: change default value of `created_at` column to current time_
462+
463+```toml
464+[[actions]]
465+type = "alter_column"
466+table = "users"
467+column = "created_at"
468+
469+ [actions.changes]
470+ default = "NOW()"
471+```
472+
316473#### Remove column
317474
318−The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
475+The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. `down` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)) . The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
476+
477+Any indices that cover the column will be removed.
319478
320−*Example: remove column `name` from table `users`*
479+_Example: remove column `name` from table `users`_
321480
322481```toml
323482[[actions]]
@@ -329,18 +488,34 @@ column = "name"
329488down = "'N/A'"
330489```
331490
491+_Example: remove `email` column from `users` table and use column from `profiles` table instead_
492+
493+```toml
494+[[actions]]
495+type = "remove_column"
496+table = "users"
497+column = "email"
498+
499+ # Our application will use the `profiles.email` column instead
500+ # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed
501+ [actions.down]
502+ table = "profiles"
503+ value = "profiles.email"
504+ where = "users.id = profiles.user_id"
505+```
506+
332507### Indices
333508
334509#### Add index
335510
336511The `add_index` action will add a new index to an existing table.
337512
338−*Example: create a `users` table with an index on the `name` column*
513+_Example: create a `users` table with a unique index on the `name` column_
339514
340515```toml
341516[[actions]]
342517type = "create_table"
343−table = "users"
518+name = "users"
344519primary_key = "id"
345520
346521 [[actions.columns]]
@@ -355,70 +530,242 @@ primary_key = "id"
355530[[actions]]
356531type = "add_index"
357532table = "users"
358−name = "name_idx"
359−columns = ["name"]
533+
534+ [actions.index]
535+ name = "name_idx"
536+ columns = ["name"]
537+
538+ # Defaults to false
539+ unique = true
540+```
541+
542+_Example: add GIN index to `data` column on `products` table_
543+
544+```toml
545+[[actions]]
546+type = "add_index"
547+table = "products"
548+
549+ [actions.index]
550+ name = "data_idx"
551+ columns = ["data"]
552+
553+ # One of: btree (default), hash, gist, spgist, gin, brin
554+ type = "gin"
555+```
556+
557+#### Remove index
558+
559+The `remove_index` action will remove an existing index. The index won't actually be removed until the migration is completed.
560+
561+_Example: remove the `name_idx` index_
562+
563+```toml
564+[[actions]]
565+type = "remove_index"
566+index = "name_idx"
567+```
568+
569+### Enums
570+
571+#### Create enum
572+
573+The `create_enum` action will create a new [enum type](https://www.postgresql.org/docs/current/datatype-enum.html) with the specified values.
574+
575+_Example: add a new `mood` enum type with three possible values_
576+
577+```toml
578+[[actions]]
579+type = "create_enum"
580+name = "mood"
581+values = ["happy", "ok", "sad"]
582+```
583+
584+#### Remove enum
585+
586+The `remove_enum` action will remove an existing [enum type](https://www.postgresql.org/docs/current/datatype-enum.html). Make sure all usages of the enum has been removed before running the migration. The enum will only be removed once the migration is completed.
587+
588+_Example: remove the `mood` enum type_
589+
590+```toml
591+[[actions]]
592+type = "remove_enum"
593+enum = "mood"
594+```
595+
596+### Custom
597+
598+The `custom` action lets you create a migration which runs custom SQL. It should be used with great care as it provides no guarantees of zero-downtime and will simply run whatever SQL is provided. Use other actions whenever possible as they are explicitly designed for zero downtime.
599+
600+There are three optional settings available which all accept SQL queries. All queries need to be idempotent, for example by using `IF NOT EXISTS` wherever available.
601+
602+- `start`: run when a migration is started using `reshape migration start`
603+- `complete`: run when a migration is completed using `reshape migration complete`
604+- `abort`: run when a migration is aborted using `reshape migration abort`
605+
606+_Example: enable PostGIS and pg_stat_statements extensions_
607+
608+```toml
609+[[actions]]
610+type = "custom"
611+
612+start = """
613+ CREATE EXTENSION IF NOT EXISTS postgis;
614+ CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
615+"""
616+
617+abort = """
618+ DROP EXTENSION IF EXISTS postgis;
619+ DROP EXTENSION IF EXISTS pg_stat_statements;
620+"""
621+```
622+
623+### Complex changes across tables
624+
625+The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.
626+
627+_Example: move `email` column from `users` to `profiles` table_
628+
629+```toml
630+[[actions]]
631+type = "add_column"
632+table = "profiles"
633+
634+ [actions.column]
635+ name = "email"
636+ type = "TEXT"
637+ nullable = false
638+
639+ # When `users` is updated in the old schema, we write the email value to `profiles`
640+ [actions.up]
641+ table = "users"
642+ value = "users.email"
643+ where = "profiles.user_id = users.id"
644+
645+[[actions]]
646+type = "remove_column"
647+table = "users"
648+column = "email"
649+
650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
651+ [actions.down]
652+ table = "profiles"
653+ value = "profiles.email"
654+ where = "users.id = profiles.user_id"
655+```
656+
657+_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_
658+
659+```toml
660+# Add `user_account_connections` as a junction table
661+[[actions]]
662+type = "create_table"
663+name = "user_account_connections"
664+primary_key = ["account_id", "user_id"]
665+
666+ [[actions.columns]]
667+ name = "account_id"
668+ type = "INTEGER"
669+
670+ [[actions.columns]]
671+ name = "user_id"
672+ type = "INTEGER"
673+
674+ # `role` is currently stored directly on the `users` table but is part of the relationship
675+ [[actions.columns]]
676+ name = "role"
677+ type = "TEXT"
678+ nullable = false
679+
680+ # Backfill the new table from `users` and uppercase the `role`
681+ [actions.up]
682+ table = "users"
683+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
684+ where = "user_account_connections.user_id = users.id"
685+
686+[[actions]]
687+type = "remove_column"
688+table = "users"
689+column = "account_id"
690+
691+ # When `user_account_connections` is updated, we write the `account_id` back to `users`
692+ [actions.down]
693+ table = "user_account_connections"
694+ value = "user_account_connections.account_id"
695+ where = "users.id = user_account_connections.user_id"
696+
697+[[actions]]
698+type = "remove_column"
699+table = "users"
700+column = "account_role"
701+
702+ # When `user_account_connections` is updated, we write the lowercase role back to `users`
703+ [actions.down]
704+ table = "user_account_connections"
705+ value = "LOWER(user_account_connections.role)"
706+ where = "users.id = user_account_connections.user_id"
360707```
361708
362709## Commands and options
363710
364−### `reshape migrate`
711+### `reshape migration start`
365712
366−Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape complete`.
713+Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape migration complete`.
367714
368715#### Options
369716
370−*See also [Connection options](#connection-options)*
717+_See also [Connection options](#connection-options)_
371718
372−| Option | Default | Description |
373−| ------ | ------- | ----------- |
374−| `--complete`, `-c` | `false` | Automatically complete migration after applying it. Useful during development. |
375−| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
719+| Option | Default | Description |
720+| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------------- |
721+| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
722+| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
376723
377−### `reshape complete`
724+### `reshape migration complete`
378725
379−Completes migrations previously started with `reshape complete`.
726+Completes migrations previously started with `reshape migration complete`.
380727
381728#### Options
382729
383730See [Connection options](#connection-options)
384731
385−### `reshape abort`
732+### `reshape migration abort`
386733
387−Aborts any migrations which haven't yet been completed.
734+Aborts any migrations which haven't yet been completed.
388735
389736#### Options
390737
391738See [Connection options](#connection-options)
392739
393−### `reshape generate-schema-query`
740+### `reshape schema-query`
394741
395742Generates the SQL query you need to run in your application before using the database. This command does not require a database connection. Instead it will generate the query based on the latest migration in the `migrations/` directory (or the directories specified by `--dirs`).
396743
744+If your application is written in Rust, we recommend using the [Rust helper library](https://github.com/fabianlindfors/reshape-helper/) instead.
745+
397746The query should look something like `SET search_path TO migration_1_initial_migration`.
398747
399748#### Options
400749
401−| Option | Default | Description |
402−| ------ | ------- | ----------- |
750+| Option | Default | Description |
751+| -------- | ------------- | --------------------------------------------------------------------------------------------------------------- |
403752| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
404753
405754### Connection options
406755
407−The options below can be used with all commands that communicate with Postgres. Use either a connection URL or specify each connection option individually.
756+The options below can be used with all commands that communicate with Postgres. Use either a [connection URL](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) or specify each connection option individually.
408757
409−| Option | Default | Description |
410−| ------ | ------- | ----------- |
411−| `--url` | | URI to connect to your Postgres database.<br>Can also be provided with the environment variable `DATABASE_URL`. |
412−| `--host` | `localhost` | Hostname to use when connecting to Postgres |
413−| `--port` | `5432` | Port which Postgres is listening on |
414−| `--database` | `postgres` | Database name |
415−| `--username` | `postgres` | Postgres username |
416−| `--password` | `postgres` | Postgres password |
758+All options can also be set using environment variables instead of flags. If a `.env` file exists, then variables will be automatically loaded from there.
417759
418−## How it works
760+| Option | Default | Environment variable | Description |
761+| ------------ | ----------- | -------------------- | ------------------------------------------- |
762+| `--url` | | `DB_URL` | URL to your Postgres database |
763+| `--host` | `localhost` | `DB_HOST` | Hostname to use when connecting to Postgres |
764+| `--port` | `5432` | `DB_PORT` | Port which Postgres is listening on |
765+| `--database` | `postgres` | `DB_NAME` | Database name |
766+| `--username` | `postgres` | `DB_USERNAME` | Postgres username |
767+| `--password` | `postgres` | `DB_PASSWORD` | Postgres password |
419768
420−Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every migration is a two phase process:
769+## License
421770
422−1. **Start migration** (`reshape migrate`): Create new views to ensure both the new and old schema are usable at the same time.
423− - After phase one is complete, you can start the roll out of your application. Once the roll out is complete, the second phase can be run.
424−2. **Complete migration** (`reshape complete`): Removes the old schema and any intermediate data.
771+Reshape is released under the [MIT license](https://choosealicense.com/licenses/mit/).
src/db.rs+155−11
@@ -1,4 +1,87 @@
1+use std::{cmp::min, time::Duration};
2+
3+use anyhow::{anyhow, Context};
14use postgres::{types::ToSql, NoTls, Row};
5+use rand::prelude::*;
6+
7+// DbLocker wraps a regular DbConn, only allowing access using the
8+// `lock` method. This method will acquire the advisory lock before
9+// allowing access to the database, and then release it afterwards.
10+//
11+// We use advisory locks to avoid multiple Reshape instances working
12+// on the same database as the same time. DbLocker is the only way to
13+// get a DbConn which ensures that all database access is protected by
14+// a lock.
15+//
16+// Postgres docs on advisory locks:
17+// https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS
18+pub struct DbLocker {
19+ client: DbConn,
20+}
21+
22+impl DbLocker {
23+ // Advisory lock keys in Postgres are 64-bit integers.
24+ // The key we use was chosen randomly.
25+ const LOCK_KEY: i64 = 4036779288569897133;
26+
27+ pub fn connect(config: &postgres::Config) -> anyhow::Result<Self> {
28+ let mut pg = config.connect(NoTls)?;
29+
30+ // When running DDL queries that acquire locks, we risk causing a "lock queue".
31+ // When attempting to acquire a lock, Postgres will wait for any long running queries to complete.
32+ // At the same time, it will block other queries until the lock has been acquired and released.
33+ // This has the bad effect of the long-running query blocking other queries because of us, forming
34+ // a queue of other queries until we release our lock.
35+ //
36+ // We set the lock_timeout setting to avoid this. This puts an upper bound for how long Postgres will
37+ // wait to acquire locks and also the maximum amount of time a long-running query can block other queries.
38+ // We should also add automatic retries to handle these timeouts gracefully.
39+ //
40+ // Reference: https://medium.com/paypal-tech/postgresql-at-scale-database-schema-changes-without-downtime-20d3749ed680
41+ //
42+ // TODO: Make lock_timeout configurable
43+ pg.simple_query("SET lock_timeout = '1s'")
44+ .context("failed to set lock_timeout")?;
45+
46+ Ok(Self {
47+ client: DbConn::new(pg),
48+ })
49+ }
50+
51+ pub fn lock(
52+ &mut self,
53+ f: impl FnOnce(&mut DbConn) -> anyhow::Result<()>,
54+ ) -> anyhow::Result<()> {
55+ self.acquire_lock()?;
56+ let result = f(&mut self.client);
57+ self.release_lock()?;
58+
59+ result
60+ }
61+
62+ fn acquire_lock(&mut self) -> anyhow::Result<()> {
63+ let success = self
64+ .client
65+ .query(&format!("SELECT pg_try_advisory_lock({})", Self::LOCK_KEY))?
66+ .first()
67+ .ok_or_else(|| anyhow!("unexpectedly failed when acquiring advisory lock"))
68+ .map(|row| row.get::<'_, _, bool>(0))?;
69+
70+ if success {
71+ Ok(())
72+ } else {
73+ Err(anyhow!("another instance of Reshape is already running"))
74+ }
75+ }
76+
77+ fn release_lock(&mut self) -> anyhow::Result<()> {
78+ self.client
79+ .query(&format!("SELECT pg_advisory_unlock({})", Self::LOCK_KEY))?
80+ .first()
81+ .ok_or_else(|| anyhow!("unexpectedly failed when releasing advisory lock"))?;
82+ Ok(())
83+ }
84+}
285
386pub trait Conn {
487 fn run(&mut self, query: &str) -> anyhow::Result<()>;
@@ -8,6 +91,7 @@ pub trait Conn {
891 query: &str,
992 params: &[&(dyn ToSql + Sync)],
1093 ) -> anyhow::Result<Vec<Row>>;
94+ fn transaction(&mut self) -> anyhow::Result<Transaction>;
1195}
1296
1397pub struct DbConn {
@@ -15,25 +99,19 @@ pub struct DbConn {
1599}
16100
17101impl DbConn {
18− pub fn connect(config: &postgres::Config) -> anyhow::Result<DbConn> {
19− let client = config.connect(NoTls)?;
20− Ok(DbConn { client })
21− }
22−
23− pub fn transaction(&mut self) -> anyhow::Result<Transaction> {
24− let transaction = self.client.transaction()?;
25− Ok(Transaction { transaction })
102+ fn new(client: postgres::Client) -> Self {
103+ DbConn { client }
26104 }
27105}
28106
29107impl Conn for DbConn {
30108 fn run(&mut self, query: &str) -> anyhow::Result<()> {
31− self.client.batch_execute(query)?;
109+ retry_automatically(|| self.client.batch_execute(query))?;
32110 Ok(())
33111 }
34112
35113 fn query(&mut self, query: &str) -> anyhow::Result<Vec<Row>> {
36− let rows = self.client.query(query, &[])?;
114+ let rows = retry_automatically(|| self.client.query(query, &[]))?;
37115 Ok(rows)
38116 }
39117
@@ -42,9 +120,14 @@ impl Conn for DbConn {
42120 query: &str,
43121 params: &[&(dyn ToSql + Sync)],
44122 ) -> anyhow::Result<Vec<Row>> {
45− let rows = self.client.query(query, params)?;
123+ let rows = retry_automatically(|| self.client.query(query, params))?;
46124 Ok(rows)
47125 }
126+
127+ fn transaction(&mut self) -> anyhow::Result<Transaction> {
128+ let transaction = self.client.transaction()?;
129+ Ok(Transaction { transaction })
130+ }
48131}
49132
50133pub struct Transaction<'a> {
@@ -56,6 +139,11 @@ impl Transaction<'_> {
56139 self.transaction.commit()?;
57140 Ok(())
58141 }
142+
143+ pub fn rollback(self) -> anyhow::Result<()> {
144+ self.transaction.rollback()?;
145+ Ok(())
146+ }
59147}
60148
61149impl Conn for Transaction<'_> {
@@ -77,4 +165,60 @@ impl Conn for Transaction<'_> {
77165 let rows = self.transaction.query(query, params)?;
78166 Ok(rows)
79167 }
168+
169+ fn transaction(&mut self) -> anyhow::Result<Transaction> {
170+ let transaction = self.transaction.transaction()?;
171+ Ok(Transaction { transaction })
172+ }
173+}
174+
175+// Retry a database operation with exponential backoff and jitter
176+fn retry_automatically<T>(
177+ mut f: impl FnMut() -> Result<T, postgres::Error>,
178+) -> Result<T, postgres::Error> {
179+ const STARTING_WAIT_TIME: u64 = 100;
180+ const MAX_WAIT_TIME: u64 = 3_200;
181+ const MAX_ATTEMPTS: u32 = 10;
182+
183+ let mut rng = rand::thread_rng();
184+ let mut attempts = 0;
185+ loop {
186+ let result = f();
187+
188+ let error = match result {
189+ Ok(_) => return result,
190+ Err(err) => err,
191+ };
192+
193+ // If we got a database error, we check if it's retryable.
194+ // If we didn't get a database error, then it's most likely some kind of connection
195+ // error which should also be retried.
196+ if let Some(db_error) = error.as_db_error() {
197+ if !error_retryable(db_error) {
198+ return Err(error);
199+ }
200+ }
201+
202+ attempts += 1;
203+ if attempts >= MAX_ATTEMPTS {
204+ return Err(error);
205+ }
206+
207+ // The wait time increases exponentially, starting at 100ms and doubling up to a max of 3.2s.
208+ let wait_time = min(
209+ MAX_WAIT_TIME,
210+ STARTING_WAIT_TIME * u64::pow(2, attempts - 1),
211+ );
212+
213+ // The jitter is up to half the wait time
214+ let jitter: u64 = rng.gen_range(0..wait_time / 2);
215+
216+ std::thread::sleep(Duration::from_millis(wait_time + jitter));
217+ }
218+}
219+
220+// Check if a database error can be retried
221+fn error_retryable(error: &postgres::error::DbError) -> bool {
222+ // LOCK_NOT_AVAILABLE is caused by lock_timeout being exceeded
223+ matches!(error.code(), &postgres::error::SqlState::LOCK_NOT_AVAILABLE)
80224}
src/helpers.rs+8−17
@@ -2,37 +2,28 @@ use anyhow::Context;
22
33use crate::db::Conn;
44
5−pub fn setup_helpers(db: &mut dyn Conn, current_migration: &Option<String>) -> anyhow::Result<()> {
6− let predicate = if let Some(current_migration) = current_migration {
7− format!(
8− "current_setting('search_path') = 'migration_{}' OR setting_bool",
9− current_migration
10− )
11− } else {
12− format!("setting_bool")
13− };
14−
5+pub fn set_up_helpers(db: &mut dyn Conn, target_migration: &str) -> anyhow::Result<()> {
156 let query = format!(
167 "
17− CREATE OR REPLACE FUNCTION reshape.is_old_schema()
8+ CREATE OR REPLACE FUNCTION reshape.is_new_schema()
189 RETURNS BOOLEAN AS $$
1910 DECLARE
20− setting TEXT := current_setting('reshape.is_old_schema', TRUE);
11+ setting TEXT := current_setting('reshape.is_new_schema', TRUE);
2112 setting_bool BOOLEAN := setting IS NOT NULL AND setting = 'YES';
2213 BEGIN
23− RETURN {};
14+ RETURN current_setting('search_path') = 'migration_{}' OR setting_bool;
2415 END
2516 $$ language 'plpgsql';
2617 ",
27− predicate
18+ target_migration,
2819 );
2920 db.query(&query)
30− .context("failed creating helper function reshape.is_old_schema()")?;
21+ .context("failed creating helper function reshape.is_new_schema()")?;
3122
3223 Ok(())
3324}
3425
35−pub fn teardown_helpers(db: &mut dyn Conn) -> anyhow::Result<()> {
36− db.query("DROP FUNCTION reshape.is_old_schema;")?;
26+pub fn tear_down_helpers(db: &mut dyn Conn) -> anyhow::Result<()> {
27+ db.query("DROP FUNCTION IF EXISTS reshape.is_new_schema;")?;
3728 Ok(())
3829}
src/lib.rs+419−198
@@ -1,10 +1,11 @@
11use crate::{
2− migrations::{Context, Migration},
2+ migrations::{Migration, MigrationContext},
33 schema::Schema,
44};
55
6+use anyhow::{anyhow, Context};
67use colored::*;
7−use db::{Conn, DbConn};
8+use db::{Conn, DbConn, DbLocker};
89use postgres::Config;
910use schema::Table;
1011
@@ -14,11 +15,10 @@ pub mod migrations;
1415mod schema;
1516mod state;
1617
17−pub use crate::state::{State, Status};
18+pub use crate::state::State;
1819
1920pub struct Reshape {
20− pub state: State,
21− db: DbConn,
21+ db: DbLocker,
2222}
2323
2424impl Reshape {
@@ -30,6 +30,7 @@ impl Reshape {
3030 pub fn new_with_options(
3131 host: &str,
3232 port: u16,
33+ database: &str,
3334 username: &str,
3435 password: &str,
3536 ) -> anyhow::Result<Reshape> {
@@ -38,268 +39,488 @@ impl Reshape {
3839 .host(host)
3940 .port(port)
4041 .user(username)
42+ .dbname(database)
4143 .password(password);
4244
4345 Self::new_with_config(&config)
4446 }
4547
4648 fn new_with_config(config: &Config) -> anyhow::Result<Reshape> {
47− let mut db = DbConn::connect(config)?;
48− let state = State::load(&mut db);
49+ let db = DbLocker::connect(config)?;
50+ Ok(Reshape { db })
51+ }
4952
50− Ok(Reshape { db, state })
53+ pub fn migrate(
54+ &mut self,
55+ migrations: impl IntoIterator<Item = Migration>,
56+ ) -> anyhow::Result<()> {
57+ self.db.lock(|db| {
58+ let mut state = State::load(db)?;
59+ migrate(db, &mut state, migrations)
60+ })
5161 }
5262
53− pub fn migrate<T>(&mut self, migrations: T) -> anyhow::Result<()>
54− where
55− T: IntoIterator<Item = Migration>,
56− {
57− self.state = State::load(&mut self.db);
63+ pub fn complete(&mut self) -> anyhow::Result<()> {
64+ self.db.lock(|db| {
65+ let mut state = State::load(db)?;
66+ complete(db, &mut state)
67+ })
68+ }
5869
59− // Make sure no migration is in progress
60− if let state::Status::InProgress { migrations: _ } = &self.state.status {
61− println!("Migration already in progress, please complete using 'reshape complete'");
62− return Ok(());
63− }
70+ pub fn abort(&mut self) -> anyhow::Result<()> {
71+ self.db.lock(|db| {
72+ let mut state = State::load(db)?;
73+ abort(db, &mut state)
74+ })
75+ }
6476
65− let current_migration = &self.state.current_migration.clone();
66− let remaining_migrations = self.state.get_remaining_migrations(migrations)?;
67− if remaining_migrations.is_empty() {
68− println!("No migrations left to apply");
69− return Ok(());
70− }
77+ pub fn remove(&mut self) -> anyhow::Result<()> {
78+ self.db.lock(|db| {
79+ let mut state = State::load(db)?;
80+
81+ // Remove migration schemas and views
82+ if let Some(current_migration) = &state::current_migration(db)? {
83+ db.run(&format!(
84+ "DROP SCHEMA IF EXISTS {} CASCADE",
85+ schema_name_for_migration(current_migration)
86+ ))?;
87+ }
7188
72− println!(" Applying {} migrations\n", remaining_migrations.len());
89+ if let State::InProgress { migrations } = &state {
90+ let target_migration = migrations.last().unwrap().name.to_string();
91+ db.run(&format!(
92+ "DROP SCHEMA IF EXISTS {} CASCADE",
93+ schema_name_for_migration(&target_migration)
94+ ))?;
95+ }
7396
74− let target_migration = remaining_migrations.last().unwrap().name.to_string();
97+ // Remove all tables
98+ let schema = Schema::new();
99+ for table in schema.get_tables(db)? {
100+ db.run(&format!(
101+ r#"
102+ DROP TABLE IF EXISTS "{}" CASCADE
103+ "#,
104+ table.real_name
105+ ))?;
106+ }
75107
76− let mut new_schema = Schema::new();
108+ // Remove all enums
109+ let enums: Vec<String> = db
110+ .query("SELECT typname FROM pg_type WHERE typcategory = 'E'")?
111+ .iter()
112+ .map(|row| row.get("typname"))
113+ .collect();
114+ for enum_type in enums {
115+ db.run(&format!("DROP TYPE {}", enum_type))?;
116+ }
77117
78− helpers::setup_helpers(&mut self.db, current_migration)?;
118+ // Reset state
119+ state.clear(db)?;
79120
80− for (migration_index, migration) in remaining_migrations.iter().enumerate() {
81− println!("Migrating '{}':", migration.name);
121+ println!("Reshape and all data has been removed");
82122
83− for (action_index, action) in migration.actions.iter().enumerate() {
84− print!(" + {} ", action.describe());
123+ Ok(())
124+ })
125+ }
126+}
85127
86− let ctx = Context::new(migration_index, action_index);
87− action.run(&ctx, &mut self.db, &new_schema)?;
88− action.update_schema(&ctx, &mut new_schema)?;
128+pub fn latest_schema_from_migrations(migrations: &[Migration]) -> Option<String> {
129+ migrations
130+ .last()
131+ .map(|migration| schema_name_for_migration(&migration.name))
132+}
89133
90− println!("{}", "done".green());
91− }
134+pub fn schema_query_for_migration(migration_name: &str) -> String {
135+ let schema_name = schema_name_for_migration(migration_name);
136+ format!("SET search_path TO {}", schema_name)
137+}
92138
93− println!("");
94− }
139+fn schema_name_for_migration(migration_name: &str) -> String {
140+ format!("migration_{}", migration_name)
141+}
95142
96− // Create schema and views for migration
97− self.create_schema_for_migration(&target_migration, &new_schema)?;
143+fn migrate(
144+ db: &mut DbConn,
145+ state: &mut State,
146+ migrations: impl IntoIterator<Item = Migration>,
147+) -> anyhow::Result<()> {
148+ // Make sure no migration is in progress
149+ if let State::InProgress { .. } = &state {
150+ println!(
151+ "Migration already in progress, please complete using 'reshape migration complete'"
152+ );
153+ return Ok(());
154+ }
98155
99− // Update state once migrations have been performed
100− self.state.in_progress(remaining_migrations);
101− self.state.save(&mut self.db)?;
156+ if let State::Completing { .. } = &state {
157+ println!(
158+ "Migration already in progress and has started completion, please finish using 'reshape migration complete'"
159+ );
160+ return Ok(());
161+ }
102162
103− // If we started from a blank slate, we can finish the migration immediately
104− if current_migration.is_none() {
105− println!("Automatically completing migrations\n");
106− self.complete_migration()?;
163+ // Determine which migrations need to be applied by comparing the provided migrations
164+ // with the already applied ones stored in the state. This will throw an error if the
165+ // two sets of migrations don't agree, for example if a new migration has been added
166+ // in between two existing ones.
167+ let remaining_migrations = state::remaining_migrations(db, migrations)?;
168+ if remaining_migrations.is_empty() {
169+ println!("No migrations left to apply");
170+ return Ok(());
171+ }
107172
108− println!("Migrations complete:");
109− println!(
110− " - Run '{}' from your application to use the latest schema",
111− schema_query_for_migration(&target_migration)
112− );
113− } else {
114− println!("Migrations have been applied and the new schema is ready for use:");
115− println!(
116− " - Run '{}' from your application to use the latest schema",
117− schema_query_for_migration(&target_migration)
118− );
119− println!(
120− " - Run 'reshape complete' once your application has been updated and the previous schema is no longer in use"
121− );
173+ // If we have already started applying some migrations we need to ensure that
174+ // they are the same ones we want to apply now
175+ if let State::Applying {
176+ migrations: existing_migrations,
177+ } = &state
178+ {
179+ if existing_migrations != &remaining_migrations {
180+ return Err(anyhow!(
181+ "a previous migration seems to have failed without cleaning up. Please run `reshape migration abort` and then run migrate again."
182+ ));
122183 }
123−
124− Ok(())
125184 }
126185
127− pub fn complete_migration(&mut self) -> anyhow::Result<()> {
128− // Make sure a migration is in progress
129− let remaining_migrations = match &self.state.status {
130− state::Status::InProgress { migrations } => migrations,
131− _ => {
132− println!("No migration in progress");
133− return Ok(());
134− }
135− };
186+ // Move to the "Applying" state which is necessary as we can't run the migrations
187+ // and state update as a single transaction. If a migration unexpectedly fails without
188+ // automatically aborting, this state saves us from dangling migrations. It forces the user
189+ // to either run migrate again (which works as all migrations are idempotent) or abort.
190+ state.applying(remaining_migrations.clone());
191+ state.save(db)?;
136192
137− helpers::teardown_helpers(&mut self.db)?;
193+ println!("Applying {} migrations\n", remaining_migrations.len());
138194
139− let mut temp_schema = Schema::new();
195+ let target_migration = remaining_migrations.last().unwrap().name.to_string();
196+ helpers::set_up_helpers(db, &target_migration).context("failed to set up helpers")?;
140197
141− // Run all the completion changes as a transaction to avoid incomplete updates
142− let mut transaction = self.db.transaction()?;
198+ let mut new_schema = Schema::new();
199+ let mut last_migration_index = usize::MAX;
200+ let mut last_action_index = usize::MAX;
201+ let mut result: anyhow::Result<()> = Ok(());
143202
144− // Remove previous migration's schema
145− if let Some(current_migration) = &self.state.current_migration {
146− transaction.run(&format!(
147− "DROP SCHEMA IF EXISTS {} CASCADE",
148− schema_name_for_migration(current_migration)
149− ))?;
150− }
203+ 'outer: for (migration_index, migration) in remaining_migrations.iter().enumerate() {
204+ println!("Migrating '{}':", migration.name);
205+ last_migration_index = migration_index;
151206
152− for (migration_index, migration) in remaining_migrations.iter().enumerate() {
153− println!("Completing '{}':", migration.name);
207+ for (action_index, action) in migration.actions.iter().enumerate() {
208+ last_action_index = action_index;
154209
155− for (action_index, action) in migration.actions.iter().enumerate() {
156− print!(" + {} ", action.describe());
210+ let description = action.describe();
211+ print!(" + {} ", description);
157212
158− let ctx = Context::new(migration_index, action_index);
159− action.complete(&ctx, &mut transaction, &temp_schema)?;
160− action.update_schema(&ctx, &mut temp_schema)?;
213+ let ctx =
214+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
215+ result = action
216+ .run(&ctx, db, &new_schema)
217+ .with_context(|| format!("failed to {}", description));
161218
219+ if result.is_ok() {
220+ action.update_schema(&ctx, &mut new_schema);
162221 println!("{}", "done".green());
222+ } else {
223+ println!("{}", "failed".red());
224+ break 'outer;
163225 }
164−
165− println!("");
166226 }
167227
168− self.state.complete()?;
169− self.state.save(&mut transaction)?;
228+ println!();
229+ }
170230
171− transaction.commit()?;
231+ // If a migration failed, we abort all the migrations that were applied
232+ if let Err(err) = result {
233+ println!("A migration failed, aborting migrations that have already been applied");
172234
173− Ok(())
174− }
235+ // Set to the Aborting state. This is to ensure that the failed
236+ // migration is fully aborted and nothing is left dangling.
237+ // If the abort is interrupted for any reason, the user can try again
238+ // by running `reshape migration abort`.
239+ state.aborting(
240+ remaining_migrations.clone(),
241+ last_migration_index + 1,
242+ last_action_index + 1,
243+ );
175244
176− fn create_schema_for_migration(
177− &mut self,
178− migration_name: &str,
179− schema: &Schema,
180− ) -> anyhow::Result<()> {
181− // Create schema for migration
182− let schema_name = schema_name_for_migration(migration_name);
183− self.db
184− .run(&format!("CREATE SCHEMA IF NOT EXISTS {}", schema_name))?;
185−
186− // Create views inside schema
187− for table in schema.get_tables(&mut self.db)? {
188− Self::create_view_for_table(&mut self.db, &table, &schema_name)?;
189− }
245+ // Abort will only
246+ abort(db, state)?;
190247
191− Ok(())
248+ return Err(err);
192249 }
193250
194− fn create_view_for_table(
195− db: &mut impl Conn,
196− table: &Table,
197− schema: &str,
198− ) -> anyhow::Result<()> {
199− let select_columns: Vec<String> = table
200− .columns
201− .iter()
202− .map(|column| format!("{} AS {}", column.real_name, column.name))
203− .collect();
251+ // Create schema and views for migration
252+ create_schema_for_migration(db, &target_migration, &new_schema)
253+ .with_context(|| format!("failed to create schema for migration {}", target_migration))?;
254+
255+ // Update state once migrations have been performed
256+ state.in_progress(remaining_migrations);
257+ state.save(db).context("failed to save in-progress state")?;
258+
259+ println!("Migrations have been applied and the new schema is ready for use:");
260+ println!(
261+ " - Run '{}' from your application to use the latest schema",
262+ schema_query_for_migration(&target_migration)
263+ );
264+ println!(
265+ " - Run 'reshape migration complete' once your application has been updated and the previous schema is no longer in use"
266+ );
267+ Ok(())
268+}
204269
270+fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
271+ // Make sure a migration is in progress
272+ let (remaining_migrations, starting_migration_index, starting_action_index) = match state.clone() {
273+ State::InProgress { migrations } => {
274+ // Move into the Completing state. Once in this state,
275+ // the migration can't be aborted and must be completed.
276+ state.completing(migrations.clone(), 0, 0);
277+ state.save(db).context("failed to save state")?;
278+
279+ (migrations, 0, 0)
280+ },
281+ State::Completing {
282+ migrations,
283+ current_migration_index,
284+ current_action_index
285+ } => (migrations, current_migration_index, current_action_index),
286+ State::Aborting { .. } => {
287+ return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape migration abort`."))
288+ }
289+ State::Applying { .. } => {
290+ return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
291+ }
292+ State::Idle => {
293+ println!("No migration in progress");
294+ return Ok(());
295+ }
296+ };
297+
298+ // Remove previous migration's schema
299+ if let Some(current_migration) = &state::current_migration(db)? {
205300 db.run(&format!(
206− "CREATE OR REPLACE VIEW {schema}.{view_name} AS
207− SELECT {columns}
208− FROM {table_name}",
209− schema = schema,
210− table_name = table.real_name,
211− view_name = table.name,
212− columns = select_columns.join(","),
213− ))?;
214−
215− Ok(())
301+ "DROP SCHEMA IF EXISTS {} CASCADE",
302+ schema_name_for_migration(current_migration)
303+ ))
304+ .context("failed to remove previous migration's schema")?;
216305 }
217306
218− pub fn remove(&mut self) -> anyhow::Result<()> {
219− // Remove migration schemas and views
220− if let Some(current_migration) = &self.state.current_migration {
221− self.db.run(&format!(
222− "DROP SCHEMA IF EXISTS {} CASCADE",
223− schema_name_for_migration(current_migration)
224− ))?;
307+ for (migration_index, migration) in remaining_migrations.iter().enumerate() {
308+ // Skip all the migrations which have already been completed
309+ if migration_index < starting_migration_index {
310+ continue;
225311 }
226312
227− if let Status::InProgress { migrations } = &self.state.status {
228− let target_migration = migrations.last().unwrap().name.to_string();
229− self.db.run(&format!(
230− "DROP SCHEMA IF EXISTS {} CASCADE",
231− schema_name_for_migration(&target_migration)
232− ))?;
233− }
313+ println!("Completing '{}':", migration.name);
234314
235− // Remove all tables
236− let schema = Schema::new();
237− for table in schema.get_tables(&mut self.db)? {
238− self.db
239− .run(&format!("DROP TABLE IF EXISTS {} CASCADE", table.real_name))?;
240− }
315+ for (action_index, action) in migration.actions.iter().enumerate() {
316+ // Skip all actions which have already been completed
317+ if migration_index == starting_migration_index && action_index < starting_action_index {
318+ continue;
319+ }
241320
242− // Reset state
243− self.state.clear(&mut self.db)?;
321+ let description = action.describe();
322+ print!(" + {} ", description);
244323
245− println!("Reshape and all data has been removed");
324+ let ctx =
325+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
246326
247− Ok(())
248− }
327+ // Update state to indicate that this action has been completed.
328+ // We won't save this new state until after the action has completed.
329+ state.completing(
330+ remaining_migrations.clone(),
331+ migration_index + 1,
332+ action_index + 1,
333+ );
249334
250− pub fn abort(&mut self) -> anyhow::Result<()> {
251− let remaining_migrations = match &self.state.status {
252− Status::InProgress { migrations } => migrations,
253− _ => {
254− println!("No migration is in progress");
255− return Ok(());
335+ // This did_save check is necessary because of the borrow checker.
336+ // The Transaction which might be returned from action.complete
337+ // contains a mutable reference to self.db. We need the Transaction
338+ // to be dropped before we can save the state using self.db instead,
339+ // which we achieve here by limiting the lifetime of the Transaction
340+ // with a new block.
341+ let did_save = {
342+ let result = action
343+ .complete(&ctx, db)
344+ .with_context(|| format!("failed to complete migration {}", migration.name))
345+ .with_context(|| format!("failed to complete action: {}", description));
346+
347+ let maybe_transaction = match result {
348+ Ok(maybe_transaction) => {
349+ println!("{}", "done".green());
350+ maybe_transaction
351+ }
352+ Err(e) => {
353+ println!("{}", "failed".red());
354+ return Err(e);
355+ }
356+ };
357+
358+ // Update state with which migrations and actions have been completed.
359+ // Each action can create and return a transaction if they need atomicity.
360+ // We use this transaction to update the state to ensure the action only completes.
361+ // once.
362+ // We want to use a single transaction for each action to keep the length of
363+ // the transaction as short as possible. Wherever possible, we don't want to
364+ // use a transaction at all.
365+ if let Some(mut transaction) = maybe_transaction {
366+ state
367+ .save(&mut transaction)
368+ .context("failed to save state after completing action")?;
369+ transaction
370+ .commit()
371+ .context("failed to commit transaction")?;
372+
373+ true
374+ } else {
375+ false
376+ }
377+ };
378+
379+ // If the action didn't return a transaction we save the state normally instead
380+ if !did_save {
381+ state
382+ .save(db)
383+ .context("failed to save state after completing action")?;
256384 }
257− };
258− let target_migration = remaining_migrations.last().unwrap().name.to_string();
385+ }
259386
260− helpers::teardown_helpers(&mut self.db)?;
387+ println!();
388+ }
261389
262− // Run all the abort changes as a transaction to avoid incomplete changes
263− let mut transaction = self.db.transaction()?;
390+ // Remove helpers which are no longer in use
391+ helpers::tear_down_helpers(db).context("failed to tear down helpers")?;
264392
265− // Remove new migration's schema
266− transaction.run(&format!(
267− "DROP SCHEMA IF EXISTS {} CASCADE",
268− schema_name_for_migration(&target_migration)
269− ))?;
270−
271− // Abort all pending migrations in reverse order
272− for (migration_index, migration) in remaining_migrations.iter().rev().enumerate() {
273− print!("Aborting'{}' ", migration.name);
274− for (action_index, action) in migration.actions.iter().rev().enumerate() {
275− let ctx = Context::new(migration_index, action_index);
276− action.abort(&ctx, &mut transaction)?;
277− }
278− println!("{}", "done".green());
393+ state
394+ .complete(db)
395+ .context("failed to update state as completed")?;
396+
397+ Ok(())
398+}
399+
400+fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
401+ let (remaining_migrations, last_migration_index, last_action_index) = match state.clone() {
402+ State::InProgress { migrations } | State::Applying { migrations } => {
403+ // Set to the Aborting state. Once this is done, the migration has to
404+ // be fully aborted and can't be completed.
405+ state.aborting(migrations.clone(), 0, 0);
406+ state.save(db)?;
407+
408+ (migrations, usize::MAX, usize::MAX)
409+ }
410+ State::Aborting {
411+ migrations,
412+ last_migration_index,
413+ last_action_index,
414+ } => (migrations, last_migration_index, last_action_index),
415+ State::Completing { .. } => {
416+ return Err(anyhow!("migration completion has already been started. Please run `reshape migration complete` again to finish it."));
417+ }
418+ State::Idle => {
419+ println!("No migration is in progress");
420+ return Ok(());
421+ }
422+ };
423+
424+ // Remove new migration's schema
425+ let target_migration = remaining_migrations.last().unwrap().name.to_string();
426+ let schema_name = schema_name_for_migration(&target_migration);
427+ db.run(&format!("DROP SCHEMA IF EXISTS {} CASCADE", schema_name,))
428+ .with_context(|| format!("failed to drop schema {}", schema_name))?;
429+
430+ // Abort all pending migrations
431+ // Abort all migrations in reverse order
432+ for (migration_index, migration) in remaining_migrations.iter().enumerate().rev() {
433+ // Skip migrations which shouldn't be aborted
434+ // The reason can be that they have already been aborted or that
435+ // the migration was never applied in the first place.
436+ if migration_index >= last_migration_index {
437+ continue;
279438 }
280439
281− let keep_count = self.state.migrations.len() - remaining_migrations.len();
282− self.state.migrations.truncate(keep_count);
283− self.state.status = state::Status::Idle;
284− self.state.save(&mut transaction)?;
440+ print!("Aborting '{}' ", migration.name);
285441
286− transaction.commit()?;
442+ for (action_index, action) in migration.actions.iter().enumerate().rev() {
443+ // Skip actions which shouldn't be aborted
444+ // The reason can be that they have already been aborted or that
445+ // the action was never applied in the first place.
446+ if migration_index == last_migration_index - 1 && action_index >= last_action_index {
447+ continue;
448+ }
287449
288− Ok(())
450+ let ctx =
451+ MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
452+ action
453+ .abort(&ctx, db)
454+ .with_context(|| format!("failed to abort migration {}", migration.name))
455+ .with_context(|| format!("failed to abort action: {}", action.describe()))?;
456+
457+ // Update state with which migrations and actions have been aborted.
458+ // We don't need to run this in a transaction as aborts are idempotent.
459+ state.aborting(remaining_migrations.to_vec(), migration_index, action_index);
460+ state.save(db).context("failed to save state")?;
461+ }
462+
463+ println!("{}", "done".green());
289464 }
290−}
291465
292−pub fn latest_schema_from_migrations(migrations: &[Migration]) -> Option<String> {
293− migrations
294− .last()
295− .map(|migration| schema_name_for_migration(&migration.name))
466+ helpers::tear_down_helpers(db).context("failed to tear down helpers")?;
467+
468+ *state = State::Idle;
469+ state.save(db).context("failed to save state")?;
470+
471+ Ok(())
296472}
297473
298−pub fn schema_query_for_migration(migration_name: &str) -> String {
474+fn create_schema_for_migration(
475+ db: &mut DbConn,
476+ migration_name: &str,
477+ schema: &Schema,
478+) -> anyhow::Result<()> {
479+ // Create schema for migration
299480 let schema_name = schema_name_for_migration(migration_name);
300− format!("SET search_path TO {}", schema_name)
481+ db.run(&format!("CREATE SCHEMA IF NOT EXISTS {}", schema_name))
482+ .with_context(|| {
483+ format!(
484+ "failed to create schema {} for migration {}",
485+ schema_name, migration_name
486+ )
487+ })?;
488+
489+ // Create views inside schema
490+ for table in schema.get_tables(db)? {
491+ create_view_for_table(db, &table, &schema_name)?;
492+ }
493+
494+ Ok(())
301495}
302496
303−fn schema_name_for_migration(migration_name: &str) -> String {
304− format!("migration_{}", migration_name)
497+fn create_view_for_table(db: &mut impl Conn, table: &Table, schema: &str) -> anyhow::Result<()> {
498+ let select_columns: Vec<String> = table
499+ .columns
500+ .iter()
501+ .map(|column| {
502+ format!(
503+ r#"
504+ "{real_name}" AS "{alias}"
505+ "#,
506+ real_name = column.real_name,
507+ alias = column.name,
508+ )
509+ })
510+ .collect();
511+
512+ db.run(&format!(
513+ r#"
514+ CREATE OR REPLACE VIEW {schema}."{view_name}" AS
515+ SELECT {columns}
516+ FROM "{table_name}"
517+ "#,
518+ schema = schema,
519+ table_name = table.real_name,
520+ view_name = table.name,
521+ columns = select_columns.join(","),
522+ ))
523+ .with_context(|| format!("failed to create view for table {}", table.name))?;
524+
525+ Ok(())
305526}
src/main.rs+98−27
@@ -4,6 +4,7 @@ use std::{
44 path::Path,
55};
66
7+use anyhow::Context;
78use clap::{Args, Parser};
89use reshape::{
910 migrations::{Action, Migration},
@@ -12,22 +13,69 @@ use reshape::{
1213use serde::{Deserialize, Serialize};
1314
1415#[derive(Parser)]
16+#[clap(name = "Reshape", version, about)]
1517struct Opts {
1618 #[clap(subcommand)]
1719 cmd: Command,
1820}
1921
2022#[derive(Parser)]
23+#[clap(about)]
2124enum Command {
25+ #[clap(subcommand)]
26+ Migration(MigrationCommand),
27+
28+ #[clap(
29+ about = "Output the query your application should use to select the right schema",
30+ display_order = 2
31+ )]
32+ SchemaQuery(FindMigrationsOptions),
33+
34+ #[clap(
35+ about = "Deprecated. Use `reshape schema-query` instead",
36+ display_order = 3
37+ )]
38+ GenerateSchemaQuery(FindMigrationsOptions),
39+
40+ #[clap(
41+ about = "Deprecated. Use `reshape migration start` instead",
42+ display_order = 4
43+ )]
2244 Migrate(MigrateOptions),
45+ #[clap(
46+ about = "Deprecated. Use `reshape migration complete` instead",
47+ display_order = 5
48+ )]
2349 Complete(ConnectionOptions),
24− Remove(ConnectionOptions),
50+ #[clap(
51+ about = "Deprecated. Use `reshape migration abort` instead",
52+ display_order = 6
53+ )]
54+ Abort(ConnectionOptions),
55+}
56+
57+#[derive(Parser)]
58+#[clap(about = "Commands for managing migrations", display_order = 1)]
59+enum MigrationCommand {
60+ #[clap(
61+ about = "Starts a new migration, applying any migrations which haven't yet been applied",
62+ display_order = 1
63+ )]
64+ Start(MigrateOptions),
65+
66+ #[clap(about = "Completes an in-progress migration", display_order = 2)]
67+ Complete(ConnectionOptions),
68+
69+ #[clap(
70+ about = "Aborts an in-progress migration without losing any data",
71+ display_order = 3
72+ )]
2573 Abort(ConnectionOptions),
26− GenerateSchemaQuery(FindMigrationsOptions),
2774}
2875
2976#[derive(Args)]
3077struct MigrateOptions {
78+ // Some comment
3179 #[clap(long, short)]
3280 complete: bool,
3381 #[clap(flatten)]
@@ -58,43 +106,35 @@ struct FindMigrationsOptions {
58106 dirs: Vec<String>,
59107}
60108
61−fn main() {
109+fn main() -> anyhow::Result<()> {
62110 let opts: Opts = Opts::parse();
63−
64− let result = run(opts);
65− if let Err(e) = result {
66− println!("Error: {}", e);
67− }
111+ run(opts)
68112}
69113
70114fn run(opts: Opts) -> anyhow::Result<()> {
71115 match opts.cmd {
72− Command::Migrate(opts) => {
116+ Command::Migration(MigrationCommand::Start(opts)) | Command::Migrate(opts) => {
73117 let mut reshape = reshape_from_connection_options(&opts.connection_options)?;
74118 let migrations = find_migrations(&opts.find_migrations_options)?;
75119 reshape.migrate(migrations)?;
76120
77121 // Automatically complete migration if --complete flag is set
78122 if opts.complete {
79− reshape.complete_migration()?;
123+ reshape.complete()?;
80124 }
81125
82126 Ok(())
83127 }
84− Command::Complete(opts) => {
85− let mut reshape = reshape_from_connection_options(&opts)?;
86− reshape.complete_migration()
87− }
88− Command::Remove(opts) => {
128+ Command::Migration(MigrationCommand::Complete(opts)) | Command::Complete(opts) => {
89129 let mut reshape = reshape_from_connection_options(&opts)?;
90− reshape.remove()
130+ reshape.complete()
91131 }
92− Command::Abort(opts) => {
132+ Command::Migration(MigrationCommand::Abort(opts)) | Command::Abort(opts) => {
93133 let mut reshape = reshape_from_connection_options(&opts)?;
94134 reshape.abort()
95135 }
96− Command::GenerateSchemaQuery(find_migrations_options) => {
97− let migrations = find_migrations(&find_migrations_options)?;
136+ Command::SchemaQuery(opts) | Command::GenerateSchemaQuery(opts) => {
137+ let migrations = find_migrations(&opts)?;
98138 let query = migrations
99139 .last()
100140 .map(|migration| reshape::schema_query_for_migration(&migration.name));
@@ -106,20 +146,42 @@ fn run(opts: Opts) -> anyhow::Result<()> {
106146}
107147
108148fn reshape_from_connection_options(opts: &ConnectionOptions) -> anyhow::Result<Reshape> {
109− let env_url = std::env::var("POSTGRES_URL").ok();
110− let url = env_url.as_ref().or(opts.url.as_ref());
149+ // Load environment variables from .env file if it exists
150+ dotenv::dotenv().ok();
151+
152+ let url_env = std::env::var("DB_URL").ok();
153+ let url = url_env.as_ref().or_else(|| opts.url.as_ref());
111154
112− match url {
113− Some(url) => Reshape::new(&url),
114− None => Reshape::new_with_options(&opts.host, opts.port, &opts.username, &opts.password),
155+ // Use the connection URL if it has been set
156+ if let Some(url) = url {
157+ return Reshape::new(url);
115158 }
159+
160+ let host_env = std::env::var("DB_HOST").ok();
161+ let host = host_env.as_ref().unwrap_or_else(|| &opts.host);
162+
163+ let port = std::env::var("DB_PORT")
164+ .ok()
165+ .and_then(|port| port.parse::<u16>().ok())
166+ .unwrap_or(opts.port);
167+
168+ let username_env = std::env::var("DB_USERNAME").ok();
169+ let username = username_env.as_ref().unwrap_or_else(|| &opts.username);
170+
171+ let password_env = std::env::var("DB_PASSWORD").ok();
172+ let password = password_env.as_ref().unwrap_or_else(|| &opts.password);
173+
174+ let database_env = std::env::var("DB_NAME").ok();
175+ let database = database_env.as_ref().unwrap_or_else(|| &opts.database);
176+
177+ Reshape::new_with_options(host, port, database, username, password)
116178}
117179
118180fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration>> {
119181 let search_paths = opts
120182 .dirs
121183 .iter()
122− .map(|path| Path::new(path))
184+ .map(Path::new)
123185 // Filter out all directories that don't exist
124186 .filter(|path| path.exists());
125187
@@ -134,7 +196,13 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
134196 }
135197
136198 // Sort all files by their file names (without extension)
137− file_paths.sort_unstable_by_key(|path| path.as_path().file_stem().unwrap().to_os_string());
199+ // The files are sorted naturally, e.g. "1_test_migration" < "10_test_migration"
200+ file_paths.sort_unstable_by(|path1, path2| {
201+ let file1 = path1.as_path().file_stem().unwrap().to_str().unwrap();
202+ let file2 = path2.as_path().file_stem().unwrap().to_str().unwrap();
203+
204+ lexical_sort::natural_cmp(file1, file2)
205+ });
138206
139207 file_paths
140208 .iter()
@@ -150,7 +218,10 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
150218 .map(|result| {
151219 result.and_then(|(path, data)| {
152220 let extension = path.extension().and_then(|ext| ext.to_str()).unwrap();
153− let file_migration = decode_migration_file(&data, extension)?;
221+ let file_migration =
222+ decode_migration_file(&data, extension).with_context(|| {
223+ format!("failed to parse migration file {}", path.display())
224+ })?;
154225
155226 let file_name = path.file_stem().and_then(|name| name.to_str()).unwrap();
156227 Ok(Migration {
src/migrations/add_column.rs+294−116
@@ -1,16 +1,31 @@
1−use super::{common, Action, Column, Context};
2−use crate::{db::Conn, schema::Schema};
1+use super::{common, Action, Column, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::{bail, Context};
37use serde::{Deserialize, Serialize};
48
59#[derive(Serialize, Deserialize, Debug)]
610pub struct AddColumn {
711 pub table: String,
8− pub up: Option<String>,
912 pub column: Column,
13+ pub up: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
1025}
1126
1227impl AddColumn {
13− fn temp_column_name(&self, ctx: &Context) -> String {
28+ fn temp_column_name(&self, ctx: &MigrationContext) -> String {
1429 format!(
1530 "{}_temp_column_{}_{}",
1631 ctx.prefix(),
@@ -19,7 +34,7 @@ impl AddColumn {
1934 )
2035 }
2136
22− fn trigger_name(&self, ctx: &Context) -> String {
37+ fn trigger_name(&self, ctx: &MigrationContext) -> String {
2338 format!(
2439 "{}_add_column_{}_{}",
2540 ctx.prefix(),
@@ -28,7 +43,16 @@ impl AddColumn {
2843 )
2944 }
3045
31− fn not_null_constraint_name(&self, ctx: &Context) -> String {
46+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_add_column_{}_{}_rev",
49+ ctx.prefix(),
50+ self.table,
51+ self.column.name
52+ )
53+ }
54+
55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
3256 format!(
3357 "{}_add_column_not_null_{}_{}",
3458 ctx.prefix(),
@@ -47,12 +71,17 @@ impl Action for AddColumn {
4771 )
4872 }
4973
50− fn run(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
74+ fn run(
75+ &self,
76+ ctx: &MigrationContext,
77+ db: &mut dyn Conn,
78+ schema: &Schema,
79+ ) -> anyhow::Result<()> {
5180 let table = schema.get_table(db, &self.table)?;
5281 let temp_column_name = self.temp_column_name(ctx);
5382
5483 let mut definition_parts = vec![
55− temp_column_name.to_string(),
84+ format!("\"{}\"", temp_column_name.to_string()),
5685 self.column.data_type.to_string(),
5786 ];
5887
@@ -68,63 +97,198 @@ impl Action for AddColumn {
6897
6998 // Add column as NOT NULL
7099 let query = format!(
71− "
72− ALTER TABLE {table}
100+ r#"
101+ ALTER TABLE "{table}"
73102 ADD COLUMN IF NOT EXISTS {definition};
74− ",
103+ "#,
75104 table = self.table,
76105 definition = definition_parts.join(" "),
77106 );
78− db.run(&query)?;
107+ db.run(&query).context("failed to add column")?;
108+
109+ let declarations: Vec<String> = table
110+ .columns
111+ .iter()
112+ .map(|column| {
113+ format!(
114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",
115+ table = table.real_name,
116+ alias = column.name,
117+ real_name = column.real_name,
118+ )
119+ })
120+ .collect();
79121
80122 if let Some(up) = &self.up {
81− let table = schema.get_table(db, &self.table)?;
82−
83− let declarations: Vec<String> = table
84− .columns
85− .iter()
86− .map(|column| {
87− format!(
88− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
89− table = table.real_name,
90− alias = column.name,
91− real_name = column.real_name,
92− )
93− })
94− .collect();
95−
96− // Add triggers to fill in values as they are inserted/updated
97− let query = format!(
98− "
99− CREATE OR REPLACE FUNCTION {trigger_name}()
100− RETURNS TRIGGER AS $$
101− BEGIN
102− IF reshape.is_old_schema() THEN
103− DECLARE
104− {declarations}
105− BEGIN
106− NEW.{temp_column_name} = {up};
107− END;
108− END IF;
109− RETURN NEW;
110− END
111− $$ language 'plpgsql';
112−
113− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
114− CREATE TRIGGER {trigger_name} BEFORE UPDATE OR INSERT ON {table} FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
115− ",
116− temp_column_name = temp_column_name,
117− trigger_name = self.trigger_name(ctx),
118− up = up,
119− table = self.table,
120− declarations = declarations.join("\n"),
121− );
122− db.run(&query)?;
123− }
123+ if let Transformation::Simple(up) = up {
124+ // Add triggers to fill in values as they are inserted/updated
125+ let query = format!(
126+ r#"
127+ CREATE OR REPLACE FUNCTION {trigger_name}()
128+ RETURNS TRIGGER AS $$
129+ BEGIN
130+ IF NOT reshape.is_new_schema() THEN
131+ DECLARE
132+ {declarations}
133+ BEGIN
134+ NEW."{temp_column_name}" = {up};
135+ END;
136+ END IF;
137+ RETURN NEW;
138+ END
139+ $$ language 'plpgsql';
140+
141+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
142+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
143+ "#,
144+ temp_column_name = temp_column_name,
145+ trigger_name = self.trigger_name(ctx),
146+ up = up,
147+ table = self.table,
148+ declarations = declarations.join("\n"),
149+ );
150+ db.run(&query).context("failed to create up trigger")?;
151+
152+ // Backfill values in batches
153+ common::batch_touch_rows(db, &table.real_name, Some(&temp_column_name))
154+ .context("failed to batch update existing rows")?;
155+ }
156+
157+ if let Transformation::Update {
158+ table: from_table,
159+ value,
160+ r#where,
161+ } = up
162+ {
163+ let existing_schema_name = match &ctx.existing_schema_name {
164+ Some(name) => name,
165+ None => bail!("can't use update without previous migration"),
166+ };
167+
168+ let from_table = schema.get_table(db, &from_table)?;
169+
170+ let from_table_assignments: Vec<String> = from_table
171+ .columns
172+ .iter()
173+ .map(|column| {
174+ format!(
175+ "{table}.{alias} = NEW.{real_name};",
176+ table = from_table.name,
177+ alias = column.name,
178+ real_name = column.real_name,
179+ )
180+ })
181+ .collect();
182+
183+ // Add triggers to fill in values as they are inserted/updated
184+ let query = format!(
185+ r#"
186+ CREATE OR REPLACE FUNCTION {trigger_name}()
187+ RETURNS TRIGGER AS $$
188+ #variable_conflict use_variable
189+ BEGIN
190+ IF NOT reshape.is_new_schema() THEN
191+ DECLARE
192+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
193+ BEGIN
194+ {assignments}
195+
196+ -- Don't trigger reverse trigger when making this update
197+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
198+
199+ UPDATE public."{changed_table_real}"
200+ SET "{temp_column_name}" = {value}
201+ WHERE {where};
124202
125− // Backfill values in batches
126− if self.up.is_some() {
127− common::batch_touch_rows(db, &table.real_name, &temp_column_name)?;
203+ perform set_config('reshape.disable_triggers', '', TRUE);
204+ END;
205+ END IF;
206+ RETURN NEW;
207+ END
208+ $$ language 'plpgsql';
209+
210+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
211+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
212+ "#,
213+ assignments = from_table_assignments.join("\n"),
214+ changed_table_real = table.real_name,
215+ from_table = from_table.name,
216+ from_table_real = from_table.real_name,
217+ trigger_name = self.trigger_name(ctx),
218+ // declarations = from_table_declarations.join("\n"),
219+ temp_column_name = temp_column_name,
220+ );
221+ db.run(&query).context("failed to create up trigger")?;
222+
223+ let from_table_columns = from_table
224+ .columns
225+ .iter()
226+ .map(|column| format!("{} as {}", column.real_name, column.name))
227+ .collect::<Vec<String>>()
228+ .join(", ");
229+
230+ let changed_table_assignments: Vec<String> = table
231+ .columns
232+ .iter()
233+ .map(|column| {
234+ format!(
235+ "{table}.{alias} := NEW.{real_name};",
236+ table = table.name,
237+ alias = column.name,
238+ real_name = column.real_name,
239+ )
240+ })
241+ .collect();
242+
243+ // Add triggers to fill in values as they are inserted/updated
244+ let query = format!(
245+ r#"
246+ CREATE OR REPLACE FUNCTION {trigger_name}()
247+ RETURNS TRIGGER AS $$
248+ #variable_conflict use_variable
249+ BEGIN
250+ IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
251+ DECLARE
252+ {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;
253+ __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;
254+ BEGIN
255+ {changed_table_assignments}
256+
257+ SELECT {from_table_columns}
258+ INTO __temp_row
259+ FROM migration_{existing_schema_name}.{from_table} {from_table}
260+ WHERE {where};
261+
262+ DECLARE
263+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
264+ BEGIN
265+ {from_table} = __temp_row;
266+ NEW.{temp_column_name} = {value};
267+ END;
268+ END;
269+ END IF;
270+ RETURN NEW;
271+ END
272+ $$ language 'plpgsql';
273+
274+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
275+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
276+ "#,
277+ changed_table_assignments = changed_table_assignments.join("\n"),
278+ changed_table_real = table.real_name,
279+ changed_table = table.name,
280+ from_table = from_table.name,
281+ trigger_name = self.reverse_trigger_name(ctx),
282+ temp_column_name = temp_column_name,
283+ // declarations = declarations.join("\n"),
284+ );
285+ db.run(&query)
286+ .context("failed to create reverse up trigger")?;
287+
288+ // Backfill values in batches by touching the from table
289+ common::batch_touch_rows(db, &from_table.real_name, None)
290+ .context("failed to batch update existing rows")?;
291+ }
128292 }
129293
130294 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
@@ -133,120 +297,134 @@ impl Action for AddColumn {
133297 // Thanks to this, we can set the full column as NOT NULL later with minimal locking.
134298 if !self.column.nullable {
135299 let query = format!(
136− "
137− ALTER TABLE {table}
138− ADD CONSTRAINT {constraint_name}
139− CHECK ({column} IS NOT NULL) NOT VALID
140− ",
300+ r#"
301+ ALTER TABLE "{table}"
302+ ADD CONSTRAINT "{constraint_name}"
303+ CHECK ("{column}" IS NOT NULL) NOT VALID
304+ "#,
141305 table = self.table,
142− constraint_name = self.not_null_constraint_name(&ctx),
306+ constraint_name = self.not_null_constraint_name(ctx),
143307 column = temp_column_name,
144308 );
145− db.run(&query)?;
309+ db.run(&query)
310+ .context("failed to add NOT NULL constraint")?;
146311 }
147312
148313 Ok(())
149314 }
150315
151− fn complete(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
152− let table = schema.get_table(db, &self.table)?;
316+ fn complete<'a>(
317+ &self,
318+ ctx: &MigrationContext,
319+ db: &'a mut dyn Conn,
320+ ) -> anyhow::Result<Option<Transaction<'a>>> {
321+ let mut transaction = db.transaction().context("failed to create transaction")?;
153322
154323 // Remove triggers and procedures
155324 let query = format!(
156− "
157− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
158− DROP FUNCTION IF EXISTS {trigger_name};
159− ",
160− table = self.table,
325+ r#"
326+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
327+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
328+ "#,
161329 trigger_name = self.trigger_name(ctx),
330+ reverse_trigger_name = self.reverse_trigger_name(ctx),
162331 );
163− db.run(&query)?;
332+ transaction
333+ .run(&query)
334+ .context("failed to drop up trigger")?;
164335
165336 // Update column to be NOT NULL if necessary
166337 if !self.column.nullable {
167338 // Validate the temporary constraint (should always be valid).
168339 // This performs a sequential scan but does not take an exclusive lock.
169340 let query = format!(
170− "
171− ALTER TABLE {table}
172− VALIDATE CONSTRAINT {constraint_name}
173− ",
341+ r#"
342+ ALTER TABLE "{table}"
343+ VALIDATE CONSTRAINT "{constraint_name}"
344+ "#,
174345 table = self.table,
175346 constraint_name = self.not_null_constraint_name(ctx),
176347 );
177− db.run(&query)?;
348+ transaction
349+ .run(&query)
350+ .context("failed to validate NOT NULL constraint")?;
178351
179352 // Update the column to be NOT NULL.
180353 // This requires an exclusive lock but since PG 12 it can check
181354 // the existing constraint for correctness which makes the lock short-lived.
182355 // Source: https://dba.stackexchange.com/a/268128
183356 let query = format!(
184− "
185− ALTER TABLE {table}
186− ALTER COLUMN {column} SET NOT NULL
187− ",
357+ r#"
358+ ALTER TABLE "{table}"
359+ ALTER COLUMN "{column}" SET NOT NULL
360+ "#,
188361 table = self.table,
189362 column = self.temp_column_name(ctx),
190363 );
191− db.run(&query)?;
364+ transaction
365+ .run(&query)
366+ .context("failed to set column as NOT NULL")?;
192367
193368 // Drop the temporary constraint
194369 let query = format!(
195− "
196− ALTER TABLE {table}
197− DROP CONSTRAINT {constraint_name}
198− ",
370+ r#"
371+ ALTER TABLE "{table}"
372+ DROP CONSTRAINT "{constraint_name}"
373+ "#,
199374 table = self.table,
200375 constraint_name = self.not_null_constraint_name(ctx),
201376 );
202− db.run(&query)?;
377+ transaction
378+ .run(&query)
379+ .context("failed to drop NOT NULL constraint")?;
203380 }
204381
205382 // Rename the temporary column to its real name
206− db.run(&format!(
207− "
208− ALTER TABLE {table}
209− RENAME COLUMN {temp_column_name} TO {column_name}
210− ",
211− table = table.real_name,
212− temp_column_name = self.temp_column_name(ctx),
213− column_name = self.column.name,
214− ))?;
383+ transaction
384+ .run(&format!(
385+ r#"
386+ ALTER TABLE "{table}"
387+ RENAME COLUMN "{temp_column_name}" TO "{column_name}"
388+ "#,
389+ table = self.table,
390+ temp_column_name = self.temp_column_name(ctx),
391+ column_name = self.column.name,
392+ ))
393+ .context("failed to rename column to final name")?;
215394
216− Ok(())
395+ Ok(Some(transaction))
217396 }
218397
219− fn update_schema(&self, ctx: &Context, schema: &mut Schema) -> anyhow::Result<()> {
398+ fn update_schema(&self, ctx: &MigrationContext, schema: &mut Schema) {
220399 schema.change_table(&self.table, |table_changes| {
221400 table_changes.change_column(&self.column.name, |column_changes| {
222401 column_changes.set_column(&self.temp_column_name(ctx));
223402 })
224403 });
225− Ok(())
226404 }
227405
228− fn abort(&self, ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()> {
229− // Remove triggers and procedures
406+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
407+ // Remove column
230408 let query = format!(
231− "
232− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
233− DROP FUNCTION IF EXISTS {trigger_name};
234− ",
409+ r#"
410+ ALTER TABLE "{table}"
411+ DROP COLUMN IF EXISTS "{column}"
412+ "#,
235413 table = self.table,
236− trigger_name = self.trigger_name(ctx),
414+ column = self.temp_column_name(ctx),
237415 );
238− db.run(&query)?;
416+ db.run(&query).context("failed to drop column")?;
239417
240− // Remove column
418+ // Remove triggers and procedures
241419 let query = format!(
242− "
243− ALTER TABLE {table}
244− DROP COLUMN IF EXISTS {column}
245− ",
246− table = self.table,
247− column = self.temp_column_name(ctx),
420+ r#"
421+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
422+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
423+ "#,
424+ trigger_name = self.trigger_name(ctx),
425+ reverse_trigger_name = self.reverse_trigger_name(ctx),
248426 );
249− db.run(&query)?;
427+ db.run(&query).context("failed to drop up trigger")?;
250428
251429 Ok(())
252430 }
src/migrations/add_foreign_key.rsnew file+122
@@ -0,0 +1,122 @@
1+use super::{common::ForeignKey, Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct AddForeignKey {
11+ pub table: String,
12+ foreign_key: ForeignKey,
13+}
14+
15+#[typetag::serde(name = "add_foreign_key")]
16+impl Action for AddForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Adding foreign key from table \"{}\" to \"{}\"",
20+ self.table, self.foreign_key.referenced_table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ let table = schema.get_table(db, &self.table)?;
31+ let referenced_table = schema.get_table(db, &self.foreign_key.referenced_table)?;
32+
33+ // Add quotes around all column names
34+ let columns: Vec<String> = table
35+ .real_column_names(&self.foreign_key.columns)
36+ .map(|col| format!("\"{}\"", col))
37+ .collect();
38+ let referenced_columns: Vec<String> = referenced_table
39+ .real_column_names(&self.foreign_key.referenced_columns)
40+ .map(|col| format!("\"{}\"", col))
41+ .collect();
42+
43+ // Create foreign key but set is as NOT VALID.
44+ // This means the foreign key will be enforced for inserts and updates
45+ // but the existing data won't be checked, that would cause a long-lived lock.
46+ db.run(&format!(
47+ r#"
48+ ALTER TABLE "{table}"
49+ ADD CONSTRAINT {constraint_name}
50+ FOREIGN KEY ({columns})
51+ REFERENCES "{referenced_table}" ({referenced_columns})
52+ NOT VALID
53+ "#,
54+ table = table.real_name,
55+ constraint_name = self.temp_constraint_name(ctx),
56+ columns = columns.join(", "),
57+ referenced_table = referenced_table.real_name,
58+ referenced_columns = referenced_columns.join(", "),
59+ ))
60+ .context("failed to create foreign key")?;
61+
62+ db.run(&format!(
63+ r#"
64+ ALTER TABLE "{table}"
65+ VALIDATE CONSTRAINT "{constraint_name}"
66+ "#,
67+ table = table.real_name,
68+ constraint_name = self.temp_constraint_name(ctx),
69+ ))
70+ .context("failed to validate foreign key")?;
71+
72+ Ok(())
73+ }
74+
75+ fn complete<'a>(
76+ &self,
77+ ctx: &MigrationContext,
78+ db: &'a mut dyn Conn,
79+ ) -> anyhow::Result<Option<Transaction<'a>>> {
80+ db.run(&format!(
81+ r#"
82+ ALTER TABLE {table}
83+ RENAME CONSTRAINT {temp_constraint_name} TO {constraint_name}
84+ "#,
85+ table = self.table,
86+ temp_constraint_name = self.temp_constraint_name(ctx),
87+ constraint_name = self.final_constraint_name(),
88+ ))
89+ .context("failed to rename temporary constraint")?;
90+ Ok(None)
91+ }
92+
93+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
94+
95+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
96+ db.run(&format!(
97+ r#"
98+ ALTER TABLE "{table}"
99+ DROP CONSTRAINT IF EXISTS "{constraint_name}"
100+ "#,
101+ table = self.table,
102+ constraint_name = self.temp_constraint_name(ctx),
103+ ))
104+ .context("failed to validate foreign key")?;
105+
106+ Ok(())
107+ }
108+}
109+
110+impl AddForeignKey {
111+ fn temp_constraint_name(&self, ctx: &MigrationContext) -> String {
112+ format!("{}_temp_fkey", ctx.prefix())
113+ }
114+
115+ fn final_constraint_name(&self) -> String {
116+ format!(
117+ "{table}_{columns}_fkey",
118+ table = self.table,
119+ columns = self.foreign_key.columns.join("_")
120+ )
121+ }
122+}
src/migrations/add_index.rs+54−23
@@ -1,57 +1,88 @@
1−use super::{Action, Context};
2−use crate::{db::Conn, schema::Schema};
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
37use serde::{Deserialize, Serialize};
48
59#[derive(Serialize, Deserialize, Debug)]
610pub struct AddIndex {
711 pub table: String,
12+ pub index: Index,
13+}
14+
15+#[derive(Serialize, Deserialize, Clone, Debug)]
16+pub struct Index {
817 pub name: String,
918 pub columns: Vec<String>,
19+ #[serde(default)]
20+ pub unique: bool,
21+ #[serde(rename = "type")]
22+ pub index_type: Option<String>,
1023}
1124
1225#[typetag::serde(name = "add_index")]
1326impl Action for AddIndex {
1427 fn describe(&self) -> String {
15− format!("Adding index \"{}\" to table \"{}\"", self.name, self.table)
28+ format!(
29+ "Adding index \"{}\" to table \"{}\"",
30+ self.index.name, self.table
31+ )
1632 }
1733
18− fn run(&self, _ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
34+ fn run(
35+ &self,
36+ _ctx: &MigrationContext,
37+ db: &mut dyn Conn,
38+ schema: &Schema,
39+ ) -> anyhow::Result<()> {
1940 let table = schema.get_table(db, &self.table)?;
2041
2142 let column_real_names: Vec<String> = table
2243 .columns
2344 .iter()
24− .filter(|column| self.columns.contains(&column.name))
25− .map(|column| column.real_name.to_string())
45+ .filter(|column| self.index.columns.contains(&column.name))
46+ .map(|column| format!("\"{}\"", column.real_name))
2647 .collect();
2748
49+ let unique = if self.index.unique { "UNIQUE" } else { "" };
50+ let index_type_def = if let Some(index_type) = &self.index.index_type {
51+ format!("USING {index_type}")
52+ } else {
53+ "".to_string()
54+ };
55+
2856 db.run(&format!(
29− "
30− CREATE INDEX CONCURRENTLY {name} ON {table} ({columns})
31− ",
32− name = self.name,
57+ r#"
58+ CREATE {unique} INDEX CONCURRENTLY "{name}" ON "{table}" {index_type_def} ({columns})
59+ "#,
60+ name = self.index.name,
3361 table = self.table,
3462 columns = column_real_names.join(", "),
35− ))?;
63+ ))
64+ .context("failed to create index")?;
3665 Ok(())
3766 }
3867
39− fn complete(&self, _ctx: &Context, _db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
40− Ok(())
68+ fn complete<'a>(
69+ &self,
70+ _ctx: &MigrationContext,
71+ _db: &'a mut dyn Conn,
72+ ) -> anyhow::Result<Option<Transaction<'a>>> {
73+ Ok(None)
4174 }
4275
43− fn update_schema(&self, _ctx: &Context, _schema: &mut Schema) -> anyhow::Result<()> {
44− Ok(())
45− }
76+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
4677
47− fn abort(&self, _ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()> {
78+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
4879 db.run(&format!(
49− "
50− DROP INDEX {name} ON {table}
51− ",
52− name = self.name,
53− table = self.table,
54− ))?;
80+ r#"
81+ DROP INDEX CONCURRENTLY IF EXISTS "{name}"
82+ "#,
83+ name = self.index.name,
84+ ))
85+ .context("failed to drop index")?;
5586 Ok(())
5687 }
5788}
src/migrations/alter_column.rs+311−205
@@ -1,6 +1,10 @@
1−use super::{Action, Context};
2−use crate::{db::Conn, migrations::common, schema::Schema};
3−use anyhow::anyhow;
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ migrations::common,
5+ schema::Schema,
6+};
7+use anyhow::{anyhow, Context};
48use serde::{Deserialize, Serialize};
59
610#[derive(Serialize, Deserialize, Debug)]
@@ -9,63 +13,31 @@ pub struct AlterColumn {
913 pub column: String,
1014 pub up: Option<String>,
1115 pub down: Option<String>,
16+ #[serde(default)]
1217 pub changes: ColumnChanges,
1318}
1419
15−#[derive(Serialize, Deserialize, Debug)]
20+#[derive(Serialize, Deserialize, Default, Debug)]
1621pub struct ColumnChanges {
1722 pub name: Option<String>,
1823 #[serde(rename = "type")]
1924 pub data_type: Option<String>,
2025 pub nullable: Option<bool>,
26+ pub default: Option<String>,
2127}
2228
23−impl AlterColumn {
24− fn temporary_column_name(&self, ctx: &Context) -> String {
25− format!("{}_new_{}", ctx.prefix(), self.column)
26− }
27−
28− fn up_trigger_name(&self, ctx: &Context) -> String {
29− format!(
30− "{}_alter_column_up_trigger_{}_{}",
31− ctx.prefix(),
32− self.table,
33− self.column
34− )
35− }
36−
37− fn down_trigger_name(&self, ctx: &Context) -> String {
38− format!(
39− "{}_alter_column_down_trigger_{}_{}",
40− ctx.prefix_inverse(),
41− self.table,
42− self.column
43− )
44− }
45−
46− fn not_null_constraint_name(&self, ctx: &Context) -> String {
47− format!(
48− "{}_alter_column_temporary_not_null_{}_{}",
49− ctx.prefix(),
50− self.table,
51− self.column
52− )
53− }
54−
55− fn can_short_circuit(&self) -> bool {
56− self.changes.name.is_some()
57− && self.changes.data_type.is_none()
58− && self.changes.nullable.is_none()
59− }
60−}
61−
62−#[typetag::serde(name = "uppercase_column")]
29+#[typetag::serde(name = "alter_column")]
6330impl Action for AlterColumn {
6431 fn describe(&self) -> String {
6532 format!("Altering column \"{}\" on \"{}\"", self.column, self.table)
6633 }
6734
68− fn run(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
35+ fn run(
36+ &self,
37+ ctx: &MigrationContext,
38+ db: &mut dyn Conn,
39+ schema: &Schema,
40+ ) -> anyhow::Result<()> {
6941 // If we are only changing the name of a column, we don't have to do anything at this stage
7042 // We'll set the new schema to point to the old column. When the migration is completed,
7143 // we rename the actual column.
@@ -73,37 +45,44 @@ impl Action for AlterColumn {
7345 return Ok(());
7446 }
7547
76− // If we couldn't short circuit, then we need up and down functions
77− let (up, down) = match (&self.up, &self.down) {
78− (Some(up), Some(down)) => (up, down),
79− _ => return Err(anyhow!("missing up or down values")),
80− };
81−
8248 let table = schema.get_table(db, &self.table)?;
8349
8450 let column = table
85− .columns
86− .iter()
87− .find(|column| column.name == self.column)
88− .ok_or_else(|| anyhow!("no such column exists"))?;
51+ .get_column(&self.column)
52+ .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
53+
54+ let temporary_column_name = self.temporary_column_name(ctx);
55+ let temporary_column_type = self.changes.data_type.as_ref().unwrap_or(&column.data_type);
56+
57+ // Add temporary, nullable column
58+ let mut temp_column_definition_parts: Vec<&str> =
59+ vec![&temporary_column_name, temporary_column_type];
8960
90− let temporary_column_type = self
61+ // Use either new default value or existing one if one exists
62+ let default_value = self
9163 .changes
92− .data_type
64+ .default
9365 .as_ref()
94− .unwrap_or_else(|| &column.data_type);
66+ .or_else(|| column.default.as_ref());
67+ if let Some(default) = default_value {
68+ temp_column_definition_parts.push("DEFAULT");
69+ temp_column_definition_parts.push(default);
70+ }
9571
96− // Add temporary, nullable column
9772 let query = format!(
98− "
99− ALTER TABLE {table}
100− ADD COLUMN IF NOT EXISTS {temp_column} {temp_column_type};
101− ",
73+ r#"
74+ ALTER TABLE "{table}"
75+ ADD COLUMN IF NOT EXISTS {temp_column_definition}
76+ "#,
10277 table = self.table,
103− temp_column = self.temporary_column_name(ctx),
104− temp_column_type = temporary_column_type,
78+ temp_column_definition = temp_column_definition_parts.join(" "),
10579 );
106− db.run(&query)?;
80+ db.run(&query).context("failed to add temporary column")?;
81+
82+ // If up or down wasn't provided, we default to simply moving the value over.
83+ // This is the correct behaviour for example when only changing the default value.
84+ let up = self.up.as_ref().unwrap_or(&self.column);
85+ let down = self.down.as_ref().unwrap_or(&self.column);
10786
10887 let declarations: Vec<String> = table
10988 .columns
@@ -119,45 +98,44 @@ impl Action for AlterColumn {
11998 })
12099 .collect();
121100
122− // Add triggers to fill in values as they are inserted/updated
123101 let query = format!(
124− "
125− CREATE OR REPLACE FUNCTION {up_trigger}()
126− RETURNS TRIGGER AS $$
127− BEGIN
128− IF reshape.is_old_schema() THEN
129− DECLARE
130− {declarations}
131− {existing_column} public.{table}.{existing_column_real}%TYPE := NEW.{existing_column_real};
132− BEGIN
133− NEW.{temp_column} = {up};
134− END;
135− END IF;
136− RETURN NEW;
137− END
138− $$ language 'plpgsql';
139−
140− DROP TRIGGER IF EXISTS {up_trigger} ON {table};
141− CREATE TRIGGER {up_trigger} BEFORE INSERT OR UPDATE ON {table} FOR EACH ROW EXECUTE PROCEDURE {up_trigger}();
142−
143− CREATE OR REPLACE FUNCTION {down_trigger}()
144− RETURNS TRIGGER AS $$
145− BEGIN
146− IF NOT reshape.is_old_schema() THEN
147− DECLARE
148− {declarations}
149− {existing_column} public.{table}.{temp_column}%TYPE := NEW.{temp_column};
150− BEGIN
151− NEW.{existing_column_real} = {down};
152− END;
153− END IF;
154− RETURN NEW;
155− END
156− $$ language 'plpgsql';
157−
158− DROP TRIGGER IF EXISTS {down_trigger} ON {table};
159− CREATE TRIGGER {down_trigger} BEFORE INSERT OR UPDATE ON {table} FOR EACH ROW EXECUTE PROCEDURE {down_trigger}();
160− ",
102+ r#"
103+ CREATE OR REPLACE FUNCTION {up_trigger}()
104+ RETURNS TRIGGER AS $$
105+ BEGIN
106+ IF NOT reshape.is_new_schema() THEN
107+ DECLARE
108+ {declarations}
109+ {existing_column} public.{table}.{existing_column_real}%TYPE := NEW.{existing_column_real};
110+ BEGIN
111+ NEW.{temp_column} = {up};
112+ END;
113+ END IF;
114+ RETURN NEW;
115+ END
116+ $$ language 'plpgsql';
117+
118+ DROP TRIGGER IF EXISTS "{up_trigger}" ON "{table}";
119+ CREATE TRIGGER "{up_trigger}" BEFORE INSERT OR UPDATE ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {up_trigger}();
120+
121+ CREATE OR REPLACE FUNCTION {down_trigger}()
122+ RETURNS TRIGGER AS $$
123+ BEGIN
124+ IF reshape.is_new_schema() THEN
125+ DECLARE
126+ {declarations}
127+ {existing_column} public.{table}.{temp_column}%TYPE := NEW.{temp_column};
128+ BEGIN
129+ NEW.{existing_column_real} = {down};
130+ END;
131+ END IF;
132+ RETURN NEW;
133+ END
134+ $$ language 'plpgsql';
135+
136+ DROP TRIGGER IF EXISTS "{down_trigger}" ON "{table}";
137+ CREATE TRIGGER "{down_trigger}" BEFORE INSERT OR UPDATE ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {down_trigger}();
138+ "#,
161139 existing_column = &self.column,
162140 existing_column_real = column.real_name,
163141 temp_column = self.temporary_column_name(ctx),
@@ -168,137 +146,224 @@ impl Action for AlterColumn {
168146 down_trigger = self.down_trigger_name(ctx),
169147 declarations = declarations.join("\n"),
170148 );
171− db.run(&query)?;
149+ db.run(&query)
150+ .context("failed to create up and down triggers")?;
172151
173152 // Backfill values in batches by touching the previous column
174− common::batch_touch_rows(db, &table.real_name, &column.real_name)?;
153+ common::batch_touch_rows(db, &table.real_name, Some(&column.real_name))
154+ .context("failed to batch update existing rows")?;
155+
156+ // Duplicate any indices to the temporary column
157+ let indices = common::get_indices_for_column(db, &table.real_name, &column.real_name)?;
158+ for index in indices {
159+ let index_columns: Vec<String> = common::get_index_columns(db, &index.name)?
160+ .into_iter()
161+ .map(|idx_column| {
162+ // Replace column with temporary column for new index
163+ if idx_column == column.real_name {
164+ temporary_column_name.to_string()
165+ } else {
166+ idx_column
167+ }
168+ })
169+ .collect();
170+ let temp_index_name = self.temp_index_name(ctx, index.oid);
171+
172+ let unique_def = if index.unique { "UNIQUE" } else { "" };
173+
174+ db.query(&format!(
175+ r#"
176+ CREATE {unique_def} INDEX CONCURRENTLY IF NOT EXISTS "{new_index_name}" ON "{table}" USING {index_type} ({columns})
177+ "#,
178+ new_index_name = temp_index_name,
179+ table = table.real_name,
180+ columns = index_columns.join(", "),
181+ index_type = index.index_type,
182+ ))
183+ .context("failed to create temporary index")?;
184+ }
175185
176186 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
177187 // This constraint is set as NOT VALID so it doesn't apply to existing rows and
178188 // the existing rows don't need to be scanned under an exclusive lock.
179189 // Thanks to this, we can set the full column as NOT NULL later with minimal locking.
180− if !column.nullable {
190+ if !self.changes.nullable.unwrap_or(column.nullable) {
181191 let query = format!(
182− "
183− ALTER TABLE {table}
184− ADD CONSTRAINT {constraint_name}
185− CHECK ({column} IS NOT NULL) NOT VALID
186− ",
192+ r#"
193+ ALTER TABLE "{table}"
194+ ADD CONSTRAINT "{constraint_name}"
195+ CHECK ("{column}" IS NOT NULL) NOT VALID
196+ "#,
187197 table = self.table,
188198 constraint_name = self.not_null_constraint_name(ctx),
189199 column = self.temporary_column_name(ctx),
190200 );
191− db.run(&query)?;
201+ db.run(&query)
202+ .context("failed to add NOT NULL constraint")?;
192203 }
193204
194205 Ok(())
195206 }
196207
197− fn complete(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
208+ fn complete<'a>(
209+ &self,
210+ ctx: &MigrationContext,
211+ db: &'a mut dyn Conn,
212+ ) -> anyhow::Result<Option<Transaction<'a>>> {
198213 if self.can_short_circuit() {
199214 if let Some(new_name) = &self.changes.name {
200215 let query = format!(
201− "
202− ALTER TABLE {table}
203− RENAME COLUMN {existing_name} TO {new_name}
204− ",
216+ r#"
217+ ALTER TABLE "{table}"
218+ RENAME COLUMN "{existing_name}" TO "{new_name}"
219+ "#,
205220 table = self.table,
206221 existing_name = self.column,
207222 new_name = new_name,
208223 );
209− db.run(&query)?;
224+ db.run(&query).context("failed to rename column")?;
210225 }
211− return Ok(());
226+ return Ok(None);
212227 }
213228
214− let table = schema.get_table(db, &self.table)?;
215−
216− let column = table
217− .columns
218− .iter()
219− .find(|column| column.name == self.column)
220− .ok_or_else(|| anyhow!("no such column exists"))?;
221− let column_name = self.changes.name.as_deref().unwrap_or(&column.real_name);
222−
223− // Remove old column
224− let query = format!(
225− "
226− ALTER TABLE {} DROP COLUMN {} CASCADE
227− ",
228− self.table, column.real_name
229− );
230− db.run(&query)?;
231−
232− // Rename temporary column
233− let query = format!(
234− "
235− ALTER TABLE {table} RENAME COLUMN {temp_column} TO {name}
236− ",
237− table = self.table,
238− temp_column = self.temporary_column_name(ctx),
239− name = column_name,
240− );
241− db.run(&query)?;
242−
243− // Remove triggers and procedures
244− let query = format!(
245− "
246− DROP TRIGGER IF EXISTS {up_trigger} ON {table};
247− DROP FUNCTION IF EXISTS {up_trigger};
248−
249− DROP TRIGGER IF EXISTS {down_trigger} ON {table};
250− DROP FUNCTION IF EXISTS {down_trigger};
251− ",
252− table = self.table,
253− up_trigger = self.up_trigger_name(ctx),
254− down_trigger = self.down_trigger_name(ctx),
255− );
256− db.run(&query)?;
257−
258229 // Update column to be NOT NULL if necessary
259− if !column.nullable {
230+ let has_not_null_constraint = !db
231+ .query_with_params(
232+ "
233+ SELECT constraint_name
234+ FROM information_schema.constraint_column_usage
235+ WHERE constraint_name = $1
236+ ",
237+ &[&self.not_null_constraint_name(ctx)],
238+ )
239+ .context("failed to get any NOT NULL constraint")?
240+ .is_empty();
241+ if has_not_null_constraint {
260242 // Validate the temporary constraint (should always be valid).
261243 // This performs a sequential scan but does not take an exclusive lock.
262244 let query = format!(
263− "
264− ALTER TABLE {table}
265− VALIDATE CONSTRAINT {constraint_name}
266− ",
245+ r#"
246+ ALTER TABLE "{table}"
247+ VALIDATE CONSTRAINT "{constraint_name}"
248+ "#,
267249 table = self.table,
268250 constraint_name = self.not_null_constraint_name(ctx),
269251 );
270− db.run(&query)?;
252+ db.run(&query)
253+ .context("failed to validate NOT NULL constraint")?;
271254
272255 // Update the column to be NOT NULL.
273256 // This requires an exclusive lock but since PG 12 it can check
274257 // the existing constraint for correctness which makes the lock short-lived.
275258 // Source: https://dba.stackexchange.com/a/268128
276259 let query = format!(
277− "
278− ALTER TABLE {table}
279− ALTER COLUMN {column} SET NOT NULL
280− ",
260+ r#"
261+ ALTER TABLE "{table}"
262+ ALTER COLUMN "{column}" SET NOT NULL
263+ "#,
281264 table = self.table,
282− column = column_name,
265+ column = self.temporary_column_name(ctx),
283266 );
284− db.run(&query)?;
267+ db.run(&query).context("failed to set column as NOT NULL")?;
285268
286269 // Drop the temporary constraint
287270 let query = format!(
288− "
289− ALTER TABLE {table}
290− DROP CONSTRAINT {constraint_name}
291− ",
271+ r#"
272+ ALTER TABLE "{table}"
273+ DROP CONSTRAINT "{constraint_name}"
274+ "#,
292275 table = self.table,
293276 constraint_name = self.not_null_constraint_name(ctx),
294277 );
295− db.run(&query)?;
278+ db.run(&query)
279+ .context("failed to drop NOT NULL constraint")?;
296280 }
297281
298− Ok(())
282+ // Replace old indices with the new temporary ones created for the temporary column
283+ let indices = common::get_indices_for_column(db, &self.table, &self.column)?;
284+ for current_index in indices {
285+ // To keep the index handling idempotent, we need to do the following:
286+ // 1. Add a prefix to the existing index
287+ // 2. Rename temporary index to its final name
288+ // 3. Drop existing index concurrently
289+
290+ // Add prefix (if not already added) to existing index
291+ let prefix = "__reshape_old";
292+ let target_index_name = current_index.name.trim_start_matches(prefix);
293+ let old_index_name = format!("{}_{}", prefix, target_index_name);
294+ db.query(&format!(
295+ r#"
296+ ALTER INDEX IF EXISTS "{current_name}" RENAME TO "{new_name}"
297+ "#,
298+ current_name = target_index_name,
299+ new_name = old_index_name,
300+ ))
301+ .context("failed to rename old index")?;
302+
303+ // Rename temporary index to real name
304+ let temp_index_name = self.temp_index_name(ctx, current_index.oid);
305+ db.query(&format!(
306+ r#"
307+ ALTER INDEX IF EXISTS "{temp_index_name}" RENAME TO "{target_index_name}"
308+ "#,
309+ temp_index_name = temp_index_name,
310+ target_index_name = target_index_name,
311+ ))
312+ .context("failed to rename temporary index")?;
313+
314+ // Drop old index concurrently
315+ db.query(&format!(
316+ r#"
317+ DROP INDEX CONCURRENTLY IF EXISTS "{old_index_name}"
318+ "#,
319+ old_index_name = old_index_name,
320+ ))
321+ .context("failed to drop old index")?;
322+ }
323+
324+ // Remove old column
325+ let query = format!(
326+ r#"
327+ ALTER TABLE "{table}" DROP COLUMN IF EXISTS "{column}" CASCADE
328+ "#,
329+ table = self.table,
330+ column = self.column,
331+ );
332+ db.run(&query).context("failed to drop old column")?;
333+
334+ // Rename temporary column
335+ let column_name = self.changes.name.as_deref().unwrap_or(&self.column);
336+ let query = format!(
337+ r#"
338+ ALTER TABLE "{table}" RENAME COLUMN "{temp_column}" TO "{name}"
339+ "#,
340+ table = self.table,
341+ temp_column = self.temporary_column_name(ctx),
342+ name = column_name,
343+ );
344+ db.run(&query)
345+ .context("failed to rename temporary column")?;
346+
347+ // Remove triggers and procedures
348+ let query = format!(
349+ r#"
350+ DROP TRIGGER IF EXISTS "{up_trigger}" ON "{table}";
351+ DROP FUNCTION IF EXISTS "{up_trigger}";
352+
353+ DROP TRIGGER IF EXISTS "{down_trigger}" ON "{table}";
354+ DROP FUNCTION IF EXISTS "{down_trigger}";
355+ "#,
356+ table = self.table,
357+ up_trigger = self.up_trigger_name(ctx),
358+ down_trigger = self.down_trigger_name(ctx),
359+ );
360+ db.run(&query)
361+ .context("failed to drop up and down triggers")?;
362+
363+ Ok(None)
299364 }
300365
301− fn update_schema(&self, ctx: &Context, schema: &mut Schema) -> anyhow::Result<()> {
366+ fn update_schema(&self, ctx: &MigrationContext, schema: &mut Schema) {
302367 // If we are only changing the name of a column, we haven't created a temporary column
303368 // Instead, we rename the schema column but point it to the old column
304369 if self.can_short_circuit() {
@@ -310,7 +375,7 @@ impl Action for AlterColumn {
310375 });
311376 }
312377
313− return Ok(());
378+ return;
314379 }
315380
316381 schema.change_table(&self.table, |table_changes| {
@@ -318,37 +383,78 @@ impl Action for AlterColumn {
318383 column_changes.set_column(&self.temporary_column_name(ctx));
319384 });
320385 });
321−
322− Ok(())
323386 }
324387
325− fn abort(&self, ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()> {
326− // Remove triggers and procedures
327− let query = format!(
328− "
329− DROP TRIGGER IF EXISTS {up_trigger} ON {table};
330− DROP FUNCTION IF EXISTS {up_trigger};
388+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
389+ // Safely remove any indices created for the temporary column
390+ let temp_column_name = self.temporary_column_name(ctx);
391+ let indices = common::get_indices_for_column(db, &self.table, &temp_column_name)?;
392+ for index in indices {
393+ let temp_index_name = self.temp_index_name(ctx, index.oid);
394+ db.query(&format!(
395+ r#"
396+ DROP INDEX CONCURRENTLY IF EXISTS "{index_name}"
397+ "#,
398+ index_name = temp_index_name,
399+ ))?;
400+ }
331401
332− DROP TRIGGER IF EXISTS {down_trigger} ON {table};
333− DROP FUNCTION IF EXISTS {down_trigger};
334− ",
402+ // Drop temporary column
403+ let query = format!(
404+ r#"
405+ ALTER TABLE "{table}"
406+ DROP COLUMN IF EXISTS "{temp_column}";
407+ "#,
335408 table = self.table,
336− up_trigger = self.up_trigger_name(ctx),
337− down_trigger = self.down_trigger_name(ctx),
409+ temp_column = self.temporary_column_name(ctx),
338410 );
339− db.run(&query)?;
411+ db.run(&query).context("failed to drop temporary column")?;
340412
341− // Drop temporary column
413+ // Remove triggers and procedures
342414 let query = format!(
343− "
344− ALTER TABLE {table}
345− DROP COLUMN IF EXISTS {temp_column};
346− ",
415+ r#"
416+ DROP TRIGGER IF EXISTS "{up_trigger}" ON "{table}";
417+ DROP FUNCTION IF EXISTS "{up_trigger}";
418+
419+ DROP TRIGGER IF EXISTS "{down_trigger}" ON "{table}";
420+ DROP FUNCTION IF EXISTS "{down_trigger}";
421+ "#,
347422 table = self.table,
348− temp_column = self.temporary_column_name(ctx),
423+ up_trigger = self.up_trigger_name(ctx),
424+ down_trigger = self.down_trigger_name(ctx),
349425 );
350− db.run(&query)?;
426+ db.run(&query)
427+ .context("failed to drop up and down triggers")?;
351428
352429 Ok(())
353430 }
354431}
432+
433+impl AlterColumn {
434+ fn temporary_column_name(&self, ctx: &MigrationContext) -> String {
435+ format!("{}_new_{}", ctx.prefix(), self.column)
436+ }
437+
438+ fn up_trigger_name(&self, ctx: &MigrationContext) -> String {
439+ format!("{}_alter_column_up_trigger", ctx.prefix())
440+ }
441+
442+ fn down_trigger_name(&self, ctx: &MigrationContext) -> String {
443+ format!("{}_alter_column_down_trigger", ctx.prefix_inverse())
444+ }
445+
446+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
447+ format!("{}_alter_column_temporary", ctx.prefix())
448+ }
449+
450+ fn temp_index_name(&self, ctx: &MigrationContext, index_oid: u32) -> String {
451+ format!("{}_alter_column_temp_index_{}", ctx.prefix(), index_oid)
452+ }
453+
454+ fn can_short_circuit(&self) -> bool {
455+ self.changes.name.is_some()
456+ && self.changes.data_type.is_none()
457+ && self.changes.nullable.is_none()
458+ && self.changes.default.is_none()
459+ }
460+}
src/migrations/common.rs+137−37
@@ -1,20 +1,17 @@
1+use anyhow::anyhow;
12use postgres::types::{FromSql, ToSql};
23use serde::{Deserialize, Serialize};
34
45use crate::db::Conn;
56
6−#[derive(Serialize, Deserialize, Debug)]
7+#[derive(Serialize, Deserialize, Clone, Debug)]
78pub struct Column {
89 pub name: String,
9−
1010 #[serde(rename = "type")]
1111 pub data_type: String,
12−
1312 #[serde(default = "nullable_default")]
1413 pub nullable: bool,
15−
1614 pub default: Option<String>,
17−
1815 pub generated: Option<String>,
1916}
2017
@@ -22,6 +19,13 @@ fn nullable_default() -> bool {
2219 true
2320}
2421
22+#[derive(Serialize, Deserialize, Clone, Debug)]
23+pub struct ForeignKey {
24+ pub columns: Vec<String>,
25+ pub referenced_table: String,
26+ pub referenced_columns: Vec<String>,
27+}
28+
2529#[derive(Debug)]
2630struct PostgresRawValue {
2731 bytes: Vec<u8>,
@@ -65,24 +69,35 @@ impl ToSql for PostgresRawValue {
6569 postgres::types::to_sql_checked!();
6670}
6771
68−pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow::Result<()> {
72+pub fn batch_touch_rows(
73+ db: &mut dyn Conn,
74+ table: &str,
75+ column: Option<&str>,
76+) -> anyhow::Result<()> {
6977 const BATCH_SIZE: u16 = 1000;
7078
71− db.query("SET reshape.is_old_schema = 'YES'")?;
72−
7379 let mut cursor: Option<PostgresRawValue> = None;
7480
7581 loop {
7682 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
7783
78− let primary_key = get_primary_key_columns_for_table(db, &table)?;
84+ let primary_key = get_primary_key_columns_for_table(db, table)?;
85+
86+ // If no column to touch is passed, we default to the first primary key column (just to make some "update")
87+ let touched_column = match column {
88+ Some(column) => column,
89+ None => primary_key.first().unwrap(),
90+ };
91+
7992 let primary_key_columns = primary_key.join(", ");
8093
8194 let primary_key_where = primary_key
8295 .iter()
8396 .map(|column| {
8497 format!(
85− "{table}.{column} = rows.{column}",
98+ r#"
99+ "{table}"."{column}" = rows."{column}"
100+ "#,
86101 table = table,
87102 column = column,
88103 )
@@ -92,7 +107,7 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
92107
93108 let returning_columns = primary_key
94109 .iter()
95− .map(|column| format!("rows.{}", column))
110+ .map(|column| format!("rows.\"{}\"", column))
96111 .collect::<Vec<String>>()
97112 .join(", ");
98113
@@ -108,31 +123,25 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
108123 };
109124
110125 let query = format!(
111− "
112− WITH rows AS (
113− SELECT {primary_key_columns}
114− FROM public.{table}
115− {cursor_where}
116− ORDER BY {primary_key_columns}
117− LIMIT {batch_size}
118− ), update AS (
119− UPDATE public.{table}
120− SET {column} = {column}
121− FROM rows
122− WHERE {primary_key_where}
123− RETURNING {returning_columns}
124− )
125− SELECT LAST_VALUE(({primary_key_columns})) OVER () AS last_value
126− FROM update
127− LIMIT 1
128− ",
129− table = table,
130− primary_key_columns = primary_key_columns,
131− cursor_where = cursor_where,
126+ r#"
127+ WITH rows AS (
128+ SELECT {primary_key_columns}
129+ FROM public."{table}"
130+ {cursor_where}
131+ ORDER BY {primary_key_columns}
132+ LIMIT {batch_size}
133+ ), update AS (
134+ UPDATE public."{table}" "{table}"
135+ SET "{touched_column}" = "{table}"."{touched_column}"
136+ FROM rows
137+ WHERE {primary_key_where}
138+ RETURNING {returning_columns}
139+ )
140+ SELECT LAST_VALUE(({primary_key_columns})) OVER () AS last_value
141+ FROM update
142+ LIMIT 1
143+ "#,
132144 batch_size = BATCH_SIZE,
133− column = column,
134− primary_key_where = primary_key_where,
135− returning_columns = returning_columns,
136145 );
137146 let last_value = db
138147 .query_with_params(&query, &params)?
@@ -146,8 +155,6 @@ pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow:
146155 cursor = last_value
147156 }
148157
149− db.query("SET reshape.is_old_schema = ''")?;
150−
151158 Ok(())
152159}
153160
@@ -173,3 +180,96 @@ fn get_primary_key_columns_for_table(
173180
174181 Ok(primary_key_columns)
175182}
183+
184+pub struct Index {
185+ pub name: String,
186+ pub oid: u32,
187+ pub unique: bool,
188+ pub index_type: String,
189+}
190+
191+pub fn get_indices_for_column(
192+ db: &mut dyn Conn,
193+ table: &str,
194+ column: &str,
195+) -> anyhow::Result<Vec<Index>> {
196+ let indices = db
197+ .query(&format!(
198+ "
199+ SELECT
200+ i.relname AS name,
201+ i.oid AS oid,
202+ ix.indisunique AS unique,
203+ am.amname AS type
204+ FROM pg_index ix
205+ JOIN pg_class t ON t.oid = ix.indrelid
206+ JOIN pg_class i ON i.oid = ix.indexrelid
207+ JOIN pg_am am ON i.relam = am.oid
208+ JOIN pg_attribute a ON
209+ a.attrelid = t.oid AND
210+ a.attnum = ANY(ix.indkey)
211+ WHERE
212+ t.relname = '{table}' AND
213+ a.attname = '{column}'
214+ ",
215+ table = table,
216+ column = column,
217+ ))?
218+ .iter()
219+ .map(|row| Index {
220+ name: row.get("name"),
221+ oid: row.get("oid"),
222+ unique: row.get("unique"),
223+ index_type: row.get("type"),
224+ })
225+ .collect();
226+
227+ Ok(indices)
228+}
229+
230+pub fn get_index_columns(db: &mut dyn Conn, index_name: &str) -> anyhow::Result<Vec<String>> {
231+ // Get all columns which are part of the index in order
232+ let (table_oid, column_nums) = db
233+ .query(&format!(
234+ "
235+ SELECT t.oid AS table_oid, ix.indkey::INTEGER[] AS columns
236+ FROM pg_index ix
237+ JOIN pg_class t ON t.oid = ix.indrelid
238+ JOIN pg_class i ON i.oid = ix.indexrelid
239+ WHERE
240+ i.relname = '{index_name}'
241+ ",
242+ index_name = index_name,
243+ ))?
244+ .first()
245+ .map(|row| {
246+ (
247+ row.get::<'_, _, u32>("table_oid"),
248+ row.get::<'_, _, Vec<i32>>("columns"),
249+ )
250+ })
251+ .ok_or_else(|| anyhow!("failed to get columns for index"))?;
252+
253+ // Get the name of each of the columns, still in order
254+ column_nums
255+ .iter()
256+ .map(|column_num| -> anyhow::Result<String> {
257+ let name: String = db
258+ .query(&format!(
259+ "
260+ SELECT attname AS name
261+ FROM pg_attribute
262+ WHERE attrelid = {table_oid}
263+ AND attnum = {column_num};
264+ ",
265+ table_oid = table_oid,
266+ column_num = column_num,
267+ ))?
268+ .first()
269+ .map(|row| row.get("name"))
270+ .ok_or_else(|| anyhow!("expected to find column"))?;
271+
272+ Ok(name)
273+ })
274+ .collect::<anyhow::Result<Vec<String>>>()
275+}
src/migrations/create_enum.rsnew file+83
@@ -0,0 +1,83 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct CreateEnum {
11+ pub name: String,
12+ pub values: Vec<String>,
13+}
14+
15+#[typetag::serde(name = "create_enum")]
16+impl Action for CreateEnum {
17+ fn describe(&self) -> String {
18+ format!("Creating enum \"{}\"", self.name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ // Check if enum already exists. CREATE TYPE doesn't have
28+ // a IF NOT EXISTS option so we have to do it manually.
29+ let enum_exists = !db
30+ .query(&format!(
31+ "
32+ SELECT typname
33+ FROM pg_catalog.pg_type
34+ WHERE typcategory = 'E'
35+ AND typname = '{name}'
36+ ",
37+ name = self.name,
38+ ))?
39+ .is_empty();
40+ if enum_exists {
41+ return Ok(());
42+ }
43+
44+ let values_def: Vec<String> = self
45+ .values
46+ .iter()
47+ .map(|value| format!("'{}'", value))
48+ .collect();
49+
50+ db.run(&format!(
51+ r#"
52+ CREATE TYPE "{name}" AS ENUM ({values})
53+ "#,
54+ name = self.name,
55+ values = values_def.join(", "),
56+ ))
57+ .context("failed to create enum")?;
58+
59+ Ok(())
60+ }
61+
62+ fn complete<'a>(
63+ &self,
64+ _ctx: &MigrationContext,
65+ _db: &'a mut dyn Conn,
66+ ) -> anyhow::Result<Option<Transaction<'a>>> {
67+ Ok(None)
68+ }
69+
70+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
71+
72+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
73+ db.run(&format!(
74+ r#"
75+ DROP TYPE IF EXISTS {name}
76+ "#,
77+ name = self.name,
78+ ))
79+ .context("failed to drop enum")?;
80+
81+ Ok(())
82+ }
83+}
src/migrations/create_table.rs+177−29
@@ -1,5 +1,12 @@
1−use super::{Action, Column, Context};
2−use crate::{db::Conn, schema::Schema};
1+use std::collections::HashMap;
2+
3+use super::{common::ForeignKey, Action, Column, MigrationContext};
4+use crate::{
5+ db::{Conn, Transaction},
6+ migrations::common,
7+ schema::Schema,
8+};
9+use anyhow::Context;
310use serde::{Deserialize, Serialize};
411
512#[derive(Serialize, Deserialize, Debug)]
@@ -7,14 +14,24 @@ pub struct CreateTable {
714 pub name: String,
815 pub columns: Vec<Column>,
916 pub primary_key: Vec<String>,
17+
18+ #[serde(default)]
1019 pub foreign_keys: Vec<ForeignKey>,
20+
21+ pub up: Option<Transformation>,
1122}
1223
1324#[derive(Serialize, Deserialize, Debug)]
14−pub struct ForeignKey {
15− pub columns: Vec<String>,
16− pub referenced_table: String,
17− pub referenced_columns: Vec<String>,
25+pub struct Transformation {
26+ table: String,
27+ values: HashMap<String, String>,
28+ upsert_constraint: Option<String>,
29+}
30+
31+impl CreateTable {
32+ fn trigger_name(&self, ctx: &MigrationContext) -> String {
33+ format!("{}_create_table_{}", ctx.prefix(), self.name)
34+ }
1835}
1936
2037#[typetag::serde(name = "create_table")]
@@ -23,12 +40,17 @@ impl Action for CreateTable {
2340 format!("Creating table \"{}\"", self.name)
2441 }
2542
26− fn run(&self, _ctx: &Context, db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
43+ fn run(
44+ &self,
45+ ctx: &MigrationContext,
46+ db: &mut dyn Conn,
47+ schema: &Schema,
48+ ) -> anyhow::Result<()> {
2749 let mut definition_rows: Vec<String> = self
2850 .columns
2951 .iter()
3052 .map(|column| {
31− let mut parts = vec![column.name.to_string(), column.data_type.to_string()];
53+ let mut parts = vec![format!("\"{}\"", column.name), column.data_type.to_string()];
3254
3355 if let Some(default) = &column.default {
3456 parts.push("DEFAULT".to_string());
@@ -48,40 +70,166 @@ impl Action for CreateTable {
4870 })
4971 .collect();
5072
51− let primary_key_columns = self.primary_key.join(", ");
73+ let primary_key_columns = self
74+ .primary_key
75+ .iter()
76+ // Add quotes around all column names
77+ .map(|col| format!("\"{}\"", col))
78+ .collect::<Vec<String>>()
79+ .join(", ");
5280 definition_rows.push(format!("PRIMARY KEY ({})", primary_key_columns));
5381
5482 for foreign_key in &self.foreign_keys {
83+ // Add quotes around all column names
84+ let columns: Vec<String> = foreign_key
85+ .columns
86+ .iter()
87+ .map(|col| format!("\"{}\"", col))
88+ .collect();
89+
90+ let referenced_table = schema.get_table(db, &foreign_key.referenced_table)?;
91+ let referenced_columns: Vec<String> = referenced_table
92+ .real_column_names(&foreign_key.referenced_columns)
93+ .map(|col| format!("\"{}\"", col))
94+ .collect();
95+
5596 definition_rows.push(format!(
56− "FOREIGN KEY ({columns}) REFERENCES {table} ({referenced_columns})",
57− columns = foreign_key.columns.join(", "),
58− table = foreign_key.referenced_table,
59− referenced_columns = foreign_key.referenced_columns.join(", "),
97+ r#"
98+ FOREIGN KEY ({columns}) REFERENCES "{table}" ({referenced_columns})
99+ "#,
100+ columns = columns.join(", "),
101+ table = referenced_table.real_name,
102+ referenced_columns = referenced_columns.join(", "),
60103 ));
61104 }
62105
63− db.run(&format!(
64− "CREATE TABLE {} (
65− {}
66− )",
67− self.name,
68− definition_rows.join(",\n"),
69− ))?;
70− Ok(())
71− }
106+ let query = &format!(
107+ r#"
108+ CREATE TABLE "{name}" (
109+ {definition}
110+ )
111+ "#,
112+ name = self.name,
113+ definition = definition_rows.join(",\n"),
114+ );
115+ db.run(query).context("failed to create table")?;
116+
117+ if let Some(Transformation {
118+ table: from_table,
119+ values,
120+ upsert_constraint,
121+ }) = &self.up
122+ {
123+ let from_table = schema.get_table(db, &from_table)?;
124+
125+ let declarations: Vec<String> = from_table
126+ .columns
127+ .iter()
128+ .map(|column| {
129+ format!(
130+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
131+ table = from_table.real_name,
132+ alias = column.name,
133+ real_name = column.real_name,
134+ )
135+ })
136+ .collect();
137+
138+ let (insert_columns, insert_values): (Vec<&str>, Vec<&str>) = values
139+ .iter()
140+ .map(|(k, v)| -> (&str, &str) { (k, v) }) // Force &String to &str
141+ .unzip();
142+
143+ let update_set: Vec<String> = values
144+ .iter()
145+ .map(|(field, value)| format!("\"{field}\" = {value}"))
146+ .collect();
147+
148+ // Constraint to check for conflicts. Defaults to the primary key constraint.
149+ let conflict_constraint_name = match upsert_constraint {
150+ Some(custom_constraint) => custom_constraint.clone(),
151+ _ => format!("{table}_pkey", table = self.name),
152+ };
153+
154+ // Add triggers to fill in values as they are inserted/updated
155+ let query = format!(
156+ r#"
157+ CREATE OR REPLACE FUNCTION {trigger_name}()
158+ RETURNS TRIGGER AS $$
159+ #variable_conflict use_variable
160+ BEGIN
161+ IF NOT reshape.is_new_schema() THEN
162+ DECLARE
163+ {declarations}
164+ BEGIN
165+ INSERT INTO public."{changed_table_real}" ({columns})
166+ VALUES ({values})
167+ ON CONFLICT ON CONSTRAINT "{conflict_constraint_name}"
168+ DO UPDATE SET
169+ {updates};
170+ END;
171+ END IF;
172+ RETURN NEW;
173+ END
174+ $$ language 'plpgsql';
175+
176+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
177+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
178+ "#,
179+ changed_table_real = self.name,
180+ from_table_real = from_table.real_name,
181+ trigger_name = self.trigger_name(ctx),
182+ declarations = declarations.join("\n"),
183+ columns = insert_columns.join(", "),
184+ values = insert_values.join(", "),
185+ updates = update_set.join(",\n"),
186+ );
187+ db.run(&query).context("failed to create up trigger")?;
188+
189+ // Backfill values in batches by touching the from table
190+ common::batch_touch_rows(db, &from_table.real_name, None)
191+ .context("failed to batch update existing rows")?;
192+ }
72193
73− fn complete(&self, _ctx: &Context, _db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
74− // Do nothing
75194 Ok(())
76195 }
77196
78− fn update_schema(&self, _ctx: &Context, _schema: &mut Schema) -> anyhow::Result<()> {
79− Ok(())
197+ fn complete<'a>(
198+ &self,
199+ ctx: &MigrationContext,
200+ db: &'a mut dyn Conn,
201+ ) -> anyhow::Result<Option<Transaction<'a>>> {
202+ // Remove triggers and procedures
203+ let query = format!(
204+ r#"
205+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
206+ "#,
207+ trigger_name = self.trigger_name(ctx),
208+ );
209+ db.run(&query).context("failed to drop up trigger")?;
210+
211+ Ok(None)
80212 }
81213
82− fn abort(&self, _ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()> {
83− let query = format!("DROP TABLE IF EXISTS {table}", table = self.name,);
84− db.run(&query)?;
214+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
215+
216+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
217+ // Remove triggers and procedures
218+ let query = format!(
219+ r#"
220+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
221+ "#,
222+ trigger_name = self.trigger_name(ctx),
223+ );
224+ db.run(&query).context("failed to drop up trigger")?;
225+
226+ db.run(&format!(
227+ r#"
228+ DROP TABLE IF EXISTS "{name}"
229+ "#,
230+ name = self.name,
231+ ))
232+ .context("failed to drop table")?;
85233
86234 Ok(())
87235 }
src/migrations/custom.rsnew file+61
@@ -0,0 +1,61 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use serde::{Deserialize, Serialize};
7+
8+#[derive(Serialize, Deserialize, Debug)]
9+pub struct Custom {
10+ #[serde(default)]
11+ pub start: Option<String>,
12+
13+ #[serde(default)]
14+ pub complete: Option<String>,
15+
16+ #[serde(default)]
17+ pub abort: Option<String>,
18+}
19+
20+#[typetag::serde(name = "custom")]
21+impl Action for Custom {
22+ fn describe(&self) -> String {
23+ "Running custom migration".to_string()
24+ }
25+
26+ fn run(
27+ &self,
28+ _ctx: &MigrationContext,
29+ db: &mut dyn Conn,
30+ _schema: &Schema,
31+ ) -> anyhow::Result<()> {
32+ if let Some(start_query) = &self.start {
33+ println!("Running query: {}", start_query);
34+ db.run(start_query)?;
35+ }
36+
37+ Ok(())
38+ }
39+
40+ fn complete<'a>(
41+ &self,
42+ _ctx: &MigrationContext,
43+ db: &'a mut dyn Conn,
44+ ) -> anyhow::Result<Option<Transaction<'a>>> {
45+ if let Some(complete_query) = &self.complete {
46+ db.run(complete_query)?;
47+ }
48+
49+ Ok(None)
50+ }
51+
52+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
53+
54+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
55+ if let Some(abort_query) = &self.abort {
56+ db.run(abort_query)?;
57+ }
58+
59+ Ok(())
60+ }
61+}
src/migrations/mod.rs+43−11
@@ -1,4 +1,7 @@
1−use crate::{db::Conn, schema::Schema};
1+use crate::{
2+ db::{Conn, Transaction},
3+ schema::Schema,
4+};
25use core::fmt::Debug;
36use serde::{Deserialize, Serialize};
47
@@ -7,7 +10,7 @@ mod common;
710pub use common::Column;
811
912mod create_table;
10−pub use create_table::{CreateTable, ForeignKey};
13+pub use create_table::CreateTable;
1114
1215mod alter_column;
1316pub use alter_column::{AlterColumn, ColumnChanges};
@@ -19,7 +22,10 @@ mod remove_column;
1922pub use remove_column::RemoveColumn;
2023
2124mod add_index;
22−pub use add_index::AddIndex;
25+pub use add_index::{AddIndex, Index};
26+
27+mod remove_index;
28+pub use remove_index::RemoveIndex;
2329
2430mod remove_table;
2531pub use remove_table::RemoveTable;
@@ -27,6 +33,21 @@ pub use remove_table::RemoveTable;
2733mod rename_table;
2834pub use rename_table::RenameTable;
2935
36+mod create_enum;
37+pub use create_enum::CreateEnum;
38+
39+mod remove_enum;
40+pub use remove_enum::RemoveEnum;
41+
42+mod custom;
43+pub use custom::Custom;
44+
45+mod add_foreign_key;
46+pub use add_foreign_key::AddForeignKey;
47+
48+mod remove_foreign_key;
49+pub use remove_foreign_key::RemoveForeignKey;
50+
3051#[derive(Serialize, Deserialize, Debug)]
3152pub struct Migration {
3253 pub name: String,
@@ -64,16 +85,22 @@ impl Clone for Migration {
6485 }
6586}
6687
67−pub struct Context {
88+pub struct MigrationContext {
6889 migration_index: usize,
6990 action_index: usize,
91+ existing_schema_name: Option<String>,
7092}
7193
72−impl Context {
73− pub fn new(migration_index: usize, action_index: usize) -> Self {
74− Context {
94+impl MigrationContext {
95+ pub fn new(
96+ migration_index: usize,
97+ action_index: usize,
98+ existing_schema_name: Option<String>,
99+ ) -> Self {
100+ MigrationContext {
75101 migration_index,
76102 action_index,
103+ existing_schema_name,
77104 }
78105 }
79106
@@ -96,8 +123,13 @@ impl Context {
96123#[typetag::serde(tag = "type")]
97124pub trait Action: Debug {
98125 fn describe(&self) -> String;
99− fn run(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()>;
100− fn complete(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()>;
101− fn update_schema(&self, ctx: &Context, schema: &mut Schema) -> anyhow::Result<()>;
102− fn abort(&self, ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()>;
126+ fn run(&self, ctx: &MigrationContext, db: &mut dyn Conn, schema: &Schema)
127+ -> anyhow::Result<()>;
128+ fn complete<'a>(
129+ &self,
130+ ctx: &MigrationContext,
131+ db: &'a mut dyn Conn,
132+ ) -> anyhow::Result<Option<Transaction<'a>>>;
133+ fn update_schema(&self, ctx: &MigrationContext, schema: &mut Schema);
134+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()>;
103135}
src/migrations/remove_column.rs+385−54
@@ -1,16 +1,31 @@
1−use super::{Action, Context};
2−use crate::{db::Conn, schema::Schema};
1+use super::{common, Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::{anyhow, bail, Context};
37use serde::{Deserialize, Serialize};
48
59#[derive(Serialize, Deserialize, Debug)]
610pub struct RemoveColumn {
711 pub table: String,
812 pub column: String,
9− pub down: Option<String>,
13+ pub down: Option<Transformation>,
14+}
15+
16+#[derive(Serialize, Deserialize, Debug)]
17+#[serde(untagged)]
18+pub enum Transformation {
19+ Simple(String),
20+ Update {
21+ table: String,
22+ value: String,
23+ r#where: String,
24+ },
1025}
1126
1227impl RemoveColumn {
13− fn trigger_name(&self, ctx: &Context) -> String {
28+ fn trigger_name(&self, ctx: &MigrationContext) -> String {
1429 format!(
1530 "{}_remove_column_{}_{}",
1631 ctx.prefix(),
@@ -18,6 +33,33 @@ impl RemoveColumn {
1833 self.column
1934 )
2035 }
36+
37+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
38+ format!(
39+ "{}_remove_column_{}_{}_rev",
40+ ctx.prefix(),
41+ self.table,
42+ self.column
43+ )
44+ }
45+
46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
47+ format!(
48+ "{}_remove_column_{}_{}_nn",
49+ ctx.prefix(),
50+ self.table,
51+ self.column
52+ )
53+ }
54+
55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
56+ format!(
57+ "{}_add_column_not_null_{}_{}",
58+ ctx.prefix(),
59+ self.table,
60+ self.column
61+ )
62+ }
2163}
2264
2365#[typetag::serde(name = "remove_column")]
@@ -29,11 +71,19 @@ impl Action for RemoveColumn {
2971 )
3072 }
3173
32− fn run(&self, ctx: &Context, db: &mut dyn Conn, schema: &Schema) -> anyhow::Result<()> {
74+ fn run(
75+ &self,
76+ ctx: &MigrationContext,
77+ db: &mut dyn Conn,
78+ schema: &Schema,
79+ ) -> anyhow::Result<()> {
80+ let table = schema.get_table(db, &self.table)?;
81+ let column = table
82+ .get_column(&self.column)
83+ .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
84+
3385 // Add down trigger
3486 if let Some(down) = &self.down {
35− let table = schema.get_table(db, &self.table)?;
36−
3787 let declarations: Vec<String> = table
3888 .columns
3989 .iter()
@@ -47,76 +97,357 @@ impl Action for RemoveColumn {
4797 })
4898 .collect();
4999
50− let query = format!(
51− "
52− CREATE OR REPLACE FUNCTION {trigger_name}()
53− RETURNS TRIGGER AS $$
54− BEGIN
55− IF NEW.{column_name} IS NULL THEN
56− DECLARE
57− {declarations}
100+ if let Transformation::Simple(down) = down {
101+ let query = format!(
102+ r#"
103+ CREATE OR REPLACE FUNCTION {trigger_name}()
104+ RETURNS TRIGGER AS $$
105+ BEGIN
106+ IF reshape.is_new_schema() THEN
107+ DECLARE
108+ {declarations}
109+ BEGIN
110+ NEW.{column_name} = {down};
111+ END;
112+ END IF;
113+ RETURN NEW;
114+ END
115+ $$ language 'plpgsql';
116+
117+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
118+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
119+ "#,
120+ column_name = self.column,
121+ trigger_name = self.trigger_name(ctx),
122+ down = down,
123+ table = self.table,
124+ declarations = declarations.join("\n"),
125+ );
126+ db.run(&query).context("failed to create down trigger")?;
127+ }
128+
129+ if let Transformation::Update {
130+ table: from_table,
131+ value,
132+ r#where,
133+ } = down
134+ {
135+ let existing_schema_name = match &ctx.existing_schema_name {
136+ Some(name) => name,
137+ None => bail!("can't use update without previous migration"),
138+ };
139+
140+ let from_table = schema.get_table(db, &from_table)?;
141+
142+ let maybe_null_check = if !column.nullable {
143+ // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.
144+ // We will add a null check to the down function on the new schema below as well to cover both cases.
145+ // As we are using a complex down function, we must remove the NOT NULL check for the new schema.
146+ // NOT NULL is not checked at the end of a transaction, but immediately upon update.
147+ let query = format!(
148+ r#"
149+ CREATE OR REPLACE FUNCTION {trigger_name}()
150+ RETURNS TRIGGER AS $$
58151 BEGIN
59− NEW.{column_name} = {down};
60− END;
61− END IF;
62− RETURN NEW;
63− END
64− $$ language 'plpgsql';
65−
66− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
67− CREATE TRIGGER {trigger_name} BEFORE UPDATE OR INSERT ON {table} FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
68− ",
69− column_name = self.column,
70− trigger_name = self.trigger_name(&ctx),
71− down = down,
72− table = self.table,
73− declarations = declarations.join("\n"),
74− );
75− db.run(&query)?;
152+ IF NOT reshape.is_new_schema() THEN
153+ IF NEW.{column} IS NULL THEN
154+ RAISE EXCEPTION '{column} can not be null';
155+ END IF;
156+ END IF;
157+ RETURN NEW;
158+ END
159+ $$ language 'plpgsql';
160+
161+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
162+
163+ CREATE CONSTRAINT TRIGGER "{trigger_name}"
164+ AFTER INSERT OR UPDATE
165+ ON "{table}"
166+ FOR EACH ROW
167+ EXECUTE PROCEDURE {trigger_name}();
168+ "#,
169+ table = self.table,
170+ trigger_name = self.not_null_constraint_trigger_name(ctx),
171+ column = self.column,
172+ );
173+ db.run(&query)
174+ .context("failed to create null constraint trigger")?;
175+
176+ db.run(&format!(
177+ r#"
178+ ALTER TABLE {table}
179+ ALTER COLUMN {column}
180+ DROP NOT NULL
181+ "#,
182+ table = self.table,
183+ column = self.column
184+ ))
185+ .context("failed to remove column not null constraint")?;
186+
187+ format!(
188+ r#"
189+ IF {value} IS NULL THEN
190+ RAISE EXCEPTION '{column_name} can not be null';
191+ END IF;
192+ "#,
193+ column_name = self.column,
194+ )
195+ } else {
196+ "".to_string()
197+ };
198+
199+ let into_variables = from_table
200+ .columns
201+ .iter()
202+ .map(|column| {
203+ format!(
204+ "NEW.{real_name} AS {alias}",
205+ alias = column.name,
206+ real_name = column.real_name,
207+ )
208+ })
209+ .collect::<Vec<String>>()
210+ .join(", ");
211+
212+ let query = format!(
213+ r#"
214+ CREATE OR REPLACE FUNCTION {trigger_name}()
215+ RETURNS TRIGGER AS $$
216+ #variable_conflict use_variable
217+ BEGIN
218+ IF reshape.is_new_schema() THEN
219+ DECLARE
220+ {from_table} record;
221+ BEGIN
222+ SELECT {into_variables}
223+ INTO {from_table};
224+
225+ {maybe_null_check}
226+
227+ -- Don't trigger reverse trigger when making this update
228+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
229+
230+ UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"
231+ SET "{column_name}" = {value}
232+ WHERE {where};
233+
234+ perform set_config('reshape.disable_triggers', '', TRUE);
235+ END;
236+ END IF;
237+ RETURN NEW;
238+ END
239+ $$ language 'plpgsql';
240+
241+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
242+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
243+ "#,
244+ changed_table = self.table,
245+ from_table = from_table.name,
246+ from_table_real = from_table.real_name,
247+ column_name = self.column,
248+ trigger_name = self.trigger_name(ctx),
249+ );
250+ db.run(&query).context("failed to create down trigger")?;
251+
252+ let changed_into_variables = table
253+ .columns
254+ .iter()
255+ .map(|column| {
256+ format!(
257+ "NEW.{real_name} AS {alias}",
258+ alias = column.name,
259+ real_name = column.real_name,
260+ )
261+ })
262+ .collect::<Vec<String>>()
263+ .join(", ");
264+
265+ let from_table_columns = from_table
266+ .columns
267+ .iter()
268+ .map(|column| format!("{} as {}", column.real_name, column.name))
269+ .collect::<Vec<String>>()
270+ .join(", ");
271+
272+ let query = format!(
273+ r#"
274+ CREATE OR REPLACE FUNCTION {trigger_name}()
275+ RETURNS TRIGGER AS $$
276+ #variable_conflict use_variable
277+ BEGIN
278+ IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
279+ DECLARE
280+ {changed_table} record;
281+ __temp_row record;
282+ BEGIN
283+ SELECT {changed_into_variables}
284+ INTO {changed_table};
285+
286+ SELECT *
287+ INTO __temp_row
288+ FROM (
289+ SELECT {from_table_columns}
290+ FROM public.{from_table_real}
291+ ) {from_table}
292+ WHERE {where};
293+
294+ DECLARE
295+ {from_table} record;
296+ BEGIN
297+ {from_table} := __temp_row;
298+ NEW.{column_name_real} = {value};
299+ END;
300+ END;
301+ END IF;
302+ RETURN NEW;
303+ END
304+ $$ language 'plpgsql';
305+
306+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
307+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
308+ "#,
309+ changed_table = table.name,
310+ changed_table_real = table.real_name,
311+ from_table = from_table.name,
312+ from_table_real = from_table.real_name,
313+ column_name_real = column.real_name,
314+ trigger_name = self.reverse_trigger_name(ctx),
315+ // declarations = declarations.join("\n"),
316+ );
317+ db.run(&query)
318+ .context("failed to create reverse down trigger")?;
319+ }
76320 }
77321
78322 Ok(())
79323 }
80324
81− fn complete(&self, ctx: &Context, db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
325+ fn complete<'a>(
326+ &self,
327+ ctx: &MigrationContext,
328+ db: &'a mut dyn Conn,
329+ ) -> anyhow::Result<Option<Transaction<'a>>> {
330+ let indices = common::get_indices_for_column(db, &self.table, &self.column)
331+ .context("failed getting column indices")?;
332+
333+ for index in indices {
334+ db.run(&format!(
335+ "
336+ DROP INDEX CONCURRENTLY IF EXISTS {name}
337+ ",
338+ name = index.name,
339+ ))
340+ .context("failed to drop index")?;
341+ }
342+
82343 // Remove column, function and trigger
83344 let query = format!(
84− "
85− ALTER TABLE {table}
86− DROP COLUMN {column};
345+ r#"
346+ ALTER TABLE "{table}"
347+ DROP COLUMN IF EXISTS "{column}";
87348
88− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
89− DROP FUNCTION IF EXISTS {trigger_name};
90− ",
349+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
350+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
351+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
352+ "#,
91353 table = self.table,
92354 column = self.column,
93− trigger_name = self.trigger_name(&ctx),
355+ trigger_name = self.trigger_name(ctx),
356+ reverse_trigger_name = self.reverse_trigger_name(ctx),
357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
94358 );
95− db.run(&query)?;
359+ db.run(&query)
360+ .context("failed to drop column and down trigger")?;
96361
97− Ok(())
362+ Ok(None)
98363 }
99364
100− fn update_schema(&self, _ctx: &Context, schema: &mut Schema) -> anyhow::Result<()> {
365+ fn update_schema(&self, _ctx: &MigrationContext, schema: &mut Schema) {
101366 schema.change_table(&self.table, |table_changes| {
102367 table_changes.change_column(&self.column, |column_changes| {
103368 column_changes.set_removed();
104369 })
105370 });
106−
107− Ok(())
108371 }
109372
110− fn abort(&self, ctx: &Context, db: &mut dyn Conn) -> anyhow::Result<()> {
373+ fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
374+ // We might have temporaily removed the NOT NULL check and have to reinstate it
375+ let has_not_null_function = !db
376+ .query_with_params(
377+ "
378+ SELECT routine_name
379+ FROM information_schema.routines
380+ WHERE routine_schema = 'public'
381+ AND routine_name = $1
382+ ",
383+ &[&self.not_null_constraint_trigger_name(ctx)],
384+ )
385+ .context("failed to get any NOT NULL function")?
386+ .is_empty();
387+
388+ if has_not_null_function {
389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint
390+ let query = format!(
391+ r#"
392+ ALTER TABLE "{table}"
393+ ADD CONSTRAINT "{constraint_name}"
394+ CHECK ("{column}" IS NOT NULL) NOT VALID
395+ "#,
396+ table = self.table,
397+ constraint_name = self.not_null_constraint_name(ctx),
398+ column = self.column,
399+ );
400+ db.run(&query)
401+ .context("failed to add NOT NULL constraint")?;
402+
403+ let query = format!(
404+ r#"
405+ ALTER TABLE "{table}"
406+ VALIDATE CONSTRAINT "{constraint_name}"
407+ "#,
408+ table = self.table,
409+ constraint_name = self.not_null_constraint_name(ctx),
410+ );
411+ db.run(&query)
412+ .context("failed to validate NOT NULL constraint")?;
413+
414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above
415+ db.run(&format!(
416+ r#"
417+ ALTER TABLE {table}
418+ ALTER COLUMN {column}
419+ SET NOT NULL
420+ "#,
421+ table = self.table,
422+ column = self.column
423+ ))
424+ .context("failed to reinstate column NOT NULL")?;
425+
426+ // Drop the temporary constraint
427+ let query = format!(
428+ r#"
429+ ALTER TABLE "{table}"
430+ DROP CONSTRAINT "{constraint_name}"
431+ "#,
432+ table = self.table,
433+ constraint_name = self.not_null_constraint_name(ctx),
434+ );
435+ db.run(&query)
436+ .context("failed to drop NOT NULL constraint")?;
437+ }
438+
111439 // Remove function and trigger
112− db.query(&format!(
113− "
114− DROP TRIGGER IF EXISTS {trigger_name} ON {table};
115− DROP FUNCTION IF EXISTS {trigger_name};
116− ",
117− table = self.table,
118− trigger_name = self.trigger_name(&ctx),
119− ))?;
440+ db.run(&format!(
441+ r#"
442+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
443+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
444+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
445+ "#,
446+ trigger_name = self.trigger_name(ctx),
447+ reverse_trigger_name = self.reverse_trigger_name(ctx),
448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
449+ ))
450+ .context("failed to drop down trigger")?;
120451
121452 Ok(())
122453 }
src/migrations/remove_enum.rsnew file+51
@@ -0,0 +1,51 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveEnum {
11+ #[serde(rename = "enum")]
12+ pub enum_name: String,
13+}
14+
15+#[typetag::serde(name = "remove_enum")]
16+impl Action for RemoveEnum {
17+ fn describe(&self) -> String {
18+ format!("Removing enum \"{}\"", self.enum_name)
19+ }
20+
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ _db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
27+ Ok(())
28+ }
29+
30+ fn complete<'a>(
31+ &self,
32+ _ctx: &MigrationContext,
33+ db: &'a mut dyn Conn,
34+ ) -> anyhow::Result<Option<Transaction<'a>>> {
35+ db.run(&format!(
36+ r#"
37+ DROP TYPE IF EXISTS {name}
38+ "#,
39+ name = self.enum_name,
40+ ))
41+ .context("failed to drop enum")?;
42+
43+ Ok(None)
44+ }
45+
46+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
47+
48+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
49+ Ok(())
50+ }
51+}
src/migrations/remove_foreign_key.rsnew file+91
@@ -0,0 +1,91 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::{anyhow, Context};
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveForeignKey {
11+ table: String,
12+ foreign_key: String,
13+}
14+
15+#[typetag::serde(name = "remove_foreign_key")]
16+impl Action for RemoveForeignKey {
17+ fn describe(&self) -> String {
18+ format!(
19+ "Removing foreign key \"{}\" from table \"{}\"",
20+ self.foreign_key, self.table
21+ )
22+ }
23+
24+ fn run(
25+ &self,
26+ _ctx: &MigrationContext,
27+ db: &mut dyn Conn,
28+ schema: &Schema,
29+ ) -> anyhow::Result<()> {
30+ // The foreign key is only removed once the migration is completed.
31+ // Removing it earlier would be hard/undesirable for several reasons:
32+ // - Postgres doesn't have an easy way to temporarily disable a foreign key check.
33+ // If it did, we could disable the FK for the new schema.
34+ // - Even if we could, it probably wouldn't be a good idea as it would cause temporary
35+ // inconsistencies for the old schema which still expects the FK to hold.
36+ // - For the same reason, we can't remove the FK when the migration is first applied.
37+ // If the migration was to be aborted, then the FK would have to be recreated with
38+ // the risk that it would no longer be valid.
39+
40+ // Ensure foreign key exists
41+ let table = schema.get_table(db, &self.table)?;
42+ let fk_exists = !db
43+ .query(&format!(
44+ r#"
45+ SELECT constraint_name
46+ FROM information_schema.table_constraints
47+ WHERE
48+ constraint_type = 'FOREIGN KEY' AND
49+ table_name = '{table_name}' AND
50+ constraint_name = '{foreign_key}'
51+ "#,
52+ table_name = table.real_name,
53+ foreign_key = self.foreign_key,
54+ ))
55+ .context("failed to check for foreign key")?
56+ .is_empty();
57+
58+ if !fk_exists {
59+ return Err(anyhow!(
60+ "no foreign key \"{}\" exists on table \"{}\"",
61+ self.foreign_key,
62+ self.table
63+ ));
64+ }
65+
66+ Ok(())
67+ }
68+
69+ fn complete<'a>(
70+ &self,
71+ _ctx: &MigrationContext,
72+ db: &'a mut dyn Conn,
73+ ) -> anyhow::Result<Option<Transaction<'a>>> {
74+ db.run(&format!(
75+ r#"
76+ ALTER TABLE {table}
77+ DROP CONSTRAINT IF EXISTS {foreign_key}
78+ "#,
79+ table = self.table,
80+ foreign_key = self.foreign_key,
81+ ))
82+ .context("failed to remove foreign key")?;
83+ Ok(None)
84+ }
85+
86+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
87+
88+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
89+ Ok(())
90+ }
91+}
src/migrations/remove_index.rsnew file+51
@@ -0,0 +1,51 @@
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
7+use serde::{Deserialize, Serialize};
8+
9+#[derive(Serialize, Deserialize, Debug)]
10+pub struct RemoveIndex {
11+ pub index: String,
12+}
13+
14+#[typetag::serde(name = "remove_index")]
15+impl Action for RemoveIndex {
16+ fn describe(&self) -> String {
17+ format!("Removing index \"{}\"", self.index)
18+ }
19+
20+ fn run(
21+ &self,
22+ _ctx: &MigrationContext,
23+ _db: &mut dyn Conn,
24+ _schema: &Schema,
25+ ) -> anyhow::Result<()> {
26+ // Do nothing, the index isn't removed until completion
27+ Ok(())
28+ }
29+
30+ fn complete<'a>(
31+ &self,
32+ _ctx: &MigrationContext,
33+ db: &'a mut dyn Conn,
34+ ) -> anyhow::Result<Option<Transaction<'a>>> {
35+ db.run(&format!(
36+ r#"
37+ DROP INDEX CONCURRENTLY IF EXISTS "{name}"
38+ "#,
39+ name = self.index
40+ ))
41+ .context("failed to drop index")?;
42+
43+ Ok(None)
44+ }
45+
46+ fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
47+
48+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
49+ Ok(())
50+ }
51+}
src/migrations/remove_table.rs+24−13
@@ -1,5 +1,9 @@
1−use super::{Action, Context};
2−use crate::{db::Conn, schema::Schema};
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
37use serde::{Deserialize, Serialize};
48
59#[derive(Serialize, Deserialize, Debug)]
@@ -13,32 +17,39 @@ impl Action for RemoveTable {
1317 format!("Removing table \"{}\"", self.table)
1418 }
1519
16− fn run(&self, _ctx: &Context, _db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
20+ fn run(
21+ &self,
22+ _ctx: &MigrationContext,
23+ _db: &mut dyn Conn,
24+ _schema: &Schema,
25+ ) -> anyhow::Result<()> {
1726 Ok(())
1827 }
1928
20− fn complete(&self, _ctx: &Context, db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
29+ fn complete<'a>(
30+ &self,
31+ _ctx: &MigrationContext,
32+ db: &'a mut dyn Conn,
33+ ) -> anyhow::Result<Option<Transaction<'a>>> {
2134 // Remove table
2235 let query = format!(
23− "
24− DROP TABLE {table};
25− ",
36+ r#"
37+ DROP TABLE IF EXISTS "{table}";
38+ "#,
2639 table = self.table,
2740 );
28− db.run(&query)?;
41+ db.run(&query).context("failed to drop table")?;
2942
30− Ok(())
43+ Ok(None)
3144 }
3245
33− fn update_schema(&self, _ctx: &Context, schema: &mut Schema) -> anyhow::Result<()> {
46+ fn update_schema(&self, _ctx: &MigrationContext, schema: &mut Schema) {
3447 schema.change_table(&self.table, |table_changes| {
3548 table_changes.set_removed();
3649 });
37−
38− Ok(())
3950 }
4051
41− fn abort(&self, _ctx: &Context, _db: &mut dyn Conn) -> anyhow::Result<()> {
52+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
4253 Ok(())
4354 }
4455}
src/migrations/rename_table.rs+25−14
@@ -1,5 +1,9 @@
1−use super::{Action, Context};
2−use crate::{db::Conn, schema::Schema};
1+use super::{Action, MigrationContext};
2+use crate::{
3+ db::{Conn, Transaction},
4+ schema::Schema,
5+};
6+use anyhow::Context;
37use serde::{Deserialize, Serialize};
48
59#[derive(Serialize, Deserialize, Debug)]
@@ -14,34 +18,41 @@ impl Action for RenameTable {
1418 format!("Renaming table \"{}\" to \"{}\"", self.table, self.new_name)
1519 }
1620
17− fn run(&self, _ctx: &Context, _db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
21+ fn run(
22+ &self,
23+ _ctx: &MigrationContext,
24+ _db: &mut dyn Conn,
25+ _schema: &Schema,
26+ ) -> anyhow::Result<()> {
1827 Ok(())
1928 }
2029
21− fn complete(&self, _ctx: &Context, db: &mut dyn Conn, _schema: &Schema) -> anyhow::Result<()> {
30+ fn complete<'a>(
31+ &self,
32+ _ctx: &MigrationContext,
33+ db: &'a mut dyn Conn,
34+ ) -> anyhow::Result<Option<Transaction<'a>>> {
2235 // Rename table
2336 let query = format!(
24− "
25− ALTER TABLE {table}
26− RENAME TO {new_name}
27− ",
37+ r#"
38+ ALTER TABLE IF EXISTS "{table}"
39+ RENAME TO "{new_name}"
40+ "#,
2841 table = self.table,
2942 new_name = self.new_name,
3043 );
31− db.run(&query)?;
44+ db.run(&query).context("failed to rename table")?;
3245
33− Ok(())
46+ Ok(None)
3447 }
3548
36− fn update_schema(&self, _ctx: &Context, schema: &mut Schema) -> anyhow::Result<()> {
49+ fn update_schema(&self, _ctx: &MigrationContext, schema: &mut Schema) {
3750 schema.change_table(&self.table, |table_changes| {
3851 table_changes.set_name(&self.new_name);
3952 });
40−
41− Ok(())
4253 }
4354
44− fn abort(&self, _ctx: &Context, _db: &mut dyn Conn) -> anyhow::Result<()> {
55+ fn abort(&self, _ctx: &MigrationContext, _db: &mut dyn Conn) -> anyhow::Result<()> {
4556 Ok(())
4657 }
4758}
src/schema.rs+42−19
@@ -52,6 +52,12 @@ impl Schema {
5252 }
5353}
5454
55+impl Default for Schema {
56+ fn default() -> Self {
57+ Self::new()
58+ }
59+}
60+
5561#[derive(Debug)]
5662pub struct TableChanges {
5763 current_name: String,
@@ -64,7 +70,7 @@ impl TableChanges {
6470 fn new(name: String) -> Self {
6571 Self {
6672 current_name: name.to_string(),
67− real_name: name.to_string(),
73+ real_name: name,
6874 column_changes: Vec::new(),
6975 removed: false,
7076 }
@@ -100,8 +106,7 @@ impl TableChanges {
100106#[derive(Debug)]
101107pub struct ColumnChanges {
102108 current_name: String,
103− original_name: String,
104− intermediate_names: Vec<String>,
109+ backing_columns: Vec<String>,
105110 removed: bool,
106111}
107112
@@ -109,8 +114,7 @@ impl ColumnChanges {
109114 fn new(name: String) -> Self {
110115 Self {
111116 current_name: name.to_string(),
112− original_name: name.to_string(),
113− intermediate_names: Vec::new(),
117+ backing_columns: vec![name],
114118 removed: false,
115119 }
116120 }
@@ -120,7 +124,7 @@ impl ColumnChanges {
120124 }
121125
122126 pub fn set_column(&mut self, column_name: &str) {
123− self.intermediate_names.push(column_name.to_string())
127+ self.backing_columns.push(column_name.to_string())
124128 }
125129
126130 pub fn set_removed(&mut self) {
@@ -128,9 +132,9 @@ impl ColumnChanges {
128132 }
129133
130134 fn real_name(&self) -> &str {
131− self.intermediate_names
135+ self.backing_columns
132136 .last()
133− .unwrap_or_else(|| &self.original_name)
137+ .expect("backing_columns should never be empty")
134138 }
135139}
136140
@@ -147,6 +151,7 @@ pub struct Column {
147151 pub real_name: String,
148152 pub data_type: String,
149153 pub nullable: bool,
154+ pub default: Option<String>,
150155}
151156
152157impl Schema {
@@ -201,10 +206,10 @@ impl Schema {
201206 .iter()
202207 .find(|changes| changes.real_name == real_table_name);
203208
204− let real_columns: Vec<(String, String, bool)> = db
209+ let real_columns: Vec<(String, String, bool, Option<String>)> = db
205210 .query(&format!(
206211 "
207− SELECT column_name, data_type, is_nullable
212+ SELECT column_name, CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END, is_nullable, column_default
208213 FROM information_schema.columns
209214 WHERE table_name = '{table}' AND table_schema = 'public'
210215 ORDER BY ordinal_position
@@ -217,6 +222,7 @@ impl Schema {
217222 row.get("column_name"),
218223 row.get("data_type"),
219224 row.get::<'_, _, String>("is_nullable") == "YES",
225+ row.get("column_default"),
220226 )
221227 })
222228 .collect();
@@ -235,21 +241,20 @@ impl Schema {
235241 );
236242 }
237243
238− if !column_changes.intermediate_names.is_empty() {
239− ignore_columns.insert(column_changes.original_name.to_string());
244+ let (_, rest) = column_changes
245+ .backing_columns
246+ .split_last()
247+ .expect("backing_columns should never be empty");
240248
241− let non_used_intermediate = &column_changes.intermediate_names
242− [..column_changes.intermediate_names.len() - 1];
243− for intermediate in non_used_intermediate {
244− ignore_columns.insert(intermediate.to_string());
245− }
249+ for column in rest {
250+ ignore_columns.insert(column.to_string());
246251 }
247252 }
248253 }
249254
250255 let mut columns: Vec<Column> = Vec::new();
251256
252− for (real_name, data_type, nullable) in real_columns {
257+ for (real_name, data_type, nullable, default) in real_columns {
253258 if ignore_columns.contains(&*real_name) {
254259 continue;
255260 }
@@ -264,6 +269,7 @@ impl Schema {
264269 real_name,
265270 data_type,
266271 nullable,
272+ default,
267273 });
268274 }
269275
@@ -274,9 +280,26 @@ impl Schema {
274280 let table = Table {
275281 name: current_table_name.to_string(),
276282 real_name: real_table_name.to_string(),
277− columns: columns,
283+ columns,
278284 };
279285
280286 Ok(table)
281287 }
282288}
289+
290+impl Table {
291+ pub fn real_column_names<'a>(
292+ &'a self,
293+ columns: &'a [String],
294+ ) -> impl Iterator<Item = &'a String> {
295+ columns.iter().map(|name| {
296+ self.get_column(name)
297+ .map(|col| &col.real_name)
298+ .unwrap_or(name)
299+ })
300+ }
301+
302+ pub fn get_column(&self, name: &str) -> Option<&Column> {
303+ self.columns.iter().find(|column| column.name == name)
304+ }
305+}
src/state.rs+200−88
@@ -4,43 +4,51 @@ use anyhow::anyhow;
44use serde::{Deserialize, Serialize};
55use version::version;
66
7−#[derive(Serialize, Deserialize, Debug)]
8−pub struct State {
9− pub version: String,
10− pub status: Status,
11− pub current_migration: Option<String>,
12− pub migrations: Vec<Migration>,
13−}
14−
15−#[derive(Serialize, Deserialize, Debug)]
16−#[serde(tag = "status")]
17−pub enum Status {
7+#[derive(Serialize, Deserialize, Clone, Debug)]
8+#[serde(tag = "state")]
9+pub enum State {
1810 #[serde(rename = "idle")]
1911 Idle,
2012
13+ #[serde(rename = "applying")]
14+ Applying { migrations: Vec<Migration> },
15+
2116 #[serde(rename = "in_progress")]
2217 InProgress { migrations: Vec<Migration> },
18+
19+ #[serde(rename = "completing")]
20+ Completing {
21+ migrations: Vec<Migration>,
22+ current_migration_index: usize,
23+ current_action_index: usize,
24+ },
25+
26+ #[serde(rename = "aborting")]
27+ Aborting {
28+ migrations: Vec<Migration>,
29+ last_migration_index: usize,
30+ last_action_index: usize,
31+ },
2332}
2433
2534impl State {
26− pub fn load(db: &mut impl Conn) -> State {
27− Self::ensure_schema_and_table(db);
35+ pub fn load(db: &mut impl Conn) -> anyhow::Result<State> {
36+ Self::ensure_schema_and_table(db)?;
2837
29− let results = db
30− .query("SELECT value FROM reshape.data WHERE key = 'state'")
31− .unwrap();
38+ let results = db.query("SELECT value FROM reshape.data WHERE key = 'state'")?;
3239
33− match results.first() {
40+ let state = match results.first() {
3441 Some(row) => {
3542 let json: serde_json::Value = row.get(0);
36− serde_json::from_value(json).unwrap()
43+ serde_json::from_value(json)?
3744 }
3845 None => Default::default(),
39− }
46+ };
47+ Ok(state)
4048 }
4149
4250 pub fn save(&self, db: &mut impl Conn) -> anyhow::Result<()> {
43− Self::ensure_schema_and_table(db);
51+ Self::ensure_schema_and_table(db)?;
4452
4553 let json = serde_json::to_value(self)?;
4654 db.query_with_params(
@@ -53,108 +61,212 @@ impl State {
5361 pub fn clear(&mut self, db: &mut impl Conn) -> anyhow::Result<()> {
5462 db.run("DROP SCHEMA reshape CASCADE")?;
5563
56− let default = Self::default();
57− self.status = default.status;
58− self.current_migration = default.current_migration;
59− self.migrations = default.migrations;
64+ *self = Self::default();
6065
6166 Ok(())
6267 }
6368
64− pub fn set_migrations<T>(&mut self, migrations: T) -> Result<(), anyhow::Error>
65− where
66− T: IntoIterator<Item = Migration>,
67− {
68− let mut new_iter = migrations.into_iter();
69+ // Complete will change the state from Completing to Idle
70+ pub fn complete(&mut self, db: &mut impl Conn) -> anyhow::Result<()> {
71+ let current_state = std::mem::replace(self, Self::Idle);
72+
73+ match current_state {
74+ Self::Completing { migrations, .. } => {
75+ // Add migrations and update state in a transaction to ensure atomicity
76+ let mut transaction = db.transaction()?;
77+ save_migrations(&mut transaction, &migrations)?;
78+ self.save(&mut transaction)?;
79+ transaction.commit()?;
80+ }
81+ _ => {
82+ // Move old state back
83+ *self = current_state;
6984
70− // Ensure the new migration match up with the existing ones
71− for pair in self.migrations.iter().zip(new_iter.by_ref()) {
72− let (existing, ref new) = pair;
73− if existing != new {
7485 return Err(anyhow!(
75− "existing migration {} does not match new migration {}",
76− existing.name,
77− new.name
86+ "couldn't update state to be completed, not in Completing state"
7887 ));
7988 }
8089 }
8190
82− // Add any new migrations
83− self.migrations.extend(new_iter);
8491 Ok(())
8592 }
8693
87− pub fn add_migrations<'a>(&mut self, new_migrations: impl IntoIterator<Item = &'a Migration>) {
88− for migration in new_migrations.into_iter() {
89− self.migrations.push(migration.clone());
90− }
94+ pub fn applying(&mut self, new_migrations: Vec<Migration>) {
95+ *self = Self::Applying {
96+ migrations: new_migrations,
97+ };
9198 }
9299
93− // Complete will change the status from InProgress to Idle
94− pub fn complete(&mut self) -> anyhow::Result<()> {
95− let current_status = std::mem::replace(&mut self.status, Status::Idle);
100+ pub fn in_progress(&mut self, new_migrations: Vec<Migration>) {
101+ *self = Self::InProgress {
102+ migrations: new_migrations,
103+ };
104+ }
96105
97− match current_status {
98− Status::Idle => {
99− // Move old status back
100− self.status = current_status;
101− return Err(anyhow!("status is not in progress"));
102− }
103− Status::InProgress { mut migrations } => {
104− let target_migration = migrations.last().unwrap().name.to_string();
105− self.migrations.append(&mut migrations);
106− self.current_migration = Some(target_migration);
107− }
106+ pub fn completing(
107+ &mut self,
108+ migrations: Vec<Migration>,
109+ current_migration_index: usize,
110+ current_action_index: usize,
111+ ) {
112+ *self = Self::Completing {
113+ migrations,
114+ current_migration_index,
115+ current_action_index,
116+ }
117+ }
118+
119+ pub fn aborting(
120+ &mut self,
121+ migrations: Vec<Migration>,
122+ last_migration_index: usize,
123+ last_action_index: usize,
124+ ) {
125+ *self = Self::Aborting {
126+ migrations,
127+ last_migration_index,
128+ last_action_index,
108129 }
130+ }
131+
132+ fn ensure_schema_and_table(db: &mut impl Conn) -> anyhow::Result<()> {
133+ db.run("CREATE SCHEMA IF NOT EXISTS reshape")?;
134+
135+ // Create data table which will be a key-value table containing
136+ // the version and current state.
137+ db.run("CREATE TABLE IF NOT EXISTS reshape.data (key TEXT PRIMARY KEY, value JSONB)")?;
138+
139+ // Create migrations table which will store all completed migrations
140+ db.run(
141+ "
142+ CREATE TABLE IF NOT EXISTS reshape.migrations (
143+ index INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
144+ name TEXT NOT NULL,
145+ description TEXT,
146+ actions JSONB NOT NULL,
147+ completed_at TIMESTAMP DEFAULT NOW()
148+ )
149+ ",
150+ )?;
151+
152+ // Update the current version
153+ let encoded_version = serde_json::to_value(version!().to_string())?;
154+ db.query_with_params(
155+ "
156+ INSERT INTO reshape.data (key, value)
157+ VALUES ('version', $1)
158+ ON CONFLICT (key) DO UPDATE SET value = $1
159+ ",
160+ &[&encoded_version],
161+ )?;
109162
110163 Ok(())
111164 }
165+}
112166
113− pub fn in_progress(&mut self, new_migrations: Vec<Migration>) {
114− self.status = Status::InProgress {
115− migrations: new_migrations,
116− };
167+impl Default for State {
168+ fn default() -> Self {
169+ Self::Idle
117170 }
171+}
172+
173+pub fn current_migration(db: &mut dyn Conn) -> anyhow::Result<Option<String>> {
174+ let name: Option<String> = db
175+ .query(
176+ "
177+ SELECT name
178+ FROM reshape.migrations
179+ ORDER BY index DESC
180+ LIMIT 1
181+ ",
182+ )?
183+ .first()
184+ .map(|row| row.get("name"));
185+ Ok(name)
186+}
187+
188+pub fn remaining_migrations(
189+ db: &mut impl Conn,
190+ new_migrations: impl IntoIterator<Item = Migration>,
191+) -> anyhow::Result<Vec<Migration>> {
192+ let mut new_iter = new_migrations.into_iter();
193+
194+ // Ensure the new migrations match up with the existing ones
195+ let mut highest_index: Option<i32> = None;
196+ loop {
197+ let migrations = get_migrations(db, highest_index)?;
198+ if migrations.is_empty() {
199+ break;
200+ }
201+
202+ for (index, existing) in migrations {
203+ highest_index = Some(index);
118204
119− pub fn get_remaining_migrations(
120− &self,
121− new_migrations: impl IntoIterator<Item = Migration>,
122− ) -> anyhow::Result<Vec<Migration>> {
123− let mut new_iter = new_migrations.into_iter();
205+ let new = match new_iter.next() {
206+ Some(migration) => migration,
207+ None => {
208+ return Err(anyhow!(
209+ "existing migration {} doesn't exist in local migrations",
210+ existing
211+ ))
212+ }
213+ };
124214
125− // Ensure the new migration match up with the existing ones
126− for pair in self.migrations.iter().zip(new_iter.by_ref()) {
127− let (existing, ref new) = pair;
128− if existing != new {
215+ if existing != new.name {
129216 return Err(anyhow!(
130217 "existing migration {} does not match new migration {}",
131− existing.name,
218+ existing,
132219 new.name
133220 ));
134221 }
135222 }
136−
137− let items: Vec<Migration> = new_iter.collect();
138−
139− // Return the remaining migrations
140− Ok(items)
141223 }
142224
143− fn ensure_schema_and_table(db: &mut impl Conn) {
144− db.run("CREATE SCHEMA IF NOT EXISTS reshape").unwrap();
225+ // Return the remaining migrations
226+ let items: Vec<Migration> = new_iter.collect();
227+ Ok(items)
228+}
145229
146− db.run("CREATE TABLE IF NOT EXISTS reshape.data (key TEXT PRIMARY KEY, value JSONB)")
147− .unwrap();
148− }
230+fn get_migrations(
231+ db: &mut impl Conn,
232+ index_larger_than: Option<i32>,
233+) -> anyhow::Result<Vec<(i32, String)>> {
234+ let rows = if let Some(index_larger_than) = index_larger_than {
235+ db.query_with_params(
236+ "
237+ SELECT index, name
238+ FROM reshape.migrations
239+ WHERE index > $1
240+ ORDER BY index ASC
241+ LIMIT 100
242+ ",
243+ &[&index_larger_than],
244+ )?
245+ } else {
246+ db.query(
247+ "
248+ SELECT index, name
249+ FROM reshape.migrations
250+ LIMIT 100
251+ ",
252+ )?
253+ };
254+
255+ let migrations = rows
256+ .iter()
257+ .map(|row| (row.get("index"), row.get("name")))
258+ .collect();
259+ Ok(migrations)
149260}
150261
151−impl Default for State {
152− fn default() -> Self {
153− State {
154− version: version!().to_string(),
155− status: Status::Idle,
156− current_migration: None,
157− migrations: vec![],
158− }
262+fn save_migrations(db: &mut impl Conn, migrations: &[Migration]) -> anyhow::Result<()> {
263+ for migration in migrations {
264+ let encoded_actions = serde_json::to_value(&migration.actions)?;
265+ db.query_with_params(
266+ "INSERT INTO reshape.migrations(name, description, actions) VALUES ($1, $2, $3)",
267+ &[&migration.name, &migration.description, &encoded_actions],
268+ )?;
159269 }
270+
271+ Ok(())
160272}
tests/add_column.rs+316−262
@@ -1,86 +1,60 @@
1−use reshape::migrations::{AddColumn, Column, CreateTable, Migration};
2−use reshape::Status;
3−
41mod common;
2+use common::Test;
53
64#[test]
75fn add_column() {
8− let (mut reshape, mut old_db, mut new_db) = common::setup();
9−
10− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![
15− Column {
16− name: "id".to_string(),
17− data_type: "SERIAL".to_string(),
18− nullable: true,
19− default: None,
20− generated: None,
21− },
22− Column {
23− name: "name".to_string(),
24− data_type: "TEXT".to_string(),
25− nullable: false,
26− default: None,
27− generated: None,
28− },
29− ],
30− });
31− let add_first_last_name_columns = Migration::new("add_first_and_last_name_columns", None)
32− .with_action(AddColumn {
33− table: "users".to_string(),
34− column: Column {
35− name: "first".to_string(),
36− data_type: "TEXT".to_string(),
37− nullable: false,
38− default: None,
39− generated: None,
40− },
41− up: Some("(STRING_TO_ARRAY(name, ' '))[1]".to_string()),
42− })
43− .with_action(AddColumn {
44− table: "users".to_string(),
45− column: Column {
46− name: "last".to_string(),
47− data_type: "TEXT".to_string(),
48− nullable: false,
49− default: None,
50− generated: None,
51− },
52− up: Some("(STRING_TO_ARRAY(name, ' '))[2]".to_string()),
53− });
54−
55− let first_migrations = vec![create_users_table.clone()];
56− let second_migrations = vec![
57− create_users_table.clone(),
58− add_first_last_name_columns.clone(),
59− ];
60−
61− // Run first migration, should automatically finish
62− reshape.migrate(first_migrations.clone()).unwrap();
63− assert!(matches!(reshape.state.status, Status::Idle));
64− assert_eq!(
65− Some(&create_users_table.name),
66− reshape.state.current_migration.as_ref()
6+ let mut test = Test::new("Add column");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
6725 );
6826
69− // Update search paths
70− old_db
71− .simple_query(&reshape::schema_query_for_migration(
72− &first_migrations.last().unwrap().name,
73− ))
74− .unwrap();
75− new_db
76− .simple_query(&reshape::schema_query_for_migration(
77− &first_migrations.last().unwrap().name,
78− ))
79− .unwrap();
27+ test.second_migration(
28+ r#"
29+ name = "add_first_and_last_name_columns"
30+
31+ [[actions]]
32+ type = "add_column"
33+ table = "users"
8034
81− // Insert some test users
82− new_db
83− .simple_query(
35+ up = "(STRING_TO_ARRAY(name, ' '))[1]"
36+
37+ [actions.column]
38+ name = "first"
39+ type = "TEXT"
40+ nullable = false
41+
42+ [[actions]]
43+ type = "add_column"
44+ table = "users"
45+
46+ up = "(STRING_TO_ARRAY(name, ' '))[2]"
47+
48+ [actions.column]
49+ name = "last"
50+ type = "TEXT"
51+ nullable = false
52+ "#,
53+ );
54+
55+ test.after_first(|db| {
56+ // Insert some test users
57+ db.simple_query(
8458 "
8559 INSERT INTO users (id, name) VALUES
8660 (1, 'John Doe'),
@@ -88,220 +62,300 @@ fn add_column() {
8862 ",
8963 )
9064 .unwrap();
65+ });
9166
92− // Run second migration
93− reshape.migrate(second_migrations.clone()).unwrap();
94− new_db
95− .simple_query(&reshape::schema_query_for_migration(
96− &second_migrations.last().unwrap().name,
97− ))
98− .unwrap();
67+ test.intermediate(|old_db, new_db| {
68+ // Check that the existing users have the new columns populated
69+ let expected = vec![("John", "Doe"), ("Jane", "Doe")];
70+ assert!(new_db
71+ .query("SELECT first, last FROM users ORDER BY id", &[],)
72+ .unwrap()
73+ .iter()
74+ .map(|row| (row.get("first"), row.get("last")))
75+ .eq(expected));
76+
77+ // Insert data using old schema and make sure the new columns are populated
78+ old_db
79+ .simple_query("INSERT INTO users (id, name) VALUES (3, 'Test Testsson')")
80+ .unwrap();
81+ let (first_name, last_name): (String, String) = new_db
82+ .query_one("SELECT first, last from users WHERE id = 3", &[])
83+ .map(|row| (row.get("first"), row.get("last")))
84+ .unwrap();
85+ assert_eq!(
86+ ("Test", "Testsson"),
87+ (first_name.as_ref(), last_name.as_ref())
88+ );
89+ });
9990
100− // Check that the existing users have the new columns populated
101− let expected = vec![("John", "Doe"), ("Jane", "Doe")];
102− assert!(new_db
103− .query("SELECT first, last FROM users ORDER BY id", &[],)
104− .unwrap()
105− .iter()
106− .map(|row| (row.get("first"), row.get("last")))
107− .eq(expected));
108−
109− // Insert data using old schema and make sure the new columns are populated
110− old_db
111− .simple_query("INSERT INTO users (id, name) VALUES (3, 'Test Testsson')")
112− .unwrap();
113− let (first_name, last_name): (String, String) = new_db
114− .query_one("SELECT first, last from users WHERE id = 3", &[])
115− .map(|row| (row.get("first"), row.get("last")))
116− .unwrap();
117− assert_eq!(
118− ("Test", "Testsson"),
119− (first_name.as_ref(), last_name.as_ref())
120− );
91+ test.after_completion(|db| {
92+ let expected = vec![("John", "Doe"), ("Jane", "Doe"), ("Test", "Testsson")];
93+ assert!(db
94+ .query("SELECT first, last FROM users ORDER BY id", &[],)
95+ .unwrap()
96+ .iter()
97+ .map(|row| (row.get("first"), row.get("last")))
98+ .eq(expected));
99+ });
100+
101+ test.after_abort(|db| {
102+ let expected = vec![("John Doe"), ("Jane Doe"), ("Test Testsson")];
103+ assert!(db
104+ .query("SELECT name FROM users ORDER BY id", &[],)
105+ .unwrap()
106+ .iter()
107+ .map(|row| row.get::<'_, _, String>("name"))
108+ .eq(expected));
109+ });
121110
122− reshape.complete_migration().unwrap();
111+ test.run()
123112}
124113
125114#[test]
126115fn add_column_nullable() {
127− let (mut reshape, mut old_db, mut new_db) = common::setup();
128−
129− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
130− name: "users".to_string(),
131− primary_key: vec!["id".to_string()],
132− foreign_keys: vec![],
133− columns: vec![Column {
134− name: "id".to_string(),
135− data_type: "INTEGER".to_string(),
136− nullable: true,
137− default: None,
138− generated: None,
139− }],
140− });
141− let add_name_column = Migration::new("add_nullable_name_column", None).with_action(AddColumn {
142− table: "users".to_string(),
143− column: Column {
144− name: "name".to_string(),
145− data_type: "TEXT".to_string(),
146− nullable: true,
147− default: None,
148− generated: None,
149− },
150− up: None,
151− });
116+ let mut test = Test::new("Add nullable column");
117+
118+ test.first_migration(
119+ r#"
120+ name = "create_users_table"
152121
153− let first_migrations = vec![create_users_table.clone()];
154− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
122+ [[actions]]
123+ type = "create_table"
124+ name = "users"
125+ primary_key = ["id"]
155126
156− // Run first migration, should automatically finish
157− reshape.migrate(first_migrations.clone()).unwrap();
158− assert!(matches!(reshape.state.status, Status::Idle));
159− assert_eq!(
160− Some(&create_users_table.name),
161− reshape.state.current_migration.as_ref()
127+ [[actions.columns]]
128+ name = "id"
129+ type = "INTEGER"
130+ "#,
162131 );
163132
164− // Update search paths
165− old_db
166− .simple_query(&reshape::schema_query_for_migration(
167− &first_migrations.last().unwrap().name,
168− ))
169− .unwrap();
170− new_db
171− .simple_query(&reshape::schema_query_for_migration(
172− &first_migrations.last().unwrap().name,
173− ))
174− .unwrap();
133+ test.second_migration(
134+ r#"
135+ name = "add_nullable_name_column"
175136
176− // Insert some test values
177− new_db
178− .simple_query(
137+ [[actions]]
138+ type = "add_column"
139+ table = "users"
140+
141+ [actions.column]
142+ name = "name"
143+ type = "TEXT"
144+ "#,
145+ );
146+
147+ test.after_first(|db| {
148+ // Insert some test values
149+ db.simple_query(
179150 "
180− INSERT INTO users (id) VALUES
181− (1),
182− (2);
151+ INSERT INTO users (id) VALUES (1), (2);
183152 ",
184153 )
185154 .unwrap();
155+ });
186156
187− // Run second migration
188− reshape.migrate(second_migrations.clone()).unwrap();
189− new_db
190− .simple_query(&reshape::schema_query_for_migration(
191− &second_migrations.last().unwrap().name,
192− ))
193− .unwrap();
194−
195− // Ensure existing data got updated
196− let expected: Vec<Option<String>> = vec![None, None];
197− assert!(new_db
198− .query("SELECT name FROM users ORDER BY id", &[],)
199− .unwrap()
200− .iter()
201− .map(|row| row.get::<_, Option<String>>("name"))
202− .eq(expected));
203−
204− // Insert data using old schema and ensure new column is NULL
205− old_db
206− .simple_query("INSERT INTO users (id) VALUES (3)")
207− .unwrap();
208− let name: Option<String> = new_db
209− .query_one("SELECT name from users WHERE id = 3", &[])
210− .map(|row| (row.get("name")))
211− .unwrap();
212− assert_eq!(None, name);
157+ test.intermediate(|old_db, new_db| {
158+ // Ensure existing data got updated
159+ let expected: Vec<Option<String>> = vec![None, None];
160+ assert!(new_db
161+ .query("SELECT name FROM users ORDER BY id", &[],)
162+ .unwrap()
163+ .iter()
164+ .map(|row| row.get::<_, Option<String>>("name"))
165+ .eq(expected));
166+
167+ // Insert data using old schema and ensure new column is NULL
168+ old_db
169+ .simple_query("INSERT INTO users (id) VALUES (3)")
170+ .unwrap();
171+ let name: Option<String> = new_db
172+ .query_one("SELECT name from users WHERE id = 3", &[])
173+ .map(|row| (row.get("name")))
174+ .unwrap();
175+ assert_eq!(None, name);
176+
177+ // Ensure data can be inserted against new schema
178+ new_db
179+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
180+ .unwrap();
181+ });
213182
214− // Ensure data can be inserted against new schema
215− new_db
216− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Test Testsson'), (5, NULL)")
217− .unwrap();
183+ test.after_completion(|db| {
184+ let expected: Vec<Option<String>> =
185+ vec![None, None, None, Some("Test Testsson".to_string()), None];
186+ let result: Vec<Option<String>> = db
187+ .query("SELECT id, name FROM users ORDER BY id", &[])
188+ .unwrap()
189+ .iter()
190+ .map(|row| row.get("name"))
191+ .collect();
192+
193+ assert_eq!(result, expected);
194+ });
218195
219− reshape.complete_migration().unwrap();
196+ test.run();
220197}
221198
222199#[test]
223200fn add_column_with_default() {
224− let (mut reshape, mut old_db, mut new_db) = common::setup();
225−
226− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
227− name: "users".to_string(),
228− primary_key: vec!["id".to_string()],
229− foreign_keys: vec![],
230− columns: vec![Column {
231− name: "id".to_string(),
232− data_type: "INTEGER".to_string(),
233− nullable: true,
234− default: None,
235− generated: None,
236− }],
201+ let mut test = Test::new("Add column with default value");
202+
203+ test.first_migration(
204+ r#"
205+ name = "create_users_table"
206+
207+ [[actions]]
208+ type = "create_table"
209+ name = "users"
210+ primary_key = ["id"]
211+
212+ [[actions.columns]]
213+ name = "id"
214+ type = "INTEGER"
215+ "#,
216+ );
217+
218+ test.second_migration(
219+ r#"
220+ name = "add_name_column_with_default"
221+
222+ [[actions]]
223+ type = "add_column"
224+ table = "users"
225+
226+ [actions.column]
227+ name = "name"
228+ type = "TEXT"
229+ nullable = false
230+ default = "'DEFAULT'"
231+ "#,
232+ );
233+
234+ test.after_first(|db| {
235+ // Insert some test values
236+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
237+ .unwrap();
237238 });
238− let add_name_column =
239− Migration::new("add_name_column_with_default", None).with_action(AddColumn {
240− table: "users".to_string(),
241− column: Column {
242− name: "name".to_string(),
243− data_type: "TEXT".to_string(),
244− nullable: false,
245− default: Some("'DEFAULT'".to_string()),
246− generated: None,
247− },
248− up: None,
249− });
250−
251− let first_migrations = vec![create_users_table.clone()];
252− let second_migrations = vec![create_users_table.clone(), add_name_column.clone()];
253−
254− // Run first migration, should automatically finish
255− reshape.migrate(first_migrations.clone()).unwrap();
256− assert!(matches!(reshape.state.status, Status::Idle));
257− assert_eq!(
258− Some(&create_users_table.name),
259− reshape.state.current_migration.as_ref()
239+
240+ test.intermediate(|old_db, new_db| {
241+ // Ensure existing data got updated with defaults
242+ let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
243+ assert!(new_db
244+ .query("SELECT name FROM users ORDER BY id", &[],)
245+ .unwrap()
246+ .iter()
247+ .map(|row| row.get::<_, String>("name"))
248+ .eq(expected));
249+
250+ // Insert data using old schema and ensure new column gets the default value
251+ old_db
252+ .simple_query("INSERT INTO users (id) VALUES (3)")
253+ .unwrap();
254+ let name: String = new_db
255+ .query_one("SELECT name from users WHERE id = 3", &[])
256+ .map(|row| row.get("name"))
257+ .unwrap();
258+ assert_eq!("DEFAULT", name);
259+ });
260+
261+ test.run();
262+}
263+
264+#[test]
265+fn add_column_with_complex_up() {
266+ let mut test = Test::new("Add column complex");
267+
268+ test.first_migration(
269+ r#"
270+ name = "create_tables"
271+
272+ [[actions]]
273+ type = "create_table"
274+ name = "users"
275+ primary_key = ["id"]
276+
277+ [[actions.columns]]
278+ name = "id"
279+ type = "INTEGER"
280+
281+ [[actions.columns]]
282+ name = "email"
283+ type = "TEXT"
284+
285+ [[actions]]
286+ type = "create_table"
287+ name = "profiles"
288+ primary_key = ["user_id"]
289+
290+ [[actions.columns]]
291+ name = "user_id"
292+ type = "INTEGER"
293+ "#,
260294 );
261295
262− // Update search paths
263− old_db
264− .simple_query(&reshape::schema_query_for_migration(
265− &first_migrations.last().unwrap().name,
266− ))
267− .unwrap();
268− new_db
269− .simple_query(&reshape::schema_query_for_migration(
270− &first_migrations.last().unwrap().name,
271− ))
272− .unwrap();
296+ test.second_migration(
297+ r#"
298+ name = "add_profiles_email_column"
273299
274− // Insert some test values
275− new_db
276− .simple_query("INSERT INTO users (id) VALUES (1), (2)")
277− .unwrap();
300+ [[actions]]
301+ type = "add_column"
302+ table = "profiles"
278303
279− // Run second migration
280− reshape.migrate(second_migrations.clone()).unwrap();
281− new_db
282− .simple_query(&reshape::schema_query_for_migration(
283− &second_migrations.last().unwrap().name,
284− ))
285− .unwrap();
304+ [actions.column]
305+ name = "email"
306+ type = "TEXT"
307+ nullable = false
286308
287− // Ensure existing data got updated with defaults
288− let expected = vec!["DEFAULT".to_string(), "DEFAULT".to_string()];
289− assert!(new_db
290− .query("SELECT name FROM users ORDER BY id", &[],)
291− .unwrap()
292− .iter()
293− .map(|row| row.get::<_, String>("name"))
294− .eq(expected));
295−
296− // Insert data using old schema and ensure new column gets the default value
297− old_db
298− .simple_query("INSERT INTO users (id) VALUES (3)")
299− .unwrap();
300− let name: String = new_db
301− .query_one("SELECT name from users WHERE id = 3", &[])
302− .map(|row| row.get("name"))
303− .unwrap();
304− assert_eq!("DEFAULT", name);
309+ [actions.up]
310+ table = "users"
311+ value = "users.email"
312+ where = "profiles.user_id = users.id"
313+ "#,
314+ );
315+
316+ test.after_first(|db| {
317+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
318+ .unwrap();
319+ db.simple_query("INSERT INTO profiles (user_id) VALUES (1)")
320+ .unwrap();
321+ });
322+
323+ test.intermediate(|old_db, new_db| {
324+ // Ensure email was backfilled on profiles
325+ let email: String = new_db
326+ .query(
327+ "
328+ SELECT email
329+ FROM profiles
330+ WHERE user_id = 1
331+ ",
332+ &[],
333+ )
334+ .unwrap()
335+ .first()
336+ .map(|row| row.get("email"))
337+ .unwrap();
338+ assert_eq!("test@example.com", email);
339+
340+ // Ensure email change in old schema is propagated to profiles table in new schema
341+ old_db
342+ .simple_query("UPDATE users SET email = 'test2@example.com' WHERE id = 1")
343+ .unwrap();
344+ let email: String = new_db
345+ .query(
346+ "
347+ SELECT email
348+ FROM profiles
349+ WHERE user_id = 1
350+ ",
351+ &[],
352+ )
353+ .unwrap()
354+ .first()
355+ .map(|row| row.get("email"))
356+ .unwrap();
357+ assert_eq!("test2@example.com", email);
358+ });
305359
306− reshape.complete_migration().unwrap();
360+ test.run();
307361}
tests/add_foreign_key.rsnew file+166
@@ -0,0 +1,166 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn add_foreign_key() {
6+ let mut test = Test::new("Add foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+ "#,
34+ );
35+
36+ test.second_migration(
37+ r#"
38+ name = "add_foreign_key"
39+
40+ [[actions]]
41+ type = "add_foreign_key"
42+ table = "items"
43+
44+ [actions.foreign_key]
45+ columns = ["user_id"]
46+ referenced_table = "users"
47+ referenced_columns = ["id"]
48+ "#,
49+ );
50+
51+ test.after_first(|db| {
52+ // Insert some test users
53+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
54+ .unwrap();
55+ });
56+
57+ test.intermediate(|db, _| {
58+ // Ensure items can be inserted if they reference valid users
59+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
60+ .unwrap();
61+
62+ // Ensure items can't be inserted if they don't reference valid users
63+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
64+ assert!(result.is_err(), "expected insert to fail");
65+ });
66+
67+ test.after_completion(|db| {
68+ // Ensure items can be inserted if they reference valid users
69+ db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 1), (4, 2)")
70+ .unwrap();
71+
72+ // Ensure items can't be inserted if they don't reference valid users
73+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)");
74+ assert!(result.is_err(), "expected insert to fail");
75+
76+ // Ensure foreign key exists with the right name
77+ let foreign_key_name: Option<String> = db
78+ .query(
79+ "
80+ SELECT tc.constraint_name
81+ FROM information_schema.table_constraints AS tc
82+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
83+ ",
84+ &[],
85+ )
86+ .unwrap()
87+ .first()
88+ .map(|row| row.get(0));
89+ assert_eq!(Some("items_user_id_fkey".to_string()), foreign_key_name);
90+ });
91+
92+ test.after_abort(|db| {
93+ // Ensure foreign key doesn't exist
94+ let fk_does_not_exist = db
95+ .query(
96+ "
97+ SELECT tc.constraint_name
98+ FROM information_schema.table_constraints AS tc
99+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
100+ ",
101+ &[],
102+ )
103+ .unwrap()
104+ .is_empty();
105+ assert!(fk_does_not_exist);
106+ });
107+
108+ test.run()
109+}
110+
111+#[test]
112+fn add_invalid_foreign_key() {
113+ let mut test = Test::new("Add invalid foreign key");
114+
115+ test.first_migration(
116+ r#"
117+ name = "create_user_table"
118+
119+ [[actions]]
120+ type = "create_table"
121+ name = "users"
122+ primary_key = ["id"]
123+
124+ [[actions.columns]]
125+ name = "id"
126+ type = "INTEGER"
127+
128+ [[actions]]
129+ type = "create_table"
130+ name = "items"
131+ primary_key = ["id"]
132+
133+ [[actions.columns]]
134+ name = "id"
135+ type = "INTEGER"
136+
137+ [[actions.columns]]
138+ name = "user_id"
139+ type = "INTEGER"
140+ "#,
141+ );
142+
143+ test.second_migration(
144+ r#"
145+ name = "add_foreign_key"
146+
147+ [[actions]]
148+ type = "add_foreign_key"
149+ table = "items"
150+
151+ [actions.foreign_key]
152+ columns = ["user_id"]
153+ referenced_table = "users"
154+ referenced_columns = ["id"]
155+ "#,
156+ );
157+
158+ test.after_first(|db| {
159+ // Insert some items which don't reference a valid user
160+ db.simple_query("INSERT INTO items (id, user_id) VALUES (1, 1), (2, 2)")
161+ .unwrap();
162+ });
163+
164+ test.expect_failure();
165+ test.run()
166+}
tests/add_index.rs+216−65
@@ -1,72 +1,223 @@
1−use reshape::migrations::{AddIndex, Column, CreateTable, Migration};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn add_index() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration =
10− Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![
15− Column {
16− name: "id".to_string(),
17− data_type: "INTEGER".to_string(),
18− nullable: true,
19− default: None,
20− generated: None,
21− },
22− Column {
23− name: "name".to_string(),
24− data_type: "TEXT".to_string(),
25− nullable: false,
26− default: None,
27− generated: None,
28− },
29− ],
30− });
31− let add_index_migration = Migration::new("add_name_index", None).with_action(AddIndex {
32− table: "users".to_string(),
33− name: "name_idx".to_string(),
34− columns: vec!["name".to_string()],
6+ let mut test = Test::new("Add index");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "add_users_name_index"
30+
31+ [[actions]]
32+ type = "add_index"
33+ table = "users"
34+
35+ [actions.index]
36+ name = "name_idx"
37+ columns = ["name"]
38+ "#,
39+ );
40+
41+ test.intermediate(|db, _| {
42+ // Ensure index is valid and ready
43+ let (is_ready, is_valid): (bool, bool) = db
44+ .query(
45+ "
46+ SELECT pg_index.indisready, pg_index.indisvalid
47+ FROM pg_catalog.pg_index
48+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
49+ WHERE pg_class.relname = 'name_idx'
50+ ",
51+ &[],
52+ )
53+ .unwrap()
54+ .first()
55+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
56+ .unwrap();
57+
58+ assert!(is_ready, "expected index to be ready");
59+ assert!(is_valid, "expected index to be valid");
60+ });
61+
62+ test.after_completion(|db| {
63+ // Ensure index is valid and ready
64+ let (is_ready, is_valid): (bool, bool) = db
65+ .query(
66+ "
67+ SELECT pg_index.indisready, pg_index.indisvalid
68+ FROM pg_catalog.pg_index
69+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
70+ WHERE pg_class.relname = 'name_idx'
71+ ",
72+ &[],
73+ )
74+ .unwrap()
75+ .first()
76+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
77+ .unwrap();
78+
79+ assert!(is_ready, "expected index to be ready");
80+ assert!(is_valid, "expected index to be valid");
81+ });
82+
83+ test.run();
84+}
85+
86+#[test]
87+fn add_index_unique() {
88+ let mut test = Test::new("Add unique index");
89+
90+ test.first_migration(
91+ r#"
92+ name = "create_users_table"
93+
94+ [[actions]]
95+ type = "create_table"
96+ name = "users"
97+ primary_key = ["id"]
98+
99+ [[actions.columns]]
100+ name = "id"
101+ type = "INTEGER"
102+
103+ [[actions.columns]]
104+ name = "name"
105+ type = "TEXT"
106+ "#,
107+ );
108+
109+ test.second_migration(
110+ r#"
111+ name = "add_name_index"
112+
113+ [[actions]]
114+ type = "add_index"
115+ table = "users"
116+
117+ [actions.index]
118+ name = "name_idx"
119+ columns = ["name"]
120+ unique = true
121+ "#,
122+ );
123+
124+ test.intermediate(|db, _| {
125+ // Ensure index is valid, ready and unique
126+ let (is_ready, is_valid, is_unique): (bool, bool, bool) = db
127+ .query(
128+ "
129+ SELECT pg_index.indisready, pg_index.indisvalid, pg_index.indisunique
130+ FROM pg_catalog.pg_index
131+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
132+ WHERE pg_class.relname = 'name_idx'
133+ ",
134+ &[],
135+ )
136+ .unwrap()
137+ .first()
138+ .map(|row| {
139+ (
140+ row.get("indisready"),
141+ row.get("indisvalid"),
142+ row.get("indisunique"),
143+ )
144+ })
145+ .unwrap();
146+
147+ assert!(is_ready, "expected index to be ready");
148+ assert!(is_valid, "expected index to be valid");
149+ assert!(is_unique, "expected index to be unique");
150+ });
151+
152+ test.run();
153+}
154+
155+#[test]
156+fn add_index_with_type() {
157+ let mut test = Test::new("Add GIN index");
158+
159+ test.first_migration(
160+ r#"
161+ name = "create_users_table"
162+
163+ [[actions]]
164+ type = "create_table"
165+ name = "users"
166+ primary_key = ["id"]
167+
168+ [[actions.columns]]
169+ name = "id"
170+ type = "INTEGER"
171+
172+ [[actions.columns]]
173+ name = "data"
174+ type = "JSONB"
175+ "#,
176+ );
177+
178+ test.second_migration(
179+ r#"
180+ name = "add_data_index"
181+
182+ [[actions]]
183+ type = "add_index"
184+ table = "users"
185+
186+ [actions.index]
187+ name = "data_idx"
188+ columns = ["data"]
189+ type = "gin"
190+ "#,
191+ );
192+
193+ test.intermediate(|db, _| {
194+ // Ensure index is valid, ready and has the right type
195+ let (is_ready, is_valid, index_type): (bool, bool, String) = db
196+ .query(
197+ "
198+ SELECT pg_index.indisready, pg_index.indisvalid, pg_am.amname
199+ FROM pg_catalog.pg_index
200+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
201+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
202+ WHERE pg_class.relname = 'data_idx'
203+ ",
204+ &[],
205+ )
206+ .unwrap()
207+ .first()
208+ .map(|row| {
209+ (
210+ row.get("indisready"),
211+ row.get("indisvalid"),
212+ row.get("amname"),
213+ )
214+ })
215+ .unwrap();
216+
217+ assert!(is_ready, "expected index to be ready");
218+ assert!(is_valid, "expected index to be valid");
219+ assert_eq!("gin", index_type, "expected index type to be GIN");
35220 });
36221
37− let first_migrations = vec![create_table_migration.clone()];
38− let second_migrations = vec![create_table_migration.clone(), add_index_migration.clone()];
39−
40− // Run migrations
41− reshape.migrate(first_migrations.clone()).unwrap();
42− reshape.migrate(second_migrations.clone()).unwrap();
43−
44− // Update schemas of Postgres connections
45− let old_schema_query =
46− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
47− let new_schema_query =
48− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
49− println!("{}", new_schema_query);
50− old_db.simple_query(&old_schema_query).unwrap();
51− new_db.simple_query(&new_schema_query).unwrap();
52−
53− // Ensure index is valid and ready
54− let result: Vec<(bool, bool)> = new_db
55− .query(
56− "
57− SELECT pg_index.indisready, pg_index.indisvalid
58− FROM pg_catalog.pg_index
59− JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
60− WHERE pg_class.relname = 'name_idx'
61− ",
62− &[],
63− )
64− .unwrap()
65− .iter()
66− .map(|row| (row.get("indisready"), row.get("indisvalid")))
67− .collect();
68−
69− assert_eq!(vec![(true, true)], result);
70−
71− reshape.complete_migration().unwrap();
222+ test.run();
72223}
tests/alter_column.rs+660−348
@@ -1,66 +1,45 @@
1−use reshape::migrations::{AlterColumn, Column, ColumnChanges, CreateTable, Migration};
2−use reshape::Status;
3−
41mod common;
2+use common::Test;
53
64#[test]
75fn alter_column_data() {
8− let (mut reshape, mut old_db, mut new_db) = common::setup();
9−
10− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![
15− Column {
16− name: "id".to_string(),
17− data_type: "INTEGER".to_string(),
18− nullable: true,
19− default: None,
20− generated: None,
21− },
22− Column {
23− name: "name".to_string(),
24− data_type: "TEXT".to_string(),
25− nullable: false,
26− default: None,
27− generated: None,
28− },
29− ],
30− });
31− let uppercase_name = Migration::new("uppercase_name", None).with_action(AlterColumn {
32− table: "users".to_string(),
33− column: "name".to_string(),
34− up: Some("UPPER(name)".to_string()),
35− down: Some("LOWER(name)".to_string()),
36− changes: ColumnChanges {
37− data_type: None,
38− nullable: None,
39− name: None,
40− },
41− });
6+ let mut test = Test::new("Alter column");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
4211
43− let first_migrations = vec![create_users_table.clone()];
44− let second_migrations = vec![create_users_table.clone(), uppercase_name.clone()];
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
4516
46− // Run first migration, should automatically finish
47− reshape.migrate(first_migrations.clone()).unwrap();
48− assert!(matches!(reshape.state.status, Status::Idle));
49− assert_eq!(
50− Some(&create_users_table.name),
51− reshape.state.current_migration.as_ref()
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
5225 );
5326
54− // Update search paths
55− old_db
56− .simple_query(&reshape::schema_query_for_migration(
57− &first_migrations.last().unwrap().name,
58− ))
59− .unwrap();
27+ test.second_migration(
28+ r#"
29+ name = "uppercase_name"
30+
31+ [[actions]]
32+ type = "alter_column"
33+ table = "users"
34+ column = "name"
35+ up = "UPPER(name)"
36+ down = "LOWER(name)"
37+ "#,
38+ );
6039
61− // Insert some test users
62− old_db
63− .simple_query(
40+ test.after_first(|db| {
41+ // Insert some test users
42+ db.simple_query(
6443 "
6544 INSERT INTO users (id, name) VALUES
6645 (1, 'john Doe'),
@@ -68,103 +47,81 @@ fn alter_column_data() {
6847 ",
6948 )
7049 .unwrap();
50+ });
7151
72− // Run second migration
73− reshape.migrate(second_migrations.clone()).unwrap();
74− new_db
75− .simple_query(&reshape::schema_query_for_migration(
76− &second_migrations.last().unwrap().name,
77− ))
78− .unwrap();
79−
80− // Check that the existing users has the altered data
81− let expected = vec!["JOHN DOE", "JANE DOE"];
82− assert!(new_db
83− .query("SELECT name FROM users ORDER BY id", &[],)
84− .unwrap()
85− .iter()
86− .map(|row| row.get::<_, String>("name"))
87− .eq(expected));
88−
89− // Insert data using old schema and make sure the new schema gets correct values
90− old_db
91− .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
92− .unwrap();
93− let result = new_db
94− .query_one("SELECT name from users WHERE id = 3", &[])
95− .unwrap();
96− assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
97−
98− // Insert data using new schema and make sure the old schema gets correct values
99− new_db
100− .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
101− .unwrap();
102− let result = old_db
103− .query_one("SELECT name from users WHERE id = 4", &[])
104− .unwrap();
105− assert_eq!("test testsson", result.get::<_, &str>("name"));
52+ test.intermediate(|old_db, new_db| {
53+ // Check that the existing users has the altered data
54+ let expected = vec!["JOHN DOE", "JANE DOE"];
55+ assert!(new_db
56+ .query("SELECT name FROM users ORDER BY id", &[],)
57+ .unwrap()
58+ .iter()
59+ .map(|row| row.get::<_, String>("name"))
60+ .eq(expected));
61+
62+ // Insert data using old schema and make sure the new schema gets correct values
63+ old_db
64+ .simple_query("INSERT INTO users (id, name) VALUES (3, 'test testsson')")
65+ .unwrap();
66+ let result = new_db
67+ .query_one("SELECT name from users WHERE id = 3", &[])
68+ .unwrap();
69+ assert_eq!("TEST TESTSSON", result.get::<_, &str>("name"));
70+
71+ // Insert data using new schema and make sure the old schema gets correct values
72+ new_db
73+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'TEST TESTSSON')")
74+ .unwrap();
75+ let result = old_db
76+ .query_one("SELECT name from users WHERE id = 4", &[])
77+ .unwrap();
78+ assert_eq!("test testsson", result.get::<_, &str>("name"));
79+ });
10680
107− reshape.complete_migration().unwrap();
81+ test.run();
10882}
10983
11084#[test]
11185fn alter_column_set_not_null() {
112− let (mut reshape, mut old_db, mut new_db) = common::setup();
113−
114− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
115− name: "users".to_string(),
116− primary_key: vec!["id".to_string()],
117− foreign_keys: vec![],
118− columns: vec![
119− Column {
120− name: "id".to_string(),
121− data_type: "INTEGER".to_string(),
122− nullable: true,
123− default: None,
124− generated: None,
125− },
126− Column {
127− name: "name".to_string(),
128− data_type: "TEXT".to_string(),
129− nullable: true,
130− default: None,
131− generated: None,
132− },
133− ],
134− });
135− let set_name_not_null = Migration::new("set_name_not_null", None).with_action(AlterColumn {
136− table: "users".to_string(),
137− column: "name".to_string(),
138− up: Some("COALESCE(name, 'TEST_DEFAULT_VALUE')".to_string()),
139− down: Some("name".to_string()),
140− changes: ColumnChanges {
141− data_type: None,
142− nullable: Some(false),
143− name: None,
144− },
145− });
86+ let mut test = Test::new("Set column not null");
87+
88+ test.first_migration(
89+ r#"
90+ name = "create_user_table"
91+
92+ [[actions]]
93+ type = "create_table"
94+ name = "users"
95+ primary_key = ["id"]
14696
147− let first_migrations = vec![create_users_table.clone()];
148− let second_migrations = vec![create_users_table.clone(), set_name_not_null.clone()];
97+ [[actions.columns]]
98+ name = "id"
99+ type = "INTEGER"
149100
150− // Run first migration, should automatically finish
151− reshape.migrate(first_migrations.clone()).unwrap();
152− assert!(matches!(reshape.state.status, Status::Idle));
153− assert_eq!(
154− Some(&create_users_table.name),
155− reshape.state.current_migration.as_ref()
101+ [[actions.columns]]
102+ name = "name"
103+ type = "TEXT"
104+ "#,
156105 );
157106
158− // Update search paths
159− old_db
160− .simple_query(&reshape::schema_query_for_migration(
161− &first_migrations.last().unwrap().name,
162− ))
163− .unwrap();
107+ test.second_migration(
108+ r#"
109+ name = "set_name_not_null"
110+
111+ [[actions]]
112+ type = "alter_column"
113+ table = "users"
114+ column = "name"
115+ up = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
116+
117+ [actions.changes]
118+ nullable = false
119+ "#,
120+ );
164121
165− // Insert some test users
166− old_db
167− .simple_query(
122+ test.after_first(|db| {
123+ // Insert some test users
124+ db.simple_query(
168125 "
169126 INSERT INTO users (id, name) VALUES
170127 (1, 'John Doe'),
@@ -172,104 +129,180 @@ fn alter_column_set_not_null() {
172129 ",
173130 )
174131 .unwrap();
132+ });
175133
176− // Run second migration
177− reshape.migrate(second_migrations.clone()).unwrap();
178− new_db
179− .simple_query(&reshape::schema_query_for_migration(
180− &second_migrations.last().unwrap().name,
181− ))
182− .unwrap();
134+ test.intermediate(|old_db, new_db| {
135+ // Check that existing users got the correct values
136+ let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
137+ assert!(new_db
138+ .query("SELECT name FROM users ORDER BY id", &[],)
139+ .unwrap()
140+ .iter()
141+ .map(|row| row.get::<_, String>("name"))
142+ .eq(expected));
143+
144+ // Insert data using old schema and make sure the new schema gets correct values
145+ old_db
146+ .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
147+ .unwrap();
148+ let result = new_db
149+ .query_one("SELECT name from users WHERE id = 3", &[])
150+ .unwrap();
151+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
152+
153+ // Insert data using new schema and make sure the old schema gets correct values
154+ new_db
155+ .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
156+ .unwrap();
157+ let result = old_db
158+ .query_one("SELECT name from users WHERE id = 4", &[])
159+ .unwrap();
160+ assert_eq!("Jane Doe", result.get::<_, &str>("name"));
161+
162+ // Ensure NULL can't be inserted using the new schema
163+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
164+ assert!(result.is_err(), "expected insert to fail");
165+ });
183166
184− // Check that existing users got the correct values
185− let expected = vec!["John Doe", "TEST_DEFAULT_VALUE"];
186− assert!(new_db
187− .query("SELECT name FROM users ORDER BY id", &[],)
188− .unwrap()
189− .iter()
190− .map(|row| row.get::<_, String>("name"))
191− .eq(expected));
192−
193− // Insert data using old schema and make sure the new schema gets correct values
194− old_db
195− .simple_query("INSERT INTO users (id, name) VALUES (3, NULL)")
196− .unwrap();
197− let result = new_db
198− .query_one("SELECT name from users WHERE id = 3", &[])
199− .unwrap();
200− assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
167+ test.after_completion(|db| {
168+ // Ensure NULL can't be inserted
169+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
170+ assert!(result.is_err(), "expected insert to fail");
171+ });
201172
202− // Insert data using new schema and make sure the old schema gets correct values
203− new_db
204− .simple_query("INSERT INTO users (id, name) VALUES (4, 'Jane Doe')")
205− .unwrap();
206− let result = old_db
207− .query_one("SELECT name from users WHERE id = 4", &[])
173+ test.after_abort(|db| {
174+ // Ensure NULL can be inserted
175+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
176+ assert!(result.is_ok(), "expected insert to succeed");
177+ });
178+
179+ test.run();
180+}
181+
182+#[test]
183+fn alter_column_set_nullable() {
184+ let mut test = Test::new("Set column nullable");
185+
186+ test.first_migration(
187+ r#"
188+ name = "create_user_table"
189+
190+ [[actions]]
191+ type = "create_table"
192+ name = "users"
193+ primary_key = ["id"]
194+
195+ [[actions.columns]]
196+ name = "id"
197+ type = "INTEGER"
198+
199+ [[actions.columns]]
200+ name = "name"
201+ type = "TEXT"
202+ nullable = false
203+ "#,
204+ );
205+
206+ test.second_migration(
207+ r#"
208+ name = "set_name_nullable"
209+
210+ [[actions]]
211+ type = "alter_column"
212+ table = "users"
213+ column = "name"
214+ down = "COALESCE(name, 'TEST_DEFAULT_VALUE')"
215+
216+ [actions.changes]
217+ nullable = true
218+ "#,
219+ );
220+
221+ test.after_first(|db| {
222+ // Insert a test user
223+ db.simple_query(
224+ "
225+ INSERT INTO users (id, name) VALUES
226+ (1, 'John Doe')
227+ ",
228+ )
208229 .unwrap();
209− assert_eq!("Jane Doe", result.get::<_, &str>("name"));
230+ });
231+
232+ test.intermediate(|old_db, new_db| {
233+ // Insert data using new schema and make sure the old schema gets correct values
234+ new_db
235+ .simple_query("INSERT INTO users (id, name) VALUES (2, NULL)")
236+ .unwrap();
237+ let result = old_db
238+ .query_one("SELECT name from users WHERE id = 2", &[])
239+ .unwrap();
240+ assert_eq!("TEST_DEFAULT_VALUE", result.get::<_, &str>("name"));
241+
242+ // Ensure NULL can't be inserted using the old schema
243+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, NULL)");
244+ assert!(result.is_err(), "expected insert to fail");
245+
246+ // Ensure NULL can be inserted using the new schema
247+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (4, NULL)");
248+ assert!(result.is_ok(), "expected insert to succeed");
249+ });
250+
251+ test.after_completion(|db| {
252+ // Ensure NULL can be inserted
253+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
254+ assert!(result.is_ok(), "expected insert to succeed");
255+ });
256+
257+ test.after_abort(|db| {
258+ // Ensure NULL can't be inserted
259+ let result = db.simple_query("INSERT INTO users (id, name) VALUES (5, NULL)");
260+ assert!(result.is_err(), "expected insert to fail");
261+ });
210262
211− reshape.complete_migration().unwrap();
263+ test.run();
212264}
213265
214266#[test]
215267fn alter_column_rename() {
216− let (mut reshape, mut old_db, mut new_db) = common::setup();
217−
218− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
219− name: "users".to_string(),
220− primary_key: vec!["id".to_string()],
221− foreign_keys: vec![],
222− columns: vec![
223− Column {
224− name: "id".to_string(),
225− data_type: "INTEGER".to_string(),
226− nullable: true,
227− default: None,
228− generated: None,
229− },
230− Column {
231− name: "name".to_string(),
232− data_type: "TEXT".to_string(),
233− nullable: true,
234− default: None,
235− generated: None,
236− },
237− ],
238− });
239− let rename_to_full_name =
240− Migration::new("rename_to_full_name", None).with_action(AlterColumn {
241− table: "users".to_string(),
242− column: "name".to_string(),
243− up: None, // up and down are not required when only renaming a column
244− down: None,
245− changes: ColumnChanges {
246− data_type: None,
247− nullable: None,
248− name: Some("full_name".to_string()),
249− },
250− });
251−
252− let first_migrations = vec![create_users_table.clone()];
253− let second_migrations = vec![create_users_table.clone(), rename_to_full_name.clone()];
254−
255− // Run first migration, should automatically finish
256− reshape.migrate(first_migrations.clone()).unwrap();
257− assert!(matches!(reshape.state.status, Status::Idle));
258− assert_eq!(
259− Some(&create_users_table.name),
260− reshape.state.current_migration.as_ref()
268+ let mut test = Test::new("Rename column");
269+
270+ test.first_migration(
271+ r#"
272+ name = "create_user_table"
273+
274+ [[actions]]
275+ type = "create_table"
276+ name = "users"
277+ primary_key = ["id"]
278+
279+ [[actions.columns]]
280+ name = "id"
281+ type = "INTEGER"
282+
283+ [[actions.columns]]
284+ name = "name"
285+ type = "TEXT"
286+ "#,
261287 );
262288
263− // Update search paths
264− old_db
265− .simple_query(&reshape::schema_query_for_migration(
266− &first_migrations.last().unwrap().name,
267− ))
268− .unwrap();
289+ test.second_migration(
290+ r#"
291+ name = "set_name_not_null"
292+
293+ [[actions]]
294+ type = "alter_column"
295+ table = "users"
296+ column = "name"
269297
270− // Insert some test data
271− old_db
272− .simple_query(
298+ [actions.changes]
299+ name = "full_name"
300+ "#,
301+ );
302+
303+ test.after_first(|db| {
304+ // Insert some test data
305+ db.simple_query(
273306 "
274307 INSERT INTO users (id, name) VALUES
275308 (1, 'John Doe'),
@@ -277,97 +310,69 @@ fn alter_column_rename() {
277310 ",
278311 )
279312 .unwrap();
313+ });
280314
281− // Run second migration
282− reshape.migrate(second_migrations.clone()).unwrap();
283− new_db
284− .simple_query(&reshape::schema_query_for_migration(
285− &second_migrations.last().unwrap().name,
286− ))
287− .unwrap();
288−
289− // Check that existing values can be queried using new column name
290− let expected = vec!["John Doe", "Jane Doe"];
291− assert!(new_db
292− .query("SELECT full_name FROM users ORDER BY id", &[],)
293− .unwrap()
294− .iter()
295− .map(|row| row.get::<_, String>("full_name"))
296− .eq(expected));
315+ test.intermediate(|_, new_db| {
316+ // Check that existing values can be queried using new column name
317+ let expected = vec!["John Doe", "Jane Doe"];
318+ assert!(new_db
319+ .query("SELECT full_name FROM users ORDER BY id", &[],)
320+ .unwrap()
321+ .iter()
322+ .map(|row| row.get::<_, String>("full_name"))
323+ .eq(expected));
324+ });
297325
298− reshape.complete_migration().unwrap();
326+ test.run();
299327}
300328
301329#[test]
302330fn alter_column_multiple() {
303− let (mut reshape, mut old_db, mut new_db) = common::setup();
304−
305− let create_users_table = Migration::new("create_users_table", None).with_action(CreateTable {
306− name: "users".to_string(),
307− primary_key: vec!["id".to_string()],
308− foreign_keys: vec![],
309− columns: vec![
310− Column {
311− name: "id".to_string(),
312− data_type: "INTEGER".to_string(),
313− nullable: true,
314− default: None,
315− generated: None,
316− },
317− Column {
318− name: "counter".to_string(),
319− data_type: "INTEGER".to_string(),
320− nullable: false,
321− default: None,
322− generated: None,
323− },
324− ],
325− });
326− let increment_counter_twice = Migration::new("increment_counter_twice", None)
327− .with_action(AlterColumn {
328− table: "users".to_string(),
329− column: "counter".to_string(),
330− up: Some("counter + 1".to_string()),
331− down: Some("counter - 1".to_string()),
332− changes: ColumnChanges {
333− data_type: None,
334− nullable: None,
335− name: None,
336− },
337− })
338− .with_action(AlterColumn {
339− table: "users".to_string(),
340− column: "counter".to_string(),
341− up: Some("counter + 1".to_string()),
342− down: Some("counter - 1".to_string()),
343− changes: ColumnChanges {
344− data_type: None,
345− nullable: None,
346− name: None,
347− },
348− });
349−
350− let first_migrations = vec![create_users_table.clone()];
351− let second_migrations = vec![create_users_table.clone(), increment_counter_twice.clone()];
352−
353− // Run first migration, should automatically finish
354− reshape.migrate(first_migrations.clone()).unwrap();
355− assert!(matches!(reshape.state.status, Status::Idle));
356− assert_eq!(
357− Some(&create_users_table.name),
358− reshape.state.current_migration.as_ref()
331+ let mut test = Test::new("Alter column value multiple times");
332+
333+ test.first_migration(
334+ r#"
335+ name = "create_user_table"
336+
337+ [[actions]]
338+ type = "create_table"
339+ name = "users"
340+ primary_key = ["id"]
341+
342+ [[actions.columns]]
343+ name = "id"
344+ type = "INTEGER"
345+
346+ [[actions.columns]]
347+ name = "counter"
348+ type = "INTEGER"
349+ nullable = false
350+ "#,
359351 );
360352
361− // Update search paths
362− old_db
363− .simple_query(&reshape::schema_query_for_migration(
364− &first_migrations.last().unwrap().name,
365− ))
366− .unwrap();
353+ test.second_migration(
354+ r#"
355+ name = "increment_counter_twice"
356+
357+ [[actions]]
358+ type = "alter_column"
359+ table = "users"
360+ column = "counter"
361+ up = "counter + 1"
362+ down = "counter - 1"
363+
364+ [[actions]]
365+ type = "alter_column"
366+ table = "users"
367+ column = "counter"
368+ up = "counter + 1"
369+ down = "counter - 1"
370+ "#,
371+ );
367372
368− // Insert some test data
369− old_db
370− .simple_query(
373+ test.after_first(|db| {
374+ // Insert some test data
375+ db.simple_query(
371376 "
372377 INSERT INTO users (id, counter) VALUES
373378 (1, 0),
@@ -375,48 +380,355 @@ fn alter_column_multiple() {
375380 ",
376381 )
377382 .unwrap();
383+ });
378384
379− // Run second migration
380− reshape.migrate(second_migrations.clone()).unwrap();
381− new_db
382− .simple_query(&reshape::schema_query_for_migration(
383− &second_migrations.last().unwrap().name,
384− ))
385− .unwrap();
385+ test.intermediate(|old_db, new_db| {
386+ // Check that the existing data has been updated
387+ let expected = vec![2, 102];
388+ let results: Vec<i32> = new_db
389+ .query("SELECT counter FROM users ORDER BY id", &[])
390+ .unwrap()
391+ .iter()
392+ .map(|row| row.get::<_, i32>("counter"))
393+ .collect();
394+ assert_eq!(expected, results);
395+
396+ // Update data using old schema and make sure it was updated for the new schema
397+ old_db
398+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
399+ .unwrap();
400+ let result: i32 = new_db
401+ .query("SELECT counter FROM users WHERE id = 1", &[])
402+ .unwrap()
403+ .iter()
404+ .map(|row| row.get("counter"))
405+ .nth(0)
406+ .unwrap();
407+ assert_eq!(52, result);
408+
409+ // Update data using new schema and make sure it was updated for the old schema
410+ new_db
411+ .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
412+ .unwrap();
413+ let result: i32 = old_db
414+ .query("SELECT counter FROM users WHERE id = 1", &[])
415+ .unwrap()
416+ .iter()
417+ .map(|row| row.get("counter"))
418+ .nth(0)
419+ .unwrap();
420+ assert_eq!(48, result);
421+ });
386422
387− // Check that the existing data has been updated
388− let expected = vec![2, 102];
389− let results: Vec<i32> = new_db
390− .query("SELECT counter FROM users ORDER BY id", &[])
391− .unwrap()
392− .iter()
393− .map(|row| row.get::<_, i32>("counter"))
394− .collect();
395− assert_eq!(expected, results);
396−
397− // Update data using old schema and make sure it was updated for the new schema
398− old_db
399− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
400− .unwrap();
401− let result: i32 = new_db
402− .query("SELECT counter FROM users WHERE id = 1", &[])
403− .unwrap()
404− .iter()
405− .map(|row| row.get("counter"))
406− .nth(0)
407− .unwrap();
408− assert_eq!(52, result);
423+ test.run();
424+}
409425
410− // Update data using new schema and make sure it was updated for the old schema
411− new_db
412− .query("UPDATE users SET counter = 50 WHERE id = 1", &[])
413− .unwrap();
414− let result: i32 = old_db
415− .query("SELECT counter FROM users WHERE id = 1", &[])
416− .unwrap()
417− .iter()
418− .map(|row| row.get("counter"))
419− .nth(0)
426+#[test]
427+fn alter_column_default() {
428+ let mut test = Test::new("Change default value for column");
429+
430+ test.first_migration(
431+ r#"
432+ name = "create_user_table"
433+
434+ [[actions]]
435+ type = "create_table"
436+ name = "users"
437+ primary_key = ["id"]
438+
439+ [[actions.columns]]
440+ name = "id"
441+ type = "INTEGER"
442+
443+ [[actions.columns]]
444+ name = "name"
445+ type = "TEXT"
446+ nullable = false
447+ default = "'DEFAULT'"
448+ "#,
449+ );
450+
451+ test.second_migration(
452+ r#"
453+ name = "change_name_default"
454+
455+ [[actions]]
456+ type = "alter_column"
457+ table = "users"
458+ column = "name"
459+
460+ [actions.changes]
461+ default = "'NEW DEFAULT'"
462+ "#,
463+ );
464+
465+ test.after_first(|db| {
466+ // Insert a test user
467+ db.simple_query(
468+ "
469+ INSERT INTO users (id) VALUES (1)
470+ ",
471+ )
420472 .unwrap();
421− assert_eq!(48, result);
473+ });
474+
475+ test.intermediate(|old_db, new_db| {
476+ // Check that the existing users has the old default value
477+ let expected = vec!["DEFAULT"];
478+ assert!(new_db
479+ .query("SELECT name FROM users", &[],)
480+ .unwrap()
481+ .iter()
482+ .map(|row| row.get::<_, String>("name"))
483+ .eq(expected));
484+
485+ // Insert data using old schema and make those get the old default value
486+ old_db
487+ .simple_query("INSERT INTO users (id) VALUES (2)")
488+ .unwrap();
489+ let result = new_db
490+ .query_one("SELECT name from users WHERE id = 2", &[])
491+ .unwrap();
492+ assert_eq!("DEFAULT", result.get::<_, &str>("name"));
493+
494+ // Insert data using new schema and make sure it gets the new default value
495+ new_db
496+ .simple_query("INSERT INTO users (id) VALUES (3)")
497+ .unwrap();
498+ let result = old_db
499+ .query_one("SELECT name from users WHERE id = 3", &[])
500+ .unwrap();
501+ assert_eq!("NEW DEFAULT", result.get::<_, &str>("name"));
502+ });
503+
504+ test.run();
505+}
506+
507+#[test]
508+fn alter_column_with_index() {
509+ let mut test = Test::new("Alter column with index");
510+
511+ test.first_migration(
512+ r#"
513+ name = "create_user_table"
514+
515+ [[actions]]
516+ type = "create_table"
517+ name = "users"
518+ primary_key = ["id"]
519+
520+ [[actions.columns]]
521+ name = "id"
522+ type = "INTEGER"
523+
524+ [[actions.columns]]
525+ name = "first_name"
526+ type = "TEXT"
527+
528+ [[actions.columns]]
529+ name = "last_name"
530+ type = "TEXT"
531+
532+ [[actions]]
533+ type = "add_index"
534+ table = "users"
535+
536+ [actions.index]
537+ name = "users_name_idx"
538+ columns = ["first_name", "last_name"]
539+ "#,
540+ );
541+
542+ test.second_migration(
543+ r#"
544+ name = "uppercase_last_name"
545+
546+ [[actions]]
547+ type = "alter_column"
548+ table = "users"
549+ column = "last_name"
550+ up = "UPPER(last_name)"
551+ down = "LOWER(last_name)"
552+ "#,
553+ );
554+
555+ test.after_completion(|db| {
556+ // Make sure index still exists
557+ let result: i64 = db
558+ .query(
559+ "
560+ SELECT COUNT(*)
561+ FROM pg_catalog.pg_index
562+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
563+ WHERE pg_class.relname = 'users_name_idx'
564+ ",
565+ &[],
566+ )
567+ .unwrap()
568+ .first()
569+ .map(|row| row.get(0))
570+ .unwrap();
571+ assert_eq!(1, result, "expected index to still exist");
572+ });
573+
574+ test.run();
575+}
576+
577+#[test]
578+fn alter_column_with_unique_index() {
579+ let mut test = Test::new("Alter column with unique index");
580+
581+ test.first_migration(
582+ r#"
583+ name = "create_user_table"
584+
585+ [[actions]]
586+ type = "create_table"
587+ name = "users"
588+ primary_key = ["id"]
589+
590+ [[actions.columns]]
591+ name = "id"
592+ type = "INTEGER"
593+
594+ [[actions.columns]]
595+ name = "name"
596+ type = "TEXT"
597+
598+ [[actions]]
599+ type = "add_index"
600+ table = "users"
601+
602+ [actions.index]
603+ name = "name_idx"
604+ columns = ["name"]
605+ unique = true
606+ "#,
607+ );
608+
609+ test.second_migration(
610+ r#"
611+ name = "uppercase_name"
612+
613+ [[actions]]
614+ type = "alter_column"
615+ table = "users"
616+ column = "name"
617+ up = "UPPER(name)"
618+ down = "LOWER(name)"
619+ "#,
620+ );
621+
622+ test.after_first(|db| {
623+ db.simple_query("INSERT INTO users (id, name) VALUES (1, 'Test')")
624+ .unwrap();
625+ });
626+
627+ test.intermediate(|old_db, new_db| {
628+ // Try inserting a value which duplicates the uppercase value of an existing row
629+ let result = new_db.simple_query("INSERT INTO users (id, name) VALUES (2, 'TEST')");
630+ assert!(
631+ result.is_err(),
632+ "expected duplicate insert to new schema to fail"
633+ );
634+
635+ // Try inserting a value which duplicates the lowercase value of an existing row
636+ new_db
637+ .simple_query("INSERT INTO users (id, name) VALUES (2, 'JOHN')")
638+ .unwrap();
639+ let result = old_db.simple_query("INSERT INTO users (id, name) VALUES (3, 'john')");
640+ assert!(
641+ result.is_err(),
642+ "expected duplicate insert to old schema to fail"
643+ );
644+ });
645+
646+ test.after_completion(|db| {
647+ // Make sure index still exists
648+ let is_unique: bool = db
649+ .query(
650+ "
651+ SELECT pg_index.indisunique
652+ FROM pg_catalog.pg_index
653+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
654+ WHERE pg_class.relname = 'name_idx'
655+ ",
656+ &[],
657+ )
658+ .unwrap()
659+ .first()
660+ .map(|row| row.get("indisunique"))
661+ .unwrap();
662+ assert!(is_unique, "expected index to still be unique");
663+ });
664+
665+ test.run();
666+}
667+
668+#[test]
669+fn alter_column_with_hash_index() {
670+ let mut test = Test::new("Alter column with custom index type");
671+
672+ test.first_migration(
673+ r#"
674+ name = "create_user_table"
675+
676+ [[actions]]
677+ type = "create_table"
678+ name = "users"
679+ primary_key = ["id"]
680+
681+ [[actions.columns]]
682+ name = "id"
683+ type = "INTEGER"
684+
685+ [[actions.columns]]
686+ name = "name"
687+ type = "TEXT"
688+
689+ [[actions]]
690+ type = "add_index"
691+ table = "users"
692+
693+ [actions.index]
694+ name = "name_idx"
695+ columns = ["name"]
696+ type = "hash"
697+ "#,
698+ );
699+
700+ test.second_migration(
701+ r#"
702+ name = "uppercase_name"
703+
704+ [[actions]]
705+ type = "alter_column"
706+ table = "users"
707+ column = "name"
708+ up = "UPPER(name)"
709+ down = "LOWER(name)"
710+ "#,
711+ );
712+
713+ test.after_completion(|db| {
714+ // Make sure index still has type GIN
715+ let index_type: String = db
716+ .query(
717+ "
718+ SELECT pg_am.amname
719+ FROM pg_catalog.pg_index
720+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
721+ JOIN pg_catalog.pg_am ON pg_class.relam = pg_am.oid
722+ WHERE pg_class.relname = 'name_idx'
723+ ",
724+ &[],
725+ )
726+ .unwrap()
727+ .first()
728+ .map(|row| row.get("amname"))
729+ .unwrap();
730+ assert_eq!("hash", index_type);
731+ });
732+
733+ test.run();
422734}
tests/common.rs+304−9
@@ -1,15 +1,310 @@
1+use colored::Colorize;
12use postgres::{Client, NoTls};
2−use reshape::Reshape;
3+use reshape::{migrations::Migration, Reshape};
34
4−pub fn setup() -> (Reshape, Client, Client) {
5− let connection_string = std::env::var("POSTGRES_CONNECTION_STRING")
6− .unwrap_or("postgres://postgres:postgres@localhost/reshape_test".to_string());
5+pub struct Test<'a> {
6+ name: &'a str,
7+ reshape: Reshape,
8+ old_db: Client,
9+ new_db: Client,
710
8− let old_db = Client::connect(&connection_string, NoTls).unwrap();
9− let new_db = Client::connect(&connection_string, NoTls).unwrap();
11+ first_migration: Option<Migration>,
12+ second_migration: Option<Migration>,
13+ expect_failure: bool,
1014
11− let mut reshape = Reshape::new(&connection_string).unwrap();
12− reshape.remove().unwrap();
15+ clear_fn: Option<fn(&mut Client) -> ()>,
16+ after_first_fn: Option<fn(&mut Client) -> ()>,
17+ intermediate_fn: Option<fn(&mut Client, &mut Client) -> ()>,
18+ after_completion_fn: Option<fn(&mut Client) -> ()>,
19+ after_abort_fn: Option<fn(&mut Client) -> ()>,
20+}
21+
22+impl Test<'_> {
23+ pub fn new<'a>(name: &'a str) -> Test<'a> {
24+ let connection_string = std::env::var("POSTGRES_CONNECTION_STRING")
25+ .unwrap_or("postgres://postgres:postgres@localhost/reshape_test".to_string());
26+
27+ let old_db = Client::connect(&connection_string, NoTls).unwrap();
28+ let new_db = Client::connect(&connection_string, NoTls).unwrap();
29+
30+ let reshape = Reshape::new(&connection_string).unwrap();
31+
32+ Test {
33+ name,
34+ reshape,
35+ old_db,
36+ new_db,
37+ first_migration: None,
38+ second_migration: None,
39+ expect_failure: false,
40+ clear_fn: None,
41+ after_first_fn: None,
42+ intermediate_fn: None,
43+ after_completion_fn: None,
44+ after_abort_fn: None,
45+ }
46+ }
47+
48+ pub fn first_migration(&mut self, migration: &str) -> &mut Self {
49+ self.first_migration = Some(Self::parse_migration(migration));
50+ self
51+ }
52+
53+ #[allow(dead_code)]
54+ pub fn second_migration(&mut self, migration: &str) -> &mut Self {
55+ self.second_migration = Some(Self::parse_migration(migration));
56+ self
57+ }
58+
59+ #[allow(dead_code)]
60+ pub fn clear(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
61+ self.clear_fn = Some(f);
62+ self
63+ }
64+
65+ #[allow(dead_code)]
66+ pub fn after_first(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
67+ self.after_first_fn = Some(f);
68+ self
69+ }
70+
71+ #[allow(dead_code)]
72+ pub fn intermediate(&mut self, f: fn(&mut Client, &mut Client) -> ()) -> &mut Self {
73+ self.intermediate_fn = Some(f);
74+ self
75+ }
76+
77+ #[allow(dead_code)]
78+ pub fn after_completion(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
79+ self.after_completion_fn = Some(f);
80+ self
81+ }
82+
83+ #[allow(dead_code)]
84+ pub fn after_abort(&mut self, f: fn(&mut Client) -> ()) -> &mut Self {
85+ self.after_abort_fn = Some(f);
86+ self
87+ }
88+
89+ #[allow(dead_code)]
90+ pub fn expect_failure(&mut self) {
91+ self.expect_failure = true;
92+ }
93+
94+ fn parse_migration(encoded: &str) -> Migration {
95+ toml::from_str(encoded).unwrap()
96+ }
97+}
98+
99+enum RunType {
100+ Simple,
101+ Completion,
102+ Abort,
103+}
104+
105+impl Test<'_> {
106+ #[allow(dead_code)]
107+ pub fn run(&mut self) {
108+ if self.second_migration.is_some() {
109+ // Run to completion
110+ print_heading(&format!("Test completion: {}", self.name));
111+ self.run_internal(RunType::Completion);
112+
113+ // Run and abort
114+ print_heading(&format!("Test abort: {}", self.name));
115+ self.run_internal(RunType::Abort);
116+ } else {
117+ print_heading(&format!("Test: {}", self.name));
118+ self.run_internal(RunType::Simple);
119+ }
120+ }
121+
122+ fn run_internal(&mut self, run_type: RunType) {
123+ print_subheading("Clearing database");
124+ self.reshape.remove().unwrap();
125+
126+ if let Some(clear_fn) = self.clear_fn {
127+ clear_fn(&mut self.old_db);
128+ }
129+
130+ // Apply first migration, will automatically complete
131+ print_subheading("Applying first migration");
132+ let first_migration = self
133+ .first_migration
134+ .as_ref()
135+ .expect("no starting migration set");
136+ self.reshape.migrate(vec![first_migration.clone()]).unwrap();
137+
138+ // Update search path
139+ self.old_db
140+ .simple_query(&reshape::schema_query_for_migration(&first_migration.name))
141+ .unwrap();
142+
143+ // Automatically complete first migration
144+ self.reshape.complete().unwrap();
145+
146+ // Run setup function
147+ if let Some(after_first_fn) = self.after_first_fn {
148+ print_subheading("Running setup and first checks");
149+ after_first_fn(&mut self.old_db);
150+ print_success();
151+ }
152+
153+ // Apply second migration
154+ if let Some(second_migration) = &self.second_migration {
155+ if self.expect_failure {
156+ print_subheading("Applying second migration (expecting failure)");
157+ let result = self
158+ .reshape
159+ .migrate(vec![first_migration.clone(), second_migration.clone()]);
160+
161+ if result.is_ok() {
162+ panic!("expected second migration to fail");
163+ }
164+ } else {
165+ print_subheading("Applying second migration");
166+ self.reshape
167+ .migrate(vec![first_migration.clone(), second_migration.clone()])
168+ .unwrap();
169+ }
170+
171+ // Update search path
172+ self.new_db
173+ .simple_query(&reshape::schema_query_for_migration(&second_migration.name))
174+ .unwrap();
175+
176+ if let Some(intermediate_fn) = self.intermediate_fn {
177+ print_subheading("Running intermediate checks");
178+ intermediate_fn(&mut self.old_db, &mut self.new_db);
179+ print_success();
180+ }
181+
182+ match run_type {
183+ RunType::Completion => {
184+ print_subheading("Completing");
185+ self.reshape.complete().unwrap();
186+
187+ if let Some(after_completion_fn) = self.after_completion_fn {
188+ print_subheading("Running post-completion checks");
189+ after_completion_fn(&mut self.new_db);
190+ print_success();
191+ }
192+ }
193+ RunType::Abort => {
194+ print_subheading("Aborting");
195+ self.reshape.abort().unwrap();
196+
197+ if let Some(after_abort_fn) = self.after_abort_fn {
198+ print_subheading("Running post-abort checks");
199+ after_abort_fn(&mut self.old_db);
200+ print_success();
201+ }
202+ }
203+ _ => {}
204+ }
205+ }
206+
207+ print_subheading("Checking cleanup");
208+ assert_cleaned_up(&mut self.new_db);
209+ print_success();
210+ }
211+}
212+
213+fn print_heading(text: &str) {
214+ let delimiter = std::iter::repeat("=").take(80).collect::<String>();
215+
216+ println!();
217+ println!();
218+ println!("{}", delimiter.blue().bold());
219+ println!("{}", add_spacer(text, "=").blue().bold());
220+ println!("{}", delimiter.blue().bold());
221+}
222+
223+fn print_subheading(text: &str) {
224+ println!();
225+ println!("{}", add_spacer(text, "=").blue());
226+}
227+
228+fn print_success() {
229+ println!("{}", add_spacer("Success", "=").green());
230+}
231+
232+fn add_spacer(text: &str, char: &str) -> String {
233+ const TARGET_WIDTH: usize = 80;
234+ let num_of_chars = (TARGET_WIDTH - text.len() - 2) / 2;
235+ let spacer = std::iter::repeat(char)
236+ .take(num_of_chars)
237+ .collect::<String>();
238+
239+ let extra = if text.len() % 2 == 0 { "" } else { char };
240+
241+ format!("{spacer} {text} {spacer}{extra}", spacer = spacer)
242+}
243+
244+pub fn assert_cleaned_up(db: &mut Client) {
245+ // Make sure no temporary columns remain
246+ let temp_columns: Vec<String> = db
247+ .query(
248+ "
249+ SELECT column_name
250+ FROM information_schema.columns
251+ WHERE table_schema = 'public'
252+ AND column_name LIKE '__reshape%'
253+ ",
254+ &[],
255+ )
256+ .unwrap()
257+ .iter()
258+ .map(|row| row.get(0))
259+ .collect();
260+
261+ assert!(
262+ temp_columns.is_empty(),
263+ "expected no temporary columns to exist, found: {}",
264+ temp_columns.join(", ")
265+ );
266+
267+ // Make sure no triggers remain
268+ let triggers: Vec<String> = db
269+ .query(
270+ "
271+ SELECT trigger_name
272+ FROM information_schema.triggers
273+ WHERE trigger_schema = 'public'
274+ AND trigger_name LIKE '__reshape%'
275+ ",
276+ &[],
277+ )
278+ .unwrap()
279+ .iter()
280+ .map(|row| row.get(0))
281+ .collect();
282+
283+ assert!(
284+ triggers.is_empty(),
285+ "expected no triggers to exist, found: {}",
286+ triggers.join(", ")
287+ );
288+
289+ // Make sure no functions remain
290+ let functions: Vec<String> = db
291+ .query(
292+ "
293+ SELECT routine_name
294+ FROM information_schema.routines
295+ WHERE routine_schema = 'public'
296+ AND routine_name LIKE '__reshape%'
297+ ",
298+ &[],
299+ )
300+ .unwrap()
301+ .iter()
302+ .map(|row| row.get(0))
303+ .collect();
13304
14− (reshape, old_db, new_db)
305+ assert!(
306+ functions.is_empty(),
307+ "expected no functions to exist, found: {}",
308+ functions.join(", ")
309+ );
15310}
tests/complex.rsnew file+379
@@ -0,0 +1,379 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn move_column_between_tables() {
6+ let mut test = Test::new("Move column between tables");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "email"
23+ type = "TEXT"
24+
25+ [[actions]]
26+ type = "create_table"
27+ name = "profiles"
28+ primary_key = ["id"]
29+
30+ [[actions.columns]]
31+ name = "id"
32+ type = "INTEGER"
33+
34+ [[actions.columns]]
35+ name = "user_id"
36+ type = "INTEGER"
37+ nullable = false
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "move_email_column"
44+
45+ [[actions]]
46+ type = "add_column"
47+ table = "profiles"
48+
49+ [actions.column]
50+ name = "email"
51+ type = "TEXT"
52+ nullable = false
53+
54+ # When `users` is updated in the old schema, we write the email value to `profiles`
55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated
56+ [actions.up]
57+ table = "users"
58+ value = "users.email"
59+ where = "profiles.user_id = users.id"
60+
61+ [[actions]]
62+ type = "remove_column"
63+ table = "users"
64+ column = "email"
65+
66+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
67+ [actions.down]
68+ table = "profiles"
69+ value = "profiles.email"
70+ where = "users.id = profiles.user_id"
71+ "#,
72+ );
73+
74+ test.after_first(|db| {
75+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")
76+ .unwrap();
77+ db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")
78+ .unwrap();
79+
80+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")
81+ .unwrap();
82+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")
83+ .unwrap();
84+ });
85+
86+ test.intermediate(|old_db, new_db| {
87+ // Ensure emails were backfilled into profiles
88+ let profiles_emails: Vec<String> = new_db
89+ .query(
90+ r#"
91+ SELECT email
92+ FROM profiles
93+ ORDER BY id
94+ "#,
95+ &[],
96+ )
97+ .unwrap()
98+ .iter()
99+ .map(|row| row.get("email"))
100+ .collect();
101+ assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);
102+
103+ // Ensure insert in old schema updates new
104+ old_db
105+ .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")
106+ .unwrap();
107+ old_db
108+ .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")
109+ .unwrap();
110+ let new_email: String = new_db
111+ .query("SELECT email FROM profiles WHERE id = 3", &[])
112+ .unwrap()
113+ .first()
114+ .map(|row| row.get("email"))
115+ .unwrap();
116+ assert_eq!("test3@test.com", new_email);
117+
118+ // Ensure updates in old schema updates new
119+ old_db
120+ .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")
121+ .unwrap();
122+ let new_email: String = new_db
123+ .query("SELECT email FROM profiles WHERE id = 3", &[])
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get("email"))
127+ .unwrap();
128+ assert_eq!("test3+updated@test.com", new_email);
129+
130+ // Ensure insert in new schema updates old
131+ new_db
132+ .simple_query(
133+ "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",
134+ )
135+ .unwrap();
136+ new_db
137+ .simple_query("INSERT INTO users (id) VALUES (4)")
138+ .unwrap();
139+ let old_email: String = old_db
140+ .query("SELECT email FROM users WHERE id = 4", &[])
141+ .unwrap()
142+ .first()
143+ .map(|row| row.get("email"))
144+ .unwrap();
145+ assert_eq!("test4@test.com", old_email);
146+
147+ // Ensure update in new schema updates old
148+ new_db
149+ .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")
150+ .unwrap();
151+ let old_email: String = old_db
152+ .query("SELECT email FROM users WHERE id = 4", &[])
153+ .unwrap()
154+ .first()
155+ .map(|row| row.get("email"))
156+ .unwrap();
157+ assert_eq!("test4+updated@test.com", old_email);
158+ });
159+
160+ test.run();
161+}
162+
163+#[test]
164+fn extract_relation_into_new_table() {
165+ let mut test = Test::new("Extract relation into new table");
166+
167+ test.first_migration(
168+ r#"
169+ name = "create_tables"
170+
171+ [[actions]]
172+ type = "create_table"
173+ name = "accounts"
174+ primary_key = ["id"]
175+
176+ [[actions.columns]]
177+ name = "id"
178+ type = "INTEGER"
179+
180+ [[actions]]
181+ type = "create_table"
182+ name = "users"
183+ primary_key = ["id"]
184+
185+ [[actions.columns]]
186+ name = "id"
187+ type = "INTEGER"
188+
189+ [[actions.columns]]
190+ name = "account_id"
191+ type = "INTEGER"
192+ nullable = false
193+
194+ [[actions.columns]]
195+ name = "account_role"
196+ type = "TEXT"
197+ nullable = false
198+ "#,
199+ );
200+
201+ test.second_migration(
202+ r#"
203+ name = "add_account_user_connection"
204+
205+ [[actions]]
206+ type = "create_table"
207+ name = "user_account_connections"
208+ primary_key = ["account_id", "user_id"]
209+
210+ [[actions.columns]]
211+ name = "account_id"
212+ type = "INTEGER"
213+
214+ [[actions.columns]]
215+ name = "user_id"
216+ type = "INTEGER"
217+
218+ [[actions.columns]]
219+ name = "role"
220+ type = "TEXT"
221+ nullable = false
222+
223+ [actions.up]
224+ table = "users"
225+ values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
226+ where = "user_account_connections.user_id = users.id"
227+
228+ [[actions]]
229+ type = "remove_column"
230+ table = "users"
231+ column = "account_id"
232+
233+ [actions.down]
234+ table = "user_account_connections"
235+ value = "user_account_connections.account_id"
236+ where = "users.id = user_account_connections.user_id"
237+
238+ [[actions]]
239+ type = "remove_column"
240+ table = "users"
241+ column = "account_role"
242+
243+ [actions.down]
244+ table = "user_account_connections"
245+ value = "LOWER(user_account_connections.role)"
246+ where = "users.id = user_account_connections.user_id"
247+ "#,
248+ );
249+
250+ test.after_first(|db| {
251+ db.simple_query("INSERT INTO accounts (id) VALUES (1)")
252+ .unwrap();
253+ db.simple_query("INSERT INTO users (id, account_id, account_role) VALUES (1, 1, 'admin')")
254+ .unwrap();
255+ });
256+
257+ test.intermediate(|old_db, new_db| {
258+ // Ensure connections was backfilled
259+ let rows: Vec<(i32, i32, String)> = new_db
260+ .query(
261+ "
262+ SELECT account_id, user_id, role
263+ FROM user_account_connections
264+ ",
265+ &[],
266+ )
267+ .unwrap()
268+ .iter()
269+ .map(|row| (row.get("account_id"), row.get("user_id"), row.get("role")))
270+ .collect();
271+ assert_eq!(1, rows.len());
272+
273+ let row = rows.first().unwrap();
274+ assert_eq!(1, row.0);
275+ assert_eq!(1, row.1);
276+ assert_eq!("ADMIN", row.2);
277+
278+ // Ensure inserted user in old schema creates a new connection
279+ old_db
280+ .simple_query(
281+ "INSERT INTO users (id, account_id, account_role) VALUES (2, 1, 'developer')",
282+ )
283+ .unwrap();
284+ assert!(
285+ new_db
286+ .query(
287+ "
288+ SELECT account_id, user_id, role
289+ FROM user_account_connections
290+ WHERE account_id = 1 AND user_id = 2 AND role = 'DEVELOPER'
291+ ",
292+ &[],
293+ )
294+ .unwrap()
295+ .len()
296+ == 1
297+ );
298+
299+ // Ensure NOT NULL constraint still applies to old schema
300+ let result = old_db
301+ .simple_query(
302+ "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",
303+ );
304+ assert!(result.is_err());
305+
306+ // Ensure updated user role in old schema updates connection in new schema
307+ old_db
308+ .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")
309+ .unwrap();
310+ assert!(
311+ new_db
312+ .query(
313+ "
314+ SELECT account_id, user_id, role
315+ FROM user_account_connections
316+ WHERE account_id = 1 AND user_id = 2 AND role = 'ADMIN'
317+ ",
318+ &[],
319+ )
320+ .unwrap()
321+ .len()
322+ == 1
323+ );
324+
325+ // Ensure updated connection in new schema updates old schema user
326+ new_db
327+ .simple_query(
328+ "UPDATE user_account_connections SET role = 'DEVELOPER' WHERE account_id = 1 AND user_id = 2",
329+ )
330+ .unwrap();
331+ assert!(
332+ old_db
333+ .query(
334+ "
335+ SELECT id
336+ FROM users
337+ WHERE id = 2 AND account_id = 1 AND account_role = 'developer'
338+ ",
339+ &[],
340+ )
341+ .unwrap()
342+ .len()
343+ == 1
344+ );
345+
346+ // Ensure insert of user with connection through new schema updates user in old schema
347+ new_db
348+ .simple_query(
349+ r#"
350+ BEGIN;
351+ INSERT INTO users (id) VALUES (3);
352+ INSERT INTO user_account_connections (user_id, account_id, role) VALUES (3, 1, 'DEVELOPER');
353+ COMMIT;
354+ "#,
355+ )
356+ .unwrap();
357+ new_db
358+ .simple_query(
359+ "",
360+ )
361+ .unwrap();
362+ assert!(
363+ old_db
364+ .query(
365+ "
366+ SELECT id
367+ FROM users
368+ WHERE id = 3 AND account_id = 1 AND account_role = 'developer'
369+ ",
370+ &[],
371+ )
372+ .unwrap()
373+ .len()
374+ == 1
375+ );
376+ });
377+
378+ test.run();
379+}
tests/create_enum.rsnew file+41
@@ -0,0 +1,41 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn create_enum() {
6+ let mut test = Test::new("Create enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum_and_table"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+
17+ [[actions]]
18+ type = "create_table"
19+ name = "updates"
20+ primary_key = ["id"]
21+
22+ [[actions.columns]]
23+ name = "id"
24+ type = "INTEGER"
25+
26+ [[actions.columns]]
27+ name = "status"
28+ type = "mood"
29+ "#,
30+ );
31+
32+ test.after_first(|db| {
33+ // Valid enum values should succeed
34+ db.simple_query(
35+ "INSERT INTO updates (id, status) VALUES (1, 'happy'), (2, 'ok'), (3, 'sad')",
36+ )
37+ .unwrap();
38+ });
39+
40+ test.run();
41+}
tests/create_table.rs+187−198
@@ -1,211 +1,200 @@
1−use reshape::{
2− migrations::{Column, CreateTable, ForeignKey, Migration},
3− Status,
4−};
5−
61mod common;
2+use common::Test;
73
84#[test]
95fn create_table() {
10− let (mut reshape, mut db, _) = common::setup();
11−
12− let create_table_migration =
13− Migration::new("create_users_table", None).with_action(CreateTable {
14− name: "users".to_string(),
15− primary_key: vec!["id".to_string()],
16− foreign_keys: vec![],
17− columns: vec![
18− Column {
19− name: "id".to_string(),
20− data_type: "INTEGER".to_string(),
21− nullable: true,
22− default: None,
23− generated: Some("ALWAYS AS IDENTITY".to_string()),
24− },
25− Column {
26− name: "name".to_string(),
27− data_type: "TEXT".to_string(),
28− nullable: true,
29− default: None,
30− generated: None,
31− },
32− Column {
33− name: "created_at".to_string(),
34− data_type: "TIMESTAMP".to_string(),
35− nullable: false,
36− default: Some("NOW()".to_string()),
37− generated: None,
38− },
39− ],
40− });
41−
42− reshape
43− .migrate(vec![create_table_migration.clone()])
44− .unwrap();
45− assert!(matches!(reshape.state.status, Status::Idle));
46− assert_eq!(
47− Some(&create_table_migration.name),
48− reshape.state.current_migration.as_ref()
6+ let mut test = Test::new("Create table");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ generated = "ALWAYS AS IDENTITY"
21+
22+ [[actions.columns]]
23+ name = "name"
24+ type = "TEXT"
25+
26+ [[actions.columns]]
27+ name = "created_at"
28+ type = "TIMESTAMP"
29+ nullable = false
30+ default = "NOW()"
31+ "#,
4932 );
5033
51− // Ensure table was created
52− let result = db
53− .query_opt(
54− "
55− SELECT table_name
56− FROM information_schema.tables
57− WHERE table_name = 'users' AND table_schema = 'public'",
58− &[],
59− )
60− .unwrap();
61− assert!(result.is_some());
62−
63− // Ensure table has the right columns
64− let result = db
65− .query(
66− "
67− SELECT column_name, column_default, is_nullable, data_type
68− FROM information_schema.columns
69− WHERE table_name = 'users' AND table_schema = 'public'
70− ORDER BY ordinal_position",
71− &[],
72− )
73− .unwrap();
74−
75− // id column
76− let id_row = &result[0];
77− assert_eq!("id", id_row.get::<_, String>("column_name"));
78− assert!(id_row.get::<_, Option<String>>("column_default").is_none());
79− assert_eq!("NO", id_row.get::<_, String>("is_nullable"));
80− assert_eq!("integer", id_row.get::<_, String>("data_type"));
81−
82− // name column
83− let name_row = &result[1];
84− assert_eq!("name", name_row.get::<_, String>("column_name"));
85− assert!(name_row
86− .get::<_, Option<String>>("column_default")
87− .is_none());
88− assert_eq!("YES", name_row.get::<_, String>("is_nullable"));
89− assert_eq!("text", name_row.get::<_, String>("data_type"));
90−
91− // created_at column
92− let created_at_column = &result[2];
93− assert_eq!(
94− "created_at",
95− created_at_column.get::<_, String>("column_name")
96− );
97− assert!(created_at_column
98− .get::<_, Option<String>>("column_default")
99− .is_some());
100− assert_eq!("NO", created_at_column.get::<_, String>("is_nullable"));
101− assert_eq!(
102− "timestamp without time zone",
103− created_at_column.get::<_, String>("data_type")
104− );
34+ test.after_first(|db| {
35+ // Ensure table was created
36+ let result = db
37+ .query_opt(
38+ "
39+ SELECT table_name
40+ FROM information_schema.tables
41+ WHERE table_name = 'users' AND table_schema = 'public'",
42+ &[],
43+ )
44+ .unwrap();
45+ assert!(result.is_some());
46+
47+ // Ensure table has the right columns
48+ let result = db
49+ .query(
50+ "
51+ SELECT column_name, column_default, is_nullable, data_type
52+ FROM information_schema.columns
53+ WHERE table_name = 'users' AND table_schema = 'public'
54+ ORDER BY ordinal_position",
55+ &[],
56+ )
57+ .unwrap();
58+
59+ // id column
60+ let id_row = &result[0];
61+ assert_eq!("id", id_row.get::<_, String>("column_name"));
62+ assert!(id_row.get::<_, Option<String>>("column_default").is_none());
63+ assert_eq!("NO", id_row.get::<_, String>("is_nullable"));
64+ assert_eq!("integer", id_row.get::<_, String>("data_type"));
65+
66+ // name column
67+ let name_row = &result[1];
68+ assert_eq!("name", name_row.get::<_, String>("column_name"));
69+ assert!(name_row
70+ .get::<_, Option<String>>("column_default")
71+ .is_none());
72+ assert_eq!("YES", name_row.get::<_, String>("is_nullable"));
73+ assert_eq!("text", name_row.get::<_, String>("data_type"));
74+
75+ // created_at column
76+ let created_at_column = &result[2];
77+ assert_eq!(
78+ "created_at",
79+ created_at_column.get::<_, String>("column_name")
80+ );
81+ assert!(created_at_column
82+ .get::<_, Option<String>>("column_default")
83+ .is_some());
84+ assert_eq!("NO", created_at_column.get::<_, String>("is_nullable"));
85+ assert_eq!(
86+ "timestamp without time zone",
87+ created_at_column.get::<_, String>("data_type")
88+ );
89+
90+ // Ensure the primary key has the right columns
91+ let primary_key_columns: Vec<String> = db
92+ .query(
93+ "
94+ SELECT a.attname AS column
95+ FROM pg_index i
96+ JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
97+ JOIN pg_class t ON t.oid = i.indrelid
98+ WHERE t.relname = 'users' AND i.indisprimary
99+ ",
100+ &[],
101+ )
102+ .unwrap()
103+ .iter()
104+ .map(|row| row.get("column"))
105+ .collect();
105106
106− // Ensure the primary key has the right columns
107− let primary_key_columns: Vec<String> = db
108− .query(
109− "
110− SELECT a.attname AS column
111− FROM pg_index i
112− JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
113− WHERE i.indrelid = 'users'::regclass AND i.indisprimary
114− ",
115− &[],
116− )
117− .unwrap()
118− .iter()
119− .map(|row| row.get("column"))
120− .collect();
121−
122− assert_eq!(vec!["id"], primary_key_columns);
107+ assert_eq!(vec!["id"], primary_key_columns);
108+ });
109+
110+ test.run();
123111}
124112
125113#[test]
126114fn create_table_with_foreign_keys() {
127− let (mut reshape, mut db, _) = common::setup();
128−
129− let create_table_migration =
130− Migration::new("create_users_table", None).with_action(CreateTable {
131− name: "users".to_string(),
132− primary_key: vec!["id".to_string()],
133− foreign_keys: vec![],
134− columns: vec![Column {
135− name: "id".to_string(),
136− data_type: "SERIAL".to_string(),
137− nullable: true, // Will be ignored by Postgres as the column is a SERIAL
138− default: None,
139− generated: None,
140− }],
141− });
142−
143− let create_second_table_migration =
144− Migration::new("create_items_table", None).with_action(CreateTable {
145− name: "items".to_string(),
146− primary_key: vec!["id".to_string()],
147− foreign_keys: vec![ForeignKey {
148− columns: vec!["user_id".to_string()],
149− referenced_table: "users".to_string(),
150− referenced_columns: vec!["id".to_string()],
151− }],
152− columns: vec![
153− Column {
154− name: "id".to_string(),
155− data_type: "SERIAL".to_string(),
156− nullable: true,
157− default: None,
158− generated: None,
159− },
160− Column {
161− name: "user_id".to_string(),
162− data_type: "INTEGER".to_string(),
163− nullable: false,
164− default: None,
165− generated: None,
166− },
167− ],
168− });
169−
170− reshape
171− .migrate(vec![
172− create_table_migration.clone(),
173− create_second_table_migration.clone(),
174− ])
175− .unwrap();
176−
177− let foreign_key_columns: Vec<(String, String, String)> = db
178− .query(
179− "
180− SELECT
181− kcu.column_name,
182− ccu.table_name AS foreign_table_name,
183− ccu.column_name AS foreign_column_name
184− FROM
185− information_schema.table_constraints AS tc
186− JOIN information_schema.key_column_usage AS kcu
187− ON tc.constraint_name = kcu.constraint_name
188− AND tc.table_schema = kcu.table_schema
189− JOIN information_schema.constraint_column_usage AS ccu
190− ON ccu.constraint_name = tc.constraint_name
191− AND ccu.table_schema = tc.table_schema
192− WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
193− ",
194− &[],
195− )
196− .unwrap()
197− .iter()
198− .map(|row| {
199− (
200− row.get("column_name"),
201− row.get("foreign_table_name"),
202− row.get("foreign_column_name"),
203− )
204− })
205− .collect();
206−
207− assert_eq!(
208− vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
209− foreign_key_columns
115+ let mut test = Test::new("Create table");
116+
117+ test.first_migration(
118+ r#"
119+ name = "create_users_table"
120+
121+ [[actions]]
122+ type = "create_table"
123+ name = "users"
124+ primary_key = ["id"]
125+
126+ [[actions.columns]]
127+ name = "id"
128+ type = "INTEGER"
129+ generated = "ALWAYS AS IDENTITY"
130+
131+ [[actions.columns]]
132+ name = "name"
133+ type = "TEXT"
134+
135+ [[actions.columns]]
136+ name = "created_at"
137+ type = "TIMESTAMP"
138+ nullable = false
139+ default = "NOW()"
140+
141+ [[actions]]
142+ type = "create_table"
143+ name = "items"
144+ primary_key = ["id"]
145+
146+ [[actions.columns]]
147+ name = "id"
148+ type = "INTEGER"
149+
150+ [[actions.columns]]
151+ name = "user_id"
152+ type = "INTEGER"
153+ nullable = false
154+
155+ [[actions.foreign_keys]]
156+ columns = ["user_id"]
157+ referenced_table = "users"
158+ referenced_columns = ["id"]
159+ "#,
210160 );
161+
162+ test.after_first(|db| {
163+ let foreign_key_columns: Vec<(String, String, String)> = db
164+ .query(
165+ "
166+ SELECT
167+ kcu.column_name,
168+ ccu.table_name AS foreign_table_name,
169+ ccu.column_name AS foreign_column_name
170+ FROM
171+ information_schema.table_constraints AS tc
172+ JOIN information_schema.key_column_usage AS kcu
173+ ON tc.constraint_name = kcu.constraint_name
174+ AND tc.table_schema = kcu.table_schema
175+ JOIN information_schema.constraint_column_usage AS ccu
176+ ON ccu.constraint_name = tc.constraint_name
177+ AND ccu.table_schema = tc.table_schema
178+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
179+ ",
180+ &[],
181+ )
182+ .unwrap()
183+ .iter()
184+ .map(|row| {
185+ (
186+ row.get("column_name"),
187+ row.get("foreign_table_name"),
188+ row.get("foreign_column_name"),
189+ )
190+ })
191+ .collect();
192+
193+ assert_eq!(
194+ vec![("user_id".to_string(), "users".to_string(), "id".to_string())],
195+ foreign_key_columns
196+ );
197+ });
198+
199+ test.run();
211200}
tests/custom.rsnew file+128
@@ -0,0 +1,128 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn custom_enable_extension() {
6+ let mut test = Test::new("Custom migration");
7+
8+ test.clear(|db| {
9+ db.simple_query(
10+ "
11+ DROP EXTENSION IF EXISTS bloom;
12+ DROP EXTENSION IF EXISTS btree_gin;
13+ DROP EXTENSION IF EXISTS btree_gist;
14+ ",
15+ )
16+ .unwrap();
17+ });
18+
19+ test.first_migration(
20+ r#"
21+ name = "empty_migration"
22+
23+ [[actions]]
24+ type = "custom"
25+ "#,
26+ );
27+
28+ test.second_migration(
29+ r#"
30+ name = "enable_extensions"
31+
32+ [[actions]]
33+ type = "custom"
34+
35+ start = """
36+ CREATE EXTENSION IF NOT EXISTS bloom;
37+ CREATE EXTENSION IF NOT EXISTS btree_gin;
38+ """
39+
40+ complete = "CREATE EXTENSION IF NOT EXISTS btree_gist"
41+
42+ abort = """
43+ DROP EXTENSION IF EXISTS bloom;
44+ DROP EXTENSION IF EXISTS btree_gin;
45+ """
46+ "#,
47+ );
48+
49+ test.intermediate(|db, _| {
50+ let bloom_activated = !db
51+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
52+ .unwrap()
53+ .is_empty();
54+ assert!(bloom_activated);
55+
56+ let btree_gin_activated = !db
57+ .query(
58+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
59+ &[],
60+ )
61+ .unwrap()
62+ .is_empty();
63+ assert!(btree_gin_activated);
64+
65+ let btree_gist_activated = !db
66+ .query(
67+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
68+ &[],
69+ )
70+ .unwrap()
71+ .is_empty();
72+ assert!(!btree_gist_activated);
73+ });
74+
75+ test.after_completion(|db| {
76+ let bloom_activated = !db
77+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
78+ .unwrap()
79+ .is_empty();
80+ assert!(bloom_activated);
81+
82+ let btree_gin_activated = !db
83+ .query(
84+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
85+ &[],
86+ )
87+ .unwrap()
88+ .is_empty();
89+ assert!(btree_gin_activated);
90+
91+ let btree_gist_activated = !db
92+ .query(
93+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
94+ &[],
95+ )
96+ .unwrap()
97+ .is_empty();
98+ assert!(btree_gist_activated);
99+ });
100+
101+ test.after_abort(|db| {
102+ let bloom_activated = !db
103+ .query("SELECT * FROM pg_extension WHERE extname = 'bloom'", &[])
104+ .unwrap()
105+ .is_empty();
106+ assert!(!bloom_activated);
107+
108+ let btree_gin_activated = !db
109+ .query(
110+ "SELECT * FROM pg_extension WHERE extname = 'btree_gin'",
111+ &[],
112+ )
113+ .unwrap()
114+ .is_empty();
115+ assert!(!btree_gin_activated);
116+
117+ let btree_gist_activated = !db
118+ .query(
119+ "SELECT * FROM pg_extension WHERE extname = 'btree_gist'",
120+ &[],
121+ )
122+ .unwrap()
123+ .is_empty();
124+ assert!(!btree_gist_activated);
125+ });
126+
127+ test.run();
128+}
tests/failure.rsnew file+51
@@ -0,0 +1,51 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn invalid_migration() {
6+ let mut test = Test::new("Invalid migration");
7+
8+ test.first_migration(
9+ r#"
10+ name = "invalid_migration"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
22+
23+ test.second_migration(
24+ r#"
25+ name = "add_invalid_column"
26+
27+ [[actions]]
28+ type = "add_column"
29+ table = "users"
30+
31+ up = "INVALID SQL"
32+
33+ [actions.column]
34+ name = "first"
35+ type = "TEXT"
36+ "#,
37+ );
38+
39+ // Insert a test user
40+ test.after_first(|db| {
41+ db.simple_query(
42+ "
43+ INSERT INTO users (id) VALUES (1)
44+ ",
45+ )
46+ .unwrap();
47+ });
48+
49+ test.expect_failure();
50+ test.run();
51+}
tests/remove_column.rs+210−75
@@ -1,82 +1,217 @@
1−use reshape::migrations::{Column, CreateTable, Migration, RemoveColumn};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn remove_column() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration =
10− Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![
15− Column {
16− name: "id".to_string(),
17− data_type: "INTEGER".to_string(),
18− nullable: true,
19− default: None,
20− generated: None,
21− },
22− Column {
23− name: "name".to_string(),
24− data_type: "TEXT".to_string(),
25− nullable: false,
26− default: None,
27− generated: None,
28− },
29− ],
30− });
31− let remove_column_migration =
32− Migration::new("remove_name_column", None).with_action(RemoveColumn {
33− table: "users".to_string(),
34− column: "name".to_string(),
35− down: Some("'TEST_DOWN_VALUE'".to_string()),
36− });
37−
38− let first_migrations = vec![create_table_migration.clone()];
39− let second_migrations = vec![
40− create_table_migration.clone(),
41− remove_column_migration.clone(),
42− ];
43−
44− // Run migrations
45− reshape.migrate(first_migrations.clone()).unwrap();
46− reshape.migrate(second_migrations.clone()).unwrap();
47−
48− // Update schemas of Postgres connections
49− let old_schema_query =
50− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
51− let new_schema_query =
52− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
53− old_db.simple_query(&old_schema_query).unwrap();
54− new_db.simple_query(&new_schema_query).unwrap();
55−
56− // Insert using old schema and ensure it can be retrieved through new schema
57− old_db
58− .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
59− .unwrap();
60− let results = new_db
61− .query("SELECT id FROM users WHERE id = 1", &[])
62− .unwrap();
63− assert_eq!(1, results.len());
64− assert_eq!(1, results[0].get::<_, i32>("id"));
65−
66− // Ensure the name column is not accesible through the new schema
67− assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
68−
69− // Insert using new schema and ensure the down function is correctly applied
70− new_db
71− .simple_query("INSERT INTO users(id) VALUES (2)")
72− .unwrap();
73− let result = old_db
74− .query_opt("SELECT name FROM users WHERE id = 2", &[])
75− .unwrap();
76− assert_eq!(
77− Some("TEST_DOWN_VALUE"),
78− result.as_ref().map(|row| row.get("name"))
6+ let mut test = Test::new("Remove column");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_user_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+ "#,
25+ );
26+
27+ test.second_migration(
28+ r#"
29+ name = "remove_name_column"
30+
31+ [[actions]]
32+ type = "remove_column"
33+ table = "users"
34+ column = "name"
35+ down = "'TEST_DOWN_VALUE'"
36+ "#,
7937 );
8038
81− reshape.complete_migration().unwrap();
39+ test.intermediate(|old_db, new_db| {
40+ // Insert using old schema and ensure it can be retrieved through new schema
41+ old_db
42+ .simple_query("INSERT INTO users(id, name) VALUES (1, 'John Doe')")
43+ .unwrap();
44+ let results = new_db
45+ .query("SELECT id FROM users WHERE id = 1", &[])
46+ .unwrap();
47+ assert_eq!(1, results.len());
48+ assert_eq!(1, results[0].get::<_, i32>("id"));
49+
50+ // Ensure the name column is not accesible through the new schema
51+ assert!(new_db.query("SELECT id, name FROM users", &[]).is_err());
52+
53+ // Insert using new schema and ensure the down function is correctly applied
54+ new_db
55+ .simple_query("INSERT INTO users(id) VALUES (2)")
56+ .unwrap();
57+ let result = old_db
58+ .query_opt("SELECT name FROM users WHERE id = 2", &[])
59+ .unwrap();
60+ assert_eq!(
61+ Some("TEST_DOWN_VALUE"),
62+ result.as_ref().map(|row| row.get("name"))
63+ );
64+ });
65+
66+ test.run();
67+}
68+
69+#[test]
70+fn remove_column_with_index() {
71+ let mut test = Test::new("Remove column");
72+
73+ test.first_migration(
74+ r#"
75+ name = "create_user_table"
76+
77+ [[actions]]
78+ type = "create_table"
79+ name = "users"
80+ primary_key = ["id"]
81+
82+ [[actions.columns]]
83+ name = "id"
84+ type = "INTEGER"
85+
86+ [[actions.columns]]
87+ name = "name"
88+ type = "TEXT"
89+
90+ [[actions]]
91+ type = "add_index"
92+ table = "users"
93+
94+ [actions.index]
95+ name = "name_idx"
96+ columns = ["name"]
97+ "#,
98+ );
99+
100+ test.second_migration(
101+ r#"
102+ name = "remove_name_column"
103+
104+ [[actions]]
105+ type = "remove_column"
106+ table = "users"
107+ column = "name"
108+ down = "'TEST_DOWN_VALUE'"
109+ "#,
110+ );
111+
112+ test.after_completion(|db| {
113+ // Ensure index has been removed after the migration is complete
114+ let count: i64 = db
115+ .query(
116+ "
117+ SELECT COUNT(*)
118+ FROM pg_catalog.pg_index
119+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
120+ WHERE pg_class.relname = 'name_idx'
121+ ",
122+ &[],
123+ )
124+ .unwrap()
125+ .first()
126+ .map(|row| row.get(0))
127+ .unwrap();
128+
129+ assert_eq!(0, count, "expected index to not exist");
130+ });
131+
132+ test.run();
133+}
134+
135+#[test]
136+fn remove_column_with_complex_down() {
137+ let mut test = Test::new("Remove column complex");
138+
139+ test.first_migration(
140+ r#"
141+ name = "create_tables"
142+
143+ [[actions]]
144+ type = "create_table"
145+ name = "users"
146+ primary_key = ["id"]
147+
148+ [[actions.columns]]
149+ name = "id"
150+ type = "INTEGER"
151+
152+ [[actions.columns]]
153+ name = "email"
154+ type = "TEXT"
155+
156+ [[actions]]
157+ type = "create_table"
158+ name = "profiles"
159+ primary_key = ["user_id"]
160+
161+ [[actions.columns]]
162+ name = "user_id"
163+ type = "INTEGER"
164+
165+ [[actions.columns]]
166+ name = "email"
167+ type = "TEXT"
168+ "#,
169+ );
170+
171+ test.second_migration(
172+ r#"
173+ name = "remove_users_email_column"
174+
175+ [[actions]]
176+ type = "remove_column"
177+ table = "users"
178+ column = "email"
179+
180+ [actions.down]
181+ table = "profiles"
182+ value = "profiles.email"
183+ where = "users.id = profiles.user_id"
184+ "#,
185+ );
186+
187+ test.after_first(|db| {
188+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
189+ .unwrap();
190+ db.simple_query("INSERT INTO profiles (user_id, email) VALUES (1, 'test@example.com')")
191+ .unwrap();
192+ });
193+
194+ test.intermediate(|old_db, new_db| {
195+ new_db
196+ .simple_query("UPDATE profiles SET email = 'test2@example.com' WHERE user_id = 1")
197+ .unwrap();
198+
199+ // Ensure new email was propagated to users table in old schema
200+ let email: String = old_db
201+ .query(
202+ "
203+ SELECT email
204+ FROM users
205+ WHERE id = 1
206+ ",
207+ &[],
208+ )
209+ .unwrap()
210+ .first()
211+ .map(|row| row.get("email"))
212+ .unwrap();
213+ assert_eq!("test2@example.com", email);
214+ });
215+
216+ test.run();
82217}
tests/remove_enum.rsnew file+69
@@ -0,0 +1,69 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_enum() {
6+ let mut test = Test::new("Remove enum");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_enum"
11+
12+ [[actions]]
13+ type = "create_enum"
14+ name = "mood"
15+ values = ["happy", "ok", "sad"]
16+ "#,
17+ );
18+
19+ test.second_migration(
20+ r#"
21+ name = "remove_enum"
22+
23+ [[actions]]
24+ type = "remove_enum"
25+ enum = "mood"
26+ "#,
27+ );
28+
29+ test.after_first(|db| {
30+ // Ensure enum was created
31+ let enum_exists = !db
32+ .query(
33+ "
34+ SELECT typname
35+ FROM pg_catalog.pg_type
36+ WHERE typcategory = 'E'
37+ AND typname = 'mood'
38+ ",
39+ &[],
40+ )
41+ .unwrap()
42+ .is_empty();
43+
44+ assert!(enum_exists, "expected mood enum to have been created");
45+ });
46+
47+ test.after_completion(|db| {
48+ // Ensure enum was removed after completion
49+ let enum_does_not_exist = db
50+ .query(
51+ "
52+ SELECT typname
53+ FROM pg_catalog.pg_type
54+ WHERE typcategory = 'E'
55+ AND typname = 'mood'
56+ ",
57+ &[],
58+ )
59+ .unwrap()
60+ .is_empty();
61+
62+ assert!(
63+ enum_does_not_exist,
64+ "expected mood enum to have been removed"
65+ );
66+ });
67+
68+ test.run();
69+}
tests/remove_foreign_key.rsnew file+124
@@ -0,0 +1,124 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_foreign_key() {
6+ let mut test = Test::new("Remove foreign key");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_tables"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions]]
22+ type = "create_table"
23+ name = "items"
24+ primary_key = ["id"]
25+
26+ [[actions.columns]]
27+ name = "id"
28+ type = "INTEGER"
29+
30+ [[actions.columns]]
31+ name = "user_id"
32+ type = "INTEGER"
33+
34+ [[actions.foreign_keys]]
35+ columns = ["user_id"]
36+ referenced_table = "users"
37+ referenced_columns = ["id"]
38+ "#,
39+ );
40+
41+ test.second_migration(
42+ r#"
43+ name = "remove_foreign_key"
44+
45+ [[actions]]
46+ type = "remove_foreign_key"
47+ table = "items"
48+ foreign_key = "items_user_id_fkey"
49+ "#,
50+ );
51+
52+ test.after_first(|db| {
53+ // Insert some test users
54+ db.simple_query("INSERT INTO users (id) VALUES (1), (2)")
55+ .unwrap();
56+ });
57+
58+ test.intermediate(|old_db, new_db| {
59+ // Ensure items can't be inserted if they don't reference valid users
60+ // The foreign key is only removed when the migration is completed so
61+ // it should still be enforced for the new and old schema.
62+ let result = old_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
63+ assert!(
64+ result.is_err(),
65+ "expected insert against old schema to fail"
66+ );
67+
68+ let result = new_db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
69+ assert!(
70+ result.is_err(),
71+ "expected insert against new schema to fail"
72+ );
73+ });
74+
75+ test.after_completion(|db| {
76+ // Ensure items can be inserted even if they don't reference valid users
77+ let result = db
78+ .simple_query("INSERT INTO items (id, user_id) VALUES (5, 3)")
79+ .map(|_| ());
80+ assert!(
81+ result.is_ok(),
82+ "expected insert to not fail, got {:?}",
83+ result
84+ );
85+
86+ // Ensure foreign key doesn't exist
87+ let foreign_keys = db
88+ .query(
89+ "
90+ SELECT tc.constraint_name
91+ FROM information_schema.table_constraints AS tc
92+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
93+ ",
94+ &[],
95+ )
96+ .unwrap();
97+ assert!(
98+ foreign_keys.is_empty(),
99+ "expected no foreign keys to exist on items table"
100+ );
101+ });
102+
103+ test.after_abort(|db| {
104+ // Ensure items can't be inserted if they don't reference valid users
105+ let result = db.simple_query("INSERT INTO items (id, user_id) VALUES (3, 3)");
106+ assert!(result.is_err(), "expected insert to fail");
107+
108+ // Ensure foreign key still exists
109+ let fk_exists = !db
110+ .query(
111+ "
112+ SELECT tc.constraint_name
113+ FROM information_schema.table_constraints AS tc
114+ WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_name='items';
115+ ",
116+ &[],
117+ )
118+ .unwrap()
119+ .is_empty();
120+ assert!(fk_exists);
121+ });
122+
123+ test.run()
124+}
tests/remove_index.rsnew file+86
@@ -0,0 +1,86 @@
1+mod common;
2+use common::Test;
3+
4+#[test]
5+fn remove_index() {
6+ let mut test = Test::new("Remove index");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+
21+ [[actions.columns]]
22+ name = "name"
23+ type = "TEXT"
24+
25+ [[actions]]
26+ type = "add_index"
27+ table = "users"
28+
29+ [actions.index]
30+ name = "name_idx"
31+ columns = ["name"]
32+ "#,
33+ );
34+
35+ test.second_migration(
36+ r#"
37+ name = "remove_name_index"
38+
39+ [[actions]]
40+ type = "remove_index"
41+ index = "name_idx"
42+ "#,
43+ );
44+
45+ test.intermediate(|db, _| {
46+ // Ensure index is still valid and ready during the migration
47+ let result: Vec<(bool, bool)> = db
48+ .query(
49+ "
50+ SELECT pg_index.indisready, pg_index.indisvalid
51+ FROM pg_catalog.pg_index
52+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
53+ WHERE pg_class.relname = 'name_idx'
54+ ",
55+ &[],
56+ )
57+ .unwrap()
58+ .iter()
59+ .map(|row| (row.get("indisready"), row.get("indisvalid")))
60+ .collect();
61+
62+ assert_eq!(vec![(true, true)], result);
63+ });
64+
65+ test.after_completion(|db| {
66+ // Ensure index has been removed after the migration is complete
67+ let count: i64 = db
68+ .query(
69+ "
70+ SELECT COUNT(*)
71+ FROM pg_catalog.pg_index
72+ JOIN pg_catalog.pg_class ON pg_index.indexrelid = pg_class.oid
73+ WHERE pg_class.relname = 'name_idx'
74+ ",
75+ &[],
76+ )
77+ .unwrap()
78+ .first()
79+ .map(|row| row.get(0))
80+ .unwrap();
81+
82+ assert_eq!(0, count, "expected index to not exist");
83+ });
84+
85+ test.run();
86+}
tests/remove_table.rs+39−49
@@ -1,54 +1,44 @@
1−use reshape::migrations::{Column, CreateTable, Migration, RemoveTable};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn remove_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration =
10− Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![Column {
15− name: "id".to_string(),
16− data_type: "INTEGER".to_string(),
17− nullable: false,
18− default: None,
19− generated: None,
20− }],
21− });
22− let remove_table_migration =
23− Migration::new("remove_users_table", None).with_action(RemoveTable {
24− table: "users".to_string(),
25− });
26−
27− let first_migrations = vec![create_table_migration.clone()];
28− let second_migrations = vec![
29− create_table_migration.clone(),
30− remove_table_migration.clone(),
31− ];
32−
33− // Run migrations
34− reshape.migrate(first_migrations.clone()).unwrap();
35− reshape.migrate(second_migrations.clone()).unwrap();
36−
37− // Update schemas of Postgres connections
38− let old_schema_query =
39− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
40− let new_schema_query =
41− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
42− old_db.simple_query(&old_schema_query).unwrap();
43− new_db.simple_query(&new_schema_query).unwrap();
44−
45− // Make sure inserts work against the old schema
46− old_db
47− .simple_query("INSERT INTO users(id) VALUES (1)")
48− .unwrap();
49−
50− // Ensure the table is not accessible through the new schema
51− assert!(new_db.query("SELECT id FROM users", &[]).is_err());
52−
53− reshape.complete_migration().unwrap();
6+ let mut test = Test::new("Remove table");
7+
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
16+
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
22+
23+ test.second_migration(
24+ r#"
25+ name = "remove_users_table"
26+
27+ [[actions]]
28+ type = "remove_table"
29+ table = "users"
30+ "#,
31+ );
32+
33+ test.intermediate(|old_db, new_db| {
34+ // Make sure inserts work against the old schema
35+ old_db
36+ .simple_query("INSERT INTO users(id) VALUES (1)")
37+ .unwrap();
38+
39+ // Ensure the table is not accessible through the new schema
40+ assert!(new_db.query("SELECT id FROM users", &[]).is_err());
41+ });
42+
43+ test.run();
5444}
tests/rename_table.rs+56−66
@@ -1,80 +1,70 @@
1−use reshape::migrations::{Column, CreateTable, Migration, RenameTable};
2−
31mod common;
2+use common::Test;
43
54#[test]
65fn rename_table() {
7− let (mut reshape, mut old_db, mut new_db) = common::setup();
8−
9− let create_table_migration =
10− Migration::new("create_users_table", None).with_action(CreateTable {
11− name: "users".to_string(),
12− primary_key: vec!["id".to_string()],
13− foreign_keys: vec![],
14− columns: vec![Column {
15− name: "id".to_string(),
16− data_type: "INTEGER".to_string(),
17− nullable: false,
18− default: None,
19− generated: None,
20− }],
21− });
22− let rename_table_migration = Migration::new("rename_users_table_to_customers", None)
23− .with_action(RenameTable {
24− table: "users".to_string(),
25− new_name: "customers".to_string(),
26− });
6+ let mut test = Test::new("Rename table");
277
28− let first_migrations = vec![create_table_migration.clone()];
29− let second_migrations = vec![
30− create_table_migration.clone(),
31− rename_table_migration.clone(),
32− ];
8+ test.first_migration(
9+ r#"
10+ name = "create_users_table"
11+
12+ [[actions]]
13+ type = "create_table"
14+ name = "users"
15+ primary_key = ["id"]
3316
34− // Run migrations
35− reshape.migrate(first_migrations.clone()).unwrap();
36− reshape.migrate(second_migrations.clone()).unwrap();
17+ [[actions.columns]]
18+ name = "id"
19+ type = "INTEGER"
20+ "#,
21+ );
3722
38− // Update schemas of Postgres connections
39− let old_schema_query =
40− reshape::schema_query_for_migration(&first_migrations.last().unwrap().name);
41− let new_schema_query =
42− reshape::schema_query_for_migration(&second_migrations.last().unwrap().name);
43− old_db.simple_query(&old_schema_query).unwrap();
44− new_db.simple_query(&new_schema_query).unwrap();
23+ test.second_migration(
24+ r#"
25+ name = "rename_users_table_to_customers"
4526
46− // Make sure inserts work using both the old and new name
47− old_db
48− .simple_query("INSERT INTO users(id) VALUES (1)")
49− .unwrap();
50− new_db
51− .simple_query("INSERT INTO customers(id) VALUES (2)")
52− .unwrap();
27+ [[actions]]
28+ type = "rename_table"
29+ table = "users"
30+ new_name = "customers"
31+ "#,
32+ );
5333
54− // Ensure the table can be queried using both the old and new name
55− let expected: Vec<i32> = vec![1, 2];
56− assert_eq!(
57− expected,
34+ test.intermediate(|old_db, new_db| {
35+ // Make sure inserts work using both the old and new name
5836 old_db
59− .query("SELECT id FROM users ORDER BY id", &[])
60− .unwrap()
61− .iter()
62− .map(|row| row.get::<_, i32>("id"))
63− .collect::<Vec<i32>>()
64− );
65− assert_eq!(
66− expected,
37+ .simple_query("INSERT INTO users(id) VALUES (1)")
38+ .unwrap();
6739 new_db
68− .query("SELECT id FROM customers ORDER BY id", &[])
69− .unwrap()
70− .iter()
71− .map(|row| row.get::<_, i32>("id"))
72− .collect::<Vec<i32>>()
73− );
40+ .simple_query("INSERT INTO customers(id) VALUES (2)")
41+ .unwrap();
42+
43+ // Ensure the table can be queried using both the old and new name
44+ let expected: Vec<i32> = vec![1, 2];
45+ assert_eq!(
46+ expected,
47+ old_db
48+ .query("SELECT id FROM users ORDER BY id", &[])
49+ .unwrap()
50+ .iter()
51+ .map(|row| row.get::<_, i32>("id"))
52+ .collect::<Vec<i32>>()
53+ );
54+ assert_eq!(
55+ expected,
56+ new_db
57+ .query("SELECT id FROM customers ORDER BY id", &[])
58+ .unwrap()
59+ .iter()
60+ .map(|row| row.get::<_, i32>("id"))
61+ .collect::<Vec<i32>>()
62+ );
7463
75− // Ensure the table can't be queried using the wrong name for the schema
76− assert!(old_db.simple_query("SELECT id FROM customers").is_err());
77− assert!(new_db.simple_query("SELECT id FROM users").is_err());
64+ // Ensure the table can't be queried using the wrong name for the schema
65+ assert!(old_db.simple_query("SELECT id FROM customers").is_err());
66+ assert!(new_db.simple_query("SELECT id FROM users").is_err());
67+ });
7868
79− reshape.complete_migration().unwrap();
69+ test.run();
8070}