git.iko.soy/reshape
on 88fca45 help sign in
a771c64
swap
0 ahead, 42 behind
20 files changed+232−1593
since splitsnapshots
unifiedsplit
.github/workflows/lint.yaml+3−3 .github/workflows/publish-crate.yaml−20 .github/workflows/publish-docker.yaml−25 Cargo.toml+3−4 Dockerfile+1−1 README.md+97−246 src/lib.rs+12−17 src/main.rs+2−12 src/migrations/add_column.rs+58−214 src/migrations/add_foreign_key.rs+1−1 src/migrations/alter_column.rs+1−1 src/migrations/common.rs+9−14 src/migrations/create_table.rs+9−122 src/migrations/mod.rs+1−7 src/migrations/remove_column.rs+34−339 src/schema.rs+1−1 tests/add_column.rs−99 tests/complex.rs−379 tests/remove_column.rs−86 tests/remove_table.rs−2
.github/workflows/lint.yaml+3−3
@@ -15,7 +15,7 @@ jobs:
1515 runs-on: ubuntu-latest
1616 steps:
1717 - name: Checkout
18− uses: actions/checkout@v4
18+ uses: actions/checkout@v2
1919 - name: Select Rust toolchain with Clippy
2020 uses: actions-rs/toolchain@v1
2121 with:
@@ -23,6 +23,6 @@ jobs:
2323 components: clippy
2424 override: true
2525 - name: Use cache for Rust dependencies
26− uses: Swatinem/rust-cache@v2
26+ uses: Swatinem/rust-cache@v1
2727 - name: Lint using Clippy
28− run: cargo clippy
28+ run: cargo clippy
.github/workflows/publish-crate.yamldeleted−20
@@ -1,20 +0,0 @@
1−name: Publish crate
2−
3−on: workflow_dispatch
4−
5−jobs:
6− publish-crate:
7− runs-on: ubuntu-latest
8− steps:
9− - name: Checkout
10− uses: actions/checkout@v2
11− - name: Select Rust toolchain
12− uses: actions-rs/toolchain@v1
13− with:
14− toolchain: stable
15− - name: Publish to crates.io
16− uses: actions-rs/cargo@v1
17− with:
18− command: publish
19− env:
20− CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/publish-docker.yamldeleted−25
@@ -1,25 +0,0 @@
1−name: Publish Docker image
2−
3−on:
4− workflow_dispatch:
5− inputs:
6− version:
7− description: "Version (without 'v' prefix)"
8− required: true
9− type: string
10−
11−jobs:
12− publish-docker:
13− runs-on: ubuntu-latest
14− steps:
15− - name: Checkout
16− uses: actions/checkout@v2
17− - name: Log in to Docker
18− env:
19− DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
20− run: |
21− docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD
22− - name: Build Docker image
23− run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
24− - name: Push Docker image
25− run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
Cargo.toml+3−4
@@ -1,6 +1,6 @@
11[package]
22name = "reshape"
3−version = "0.7.0"
3+version = "0.5.1"
44description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
55homepage = "https://github.com/fabianlindfors/reshape"
66documentation = "https://github.com/fabianlindfors/reshape"
@@ -9,7 +9,7 @@ license = "MIT"
99keywords = ["postgres", "migrations"]
1010edition = "2021"
1111authors = ["Fabian Lindfors"]
12−rust-version = "1.70"
12+rust-version = "1.58"
1313
1414[dependencies]
1515postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
@@ -22,5 +22,4 @@ toml = "0.5"
2222version = "3.0.0"
2323colored = "2"
2424rand = "0.8"
25−dotenv = "0.15.0"
26−lexical-sort = "0.3.1"
25+dotenv = "0.15.0"
Dockerfile+1−1
@@ -1,4 +1,4 @@
1−FROM rust:1.70.0 AS builder
1+FROM rust:1.58.0 AS builder
22WORKDIR /usr/src/reshape
33COPY . .
44RUN cargo build --release
README.md+97−246
@@ -2,56 +2,56 @@
22
33[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)
44
5−> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!
6−
75Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).
86
97Designed for Postgres 12 and later.
108
9+*Note: Reshape is **experimental** and should not be used in production. It can (and probably will) break your application.*
10+
1111- [How it works](#how-it-works)
1212- [Getting started](#getting-started)
13− - [Installation](#installation)
14− - [Creating your first migration](#creating-your-first-migration)
15− - [Preparing your application](#preparing-your-application)
16− - [Running your migration](#running-your-migration)
17− - [Using during development](#using-during-development)
13+ - [Installation](#installation)
14+ - [Creating your first migration](#creating-your-first-migration)
15+ - [Preparing your application](#preparing-your-application)
16+ - [Running your migration](#running-your-migration)
17+ - [Using during development](#using-during-development)
1818- [Writing migrations](#writing-migrations)
19− - [Basics](#basics)
20− - [Tables](#tables)
21− - [Create table](#create-table)
22− - [Rename table](#rename-table)
23− - [Remove table](#remove-table)
24− - [Add foreign key](#add-foreign-key)
25− - [Remove foreign key](#remove-foreign-key)
26− - [Columns](#columns)
27− - [Add column](#add-column)
28− - [Alter column](#alter-column)
29− - [Remove column](#remove-column)
30− - [Indices](#indices)
31− - [Add index](#add-index)
32− - [Remove index](#remove-index)
33− - [Enums](#enums)
34− - [Create enum](#create-enum)
35− - [Remove enum](#remove-enum)
36− - [Custom](#custom)
37− - [Complex changes across tables](#complex-changes-across-tables)
19+ - [Basics](#basics)
20+ - [Tables](#tables)
21+ - [Create table](#create-table)
22+ - [Rename table](#rename-table)
23+ - [Remove table](#remove-table)
24+ - [Add foreign key](#add-foreign-key)
25+ - [Remove foreign key](#remove-foreign-key)
26+ - [Columns](#columns)
27+ - [Add column](#add-column)
28+ - [Alter column](#alter-column)
29+ - [Remove column](#remove-column)
30+ - [Indices](#indices)
31+ - [Add index](#add-index)
32+ - [Remove index](#remove-index)
33+ - [Enums](#enums)
34+ - [Create enum](#create-enum)
35+ - [Remove enum](#remove-enum)
36+ - [Custom](#custom)
3837- [Commands and options](#commands-and-options)
39− - [`reshape migration start`](#reshape-migration-start)
40− - [`reshape migration complete`](#reshape-migration-complete)
41− - [`reshape migration abort`](#reshape-migration-abort)
42− - [`reshape schema-query`](#reshape-schema-query)
43− - [Connection options](#connection-options)
38+ - [`reshape migrate`](#reshape-migrate)
39+ - [`reshape complete`](#reshape-complete)
40+ - [`reshape abort`](#reshape-abort)
41+ - [`reshape generate-schema-query`](#reshape-generate-schema-query)
42+ - [Connection options](#connection-options)
4443- [License](#license)
4544
45+
4646## How it works
4747
4848Reshape works by creating views that encapsulate the underlying tables, which your application will interact with. During a migration, Reshape will automatically create a new set of views and set up triggers to translate inserts and updates between the old and new schema. This means that every deployment is a three-phase process:
4949
50−1. **Start migration** (`reshape migration start`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
50+1. **Start migration** (`reshape migrate`): Sets up views and triggers to ensure both the new and old schema are usable at the same time.
51512. **Roll out application**: Your application can be gradually rolled out without downtime. The existing deployment will continue using the old schema whilst the new deployment uses the new schema.
52−3. **Complete migration** (`reshape migration complete`): Removes the old schema and any intermediate data and triggers.
52+3. **Complete migration** (`reshape complete`): Removes the old schema and any intermediate data and triggers.
5353
54−If the application deployment fails, you should run `reshape migration abort` which will roll back any changes made by `reshape migration start` without losing data.
54+If the application deployment fails, you should run `reshape abort` which will roll back any changes made by `reshape migrate` without losing data.
5555
5656## Getting started
5757
@@ -71,10 +71,10 @@ cargo install reshape
7171
7272#### Docker
7373
74−Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
74+Reshape is available as a Docker image on [Docker Hub](https://hub.docker.com/repository/docker/fabianlindfors/reshape).
7575
7676```shell
77−docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migration start
77+docker run -v $(pwd):/usr/share/app fabianlindfors/reshape reshape migrate
7878```
7979
8080### Creating your first migration
@@ -103,14 +103,9 @@ This is the equivalent of running `CREATE TABLE users (id INTEGER GENERATED ALWA
103103
104104### Preparing your application
105105
106−Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. The simplest way to do this is to use one of the helper libraries:
106+Reshape relies on your application using a specific schema. When establishing the connection to Postgres in your application, you need to run a query to select the most recent schema. This query can be generated using: `reshape generate-schema-query`.
107107
108−- [Rust](https://github.com/fabianlindfors/reshape-helper)
109−- [Ruby (and Rails)](https://github.com/fabianlindfors/reshape-ruby)
110−- [Python (and Django)](https://github.com/fabianlindfors/reshape-python)
111−- [Go](https://github.com/leourbina/reshape-helper)
112−
113−If your application is not using one of the languages with an available helper library, you can instead generate the query with the command: `reshape schema-query`. To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape schema-query)`. Then in your application:
108+To pass it along to your application, you can for example use an environment variable in your run script: `RESHAPE_SCHEMA_QUERY=$(reshape generate-schema-query)`. Then in your application:
114109
115110```python
116111# Example for Python
@@ -118,21 +113,23 @@ reshape_schema_query = os.getenv("RESHAPE_SCHEMA_QUERY")
118113db.execute(reshape_schema_query)
119114```
120115
116+If your application is written in Rust, you might prefer the [Rust helper library](https://github.com/fabianlindfors/reshape-helper) which embeds the query directly in your application with a macro.
117+
121118### Running your migration
122119
123120To create your new `users` table, run:
124121
125122```bash
126−reshape migration start --complete
123+reshape migrate --complete
127124```
128125
129−We use the `--complete` flag to automatically complete the migration. During a production deployment, you should first run `reshape migration start` followed by `reshape migration complete` once your application has been fully rolled out.
126+We use the `--complete` flag to automatically complete the migration. During a production deployment, you should first run `reshape migrate` followed by `reshape complete` once your application has been fully rolled out.
130127
131128If nothing else is specified, Reshape will try to connect to a Postgres database running on `localhost` using `postgres` as both username and password. See [Connection options](#connection-options) for details on how to change the connection settings.
132129
133130### Using during development
134131
135−When adding new migrations during development, we recommend running `reshape migration start` but skipping `reshape migration complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape migration abort` followed by `reshape migration start`.
132+When adding new migrations during development, we recommend running `reshape migrate` but skipping `reshape complete`. This way, the new migrations can be iterated on by updating the migration file and running `reshape abort` followed by `reshape migrate`.
136133
137134## Writing migrations
138135
@@ -167,9 +164,9 @@ Every action has a `type`. The supported types are detailed below.
167164
168165#### Create table
169166
170−The `create_table` action will create a new table with the specified columns, indices and constraints. You can optionally provide an `up` option to backfill values from an existing table.
167+The `create_table` action will create a new table with the specified columns, indices and constraints.
171168
172−_Example: create a `customers` table with a few columns and a primary key_
169+*Example: create a `customers` table with a few columns and a primary key*
173170
174171```toml
175172[[actions]]
@@ -187,13 +184,13 @@ primary_key = ["id"]
187184 type = "TEXT"
188185
189186 # Columns default to nullable
190− nullable = false
187+ nullable = false
191188
192189 # default can be any valid SQL value, in this case a string literal
193− default = "'PLACEHOLDER'"
190+ default = "'PLACEHOLDER'"
194191```
195192
196−_Example: create `users` and `items` tables with a foreign key between them_
193+*Example: create `users` and `items` tables with a foreign key between them*
197194
198195```toml
199196[[actions]]
@@ -226,34 +223,11 @@ primary_key = ["id"]
226223 referenced_columns = ["id"]
227224```
228225
229−_Example: create `profiles` table based on existing `users` table_
230−
231−```toml
232−[[actions]]
233−type = "create_table"
234−name = "profiles"
235−primary_key = ["user_id"]
236−
237− [[actions.columns]]
238− name = "user_id"
239− type = "INTEGER"
240−
241− [[actions.columns]]
242− name = "user_email"
243− type = "TEXT"
244−
245− # Backfill from `users` table and copy `users.email` to `user_email` column
246− # This will perform an upsert based on the primary key to avoid duplicate rows
247− [actions.up]
248− table = "users"
249− values = { user_id = "id", user_email = "email" }
250−```
251−
252226#### Rename table
253227
254228The `rename_table` action will change the name of an existing table.
255229
256−_Example: change name of `users` table to `customers`_
230+*Example: change name of `users` table to `customers`*
257231
258232```toml
259233[[actions]]
@@ -266,7 +240,7 @@ new_name = "customers"
266240
267241The `remove_table` action will remove an existing table.
268242
269−_Example: remove `users` table_
243+*Example: remove `users` table*
270244
271245```toml
272246[[actions]]
@@ -278,7 +252,7 @@ table = "users"
278252
279253The `add_foreign_key` action will add a foreign key between two existing tables. The migration will fail if the existing column values aren't valid references.
280254
281−_Example: create foreign key from `items` to `users` table_
255+*Example: create foreign key from `items` to `users` table*
282256
283257```toml
284258[[actions]]
@@ -295,7 +269,7 @@ table = "items"
295269
296270The `remove_foreign_key` action will remove an existing foreign key. The foreign key will only be removed once the migration is completed, which means that your new application must continue to adhere to the foreign key constraint.
297271
298−_Example: remove foreign key `items_user_id_fkey` from `users` table_
272+*Example: remove foreign key `items_user_id_fkey` from `users` table*
299273
300274```toml
301275[[actions]]
@@ -308,9 +282,9 @@ foreign_key = "items_user_id_fkey"
308282
309283#### Add column
310284
311−The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column. `up` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)).
285+The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column.
312286
313−_Example: add a new column `reference` to table `products`_
287+*Example: add a new column `reference` to table `products`*
314288
315289```toml
316290[[actions]]
@@ -324,7 +298,7 @@ table = "products"
324298 default = "10"
325299```
326300
327−_Example: replace an existing `name` column with two new columns, `first_name` and `last_name`_
301+*Example: replace an existing `name` column with two new columns, `first_name` and `last_name`*
328302
329303```toml
330304[[actions]]
@@ -360,7 +334,7 @@ column = "name"
360334down = "first_name || ' ' || last_name"
361335```
362336
363−_Example: extract nested value from unstructured JSON `data` column to new `name` column_
337+*Example: extract nested value from unstructured JSON `data` column to new `name` column*
364338
365339```toml
366340[[actions]]
@@ -375,25 +349,6 @@ up = "data['path']['to']['value'] #>> '{}'"
375349 type = "TEXT"
376350```
377351
378−_Example: duplicate `email` column from `users` to `profiles` table_
379−
380−```toml
381−# `profiles` has `user_id` column which maps to `users.id`
382−[[actions]]
383−type = "add_column"
384−table = "profiles"
385−
386− [actions.column]
387− name = "email"
388− type = "TEXT"
389− nullable = false
390−
391− # When `users` is updated in the old schema, we write the email value to `profiles`
392− [actions.up]
393− table = "users"
394− value = "email"
395− where = "user_id = id"
396−```
397352
398353#### Alter column
399354
@@ -401,7 +356,7 @@ The `alter_column` action enables many different changes to an existing column,
401356
402357When performing more complex changes than a rename, `up` and `down` should be provided. These should be SQL expressions which determine how to transform between the new and old version of the column. Inside those expressions, you can reference the current column value by the column name.
403358
404−_Example: rename `last_name` column on `users` table to `family_name`_
359+*Example: rename `last_name` column on `users` table to `family_name`*
405360
406361```toml
407362[[actions]]
@@ -413,7 +368,7 @@ column = "last_name"
413368 name = "family_name"
414369```
415370
416−_Example: change the type of `reference` column from `INTEGER` to `TEXT`_
371+*Example: change the type of `reference` column from `INTEGER` to `TEXT`*
417372
418373```toml
419374[[actions]]
@@ -428,7 +383,7 @@ down = "CAST(reference AS INTEGER)" # Converts from text value to integer
428383 type = "TEXT" # Previous type was 'INTEGER'
429384```
430385
431−_Example: increment all values of an `index` column by one_
386+*Example: increment all values of an `index` column by one*
432387
433388```toml
434389[[actions]]
@@ -443,7 +398,7 @@ down = "index - 1" # Decrement to revert for old schema
443398 name = "index"
444399```
445400
446−_Example: make `name` column not nullable_
401+*Example: make `name` column not nullable*
447402
448403```toml
449404[[actions]]
@@ -458,7 +413,7 @@ up = "COALESCE(name, 'N/A')"
458413 nullable = false
459414```
460415
461−_Example: change default value of `created_at` column to current time_
416+*Example: change default value of `created_at` column to current time*
462417
463418```toml
464419[[actions]]
@@ -472,11 +427,11 @@ column = "created_at"
472427
473428#### Remove column
474429
475−The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. `down` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)) . The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
430+The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.
476431
477432Any indices that cover the column will be removed.
478433
479−_Example: remove column `name` from table `users`_
434+*Example: remove column `name` from table `users`*
480435
481436```toml
482437[[actions]]
@@ -488,34 +443,18 @@ column = "name"
488443down = "'N/A'"
489444```
490445
491−_Example: remove `email` column from `users` table and use column from `profiles` table instead_
492−
493−```toml
494−[[actions]]
495−type = "remove_column"
496−table = "users"
497−column = "email"
498−
499− # Our application will use the `profiles.email` column instead
500− # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed
501− [actions.down]
502− table = "profiles"
503− value = "profiles.email"
504− where = "users.id = profiles.user_id"
505−```
506−
507446### Indices
508447
509448#### Add index
510449
511450The `add_index` action will add a new index to an existing table.
512451
513−_Example: create a `users` table with a unique index on the `name` column_
452+*Example: create a `users` table with a unique index on the `name` column*
514453
515454```toml
516455[[actions]]
517456type = "create_table"
518−name = "users"
457+table = "users"
519458primary_key = "id"
520459
521460 [[actions.columns]]
@@ -539,7 +478,7 @@ table = "users"
539478 unique = true
540479```
541480
542−_Example: add GIN index to `data` column on `products` table_
481+*Example: add GIN index to `data` column on `products` table*
543482
544483```toml
545484[[actions]]
@@ -558,7 +497,7 @@ table = "products"
558497
559498The `remove_index` action will remove an existing index. The index won't actually be removed until the migration is completed.
560499
561−_Example: remove the `name_idx` index_
500+*Example: remove the `name_idx` index*
562501
563502```toml
564503[[actions]]
@@ -572,7 +511,7 @@ index = "name_idx"
572511
573512The `create_enum` action will create a new [enum type](https://www.postgresql.org/docs/current/datatype-enum.html) with the specified values.
574513
575−_Example: add a new `mood` enum type with three possible values_
514+*Example: add a new `mood` enum type with three possible values*
576515
577516```toml
578517[[actions]]
@@ -585,7 +524,7 @@ values = ["happy", "ok", "sad"]
585524
586525The `remove_enum` action will remove an existing [enum type](https://www.postgresql.org/docs/current/datatype-enum.html). Make sure all usages of the enum has been removed before running the migration. The enum will only be removed once the migration is completed.
587526
588−_Example: remove the `mood` enum type_
527+*Example: remove the `mood` enum type*
589528
590529```toml
591530[[actions]]
@@ -599,11 +538,11 @@ The `custom` action lets you create a migration which runs custom SQL. It should
599538
600539There are three optional settings available which all accept SQL queries. All queries need to be idempotent, for example by using `IF NOT EXISTS` wherever available.
601540
602−- `start`: run when a migration is started using `reshape migration start`
603−- `complete`: run when a migration is completed using `reshape migration complete`
604−- `abort`: run when a migration is aborted using `reshape migration abort`
541+- `start`: run when a migration is started using `reshape migrate`
542+- `complete`: run when a migration is completed using `reshape complete`
543+- `abort`: run when a migration is aborted using `reshape abort`
605544
606−_Example: enable PostGIS and pg_stat_statements extensions_
545+*Example: enable PostGIS and pg_stat_statements extensions*
607546
608547```toml
609548[[actions]]
@@ -615,140 +554,52 @@ start = """
615554"""
616555
617556abort = """
618− DROP EXTENSION IF EXISTS postgis;
619− DROP EXTENSION IF EXISTS pg_stat_statements;
557+ DROP EXTENSION IF NOT EXISTS postgis;
558+ DROP EXTENSION IF NOT EXISTS pg_stat_statements;
620559"""
621560```
622561
623−### Complex changes across tables
624−
625−The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.
626−
627−_Example: move `email` column from `users` to `profiles` table_
628−
629−```toml
630−[[actions]]
631−type = "add_column"
632−table = "profiles"
633−
634− [actions.column]
635− name = "email"
636− type = "TEXT"
637− nullable = false
638−
639− # When `users` is updated in the old schema, we write the email value to `profiles`
640− [actions.up]
641− table = "users"
642− value = "users.email"
643− where = "profiles.user_id = users.id"
644−
645−[[actions]]
646−type = "remove_column"
647−table = "users"
648−column = "email"
649−
650− # When `profiles` is changed in the new schema, we write the email address back to the removed column
651− [actions.down]
652− table = "profiles"
653− value = "profiles.email"
654− where = "users.id = profiles.user_id"
655−```
656−
657−_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_
658−
659−```toml
660−# Add `user_account_connections` as a junction table
661−[[actions]]
662−type = "create_table"
663−name = "user_account_connections"
664−primary_key = ["account_id", "user_id"]
665−
666− [[actions.columns]]
667− name = "account_id"
668− type = "INTEGER"
669−
670− [[actions.columns]]
671− name = "user_id"
672− type = "INTEGER"
673−
674− # `role` is currently stored directly on the `users` table but is part of the relationship
675− [[actions.columns]]
676− name = "role"
677− type = "TEXT"
678− nullable = false
679−
680− # Backfill the new table from `users` and uppercase the `role`
681− [actions.up]
682− table = "users"
683− values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
684− where = "user_account_connections.user_id = users.id"
685−
686−[[actions]]
687−type = "remove_column"
688−table = "users"
689−column = "account_id"
690−
691− # When `user_account_connections` is updated, we write the `account_id` back to `users`
692− [actions.down]
693− table = "user_account_connections"
694− value = "user_account_connections.account_id"
695− where = "users.id = user_account_connections.user_id"
696−
697−[[actions]]
698−type = "remove_column"
699−table = "users"
700−column = "account_role"
701−
702− # When `user_account_connections` is updated, we write the lowercase role back to `users`
703− [actions.down]
704− table = "user_account_connections"
705− value = "LOWER(user_account_connections.role)"
706− where = "users.id = user_account_connections.user_id"
707−```
708−
709562## Commands and options
710563
711−### `reshape migration start`
564+### `reshape migrate`
712565
713−Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape migration complete`.
566+Starts a new migration, applying all migrations under `migrations/` that haven't yet been applied. After the command has completed, both the old and new schema will be usable at the same time. When you have rolled out the new version of your application which uses the new schema, you should run `reshape complete`.
714567
715568#### Options
716569
717−_See also [Connection options](#connection-options)_
570+*See also [Connection options](#connection-options)*
718571
719−| Option | Default | Description |
720−| ------------------ | ------------- | --------------------------------------------------------------------------------------------------------------- |
721−| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
722−| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
572+| Option | Default | Description |
573+| ------ | ------- | ----------- |
574+| `--complete`, `-c` | `false` | Automatically complete migration after applying it. |
575+| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
723576
724−### `reshape migration complete`
577+### `reshape complete`
725578
726−Completes migrations previously started with `reshape migration complete`.
579+Completes migrations previously started with `reshape complete`.
727580
728581#### Options
729582
730583See [Connection options](#connection-options)
731584
732−### `reshape migration abort`
585+### `reshape abort`
733586
734−Aborts any migrations which haven't yet been completed.
587+Aborts any migrations which haven't yet been completed.
735588
736589#### Options
737590
738591See [Connection options](#connection-options)
739592
740−### `reshape schema-query`
593+### `reshape generate-schema-query`
741594
742595Generates the SQL query you need to run in your application before using the database. This command does not require a database connection. Instead it will generate the query based on the latest migration in the `migrations/` directory (or the directories specified by `--dirs`).
743596
744−If your application is written in Rust, we recommend using the [Rust helper library](https://github.com/fabianlindfors/reshape-helper/) instead.
745−
746597The query should look something like `SET search_path TO migration_1_initial_migration`.
747598
748599#### Options
749600
750−| Option | Default | Description |
751−| -------- | ------------- | --------------------------------------------------------------------------------------------------------------- |
601+| Option | Default | Description |
602+| ------ | ------- | ----------- |
752603| `--dirs` | `migrations/` | Directories to search for migration files. Multiple directories can be specified using `--dirs dir1 dir2 dir3`. |
753604
754605### Connection options
@@ -757,14 +608,14 @@ The options below can be used with all commands that communicate with Postgres.
757608
758609All options can also be set using environment variables instead of flags. If a `.env` file exists, then variables will be automatically loaded from there.
759610
760−| Option | Default | Environment variable | Description |
761−| ------------ | ----------- | -------------------- | ------------------------------------------- |
762−| `--url` | | `DB_URL` | URL to your Postgres database |
763−| `--host` | `localhost` | `DB_HOST` | Hostname to use when connecting to Postgres |
764−| `--port` | `5432` | `DB_PORT` | Port which Postgres is listening on |
765−| `--database` | `postgres` | `DB_NAME` | Database name |
766−| `--username` | `postgres` | `DB_USERNAME` | Postgres username |
767−| `--password` | `postgres` | `DB_PASSWORD` | Postgres password |
611+| Option | Default | Environment variable | Description |
612+| ------ | ------- | -------------------- | ----------- |
613+| `--url` | | `DB_URL` | URL to your Postgres database |
614+| `--host` | `localhost` | `DB_HOST` | Hostname to use when connecting to Postgres |
615+| `--port` | `5432` | `DB_PORT` | Port which Postgres is listening on |
616+| `--database` | `postgres` | `DB_NAME` | Database name |
617+| `--username` | `postgres` | `DB_USERNAME` | Postgres username |
618+| `--password` | `postgres` | `DB_PASSWORD` | Postgres password |
768619
769620## License
770621
src/lib.rs+12−17
@@ -147,15 +147,13 @@ fn migrate(
147147) -> anyhow::Result<()> {
148148 // Make sure no migration is in progress
149149 if let State::InProgress { .. } = &state {
150− println!(
151− "Migration already in progress, please complete using 'reshape migration complete'"
152− );
150+ println!("Migration already in progress, please complete using 'reshape complete'");
153151 return Ok(());
154152 }
155153
156154 if let State::Completing { .. } = &state {
157155 println!(
158− "Migration already in progress and has started completion, please finish using 'reshape migration complete'"
156+ "Migration already in progress and has started completion, please finish using 'reshape complete'"
159157 );
160158 return Ok(());
161159 }
@@ -178,7 +176,7 @@ fn migrate(
178176 {
179177 if existing_migrations != &remaining_migrations {
180178 return Err(anyhow!(
181− "a previous migration seems to have failed without cleaning up. Please run `reshape migration abort` and then run migrate again."
179+ "a previous migration seems to have failed without cleaning up. Please run `reshape abort` and then run migrate again."
182180 ));
183181 }
184182 }
@@ -200,7 +198,7 @@ fn migrate(
200198 let mut last_action_index = usize::MAX;
201199 let mut result: anyhow::Result<()> = Ok(());
202200
203− 'outer: for (migration_index, migration) in remaining_migrations.iter().enumerate() {
201+ for (migration_index, migration) in remaining_migrations.iter().enumerate() {
204202 println!("Migrating '{}':", migration.name);
205203 last_migration_index = migration_index;
206204
@@ -210,8 +208,7 @@ fn migrate(
210208 let description = action.describe();
211209 print!(" + {} ", description);
212210
213− let ctx =
214− MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
211+ let ctx = MigrationContext::new(migration_index, action_index);
215212 result = action
216213 .run(&ctx, db, &new_schema)
217214 .with_context(|| format!("failed to {}", description));
@@ -221,7 +218,7 @@ fn migrate(
221218 println!("{}", "done".green());
222219 } else {
223220 println!("{}", "failed".red());
224− break 'outer;
221+ break;
225222 }
226223 }
227224
@@ -235,7 +232,7 @@ fn migrate(
235232 // Set to the Aborting state. This is to ensure that the failed
236233 // migration is fully aborted and nothing is left dangling.
237234 // If the abort is interrupted for any reason, the user can try again
238− // by running `reshape migration abort`.
235+ // by running `reshape abort`.
239236 state.aborting(
240237 remaining_migrations.clone(),
241238 last_migration_index + 1,
@@ -262,7 +259,7 @@ fn migrate(
262259 schema_query_for_migration(&target_migration)
263260 );
264261 println!(
265− " - Run 'reshape migration complete' once your application has been updated and the previous schema is no longer in use"
262+ " - Run 'reshape complete' once your application has been updated and the previous schema is no longer in use"
266263 );
267264 Ok(())
268265}
@@ -284,7 +281,7 @@ fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
284281 current_action_index
285282 } => (migrations, current_migration_index, current_action_index),
286283 State::Aborting { .. } => {
287− return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape migration abort`."))
284+ return Err(anyhow!("migration been aborted and can't be completed. Please finish using `reshape abort`."))
288285 }
289286 State::Applying { .. } => {
290287 return Err(anyhow!("a previous migration unexpectedly failed. Please run `reshape migrate` to try applying the migration again."))
@@ -321,8 +318,7 @@ fn complete(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
321318 let description = action.describe();
322319 print!(" + {} ", description);
323320
324− let ctx =
325− MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
321+ let ctx = MigrationContext::new(migration_index, action_index);
326322
327323 // Update state to indicate that this action has been completed.
328324 // We won't save this new state until after the action has completed.
@@ -413,7 +409,7 @@ fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
413409 last_action_index,
414410 } => (migrations, last_migration_index, last_action_index),
415411 State::Completing { .. } => {
416− return Err(anyhow!("migration completion has already been started. Please run `reshape migration complete` again to finish it."));
412+ return Err(anyhow!("Migration completion has already been started. Please run `reshape complete` again to finish it."));
417413 }
418414 State::Idle => {
419415 println!("No migration is in progress");
@@ -447,8 +443,7 @@ fn abort(db: &mut DbConn, state: &mut State) -> anyhow::Result<()> {
447443 continue;
448444 }
449445
450− let ctx =
451− MigrationContext::new(migration_index, action_index, state::current_migration(db)?);
446+ let ctx = MigrationContext::new(migration_index, action_index);
452447 action
453448 .abort(&ctx, db)
454449 .with_context(|| format!("failed to abort migration {}", migration.name))
src/main.rs+2−12
@@ -4,7 +4,6 @@ use std::{
44 path::Path,
55};
66
7−use anyhow::Context;
87use clap::{Args, Parser};
98use reshape::{
109 migrations::{Action, Migration},
@@ -196,13 +195,7 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
196195 }
197196
198197 // Sort all files by their file names (without extension)
199− // The files are sorted naturally, e.g. "1_test_migration" < "10_test_migration"
200− file_paths.sort_unstable_by(|path1, path2| {
201− let file1 = path1.as_path().file_stem().unwrap().to_str().unwrap();
202− let file2 = path2.as_path().file_stem().unwrap().to_str().unwrap();
203−
204− lexical_sort::natural_cmp(file1, file2)
205− });
198+ file_paths.sort_unstable_by_key(|path| path.as_path().file_stem().unwrap().to_os_string());
206199
207200 file_paths
208201 .iter()
@@ -218,10 +211,7 @@ fn find_migrations(opts: &FindMigrationsOptions) -> anyhow::Result<Vec<Migration
218211 .map(|result| {
219212 result.and_then(|(path, data)| {
220213 let extension = path.extension().and_then(|ext| ext.to_str()).unwrap();
221− let file_migration =
222− decode_migration_file(&data, extension).with_context(|| {
223− format!("failed to parse migration file {}", path.display())
224− })?;
214+ let file_migration = decode_migration_file(&data, extension)?;
225215
226216 let file_name = path.file_stem().and_then(|name| name.to_str()).unwrap();
227217 Ok(Migration {
src/migrations/add_column.rs+58−214
@@ -3,25 +3,14 @@ use crate::{
33 db::{Conn, Transaction},
44 schema::Schema,
55};
6−use anyhow::{bail, Context};
6+use anyhow::Context;
77use serde::{Deserialize, Serialize};
88
99#[derive(Serialize, Deserialize, Debug)]
1010pub struct AddColumn {
1111 pub table: String,
12+ pub up: Option<String>,
1213 pub column: Column,
13− pub up: Option<Transformation>,
14−}
15−
16−#[derive(Serialize, Deserialize, Debug)]
17−#[serde(untagged)]
18−pub enum Transformation {
19− Simple(String),
20− Update {
21− table: String,
22− value: String,
23− r#where: String,
24− },
2514}
2615
2716impl AddColumn {
@@ -43,15 +32,6 @@ impl AddColumn {
4332 )
4433 }
4534
46− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
47− format!(
48− "{}_add_column_{}_{}_rev",
49− ctx.prefix(),
50− self.table,
51− self.column.name
52− )
53− }
54−
5535 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
5636 format!(
5737 "{}_add_column_not_null_{}_{}",
@@ -81,7 +61,7 @@ impl Action for AddColumn {
8161 let temp_column_name = self.temp_column_name(ctx);
8262
8363 let mut definition_parts = vec![
84− format!("\"{}\"", temp_column_name.to_string()),
64+ temp_column_name.to_string(),
8565 self.column.data_type.to_string(),
8666 ];
8767
@@ -106,189 +86,53 @@ impl Action for AddColumn {
10686 );
10787 db.run(&query).context("failed to add column")?;
10888
109− let declarations: Vec<String> = table
110− .columns
111− .iter()
112− .map(|column| {
113− format!(
114− "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",
115− table = table.real_name,
116− alias = column.name,
117− real_name = column.real_name,
118− )
119− })
120− .collect();
121−
12289 if let Some(up) = &self.up {
123− if let Transformation::Simple(up) = up {
124− // Add triggers to fill in values as they are inserted/updated
125− let query = format!(
126− r#"
127− CREATE OR REPLACE FUNCTION {trigger_name}()
128− RETURNS TRIGGER AS $$
129− BEGIN
130− IF NOT reshape.is_new_schema() THEN
131− DECLARE
132− {declarations}
133− BEGIN
134− NEW."{temp_column_name}" = {up};
135− END;
136− END IF;
137− RETURN NEW;
138− END
139− $$ language 'plpgsql';
140−
141− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
142− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
143− "#,
144− temp_column_name = temp_column_name,
145− trigger_name = self.trigger_name(ctx),
146− up = up,
147− table = self.table,
148− declarations = declarations.join("\n"),
149− );
150− db.run(&query).context("failed to create up trigger")?;
151−
152− // Backfill values in batches
153− common::batch_touch_rows(db, &table.real_name, Some(&temp_column_name))
154− .context("failed to batch update existing rows")?;
155− }
156−
157− if let Transformation::Update {
158− table: from_table,
159− value,
160− r#where,
161− } = up
162− {
163− let existing_schema_name = match &ctx.existing_schema_name {
164− Some(name) => name,
165− None => bail!("can't use update without previous migration"),
166− };
167−
168− let from_table = schema.get_table(db, &from_table)?;
169−
170− let from_table_assignments: Vec<String> = from_table
171− .columns
172− .iter()
173− .map(|column| {
174− format!(
175− "{table}.{alias} = NEW.{real_name};",
176− table = from_table.name,
177− alias = column.name,
178− real_name = column.real_name,
179− )
180− })
181− .collect();
182−
183− // Add triggers to fill in values as they are inserted/updated
184− let query = format!(
185− r#"
186− CREATE OR REPLACE FUNCTION {trigger_name}()
187− RETURNS TRIGGER AS $$
188− #variable_conflict use_variable
189− BEGIN
190− IF NOT reshape.is_new_schema() THEN
191− DECLARE
192− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
193− BEGIN
194− {assignments}
195−
196− -- Don't trigger reverse trigger when making this update
197− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
198−
199− UPDATE public."{changed_table_real}"
200− SET "{temp_column_name}" = {value}
201− WHERE {where};
202−
203− perform set_config('reshape.disable_triggers', '', TRUE);
204− END;
205− END IF;
206− RETURN NEW;
207− END
208− $$ language 'plpgsql';
209−
210− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
211− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
212− "#,
213− assignments = from_table_assignments.join("\n"),
214− changed_table_real = table.real_name,
215− from_table = from_table.name,
216− from_table_real = from_table.real_name,
217− trigger_name = self.trigger_name(ctx),
218− // declarations = from_table_declarations.join("\n"),
219− temp_column_name = temp_column_name,
220− );
221− db.run(&query).context("failed to create up trigger")?;
222−
223− let from_table_columns = from_table
224− .columns
225− .iter()
226− .map(|column| format!("{} as {}", column.real_name, column.name))
227− .collect::<Vec<String>>()
228− .join(", ");
229−
230− let changed_table_assignments: Vec<String> = table
231− .columns
232− .iter()
233− .map(|column| {
234− format!(
235− "{table}.{alias} := NEW.{real_name};",
236− table = table.name,
237− alias = column.name,
238− real_name = column.real_name,
239− )
240− })
241− .collect();
242−
243− // Add triggers to fill in values as they are inserted/updated
244− let query = format!(
245− r#"
246− CREATE OR REPLACE FUNCTION {trigger_name}()
247− RETURNS TRIGGER AS $$
248− #variable_conflict use_variable
249− BEGIN
250− IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
251− DECLARE
252− {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;
253− __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;
254− BEGIN
255− {changed_table_assignments}
256−
257− SELECT {from_table_columns}
258− INTO __temp_row
259− FROM migration_{existing_schema_name}.{from_table} {from_table}
260− WHERE {where};
261−
262− DECLARE
263− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;
264− BEGIN
265− {from_table} = __temp_row;
266− NEW.{temp_column_name} = {value};
267− END;
268− END;
269− END IF;
270− RETURN NEW;
271− END
272− $$ language 'plpgsql';
273−
274− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
275− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
276− "#,
277− changed_table_assignments = changed_table_assignments.join("\n"),
278− changed_table_real = table.real_name,
279− changed_table = table.name,
280− from_table = from_table.name,
281− trigger_name = self.reverse_trigger_name(ctx),
282− temp_column_name = temp_column_name,
283− // declarations = declarations.join("\n"),
284− );
285− db.run(&query)
286− .context("failed to create reverse up trigger")?;
90+ let declarations: Vec<String> = table
91+ .columns
92+ .iter()
93+ .map(|column| {
94+ format!(
95+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
96+ table = table.real_name,
97+ alias = column.name,
98+ real_name = column.real_name,
99+ )
100+ })
101+ .collect();
102+
103+ // Add triggers to fill in values as they are inserted/updated
104+ let query = format!(
105+ r#"
106+ CREATE OR REPLACE FUNCTION {trigger_name}()
107+ RETURNS TRIGGER AS $$
108+ BEGIN
109+ IF NOT reshape.is_new_schema() THEN
110+ DECLARE
111+ {declarations}
112+ BEGIN
113+ NEW.{temp_column_name} = {up};
114+ END;
115+ END IF;
116+ RETURN NEW;
117+ END
118+ $$ language 'plpgsql';
119+
120+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
121+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
122+ "#,
123+ temp_column_name = temp_column_name,
124+ trigger_name = self.trigger_name(ctx),
125+ up = up,
126+ table = self.table,
127+ declarations = declarations.join("\n"),
128+ );
129+ db.run(&query).context("failed to create up trigger")?;
130+ }
287131
288− // Backfill values in batches by touching the from table
289− common::batch_touch_rows(db, &from_table.real_name, None)
290− .context("failed to batch update existing rows")?;
291− }
132+ // Backfill values in batches
133+ if self.up.is_some() {
134+ common::batch_touch_rows(db, &table.real_name, &temp_column_name)
135+ .context("failed to batch update existing rows")?;
292136 }
293137
294138 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.
@@ -298,10 +142,10 @@ impl Action for AddColumn {
298142 if !self.column.nullable {
299143 let query = format!(
300144 r#"
301− ALTER TABLE "{table}"
302− ADD CONSTRAINT "{constraint_name}"
303− CHECK ("{column}" IS NOT NULL) NOT VALID
304− "#,
145+ ALTER TABLE "{table}"
146+ ADD CONSTRAINT "{constraint_name}"
147+ CHECK ("{column}" IS NOT NULL) NOT VALID
148+ "#,
305149 table = self.table,
306150 constraint_name = self.not_null_constraint_name(ctx),
307151 column = temp_column_name,
@@ -323,11 +167,11 @@ impl Action for AddColumn {
323167 // Remove triggers and procedures
324168 let query = format!(
325169 r#"
326− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
327− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
170+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
171+ DROP FUNCTION IF EXISTS "{trigger_name}";
328172 "#,
173+ table = self.table,
329174 trigger_name = self.trigger_name(ctx),
330− reverse_trigger_name = self.reverse_trigger_name(ctx),
331175 );
332176 transaction
333177 .run(&query)
@@ -418,11 +262,11 @@ impl Action for AddColumn {
418262 // Remove triggers and procedures
419263 let query = format!(
420264 r#"
421− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
422− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
265+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
266+ DROP FUNCTION IF EXISTS "{trigger_name}";
423267 "#,
268+ table = self.table,
424269 trigger_name = self.trigger_name(ctx),
425− reverse_trigger_name = self.reverse_trigger_name(ctx),
426270 );
427271 db.run(&query).context("failed to drop up trigger")?;
428272
src/migrations/add_foreign_key.rs+1−1
@@ -48,7 +48,7 @@ impl Action for AddForeignKey {
4848 ALTER TABLE "{table}"
4949 ADD CONSTRAINT {constraint_name}
5050 FOREIGN KEY ({columns})
51− REFERENCES "{referenced_table}" ({referenced_columns})
51+ REFERENCES {referenced_table} ({referenced_columns})
5252 NOT VALID
5353 "#,
5454 table = table.real_name,
src/migrations/alter_column.rs+1−1
@@ -150,7 +150,7 @@ impl Action for AlterColumn {
150150 .context("failed to create up and down triggers")?;
151151
152152 // Backfill values in batches by touching the previous column
153− common::batch_touch_rows(db, &table.real_name, Some(&column.real_name))
153+ common::batch_touch_rows(db, &table.real_name, &column.real_name)
154154 .context("failed to batch update existing rows")?;
155155
156156 // Duplicate any indices to the temporary column
src/migrations/common.rs+9−14
@@ -69,11 +69,7 @@ impl ToSql for PostgresRawValue {
6969 postgres::types::to_sql_checked!();
7070}
7171
72−pub fn batch_touch_rows(
73− db: &mut dyn Conn,
74− table: &str,
75− column: Option<&str>,
76−) -> anyhow::Result<()> {
72+pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow::Result<()> {
7773 const BATCH_SIZE: u16 = 1000;
7874
7975 let mut cursor: Option<PostgresRawValue> = None;
@@ -82,13 +78,6 @@ pub fn batch_touch_rows(
8278 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();
8379
8480 let primary_key = get_primary_key_columns_for_table(db, table)?;
85−
86− // If no column to touch is passed, we default to the first primary key column (just to make some "update")
87− let touched_column = match column {
88− Some(column) => column,
89− None => primary_key.first().unwrap(),
90− };
91−
9281 let primary_key_columns = primary_key.join(", ");
9382
9483 let primary_key_where = primary_key
@@ -131,8 +120,8 @@ pub fn batch_touch_rows(
131120 ORDER BY {primary_key_columns}
132121 LIMIT {batch_size}
133122 ), update AS (
134− UPDATE public."{table}" "{table}"
135− SET "{touched_column}" = "{table}"."{touched_column}"
123+ UPDATE public."{table}"
124+ SET "{column}" = "{column}"
136125 FROM rows
137126 WHERE {primary_key_where}
138127 RETURNING {returning_columns}
@@ -141,7 +130,13 @@ pub fn batch_touch_rows(
141130 FROM update
142131 LIMIT 1
143132 "#,
133+ table = table,
134+ primary_key_columns = primary_key_columns,
135+ cursor_where = cursor_where,
144136 batch_size = BATCH_SIZE,
137+ column = column,
138+ primary_key_where = primary_key_where,
139+ returning_columns = returning_columns,
145140 );
146141 let last_value = db
147142 .query_with_params(&query, &params)?
src/migrations/create_table.rs+9−122
@@ -1,9 +1,6 @@
1−use std::collections::HashMap;
2−
31use super::{common::ForeignKey, Action, Column, MigrationContext};
42use crate::{
53 db::{Conn, Transaction},
6− migrations::common,
74 schema::Schema,
85};
96use anyhow::Context;
@@ -17,21 +14,6 @@ pub struct CreateTable {
1714
1815 #[serde(default)]
1916 pub foreign_keys: Vec<ForeignKey>,
20−
21− pub up: Option<Transformation>,
22−}
23−
24−#[derive(Serialize, Deserialize, Debug)]
25−pub struct Transformation {
26− table: String,
27− values: HashMap<String, String>,
28− upsert_constraint: Option<String>,
29−}
30−
31−impl CreateTable {
32− fn trigger_name(&self, ctx: &MigrationContext) -> String {
33− format!("{}_create_table_{}", ctx.prefix(), self.name)
34− }
3517}
3618
3719#[typetag::serde(name = "create_table")]
@@ -42,7 +24,7 @@ impl Action for CreateTable {
4224
4325 fn run(
4426 &self,
45− ctx: &MigrationContext,
27+ _ctx: &MigrationContext,
4628 db: &mut dyn Conn,
4729 schema: &Schema,
4830 ) -> anyhow::Result<()> {
@@ -103,7 +85,7 @@ impl Action for CreateTable {
10385 ));
10486 }
10587
106− let query = &format!(
88+ db.run(&format!(
10789 r#"
10890 CREATE TABLE "{name}" (
10991 {definition}
@@ -111,121 +93,26 @@ impl Action for CreateTable {
11193 "#,
11294 name = self.name,
11395 definition = definition_rows.join(",\n"),
114− );
115− db.run(query).context("failed to create table")?;
116−
117− if let Some(Transformation {
118− table: from_table,
119− values,
120− upsert_constraint,
121− }) = &self.up
122− {
123− let from_table = schema.get_table(db, &from_table)?;
124−
125− let declarations: Vec<String> = from_table
126− .columns
127− .iter()
128− .map(|column| {
129− format!(
130− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
131− table = from_table.real_name,
132− alias = column.name,
133− real_name = column.real_name,
134− )
135− })
136− .collect();
137−
138− let (insert_columns, insert_values): (Vec<&str>, Vec<&str>) = values
139− .iter()
140− .map(|(k, v)| -> (&str, &str) { (k, v) }) // Force &String to &str
141− .unzip();
142−
143− let update_set: Vec<String> = values
144− .iter()
145− .map(|(field, value)| format!("\"{field}\" = {value}"))
146− .collect();
147−
148− // Constraint to check for conflicts. Defaults to the primary key constraint.
149− let conflict_constraint_name = match upsert_constraint {
150− Some(custom_constraint) => custom_constraint.clone(),
151− _ => format!("{table}_pkey", table = self.name),
152− };
153−
154− // Add triggers to fill in values as they are inserted/updated
155− let query = format!(
156− r#"
157− CREATE OR REPLACE FUNCTION {trigger_name}()
158− RETURNS TRIGGER AS $$
159− #variable_conflict use_variable
160− BEGIN
161− IF NOT reshape.is_new_schema() THEN
162− DECLARE
163− {declarations}
164− BEGIN
165− INSERT INTO public."{changed_table_real}" ({columns})
166− VALUES ({values})
167− ON CONFLICT ON CONSTRAINT "{conflict_constraint_name}"
168− DO UPDATE SET
169− {updates};
170− END;
171− END IF;
172− RETURN NEW;
173− END
174− $$ language 'plpgsql';
175−
176− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
177− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
178− "#,
179− changed_table_real = self.name,
180− from_table_real = from_table.real_name,
181− trigger_name = self.trigger_name(ctx),
182− declarations = declarations.join("\n"),
183− columns = insert_columns.join(", "),
184− values = insert_values.join(", "),
185− updates = update_set.join(",\n"),
186− );
187− db.run(&query).context("failed to create up trigger")?;
188−
189− // Backfill values in batches by touching the from table
190− common::batch_touch_rows(db, &from_table.real_name, None)
191− .context("failed to batch update existing rows")?;
192− }
193−
96+ ))
97+ .context("failed to create table")?;
19498 Ok(())
19599 }
196100
197101 fn complete<'a>(
198102 &self,
199− ctx: &MigrationContext,
200− db: &'a mut dyn Conn,
103+ _ctx: &MigrationContext,
104+ _db: &'a mut dyn Conn,
201105 ) -> anyhow::Result<Option<Transaction<'a>>> {
202− // Remove triggers and procedures
203− let query = format!(
204− r#"
205− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
206− "#,
207− trigger_name = self.trigger_name(ctx),
208− );
209− db.run(&query).context("failed to drop up trigger")?;
210−
106+ // Do nothing
211107 Ok(None)
212108 }
213109
214110 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}
215111
216− fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
217− // Remove triggers and procedures
218− let query = format!(
219− r#"
220− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
221− "#,
222− trigger_name = self.trigger_name(ctx),
223− );
224− db.run(&query).context("failed to drop up trigger")?;
225−
112+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
226113 db.run(&format!(
227114 r#"
228− DROP TABLE IF EXISTS "{name}"
115+ DROP TABLE IF EXISTS {name}
229116 "#,
230117 name = self.name,
231118 ))
src/migrations/mod.rs+1−7
@@ -88,19 +88,13 @@ impl Clone for Migration {
8888pub struct MigrationContext {
8989 migration_index: usize,
9090 action_index: usize,
91− existing_schema_name: Option<String>,
9291}
9392
9493impl MigrationContext {
95− pub fn new(
96− migration_index: usize,
97− action_index: usize,
98− existing_schema_name: Option<String>,
99− ) -> Self {
94+ pub fn new(migration_index: usize, action_index: usize) -> Self {
10095 MigrationContext {
10196 migration_index,
10297 action_index,
103− existing_schema_name,
10498 }
10599 }
106100
src/migrations/remove_column.rs+34−339
@@ -3,25 +3,14 @@ use crate::{
33 db::{Conn, Transaction},
44 schema::Schema,
55};
6−use anyhow::{anyhow, bail, Context};
6+use anyhow::Context;
77use serde::{Deserialize, Serialize};
88
99#[derive(Serialize, Deserialize, Debug)]
1010pub struct RemoveColumn {
1111 pub table: String,
1212 pub column: String,
13− pub down: Option<Transformation>,
14−}
15−
16−#[derive(Serialize, Deserialize, Debug)]
17−#[serde(untagged)]
18−pub enum Transformation {
19− Simple(String),
20− Update {
21− table: String,
22− value: String,
23− r#where: String,
24− },
13+ pub down: Option<String>,
2514}
2615
2716impl RemoveColumn {
@@ -33,33 +22,6 @@ impl RemoveColumn {
3322 self.column
3423 )
3524 }
36−
37− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {
38− format!(
39− "{}_remove_column_{}_{}_rev",
40− ctx.prefix(),
41− self.table,
42− self.column
43− )
44− }
45−
46− fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
47− format!(
48− "{}_remove_column_{}_{}_nn",
49− ctx.prefix(),
50− self.table,
51− self.column
52− )
53− }
54−
55− fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
56− format!(
57− "{}_add_column_not_null_{}_{}",
58− ctx.prefix(),
59− self.table,
60− self.column
61− )
62− }
6325}
6426
6527#[typetag::serde(name = "remove_column")]
@@ -77,13 +39,10 @@ impl Action for RemoveColumn {
7739 db: &mut dyn Conn,
7840 schema: &Schema,
7941 ) -> anyhow::Result<()> {
80− let table = schema.get_table(db, &self.table)?;
81− let column = table
82− .get_column(&self.column)
83− .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;
84−
8542 // Add down trigger
8643 if let Some(down) = &self.down {
44+ let table = schema.get_table(db, &self.table)?;
45+
8746 let declarations: Vec<String> = table
8847 .columns
8948 .iter()
@@ -97,226 +56,32 @@ impl Action for RemoveColumn {
9756 })
9857 .collect();
9958
100− if let Transformation::Simple(down) = down {
101− let query = format!(
102− r#"
103− CREATE OR REPLACE FUNCTION {trigger_name}()
104− RETURNS TRIGGER AS $$
105− BEGIN
106− IF reshape.is_new_schema() THEN
107− DECLARE
108− {declarations}
109− BEGIN
110− NEW.{column_name} = {down};
111− END;
112− END IF;
113− RETURN NEW;
114− END
115− $$ language 'plpgsql';
116−
117− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
118− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
119− "#,
120− column_name = self.column,
121− trigger_name = self.trigger_name(ctx),
122− down = down,
123− table = self.table,
124− declarations = declarations.join("\n"),
125− );
126− db.run(&query).context("failed to create down trigger")?;
127− }
128−
129− if let Transformation::Update {
130− table: from_table,
131− value,
132− r#where,
133− } = down
134− {
135− let existing_schema_name = match &ctx.existing_schema_name {
136− Some(name) => name,
137− None => bail!("can't use update without previous migration"),
138− };
139−
140− let from_table = schema.get_table(db, &from_table)?;
141−
142− let maybe_null_check = if !column.nullable {
143− // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.
144− // We will add a null check to the down function on the new schema below as well to cover both cases.
145− // As we are using a complex down function, we must remove the NOT NULL check for the new schema.
146− // NOT NULL is not checked at the end of a transaction, but immediately upon update.
147− let query = format!(
148− r#"
149− CREATE OR REPLACE FUNCTION {trigger_name}()
150− RETURNS TRIGGER AS $$
59+ let query = format!(
60+ r#"
61+ CREATE OR REPLACE FUNCTION {trigger_name}()
62+ RETURNS TRIGGER AS $$
63+ BEGIN
64+ IF reshape.is_new_schema() THEN
65+ DECLARE
66+ {declarations}
15167 BEGIN
152− IF NOT reshape.is_new_schema() THEN
153− IF NEW.{column} IS NULL THEN
154− RAISE EXCEPTION '{column} can not be null';
155− END IF;
156− END IF;
157− RETURN NEW;
158− END
159− $$ language 'plpgsql';
160−
161− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
162−
163− CREATE CONSTRAINT TRIGGER "{trigger_name}"
164− AFTER INSERT OR UPDATE
165− ON "{table}"
166− FOR EACH ROW
167− EXECUTE PROCEDURE {trigger_name}();
168− "#,
169− table = self.table,
170− trigger_name = self.not_null_constraint_trigger_name(ctx),
171− column = self.column,
172− );
173− db.run(&query)
174− .context("failed to create null constraint trigger")?;
175−
176− db.run(&format!(
177− r#"
178− ALTER TABLE {table}
179− ALTER COLUMN {column}
180− DROP NOT NULL
181− "#,
182− table = self.table,
183− column = self.column
184− ))
185− .context("failed to remove column not null constraint")?;
186−
187− format!(
188− r#"
189− IF {value} IS NULL THEN
190− RAISE EXCEPTION '{column_name} can not be null';
191− END IF;
192− "#,
193− column_name = self.column,
194− )
195− } else {
196− "".to_string()
197− };
198−
199− let into_variables = from_table
200− .columns
201− .iter()
202− .map(|column| {
203− format!(
204− "NEW.{real_name} AS {alias}",
205− alias = column.name,
206− real_name = column.real_name,
207− )
208− })
209− .collect::<Vec<String>>()
210− .join(", ");
211−
212− let query = format!(
213− r#"
214− CREATE OR REPLACE FUNCTION {trigger_name}()
215− RETURNS TRIGGER AS $$
216− #variable_conflict use_variable
217− BEGIN
218− IF reshape.is_new_schema() THEN
219− DECLARE
220− {from_table} record;
221− BEGIN
222− SELECT {into_variables}
223− INTO {from_table};
224−
225− {maybe_null_check}
226−
227− -- Don't trigger reverse trigger when making this update
228− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);
229−
230− UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"
231− SET "{column_name}" = {value}
232− WHERE {where};
233−
234− perform set_config('reshape.disable_triggers', '', TRUE);
235− END;
236− END IF;
237− RETURN NEW;
238− END
239− $$ language 'plpgsql';
240−
241− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";
242− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
243− "#,
244− changed_table = self.table,
245− from_table = from_table.name,
246− from_table_real = from_table.real_name,
247− column_name = self.column,
248− trigger_name = self.trigger_name(ctx),
249− );
250− db.run(&query).context("failed to create down trigger")?;
251−
252− let changed_into_variables = table
253− .columns
254− .iter()
255− .map(|column| {
256− format!(
257− "NEW.{real_name} AS {alias}",
258− alias = column.name,
259− real_name = column.real_name,
260− )
261− })
262− .collect::<Vec<String>>()
263− .join(", ");
264−
265− let from_table_columns = from_table
266− .columns
267− .iter()
268− .map(|column| format!("{} as {}", column.real_name, column.name))
269− .collect::<Vec<String>>()
270− .join(", ");
271−
272− let query = format!(
273− r#"
274− CREATE OR REPLACE FUNCTION {trigger_name}()
275− RETURNS TRIGGER AS $$
276− #variable_conflict use_variable
277− BEGIN
278− IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN
279− DECLARE
280− {changed_table} record;
281− __temp_row record;
282− BEGIN
283− SELECT {changed_into_variables}
284− INTO {changed_table};
285−
286− SELECT *
287− INTO __temp_row
288− FROM (
289− SELECT {from_table_columns}
290− FROM public.{from_table_real}
291− ) {from_table}
292− WHERE {where};
293−
294− DECLARE
295− {from_table} record;
296− BEGIN
297− {from_table} := __temp_row;
298− NEW.{column_name_real} = {value};
299− END;
300− END;
301− END IF;
302− RETURN NEW;
303− END
304− $$ language 'plpgsql';
305−
306− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";
307− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
308− "#,
309− changed_table = table.name,
310− changed_table_real = table.real_name,
311− from_table = from_table.name,
312− from_table_real = from_table.real_name,
313− column_name_real = column.real_name,
314− trigger_name = self.reverse_trigger_name(ctx),
315− // declarations = declarations.join("\n"),
316− );
317− db.run(&query)
318− .context("failed to create reverse down trigger")?;
319− }
68+ NEW.{column_name} = {down};
69+ END;
70+ END IF;
71+ RETURN NEW;
72+ END
73+ $$ language 'plpgsql';
74+
75+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
76+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();
77+ "#,
78+ column_name = self.column,
79+ trigger_name = self.trigger_name(ctx),
80+ down = down,
81+ table = self.table,
82+ declarations = declarations.join("\n"),
83+ );
84+ db.run(&query).context("failed to create down trigger")?;
32085 }
32186
32287 Ok(())
@@ -346,15 +111,12 @@ impl Action for RemoveColumn {
346111 ALTER TABLE "{table}"
347112 DROP COLUMN IF EXISTS "{column}";
348113
349− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
350− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
351− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
114+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
115+ DROP FUNCTION IF EXISTS "{trigger_name}";
352116 "#,
353117 table = self.table,
354118 column = self.column,
355119 trigger_name = self.trigger_name(ctx),
356− reverse_trigger_name = self.reverse_trigger_name(ctx),
357− null_trigger_name = self.not_null_constraint_trigger_name(ctx),
358120 );
359121 db.run(&query)
360122 .context("failed to drop column and down trigger")?;
@@ -371,81 +133,14 @@ impl Action for RemoveColumn {
371133 }
372134
373135 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
374− // We might have temporaily removed the NOT NULL check and have to reinstate it
375− let has_not_null_function = !db
376− .query_with_params(
377− "
378− SELECT routine_name
379− FROM information_schema.routines
380− WHERE routine_schema = 'public'
381− AND routine_name = $1
382− ",
383− &[&self.not_null_constraint_trigger_name(ctx)],
384− )
385− .context("failed to get any NOT NULL function")?
386− .is_empty();
387−
388− if has_not_null_function {
389− // Make column NOT NULL again without taking any long lived locks with a temporary constraint
390− let query = format!(
391− r#"
392− ALTER TABLE "{table}"
393− ADD CONSTRAINT "{constraint_name}"
394− CHECK ("{column}" IS NOT NULL) NOT VALID
395− "#,
396− table = self.table,
397− constraint_name = self.not_null_constraint_name(ctx),
398− column = self.column,
399− );
400− db.run(&query)
401− .context("failed to add NOT NULL constraint")?;
402−
403− let query = format!(
404− r#"
405− ALTER TABLE "{table}"
406− VALIDATE CONSTRAINT "{constraint_name}"
407− "#,
408− table = self.table,
409− constraint_name = self.not_null_constraint_name(ctx),
410− );
411− db.run(&query)
412− .context("failed to validate NOT NULL constraint")?;
413−
414− // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above
415− db.run(&format!(
416− r#"
417− ALTER TABLE {table}
418− ALTER COLUMN {column}
419− SET NOT NULL
420− "#,
421− table = self.table,
422− column = self.column
423− ))
424− .context("failed to reinstate column NOT NULL")?;
425−
426− // Drop the temporary constraint
427− let query = format!(
428− r#"
429− ALTER TABLE "{table}"
430− DROP CONSTRAINT "{constraint_name}"
431− "#,
432− table = self.table,
433− constraint_name = self.not_null_constraint_name(ctx),
434− );
435− db.run(&query)
436− .context("failed to drop NOT NULL constraint")?;
437− }
438−
439136 // Remove function and trigger
440137 db.run(&format!(
441138 r#"
442− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;
443− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;
444− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;
139+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";
140+ DROP FUNCTION IF EXISTS "{trigger_name}";
445141 "#,
142+ table = self.table,
446143 trigger_name = self.trigger_name(ctx),
447− reverse_trigger_name = self.reverse_trigger_name(ctx),
448− null_trigger_name = self.not_null_constraint_trigger_name(ctx),
449144 ))
450145 .context("failed to drop down trigger")?;
451146
src/schema.rs+1−1
@@ -209,7 +209,7 @@ impl Schema {
209209 let real_columns: Vec<(String, String, bool, Option<String>)> = db
210210 .query(&format!(
211211 "
212− SELECT column_name, CASE WHEN data_type = 'USER-DEFINED' THEN udt_name ELSE data_type END, is_nullable, column_default
212+ SELECT column_name, data_type, is_nullable, column_default
213213 FROM information_schema.columns
214214 WHERE table_name = '{table}' AND table_schema = 'public'
215215 ORDER BY ordinal_position
tests/add_column.rs−99
@@ -260,102 +260,3 @@ fn add_column_with_default() {
260260
261261 test.run();
262262}
263−
264−#[test]
265−fn add_column_with_complex_up() {
266− let mut test = Test::new("Add column complex");
267−
268− test.first_migration(
269− r#"
270− name = "create_tables"
271−
272− [[actions]]
273− type = "create_table"
274− name = "users"
275− primary_key = ["id"]
276−
277− [[actions.columns]]
278− name = "id"
279− type = "INTEGER"
280−
281− [[actions.columns]]
282− name = "email"
283− type = "TEXT"
284−
285− [[actions]]
286− type = "create_table"
287− name = "profiles"
288− primary_key = ["user_id"]
289−
290− [[actions.columns]]
291− name = "user_id"
292− type = "INTEGER"
293− "#,
294− );
295−
296− test.second_migration(
297− r#"
298− name = "add_profiles_email_column"
299−
300− [[actions]]
301− type = "add_column"
302− table = "profiles"
303−
304− [actions.column]
305− name = "email"
306− type = "TEXT"
307− nullable = false
308−
309− [actions.up]
310− table = "users"
311− value = "users.email"
312− where = "profiles.user_id = users.id"
313− "#,
314− );
315−
316− test.after_first(|db| {
317− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
318− .unwrap();
319− db.simple_query("INSERT INTO profiles (user_id) VALUES (1)")
320− .unwrap();
321− });
322−
323− test.intermediate(|old_db, new_db| {
324− // Ensure email was backfilled on profiles
325− let email: String = new_db
326− .query(
327− "
328− SELECT email
329− FROM profiles
330− WHERE user_id = 1
331− ",
332− &[],
333− )
334− .unwrap()
335− .first()
336− .map(|row| row.get("email"))
337− .unwrap();
338− assert_eq!("test@example.com", email);
339−
340− // Ensure email change in old schema is propagated to profiles table in new schema
341− old_db
342− .simple_query("UPDATE users SET email = 'test2@example.com' WHERE id = 1")
343− .unwrap();
344− let email: String = new_db
345− .query(
346− "
347− SELECT email
348− FROM profiles
349− WHERE user_id = 1
350− ",
351− &[],
352− )
353− .unwrap()
354− .first()
355− .map(|row| row.get("email"))
356− .unwrap();
357− assert_eq!("test2@example.com", email);
358− });
359−
360− test.run();
361−}
tests/complex.rsdeleted−379
@@ -1,379 +0,0 @@
1−mod common;
2−use common::Test;
3−
4−#[test]
5−fn move_column_between_tables() {
6− let mut test = Test::new("Move column between tables");
7−
8− test.first_migration(
9− r#"
10− name = "create_tables"
11−
12− [[actions]]
13− type = "create_table"
14− name = "users"
15− primary_key = ["id"]
16−
17− [[actions.columns]]
18− name = "id"
19− type = "INTEGER"
20−
21− [[actions.columns]]
22− name = "email"
23− type = "TEXT"
24−
25− [[actions]]
26− type = "create_table"
27− name = "profiles"
28− primary_key = ["id"]
29−
30− [[actions.columns]]
31− name = "id"
32− type = "INTEGER"
33−
34− [[actions.columns]]
35− name = "user_id"
36− type = "INTEGER"
37− nullable = false
38− "#,
39− );
40−
41− test.second_migration(
42− r#"
43− name = "move_email_column"
44−
45− [[actions]]
46− type = "add_column"
47− table = "profiles"
48−
49− [actions.column]
50− name = "email"
51− type = "TEXT"
52− nullable = false
53−
54− # When `users` is updated in the old schema, we write the email value to `profiles`
55− # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated
56− [actions.up]
57− table = "users"
58− value = "users.email"
59− where = "profiles.user_id = users.id"
60−
61− [[actions]]
62− type = "remove_column"
63− table = "users"
64− column = "email"
65−
66− # When `profiles` is changed in the new schema, we write the email address back to the removed column
67− [actions.down]
68− table = "profiles"
69− value = "profiles.email"
70− where = "users.id = profiles.user_id"
71− "#,
72− );
73−
74− test.after_first(|db| {
75− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")
76− .unwrap();
77− db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")
78− .unwrap();
79−
80− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")
81− .unwrap();
82− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")
83− .unwrap();
84− });
85−
86− test.intermediate(|old_db, new_db| {
87− // Ensure emails were backfilled into profiles
88− let profiles_emails: Vec<String> = new_db
89− .query(
90− r#"
91− SELECT email
92− FROM profiles
93− ORDER BY id
94− "#,
95− &[],
96− )
97− .unwrap()
98− .iter()
99− .map(|row| row.get("email"))
100− .collect();
101− assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);
102−
103− // Ensure insert in old schema updates new
104− old_db
105− .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")
106− .unwrap();
107− old_db
108− .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")
109− .unwrap();
110− let new_email: String = new_db
111− .query("SELECT email FROM profiles WHERE id = 3", &[])
112− .unwrap()
113− .first()
114− .map(|row| row.get("email"))
115− .unwrap();
116− assert_eq!("test3@test.com", new_email);
117−
118− // Ensure updates in old schema updates new
119− old_db
120− .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")
121− .unwrap();
122− let new_email: String = new_db
123− .query("SELECT email FROM profiles WHERE id = 3", &[])
124− .unwrap()
125− .first()
126− .map(|row| row.get("email"))
127− .unwrap();
128− assert_eq!("test3+updated@test.com", new_email);
129−
130− // Ensure insert in new schema updates old
131− new_db
132− .simple_query(
133− "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",
134− )
135− .unwrap();
136− new_db
137− .simple_query("INSERT INTO users (id) VALUES (4)")
138− .unwrap();
139− let old_email: String = old_db
140− .query("SELECT email FROM users WHERE id = 4", &[])
141− .unwrap()
142− .first()
143− .map(|row| row.get("email"))
144− .unwrap();
145− assert_eq!("test4@test.com", old_email);
146−
147− // Ensure update in new schema updates old
148− new_db
149− .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")
150− .unwrap();
151− let old_email: String = old_db
152− .query("SELECT email FROM users WHERE id = 4", &[])
153− .unwrap()
154− .first()
155− .map(|row| row.get("email"))
156− .unwrap();
157− assert_eq!("test4+updated@test.com", old_email);
158− });
159−
160− test.run();
161−}
162−
163−#[test]
164−fn extract_relation_into_new_table() {
165− let mut test = Test::new("Extract relation into new table");
166−
167− test.first_migration(
168− r#"
169− name = "create_tables"
170−
171− [[actions]]
172− type = "create_table"
173− name = "accounts"
174− primary_key = ["id"]
175−
176− [[actions.columns]]
177− name = "id"
178− type = "INTEGER"
179−
180− [[actions]]
181− type = "create_table"
182− name = "users"
183− primary_key = ["id"]
184−
185− [[actions.columns]]
186− name = "id"
187− type = "INTEGER"
188−
189− [[actions.columns]]
190− name = "account_id"
191− type = "INTEGER"
192− nullable = false
193−
194− [[actions.columns]]
195− name = "account_role"
196− type = "TEXT"
197− nullable = false
198− "#,
199− );
200−
201− test.second_migration(
202− r#"
203− name = "add_account_user_connection"
204−
205− [[actions]]
206− type = "create_table"
207− name = "user_account_connections"
208− primary_key = ["account_id", "user_id"]
209−
210− [[actions.columns]]
211− name = "account_id"
212− type = "INTEGER"
213−
214− [[actions.columns]]
215− name = "user_id"
216− type = "INTEGER"
217−
218− [[actions.columns]]
219− name = "role"
220− type = "TEXT"
221− nullable = false
222−
223− [actions.up]
224− table = "users"
225− values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }
226− where = "user_account_connections.user_id = users.id"
227−
228− [[actions]]
229− type = "remove_column"
230− table = "users"
231− column = "account_id"
232−
233− [actions.down]
234− table = "user_account_connections"
235− value = "user_account_connections.account_id"
236− where = "users.id = user_account_connections.user_id"
237−
238− [[actions]]
239− type = "remove_column"
240− table = "users"
241− column = "account_role"
242−
243− [actions.down]
244− table = "user_account_connections"
245− value = "LOWER(user_account_connections.role)"
246− where = "users.id = user_account_connections.user_id"
247− "#,
248− );
249−
250− test.after_first(|db| {
251− db.simple_query("INSERT INTO accounts (id) VALUES (1)")
252− .unwrap();
253− db.simple_query("INSERT INTO users (id, account_id, account_role) VALUES (1, 1, 'admin')")
254− .unwrap();
255− });
256−
257− test.intermediate(|old_db, new_db| {
258− // Ensure connections was backfilled
259− let rows: Vec<(i32, i32, String)> = new_db
260− .query(
261− "
262− SELECT account_id, user_id, role
263− FROM user_account_connections
264− ",
265− &[],
266− )
267− .unwrap()
268− .iter()
269− .map(|row| (row.get("account_id"), row.get("user_id"), row.get("role")))
270− .collect();
271− assert_eq!(1, rows.len());
272−
273− let row = rows.first().unwrap();
274− assert_eq!(1, row.0);
275− assert_eq!(1, row.1);
276− assert_eq!("ADMIN", row.2);
277−
278− // Ensure inserted user in old schema creates a new connection
279− old_db
280− .simple_query(
281− "INSERT INTO users (id, account_id, account_role) VALUES (2, 1, 'developer')",
282− )
283− .unwrap();
284− assert!(
285− new_db
286− .query(
287− "
288− SELECT account_id, user_id, role
289− FROM user_account_connections
290− WHERE account_id = 1 AND user_id = 2 AND role = 'DEVELOPER'
291− ",
292− &[],
293− )
294− .unwrap()
295− .len()
296− == 1
297− );
298−
299− // Ensure NOT NULL constraint still applies to old schema
300− let result = old_db
301− .simple_query(
302− "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",
303− );
304− assert!(result.is_err());
305−
306− // Ensure updated user role in old schema updates connection in new schema
307− old_db
308− .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")
309− .unwrap();
310− assert!(
311− new_db
312− .query(
313− "
314− SELECT account_id, user_id, role
315− FROM user_account_connections
316− WHERE account_id = 1 AND user_id = 2 AND role = 'ADMIN'
317− ",
318− &[],
319− )
320− .unwrap()
321− .len()
322− == 1
323− );
324−
325− // Ensure updated connection in new schema updates old schema user
326− new_db
327− .simple_query(
328− "UPDATE user_account_connections SET role = 'DEVELOPER' WHERE account_id = 1 AND user_id = 2",
329− )
330− .unwrap();
331− assert!(
332− old_db
333− .query(
334− "
335− SELECT id
336− FROM users
337− WHERE id = 2 AND account_id = 1 AND account_role = 'developer'
338− ",
339− &[],
340− )
341− .unwrap()
342− .len()
343− == 1
344− );
345−
346− // Ensure insert of user with connection through new schema updates user in old schema
347− new_db
348− .simple_query(
349− r#"
350− BEGIN;
351− INSERT INTO users (id) VALUES (3);
352− INSERT INTO user_account_connections (user_id, account_id, role) VALUES (3, 1, 'DEVELOPER');
353− COMMIT;
354− "#,
355− )
356− .unwrap();
357− new_db
358− .simple_query(
359− "",
360− )
361− .unwrap();
362− assert!(
363− old_db
364− .query(
365− "
366− SELECT id
367− FROM users
368− WHERE id = 3 AND account_id = 1 AND account_role = 'developer'
369− ",
370− &[],
371− )
372− .unwrap()
373− .len()
374− == 1
375− );
376− });
377−
378− test.run();
379−}
tests/remove_column.rs−86
@@ -128,90 +128,4 @@ fn remove_column_with_index() {
128128
129129 assert_eq!(0, count, "expected index to not exist");
130130 });
131−
132− test.run();
133−}
134−
135−#[test]
136−fn remove_column_with_complex_down() {
137− let mut test = Test::new("Remove column complex");
138−
139− test.first_migration(
140− r#"
141− name = "create_tables"
142−
143− [[actions]]
144− type = "create_table"
145− name = "users"
146− primary_key = ["id"]
147−
148− [[actions.columns]]
149− name = "id"
150− type = "INTEGER"
151−
152− [[actions.columns]]
153− name = "email"
154− type = "TEXT"
155−
156− [[actions]]
157− type = "create_table"
158− name = "profiles"
159− primary_key = ["user_id"]
160−
161− [[actions.columns]]
162− name = "user_id"
163− type = "INTEGER"
164−
165− [[actions.columns]]
166− name = "email"
167− type = "TEXT"
168− "#,
169− );
170−
171− test.second_migration(
172− r#"
173− name = "remove_users_email_column"
174−
175− [[actions]]
176− type = "remove_column"
177− table = "users"
178− column = "email"
179−
180− [actions.down]
181− table = "profiles"
182− value = "profiles.email"
183− where = "users.id = profiles.user_id"
184− "#,
185− );
186−
187− test.after_first(|db| {
188− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")
189− .unwrap();
190− db.simple_query("INSERT INTO profiles (user_id, email) VALUES (1, 'test@example.com')")
191− .unwrap();
192− });
193−
194− test.intermediate(|old_db, new_db| {
195− new_db
196− .simple_query("UPDATE profiles SET email = 'test2@example.com' WHERE user_id = 1")
197− .unwrap();
198−
199− // Ensure new email was propagated to users table in old schema
200− let email: String = old_db
201− .query(
202− "
203− SELECT email
204− FROM users
205− WHERE id = 1
206− ",
207− &[],
208− )
209− .unwrap()
210− .first()
211− .map(|row| row.get("email"))
212− .unwrap();
213− assert_eq!("test2@example.com", email);
214− });
215−
216− test.run();
217131}
tests/remove_table.rs−2
@@ -39,6 +39,4 @@ fn remove_table() {
3939 // Ensure the table is not accessible through the new schema
4040 assert!(new_db.query("SELECT id FROM users", &[]).is_err());
4141 });
42−
43− test.run();
4442}