on main help sign in
11 files changed+150−44
since splitsnapshots
unifiedsplit
.github/workflows/lint.yaml+3−3 .github/workflows/publish-crate.yaml+20 .github/workflows/publish-docker.yaml+25 Cargo.toml+3−3 Dockerfile+1−1 README.md+12−11 src/migrations/add_column.rs+3−3 src/migrations/remove_column.rs+78−18 tests/add_column.rs+2−2 tests/complex.rs+1−1 tests/remove_column.rs+2−2
.github/workflows/lint.yaml+3−3
@@ -15,7 +15,7 @@ jobs:
1515 runs-on: ubuntu-latest
1616 steps:
1717 - name: Checkout
18− uses: actions/checkout@v2
18+ uses: actions/checkout@v4
1919 - name: Select Rust toolchain with Clippy
2020 uses: actions-rs/toolchain@v1
2121 with:
@@ -23,6 +23,6 @@ jobs:
2323 components: clippy
2424 override: true
2525 - name: Use cache for Rust dependencies
26− uses: Swatinem/rust-cache@v1
26+ uses: Swatinem/rust-cache@v2
2727 - name: Lint using Clippy
28− run: cargo clippy
28+ run: cargo clippy
.github/workflows/publish-crate.yamlnew file+20
@@ -0,0 +1,20 @@
1+name: Publish crate
2+
3+on: workflow_dispatch
4+
5+jobs:
6+ publish-crate:
7+ runs-on: ubuntu-latest
8+ steps:
9+ - name: Checkout
10+ uses: actions/checkout@v2
11+ - name: Select Rust toolchain
12+ uses: actions-rs/toolchain@v1
13+ with:
14+ toolchain: stable
15+ - name: Publish to crates.io
16+ uses: actions-rs/cargo@v1
17+ with:
18+ command: publish
19+ env:
20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
.github/workflows/publish-docker.yamlnew file+25
@@ -0,0 +1,25 @@
1+name: Publish Docker image
2+
3+on:
4+ workflow_dispatch:
5+ inputs:
6+ version:
7+ description: "Version (without 'v' prefix)"
8+ required: true
9+ type: string
10+
11+jobs:
12+ publish-docker:
13+ runs-on: ubuntu-latest
14+ steps:
15+ - name: Checkout
16+ uses: actions/checkout@v2
17+ - name: Log in to Docker
18+ env:
19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
20+ run: |
21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD
22+ - name: Build Docker image
23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest
24+ - name: Push Docker image
25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape
Cargo.toml+3−3
@@ -1,6 +1,6 @@
11[package]
22name = "reshape"
3−version = "0.6.1"
3+version = "0.7.0"
44description = "An easy-to-use, zero-downtime schema migration tool for Postgres"
55homepage = "https://github.com/fabianlindfors/reshape"
66documentation = "https://github.com/fabianlindfors/reshape"
@@ -9,7 +9,7 @@ license = "MIT"
99keywords = ["postgres", "migrations"]
1010edition = "2021"
1111authors = ["Fabian Lindfors"]
12−rust-version = "1.58"
12+rust-version = "1.70"
1313
1414[dependencies]
1515postgres = { version = "0.19.2", features = ["with-serde_json-1"] }
@@ -23,4 +23,4 @@ version = "3.0.0"
2323colored = "2"
2424rand = "0.8"
2525dotenv = "0.15.0"
26−lexical-sort = "0.3.1"
26+lexical-sort = "0.3.1"
Dockerfile+1−1
@@ -1,4 +1,4 @@
1−FROM rust:1.58.0 AS builder
1+FROM rust:1.70.0 AS builder
22WORKDIR /usr/src/reshape
33COPY . .
44RUN cargo build --release
README.md+12−11
@@ -627,17 +627,6 @@ The `up` and `down` options available when creating tables, adding columns and r
627627_Example: move `email` column from `users` to `profiles` table_
628628
629629```toml
630−[[actions]]
631−type = "remove_column"
632−table = "users"
633−column = "email"
634−
635− # When `profiles` is changed in the new schema, we write the email address back to the removed column
636− [actions.down]
637− table = "profiles"
638− value = "profiles.email"
639− where = "users.id = profiles.user_id"
640−
641630[[actions]]
642631type = "add_column"
643632table = "profiles"
@@ -652,6 +641,17 @@ table = "profiles"
652641 table = "users"
653642 value = "users.email"
654643 where = "profiles.user_id = users.id"
644+
645+[[actions]]
646+type = "remove_column"
647+table = "users"
648+column = "email"
649+
650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column
651+ [actions.down]
652+ table = "profiles"
653+ value = "profiles.email"
654+ where = "users.id = profiles.user_id"
655655```
656656
657657_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_
@@ -675,6 +675,7 @@ primary_key = ["account_id", "user_id"]
675675 [[actions.columns]]
676676 name = "role"
677677 type = "TEXT"
678+ nullable = false
678679
679680 # Backfill the new table from `users` and uppercase the `role`
680681 [actions.up]
src/migrations/add_column.rs+3−3
@@ -81,7 +81,7 @@ impl Action for AddColumn {
8181 let temp_column_name = self.temp_column_name(ctx);
8282
8383 let mut definition_parts = vec![
84− temp_column_name.to_string(),
84+ format!("\"{}\"", temp_column_name.to_string()),
8585 self.column.data_type.to_string(),
8686 ];
8787
@@ -111,7 +111,7 @@ impl Action for AddColumn {
111111 .iter()
112112 .map(|column| {
113113 format!(
114− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",
114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",
115115 table = table.real_name,
116116 alias = column.name,
117117 real_name = column.real_name,
@@ -131,7 +131,7 @@ impl Action for AddColumn {
131131 DECLARE
132132 {declarations}
133133 BEGIN
134− NEW.{temp_column_name} = {up};
134+ NEW."{temp_column_name}" = {up};
135135 END;
136136 END IF;
137137 RETURN NEW;
src/migrations/remove_column.rs+78−18
@@ -43,7 +43,7 @@ impl RemoveColumn {
4343 )
4444 }
4545
46− fn null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {
4747 format!(
4848 "{}_remove_column_{}_{}_nn",
4949 ctx.prefix(),
@@ -51,6 +51,15 @@ impl RemoveColumn {
5151 self.column
5252 )
5353 }
54+
55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {
56+ format!(
57+ "{}_add_column_not_null_{}_{}",
58+ ctx.prefix(),
59+ self.table,
60+ self.column
61+ )
62+ }
5463}
5564
5665#[typetag::serde(name = "remove_column")]
@@ -132,6 +141,7 @@ impl Action for RemoveColumn {
132141
133142 let maybe_null_check = if !column.nullable {
134143 // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.
144+ // We will add a null check to the down function on the new schema below as well to cover both cases.
135145 // As we are using a complex down function, we must remove the NOT NULL check for the new schema.
136146 // NOT NULL is not checked at the end of a transaction, but immediately upon update.
137147 let query = format!(
@@ -157,7 +167,7 @@ impl Action for RemoveColumn {
157167 EXECUTE PROCEDURE {trigger_name}();
158168 "#,
159169 table = self.table,
160− trigger_name = self.null_constraint_trigger_name(ctx),
170+ trigger_name = self.not_null_constraint_trigger_name(ctx),
161171 column = self.column,
162172 );
163173 db.run(&query)
@@ -236,7 +246,6 @@ impl Action for RemoveColumn {
236246 from_table_real = from_table.real_name,
237247 column_name = self.column,
238248 trigger_name = self.trigger_name(ctx),
239− // declarations = declarations.join("\n"),
240249 );
241250 db.run(&query).context("failed to create down trigger")?;
242251
@@ -310,8 +319,6 @@ impl Action for RemoveColumn {
310319 }
311320 }
312321
313− if !column.nullable {}
314−
315322 Ok(())
316323 }
317324
@@ -347,7 +354,7 @@ impl Action for RemoveColumn {
347354 column = self.column,
348355 trigger_name = self.trigger_name(ctx),
349356 reverse_trigger_name = self.reverse_trigger_name(ctx),
350− null_trigger_name = self.null_constraint_trigger_name(ctx),
357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
351358 );
352359 db.run(&query)
353360 .context("failed to drop column and down trigger")?;
@@ -364,17 +371,70 @@ impl Action for RemoveColumn {
364371 }
365372
366373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {
367− // We might have removed the NOT NULL check so we reinstate it
368− db.run(&format!(
369− r#"
370− ALTER TABLE {table}
371− ALTER COLUMN {column}
372− SET NOT NULL
373− "#,
374− table = self.table,
375− column = self.column
376− ))
377− .context("failed to reinstate column not null constraint")?;
374+ // We might have temporaily removed the NOT NULL check and have to reinstate it
375+ let has_not_null_function = !db
376+ .query_with_params(
377+ "
378+ SELECT routine_name
379+ FROM information_schema.routines
380+ WHERE routine_schema = 'public'
381+ AND routine_name = $1
382+ ",
383+ &[&self.not_null_constraint_trigger_name(ctx)],
384+ )
385+ .context("failed to get any NOT NULL function")?
386+ .is_empty();
387+
388+ if has_not_null_function {
389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint
390+ let query = format!(
391+ r#"
392+ ALTER TABLE "{table}"
393+ ADD CONSTRAINT "{constraint_name}"
394+ CHECK ("{column}" IS NOT NULL) NOT VALID
395+ "#,
396+ table = self.table,
397+ constraint_name = self.not_null_constraint_name(ctx),
398+ column = self.column,
399+ );
400+ db.run(&query)
401+ .context("failed to add NOT NULL constraint")?;
402+
403+ let query = format!(
404+ r#"
405+ ALTER TABLE "{table}"
406+ VALIDATE CONSTRAINT "{constraint_name}"
407+ "#,
408+ table = self.table,
409+ constraint_name = self.not_null_constraint_name(ctx),
410+ );
411+ db.run(&query)
412+ .context("failed to validate NOT NULL constraint")?;
413+
414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above
415+ db.run(&format!(
416+ r#"
417+ ALTER TABLE {table}
418+ ALTER COLUMN {column}
419+ SET NOT NULL
420+ "#,
421+ table = self.table,
422+ column = self.column
423+ ))
424+ .context("failed to reinstate column NOT NULL")?;
425+
426+ // Drop the temporary constraint
427+ let query = format!(
428+ r#"
429+ ALTER TABLE "{table}"
430+ DROP CONSTRAINT "{constraint_name}"
431+ "#,
432+ table = self.table,
433+ constraint_name = self.not_null_constraint_name(ctx),
434+ );
435+ db.run(&query)
436+ .context("failed to drop NOT NULL constraint")?;
437+ }
378438
379439 // Remove function and trigger
380440 db.run(&format!(
@@ -385,7 +445,7 @@ impl Action for RemoveColumn {
385445 "#,
386446 trigger_name = self.trigger_name(ctx),
387447 reverse_trigger_name = self.reverse_trigger_name(ctx),
388− null_trigger_name = self.null_constraint_trigger_name(ctx),
448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),
389449 ))
390450 .context("failed to drop down trigger")?;
391451
tests/add_column.rs+2−2
@@ -308,8 +308,8 @@ fn add_column_with_complex_up() {
308308
309309 [actions.up]
310310 table = "users"
311− value = "email"
312− where = "user_id = id"
311+ value = "users.email"
312+ where = "profiles.user_id = users.id"
313313 "#,
314314 );
315315
tests/complex.rs+1−1
@@ -52,7 +52,7 @@ fn move_column_between_tables() {
5252 nullable = false
5353
5454 # When `users` is updated in the old schema, we write the email value to `profiles`
55− # When `profiles` is updated in the old schema, we touch the equivalent `users` row to update it
55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated
5656 [actions.up]
5757 table = "users"
5858 value = "users.email"
tests/remove_column.rs+2−2
@@ -179,8 +179,8 @@ fn remove_column_with_complex_down() {
179179
180180 [actions.down]
181181 table = "profiles"
182− value = "email"
183− where = "id = user_id"
182+ value = "profiles.email"
183+ where = "users.id = profiles.user_id"
184184 "#,
185185 );
186186