bf1fe13...main · reshape

git.iko.soy/reshape

on main help sign in

.github/workflows/lint.yaml+3−3

since splitsnapshots

unifiedsplit

@@ -15,7 +15,7 @@ jobs:

15 runs-on: ubuntu-latest

16 steps:

17 - name: Checkout

18− uses: actions/checkout@v2

18+ uses: actions/checkout@v4

19 - name: Select Rust toolchain with Clippy

20 uses: actions-rs/toolchain@v1

21 with:

@@ -23,6 +23,6 @@ jobs:

23 components: clippy

24 override: true

25 - name: Use cache for Rust dependencies

26− uses: Swatinem/rust-cache@v1

26+ uses: Swatinem/rust-cache@v2

27 - name: Lint using Clippy

28− run: cargo clippy

28+ run: cargo clippy

.github/workflows/publish-crate.yamlnew file+20

@@ -0,0 +1,20 @@

1+name: Publish crate

3+on: workflow_dispatch

5+jobs:

6+ publish-crate:

7+ runs-on: ubuntu-latest

8+ steps:

9+ - name: Checkout

10+ uses: actions/checkout@v2

11+ - name: Select Rust toolchain

12+ uses: actions-rs/toolchain@v1

13+ with:

14+ toolchain: stable

15+ - name: Publish to crates.io

16+ uses: actions-rs/cargo@v1

17+ with:

18+ command: publish

19+ env:

20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.github/workflows/publish-docker.yamlnew file+25

@@ -0,0 +1,25 @@

1+name: Publish Docker image

3+on:

4+ workflow_dispatch:

5+ inputs:

6+ version:

7+ description: "Version (without 'v' prefix)"

8+ required: true

9+ type: string

10+

11+jobs:

12+ publish-docker:

13+ runs-on: ubuntu-latest

14+ steps:

15+ - name: Checkout

16+ uses: actions/checkout@v2

17+ - name: Log in to Docker

18+ env:

19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}

20+ run: |

21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD

22+ - name: Build Docker image

23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest

24+ - name: Push Docker image

25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape

Cargo.toml+3−3

@@ -1,6 +1,6 @@

1[package]

2name = "reshape"

3−version = "0.6.1"

3+version = "0.7.0"

4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"

5homepage = "https://github.com/fabianlindfors/reshape"

6documentation = "https://github.com/fabianlindfors/reshape"

@@ -9,7 +9,7 @@ license = "MIT"

9keywords = ["postgres", "migrations"]

10edition = "2021"

11authors = ["Fabian Lindfors"]

12−rust-version = "1.58"

12+rust-version = "1.70"

14[dependencies]

15postgres = { version = "0.19.2", features = ["with-serde_json-1"] }

@@ -23,4 +23,4 @@ version = "3.0.0"

23colored = "2"

24rand = "0.8"

25dotenv = "0.15.0"

26−lexical-sort = "0.3.1"

26+lexical-sort = "0.3.1"

Dockerfile+1−1

@@ -1,4 +1,4 @@

1−FROM rust:1.58.0 AS builder

1+FROM rust:1.70.0 AS builder

2WORKDIR /usr/src/reshape

3COPY . .

4RUN cargo build --release

README.md+21−20

@@ -500,8 +500,8 @@ column = "email"

500 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

501 [actions.down]

502 table = "profiles"

503− value = "email"

503+ value = "profiles.email"

504− where = "id = user_id"

504+ where = "users.id = profiles.user_id"

505```

506

507### Indices

@@ -627,17 +627,6 @@ The `up` and `down` options available when creating tables, adding columns and r

627_Example: move `email` column from `users` to `profiles` table_

628

629```toml

630−[[actions]]

631−type = "remove_column"

632−table = "users"

633−column = "email"

634−

635− # When `profiles` is changed in the new schema, we write the email address back to the removed column

636− [actions.down]

637− table = "profiles"

638− value = "email"

639− where = "id = user_id"

640−

641[[actions]]

630[[actions]]

642type = "add_column"

631type = "add_column"

643table = "profiles"

632table = "profiles"

@@ -650,8 +639,19 @@ table = "profiles"

650 # When `users` is updated in the old schema, we write the email value to `profiles`

639 # When `users` is updated in the old schema, we write the email value to `profiles`

651 [actions.up]

640 [actions.up]

652 table = "users"

641 table = "users"

653− value = "email"

642+ value = "users.email"

654− where = "user_id = id"

643+ where = "profiles.user_id = users.id"

644+

645+[[actions]]

646+type = "remove_column"

647+table = "users"

648+column = "email"

649+

650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column

651+ [actions.down]

652+ table = "profiles"

653+ value = "profiles.email"

654+ where = "users.id = profiles.user_id"

655```

656

657_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_

@@ -675,12 +675,13 @@ primary_key = ["account_id", "user_id"]

675 [[actions.columns]]

676 name = "role"

677 type = "TEXT"

678+ nullable = false

678

679

679 # Backfill the new table from `users` and uppercase the `role`

680 # Backfill the new table from `users` and uppercase the `role`

680 [actions.up]

681 [actions.up]

681 table = "users"

682 table = "users"

682 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

683 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

683− where = "user_id = id"

684+ where = "user_account_connections.user_id = users.id"

684

685

685[[actions]]

686[[actions]]

686type = "remove_column"

687type = "remove_column"

@@ -690,8 +691,8 @@ column = "account_id"

690 # When `user_account_connections` is updated, we write the `account_id` back to `users`

691 # When `user_account_connections` is updated, we write the `account_id` back to `users`

691 [actions.down]

692 [actions.down]

692 table = "user_account_connections"

693 table = "user_account_connections"

693− value = "account_id"

694+ value = "user_account_connections.account_id"

694− where = "id = user_id"

695+ where = "users.id = user_account_connections.user_id"

695

696

696[[actions]]

697[[actions]]

697type = "remove_column"

698type = "remove_column"

@@ -701,8 +702,8 @@ column = "account_role"

701 # When `user_account_connections` is updated, we write the lowercase role back to `users`

702 # When `user_account_connections` is updated, we write the lowercase role back to `users`

702 [actions.down]

703 [actions.down]

703 table = "user_account_connections"

704 table = "user_account_connections"

704− value = "LOWER(role)"

705+ value = "LOWER(user_account_connections.role)"

705− where = "id = user_id"

706+ where = "users.id = user_account_connections.user_id"

706```

707```

707

708

708## Commands and options

709## Commands and options

src/migrations/add_column.rs+3−3

@@ -81,7 +81,7 @@ impl Action for AddColumn {

81 let temp_column_name = self.temp_column_name(ctx);

83 let mut definition_parts = vec![

84− temp_column_name.to_string(),

84+ format!("\"{}\"", temp_column_name.to_string()),

85 self.column.data_type.to_string(),

86 ];

@@ -111,7 +111,7 @@ impl Action for AddColumn {

111 .iter()

112 .map(|column| {

113 format!(

114− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",

115 table = table.real_name,

116 alias = column.name,

117 real_name = column.real_name,

@@ -131,7 +131,7 @@ impl Action for AddColumn {

131 DECLARE

132 {declarations}

133 BEGIN

134− NEW.{temp_column_name} = {up};

134+ NEW."{temp_column_name}" = {up};

135 END;

136 END IF;

137 RETURN NEW;

src/migrations/remove_column.rs+78−18

@@ -43,7 +43,7 @@ impl RemoveColumn {

43 )

44 }

46− fn null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

47 format!(

48 "{}_remove_column_{}_{}_nn",

49 ctx.prefix(),

@@ -51,6 +51,15 @@ impl RemoveColumn {

51 self.column

52 )

53 }

54+

55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56+ format!(

57+ "{}_add_column_not_null_{}_{}",

58+ ctx.prefix(),

59+ self.table,

60+ self.column

61+ )

62+ }

54}

63}

56#[typetag::serde(name = "remove_column")]

65#[typetag::serde(name = "remove_column")]

@@ -132,6 +141,7 @@ impl Action for RemoveColumn {

132

141

133 let maybe_null_check = if !column.nullable {

142 let maybe_null_check = if !column.nullable {

134 // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

143 // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

144+ // We will add a null check to the down function on the new schema below as well to cover both cases.

135 // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

145 // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

136 // NOT NULL is not checked at the end of a transaction, but immediately upon update.

146 // NOT NULL is not checked at the end of a transaction, but immediately upon update.

137 let query = format!(

147 let query = format!(

@@ -157,7 +167,7 @@ impl Action for RemoveColumn {

157 EXECUTE PROCEDURE {trigger_name}();

167 EXECUTE PROCEDURE {trigger_name}();

158 "#,

168 "#,

159 table = self.table,

169 table = self.table,

160− trigger_name = self.null_constraint_trigger_name(ctx),

170+ trigger_name = self.not_null_constraint_trigger_name(ctx),

161 column = self.column,

171 column = self.column,

162 );

172 );

163 db.run(&query)

173 db.run(&query)

@@ -236,7 +246,6 @@ impl Action for RemoveColumn {

236 from_table_real = from_table.real_name,

246 from_table_real = from_table.real_name,

237 column_name = self.column,

247 column_name = self.column,

238 trigger_name = self.trigger_name(ctx),

248 trigger_name = self.trigger_name(ctx),

239− // declarations = declarations.join("\n"),

240 );

249 );

241 db.run(&query).context("failed to create down trigger")?;

250 db.run(&query).context("failed to create down trigger")?;

242

251

@@ -310,8 +319,6 @@ impl Action for RemoveColumn {

310 }

319 }

311 }

320 }

312

321

313− if !column.nullable {}

314−

315 Ok(())

322 Ok(())

316 }

323 }

317

324

@@ -347,7 +354,7 @@ impl Action for RemoveColumn {

347 column = self.column,

354 column = self.column,

348 trigger_name = self.trigger_name(ctx),

355 trigger_name = self.trigger_name(ctx),

349 reverse_trigger_name = self.reverse_trigger_name(ctx),

356 reverse_trigger_name = self.reverse_trigger_name(ctx),

350− null_trigger_name = self.null_constraint_trigger_name(ctx),

357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),

351 );

358 );

352 db.run(&query)

359 db.run(&query)

353 .context("failed to drop column and down trigger")?;

360 .context("failed to drop column and down trigger")?;

@@ -364,17 +371,70 @@ impl Action for RemoveColumn {

364 }

371 }

365

372

366 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

367− // We might have removed the NOT NULL check so we reinstate it

374+ // We might have temporaily removed the NOT NULL check and have to reinstate it

375+ let has_not_null_function = !db

376+ .query_with_params(

377+ "

378+ SELECT routine_name

379+ FROM information_schema.routines

380+ WHERE routine_schema = 'public'

381+ AND routine_name = $1

382+ ",

383+ &[&self.not_null_constraint_trigger_name(ctx)],

384+ )

385+ .context("failed to get any NOT NULL function")?

386+ .is_empty();

387+

388+ if has_not_null_function {

389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint

368− db.run(&format!(

390+ let query = format!(

369− r#"

391+ r#"

392+ ALTER TABLE "{table}"

393+ ADD CONSTRAINT "{constraint_name}"

394+ CHECK ("{column}" IS NOT NULL) NOT VALID

395+ "#,

396+ table = self.table,

397+ constraint_name = self.not_null_constraint_name(ctx),

398+ column = self.column,

399+ );

400+ db.run(&query)

401+ .context("failed to add NOT NULL constraint")?;

402+

403+ let query = format!(

404+ r#"

370− ALTER TABLE {table}

405+ ALTER TABLE "{table}"

406+ VALIDATE CONSTRAINT "{constraint_name}"

407+ "#,

408+ table = self.table,

409+ constraint_name = self.not_null_constraint_name(ctx),

410+ );

411+ db.run(&query)

412+ .context("failed to validate NOT NULL constraint")?;

413+

414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above

415+ db.run(&format!(

416+ r#"

417+ ALTER TABLE {table}

371− ALTER COLUMN {column}

418+ ALTER COLUMN {column}

372− SET NOT NULL

419+ SET NOT NULL

373− "#,

420+ "#,

374− table = self.table,

421+ table = self.table,

375− column = self.column

422+ column = self.column

376− ))

423+ ))

377− .context("failed to reinstate column not null constraint")?;

424+ .context("failed to reinstate column NOT NULL")?;

425+

426+ // Drop the temporary constraint

427+ let query = format!(

428+ r#"

429+ ALTER TABLE "{table}"

430+ DROP CONSTRAINT "{constraint_name}"

431+ "#,

432+ table = self.table,

433+ constraint_name = self.not_null_constraint_name(ctx),

434+ );

435+ db.run(&query)

436+ .context("failed to drop NOT NULL constraint")?;

437+ }

378

438

379 // Remove function and trigger

439 // Remove function and trigger

380 db.run(&format!(

440 db.run(&format!(

@@ -385,7 +445,7 @@ impl Action for RemoveColumn {

385 "#,

445 "#,

386 trigger_name = self.trigger_name(ctx),

446 trigger_name = self.trigger_name(ctx),

387 reverse_trigger_name = self.reverse_trigger_name(ctx),

447 reverse_trigger_name = self.reverse_trigger_name(ctx),

388− null_trigger_name = self.null_constraint_trigger_name(ctx),

448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),

389 ))

449 ))

390 .context("failed to drop down trigger")?;

450 .context("failed to drop down trigger")?;

391

451

tests/add_column.rs+2−2

@@ -308,8 +308,8 @@ fn add_column_with_complex_up() {

308

309 [actions.up]

310 table = "users"

311− value = "email"

311+ value = "users.email"

312− where = "user_id = id"

312+ where = "profiles.user_id = users.id"

313 "#,

314 );

315

tests/complex.rs+1−1

@@ -52,7 +52,7 @@ fn move_column_between_tables() {

52 nullable = false

54 # When `users` is updated in the old schema, we write the email value to `profiles`

55− # When `profiles` is updated in the old schema, we touch the equivalent `users` row to update it

55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated

56 [actions.up]

57 table = "users"

58 value = "users.email"

tests/remove_column.rs+2−2

@@ -179,8 +179,8 @@ fn remove_column_with_complex_down() {

179

180 [actions.down]

181 table = "profiles"

182− value = "email"

182+ value = "profiles.email"

183− where = "id = user_id"

183+ where = "users.id = profiles.user_id"

184 "#,

185 );

186