main..647ba2d · reshape

git.iko.soy/reshape

on 647ba2d help sign in

.github/workflows/lint.yaml+3−3

since splitsnapshots

unifiedsplit

@@ -15,7 +15,7 @@ jobs:

15 runs-on: ubuntu-latest

16 steps:

17 - name: Checkout

18− uses: actions/checkout@v4

18+ uses: actions/checkout@v2

19 - name: Select Rust toolchain with Clippy

20 uses: actions-rs/toolchain@v1

21 with:

@@ -23,6 +23,6 @@ jobs:

23 components: clippy

24 override: true

25 - name: Use cache for Rust dependencies

26− uses: Swatinem/rust-cache@v2

26+ uses: Swatinem/rust-cache@v1

27 - name: Lint using Clippy

28− run: cargo clippy

28+ run: cargo clippy

.github/workflows/publish-crate.yamldeleted−20

@@ -1,20 +0,0 @@

1−name: Publish crate

2−

3−on: workflow_dispatch

4−

5−jobs:

6− publish-crate:

7− runs-on: ubuntu-latest

8− steps:

9− - name: Checkout

10− uses: actions/checkout@v2

11− - name: Select Rust toolchain

12− uses: actions-rs/toolchain@v1

13− with:

14− toolchain: stable

15− - name: Publish to crates.io

16− uses: actions-rs/cargo@v1

17− with:

18− command: publish

19− env:

20− CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.github/workflows/publish-docker.yamldeleted−25

@@ -1,25 +0,0 @@

1−name: Publish Docker image

2−

3−on:

4− workflow_dispatch:

5− inputs:

6− version:

7− description: "Version (without 'v' prefix)"

8− required: true

9− type: string

10−

11−jobs:

12− publish-docker:

13− runs-on: ubuntu-latest

14− steps:

15− - name: Checkout

16− uses: actions/checkout@v2

17− - name: Log in to Docker

18− env:

19− DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}

20− run: |

21− docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD

22− - name: Build Docker image

23− run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest

24− - name: Push Docker image

25− run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape

Cargo.toml+3−3

@@ -1,6 +1,6 @@

1[package]

2name = "reshape"

3−version = "0.7.0"

3+version = "0.6.1"

4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"

5homepage = "https://github.com/fabianlindfors/reshape"

6documentation = "https://github.com/fabianlindfors/reshape"

@@ -9,7 +9,7 @@ license = "MIT"

9keywords = ["postgres", "migrations"]

10edition = "2021"

11authors = ["Fabian Lindfors"]

12−rust-version = "1.70"

12+rust-version = "1.58"

14[dependencies]

15postgres = { version = "0.19.2", features = ["with-serde_json-1"] }

@@ -23,4 +23,4 @@ version = "3.0.0"

23colored = "2"

24rand = "0.8"

25dotenv = "0.15.0"

26−lexical-sort = "0.3.1"

26+lexical-sort = "0.3.1"

Dockerfile+1−1

@@ -1,4 +1,4 @@

1−FROM rust:1.70.0 AS builder

1+FROM rust:1.58.0 AS builder

2WORKDIR /usr/src/reshape

3COPY . .

4RUN cargo build --release

README.md+13−66

@@ -2,7 +2,7 @@

3[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)

5−> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!

5+> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible

7Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).

@@ -243,7 +243,6 @@ primary_key = ["user_id"]

243 type = "TEXT"

244

245 # Backfill from `users` table and copy `users.email` to `user_email` column

246− # This will perform an upsert based on the primary key to avoid duplicate rows

247 [actions.up]

246 [actions.up]

248 table = "users"

247 table = "users"

249 values = { user_id = "id", user_email = "email" }

248 values = { user_id = "id", user_email = "email" }

@@ -500,8 +499,8 @@ column = "email"

500 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

499 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

501 [actions.down]

500 [actions.down]

502 table = "profiles"

501 table = "profiles"

503− value = "profiles.email"

502+ value = "email"

504− where = "users.id = profiles.user_id"

503+ where = "id = user_id"

505```

504```

506

505

507### Indices

506### Indices

@@ -622,26 +621,11 @@ abort = """

622

621

623### Complex changes across tables

622### Complex changes across tables

624

623

625−The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.

624+The `up` and `down` options available for many actions can also perform more complex changes that span tables.

626

625

627_Example: move `email` column from `users` to `profiles` table_

626_Example: move `email` column from `users` to `profiles` table_

628

627

629```toml

628```toml

630−[[actions]]

631−type = "add_column"

632−table = "profiles"

633−

634− [actions.column]

635− name = "email"

636− type = "TEXT"

637− nullable = false

638−

639− # When `users` is updated in the old schema, we write the email value to `profiles`

640− [actions.up]

641− table = "users"

642− value = "users.email"

643− where = "profiles.user_id = users.id"

644−

645[[actions]]

629[[actions]]

646type = "remove_column"

630type = "remove_column"

647table = "users"

631table = "users"

@@ -650,60 +634,23 @@ column = "email"

650 # When `profiles` is changed in the new schema, we write the email address back to the removed column

634 # When `profiles` is changed in the new schema, we write the email address back to the removed column

651 [actions.down]

635 [actions.down]

652 table = "profiles"

636 table = "profiles"

653− value = "profiles.email"

637+ value = "email"

654− where = "users.id = profiles.user_id"

638+ where = "id = user_id"

655−```

656−

657−_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_

658

639

659−```toml

660−# Add `user_account_connections` as a junction table

661[[actions]]

640[[actions]]

662−type = "create_table"

641+type = "add_column"

663−name = "user_account_connections"

642+table = "profiles"

664−primary_key = ["account_id", "user_id"]

665−

666− [[actions.columns]]

667− name = "account_id"

668− type = "INTEGER"

669−

670− [[actions.columns]]

671− name = "user_id"

672− type = "INTEGER"

673

643

674− # `role` is currently stored directly on the `users` table but is part of the relationship

675− [[actions.columns]]

644+ [actions.column]

676− name = "role"

645+ name = "email"

677 type = "TEXT"

646 type = "TEXT"

678 nullable = false

647 nullable = false

679

648

680− # Backfill the new table from `users` and uppercase the `role`

649+ # When `users` is updated in the old schema, we write the email value to `profiles`

681 [actions.up]

650 [actions.up]

682 table = "users"

651 table = "users"

683− values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

652+ value = "email"

684− where = "user_account_connections.user_id = users.id"

685−

686−[[actions]]

687−type = "remove_column"

688−table = "users"

689−column = "account_id"

690−

691− # When `user_account_connections` is updated, we write the `account_id` back to `users`

692− [actions.down]

693− table = "user_account_connections"

694− value = "user_account_connections.account_id"

695− where = "users.id = user_account_connections.user_id"

653+ where = "user_id = id"

696−

697−[[actions]]

698−type = "remove_column"

699−table = "users"

700−column = "account_role"

701−

702− # When `user_account_connections` is updated, we write the lowercase role back to `users`

703− [actions.down]

704− table = "user_account_connections"

705− value = "LOWER(user_account_connections.role)"

706− where = "users.id = user_account_connections.user_id"

707```

654```

708

655

709## Commands and options

656## Commands and options

src/migrations/add_column.rs+25−117

@@ -3,7 +3,7 @@ use crate::{

3 db::{Conn, Transaction},

4 schema::Schema,

5};

6−use anyhow::{bail, Context};

6+use anyhow::Context;

7use serde::{Deserialize, Serialize};

9#[derive(Serialize, Deserialize, Debug)]

@@ -43,15 +43,6 @@ impl AddColumn {

43 )

44 }

46− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

47− format!(

48− "{}_add_column_{}_{}_rev",

49− ctx.prefix(),

50− self.table,

51− self.column.name

52− )

53− }

54−

55 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

46 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56 format!(

47 format!(

57 "{}_add_column_not_null_{}_{}",

48 "{}_add_column_not_null_{}_{}",

@@ -81,7 +72,7 @@ impl Action for AddColumn {

81 let temp_column_name = self.temp_column_name(ctx);

72 let temp_column_name = self.temp_column_name(ctx);

83 let mut definition_parts = vec![

74 let mut definition_parts = vec![

84− format!("\"{}\"", temp_column_name.to_string()),

75+ temp_column_name.to_string(),

85 self.column.data_type.to_string(),

76 self.column.data_type.to_string(),

86 ];

77 ];

@@ -106,21 +97,21 @@ impl Action for AddColumn {

106 );

97 );

107 db.run(&query).context("failed to add column")?;

98 db.run(&query).context("failed to add column")?;

108

109− let declarations: Vec<String> = table

110− .columns

111− .iter()

112− .map(|column| {

113− format!(

114− "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",

115− table = table.real_name,

116− alias = column.name,

117− real_name = column.real_name,

118− )

119− })

120− .collect();

121−

122 if let Some(up) = &self.up {

100 if let Some(up) = &self.up {

123 if let Transformation::Simple(up) = up {

101 if let Transformation::Simple(up) = up {

102+ let declarations: Vec<String> = table

103+ .columns

104+ .iter()

105+ .map(|column| {

106+ format!(

107+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

108+ table = table.real_name,

109+ alias = column.name,

110+ real_name = column.real_name,

111+ )

112+ })

113+ .collect();

114+

124 // Add triggers to fill in values as they are inserted/updated

115 // Add triggers to fill in values as they are inserted/updated

125 let query = format!(

116 let query = format!(

126 r#"

117 r#"

@@ -131,7 +122,7 @@ impl Action for AddColumn {

131 DECLARE

122 DECLARE

132 {declarations}

123 {declarations}

133 BEGIN

124 BEGIN

134− NEW."{temp_column_name}" = {up};

125+ NEW.{temp_column_name} = {up};

135 END;

126 END;

136 END IF;

127 END IF;

137 RETURN NEW;

128 RETURN NEW;

@@ -160,20 +151,15 @@ impl Action for AddColumn {

160 r#where,

151 r#where,

161 } = up

152 } = up

162 {

153 {

163− let existing_schema_name = match &ctx.existing_schema_name {

164− Some(name) => name,

165− None => bail!("can't use update without previous migration"),

166− };

167−

168 let from_table = schema.get_table(db, &from_table)?;

154 let from_table = schema.get_table(db, &from_table)?;

169

155

170− let from_table_assignments: Vec<String> = from_table

156+ let declarations: Vec<String> = from_table

171 .columns

157 .columns

172 .iter()

158 .iter()

173 .map(|column| {

159 .map(|column| {

174 format!(

160 format!(

175− "{table}.{alias} = NEW.{real_name};",

161+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

176− table = from_table.name,

162+ table = from_table.real_name,

177 alias = column.name,

163 alias = column.name,

178 real_name = column.real_name,

164 real_name = column.real_name,

179 )

165 )

@@ -189,18 +175,11 @@ impl Action for AddColumn {

189 BEGIN

175 BEGIN

190 IF NOT reshape.is_new_schema() THEN

176 IF NOT reshape.is_new_schema() THEN

191 DECLARE

177 DECLARE

192− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

178+ {declarations}

193 BEGIN

179 BEGIN

194− {assignments}

195−

196− -- Don't trigger reverse trigger when making this update

197− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

198−

199 UPDATE public."{changed_table_real}"

180 UPDATE public."{changed_table_real}"

200 SET "{temp_column_name}" = {value}

181 SET "{temp_column_name}" = {value}

201 WHERE {where};

182 WHERE {where};

202−

203− perform set_config('reshape.disable_triggers', '', TRUE);

204 END;

183 END;

205 END IF;

184 END IF;

206 RETURN NEW;

185 RETURN NEW;

@@ -210,81 +189,14 @@ impl Action for AddColumn {

210 DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

189 DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

211 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

190 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

212 "#,

191 "#,

213− assignments = from_table_assignments.join("\n"),

214 changed_table_real = table.real_name,

192 changed_table_real = table.real_name,

215− from_table = from_table.name,

216 from_table_real = from_table.real_name,

193 from_table_real = from_table.real_name,

217 trigger_name = self.trigger_name(ctx),

194 trigger_name = self.trigger_name(ctx),

218− // declarations = from_table_declarations.join("\n"),

195+ declarations = declarations.join("\n"),

219 temp_column_name = temp_column_name,

196 temp_column_name = temp_column_name,

220 );

197 );

221 db.run(&query).context("failed to create up trigger")?;

198 db.run(&query).context("failed to create up trigger")?;

222

199

223− let from_table_columns = from_table

224− .columns

225− .iter()

226− .map(|column| format!("{} as {}", column.real_name, column.name))

227− .collect::<Vec<String>>()

228− .join(", ");

229−

230− let changed_table_assignments: Vec<String> = table

231− .columns

232− .iter()

233− .map(|column| {

234− format!(

235− "{table}.{alias} := NEW.{real_name};",

236− table = table.name,

237− alias = column.name,

238− real_name = column.real_name,

239− )

240− })

241− .collect();

242−

243− // Add triggers to fill in values as they are inserted/updated

244− let query = format!(

245− r#"

246− CREATE OR REPLACE FUNCTION {trigger_name}()

247− RETURNS TRIGGER AS $$

248− #variable_conflict use_variable

249− BEGIN

250− IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

251− DECLARE

252− {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;

253− __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;

254− BEGIN

255− {changed_table_assignments}

256−

257− SELECT {from_table_columns}

258− INTO __temp_row

259− FROM migration_{existing_schema_name}.{from_table} {from_table}

260− WHERE {where};

261−

262− DECLARE

263− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

264− BEGIN

265− {from_table} = __temp_row;

266− NEW.{temp_column_name} = {value};

267− END;

268− END;

269− END IF;

270− RETURN NEW;

271− END

272− $$ language 'plpgsql';

273−

274− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

275− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

276− "#,

277− changed_table_assignments = changed_table_assignments.join("\n"),

278− changed_table_real = table.real_name,

279− changed_table = table.name,

280− from_table = from_table.name,

281− trigger_name = self.reverse_trigger_name(ctx),

282− temp_column_name = temp_column_name,

283− // declarations = declarations.join("\n"),

284− );

285− db.run(&query)

286− .context("failed to create reverse up trigger")?;

287−

288 // Backfill values in batches by touching the from table

200 // Backfill values in batches by touching the from table

289 common::batch_touch_rows(db, &from_table.real_name, None)

201 common::batch_touch_rows(db, &from_table.real_name, None)

290 .context("failed to batch update existing rows")?;

202 .context("failed to batch update existing rows")?;

@@ -298,10 +210,10 @@ impl Action for AddColumn {

298 if !self.column.nullable {

210 if !self.column.nullable {

299 let query = format!(

211 let query = format!(

300 r#"

212 r#"

301− ALTER TABLE "{table}"

213+ ALTER TABLE "{table}"

302− ADD CONSTRAINT "{constraint_name}"

214+ ADD CONSTRAINT "{constraint_name}"

303− CHECK ("{column}" IS NOT NULL) NOT VALID

215+ CHECK ("{column}" IS NOT NULL) NOT VALID

304− "#,

216+ "#,

305 table = self.table,

217 table = self.table,

306 constraint_name = self.not_null_constraint_name(ctx),

218 constraint_name = self.not_null_constraint_name(ctx),

307 column = temp_column_name,

219 column = temp_column_name,

@@ -324,10 +236,8 @@ impl Action for AddColumn {

324 let query = format!(

236 let query = format!(

325 r#"

237 r#"

326 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

238 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

327− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

328 "#,

239 "#,

329 trigger_name = self.trigger_name(ctx),

240 trigger_name = self.trigger_name(ctx),

330− reverse_trigger_name = self.reverse_trigger_name(ctx),

331 );

241 );

332 transaction

242 transaction

333 .run(&query)

243 .run(&query)

@@ -419,10 +329,8 @@ impl Action for AddColumn {

419 let query = format!(

329 let query = format!(

420 r#"

330 r#"

421 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

331 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

422− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

423 "#,

332 "#,

424 trigger_name = self.trigger_name(ctx),

333 trigger_name = self.trigger_name(ctx),

425− reverse_trigger_name = self.reverse_trigger_name(ctx),

426 );

334 );

427 db.run(&query).context("failed to drop up trigger")?;

335 db.run(&query).context("failed to drop up trigger")?;

428

336

src/migrations/remove_column.rs+30−247

@@ -3,7 +3,7 @@ use crate::{

3 db::{Conn, Transaction},

4 schema::Schema,

5};

6−use anyhow::{anyhow, bail, Context};

6+use anyhow::{bail, Context};

7use serde::{Deserialize, Serialize};

9#[derive(Serialize, Deserialize, Debug)]

@@ -33,33 +33,6 @@ impl RemoveColumn {

33 self.column

34 )

35 }

36−

37− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

38− format!(

39− "{}_remove_column_{}_{}_rev",

40− ctx.prefix(),

41− self.table,

42− self.column

43− )

44− }

45−

46− fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

47− format!(

48− "{}_remove_column_{}_{}_nn",

49− ctx.prefix(),

50− self.table,

51− self.column

52− )

53− }

54−

55− fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56− format!(

57− "{}_add_column_not_null_{}_{}",

58− ctx.prefix(),

59− self.table,

60− self.column

61− )

62− }

63}

36}

65#[typetag::serde(name = "remove_column")]

38#[typetag::serde(name = "remove_column")]

@@ -78,9 +51,6 @@ impl Action for RemoveColumn {

78 schema: &Schema,

51 schema: &Schema,

79 ) -> anyhow::Result<()> {

52 ) -> anyhow::Result<()> {

80 let table = schema.get_table(db, &self.table)?;

53 let table = schema.get_table(db, &self.table)?;

81− let column = table

82− .get_column(&self.column)

83− .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;

85 // Add down trigger

55 // Add down trigger

86 if let Some(down) = &self.down {

56 if let Some(down) = &self.down {

@@ -139,76 +109,22 @@ impl Action for RemoveColumn {

139

109

140 let from_table = schema.get_table(db, &from_table)?;

110 let from_table = schema.get_table(db, &from_table)?;

141

111

142− let maybe_null_check = if !column.nullable {

112+ let declarations: Vec<String> = from_table

143− // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

144− // We will add a null check to the down function on the new schema below as well to cover both cases.

145− // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

146− // NOT NULL is not checked at the end of a transaction, but immediately upon update.

147− let query = format!(

148− r#"

149− CREATE OR REPLACE FUNCTION {trigger_name}()

150− RETURNS TRIGGER AS $$

151− BEGIN

152− IF NOT reshape.is_new_schema() THEN

153− IF NEW.{column} IS NULL THEN

154− RAISE EXCEPTION '{column} can not be null';

155− END IF;

156− END IF;

157− RETURN NEW;

158− END

159− $$ language 'plpgsql';

160−

161− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

162−

163− CREATE CONSTRAINT TRIGGER "{trigger_name}"

164− AFTER INSERT OR UPDATE

165− ON "{table}"

166− FOR EACH ROW

167− EXECUTE PROCEDURE {trigger_name}();

168− "#,

169− table = self.table,

170− trigger_name = self.not_null_constraint_trigger_name(ctx),

171− column = self.column,

172− );

173− db.run(&query)

174− .context("failed to create null constraint trigger")?;

175−

176− db.run(&format!(

177− r#"

178− ALTER TABLE {table}

179− ALTER COLUMN {column}

180− DROP NOT NULL

181− "#,

182− table = self.table,

183− column = self.column

184− ))

185− .context("failed to remove column not null constraint")?;

186−

187− format!(

188− r#"

189− IF {value} IS NULL THEN

190− RAISE EXCEPTION '{column_name} can not be null';

191− END IF;

192− "#,

193− column_name = self.column,

194− )

195− } else {

196− "".to_string()

197− };

198−

199− let into_variables = from_table

200 .columns

113 .columns

201 .iter()

114 .iter()

202 .map(|column| {

115 .map(|column| {

203 format!(

116 format!(

204− "NEW.{real_name} AS {alias}",

117+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

118+ table = from_table.real_name,

205 alias = column.name,

119 alias = column.name,

206 real_name = column.real_name,

120 real_name = column.real_name,

207 )

121 )

208 })

122 })

209− .collect::<Vec<String>>()

123+ .collect();

210− .join(", ");

211

124

125+ // TODO: If column is NOT NULL, remove the constraint and perform a NULL check in some other way.

126+ // Otherwise, it's not possible to update the NOT NULL column from another table even in the same transaction.

127+ // Either make the NULL check in here or maybe use a constraint trigger: https://www.postgresql.org/docs/9.0/sql-createconstraint.html.

212 let query = format!(

128 let query = format!(

213 r#"

129 r#"

214 CREATE OR REPLACE FUNCTION {trigger_name}()

130 CREATE OR REPLACE FUNCTION {trigger_name}()

@@ -217,21 +133,11 @@ impl Action for RemoveColumn {

217 BEGIN

133 BEGIN

218 IF reshape.is_new_schema() THEN

134 IF reshape.is_new_schema() THEN

219 DECLARE

135 DECLARE

220− {from_table} record;

136+ {declarations}

221 BEGIN

137 BEGIN

222− SELECT {into_variables}

223− INTO {from_table};

224−

225− {maybe_null_check}

226−

227− -- Don't trigger reverse trigger when making this update

228− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

229−

230 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

138 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

231 SET "{column_name}" = {value}

139 SET "{column_name}" = {value}

232 WHERE {where};

140 WHERE {where};

233−

234− perform set_config('reshape.disable_triggers', '', TRUE);

235 END;

141 END;

236 END IF;

142 END IF;

237 RETURN NEW;

143 RETURN NEW;

@@ -242,80 +148,12 @@ impl Action for RemoveColumn {

242 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

148 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

243 "#,

149 "#,

244 changed_table = self.table,

150 changed_table = self.table,

245− from_table = from_table.name,

246 from_table_real = from_table.real_name,

151 from_table_real = from_table.real_name,

247 column_name = self.column,

152 column_name = self.column,

248 trigger_name = self.trigger_name(ctx),

153 trigger_name = self.trigger_name(ctx),

154+ declarations = declarations.join("\n"),

249 );

155 );

250 db.run(&query).context("failed to create down trigger")?;

156 db.run(&query).context("failed to create down trigger")?;

251−

252− let changed_into_variables = table

253− .columns

254− .iter()

255− .map(|column| {

256− format!(

257− "NEW.{real_name} AS {alias}",

258− alias = column.name,

259− real_name = column.real_name,

260− )

261− })

262− .collect::<Vec<String>>()

263− .join(", ");

264−

265− let from_table_columns = from_table

266− .columns

267− .iter()

268− .map(|column| format!("{} as {}", column.real_name, column.name))

269− .collect::<Vec<String>>()

270− .join(", ");

271−

272− let query = format!(

273− r#"

274− CREATE OR REPLACE FUNCTION {trigger_name}()

275− RETURNS TRIGGER AS $$

276− #variable_conflict use_variable

277− BEGIN

278− IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

279− DECLARE

280− {changed_table} record;

281− __temp_row record;

282− BEGIN

283− SELECT {changed_into_variables}

284− INTO {changed_table};

285−

286− SELECT *

287− INTO __temp_row

288− FROM (

289− SELECT {from_table_columns}

290− FROM public.{from_table_real}

291− ) {from_table}

292− WHERE {where};

293−

294− DECLARE

295− {from_table} record;

296− BEGIN

297− {from_table} := __temp_row;

298− NEW.{column_name_real} = {value};

299− END;

300− END;

301− END IF;

302− RETURN NEW;

303− END

304− $$ language 'plpgsql';

305−

306− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

307− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

308− "#,

309− changed_table = table.name,

310− changed_table_real = table.real_name,

311− from_table = from_table.name,

312− from_table_real = from_table.real_name,

313− column_name_real = column.real_name,

314− trigger_name = self.reverse_trigger_name(ctx),

315− // declarations = declarations.join("\n"),

316− );

317− db.run(&query)

318− .context("failed to create reverse down trigger")?;

319 }

157 }

320 }

158 }

321

159

@@ -341,20 +179,25 @@ impl Action for RemoveColumn {

341 }

179 }

342

180

343 // Remove column, function and trigger

181 // Remove column, function and trigger

182+ let trigger_table = match &self.down {

183+ Some(Transformation::Update {

184+ table,

185+ value: _,

186+ r#where: _,

187+ }) => table,

188+ _ => &self.table,

189+ };

344 let query = format!(

190 let query = format!(

345 r#"

191 r#"

346 ALTER TABLE "{table}"

192 ALTER TABLE "{table}"

347 DROP COLUMN IF EXISTS "{column}";

193 DROP COLUMN IF EXISTS "{column}";

348

194

349− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

195+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

350− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

351− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

196+ DROP FUNCTION IF EXISTS "{trigger_name}";

352 "#,

197 "#,

353 table = self.table,

198 table = self.table,

354 column = self.column,

199 column = self.column,

355 trigger_name = self.trigger_name(ctx),

200 trigger_name = self.trigger_name(ctx),

356− reverse_trigger_name = self.reverse_trigger_name(ctx),

357− null_trigger_name = self.not_null_constraint_trigger_name(ctx),

358 );

201 );

359 db.run(&query)

202 db.run(&query)

360 .context("failed to drop column and down trigger")?;

203 .context("failed to drop column and down trigger")?;

@@ -371,81 +214,21 @@ impl Action for RemoveColumn {

371 }

214 }

372

215

373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

216 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

374− // We might have temporaily removed the NOT NULL check and have to reinstate it

375− let has_not_null_function = !db

376− .query_with_params(

377− "

378− SELECT routine_name

379− FROM information_schema.routines

380− WHERE routine_schema = 'public'

381− AND routine_name = $1

382− ",

383− &[&self.not_null_constraint_trigger_name(ctx)],

384− )

385− .context("failed to get any NOT NULL function")?

386− .is_empty();

387−

388− if has_not_null_function {

389− // Make column NOT NULL again without taking any long lived locks with a temporary constraint

390− let query = format!(

391− r#"

392− ALTER TABLE "{table}"

393− ADD CONSTRAINT "{constraint_name}"

394− CHECK ("{column}" IS NOT NULL) NOT VALID

395− "#,

396− table = self.table,

397− constraint_name = self.not_null_constraint_name(ctx),

398− column = self.column,

399− );

400− db.run(&query)

401− .context("failed to add NOT NULL constraint")?;

402−

403− let query = format!(

404− r#"

405− ALTER TABLE "{table}"

406− VALIDATE CONSTRAINT "{constraint_name}"

407− "#,

408− table = self.table,

409− constraint_name = self.not_null_constraint_name(ctx),

410− );

411− db.run(&query)

412− .context("failed to validate NOT NULL constraint")?;

413−

414− // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above

415− db.run(&format!(

416− r#"

417− ALTER TABLE {table}

418− ALTER COLUMN {column}

419− SET NOT NULL

420− "#,

421− table = self.table,

422− column = self.column

423− ))

424− .context("failed to reinstate column NOT NULL")?;

425−

426− // Drop the temporary constraint

427− let query = format!(

428− r#"

429− ALTER TABLE "{table}"

430− DROP CONSTRAINT "{constraint_name}"

431− "#,

432− table = self.table,

433− constraint_name = self.not_null_constraint_name(ctx),

434− );

435− db.run(&query)

436− .context("failed to drop NOT NULL constraint")?;

437− }

438−

439 // Remove function and trigger

217 // Remove function and trigger

218+ let trigger_table = match &self.down {

219+ Some(Transformation::Update {

220+ table,

221+ value: _,

222+ r#where: _,

223+ }) => table,

224+ _ => &self.table,

225+ };

440 db.run(&format!(

226 db.run(&format!(

441 r#"

227 r#"

442− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

228+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

443− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

444− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

229+ DROP FUNCTION IF EXISTS "{trigger_name}";

445 "#,

230 "#,

446 trigger_name = self.trigger_name(ctx),

231 trigger_name = self.trigger_name(ctx),

447− reverse_trigger_name = self.reverse_trigger_name(ctx),

448− null_trigger_name = self.not_null_constraint_trigger_name(ctx),

449 ))

232 ))

450 .context("failed to drop down trigger")?;

233 .context("failed to drop down trigger")?;

451

234

tests/add_column.rs+2−2

@@ -308,8 +308,8 @@ fn add_column_with_complex_up() {

308

309 [actions.up]

310 table = "users"

311− value = "users.email"

311+ value = "email"

312− where = "profiles.user_id = users.id"

312+ where = "user_id = id"

313 "#,

314 );

315

tests/complex.rs+5−174

@@ -1,165 +1,6 @@

1mod common;

2use common::Test;

4−#[test]

5−fn move_column_between_tables() {

6− let mut test = Test::new("Move column between tables");

7−

8− test.first_migration(

9− r#"

10− name = "create_tables"

11−

12− [[actions]]

13− type = "create_table"

14− name = "users"

15− primary_key = ["id"]

16−

17− [[actions.columns]]

18− name = "id"

19− type = "INTEGER"

20−

21− [[actions.columns]]

22− name = "email"

23− type = "TEXT"

24−

25− [[actions]]

26− type = "create_table"

27− name = "profiles"

28− primary_key = ["id"]

29−

30− [[actions.columns]]

31− name = "id"

32− type = "INTEGER"

33−

34− [[actions.columns]]

35− name = "user_id"

36− type = "INTEGER"

37− nullable = false

38− "#,

39− );

40−

41− test.second_migration(

42− r#"

43− name = "move_email_column"

44−

45− [[actions]]

46− type = "add_column"

47− table = "profiles"

48−

49− [actions.column]

50− name = "email"

51− type = "TEXT"

52− nullable = false

53−

54− # When `users` is updated in the old schema, we write the email value to `profiles`

55− # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated

56− [actions.up]

57− table = "users"

58− value = "users.email"

59− where = "profiles.user_id = users.id"

60−

61− [[actions]]

62− type = "remove_column"

63− table = "users"

64− column = "email"

65−

66− # When `profiles` is changed in the new schema, we write the email address back to the removed column

67− [actions.down]

68− table = "profiles"

69− value = "profiles.email"

70− where = "users.id = profiles.user_id"

71− "#,

72− );

73−

74− test.after_first(|db| {

75− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")

76− .unwrap();

77− db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")

78− .unwrap();

79−

80− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")

81− .unwrap();

82− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")

83− .unwrap();

84− });

85−

86− test.intermediate(|old_db, new_db| {

87− // Ensure emails were backfilled into profiles

88− let profiles_emails: Vec<String> = new_db

89− .query(

90− r#"

91− SELECT email

92− FROM profiles

93− ORDER BY id

94− "#,

95− &[],

96− )

97− .unwrap()

98− .iter()

99− .map(|row| row.get("email"))

100− .collect();

101− assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);

102−

103− // Ensure insert in old schema updates new

104− old_db

105− .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")

106− .unwrap();

107− old_db

108− .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")

109− .unwrap();

110− let new_email: String = new_db

111− .query("SELECT email FROM profiles WHERE id = 3", &[])

112− .unwrap()

113− .first()

114− .map(|row| row.get("email"))

115− .unwrap();

116− assert_eq!("test3@test.com", new_email);

117−

118− // Ensure updates in old schema updates new

119− old_db

120− .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")

121− .unwrap();

122− let new_email: String = new_db

123− .query("SELECT email FROM profiles WHERE id = 3", &[])

124− .unwrap()

125− .first()

126− .map(|row| row.get("email"))

127− .unwrap();

128− assert_eq!("test3+updated@test.com", new_email);

129−

130− // Ensure insert in new schema updates old

131− new_db

132− .simple_query(

133− "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",

134− )

135− .unwrap();

136− new_db

137− .simple_query("INSERT INTO users (id) VALUES (4)")

138− .unwrap();

139− let old_email: String = old_db

140− .query("SELECT email FROM users WHERE id = 4", &[])

141− .unwrap()

142− .first()

143− .map(|row| row.get("email"))

144− .unwrap();

145− assert_eq!("test4@test.com", old_email);

146−

147− // Ensure update in new schema updates old

148− new_db

149− .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")

150− .unwrap();

151− let old_email: String = old_db

152− .query("SELECT email FROM users WHERE id = 4", &[])

153− .unwrap()

154− .first()

155− .map(|row| row.get("email"))

156− .unwrap();

157− assert_eq!("test4+updated@test.com", old_email);

158− });

159−

160− test.run();

161−}

162−

163#[test]

4#[test]

164fn extract_relation_into_new_table() {

5fn extract_relation_into_new_table() {

165 let mut test = Test::new("Extract relation into new table");

6 let mut test = Test::new("Extract relation into new table");

@@ -189,12 +30,10 @@ fn extract_relation_into_new_table() {

189 [[actions.columns]]

30 [[actions.columns]]

190 name = "account_id"

31 name = "account_id"

191 type = "INTEGER"

32 type = "INTEGER"

192− nullable = false

193

194 [[actions.columns]]

34 [[actions.columns]]

195 name = "account_role"

35 name = "account_role"

196 type = "TEXT"

36 type = "TEXT"

197− nullable = false

198 "#,

37 "#,

199 );

38 );

200

@@ -218,12 +57,11 @@ fn extract_relation_into_new_table() {

218 [[actions.columns]]

57 [[actions.columns]]

219 name = "role"

58 name = "role"

220 type = "TEXT"

59 type = "TEXT"

221− nullable = false

222

223 [actions.up]

61 [actions.up]

224 table = "users"

62 table = "users"

225 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

63 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

226− where = "user_account_connections.user_id = users.id"

64+ where = "user_id = id"

227

228 [[actions]]

66 [[actions]]

229 type = "remove_column"

67 type = "remove_column"

@@ -232,8 +70,8 @@ fn extract_relation_into_new_table() {

232

233 [actions.down]

71 [actions.down]

234 table = "user_account_connections"

72 table = "user_account_connections"

235− value = "user_account_connections.account_id"

73+ value = "account_id"

236− where = "users.id = user_account_connections.user_id"

74+ where = "id = user_id"

237

238 [[actions]]

76 [[actions]]

239 type = "remove_column"

77 type = "remove_column"

@@ -242,8 +80,8 @@ fn extract_relation_into_new_table() {

242

243 [actions.down]

81 [actions.down]

244 table = "user_account_connections"

82 table = "user_account_connections"

245− value = "LOWER(user_account_connections.role)"

83+ value = "LOWER(role)"

246− where = "users.id = user_account_connections.user_id"

84+ where = "id = user_id"

247 "#,

85 "#,

248 );

86 );

249

@@ -296,13 +134,6 @@ fn extract_relation_into_new_table() {

296 == 1

134 == 1

297 );

135 );

298

136

299− // Ensure NOT NULL constraint still applies to old schema

300− let result = old_db

301− .simple_query(

302− "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",

303− );

304− assert!(result.is_err());

305−

306 // Ensure updated user role in old schema updates connection in new schema

137 // Ensure updated user role in old schema updates connection in new schema

307 old_db

138 old_db

308 .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")

139 .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")

tests/remove_column.rs+2−2

@@ -179,8 +179,8 @@ fn remove_column_with_complex_down() {

179

180 [actions.down]

181 table = "profiles"

182− value = "profiles.email"

182+ value = "email"

183− where = "users.id = profiles.user_id"

183+ where = "id = user_id"

184 "#,

185 );

186