de44fdd..main · reshape

git.iko.soy/reshape

on main help sign in

.github/workflows/lint.yaml+3−3

since splitsnapshots

unifiedsplit

@@ -15,7 +15,7 @@ jobs:

15 runs-on: ubuntu-latest

16 steps:

17 - name: Checkout

18− uses: actions/checkout@v2

18+ uses: actions/checkout@v4

19 - name: Select Rust toolchain with Clippy

20 uses: actions-rs/toolchain@v1

21 with:

@@ -23,6 +23,6 @@ jobs:

23 components: clippy

24 override: true

25 - name: Use cache for Rust dependencies

26− uses: Swatinem/rust-cache@v1

26+ uses: Swatinem/rust-cache@v2

27 - name: Lint using Clippy

28− run: cargo clippy

28+ run: cargo clippy

.github/workflows/publish-crate.yamlnew file+20

@@ -0,0 +1,20 @@

1+name: Publish crate

3+on: workflow_dispatch

5+jobs:

6+ publish-crate:

7+ runs-on: ubuntu-latest

8+ steps:

9+ - name: Checkout

10+ uses: actions/checkout@v2

11+ - name: Select Rust toolchain

12+ uses: actions-rs/toolchain@v1

13+ with:

14+ toolchain: stable

15+ - name: Publish to crates.io

16+ uses: actions-rs/cargo@v1

17+ with:

18+ command: publish

19+ env:

20+ CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.github/workflows/publish-docker.yamlnew file+25

@@ -0,0 +1,25 @@

1+name: Publish Docker image

3+on:

4+ workflow_dispatch:

5+ inputs:

6+ version:

7+ description: "Version (without 'v' prefix)"

8+ required: true

9+ type: string

10+

11+jobs:

12+ publish-docker:

13+ runs-on: ubuntu-latest

14+ steps:

15+ - name: Checkout

16+ uses: actions/checkout@v2

17+ - name: Log in to Docker

18+ env:

19+ DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}

20+ run: |

21+ docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD

22+ - name: Build Docker image

23+ run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest

24+ - name: Push Docker image

25+ run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape

Cargo.toml+3−3

@@ -1,6 +1,6 @@

1[package]

2name = "reshape"

3−version = "0.6.1"

3+version = "0.7.0"

4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"

5homepage = "https://github.com/fabianlindfors/reshape"

6documentation = "https://github.com/fabianlindfors/reshape"

@@ -9,7 +9,7 @@ license = "MIT"

9keywords = ["postgres", "migrations"]

10edition = "2021"

11authors = ["Fabian Lindfors"]

12−rust-version = "1.58"

12+rust-version = "1.70"

14[dependencies]

15postgres = { version = "0.19.2", features = ["with-serde_json-1"] }

@@ -23,4 +23,4 @@ version = "3.0.0"

23colored = "2"

24rand = "0.8"

25dotenv = "0.15.0"

26−lexical-sort = "0.3.1"

26+lexical-sort = "0.3.1"

Dockerfile+1−1

@@ -1,4 +1,4 @@

1−FROM rust:1.58.0 AS builder

1+FROM rust:1.70.0 AS builder

2WORKDIR /usr/src/reshape

3COPY . .

4RUN cargo build --release

README.md+21−20

@@ -500,8 +500,8 @@ column = "email"

500 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

501 [actions.down]

502 table = "profiles"

503− value = "email"

503+ value = "profiles.email"

504− where = "id = user_id"

504+ where = "users.id = profiles.user_id"

505```

506

507### Indices

@@ -627,17 +627,6 @@ The `up` and `down` options available when creating tables, adding columns and r

627_Example: move `email` column from `users` to `profiles` table_

628

629```toml

630−[[actions]]

631−type = "remove_column"

632−table = "users"

633−column = "email"

634−

635− # When `profiles` is changed in the new schema, we write the email address back to the removed column

636− [actions.down]

637− table = "profiles"

638− value = "email"

639− where = "id = user_id"

640−

641[[actions]]

630[[actions]]

642type = "add_column"

631type = "add_column"

643table = "profiles"

632table = "profiles"

@@ -650,8 +639,19 @@ table = "profiles"

650 # When `users` is updated in the old schema, we write the email value to `profiles`

639 # When `users` is updated in the old schema, we write the email value to `profiles`

651 [actions.up]

640 [actions.up]

652 table = "users"

641 table = "users"

653− value = "email"

642+ value = "users.email"

654− where = "user_id = id"

643+ where = "profiles.user_id = users.id"

644+

645+[[actions]]

646+type = "remove_column"

647+table = "users"

648+column = "email"

649+

650+ # When `profiles` is changed in the new schema, we write the email address back to the removed column

651+ [actions.down]

652+ table = "profiles"

653+ value = "profiles.email"

654+ where = "users.id = profiles.user_id"

655```

656

657_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_

@@ -675,12 +675,13 @@ primary_key = ["account_id", "user_id"]

675 [[actions.columns]]

676 name = "role"

677 type = "TEXT"

678+ nullable = false

678

679

679 # Backfill the new table from `users` and uppercase the `role`

680 # Backfill the new table from `users` and uppercase the `role`

680 [actions.up]

681 [actions.up]

681 table = "users"

682 table = "users"

682 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

683 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

683− where = "user_id = id"

684+ where = "user_account_connections.user_id = users.id"

684

685

685[[actions]]

686[[actions]]

686type = "remove_column"

687type = "remove_column"

@@ -690,8 +691,8 @@ column = "account_id"

690 # When `user_account_connections` is updated, we write the `account_id` back to `users`

691 # When `user_account_connections` is updated, we write the `account_id` back to `users`

691 [actions.down]

692 [actions.down]

692 table = "user_account_connections"

693 table = "user_account_connections"

693− value = "account_id"

694+ value = "user_account_connections.account_id"

694− where = "id = user_id"

695+ where = "users.id = user_account_connections.user_id"

695

696

696[[actions]]

697[[actions]]

697type = "remove_column"

698type = "remove_column"

@@ -701,8 +702,8 @@ column = "account_role"

701 # When `user_account_connections` is updated, we write the lowercase role back to `users`

702 # When `user_account_connections` is updated, we write the lowercase role back to `users`

702 [actions.down]

703 [actions.down]

703 table = "user_account_connections"

704 table = "user_account_connections"

704− value = "LOWER(role)"

705+ value = "LOWER(user_account_connections.role)"

705− where = "id = user_id"

706+ where = "users.id = user_account_connections.user_id"

706```

707```

707

708

708## Commands and options

709## Commands and options

src/migrations/add_column.rs+117−25

@@ -3,7 +3,7 @@ use crate::{

3 db::{Conn, Transaction},

4 schema::Schema,

5};

6−use anyhow::Context;

6+use anyhow::{bail, Context};

7use serde::{Deserialize, Serialize};

9#[derive(Serialize, Deserialize, Debug)]

@@ -43,6 +43,15 @@ impl AddColumn {

43 )

44 }

46+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

47+ format!(

48+ "{}_add_column_{}_{}_rev",

49+ ctx.prefix(),

50+ self.table,

51+ self.column.name

52+ )

53+ }

54+

46 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

55 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

47 format!(

56 format!(

48 "{}_add_column_not_null_{}_{}",

57 "{}_add_column_not_null_{}_{}",

@@ -72,7 +81,7 @@ impl Action for AddColumn {

72 let temp_column_name = self.temp_column_name(ctx);

81 let temp_column_name = self.temp_column_name(ctx);

74 let mut definition_parts = vec![

83 let mut definition_parts = vec![

75− temp_column_name.to_string(),

84+ format!("\"{}\"", temp_column_name.to_string()),

76 self.column.data_type.to_string(),

85 self.column.data_type.to_string(),

77 ];

86 ];

@@ -97,21 +106,21 @@ impl Action for AddColumn {

97 );

106 );

98 db.run(&query).context("failed to add column")?;

107 db.run(&query).context("failed to add column")?;

108

109+ let declarations: Vec<String> = table

110+ .columns

111+ .iter()

112+ .map(|column| {

113+ format!(

114+ "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",

115+ table = table.real_name,

116+ alias = column.name,

117+ real_name = column.real_name,

118+ )

119+ })

120+ .collect();

121+

100 if let Some(up) = &self.up {

122 if let Some(up) = &self.up {

101 if let Transformation::Simple(up) = up {

123 if let Transformation::Simple(up) = up {

102− let declarations: Vec<String> = table

103− .columns

104− .iter()

105− .map(|column| {

106− format!(

107− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

108− table = table.real_name,

109− alias = column.name,

110− real_name = column.real_name,

111− )

112− })

113− .collect();

114−

115 // Add triggers to fill in values as they are inserted/updated

124 // Add triggers to fill in values as they are inserted/updated

116 let query = format!(

125 let query = format!(

117 r#"

126 r#"

@@ -122,7 +131,7 @@ impl Action for AddColumn {

122 DECLARE

131 DECLARE

123 {declarations}

132 {declarations}

124 BEGIN

133 BEGIN

125− NEW.{temp_column_name} = {up};

134+ NEW."{temp_column_name}" = {up};

126 END;

135 END;

127 END IF;

136 END IF;

128 RETURN NEW;

137 RETURN NEW;

@@ -151,15 +160,20 @@ impl Action for AddColumn {

151 r#where,

160 r#where,

152 } = up

161 } = up

153 {

162 {

163+ let existing_schema_name = match &ctx.existing_schema_name {

164+ Some(name) => name,

165+ None => bail!("can't use update without previous migration"),

166+ };

167+

154 let from_table = schema.get_table(db, &from_table)?;

168 let from_table = schema.get_table(db, &from_table)?;

155

169

156− let declarations: Vec<String> = from_table

170+ let from_table_assignments: Vec<String> = from_table

157 .columns

171 .columns

158 .iter()

172 .iter()

159 .map(|column| {

173 .map(|column| {

160 format!(

174 format!(

161− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

175+ "{table}.{alias} = NEW.{real_name};",

162− table = from_table.real_name,

176+ table = from_table.name,

163 alias = column.name,

177 alias = column.name,

164 real_name = column.real_name,

178 real_name = column.real_name,

165 )

179 )

@@ -175,11 +189,18 @@ impl Action for AddColumn {

175 BEGIN

189 BEGIN

176 IF NOT reshape.is_new_schema() THEN

190 IF NOT reshape.is_new_schema() THEN

177 DECLARE

191 DECLARE

178− {declarations}

192+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

179 BEGIN

193 BEGIN

194+ {assignments}

195+

196+ -- Don't trigger reverse trigger when making this update

197+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

198+

180 UPDATE public."{changed_table_real}"

199 UPDATE public."{changed_table_real}"

181 SET "{temp_column_name}" = {value}

200 SET "{temp_column_name}" = {value}

182 WHERE {where};

201 WHERE {where};

202+

203+ perform set_config('reshape.disable_triggers', '', TRUE);

183 END;

204 END;

184 END IF;

205 END IF;

185 RETURN NEW;

206 RETURN NEW;

@@ -189,14 +210,81 @@ impl Action for AddColumn {

189 DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

210 DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

190 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

211 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

191 "#,

212 "#,

213+ assignments = from_table_assignments.join("\n"),

192 changed_table_real = table.real_name,

214 changed_table_real = table.real_name,

215+ from_table = from_table.name,

193 from_table_real = from_table.real_name,

216 from_table_real = from_table.real_name,

194 trigger_name = self.trigger_name(ctx),

217 trigger_name = self.trigger_name(ctx),

195− declarations = declarations.join("\n"),

218+ // declarations = from_table_declarations.join("\n"),

196 temp_column_name = temp_column_name,

219 temp_column_name = temp_column_name,

197 );

220 );

198 db.run(&query).context("failed to create up trigger")?;

221 db.run(&query).context("failed to create up trigger")?;

199

222

223+ let from_table_columns = from_table

224+ .columns

225+ .iter()

226+ .map(|column| format!("{} as {}", column.real_name, column.name))

227+ .collect::<Vec<String>>()

228+ .join(", ");

229+

230+ let changed_table_assignments: Vec<String> = table

231+ .columns

232+ .iter()

233+ .map(|column| {

234+ format!(

235+ "{table}.{alias} := NEW.{real_name};",

236+ table = table.name,

237+ alias = column.name,

238+ real_name = column.real_name,

239+ )

240+ })

241+ .collect();

242+

243+ // Add triggers to fill in values as they are inserted/updated

244+ let query = format!(

245+ r#"

246+ CREATE OR REPLACE FUNCTION {trigger_name}()

247+ RETURNS TRIGGER AS $$

248+ #variable_conflict use_variable

249+ BEGIN

250+ IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

251+ DECLARE

252+ {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;

253+ __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;

254+ BEGIN

255+ {changed_table_assignments}

256+

257+ SELECT {from_table_columns}

258+ INTO __temp_row

259+ FROM migration_{existing_schema_name}.{from_table} {from_table}

260+ WHERE {where};

261+

262+ DECLARE

263+ {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

264+ BEGIN

265+ {from_table} = __temp_row;

266+ NEW.{temp_column_name} = {value};

267+ END;

268+ END;

269+ END IF;

270+ RETURN NEW;

271+ END

272+ $$ language 'plpgsql';

273+

274+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

275+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

276+ "#,

277+ changed_table_assignments = changed_table_assignments.join("\n"),

278+ changed_table_real = table.real_name,

279+ changed_table = table.name,

280+ from_table = from_table.name,

281+ trigger_name = self.reverse_trigger_name(ctx),

282+ temp_column_name = temp_column_name,

283+ // declarations = declarations.join("\n"),

284+ );

285+ db.run(&query)

286+ .context("failed to create reverse up trigger")?;

287+

200 // Backfill values in batches by touching the from table

288 // Backfill values in batches by touching the from table

201 common::batch_touch_rows(db, &from_table.real_name, None)

289 common::batch_touch_rows(db, &from_table.real_name, None)

202 .context("failed to batch update existing rows")?;

290 .context("failed to batch update existing rows")?;

@@ -210,10 +298,10 @@ impl Action for AddColumn {

210 if !self.column.nullable {

298 if !self.column.nullable {

211 let query = format!(

299 let query = format!(

212 r#"

300 r#"

213− ALTER TABLE "{table}"

301+ ALTER TABLE "{table}"

214− ADD CONSTRAINT "{constraint_name}"

302+ ADD CONSTRAINT "{constraint_name}"

215− CHECK ("{column}" IS NOT NULL) NOT VALID

303+ CHECK ("{column}" IS NOT NULL) NOT VALID

216− "#,

304+ "#,

217 table = self.table,

305 table = self.table,

218 constraint_name = self.not_null_constraint_name(ctx),

306 constraint_name = self.not_null_constraint_name(ctx),

219 column = temp_column_name,

307 column = temp_column_name,

@@ -236,8 +324,10 @@ impl Action for AddColumn {

236 let query = format!(

324 let query = format!(

237 r#"

325 r#"

238 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

326 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

327+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

239 "#,

328 "#,

240 trigger_name = self.trigger_name(ctx),

329 trigger_name = self.trigger_name(ctx),

330+ reverse_trigger_name = self.reverse_trigger_name(ctx),

241 );

331 );

242 transaction

332 transaction

243 .run(&query)

333 .run(&query)

@@ -329,8 +419,10 @@ impl Action for AddColumn {

329 let query = format!(

419 let query = format!(

330 r#"

420 r#"

331 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

421 DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

422+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

332 "#,

423 "#,

333 trigger_name = self.trigger_name(ctx),

424 trigger_name = self.trigger_name(ctx),

425+ reverse_trigger_name = self.reverse_trigger_name(ctx),

334 );

426 );

335 db.run(&query).context("failed to drop up trigger")?;

427 db.run(&query).context("failed to drop up trigger")?;

336

428

src/migrations/remove_column.rs+187−59

@@ -34,7 +34,16 @@ impl RemoveColumn {

34 )

35 }

37+ fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

38+ format!(

39+ "{}_remove_column_{}_{}_rev",

40+ ctx.prefix(),

41+ self.table,

42+ self.column

43+ )

44+ }

45+

37− fn null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

46+ fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

38 format!(

47 format!(

39 "{}_remove_column_{}_{}_nn",

48 "{}_remove_column_{}_{}_nn",

40 ctx.prefix(),

49 ctx.prefix(),

@@ -42,6 +51,15 @@ impl RemoveColumn {

42 self.column

51 self.column

43 )

52 )

44 }

53 }

54+

55+ fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56+ format!(

57+ "{}_add_column_not_null_{}_{}",

58+ ctx.prefix(),

59+ self.table,

60+ self.column

61+ )

62+ }

45}

63}

47#[typetag::serde(name = "remove_column")]

65#[typetag::serde(name = "remove_column")]

@@ -121,21 +139,9 @@ impl Action for RemoveColumn {

121

139

122 let from_table = schema.get_table(db, &from_table)?;

140 let from_table = schema.get_table(db, &from_table)?;

123

141

124− let declarations: Vec<String> = from_table

125− .columns

126− .iter()

127− .map(|column| {

128− format!(

129− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

130− table = from_table.real_name,

131− alias = column.name,

132− real_name = column.real_name,

133− )

134− })

135− .collect();

136−

137 let maybe_null_check = if !column.nullable {

142 let maybe_null_check = if !column.nullable {

138 // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

143 // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

144+ // We will add a null check to the down function on the new schema below as well to cover both cases.

139 // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

145 // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

140 // NOT NULL is not checked at the end of a transaction, but immediately upon update.

146 // NOT NULL is not checked at the end of a transaction, but immediately upon update.

141 let query = format!(

147 let query = format!(

@@ -161,7 +167,7 @@ impl Action for RemoveColumn {

161 EXECUTE PROCEDURE {trigger_name}();

167 EXECUTE PROCEDURE {trigger_name}();

162 "#,

168 "#,

163 table = self.table,

169 table = self.table,

164− trigger_name = self.null_constraint_trigger_name(ctx),

170+ trigger_name = self.not_null_constraint_trigger_name(ctx),

165 column = self.column,

171 column = self.column,

166 );

172 );

167 db.run(&query)

173 db.run(&query)

@@ -190,6 +196,19 @@ impl Action for RemoveColumn {

190 "".to_string()

196 "".to_string()

191 };

197 };

192

198

199+ let into_variables = from_table

200+ .columns

201+ .iter()

202+ .map(|column| {

203+ format!(

204+ "NEW.{real_name} AS {alias}",

205+ alias = column.name,

206+ real_name = column.real_name,

207+ )

208+ })

209+ .collect::<Vec<String>>()

210+ .join(", ");

211+

193 let query = format!(

212 let query = format!(

194 r#"

213 r#"

195 CREATE OR REPLACE FUNCTION {trigger_name}()

214 CREATE OR REPLACE FUNCTION {trigger_name}()

@@ -198,12 +217,21 @@ impl Action for RemoveColumn {

198 BEGIN

217 BEGIN

199 IF reshape.is_new_schema() THEN

218 IF reshape.is_new_schema() THEN

200 DECLARE

219 DECLARE

201− {declarations}

220+ {from_table} record;

202 BEGIN

221 BEGIN

222+ SELECT {into_variables}

223+ INTO {from_table};

224+

203 {maybe_null_check}

225 {maybe_null_check}

226+

227+ -- Don't trigger reverse trigger when making this update

228+ perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

229+

204 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

230 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

205 SET "{column_name}" = {value}

231 SET "{column_name}" = {value}

206 WHERE {where};

232 WHERE {where};

233+

234+ perform set_config('reshape.disable_triggers', '', TRUE);

207 END;

235 END;

208 END IF;

236 END IF;

209 RETURN NEW;

237 RETURN NEW;

@@ -214,17 +242,83 @@ impl Action for RemoveColumn {

214 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

242 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

215 "#,

243 "#,

216 changed_table = self.table,

244 changed_table = self.table,

245+ from_table = from_table.name,

217 from_table_real = from_table.real_name,

246 from_table_real = from_table.real_name,

218 column_name = self.column,

247 column_name = self.column,

219 trigger_name = self.trigger_name(ctx),

248 trigger_name = self.trigger_name(ctx),

220− declarations = declarations.join("\n"),

221 );

249 );

222 db.run(&query).context("failed to create down trigger")?;

250 db.run(&query).context("failed to create down trigger")?;

251+

252+ let changed_into_variables = table

253+ .columns

254+ .iter()

255+ .map(|column| {

256+ format!(

257+ "NEW.{real_name} AS {alias}",

258+ alias = column.name,

259+ real_name = column.real_name,

260+ )

261+ })

262+ .collect::<Vec<String>>()

263+ .join(", ");

264+

265+ let from_table_columns = from_table

266+ .columns

267+ .iter()

268+ .map(|column| format!("{} as {}", column.real_name, column.name))

269+ .collect::<Vec<String>>()

270+ .join(", ");

271+

272+ let query = format!(

273+ r#"

274+ CREATE OR REPLACE FUNCTION {trigger_name}()

275+ RETURNS TRIGGER AS $$

276+ #variable_conflict use_variable

277+ BEGIN

278+ IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

279+ DECLARE

280+ {changed_table} record;

281+ __temp_row record;

282+ BEGIN

283+ SELECT {changed_into_variables}

284+ INTO {changed_table};

285+

286+ SELECT *

287+ INTO __temp_row

288+ FROM (

289+ SELECT {from_table_columns}

290+ FROM public.{from_table_real}

291+ ) {from_table}

292+ WHERE {where};

293+

294+ DECLARE

295+ {from_table} record;

296+ BEGIN

297+ {from_table} := __temp_row;

298+ NEW.{column_name_real} = {value};

299+ END;

300+ END;

301+ END IF;

302+ RETURN NEW;

303+ END

304+ $$ language 'plpgsql';

305+

306+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

307+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

308+ "#,

309+ changed_table = table.name,

310+ changed_table_real = table.real_name,

311+ from_table = from_table.name,

312+ from_table_real = from_table.real_name,

313+ column_name_real = column.real_name,

314+ trigger_name = self.reverse_trigger_name(ctx),

315+ // declarations = declarations.join("\n"),

316+ );

317+ db.run(&query)

318+ .context("failed to create reverse down trigger")?;

223 }

319 }

224 }

320 }

225

321

226− if !column.nullable {}

227−

228 Ok(())

322 Ok(())

229 }

323 }

230

324

@@ -247,29 +341,20 @@ impl Action for RemoveColumn {

247 }

341 }

248

342

249 // Remove column, function and trigger

343 // Remove column, function and trigger

250− let trigger_table = match &self.down {

251− Some(Transformation::Update {

252− table,

253− value: _,

254− r#where: _,

255− }) => table,

256− _ => &self.table,

257− };

258 let query = format!(

344 let query = format!(

259 r#"

345 r#"

260 ALTER TABLE "{table}"

346 ALTER TABLE "{table}"

261 DROP COLUMN IF EXISTS "{column}";

347 DROP COLUMN IF EXISTS "{column}";

262

348

263− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

349+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

264− DROP FUNCTION IF EXISTS "{trigger_name}";

350+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

265−

266− DROP TRIGGER IF EXISTS "{null_trigger_name}" ON "{table}";

267− DROP FUNCTION IF EXISTS "{null_trigger_name}";

351+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

268 "#,

352 "#,

269 table = self.table,

353 table = self.table,

270 column = self.column,

354 column = self.column,

271 trigger_name = self.trigger_name(ctx),

355 trigger_name = self.trigger_name(ctx),

356+ reverse_trigger_name = self.reverse_trigger_name(ctx),

272− null_trigger_name = self.null_constraint_trigger_name(ctx),

357+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),

273 );

358 );

274 db.run(&query)

359 db.run(&query)

275 .context("failed to drop column and down trigger")?;

360 .context("failed to drop column and down trigger")?;

@@ -286,38 +371,81 @@ impl Action for RemoveColumn {

286 }

371 }

287

372

288 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

289− // We might have removed the NOT NULL check so we reinstate it

374+ // We might have temporaily removed the NOT NULL check and have to reinstate it

375+ let has_not_null_function = !db

376+ .query_with_params(

377+ "

378+ SELECT routine_name

379+ FROM information_schema.routines

380+ WHERE routine_schema = 'public'

381+ AND routine_name = $1

382+ ",

383+ &[&self.not_null_constraint_trigger_name(ctx)],

384+ )

385+ .context("failed to get any NOT NULL function")?

386+ .is_empty();

387+

388+ if has_not_null_function {

389+ // Make column NOT NULL again without taking any long lived locks with a temporary constraint

290− db.run(&format!(

390+ let query = format!(

291− r#"

391+ r#"

392+ ALTER TABLE "{table}"

393+ ADD CONSTRAINT "{constraint_name}"

394+ CHECK ("{column}" IS NOT NULL) NOT VALID

395+ "#,

396+ table = self.table,

397+ constraint_name = self.not_null_constraint_name(ctx),

398+ column = self.column,

399+ );

400+ db.run(&query)

401+ .context("failed to add NOT NULL constraint")?;

402+

403+ let query = format!(

404+ r#"

292− ALTER TABLE {table}

405+ ALTER TABLE "{table}"

406+ VALIDATE CONSTRAINT "{constraint_name}"

407+ "#,

408+ table = self.table,

409+ constraint_name = self.not_null_constraint_name(ctx),

410+ );

411+ db.run(&query)

412+ .context("failed to validate NOT NULL constraint")?;

413+

414+ // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above

415+ db.run(&format!(

416+ r#"

417+ ALTER TABLE {table}

293− ALTER COLUMN {column}

418+ ALTER COLUMN {column}

294− SET NOT NULL

419+ SET NOT NULL

295− "#,

420+ "#,

296− table = self.table,

421+ table = self.table,

297− column = self.column

422+ column = self.column

298− ))

423+ ))

299− .context("failed to reinstate column not null constraint")?;

424+ .context("failed to reinstate column NOT NULL")?;

425+

426+ // Drop the temporary constraint

427+ let query = format!(

428+ r#"

429+ ALTER TABLE "{table}"

430+ DROP CONSTRAINT "{constraint_name}"

431+ "#,

432+ table = self.table,

433+ constraint_name = self.not_null_constraint_name(ctx),

434+ );

435+ db.run(&query)

436+ .context("failed to drop NOT NULL constraint")?;

437+ }

300

438

301 // Remove function and trigger

439 // Remove function and trigger

302− let trigger_table = match &self.down {

303− Some(Transformation::Update {

304− table,

305− value: _,

306− r#where: _,

307− }) => table,

308− _ => &self.table,

309− };

310 db.run(&format!(

440 db.run(&format!(

311 r#"

441 r#"

312− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

442+ DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

313− DROP FUNCTION IF EXISTS "{trigger_name}";

443+ DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

314−

315− DROP TRIGGER IF EXISTS "{null_trigger_name}" ON "{table}";

316− DROP FUNCTION IF EXISTS "{null_trigger_name}";

444+ DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

317 "#,

445 "#,

318− table = self.table,

319 trigger_name = self.trigger_name(ctx),

446 trigger_name = self.trigger_name(ctx),

447+ reverse_trigger_name = self.reverse_trigger_name(ctx),

320− null_trigger_name = self.null_constraint_trigger_name(ctx),

448+ null_trigger_name = self.not_null_constraint_trigger_name(ctx),

321 ))

449 ))

322 .context("failed to drop down trigger")?;

450 .context("failed to drop down trigger")?;

323

451

tests/add_column.rs+2−2

@@ -308,8 +308,8 @@ fn add_column_with_complex_up() {

308

309 [actions.up]

310 table = "users"

311− value = "email"

311+ value = "users.email"

312− where = "user_id = id"

312+ where = "profiles.user_id = users.id"

313 "#,

314 );

315

tests/complex.rs+164−5

@@ -1,6 +1,165 @@

1mod common;

2use common::Test;

4+#[test]

5+fn move_column_between_tables() {

6+ let mut test = Test::new("Move column between tables");

8+ test.first_migration(

9+ r#"

10+ name = "create_tables"

11+

12+ [[actions]]

13+ type = "create_table"

14+ name = "users"

15+ primary_key = ["id"]

16+

17+ [[actions.columns]]

18+ name = "id"

19+ type = "INTEGER"

20+

21+ [[actions.columns]]

22+ name = "email"

23+ type = "TEXT"

24+

25+ [[actions]]

26+ type = "create_table"

27+ name = "profiles"

28+ primary_key = ["id"]

29+

30+ [[actions.columns]]

31+ name = "id"

32+ type = "INTEGER"

33+

34+ [[actions.columns]]

35+ name = "user_id"

36+ type = "INTEGER"

37+ nullable = false

38+ "#,

39+ );

40+

41+ test.second_migration(

42+ r#"

43+ name = "move_email_column"

44+

45+ [[actions]]

46+ type = "add_column"

47+ table = "profiles"

48+

49+ [actions.column]

50+ name = "email"

51+ type = "TEXT"

52+ nullable = false

53+

54+ # When `users` is updated in the old schema, we write the email value to `profiles`

55+ # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated

56+ [actions.up]

57+ table = "users"

58+ value = "users.email"

59+ where = "profiles.user_id = users.id"

60+

61+ [[actions]]

62+ type = "remove_column"

63+ table = "users"

64+ column = "email"

65+

66+ # When `profiles` is changed in the new schema, we write the email address back to the removed column

67+ [actions.down]

68+ table = "profiles"

69+ value = "profiles.email"

70+ where = "users.id = profiles.user_id"

71+ "#,

72+ );

73+

74+ test.after_first(|db| {

75+ db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")

76+ .unwrap();

77+ db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")

78+ .unwrap();

79+

80+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")

81+ .unwrap();

82+ db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")

83+ .unwrap();

84+ });

85+

86+ test.intermediate(|old_db, new_db| {

87+ // Ensure emails were backfilled into profiles

88+ let profiles_emails: Vec<String> = new_db

89+ .query(

90+ r#"

91+ SELECT email

92+ FROM profiles

93+ ORDER BY id

94+ "#,

95+ &[],

96+ )

97+ .unwrap()

98+ .iter()

99+ .map(|row| row.get("email"))

100+ .collect();

101+ assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);

102+

103+ // Ensure insert in old schema updates new

104+ old_db

105+ .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")

106+ .unwrap();

107+ old_db

108+ .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")

109+ .unwrap();

110+ let new_email: String = new_db

111+ .query("SELECT email FROM profiles WHERE id = 3", &[])

112+ .unwrap()

113+ .first()

114+ .map(|row| row.get("email"))

115+ .unwrap();

116+ assert_eq!("test3@test.com", new_email);

117+

118+ // Ensure updates in old schema updates new

119+ old_db

120+ .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")

121+ .unwrap();

122+ let new_email: String = new_db

123+ .query("SELECT email FROM profiles WHERE id = 3", &[])

124+ .unwrap()

125+ .first()

126+ .map(|row| row.get("email"))

127+ .unwrap();

128+ assert_eq!("test3+updated@test.com", new_email);

129+

130+ // Ensure insert in new schema updates old

131+ new_db

132+ .simple_query(

133+ "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",

134+ )

135+ .unwrap();

136+ new_db

137+ .simple_query("INSERT INTO users (id) VALUES (4)")

138+ .unwrap();

139+ let old_email: String = old_db

140+ .query("SELECT email FROM users WHERE id = 4", &[])

141+ .unwrap()

142+ .first()

143+ .map(|row| row.get("email"))

144+ .unwrap();

145+ assert_eq!("test4@test.com", old_email);

146+

147+ // Ensure update in new schema updates old

148+ new_db

149+ .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")

150+ .unwrap();

151+ let old_email: String = old_db

152+ .query("SELECT email FROM users WHERE id = 4", &[])

153+ .unwrap()

154+ .first()

155+ .map(|row| row.get("email"))

156+ .unwrap();

157+ assert_eq!("test4+updated@test.com", old_email);

158+ });

159+

160+ test.run();

161+}

162+

4#[test]

163#[test]

5fn extract_relation_into_new_table() {

164fn extract_relation_into_new_table() {

6 let mut test = Test::new("Extract relation into new table");

165 let mut test = Test::new("Extract relation into new table");

@@ -64,7 +223,7 @@ fn extract_relation_into_new_table() {

64 [actions.up]

223 [actions.up]

65 table = "users"

224 table = "users"

66 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

225 values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

67− where = "user_id = id"

226+ where = "user_account_connections.user_id = users.id"

227

69 [[actions]]

228 [[actions]]

70 type = "remove_column"

229 type = "remove_column"

@@ -73,8 +232,8 @@ fn extract_relation_into_new_table() {

232

74 [actions.down]

233 [actions.down]

75 table = "user_account_connections"

234 table = "user_account_connections"

76− value = "account_id"

235+ value = "user_account_connections.account_id"

77− where = "id = user_id"

236+ where = "users.id = user_account_connections.user_id"

237

79 [[actions]]

238 [[actions]]

80 type = "remove_column"

239 type = "remove_column"

@@ -83,8 +242,8 @@ fn extract_relation_into_new_table() {

242

84 [actions.down]

243 [actions.down]

85 table = "user_account_connections"

244 table = "user_account_connections"

86− value = "LOWER(role)"

245+ value = "LOWER(user_account_connections.role)"

87− where = "id = user_id"

246+ where = "users.id = user_account_connections.user_id"

88 "#,

247 "#,

89 );

248 );

249

tests/remove_column.rs+2−2

@@ -179,8 +179,8 @@ fn remove_column_with_complex_down() {

179

180 [actions.down]

181 table = "profiles"

182− value = "email"

182+ value = "profiles.email"

183− where = "id = user_id"

183+ where = "users.id = profiles.user_id"

184 "#,

185 );

186