main..b4baef2 · reshape

git.iko.soy/reshape

on b4baef2 help sign in

.github/workflows/lint.yaml+3−3

since splitsnapshots

unifiedsplit

@@ -15,7 +15,7 @@ jobs:

15 runs-on: ubuntu-latest

16 steps:

17 - name: Checkout

18− uses: actions/checkout@v4

18+ uses: actions/checkout@v2

19 - name: Select Rust toolchain with Clippy

20 uses: actions-rs/toolchain@v1

21 with:

@@ -23,6 +23,6 @@ jobs:

23 components: clippy

24 override: true

25 - name: Use cache for Rust dependencies

26− uses: Swatinem/rust-cache@v2

26+ uses: Swatinem/rust-cache@v1

27 - name: Lint using Clippy

28− run: cargo clippy

28+ run: cargo clippy

.github/workflows/publish-crate.yamldeleted−20

@@ -1,20 +0,0 @@

1−name: Publish crate

2−

3−on: workflow_dispatch

4−

5−jobs:

6− publish-crate:

7− runs-on: ubuntu-latest

8− steps:

9− - name: Checkout

10− uses: actions/checkout@v2

11− - name: Select Rust toolchain

12− uses: actions-rs/toolchain@v1

13− with:

14− toolchain: stable

15− - name: Publish to crates.io

16− uses: actions-rs/cargo@v1

17− with:

18− command: publish

19− env:

20− CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}

.github/workflows/publish-docker.yamldeleted−25

@@ -1,25 +0,0 @@

1−name: Publish Docker image

2−

3−on:

4− workflow_dispatch:

5− inputs:

6− version:

7− description: "Version (without 'v' prefix)"

8− required: true

9− type: string

10−

11−jobs:

12− publish-docker:

13− runs-on: ubuntu-latest

14− steps:

15− - name: Checkout

16− uses: actions/checkout@v2

17− - name: Log in to Docker

18− env:

19− DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}

20− run: |

21− docker login -u ${{ secrets.DOCKER_USER }} -p $DOCKER_PASSWORD

22− - name: Build Docker image

23− run: docker build . --tag ${{ secrets.DOCKER_USER }}/reshape:${{ inputs.version }} --tag ${{ secrets.DOCKER_USER }}/reshape:latest

24− - name: Push Docker image

25− run: docker push --all-tags ${{ secrets.DOCKER_USER }}/reshape

Cargo.toml+3−3

@@ -1,6 +1,6 @@

1[package]

2name = "reshape"

3−version = "0.7.0"

3+version = "0.6.1"

4description = "An easy-to-use, zero-downtime schema migration tool for Postgres"

5homepage = "https://github.com/fabianlindfors/reshape"

6documentation = "https://github.com/fabianlindfors/reshape"

@@ -9,7 +9,7 @@ license = "MIT"

9keywords = ["postgres", "migrations"]

10edition = "2021"

11authors = ["Fabian Lindfors"]

12−rust-version = "1.70"

12+rust-version = "1.58"

14[dependencies]

15postgres = { version = "0.19.2", features = ["with-serde_json-1"] }

@@ -23,4 +23,4 @@ version = "3.0.0"

23colored = "2"

24rand = "0.8"

25dotenv = "0.15.0"

26−lexical-sort = "0.3.1"

26+lexical-sort = "0.3.1"

Dockerfile+1−1

@@ -1,4 +1,4 @@

1−FROM rust:1.70.0 AS builder

1+FROM rust:1.58.0 AS builder

2WORKDIR /usr/src/reshape

3COPY . .

4RUN cargo build --release

README.md+7−137

@@ -2,7 +2,7 @@

3[![Test status badge](https://github.com/fabianlindfors/Reshape/actions/workflows/test.yaml/badge.svg)](https://github.com/fabianlindfors/reshape/actions/workflows/test.yaml) [![Latest release](https://shields.io/github/v/release/fabianlindfors/reshape?display_name=tag&sort=semver&color=blue)](https://github.com/fabianlindfors/reshape/releases)

5−> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible. If you'd like to chat about it, please [reach out](contact@reshapedb.com)!

5+> Also check out [ReshapeDB](https://reshapedb.com), a new database built from the ground up to make zero-downtime schema and data migrations as simple and safe as possible

7Reshape is an easy-to-use, zero-downtime schema migration tool for Postgres. It automatically handles complex migrations that would normally require downtime or manual multi-step changes. During a migration, Reshape ensures both the old and new schema are available at the same time, allowing you to gradually roll out your application. It will also perform all changes without excessive locking, avoiding downtime caused by blocking other queries. For a more thorough introduction to Reshape, check out the [introductory blog post](https://fabianlindfors.se/blog/schema-migrations-in-postgres-using-reshape/).

@@ -34,7 +34,6 @@ Designed for Postgres 12 and later.

34 - [Create enum](#create-enum)

35 - [Remove enum](#remove-enum)

36 - [Custom](#custom)

37− - [Complex changes across tables](#complex-changes-across-tables)

38- [Commands and options](#commands-and-options)

37- [Commands and options](#commands-and-options)

39 - [`reshape migration start`](#reshape-migration-start)

38 - [`reshape migration start`](#reshape-migration-start)

40 - [`reshape migration complete`](#reshape-migration-complete)

39 - [`reshape migration complete`](#reshape-migration-complete)

@@ -167,7 +166,7 @@ Every action has a `type`. The supported types are detailed below.

167

166

168#### Create table

167#### Create table

169

168

170−The `create_table` action will create a new table with the specified columns, indices and constraints. You can optionally provide an `up` option to backfill values from an existing table.

169+The `create_table` action will create a new table with the specified columns, indices and constraints.

171

170

172_Example: create a `customers` table with a few columns and a primary key_

171_Example: create a `customers` table with a few columns and a primary key_

173

172

@@ -226,29 +225,6 @@ primary_key = ["id"]

226 referenced_columns = ["id"]

225 referenced_columns = ["id"]

227```

226```

228

227

229−_Example: create `profiles` table based on existing `users` table_

230−

231−```toml

232−[[actions]]

233−type = "create_table"

234−name = "profiles"

235−primary_key = ["user_id"]

236−

237− [[actions.columns]]

238− name = "user_id"

239− type = "INTEGER"

240−

241− [[actions.columns]]

242− name = "user_email"

243− type = "TEXT"

244−

245− # Backfill from `users` table and copy `users.email` to `user_email` column

246− # This will perform an upsert based on the primary key to avoid duplicate rows

247− [actions.up]

248− table = "users"

249− values = { user_id = "id", user_email = "email" }

250−```

251−

252#### Rename table

228#### Rename table

253

229

254The `rename_table` action will change the name of an existing table.

230The `rename_table` action will change the name of an existing table.

@@ -308,7 +284,7 @@ foreign_key = "items_user_id_fkey"

308

284

309#### Add column

285#### Add column

310

286

311−The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column. `up` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)).

287+The `add_column` action will add a new column to an existing table. You can optionally provide an `up` setting. This should be an SQL expression which will be run for all existing rows to backfill the new column.

312

288

313_Example: add a new column `reference` to table `products`_

289_Example: add a new column `reference` to table `products`_

314

290

@@ -375,26 +351,6 @@ up = "data['path']['to']['value'] #>> '{}'"

375 type = "TEXT"

351 type = "TEXT"

376```

352```

377

353

378−_Example: duplicate `email` column from `users` to `profiles` table_

379−

380−```toml

381−# `profiles` has `user_id` column which maps to `users.id`

382−[[actions]]

383−type = "add_column"

384−table = "profiles"

385−

386− [actions.column]

387− name = "email"

388− type = "TEXT"

389− nullable = false

390−

391− # When `users` is updated in the old schema, we write the email value to `profiles`

392− [actions.up]

393− table = "users"

394− value = "email"

395− where = "user_id = id"

396−```

397−

398#### Alter column

354#### Alter column

399

355

400The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.

356The `alter_column` action enables many different changes to an existing column, for example renaming, changing type and changing existing values.

@@ -472,7 +428,7 @@ column = "created_at"

472

428

473#### Remove column

429#### Remove column

474

430

475−The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. `down` may also reference another table to perform cross-table migrations (see ["Complex changes across tables"](#complex-changes-across-tables)) . The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.

431+The `remove_column` action will remove an existing column from a table. You can optionally provide a `down` setting. This should be an SQL expression which will be used to determine values for the old schema when inserting or updating rows using the new schema. The `down` setting must be provided when the removed column is `NOT NULL` or doesn't have a default value.

476

432

477Any indices that cover the column will be removed.

433Any indices that cover the column will be removed.

478

434

@@ -493,15 +449,15 @@ _Example: remove `email` column from `users` table and use column from `profiles

493```toml

449```toml

494[[actions]]

450[[actions]]

495type = "remove_column"

451type = "remove_column"

496−table = "users"

452+name = "users"

497column = "email"

453column = "email"

498

454

499 # Our application will use the `profiles.email` column instead

455 # Our application will use the `profiles.email` column instead

500 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

456 # For backwards compatibility, we will write back to the removed `email` column whenever `profiles` is changed

501 [actions.down]

457 [actions.down]

502 table = "profiles"

458 table = "profiles"

503− value = "profiles.email"

459+ value = "email"

504− where = "users.id = profiles.user_id"

460+ where = "id = user_id"

505```

461```

506

462

507### Indices

463### Indices

@@ -620,92 +576,6 @@ abort = """

620"""

576"""

621```

577```

622

578

623−### Complex changes across tables

624−

625−The `up` and `down` options available when creating tables, adding columns and removing columns can also perform more complex changes that span tables.

626−

627−_Example: move `email` column from `users` to `profiles` table_

628−

629−```toml

630−[[actions]]

631−type = "add_column"

632−table = "profiles"

633−

634− [actions.column]

635− name = "email"

636− type = "TEXT"

637− nullable = false

638−

639− # When `users` is updated in the old schema, we write the email value to `profiles`

640− [actions.up]

641− table = "users"

642− value = "users.email"

643− where = "profiles.user_id = users.id"

644−

645−[[actions]]

646−type = "remove_column"

647−table = "users"

648−column = "email"

649−

650− # When `profiles` is changed in the new schema, we write the email address back to the removed column

651− [actions.down]

652− table = "profiles"

653− value = "profiles.email"

654− where = "users.id = profiles.user_id"

655−```

656−

657−_Example: turn a 1:N relationship between `users` and `accounts` into N:M and change the format of the associated `role`_

658−

659−```toml

660−# Add `user_account_connections` as a junction table

661−[[actions]]

662−type = "create_table"

663−name = "user_account_connections"

664−primary_key = ["account_id", "user_id"]

665−

666− [[actions.columns]]

667− name = "account_id"

668− type = "INTEGER"

669−

670− [[actions.columns]]

671− name = "user_id"

672− type = "INTEGER"

673−

674− # `role` is currently stored directly on the `users` table but is part of the relationship

675− [[actions.columns]]

676− name = "role"

677− type = "TEXT"

678− nullable = false

679−

680− # Backfill the new table from `users` and uppercase the `role`

681− [actions.up]

682− table = "users"

683− values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

684− where = "user_account_connections.user_id = users.id"

685−

686−[[actions]]

687−type = "remove_column"

688−table = "users"

689−column = "account_id"

690−

691− # When `user_account_connections` is updated, we write the `account_id` back to `users`

692− [actions.down]

693− table = "user_account_connections"

694− value = "user_account_connections.account_id"

695− where = "users.id = user_account_connections.user_id"

696−

697−[[actions]]

698−type = "remove_column"

699−table = "users"

700−column = "account_role"

701−

702− # When `user_account_connections` is updated, we write the lowercase role back to `users`

703− [actions.down]

704− table = "user_account_connections"

705− value = "LOWER(user_account_connections.role)"

706− where = "users.id = user_account_connections.user_id"

707−```

708−

709## Commands and options

579## Commands and options

710

580

711### `reshape migration start`

581### `reshape migration start`

src/migrations/add_column.rs+58−214

@@ -3,25 +3,14 @@ use crate::{

3 db::{Conn, Transaction},

4 schema::Schema,

5};

6−use anyhow::{bail, Context};

6+use anyhow::Context;

7use serde::{Deserialize, Serialize};

9#[derive(Serialize, Deserialize, Debug)]

10pub struct AddColumn {

11 pub table: String,

12+ pub up: Option<String>,

12 pub column: Column,

13 pub column: Column,

13− pub up: Option<Transformation>,

14−}

15−

16−#[derive(Serialize, Deserialize, Debug)]

17−#[serde(untagged)]

18−pub enum Transformation {

19− Simple(String),

20− Update {

21− table: String,

22− value: String,

23− r#where: String,

24− },

25}

14}

27impl AddColumn {

16impl AddColumn {

@@ -43,15 +32,6 @@ impl AddColumn {

43 )

32 )

44 }

33 }

46− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

47− format!(

48− "{}_add_column_{}_{}_rev",

49− ctx.prefix(),

50− self.table,

51− self.column.name

52− )

53− }

54−

55 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

35 fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56 format!(

36 format!(

57 "{}_add_column_not_null_{}_{}",

37 "{}_add_column_not_null_{}_{}",

@@ -81,7 +61,7 @@ impl Action for AddColumn {

81 let temp_column_name = self.temp_column_name(ctx);

61 let temp_column_name = self.temp_column_name(ctx);

83 let mut definition_parts = vec![

63 let mut definition_parts = vec![

84− format!("\"{}\"", temp_column_name.to_string()),

64+ temp_column_name.to_string(),

85 self.column.data_type.to_string(),

65 self.column.data_type.to_string(),

86 ];

66 ];

@@ -106,189 +86,53 @@ impl Action for AddColumn {

106 );

86 );

107 db.run(&query).context("failed to add column")?;

87 db.run(&query).context("failed to add column")?;

108

109− let declarations: Vec<String> = table

110− .columns

111− .iter()

112− .map(|column| {

113− format!(

114− "\"{alias}\" public.{table}.{real_name}%TYPE := NEW.{real_name};",

115− table = table.real_name,

116− alias = column.name,

117− real_name = column.real_name,

118− )

119− })

120− .collect();

121−

122 if let Some(up) = &self.up {

89 if let Some(up) = &self.up {

123− if let Transformation::Simple(up) = up {

124− // Add triggers to fill in values as they are inserted/updated

125− let query = format!(

126− r#"

127− CREATE OR REPLACE FUNCTION {trigger_name}()

128− RETURNS TRIGGER AS $$

129− BEGIN

130− IF NOT reshape.is_new_schema() THEN

131− DECLARE

132− {declarations}

90+ let declarations: Vec<String> = table

133− BEGIN

134− NEW."{temp_column_name}" = {up};

135− END;

136− END IF;

137− RETURN NEW;

138− END

139− $$ language 'plpgsql';

140−

141− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

142− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

143− "#,

144− temp_column_name = temp_column_name,

145− trigger_name = self.trigger_name(ctx),

146− up = up,

147− table = self.table,

148− declarations = declarations.join("\n"),

149− );

150− db.run(&query).context("failed to create up trigger")?;

151−

152− // Backfill values in batches

153− common::batch_touch_rows(db, &table.real_name, Some(&temp_column_name))

154− .context("failed to batch update existing rows")?;

155− }

156−

157− if let Transformation::Update {

158− table: from_table,

159− value,

160− r#where,

161− } = up

162− {

163− let existing_schema_name = match &ctx.existing_schema_name {

164− Some(name) => name,

165− None => bail!("can't use update without previous migration"),

166− };

167−

168− let from_table = schema.get_table(db, &from_table)?;

169−

170− let from_table_assignments: Vec<String> = from_table

171− .columns

91+ .columns

172− .iter()

92+ .iter()

173− .map(|column| {

93+ .map(|column| {

174− format!(

94+ format!(

175− "{table}.{alias} = NEW.{real_name};",

95+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

176− table = from_table.name,

177− alias = column.name,

178− real_name = column.real_name,

179− )

180− })

181− .collect();

182−

183− // Add triggers to fill in values as they are inserted/updated

184− let query = format!(

185− r#"

186− CREATE OR REPLACE FUNCTION {trigger_name}()

187− RETURNS TRIGGER AS $$

188− #variable_conflict use_variable

189− BEGIN

190− IF NOT reshape.is_new_schema() THEN

191− DECLARE

192− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

193− BEGIN

194− {assignments}

195−

196− -- Don't trigger reverse trigger when making this update

197− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

198−

199− UPDATE public."{changed_table_real}"

200− SET "{temp_column_name}" = {value}

201− WHERE {where};

202−

203− perform set_config('reshape.disable_triggers', '', TRUE);

204− END;

205− END IF;

206− RETURN NEW;

207− END

208− $$ language 'plpgsql';

209−

210− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

211− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

212− "#,

213− assignments = from_table_assignments.join("\n"),

214− changed_table_real = table.real_name,

215− from_table = from_table.name,

216− from_table_real = from_table.real_name,

217− trigger_name = self.trigger_name(ctx),

218− // declarations = from_table_declarations.join("\n"),

219− temp_column_name = temp_column_name,

220− );

221− db.run(&query).context("failed to create up trigger")?;

222−

223− let from_table_columns = from_table

224− .columns

225− .iter()

226− .map(|column| format!("{} as {}", column.real_name, column.name))

227− .collect::<Vec<String>>()

228− .join(", ");

229−

230− let changed_table_assignments: Vec<String> = table

231− .columns

232− .iter()

233− .map(|column| {

234− format!(

235− "{table}.{alias} := NEW.{real_name};",

236− table = table.name,

96+ table = table.real_name,

237− alias = column.name,

97+ alias = column.name,

238− real_name = column.real_name,

98+ real_name = column.real_name,

239− )

99+ )

240− })

100+ })

241− .collect();

101+ .collect();

242−

102+

243− // Add triggers to fill in values as they are inserted/updated

103+ // Add triggers to fill in values as they are inserted/updated

244− let query = format!(

104+ let query = format!(

245− r#"

105+ r#"

246− CREATE OR REPLACE FUNCTION {trigger_name}()

106+ CREATE OR REPLACE FUNCTION {trigger_name}()

247− RETURNS TRIGGER AS $$

107+ RETURNS TRIGGER AS $$

248− #variable_conflict use_variable

108+ BEGIN

249− BEGIN

250− IF NOT reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

109+ IF NOT reshape.is_new_schema() THEN

251− DECLARE

110+ DECLARE

252− {changed_table} migration_{existing_schema_name}.{changed_table}%ROWTYPE;

111+ {declarations}

253− __temp_row migration_{existing_schema_name}.{from_table}%ROWTYPE;

254− BEGIN

255− {changed_table_assignments}

256−

257− SELECT {from_table_columns}

258− INTO __temp_row

259− FROM migration_{existing_schema_name}.{from_table} {from_table}

260− WHERE {where};

261−

262− DECLARE

263− {from_table} migration_{existing_schema_name}.{from_table}%ROWTYPE;

264− BEGIN

112+ BEGIN

265− {from_table} = __temp_row;

266− NEW.{temp_column_name} = {value};

113+ NEW.{temp_column_name} = {up};

267− END;

268− END;

114+ END;

269− END IF;

115+ END IF;

270− RETURN NEW;

116+ RETURN NEW;

271− END

117+ END

272− $$ language 'plpgsql';

118+ $$ language 'plpgsql';

273−

119+

274− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

120+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

275− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

121+ CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{table}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

276− "#,

122+ "#,

277− changed_table_assignments = changed_table_assignments.join("\n"),

123+ temp_column_name = temp_column_name,

278− changed_table_real = table.real_name,

124+ trigger_name = self.trigger_name(ctx),

279− changed_table = table.name,

125+ up = up,

280− from_table = from_table.name,

126+ table = self.table,

281− trigger_name = self.reverse_trigger_name(ctx),

282− temp_column_name = temp_column_name,

283− // declarations = declarations.join("\n"),

127+ declarations = declarations.join("\n"),

284− );

128+ );

285− db.run(&query)

286− .context("failed to create reverse up trigger")?;

129+ db.run(&query).context("failed to create up trigger")?;

130+ }

287

131

288− // Backfill values in batches by touching the from table

132+ // Backfill values in batches

133+ if self.up.is_some() {

289− common::batch_touch_rows(db, &from_table.real_name, None)

134+ common::batch_touch_rows(db, &table.real_name, &temp_column_name)

290− .context("failed to batch update existing rows")?;

135+ .context("failed to batch update existing rows")?;

291− }

292 }

136 }

293

137

294 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.

138 // Add a temporary NOT NULL constraint if the column shouldn't be nullable.

@@ -298,10 +142,10 @@ impl Action for AddColumn {

298 if !self.column.nullable {

142 if !self.column.nullable {

299 let query = format!(

143 let query = format!(

300 r#"

144 r#"

301− ALTER TABLE "{table}"

145+ ALTER TABLE "{table}"

302− ADD CONSTRAINT "{constraint_name}"

146+ ADD CONSTRAINT "{constraint_name}"

303− CHECK ("{column}" IS NOT NULL) NOT VALID

147+ CHECK ("{column}" IS NOT NULL) NOT VALID

304− "#,

148+ "#,

305 table = self.table,

149 table = self.table,

306 constraint_name = self.not_null_constraint_name(ctx),

150 constraint_name = self.not_null_constraint_name(ctx),

307 column = temp_column_name,

151 column = temp_column_name,

@@ -323,11 +167,11 @@ impl Action for AddColumn {

323 // Remove triggers and procedures

167 // Remove triggers and procedures

324 let query = format!(

168 let query = format!(

325 r#"

169 r#"

326− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

170+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

327− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

171+ DROP FUNCTION IF EXISTS "{trigger_name}";

328 "#,

172 "#,

173+ table = self.table,

329 trigger_name = self.trigger_name(ctx),

174 trigger_name = self.trigger_name(ctx),

330− reverse_trigger_name = self.reverse_trigger_name(ctx),

331 );

175 );

332 transaction

176 transaction

333 .run(&query)

177 .run(&query)

@@ -418,11 +262,11 @@ impl Action for AddColumn {

418 // Remove triggers and procedures

262 // Remove triggers and procedures

419 let query = format!(

263 let query = format!(

420 r#"

264 r#"

421− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

265+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

422− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

266+ DROP FUNCTION IF EXISTS "{trigger_name}";

423 "#,

267 "#,

268+ table = self.table,

424 trigger_name = self.trigger_name(ctx),

269 trigger_name = self.trigger_name(ctx),

425− reverse_trigger_name = self.reverse_trigger_name(ctx),

426 );

270 );

427 db.run(&query).context("failed to drop up trigger")?;

271 db.run(&query).context("failed to drop up trigger")?;

428

272

src/migrations/alter_column.rs+1−1

@@ -150,7 +150,7 @@ impl Action for AlterColumn {

150 .context("failed to create up and down triggers")?;

151

152 // Backfill values in batches by touching the previous column

153− common::batch_touch_rows(db, &table.real_name, Some(&column.real_name))

153+ common::batch_touch_rows(db, &table.real_name, &column.real_name)

154 .context("failed to batch update existing rows")?;

155

156 // Duplicate any indices to the temporary column

src/migrations/common.rs+9−14

@@ -69,11 +69,7 @@ impl ToSql for PostgresRawValue {

69 postgres::types::to_sql_checked!();

70}

72−pub fn batch_touch_rows(

72+pub fn batch_touch_rows(db: &mut dyn Conn, table: &str, column: &str) -> anyhow::Result<()> {

73− db: &mut dyn Conn,

74− table: &str,

75− column: Option<&str>,

76−) -> anyhow::Result<()> {

77 const BATCH_SIZE: u16 = 1000;

73 const BATCH_SIZE: u16 = 1000;

79 let mut cursor: Option<PostgresRawValue> = None;

75 let mut cursor: Option<PostgresRawValue> = None;

@@ -82,13 +78,6 @@ pub fn batch_touch_rows(

82 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();

78 let mut params: Vec<&(dyn ToSql + Sync)> = Vec::new();

84 let primary_key = get_primary_key_columns_for_table(db, table)?;

80 let primary_key = get_primary_key_columns_for_table(db, table)?;

85−

86− // If no column to touch is passed, we default to the first primary key column (just to make some "update")

87− let touched_column = match column {

88− Some(column) => column,

89− None => primary_key.first().unwrap(),

90− };

91−

92 let primary_key_columns = primary_key.join(", ");

81 let primary_key_columns = primary_key.join(", ");

94 let primary_key_where = primary_key

83 let primary_key_where = primary_key

@@ -131,8 +120,8 @@ pub fn batch_touch_rows(

131 ORDER BY {primary_key_columns}

120 ORDER BY {primary_key_columns}

132 LIMIT {batch_size}

121 LIMIT {batch_size}

133 ), update AS (

122 ), update AS (

134− UPDATE public."{table}" "{table}"

123+ UPDATE public."{table}"

135− SET "{touched_column}" = "{table}"."{touched_column}"

124+ SET "{column}" = "{column}"

136 FROM rows

125 FROM rows

137 WHERE {primary_key_where}

126 WHERE {primary_key_where}

138 RETURNING {returning_columns}

127 RETURNING {returning_columns}

@@ -141,7 +130,13 @@ pub fn batch_touch_rows(

141 FROM update

130 FROM update

142 LIMIT 1

131 LIMIT 1

143 "#,

132 "#,

133+ table = table,

134+ primary_key_columns = primary_key_columns,

135+ cursor_where = cursor_where,

144 batch_size = BATCH_SIZE,

136 batch_size = BATCH_SIZE,

137+ column = column,

138+ primary_key_where = primary_key_where,

139+ returning_columns = returning_columns,

145 );

140 );

146 let last_value = db

141 let last_value = db

147 .query_with_params(&query, &params)?

142 .query_with_params(&query, &params)?

src/migrations/create_table.rs+8−121

@@ -1,9 +1,6 @@

1−use std::collections::HashMap;

2−

3use super::{common::ForeignKey, Action, Column, MigrationContext};

1use super::{common::ForeignKey, Action, Column, MigrationContext};

4use crate::{

2use crate::{

5 db::{Conn, Transaction},

3 db::{Conn, Transaction},

6− migrations::common,

7 schema::Schema,

4 schema::Schema,

8};

5};

9use anyhow::Context;

6use anyhow::Context;

@@ -17,21 +14,6 @@ pub struct CreateTable {

18 #[serde(default)]

15 #[serde(default)]

19 pub foreign_keys: Vec<ForeignKey>,

16 pub foreign_keys: Vec<ForeignKey>,

20−

21− pub up: Option<Transformation>,

22−}

23−

24−#[derive(Serialize, Deserialize, Debug)]

25−pub struct Transformation {

26− table: String,

27− values: HashMap<String, String>,

28− upsert_constraint: Option<String>,

29−}

30−

31−impl CreateTable {

32− fn trigger_name(&self, ctx: &MigrationContext) -> String {

33− format!("{}_create_table_{}", ctx.prefix(), self.name)

34− }

35}

17}

37#[typetag::serde(name = "create_table")]

19#[typetag::serde(name = "create_table")]

@@ -42,7 +24,7 @@ impl Action for CreateTable {

43 fn run(

25 fn run(

44 &self,

26 &self,

45− ctx: &MigrationContext,

27+ _ctx: &MigrationContext,

46 db: &mut dyn Conn,

28 db: &mut dyn Conn,

47 schema: &Schema,

29 schema: &Schema,

48 ) -> anyhow::Result<()> {

30 ) -> anyhow::Result<()> {

@@ -103,7 +85,7 @@ impl Action for CreateTable {

103 ));

85 ));

104 }

86 }

105

106− let query = &format!(

88+ db.run(&format!(

107 r#"

89 r#"

108 CREATE TABLE "{name}" (

90 CREATE TABLE "{name}" (

109 {definition}

91 {definition}

@@ -111,118 +93,23 @@ impl Action for CreateTable {

111 "#,

93 "#,

112 name = self.name,

94 name = self.name,

113 definition = definition_rows.join(",\n"),

95 definition = definition_rows.join(",\n"),

114− );

96+ ))

115− db.run(query).context("failed to create table")?;

116−

117− if let Some(Transformation {

118− table: from_table,

119− values,

120− upsert_constraint,

121− }) = &self.up

122− {

123− let from_table = schema.get_table(db, &from_table)?;

124−

125− let declarations: Vec<String> = from_table

126− .columns

127− .iter()

128− .map(|column| {

129− format!(

130− "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

131− table = from_table.real_name,

132− alias = column.name,

133− real_name = column.real_name,

134− )

135− })

136− .collect();

137−

138− let (insert_columns, insert_values): (Vec<&str>, Vec<&str>) = values

139− .iter()

140− .map(|(k, v)| -> (&str, &str) { (k, v) }) // Force &String to &str

141− .unzip();

142−

143− let update_set: Vec<String> = values

144− .iter()

145− .map(|(field, value)| format!("\"{field}\" = {value}"))

146− .collect();

147−

148− // Constraint to check for conflicts. Defaults to the primary key constraint.

149− let conflict_constraint_name = match upsert_constraint {

150− Some(custom_constraint) => custom_constraint.clone(),

151− _ => format!("{table}_pkey", table = self.name),

152− };

153−

154− // Add triggers to fill in values as they are inserted/updated

155− let query = format!(

156− r#"

157− CREATE OR REPLACE FUNCTION {trigger_name}()

158− RETURNS TRIGGER AS $$

159− #variable_conflict use_variable

160− BEGIN

161− IF NOT reshape.is_new_schema() THEN

162− DECLARE

163− {declarations}

164− BEGIN

165− INSERT INTO public."{changed_table_real}" ({columns})

166− VALUES ({values})

167− ON CONFLICT ON CONSTRAINT "{conflict_constraint_name}"

168− DO UPDATE SET

169− {updates};

170− END;

171− END IF;

172− RETURN NEW;

173− END

174− $$ language 'plpgsql';

175−

176− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{from_table_real}";

177− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

178− "#,

179− changed_table_real = self.name,

180− from_table_real = from_table.real_name,

181− trigger_name = self.trigger_name(ctx),

182− declarations = declarations.join("\n"),

183− columns = insert_columns.join(", "),

184− values = insert_values.join(", "),

185− updates = update_set.join(",\n"),

186− );

187− db.run(&query).context("failed to create up trigger")?;

97+ .context("failed to create table")?;

188−

189− // Backfill values in batches by touching the from table

190− common::batch_touch_rows(db, &from_table.real_name, None)

191− .context("failed to batch update existing rows")?;

192− }

193−

194 Ok(())

98 Ok(())

195 }

99 }

196

100

197 fn complete<'a>(

101 fn complete<'a>(

198 &self,

102 &self,

199− ctx: &MigrationContext,

103+ _ctx: &MigrationContext,

200− db: &'a mut dyn Conn,

104+ _db: &'a mut dyn Conn,

201 ) -> anyhow::Result<Option<Transaction<'a>>> {

105 ) -> anyhow::Result<Option<Transaction<'a>>> {

202− // Remove triggers and procedures

106+ // Do nothing

203− let query = format!(

204− r#"

205− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

206− "#,

207− trigger_name = self.trigger_name(ctx),

208− );

209− db.run(&query).context("failed to drop up trigger")?;

210−

211 Ok(None)

107 Ok(None)

212 }

108 }

213

109

214 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}

110 fn update_schema(&self, _ctx: &MigrationContext, _schema: &mut Schema) {}

215

111

216− fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

112+ fn abort(&self, _ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

217− // Remove triggers and procedures

218− let query = format!(

219− r#"

220− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

221− "#,

222− trigger_name = self.trigger_name(ctx),

223− );

224− db.run(&query).context("failed to drop up trigger")?;

225−

226 db.run(&format!(

113 db.run(&format!(

227 r#"

114 r#"

228 DROP TABLE IF EXISTS "{name}"

115 DROP TABLE IF EXISTS "{name}"

src/migrations/remove_column.rs+27−247

@@ -3,7 +3,7 @@ use crate::{

3 db::{Conn, Transaction},

4 schema::Schema,

5};

6−use anyhow::{anyhow, bail, Context};

6+use anyhow::{bail, Context};

7use serde::{Deserialize, Serialize};

9#[derive(Serialize, Deserialize, Debug)]

@@ -33,33 +33,6 @@ impl RemoveColumn {

33 self.column

34 )

35 }

36−

37− fn reverse_trigger_name(&self, ctx: &MigrationContext) -> String {

38− format!(

39− "{}_remove_column_{}_{}_rev",

40− ctx.prefix(),

41− self.table,

42− self.column

43− )

44− }

45−

46− fn not_null_constraint_trigger_name(&self, ctx: &MigrationContext) -> String {

47− format!(

48− "{}_remove_column_{}_{}_nn",

49− ctx.prefix(),

50− self.table,

51− self.column

52− )

53− }

54−

55− fn not_null_constraint_name(&self, ctx: &MigrationContext) -> String {

56− format!(

57− "{}_add_column_not_null_{}_{}",

58− ctx.prefix(),

59− self.table,

60− self.column

61− )

62− }

63}

36}

65#[typetag::serde(name = "remove_column")]

38#[typetag::serde(name = "remove_column")]

@@ -78,9 +51,6 @@ impl Action for RemoveColumn {

78 schema: &Schema,

51 schema: &Schema,

79 ) -> anyhow::Result<()> {

52 ) -> anyhow::Result<()> {

80 let table = schema.get_table(db, &self.table)?;

53 let table = schema.get_table(db, &self.table)?;

81− let column = table

82− .get_column(&self.column)

83− .ok_or_else(|| anyhow!("no such column {} exists", self.column))?;

85 // Add down trigger

55 // Add down trigger

86 if let Some(down) = &self.down {

56 if let Some(down) = &self.down {

@@ -139,75 +109,18 @@ impl Action for RemoveColumn {

139

109

140 let from_table = schema.get_table(db, &from_table)?;

110 let from_table = schema.get_table(db, &from_table)?;

141

111

142− let maybe_null_check = if !column.nullable {

112+ let declarations: Vec<String> = from_table

143− // Replace NOT NULL constraint with a constraint trigger that only triggers on the old schema.

144− // We will add a null check to the down function on the new schema below as well to cover both cases.

145− // As we are using a complex down function, we must remove the NOT NULL check for the new schema.

146− // NOT NULL is not checked at the end of a transaction, but immediately upon update.

147− let query = format!(

148− r#"

149− CREATE OR REPLACE FUNCTION {trigger_name}()

150− RETURNS TRIGGER AS $$

151− BEGIN

152− IF NOT reshape.is_new_schema() THEN

153− IF NEW.{column} IS NULL THEN

154− RAISE EXCEPTION '{column} can not be null';

155− END IF;

156− END IF;

157− RETURN NEW;

158− END

159− $$ language 'plpgsql';

160−

161− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{table}";

162−

163− CREATE CONSTRAINT TRIGGER "{trigger_name}"

164− AFTER INSERT OR UPDATE

165− ON "{table}"

166− FOR EACH ROW

167− EXECUTE PROCEDURE {trigger_name}();

168− "#,

169− table = self.table,

170− trigger_name = self.not_null_constraint_trigger_name(ctx),

171− column = self.column,

172− );

173− db.run(&query)

174− .context("failed to create null constraint trigger")?;

175−

176− db.run(&format!(

177− r#"

178− ALTER TABLE {table}

179− ALTER COLUMN {column}

180− DROP NOT NULL

181− "#,

182− table = self.table,

183− column = self.column

184− ))

185− .context("failed to remove column not null constraint")?;

186−

187− format!(

188− r#"

189− IF {value} IS NULL THEN

190− RAISE EXCEPTION '{column_name} can not be null';

191− END IF;

192− "#,

193− column_name = self.column,

194− )

195− } else {

196− "".to_string()

197− };

198−

199− let into_variables = from_table

200 .columns

113 .columns

201 .iter()

114 .iter()

202 .map(|column| {

115 .map(|column| {

203 format!(

116 format!(

204− "NEW.{real_name} AS {alias}",

117+ "{alias} public.{table}.{real_name}%TYPE := NEW.{real_name};",

118+ table = from_table.real_name,

205 alias = column.name,

119 alias = column.name,

206 real_name = column.real_name,

120 real_name = column.real_name,

207 )

121 )

208 })

122 })

209− .collect::<Vec<String>>()

123+ .collect();

210− .join(", ");

211

124

212 let query = format!(

125 let query = format!(

213 r#"

126 r#"

@@ -217,21 +130,11 @@ impl Action for RemoveColumn {

217 BEGIN

130 BEGIN

218 IF reshape.is_new_schema() THEN

131 IF reshape.is_new_schema() THEN

219 DECLARE

132 DECLARE

220− {from_table} record;

133+ {declarations}

221 BEGIN

134 BEGIN

222− SELECT {into_variables}

223− INTO {from_table};

224−

225− {maybe_null_check}

226−

227− -- Don't trigger reverse trigger when making this update

228− perform set_config('reshape.disable_triggers', 'TRUE', TRUE);

229−

230 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

135 UPDATE "migration_{existing_schema_name}"."{changed_table}" "{changed_table}"

231 SET "{column_name}" = {value}

136 SET "{column_name}" = {value}

232 WHERE {where};

137 WHERE {where};

233−

234− perform set_config('reshape.disable_triggers', '', TRUE);

235 END;

138 END;

236 END IF;

139 END IF;

237 RETURN NEW;

140 RETURN NEW;

@@ -242,80 +145,12 @@ impl Action for RemoveColumn {

242 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

145 CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{from_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

243 "#,

146 "#,

244 changed_table = self.table,

147 changed_table = self.table,

245− from_table = from_table.name,

246 from_table_real = from_table.real_name,

148 from_table_real = from_table.real_name,

247 column_name = self.column,

149 column_name = self.column,

248 trigger_name = self.trigger_name(ctx),

150 trigger_name = self.trigger_name(ctx),

151+ declarations = declarations.join("\n"),

249 );

152 );

250 db.run(&query).context("failed to create down trigger")?;

153 db.run(&query).context("failed to create down trigger")?;

251−

252− let changed_into_variables = table

253− .columns

254− .iter()

255− .map(|column| {

256− format!(

257− "NEW.{real_name} AS {alias}",

258− alias = column.name,

259− real_name = column.real_name,

260− )

261− })

262− .collect::<Vec<String>>()

263− .join(", ");

264−

265− let from_table_columns = from_table

266− .columns

267− .iter()

268− .map(|column| format!("{} as {}", column.real_name, column.name))

269− .collect::<Vec<String>>()

270− .join(", ");

271−

272− let query = format!(

273− r#"

274− CREATE OR REPLACE FUNCTION {trigger_name}()

275− RETURNS TRIGGER AS $$

276− #variable_conflict use_variable

277− BEGIN

278− IF reshape.is_new_schema() AND NOT current_setting('reshape.disable_triggers', TRUE) = 'TRUE' THEN

279− DECLARE

280− {changed_table} record;

281− __temp_row record;

282− BEGIN

283− SELECT {changed_into_variables}

284− INTO {changed_table};

285−

286− SELECT *

287− INTO __temp_row

288− FROM (

289− SELECT {from_table_columns}

290− FROM public.{from_table_real}

291− ) {from_table}

292− WHERE {where};

293−

294− DECLARE

295− {from_table} record;

296− BEGIN

297− {from_table} := __temp_row;

298− NEW.{column_name_real} = {value};

299− END;

300− END;

301− END IF;

302− RETURN NEW;

303− END

304− $$ language 'plpgsql';

305−

306− DROP TRIGGER IF EXISTS "{trigger_name}" ON "{changed_table_real}";

307− CREATE TRIGGER "{trigger_name}" BEFORE UPDATE OR INSERT ON "{changed_table_real}" FOR EACH ROW EXECUTE PROCEDURE {trigger_name}();

308− "#,

309− changed_table = table.name,

310− changed_table_real = table.real_name,

311− from_table = from_table.name,

312− from_table_real = from_table.real_name,

313− column_name_real = column.real_name,

314− trigger_name = self.reverse_trigger_name(ctx),

315− // declarations = declarations.join("\n"),

316− );

317− db.run(&query)

318− .context("failed to create reverse down trigger")?;

319 }

154 }

320 }

155 }

321

156

@@ -341,20 +176,25 @@ impl Action for RemoveColumn {

341 }

176 }

342

177

343 // Remove column, function and trigger

178 // Remove column, function and trigger

179+ let trigger_table = match &self.down {

180+ Some(Transformation::Update {

181+ table,

182+ value: _,

183+ r#where: _,

184+ }) => table,

185+ _ => &self.table,

186+ };

344 let query = format!(

187 let query = format!(

345 r#"

188 r#"

346 ALTER TABLE "{table}"

189 ALTER TABLE "{table}"

347 DROP COLUMN IF EXISTS "{column}";

190 DROP COLUMN IF EXISTS "{column}";

348

191

349− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

192+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

350− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

351− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

193+ DROP FUNCTION IF EXISTS "{trigger_name}";

352 "#,

194 "#,

353 table = self.table,

195 table = self.table,

354 column = self.column,

196 column = self.column,

355 trigger_name = self.trigger_name(ctx),

197 trigger_name = self.trigger_name(ctx),

356− reverse_trigger_name = self.reverse_trigger_name(ctx),

357− null_trigger_name = self.not_null_constraint_trigger_name(ctx),

358 );

198 );

359 db.run(&query)

199 db.run(&query)

360 .context("failed to drop column and down trigger")?;

200 .context("failed to drop column and down trigger")?;

@@ -371,81 +211,21 @@ impl Action for RemoveColumn {

371 }

211 }

372

212

373 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

213 fn abort(&self, ctx: &MigrationContext, db: &mut dyn Conn) -> anyhow::Result<()> {

374− // We might have temporaily removed the NOT NULL check and have to reinstate it

375− let has_not_null_function = !db

376− .query_with_params(

377− "

378− SELECT routine_name

379− FROM information_schema.routines

380− WHERE routine_schema = 'public'

381− AND routine_name = $1

382− ",

383− &[&self.not_null_constraint_trigger_name(ctx)],

384− )

385− .context("failed to get any NOT NULL function")?

386− .is_empty();

387−

388− if has_not_null_function {

389− // Make column NOT NULL again without taking any long lived locks with a temporary constraint

390− let query = format!(

391− r#"

392− ALTER TABLE "{table}"

393− ADD CONSTRAINT "{constraint_name}"

394− CHECK ("{column}" IS NOT NULL) NOT VALID

395− "#,

396− table = self.table,

397− constraint_name = self.not_null_constraint_name(ctx),

398− column = self.column,

399− );

400− db.run(&query)

401− .context("failed to add NOT NULL constraint")?;

402−

403− let query = format!(

404− r#"

405− ALTER TABLE "{table}"

406− VALIDATE CONSTRAINT "{constraint_name}"

407− "#,

408− table = self.table,

409− constraint_name = self.not_null_constraint_name(ctx),

410− );

411− db.run(&query)

412− .context("failed to validate NOT NULL constraint")?;

413−

414− // This ALTER TABLE call will not require any exclusive locks as it can use the validated constraint from above

415− db.run(&format!(

416− r#"

417− ALTER TABLE {table}

418− ALTER COLUMN {column}

419− SET NOT NULL

420− "#,

421− table = self.table,

422− column = self.column

423− ))

424− .context("failed to reinstate column NOT NULL")?;

425−

426− // Drop the temporary constraint

427− let query = format!(

428− r#"

429− ALTER TABLE "{table}"

430− DROP CONSTRAINT "{constraint_name}"

431− "#,

432− table = self.table,

433− constraint_name = self.not_null_constraint_name(ctx),

434− );

435− db.run(&query)

436− .context("failed to drop NOT NULL constraint")?;

437− }

438−

439 // Remove function and trigger

214 // Remove function and trigger

215+ let trigger_table = match &self.down {

216+ Some(Transformation::Update {

217+ table,

218+ value: _,

219+ r#where: _,

220+ }) => table,

221+ _ => &self.table,

222+ };

440 db.run(&format!(

223 db.run(&format!(

441 r#"

224 r#"

442− DROP FUNCTION IF EXISTS "{trigger_name}" CASCADE;

225+ DROP TRIGGER IF EXISTS "{trigger_name}" ON "{trigger_table}";

443− DROP FUNCTION IF EXISTS "{reverse_trigger_name}" CASCADE;

444− DROP FUNCTION IF EXISTS "{null_trigger_name}" CASCADE;

226+ DROP FUNCTION IF EXISTS "{trigger_name}";

445 "#,

227 "#,

446 trigger_name = self.trigger_name(ctx),

228 trigger_name = self.trigger_name(ctx),

447− reverse_trigger_name = self.reverse_trigger_name(ctx),

448− null_trigger_name = self.not_null_constraint_trigger_name(ctx),

449 ))

229 ))

450 .context("failed to drop down trigger")?;

230 .context("failed to drop down trigger")?;

451

231

tests/add_column.rs−99

@@ -260,102 +260,3 @@ fn add_column_with_default() {

260

261 test.run();

262}

263−

264−#[test]

265−fn add_column_with_complex_up() {

266− let mut test = Test::new("Add column complex");

267−

268− test.first_migration(

269− r#"

270− name = "create_tables"

271−

272− [[actions]]

273− type = "create_table"

274− name = "users"

275− primary_key = ["id"]

276−

277− [[actions.columns]]

278− name = "id"

279− type = "INTEGER"

280−

281− [[actions.columns]]

282− name = "email"

283− type = "TEXT"

284−

285− [[actions]]

286− type = "create_table"

287− name = "profiles"

288− primary_key = ["user_id"]

289−

290− [[actions.columns]]

291− name = "user_id"

292− type = "INTEGER"

293− "#,

294− );

295−

296− test.second_migration(

297− r#"

298− name = "add_profiles_email_column"

299−

300− [[actions]]

301− type = "add_column"

302− table = "profiles"

303−

304− [actions.column]

305− name = "email"

306− type = "TEXT"

307− nullable = false

308−

309− [actions.up]

310− table = "users"

311− value = "users.email"

312− where = "profiles.user_id = users.id"

313− "#,

314− );

315−

316− test.after_first(|db| {

317− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test@example.com')")

318− .unwrap();

319− db.simple_query("INSERT INTO profiles (user_id) VALUES (1)")

320− .unwrap();

321− });

322−

323− test.intermediate(|old_db, new_db| {

324− // Ensure email was backfilled on profiles

325− let email: String = new_db

326− .query(

327− "

328− SELECT email

329− FROM profiles

330− WHERE user_id = 1

331− ",

332− &[],

333− )

334− .unwrap()

335− .first()

336− .map(|row| row.get("email"))

337− .unwrap();

338− assert_eq!("test@example.com", email);

339−

340− // Ensure email change in old schema is propagated to profiles table in new schema

341− old_db

342− .simple_query("UPDATE users SET email = 'test2@example.com' WHERE id = 1")

343− .unwrap();

344− let email: String = new_db

345− .query(

346− "

347− SELECT email

348− FROM profiles

349− WHERE user_id = 1

350− ",

351− &[],

352− )

353− .unwrap()

354− .first()

355− .map(|row| row.get("email"))

356− .unwrap();

357− assert_eq!("test2@example.com", email);

358− });

359−

360− test.run();

361−}

tests/complex.rsdeleted−379

@@ -1,379 +0,0 @@

1−mod common;

2−use common::Test;

3−

4−#[test]

5−fn move_column_between_tables() {

6− let mut test = Test::new("Move column between tables");

7−

8− test.first_migration(

9− r#"

10− name = "create_tables"

11−

12− [[actions]]

13− type = "create_table"

14− name = "users"

15− primary_key = ["id"]

16−

17− [[actions.columns]]

18− name = "id"

19− type = "INTEGER"

20−

21− [[actions.columns]]

22− name = "email"

23− type = "TEXT"

24−

25− [[actions]]

26− type = "create_table"

27− name = "profiles"

28− primary_key = ["id"]

29−

30− [[actions.columns]]

31− name = "id"

32− type = "INTEGER"

33−

34− [[actions.columns]]

35− name = "user_id"

36− type = "INTEGER"

37− nullable = false

38− "#,

39− );

40−

41− test.second_migration(

42− r#"

43− name = "move_email_column"

44−

45− [[actions]]

46− type = "add_column"

47− table = "profiles"

48−

49− [actions.column]

50− name = "email"

51− type = "TEXT"

52− nullable = false

53−

54− # When `users` is updated in the old schema, we write the email value to `profiles`

55− # When `profiles` is updated in the old schema, the equivalent `users.email` will also be updated

56− [actions.up]

57− table = "users"

58− value = "users.email"

59− where = "profiles.user_id = users.id"

60−

61− [[actions]]

62− type = "remove_column"

63− table = "users"

64− column = "email"

65−

66− # When `profiles` is changed in the new schema, we write the email address back to the removed column

67− [actions.down]

68− table = "profiles"

69− value = "profiles.email"

70− where = "users.id = profiles.user_id"

71− "#,

72− );

73−

74− test.after_first(|db| {

75− db.simple_query("INSERT INTO users (id, email) VALUES (1, 'test1@test.com')")

76− .unwrap();

77− db.simple_query("INSERT INTO users (id, email) VALUES (2, 'test2@test.com')")

78− .unwrap();

79−

80− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (1, 1)")

81− .unwrap();

82− db.simple_query("INSERT INTO profiles (id, user_id) VALUES (2, 2)")

83− .unwrap();

84− });

85−

86− test.intermediate(|old_db, new_db| {

87− // Ensure emails were backfilled into profiles

88− let profiles_emails: Vec<String> = new_db

89− .query(

90− r#"

91− SELECT email

92− FROM profiles

93− ORDER BY id

94− "#,

95− &[],

96− )

97− .unwrap()

98− .iter()

99− .map(|row| row.get("email"))

100− .collect();

101− assert_eq!(vec!("test1@test.com", "test2@test.com"), profiles_emails);

102−

103− // Ensure insert in old schema updates new

104− old_db

105− .simple_query("INSERT INTO users (id, email) VALUES (3, 'test3@test.com')")

106− .unwrap();

107− old_db

108− .simple_query("INSERT INTO profiles (id, user_id) VALUES (3, 3)")

109− .unwrap();

110− let new_email: String = new_db

111− .query("SELECT email FROM profiles WHERE id = 3", &[])

112− .unwrap()

113− .first()

114− .map(|row| row.get("email"))

115− .unwrap();

116− assert_eq!("test3@test.com", new_email);

117−

118− // Ensure updates in old schema updates new

119− old_db

120− .simple_query("UPDATE users SET email = 'test3+updated@test.com' WHERE id = 3")

121− .unwrap();

122− let new_email: String = new_db

123− .query("SELECT email FROM profiles WHERE id = 3", &[])

124− .unwrap()

125− .first()

126− .map(|row| row.get("email"))

127− .unwrap();

128− assert_eq!("test3+updated@test.com", new_email);

129−

130− // Ensure insert in new schema updates old

131− new_db

132− .simple_query(

133− "INSERT INTO profiles (id, user_id, email) VALUES (4, 4, 'test4@test.com')",

134− )

135− .unwrap();

136− new_db

137− .simple_query("INSERT INTO users (id) VALUES (4)")

138− .unwrap();

139− let old_email: String = old_db

140− .query("SELECT email FROM users WHERE id = 4", &[])

141− .unwrap()

142− .first()

143− .map(|row| row.get("email"))

144− .unwrap();

145− assert_eq!("test4@test.com", old_email);

146−

147− // Ensure update in new schema updates old

148− new_db

149− .simple_query("UPDATE profiles SET email = 'test4+updated@test.com' WHERE id = 4")

150− .unwrap();

151− let old_email: String = old_db

152− .query("SELECT email FROM users WHERE id = 4", &[])

153− .unwrap()

154− .first()

155− .map(|row| row.get("email"))

156− .unwrap();

157− assert_eq!("test4+updated@test.com", old_email);

158− });

159−

160− test.run();

161−}

162−

163−#[test]

164−fn extract_relation_into_new_table() {

165− let mut test = Test::new("Extract relation into new table");

166−

167− test.first_migration(

168− r#"

169− name = "create_tables"

170−

171− [[actions]]

172− type = "create_table"

173− name = "accounts"

174− primary_key = ["id"]

175−

176− [[actions.columns]]

177− name = "id"

178− type = "INTEGER"

179−

180− [[actions]]

181− type = "create_table"

182− name = "users"

183− primary_key = ["id"]

184−

185− [[actions.columns]]

186− name = "id"

187− type = "INTEGER"

188−

189− [[actions.columns]]

190− name = "account_id"

191− type = "INTEGER"

192− nullable = false

193−

194− [[actions.columns]]

195− name = "account_role"

196− type = "TEXT"

197− nullable = false

198− "#,

199− );

200−

201− test.second_migration(

202− r#"

203− name = "add_account_user_connection"

204−

205− [[actions]]

206− type = "create_table"

207− name = "user_account_connections"

208− primary_key = ["account_id", "user_id"]

209−

210− [[actions.columns]]

211− name = "account_id"

212− type = "INTEGER"

213−

214− [[actions.columns]]

215− name = "user_id"

216− type = "INTEGER"

217−

218− [[actions.columns]]

219− name = "role"

220− type = "TEXT"

221− nullable = false

222−

223− [actions.up]

224− table = "users"

225− values = { user_id = "id", account_id = "account_id", role = "UPPER(account_role)" }

226− where = "user_account_connections.user_id = users.id"

227−

228− [[actions]]

229− type = "remove_column"

230− table = "users"

231− column = "account_id"

232−

233− [actions.down]

234− table = "user_account_connections"

235− value = "user_account_connections.account_id"

236− where = "users.id = user_account_connections.user_id"

237−

238− [[actions]]

239− type = "remove_column"

240− table = "users"

241− column = "account_role"

242−

243− [actions.down]

244− table = "user_account_connections"

245− value = "LOWER(user_account_connections.role)"

246− where = "users.id = user_account_connections.user_id"

247− "#,

248− );

249−

250− test.after_first(|db| {

251− db.simple_query("INSERT INTO accounts (id) VALUES (1)")

252− .unwrap();

253− db.simple_query("INSERT INTO users (id, account_id, account_role) VALUES (1, 1, 'admin')")

254− .unwrap();

255− });

256−

257− test.intermediate(|old_db, new_db| {

258− // Ensure connections was backfilled

259− let rows: Vec<(i32, i32, String)> = new_db

260− .query(

261− "

262− SELECT account_id, user_id, role

263− FROM user_account_connections

264− ",

265− &[],

266− )

267− .unwrap()

268− .iter()

269− .map(|row| (row.get("account_id"), row.get("user_id"), row.get("role")))

270− .collect();

271− assert_eq!(1, rows.len());

272−

273− let row = rows.first().unwrap();

274− assert_eq!(1, row.0);

275− assert_eq!(1, row.1);

276− assert_eq!("ADMIN", row.2);

277−

278− // Ensure inserted user in old schema creates a new connection

279− old_db

280− .simple_query(

281− "INSERT INTO users (id, account_id, account_role) VALUES (2, 1, 'developer')",

282− )

283− .unwrap();

284− assert!(

285− new_db

286− .query(

287− "

288− SELECT account_id, user_id, role

289− FROM user_account_connections

290− WHERE account_id = 1 AND user_id = 2 AND role = 'DEVELOPER'

291− ",

292− &[],

293− )

294− .unwrap()

295− .len()

296− == 1

297− );

298−

299− // Ensure NOT NULL constraint still applies to old schema

300− let result = old_db

301− .simple_query(

302− "INSERT INTO users (id, account_id, account_role) VALUES (2, NULL, 'developer')",

303− );

304− assert!(result.is_err());

305−

306− // Ensure updated user role in old schema updates connection in new schema

307− old_db

308− .simple_query("UPDATE users SET account_role = 'admin' WHERE id = 2")

309− .unwrap();

310− assert!(

311− new_db

312− .query(

313− "

314− SELECT account_id, user_id, role

315− FROM user_account_connections

316− WHERE account_id = 1 AND user_id = 2 AND role = 'ADMIN'

317− ",

318− &[],

319− )

320− .unwrap()

321− .len()

322− == 1

323− );

324−

325− // Ensure updated connection in new schema updates old schema user

326− new_db

327− .simple_query(

328− "UPDATE user_account_connections SET role = 'DEVELOPER' WHERE account_id = 1 AND user_id = 2",

329− )

330− .unwrap();

331− assert!(

332− old_db

333− .query(

334− "

335− SELECT id

336− FROM users

337− WHERE id = 2 AND account_id = 1 AND account_role = 'developer'

338− ",

339− &[],

340− )

341− .unwrap()

342− .len()

343− == 1

344− );

345−

346− // Ensure insert of user with connection through new schema updates user in old schema

347− new_db

348− .simple_query(

349− r#"

350− BEGIN;

351− INSERT INTO users (id) VALUES (3);

352− INSERT INTO user_account_connections (user_id, account_id, role) VALUES (3, 1, 'DEVELOPER');

353− COMMIT;

354− "#,

355− )

356− .unwrap();

357− new_db

358− .simple_query(

359− "",

360− )

361− .unwrap();

362− assert!(

363− old_db

364− .query(

365− "

366− SELECT id

367− FROM users

368− WHERE id = 3 AND account_id = 1 AND account_role = 'developer'

369− ",

370− &[],

371− )

372− .unwrap()

373− .len()

374− == 1

375− );

376− });

377−

378− test.run();

379−}

tests/remove_column.rs+2−2

@@ -179,8 +179,8 @@ fn remove_column_with_complex_down() {

179

180 [actions.down]

181 table = "profiles"

182− value = "profiles.email"

182+ value = "email"

183− where = "users.id = profiles.user_id"

183+ where = "id = user_id"

184 "#,

185 );

186