2023-09-22 13:31:49 +03:00
// SPDX-License-Identifier: Apache-2.0
2023-06-28 12:10:03 +03:00
package state
import (
"context"
"database/sql"
"encoding/json"
2023-07-03 16:18:31 +03:00
"errors"
2023-06-28 12:10:03 +03:00
"fmt"
"github.com/lib/pq"
2023-09-22 13:50:31 +03:00
"github.com/xataio/pgroll/pkg/migrations"
"github.com/xataio/pgroll/pkg/schema"
2023-06-28 12:10:03 +03:00
)
const sqlInit = `
CREATE SCHEMA IF NOT EXISTS % [ 1 ] s ;
CREATE TABLE IF NOT EXISTS % [ 1 ] s . migrations (
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
schema NAME NOT NULL ,
name TEXT NOT NULL ,
migration JSONB NOT NULL ,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ,
2023-06-28 12:10:03 +03:00
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
parent TEXT ,
done BOOLEAN NOT NULL DEFAULT false ,
resulting_schema JSONB NOT NULL DEFAULT ' { } ' : : jsonb ,
2023-06-28 12:10:03 +03:00
PRIMARY KEY ( schema , name ) ,
FOREIGN KEY ( schema , parent ) REFERENCES % [ 1 ] s . migrations ( schema , name )
) ;
-- Only one migration can be active at a time
CREATE UNIQUE INDEX IF NOT EXISTS only_one_active ON % [ 1 ] s . migrations ( schema , name , done ) WHERE done = false ;
-- Only first migration can exist without parent
2023-07-11 16:57:54 +03:00
CREATE UNIQUE INDEX IF NOT EXISTS only_first_migration_without_parent ON % [ 1 ] s . migrations ( schema ) WHERE parent IS NULL ;
2023-06-28 12:10:03 +03:00
-- History is linear
CREATE UNIQUE INDEX IF NOT EXISTS history_is_linear ON % [ 1 ] s . migrations ( schema , parent ) ;
2023-11-03 10:08:01 +03:00
-- Add a column to tell whether the row represents an auto - detected DDL capture or a pgroll migration
ALTER TABLE % [ 1 ] s . migrations ADD COLUMN IF NOT EXISTS migration_type
VARCHAR ( 32 )
DEFAULT ' pgroll '
CONSTRAINT migration_type_check CHECK ( migration_type IN ( ' pgroll ' , ' inferred ' )
) ;
2023-06-28 12:10:03 +03:00
-- Helper functions
-- Are we in the middle of a migration ?
CREATE OR REPLACE FUNCTION % [ 1 ] s . is_active_migration_period ( schemaname NAME ) RETURNS boolean
AS $ $ SELECT EXISTS ( SELECT 1 FROM % [ 1 ] s . migrations WHERE schema = schemaname AND done = false ) $ $
LANGUAGE SQL
STABLE ;
-- Get the latest version name ( this is the one with child migrations )
CREATE OR REPLACE FUNCTION % [ 1 ] s . latest_version ( schemaname NAME ) RETURNS text
2023-10-25 10:59:42 +03:00
SECURITY DEFINER
SET search_path = % [ 1 ] s , pg_catalog , pg_temp
2023-07-11 16:57:54 +03:00
AS $ $
SELECT p . name FROM % [ 1 ] s . migrations p
WHERE NOT EXISTS (
SELECT 1 FROM % [ 1 ] s . migrations c WHERE schema = schemaname AND c . parent = p . name
)
AND schema = schemaname $ $
2023-06-28 12:10:03 +03:00
LANGUAGE SQL
STABLE ;
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
2023-07-05 15:58:55 +03:00
-- Get the name of the previous version of the schema , or NULL if there is none .
CREATE OR REPLACE FUNCTION % [ 1 ] s . previous_version ( schemaname NAME ) RETURNS text
AS $ $
2023-11-03 10:08:01 +03:00
WITH RECURSIVE find_ancestor AS (
2024-02-22 19:14:00 +03:00
SELECT schema , name , parent , migration_type FROM % [ 1 ] s . migrations
2023-11-03 10:08:01 +03:00
WHERE name = ( SELECT % [ 1 ] s . latest_version ( schemaname ) ) AND schema = schemaname
UNION ALL
2024-02-22 19:14:00 +03:00
SELECT m . schema , m . name , m . parent , m . migration_type FROM % [ 1 ] s . migrations m
2023-11-03 10:08:01 +03:00
INNER JOIN find_ancestor fa ON fa . parent = m . name AND fa . schema = m . schema
WHERE m . migration_type = ' inferred '
)
SELECT a . parent
FROM find_ancestor AS a
2024-02-22 19:14:00 +03:00
JOIN % [ 1 ] s . migrations AS b ON a . parent = b . name AND a . schema = b . schema
2023-11-03 10:08:01 +03:00
WHERE b . migration_type = ' pgroll ' ;
2023-07-05 15:58:55 +03:00
$ $
LANGUAGE SQL
STABLE ;
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
-- Get the JSON representation of the current schema
CREATE OR REPLACE FUNCTION % [ 1 ] s . read_schema ( schemaname text ) RETURNS jsonb
LANGUAGE plpgsql AS $ $
DECLARE
tables jsonb ;
BEGIN
SELECT json_build_object (
2023-08-29 16:58:24 +03:00
' name ' , schemaname ,
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
' tables ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_object_agg ( t . relname , jsonb_build_object (
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
' name ' , t . relname ,
' oid ' , t . oid ,
' comment ' , descr . description ,
' columns ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_object_agg ( name , c ) , ' { } ' : : json ) FROM (
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
SELECT
attr . attname AS name ,
pg_get_expr ( def . adbin , def . adrelid ) AS default ,
NOT (
attr . attnotnull
OR tp . typtype = 'd'
AND tp . typnotnull
) AS nullable ,
CASE
WHEN ' character varying ' : : regtype = ANY ( ARRAY [ attr . atttypid , tp . typelem ] ) THEN REPLACE (
format_type ( attr . atttypid , attr . atttypmod ) ,
' character varying ' ,
' varchar '
)
WHEN ' timestamp with time zone ' : : regtype = ANY ( ARRAY [ attr . atttypid , tp . typelem ] ) THEN REPLACE (
format_type ( attr . atttypid , attr . atttypmod ) ,
' timestamp with time zone ' ,
' timestamptz '
)
ELSE format_type ( attr . atttypid , attr . atttypmod )
END AS type ,
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
descr . description AS comment ,
( EXISTS (
SELECT 1
FROM pg_constraint
WHERE conrelid = attr . attrelid
2024-02-27 18:16:44 +03:00
AND ARRAY [ attr . attnum : : int ] @ > conkey : : int [ ]
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
AND contype = 'u'
) OR EXISTS (
SELECT 1
FROM pg_index
JOIN pg_class ON pg_class . oid = pg_index . indexrelid
WHERE indrelid = attr . attrelid
AND indisunique
2024-02-27 18:16:44 +03:00
AND ARRAY [ attr . attnum : : int ] @ > pg_index . indkey : : int [ ]
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
) ) AS unique
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
FROM
pg_attribute AS attr
INNER JOIN pg_type AS tp ON attr . atttypid = tp . oid
LEFT JOIN pg_attrdef AS def ON attr . attrelid = def . adrelid
AND attr . attnum = def . adnum
LEFT JOIN pg_description AS descr ON attr . attrelid = descr . objoid
AND attr . attnum = descr . objsubid
WHERE
attr . attnum > 0
AND NOT attr . attisdropped
AND attr . attrelid = t . oid
ORDER BY
attr . attnum
) c
Store indexes in internal schema representation (#57)
Add information about indexes on a table to `pg-roll`'s internal state
storage.
For each table, store an additional JSON object mapping each index name
on the table to details of the index (initially just its name).
An example of the resulting JSON is:
```json
{
"tables": {
"fruits": {
"oid": "16497",
"name": "fruits",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval('_pgroll_new_fruits_id_seq'::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"_pgroll_idx_fruits_name": {
"name": "_pgroll_idx_fruits_name"
},
"_pgroll_new_fruits_pkey": {
"name": "_pgroll_new_fruits_pkey"
},
"_pgroll_new_fruits_name_key": {
"name": "_pgroll_new_fruits_name_key"
}
}
}
}
}
```
Also add fields to the `Schema` model structs to allow the new `indexes`
field to be unmarshalled.
2023-08-17 16:26:44 +03:00
) ,
Store primary keys in `pgroll` internal schema (#135)
Add a `primaryKey` field to each table in the internal schema store to
record the column(s) that make up a table's primary key.
This will be used when backfilling rows
(https://github.com/xataio/pgroll/issues/127)
An example schema now looks like:
```json
{
"name": "public",
"tables": {
"users": {
"oid": "16412",
"name": "users",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval('users_id_seq'::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null,
"indexes": {
"users_pkey": {
"name": "users_pkey"
}
},
"primaryKey": [
"id"
]
}
}
}
```
Where the `primaryKey` field is the new field.
2023-09-26 09:08:21 +03:00
' primaryKey ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_agg ( pg_attribute . attname ) , ' [ ] ' : : json ) AS primary_key_columns
2023-11-03 19:16:04 +03:00
FROM pg_index , pg_attribute
WHERE
indrelid = t . oid AND
nspname = schemaname AND
pg_attribute . attrelid = t . oid AND
pg_attribute . attnum = any ( pg_index . indkey )
AND indisprimary
Store primary keys in `pgroll` internal schema (#135)
Add a `primaryKey` field to each table in the internal schema store to
record the column(s) that make up a table's primary key.
This will be used when backfilling rows
(https://github.com/xataio/pgroll/issues/127)
An example schema now looks like:
```json
{
"name": "public",
"tables": {
"users": {
"oid": "16412",
"name": "users",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval('users_id_seq'::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null,
"indexes": {
"users_pkey": {
"name": "users_pkey"
}
},
"primaryKey": [
"id"
]
}
}
}
```
Where the `primaryKey` field is the new field.
2023-09-26 09:08:21 +03:00
) ,
Store indexes in internal schema representation (#57)
Add information about indexes on a table to `pg-roll`'s internal state
storage.
For each table, store an additional JSON object mapping each index name
on the table to details of the index (initially just its name).
An example of the resulting JSON is:
```json
{
"tables": {
"fruits": {
"oid": "16497",
"name": "fruits",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval('_pgroll_new_fruits_id_seq'::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"_pgroll_idx_fruits_name": {
"name": "_pgroll_idx_fruits_name"
},
"_pgroll_new_fruits_pkey": {
"name": "_pgroll_new_fruits_pkey"
},
"_pgroll_new_fruits_name_key": {
"name": "_pgroll_new_fruits_name_key"
}
}
}
}
}
```
Also add fields to the `Schema` model structs to allow the new `indexes`
field to be unmarshalled.
2023-08-17 16:26:44 +03:00
' indexes ' , (
2024-02-07 15:08:25 +03:00
SELECT COALESCE ( json_object_agg ( ix_details . name , json_build_object (
' name ' , ix_details . name ,
2024-01-24 19:28:35 +03:00
' unique ' , ix_details . indisunique ,
' columns ' , ix_details . columns
2024-02-05 18:35:22 +03:00
) ) , ' { } ' : : json )
2024-01-24 19:28:35 +03:00
FROM (
SELECT
2024-05-16 14:08:58 +03:00
replace ( reverse ( split_part ( reverse ( pi . indexrelid : : regclass : : text ) , '.' , 1 ) ) , '"' , ' ' ) as name ,
2024-01-24 19:28:35 +03:00
pi . indisunique ,
array_agg ( a . attname ) AS columns
FROM pg_index pi
JOIN pg_attribute a ON a . attrelid = pi . indrelid AND a . attnum = ANY ( pi . indkey )
WHERE indrelid = t . oid : : regclass
2024-01-29 19:49:08 +03:00
GROUP BY pi . indexrelid , pi . indisunique
2024-01-24 19:28:35 +03:00
) as ix_details
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
) ,
2024-01-18 15:06:36 +03:00
' checkConstraints ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_object_agg ( cc_details . conname , json_build_object (
2024-01-18 15:06:36 +03:00
' name ' , cc_details . conname ,
' columns ' , cc_details . columns ,
' definition ' , cc_details . definition
2024-02-05 18:35:22 +03:00
) ) , ' { } ' : : json )
2024-01-18 15:06:36 +03:00
FROM (
SELECT
cc_constraint . conname ,
array_agg ( cc_attr . attname ORDER BY cc_constraint . conkey : : int [ ] ) AS columns ,
pg_get_constraintdef ( cc_constraint . oid ) AS definition
FROM pg_constraint AS cc_constraint
INNER JOIN pg_attribute cc_attr ON cc_attr . attrelid = cc_constraint . conrelid AND cc_attr . attnum = ANY ( cc_constraint . conkey )
WHERE cc_constraint . conrelid = t . oid
AND cc_constraint . contype = 'c'
2024-01-29 19:49:08 +03:00
GROUP BY cc_constraint . oid , cc_constraint . conname
2024-01-18 15:06:36 +03:00
) AS cc_details
2024-01-18 15:08:53 +03:00
) ,
' uniqueConstraints ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_object_agg ( uc_details . conname , json_build_object (
2024-01-18 15:08:53 +03:00
' name ' , uc_details . conname ,
' columns ' , uc_details . columns
2024-02-05 18:35:22 +03:00
) ) , ' { } ' : : json )
2024-01-18 15:08:53 +03:00
FROM (
SELECT
uc_constraint . conname ,
array_agg ( uc_attr . attname ORDER BY uc_constraint . conkey : : int [ ] ) AS columns ,
pg_get_constraintdef ( uc_constraint . oid ) AS definition
FROM pg_constraint AS uc_constraint
INNER JOIN pg_attribute uc_attr ON uc_attr . attrelid = uc_constraint . conrelid AND uc_attr . attnum = ANY ( uc_constraint . conkey )
WHERE uc_constraint . conrelid = t . oid
AND uc_constraint . contype = 'u'
2024-01-29 19:49:08 +03:00
GROUP BY uc_constraint . oid , uc_constraint . conname
2024-01-18 15:08:53 +03:00
) AS uc_details
2024-01-18 15:06:36 +03:00
) ,
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
' foreignKeys ' , (
2024-02-05 18:35:22 +03:00
SELECT COALESCE ( json_object_agg ( fk_details . conname , json_build_object (
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
' name ' , fk_details . conname ,
' columns ' , fk_details . columns ,
' referencedTable ' , fk_details . referencedTable ,
2024-03-07 11:59:45 +03:00
' referencedColumns ' , fk_details . referencedColumns ,
' onDelete ' , fk_details . onDelete
2024-02-05 18:35:22 +03:00
) ) , ' { } ' : : json )
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
FROM (
SELECT
fk_constraint . conname ,
array_agg ( fk_attr . attname ORDER BY fk_constraint . conkey : : int [ ] ) AS columns ,
fk_cl . relname AS referencedTable ,
2024-03-07 11:59:45 +03:00
array_agg ( ref_attr . attname ORDER BY fk_constraint . confkey : : int [ ] ) AS referencedColumns ,
CASE
WHEN fk_constraint . confdeltype = 'a' THEN ' NO ACTION '
WHEN fk_constraint . confdeltype = 'r' THEN ' RESTRICT '
WHEN fk_constraint . confdeltype = 'c' THEN ' CASCADE '
WHEN fk_constraint . confdeltype = 'd' THEN ' SET DEFAULT '
WHEN fk_constraint . confdeltype = 'n' THEN ' SET NULL '
END as onDelete
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
FROM pg_constraint AS fk_constraint
INNER JOIN pg_class fk_cl ON fk_constraint . confrelid = fk_cl . oid
INNER JOIN pg_attribute fk_attr ON fk_attr . attrelid = fk_constraint . conrelid AND fk_attr . attnum = ANY ( fk_constraint . conkey )
INNER JOIN pg_attribute ref_attr ON ref_attr . attrelid = fk_constraint . confrelid AND ref_attr . attnum = ANY ( fk_constraint . confkey )
WHERE fk_constraint . conrelid = t . oid
AND fk_constraint . contype = 'f'
2024-03-07 11:59:45 +03:00
GROUP BY fk_constraint . conname , fk_cl . relname , fk_constraint . confdeltype
Add unique & FK constraints info to the schema (#218)
This info is useful to better validate incoming migrations, also it
reflects better the resulting schema
example output:
```
{
"name": "public",
"tables": {
"table1": {
"oid": "66508",
"name": "table1",
"columns": {
"id": {
"name": "id",
"type": "integer",
"unique": true,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": {
"table1_pkey": {
"name": "table1_pkey"
}
},
"primaryKey": [
"id"
],
"foreignKeys": null
},
"table2": {
"oid": "66513",
"name": "table2",
"columns": {
"fk": {
"name": "fk",
"type": "integer",
"unique": false,
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null,
"indexes": null,
"primaryKey": null,
"foreignKeys": {
"fk_fkey": {
"name": "fk_fkey",
"columns": [
"fk"
],
"referencedTable": "table1",
"referencedColumns": [
"id"
]
}
}
}
}
}
```
2023-12-20 16:21:50 +03:00
) AS fk_details
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
)
2024-02-05 18:35:22 +03:00
) ) , ' { } ' : : json ) FROM pg_class AS t
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
INNER JOIN pg_namespace AS ns ON t . relnamespace = ns . oid
LEFT JOIN pg_description AS descr ON t . oid = descr . objoid
AND descr . objsubid = 0
WHERE
ns . nspname = schemaname
AND t . relkind IN ( 'r' , 'p' ) -- tables only ( ignores views , materialized views & foreign tables )
)
)
INTO tables ;
RETURN tables ;
END ;
$ $ ;
2023-08-30 12:50:59 +03:00
CREATE OR REPLACE FUNCTION % [ 1 ] s . raw_migration ( ) RETURNS event_trigger
2023-10-25 10:59:42 +03:00
LANGUAGE plpgsql
SECURITY DEFINER
SET search_path = % [ 1 ] s , pg_catalog , pg_temp AS $ $
2023-08-30 12:50:59 +03:00
DECLARE
schemaname TEXT ;
2024-02-01 18:01:24 +03:00
migration_id TEXT ;
2023-08-30 12:50:59 +03:00
BEGIN
2023-09-22 13:50:31 +03:00
-- Ignore migrations done by pgroll
2023-10-25 10:59:42 +03:00
IF ( pg_catalog . current_setting ( ' pgroll . internal ' , ' TRUE ' ) < > ' TRUE ' ) THEN
2023-08-30 12:50:59 +03:00
RETURN ;
END IF ;
2024-03-06 12:56:52 +03:00
IF tg_event = ' sql_drop ' and tg_tag != ' ALTER TABLE ' THEN
2023-09-19 12:15:38 +03:00
-- Guess the schema from drop commands
2023-10-25 10:59:42 +03:00
SELECT schema_name INTO schemaname FROM pg_catalog . pg_event_trigger_dropped_objects ( ) WHERE schema_name IS NOT NULL ;
2023-09-19 12:15:38 +03:00
ELSIF tg_event = ' ddl_command_end ' THEN
-- Guess the schema from ddl commands , ignore migrations that touch several schemas
2023-10-25 10:59:42 +03:00
IF ( SELECT pg_catalog . count ( DISTINCT schema_name ) FROM pg_catalog . pg_event_trigger_ddl_commands ( ) WHERE schema_name IS NOT NULL ) > 1 THEN
2023-09-19 12:15:38 +03:00
RETURN ;
END IF ;
2023-10-25 10:59:42 +03:00
SELECT schema_name INTO schemaname FROM pg_catalog . pg_event_trigger_ddl_commands ( ) WHERE schema_name IS NOT NULL ;
2023-08-30 12:50:59 +03:00
END IF ;
IF schemaname IS NULL THEN
RETURN ;
END IF ;
-- Ignore migrations done during a migration period
IF % [ 1 ] s . is_active_migration_period ( schemaname ) THEN
RETURN ;
END IF ;
2023-09-22 13:50:31 +03:00
-- Someone did a schema change without pgroll , include it in the history
2024-02-01 18:01:24 +03:00
SELECT INTO migration_id pg_catalog . format ( ' sql_ % % s ' , pg_catalog . substr ( pg_catalog . md5 ( pg_catalog . random ( ) : : text ) , 0 , 15 ) ) ;
2023-11-03 10:08:01 +03:00
INSERT INTO % [ 1 ] s . migrations ( schema , name , migration , resulting_schema , done , parent , migration_type )
2023-08-30 12:50:59 +03:00
VALUES (
schemaname ,
2024-02-01 18:01:24 +03:00
migration_id ,
pg_catalog . json_build_object (
' name ' , migration_id ,
' operations ' , (
SELECT pg_catalog . json_agg (
pg_catalog . json_build_object (
' sql ' , pg_catalog . json_build_object (
' up ' , pg_catalog . current_query ( )
)
)
)
)
) ,
2023-08-30 12:50:59 +03:00
% [ 1 ] s . read_schema ( schemaname ) ,
true ,
2023-11-03 10:08:01 +03:00
% [ 1 ] s . latest_version ( schemaname ) ,
' inferred '
2023-08-30 12:50:59 +03:00
) ;
END ;
$ $ ;
DROP EVENT TRIGGER IF EXISTS pg_roll_handle_ddl ;
CREATE EVENT TRIGGER pg_roll_handle_ddl ON ddl_command_end
2023-09-19 12:15:38 +03:00
EXECUTE FUNCTION % [ 1 ] s . raw_migration ( ) ;
DROP EVENT TRIGGER IF EXISTS pg_roll_handle_drop ;
CREATE EVENT TRIGGER pg_roll_handle_drop ON sql_drop
EXECUTE FUNCTION % [ 1 ] s . raw_migration ( ) ;
2023-06-28 12:10:03 +03:00
`
type State struct {
pgConn * sql . DB
schema string
}
func New ( ctx context . Context , pgURL , stateSchema string ) ( * State , error ) {
2024-04-22 10:38:11 +03:00
dsn , err := pq . ParseURL ( pgURL )
if err != nil {
dsn = pgURL
}
dsn += " search_path=" + stateSchema
conn , err := sql . Open ( "postgres" , dsn )
2023-06-28 12:10:03 +03:00
if err != nil {
return nil , err
}
2023-11-02 13:48:07 +03:00
if err := conn . PingContext ( ctx ) ; err != nil {
return nil , err
}
2023-08-30 12:50:59 +03:00
_ , err = conn . ExecContext ( ctx , "SET LOCAL pgroll.internal to 'TRUE'" )
if err != nil {
return nil , fmt . Errorf ( "unable to set pgroll.internal to true: %w" , err )
}
2023-06-28 12:10:03 +03:00
return & State {
pgConn : conn ,
schema : stateSchema ,
} , nil
}
func ( s * State ) Init ( ctx context . Context ) error {
2024-02-26 12:04:54 +03:00
tx , err := s . pgConn . Begin ( )
if err != nil {
return err
}
defer tx . Rollback ( )
// Try to obtain an advisory lock.
// The key is an arbitrary number, used to distinguish the lock from other locks.
// The lock is automatically released when the transaction is committed or rolled back.
const key int64 = 0x2c03057fb9525b
_ , err = tx . ExecContext ( ctx , "SELECT pg_advisory_xact_lock($1)" , key )
if err != nil {
return err
}
// Perform pgroll state initialization
_ , err = tx . ExecContext ( ctx , fmt . Sprintf ( sqlInit , pq . QuoteIdentifier ( s . schema ) ) )
if err != nil {
return err
}
2023-06-28 12:10:03 +03:00
2024-02-26 12:04:54 +03:00
return tx . Commit ( )
2023-06-28 12:10:03 +03:00
}
func ( s * State ) Close ( ) error {
return s . pgConn . Close ( )
}
2023-07-20 08:37:03 +03:00
func ( s * State ) Schema ( ) string {
return s . schema
}
2023-06-28 12:10:03 +03:00
// IsActiveMigrationPeriod returns true if there is an active migration
func ( s * State ) IsActiveMigrationPeriod ( ctx context . Context , schema string ) ( bool , error ) {
var isActive bool
err := s . pgConn . QueryRowContext ( ctx , fmt . Sprintf ( "SELECT %s.is_active_migration_period($1)" , pq . QuoteIdentifier ( s . schema ) ) , schema ) . Scan ( & isActive )
if err != nil {
return false , err
}
2023-08-30 12:50:59 +03:00
return isActive , nil
2023-06-28 12:10:03 +03:00
}
// GetActiveMigration returns the name & raw content of the active migration (if any), errors out otherwise
func ( s * State ) GetActiveMigration ( ctx context . Context , schema string ) ( * migrations . Migration , error ) {
var name , rawMigration string
err := s . pgConn . QueryRowContext ( ctx , fmt . Sprintf ( "SELECT name, migration FROM %s.migrations WHERE schema=$1 AND done=false" , pq . QuoteIdentifier ( s . schema ) ) , schema ) . Scan ( & name , & rawMigration )
if err != nil {
2023-07-03 16:18:31 +03:00
if errors . Is ( err , sql . ErrNoRows ) {
return nil , ErrNoActiveMigration
}
2023-06-28 12:10:03 +03:00
return nil , err
}
var migration migrations . Migration
err = json . Unmarshal ( [ ] byte ( rawMigration ) , & migration )
if err != nil {
return nil , fmt . Errorf ( "unable to unmarshal migration: %w" , err )
}
return & migration , nil
}
2023-07-11 13:07:18 +03:00
// LatestVersion returns the name of the latest version schema
func ( s * State ) LatestVersion ( ctx context . Context , schema string ) ( * string , error ) {
var version * string
err := s . pgConn . QueryRowContext ( ctx ,
fmt . Sprintf ( "SELECT %s.latest_version($1)" , pq . QuoteIdentifier ( s . schema ) ) ,
schema ) . Scan ( & version )
if err != nil {
return nil , err
}
return version , nil
}
// PreviousVersion returns the name of the previous version schema
2023-07-05 15:58:55 +03:00
func ( s * State ) PreviousVersion ( ctx context . Context , schema string ) ( * string , error ) {
var parent * string
err := s . pgConn . QueryRowContext ( ctx ,
fmt . Sprintf ( "SELECT %s.previous_version($1)" , pq . QuoteIdentifier ( s . schema ) ) ,
2023-07-11 13:07:18 +03:00
schema ) . Scan ( & parent )
2023-07-05 15:58:55 +03:00
if err != nil {
return nil , err
}
return parent , nil
}
2023-11-22 15:36:54 +03:00
// Status returns the current migration status of the specified schema
func ( s * State ) Status ( ctx context . Context , schema string ) ( * Status , error ) {
latestVersion , err := s . LatestVersion ( ctx , schema )
if err != nil {
return nil , err
}
if latestVersion == nil {
latestVersion = new ( string )
}
isActive , err := s . IsActiveMigrationPeriod ( ctx , schema )
if err != nil {
return nil , err
}
var status MigrationStatus
if * latestVersion == "" {
status = NoneMigrationStatus
} else if isActive {
status = InProgressMigrationStatus
} else {
status = CompleteMigrationStatus
}
return & Status {
Schema : schema ,
Version : * latestVersion ,
Status : status ,
} , nil
}
2023-07-03 16:18:31 +03:00
// Start creates a new migration, storing its name and raw content
2023-06-28 12:10:03 +03:00
// this will effectively activate a new migration period, so `IsActiveMigrationPeriod` will return true
// until the migration is completed
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
// This method will return the current schema (before the migration is applied)
func ( s * State ) Start ( ctx context . Context , schemaname string , migration * migrations . Migration ) ( * schema . Schema , error ) {
2023-06-28 12:10:03 +03:00
rawMigration , err := json . Marshal ( migration )
if err != nil {
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
return nil , fmt . Errorf ( "unable to marshal migration: %w" , err )
2023-06-28 12:10:03 +03:00
}
2023-09-21 23:41:38 +03:00
// create a new migration object and return the previous known schema
// if there is no previous migration, read the schema from postgres
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
stmt := fmt . Sprintf ( `
INSERT INTO % [ 1 ] s . migrations ( schema , name , parent , migration ) VALUES ( $ 1 , $ 2 , % [ 1 ] s . latest_version ( $ 1 ) , $ 3 )
RETURNING (
SELECT COALESCE (
( SELECT resulting_schema FROM % [ 1 ] s . migrations WHERE schema = $ 1 AND name = % [ 1 ] s . latest_version ( $ 1 ) ) ,
2023-09-21 23:41:38 +03:00
% [ 1 ] s . read_schema ( $ 1 ) )
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
) ` , pq . QuoteIdentifier ( s . schema ) )
2023-06-28 12:10:03 +03:00
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
var rawSchema string
err = s . pgConn . QueryRowContext ( ctx , stmt , schemaname , migration . Name , rawMigration ) . Scan ( & rawSchema )
if err != nil {
return nil , err
}
var schema schema . Schema
err = json . Unmarshal ( [ ] byte ( rawSchema ) , & schema )
if err != nil {
return nil , fmt . Errorf ( "unable to unmarshal schema: %w" , err )
}
return & schema , nil
2023-06-28 12:10:03 +03:00
}
// Complete marks a migration as completed
func ( s * State ) Complete ( ctx context . Context , schema , name string ) error {
Make migrations schema aware (#12)
This change will retrieve and store the resulting schema after a
migration is completed. This schema will be used as the base to execute
the next migration, making it possible to create views that are aware of
the full schema, and not only the one created by the last migration.
We use a function to retrieve the schema directly from Postgres instead
of building it from the migration files. This allows for more features
in the future, like doing an initial sync on top of the existing schema
or automatically detecting and storing out of band migrations from
triggers.
Example JSON stored schema:
```
{
"tables": {
"bills": {
"oid": "18272",
"name": "bills",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"date": {
"name": "date",
"type": "time with time zone",
"comment": null,
"default": null,
"nullable": false
},
"quantity": {
"name": "quantity",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"products": {
"oid": "18286",
"name": "products",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": "nextval(_pgroll_new_products_id_seq::regclass)",
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"price": {
"name": "price",
"type": "numeric(10,2)",
"comment": null,
"default": null,
"nullable": false
}
},
"comment": null
},
"customers": {
"oid": "18263",
"name": "customers",
"columns": {
"id": {
"name": "id",
"type": "integer",
"comment": null,
"default": null,
"nullable": false
},
"name": {
"name": "name",
"type": "varchar(255)",
"comment": null,
"default": null,
"nullable": false
},
"credit_card": {
"name": "credit_card",
"type": "text",
"comment": null,
"default": null,
"nullable": true
}
},
"comment": null
}
}
}
```
After this change, I believe that the `create_table` operation is
feature complete and can be used for many sequential migrations.
2023-07-05 14:20:59 +03:00
res , err := s . pgConn . ExecContext ( ctx , fmt . Sprintf ( "UPDATE %[1]s.migrations SET done=$1, resulting_schema=(SELECT %[1]s.read_schema($2)) WHERE schema=$2 AND name=$3 AND done=$4" , pq . QuoteIdentifier ( s . schema ) ) , true , schema , name , false )
2023-06-28 12:10:03 +03:00
if err != nil {
return err
}
rows , err := res . RowsAffected ( )
if err != nil {
return err
}
if rows == 0 {
return fmt . Errorf ( "no migration found with name %s" , name )
}
return err
}
Reimplement `analyze` subcommand (#24)
The hidden `analyze` subcommand to dump the inferred database schema to
stdout was implemented in #1.
We've since changed how the schema is inferred (in #12).
This PR updates the `analyze` command to use the schema retrieval
implemented in #12.
Example
`go run . analyze public`:
```json
{
"tables": {
"bills": {
"oid": "16417",
"name": "bills",
"comment": "",
"columns": {
"date": {
"name": "date",
"type": "time with time zone",
"default": null,
"nullable": false,
"comment": ""
},
"id": {
"name": "id",
"type": "integer",
"default": null,
"nullable": false,
"comment": ""
},
"quantity": {
"name": "quantity",
"type": "integer",
"default": null,
"nullable": false,
"comment": ""
}
}
},
"customers": {
"oid": "16408",
"name": "customers",
"comment": "",
"columns": {
"credit_card": {
"name": "credit_card",
"type": "text",
"default": null,
"nullable": true,
"comment": ""
},
"id": {
"name": "id",
"type": "integer",
"default": null,
"nullable": false,
"comment": ""
},
"name": {
"name": "name",
"type": "varchar(255)",
"default": null,
"nullable": false,
"comment": ""
}
}
}
}
}
```
2023-07-11 10:13:23 +03:00
func ( s * State ) ReadSchema ( ctx context . Context , schemaName string ) ( * schema . Schema , error ) {
var rawSchema [ ] byte
err := s . pgConn . QueryRowContext ( ctx , fmt . Sprintf ( "SELECT %s.read_schema($1)" , s . schema ) , schemaName ) . Scan ( & rawSchema )
if err != nil {
return nil , err
}
var sc schema . Schema
err = json . Unmarshal ( rawSchema , & sc )
if err != nil {
return nil , fmt . Errorf ( "unable to unmarshal schema: %w" , err )
}
return & sc , nil
}
2023-06-28 12:10:03 +03:00
// Rollback removes a migration from the state (we consider it rolled back, as if it never started)
func ( s * State ) Rollback ( ctx context . Context , schema , name string ) error {
res , err := s . pgConn . ExecContext ( ctx , fmt . Sprintf ( "DELETE FROM %s.migrations WHERE schema=$1 AND name=$2 AND done=$3" , pq . QuoteIdentifier ( s . schema ) ) , schema , name , false )
if err != nil {
return err
}
rows , err := res . RowsAffected ( )
if err != nil {
return err
}
if rows == 0 {
return fmt . Errorf ( "no migration found with name %s" , name )
}
return nil
}