diff --git a/.circleci/config.yml b/.circleci/config.yml index c209b68..d301472 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: working_directory: ~/circleci docker: - image: circleci/python:3.8.6 - - image: circleci/postgres:12 + - image: circleci/postgres:13 environment: POSTGRES_USER: circleci POSTGRES_DB: circleci diff --git a/migra/changes.py b/migra/changes.py index c246fc3..14be97f 100644 --- a/migra/changes.py +++ b/migra/changes.py @@ -432,6 +432,10 @@ def get_selectable_changes( sequences_from, sequences_target, add_dependents_for_modified=True, + tables_only=False, + non_tables_only=False, + drops_only=False, + creations_only=False, ): ( tables_from, @@ -455,53 +459,67 @@ def get_selectable_changes( def functions(d): return {k: v for k, v in d.items() if v.relationtype == "f"} - statements += statements_from_differences( - added_other, - removed_other, - modified_other, - replaceable=replaceable, - drops_only=True, - dependency_ordering=True, - old=selectables_from, - ) + if not tables_only: + if not creations_only: + statements += statements_from_differences( + added_other, + removed_other, + modified_other, + replaceable=replaceable, + drops_only=True, + dependency_ordering=True, + old=selectables_from, + ) - statements += get_table_changes( - tables_from, - tables_target, - enums_from, - enums_target, - sequences_from, - sequences_target, - ) + if not non_tables_only: + statements += get_table_changes( + tables_from, + tables_target, + enums_from, + enums_target, + sequences_from, + sequences_target, + ) - if any([functions(added_other), functions(modified_other)]): - statements += ["set check_function_bodies = off;"] + if not tables_only: + if not drops_only: + if any([functions(added_other), functions(modified_other)]): + statements += ["set check_function_bodies = off;"] - statements += statements_from_differences( - added_other, - removed_other, - modified_other, - replaceable=replaceable, - creations_only=True, - dependency_ordering=True, - old=selectables_from, - ) + statements += statements_from_differences( + added_other, + removed_other, + modified_other, + replaceable=replaceable, + creations_only=True, + dependency_ordering=True, + old=selectables_from, + ) return statements class Changes(object): - def __init__(self, i_from, i_target): + def __init__(self, i_from, i_target, ignore_extension_versions=False): self.i_from = i_from self.i_target = i_target + self.ignore_extension_versions = ignore_extension_versions @property def extensions(self): - return partial( - statements_for_changes, - self.i_from.extensions, - self.i_target.extensions, - modifications_as_alters=True, - ) + if self.ignore_extension_versions: + return partial( + statements_for_changes, + self.i_from.extensions, + self.i_target.extensions, + modifications=False, + ) + else: + return partial( + statements_for_changes, + self.i_from.extensions, + self.i_target.extensions, + modifications_as_alters=True, + ) @property def selectables(self): @@ -515,6 +533,47 @@ class Changes(object): self.i_target.sequences, ) + @property + def tables_only_selectables(self): + return partial( + get_selectable_changes, + od(sorted(self.i_from.selectables.items())), + od(sorted(self.i_target.selectables.items())), + self.i_from.enums, + self.i_target.enums, + self.i_from.sequences, + self.i_target.sequences, + tables_only=True, + ) + + @property + def non_table_selectable_drops(self): + return partial( + get_selectable_changes, + od(sorted(self.i_from.selectables.items())), + od(sorted(self.i_target.selectables.items())), + self.i_from.enums, + self.i_target.enums, + self.i_from.sequences, + self.i_target.sequences, + drops_only=True, + non_tables_only=True, + ) + + @property + def non_table_selectable_creations(self): + return partial( + get_selectable_changes, + od(sorted(self.i_from.selectables.items())), + od(sorted(self.i_target.selectables.items())), + self.i_from.enums, + self.i_target.enums, + self.i_from.sequences, + self.i_target.sequences, + creations_only=True, + non_tables_only=True, + ) + @property def non_pk_constraints(self): a = self.i_from.constraints.items() diff --git a/migra/migra.py b/migra/migra.py index 0e57ef7..2ff5e51 100644 --- a/migra/migra.py +++ b/migra/migra.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals -from sqlbag import raw_execute - from schemainspect import DBInspector, get_inspector +from sqlbag import raw_execute from .changes import Changes from .statements import Statements @@ -88,10 +87,13 @@ class Migration(object): if privileges: self.add(self.changes.privileges(drops_only=True)) self.add(self.changes.non_pk_constraints(drops_only=True)) + + self.add(self.changes.non_table_selectable_drops()) + self.add(self.changes.pk_constraints(drops_only=True)) self.add(self.changes.indexes(drops_only=True)) - self.add(self.changes.selectables()) + self.add(self.changes.tables_only_selectables()) self.add(self.changes.sequences(drops_only=True)) self.add(self.changes.enums(drops_only=True, modifications=False)) @@ -99,6 +101,9 @@ class Migration(object): self.add(self.changes.indexes(creations_only=True)) self.add(self.changes.pk_constraints(creations_only=True)) self.add(self.changes.non_pk_constraints(creations_only=True)) + + self.add(self.changes.non_table_selectable_creations()) + if privileges: self.add(self.changes.privileges(creations_only=True)) self.add(self.changes.rlspolicies(creations_only=True)) diff --git a/tests/FIXTURES/dependencies4/a.sql b/tests/FIXTURES/dependencies4/a.sql new file mode 100644 index 0000000..6de8fa1 --- /dev/null +++ b/tests/FIXTURES/dependencies4/a.sql @@ -0,0 +1 @@ +create table t2(a int); \ No newline at end of file diff --git a/tests/FIXTURES/dependencies4/additions.sql b/tests/FIXTURES/dependencies4/additions.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/dependencies4/b.sql b/tests/FIXTURES/dependencies4/b.sql new file mode 100644 index 0000000..65b2764 --- /dev/null +++ b/tests/FIXTURES/dependencies4/b.sql @@ -0,0 +1,10 @@ +create table t ( + id integer not null primary key, + a text, + b integer +); + +create view v as +select id, a, max(b) +from t +group by id; -- "a" is implied because "id" is primary key diff --git a/tests/FIXTURES/dependencies4/expected.sql b/tests/FIXTURES/dependencies4/expected.sql new file mode 100644 index 0000000..ee500bd --- /dev/null +++ b/tests/FIXTURES/dependencies4/expected.sql @@ -0,0 +1,18 @@ +drop table "public"."t2"; + +create table "public"."t" ( + "id" integer not null, + "a" text, + "b" integer +); + + +CREATE UNIQUE INDEX t_pkey ON public.t USING btree (id); + +alter table "public"."t" add constraint "t_pkey" PRIMARY KEY using index "t_pkey"; + +create or replace view "public"."v" as SELECT t.id, + t.a, + max(t.b) AS max + FROM t + GROUP BY t.id; \ No newline at end of file diff --git a/tests/FIXTURES/dependencies4/expected2.sql b/tests/FIXTURES/dependencies4/expected2.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/everything/expected.sql b/tests/FIXTURES/everything/expected.sql index 32d07be..b17a37d 100644 --- a/tests/FIXTURES/everything/expected.sql +++ b/tests/FIXTURES/everything/expected.sql @@ -2,7 +2,7 @@ create schema if not exists "evenbetterschema"; create extension if not exists "citext" with schema "public" version '1.6'; -create extension if not exists "hstore" with schema "public" version '1.6'; +create extension if not exists "hstore" with schema "public" version '1.7'; create type "public"."bug_status" as enum ('new', 'open', 'closed'); @@ -22,6 +22,10 @@ alter table "public"."products" drop constraint "products_zz_fkey"; alter table "public"."products" drop constraint "x"; +drop materialized view if exists "public"."matvvv"; + +drop view if exists "public"."vvv"; + alter table "public"."aunwanted" drop constraint "aunwanted_pkey"; drop index if exists "public"."aunwanted_pkey"; @@ -36,10 +40,6 @@ drop index if exists "public"."products_x_idx"; drop index if exists "public"."products_x_key"; -drop materialized view if exists "public"."matvvv"; - -drop view if exists "public"."vvv"; - drop table "public"."aunwanted"; drop table "public"."columnless_table"; @@ -116,6 +116,32 @@ alter sequence "public"."bug_id_seq" owned by "public"."bug"."id"; alter sequence "public"."products_product_no_seq" owned by "public"."products"."product_no"; +drop sequence if exists "public"."aunwanted_id_seq"; + +drop sequence if exists "public"."orders_order_id_seq"; + +drop type "public"."unwanted_enum"; + +drop extension if exists "pg_trgm"; + +CREATE UNIQUE INDEX order_items_pkey ON public.order_items USING btree (product_no, order_id); + +CREATE INDEX products_name_idx ON public.products USING btree (name); + +CREATE UNIQUE INDEX products_pkey ON public.products USING btree (product_no); + +alter table "public"."order_items" add constraint "order_items_pkey" PRIMARY KEY using index "order_items_pkey"; + +alter table "public"."products" add constraint "products_pkey" PRIMARY KEY using index "products_pkey"; + +alter table "public"."order_items" add constraint "order_items_order_id_fkey" FOREIGN KEY (order_id) REFERENCES orders(order_id) ON DELETE CASCADE; + +alter table "public"."order_items" add constraint "order_items_product_no_fkey" FOREIGN KEY (product_no) REFERENCES products(product_no) ON DELETE RESTRICT; + +alter table "public"."products" add constraint "y" CHECK ((price > (0)::numeric)); + +alter table "public"."products" add constraint "x" CHECK ((price > (10)::numeric)); + set check_function_bodies = off; CREATE OR REPLACE FUNCTION public.newfunc(i integer, t text[]) @@ -150,32 +176,6 @@ create materialized view "public"."matvvv" as SELECT 2; create or replace view "public"."vvv" as SELECT 2; -drop sequence if exists "public"."aunwanted_id_seq"; - -drop sequence if exists "public"."orders_order_id_seq"; - -drop type "public"."unwanted_enum"; - -drop extension if exists "pg_trgm"; - -CREATE UNIQUE INDEX order_items_pkey ON public.order_items USING btree (product_no, order_id); - -CREATE INDEX products_name_idx ON public.products USING btree (name); - -CREATE UNIQUE INDEX products_pkey ON public.products USING btree (product_no); - -alter table "public"."order_items" add constraint "order_items_pkey" PRIMARY KEY using index "order_items_pkey"; - -alter table "public"."products" add constraint "products_pkey" PRIMARY KEY using index "products_pkey"; - -alter table "public"."order_items" add constraint "order_items_order_id_fkey" FOREIGN KEY (order_id) REFERENCES orders(order_id) ON DELETE CASCADE; - -alter table "public"."order_items" add constraint "order_items_product_no_fkey" FOREIGN KEY (product_no) REFERENCES products(product_no) ON DELETE RESTRICT; - -alter table "public"."products" add constraint "y" CHECK ((price > (0)::numeric)); - -alter table "public"."products" add constraint "x" CHECK ((price > (10)::numeric)); - grant update on table "public"."products" to "postgres"; drop schema if exists "badschema"; diff --git a/tests/FIXTURES/everything/expected2.sql b/tests/FIXTURES/everything/expected2.sql index bbb9e91..cbd5cb6 100644 --- a/tests/FIXTURES/everything/expected2.sql +++ b/tests/FIXTURES/everything/expected2.sql @@ -2,7 +2,7 @@ create schema if not exists "evenbetterschema"; create extension if not exists "citext" with schema "public" version '1.6'; -create extension if not exists "hstore" with schema "public" version '1.6'; +create extension if not exists "hstore" with schema "public" version '1.7'; create type "public"."bug_status" as enum ('new', 'open', 'closed'); @@ -22,6 +22,10 @@ alter table "public"."products" drop constraint "products_zz_fkey"; alter table "public"."products" drop constraint "x"; +drop materialized view if exists "public"."matvvv"; + +drop view if exists "public"."vvv"; + alter table "public"."aunwanted" drop constraint "aunwanted_pkey"; drop index if exists "public"."aunwanted_pkey"; @@ -36,10 +40,6 @@ drop index if exists "public"."products_x_idx"; drop index if exists "public"."products_x_key"; -drop materialized view if exists "public"."matvvv"; - -drop view if exists "public"."vvv"; - drop table "public"."aunwanted"; drop table "public"."columnless_table"; @@ -112,6 +112,32 @@ alter sequence "public"."bug_id_seq" owned by "public"."bug"."id"; alter sequence "public"."products_product_no_seq" owned by "public"."products"."product_no"; +drop sequence if exists "public"."aunwanted_id_seq"; + +drop sequence if exists "public"."orders_order_id_seq"; + +drop type "public"."unwanted_enum"; + +drop extension if exists "pg_trgm"; + +CREATE UNIQUE INDEX order_items_pkey ON public.order_items USING btree (product_no, order_id); + +CREATE INDEX products_name_idx ON public.products USING btree (name); + +CREATE UNIQUE INDEX products_pkey ON public.products USING btree (product_no); + +alter table "public"."order_items" add constraint "order_items_pkey" PRIMARY KEY using index "order_items_pkey"; + +alter table "public"."products" add constraint "products_pkey" PRIMARY KEY using index "products_pkey"; + +alter table "public"."order_items" add constraint "order_items_order_id_fkey" FOREIGN KEY (order_id) REFERENCES orders(order_id) ON DELETE CASCADE; + +alter table "public"."order_items" add constraint "order_items_product_no_fkey" FOREIGN KEY (product_no) REFERENCES products(product_no) ON DELETE RESTRICT; + +alter table "public"."products" add constraint "y" CHECK ((price > (0)::numeric)); + +alter table "public"."products" add constraint "x" CHECK ((price > (10)::numeric)); + set check_function_bodies = off; CREATE OR REPLACE FUNCTION public.newfunc(i integer, t text[]) @@ -146,32 +172,6 @@ create materialized view "public"."matvvv" as SELECT 2; create or replace view "public"."vvv" as SELECT 2; -drop sequence if exists "public"."aunwanted_id_seq"; - -drop sequence if exists "public"."orders_order_id_seq"; - -drop type "public"."unwanted_enum"; - -drop extension if exists "pg_trgm"; - -CREATE UNIQUE INDEX order_items_pkey ON public.order_items USING btree (product_no, order_id); - -CREATE INDEX products_name_idx ON public.products USING btree (name); - -CREATE UNIQUE INDEX products_pkey ON public.products USING btree (product_no); - -alter table "public"."order_items" add constraint "order_items_pkey" PRIMARY KEY using index "order_items_pkey"; - -alter table "public"."products" add constraint "products_pkey" PRIMARY KEY using index "products_pkey"; - -alter table "public"."order_items" add constraint "order_items_order_id_fkey" FOREIGN KEY (order_id) REFERENCES orders(order_id) ON DELETE CASCADE; - -alter table "public"."order_items" add constraint "order_items_product_no_fkey" FOREIGN KEY (product_no) REFERENCES products(product_no) ON DELETE RESTRICT; - -alter table "public"."products" add constraint "y" CHECK ((price > (0)::numeric)); - -alter table "public"."products" add constraint "x" CHECK ((price > (10)::numeric)); - grant update on table "public"."products" to "postgres"; drop schema if exists "badschema"; diff --git a/tests/test_migra.py b/tests/test_migra.py index ab1d2b0..165b886 100644 --- a/tests/test_migra.py +++ b/tests/test_migra.py @@ -4,13 +4,14 @@ import io from difflib import ndiff as difflib_diff import pytest + # import yaml from pytest import raises +from schemainspect import get_inspector from sqlbag import S, load_sql_from_file, temporary_database from migra import Migration, Statements, UnsafeMigrationException from migra.command import parse_args, run -from schemainspect import get_inspector def textdiff(a, b): @@ -43,37 +44,12 @@ def outs(): return io.StringIO(), io.StringIO() -def test_deps(): - for FIXTURE_NAME in ["dependencies", "dependencies2", "dependencies3"]: - do_fixture_test(FIXTURE_NAME) - - -def test_partitioning(): - for FIXTURE_NAME in ["partitioning"]: - do_fixture_test(FIXTURE_NAME) - - -def test_identitycols(): - for FIXTURE_NAME in ["identitycols"]: - do_fixture_test(FIXTURE_NAME) - - -def test_collations(): - for FIXTURE_NAME in ["collations"]: - do_fixture_test(FIXTURE_NAME) - - -def test_triggers(): - for FIXTURE_NAME in ["triggers", "triggers2", "triggers3"]: - do_fixture_test(FIXTURE_NAME) - - -def test_singleschemea(): +def test_singleschema(): for FIXTURE_NAME in ["singleschema"]: do_fixture_test(FIXTURE_NAME, schema="goodschema") -def test_excludeschemea(): +def test_excludeschema(): for FIXTURE_NAME in ["excludeschema"]: do_fixture_test(FIXTURE_NAME, exclude_schema="excludedschema") @@ -85,6 +61,9 @@ def test_singleschema_ext(): fixtures = """\ everything +collations +identitycols +partitioning privileges enumdefaults enumdeps @@ -92,6 +71,13 @@ extversions seq inherit inherit2 +triggers +triggers2 +triggers3 +dependencies +dependencies2 +dependencies3 +dependencies4 """.split() # fixtures = [(_, ) for _ in fixtures] @@ -175,8 +161,11 @@ def do_fixture_test( assert run(args, out=out, err=err) == 2 assert err.getvalue() == "" + output = out.getvalue().strip() + if check_expected: - assert out.getvalue().strip() == EXPECTED + assert output == EXPECTED + ADDITIONS = io.open(fixture_path + "additions.sql").read().strip() EXPECTED2 = io.open(fixture_path + "expected2.sql").read().strip()