Merge pull request #57 from djrobstep/partitioning-and-collations

partitioning, collations support

Makefile

@@ -17,6 +17,7 @@ gitclean:
 clean:
 	find . -name \*.pyc -delete
 	rm -rf .cache
+	rm -rf build
 
 fmt:
 	isort -rc .

deploy/vbump.py

@@ -1,7 +1,8 @@
 from pathlib import Path
 from time import time
 
-from toml import dumps, loads, TomlPreserveInlineDictEncoder as tpide
+from toml import TomlPreserveInlineDictEncoder as tpide
+from toml import dumps, loads
 
 PYPROJECT = "pyproject.toml"
 

migra/changes.py

@@ -16,6 +16,7 @@ THINGS = [
     "indexes",
     "extensions",
     "privileges",
+    "collations",
 ]
 PK = "PRIMARY KEY"
 
@@ -151,8 +152,18 @@ def get_table_changes(tables_from, tables_target, enums_from, enums_target):
     statements += get_enum_modifications(
         tables_from, tables_target, enums_from, enums_target
     )
+
     for t, v in modified.items():
         before = tables_from[t]
+        # attach/detach tables with changed parent tables
+        statements += v.attach_detach_statements(before)
+
+    for t, v in modified.items():
+        before = tables_from[t]
+
+        if not v.is_alterable:
+            continue
+
         c_added, c_removed, c_modified, _ = differences(before.columns, v.columns)
         for k, c in c_removed.items():
             alter = v.alter_table_statement(c.drop_column_clause)
@@ -172,19 +183,11 @@ def get_selectable_changes(
     enums_target,
     add_dependents_for_modified=True,
 ):
-    tables_from = od(
-        (k, v) for k, v in selectables_from.items() if v.relationtype == "r"
-    )
-    tables_target = od(
-        (k, v) for k, v in selectables_target.items() if v.relationtype == "r"
-    )
+    tables_from = od((k, v) for k, v in selectables_from.items() if v.is_table)
+    tables_target = od((k, v) for k, v in selectables_target.items() if v.is_table)
 
-    other_from = od(
-        (k, v) for k, v in selectables_from.items() if v.relationtype != "r"
-    )
-    other_target = od(
-        (k, v) for k, v in selectables_target.items() if v.relationtype != "r"
-    )
+    other_from = od((k, v) for k, v in selectables_from.items() if not v.is_table)
+    other_target = od((k, v) for k, v in selectables_target.items() if not v.is_table)
 
     added_tables, removed_tables, modified_tables, unmodified_tables = differences(
         tables_from, tables_target

migra/migra.py

@@ -64,6 +64,7 @@ class Migration(object):
     def add_all_changes(self, privileges=False):
         self.add(self.changes.schemas(creations_only=True))
         self.add(self.changes.extensions(creations_only=True))
+        self.add(self.changes.collations(creations_only=True))
         self.add(self.changes.enums(creations_only=True, modifications=False))
         self.add(self.changes.sequences(creations_only=True))
         if privileges:
@@ -82,6 +83,7 @@ class Migration(object):
         self.add(self.changes.non_pk_constraints(creations_only=True))
         if privileges:
             self.add(self.changes.privileges(creations_only=True))
+        self.add(self.changes.collations(drops_only=True))
         self.add(self.changes.schemas(drops_only=True))
 
     @property

migra/statements.py

@@ -4,7 +4,7 @@ import re
 
 
 def check_for_drop(s):
-    return not not re.search(r"(drop\s+)", s, re.IGNORECASE)
+    return bool(re.search(r"(drop\s+)", s, re.IGNORECASE))
 
 
 class Statements(list):

pyproject.toml

@@ -24,7 +24,7 @@ pytest-sugar = "*"
 psycopg2-binary = "*"
 flake8 = "*"
 isort = "*"
-black = {version = "*", python = ">=3.6"}
+black = {python=">3.6", version=">=18.9b0", allow_prereleases=true}
 
 [tool.poetry.scripts]
 migra = 'migra:do_command'

tests/FIXTURES/collations/a.sql

@@ -0,0 +1,8 @@
+CREATE COLLATION posix FROM "POSIX";
+
+create table t(
+  a text,
+  b text collate posix
+);
+
+CREATE COLLATION numeric (provider = icu, locale = 'en-u-kn-true');

tests/FIXTURES/collations/b.sql

@@ -0,0 +1,10 @@
+
+
+
+CREATE COLLATION numeric (provider = icu, locale = 'en-u-kn-true');
+
+create table t(
+  a text,
+  b text collate numeric,
+  c text collate numeric
+);

tests/FIXTURES/collations/expected.sql

@@ -0,0 +1,5 @@
+alter table "public"."t" add column "c" text collate "numeric";
+
+alter table "public"."t" alter column "b" set data type text collate "numeric";
+
+drop collation if exists "public"."posix";

tests/FIXTURES/partitioning/a.sql

@@ -0,0 +1,14 @@
+CREATE TABLE measurement (
+    city_id         int not null,
+    logdate         date not null,
+    peaktemp        int,
+    unitsales       int
+) PARTITION BY RANGE (logdate);
+
+CREATE TABLE measurement_y2006m02 PARTITION OF measurement
+    FOR VALUES FROM ('2006-02-01') TO ('2006-03-01');
+
+CREATE TABLE measurement_y2006m03 PARTITION OF measurement
+    FOR VALUES FROM ('2006-03-01') TO ('2006-04-01');
+
+CREATE INDEX ON measurement_y2006m02 (logdate);

tests/FIXTURES/partitioning/b.sql

@@ -0,0 +1,20 @@
+CREATE TABLE measurement (
+    city_id         int not null,
+    logdate         date not null,
+    peaktemp        int,
+    unitsales       int,
+    extra           text
+) PARTITION BY RANGE (logdate);
+
+CREATE TABLE measurement_y2006m02 PARTITION OF measurement
+    FOR VALUES FROM ('2006-02-01') TO ('2006-03-01');
+
+CREATE TABLE measurement_y2006m03 (
+    city_id         int not null,
+    logdate         date not null,
+    peaktemp        int,
+    unitsales       int
+);
+
+
+--CREATE INDEX ON measurement (logdate);

tests/FIXTURES/partitioning/expected.sql

@@ -0,0 +1,5 @@
+drop index if exists "public"."measurement_y2006m02_logdate_idx";
+
+alter table "public"."measurement" detach partition "public"."measurement_y2006m03";
+
+alter table "public"."measurement" add column "extra" text;

tests/test_migra.py

@@ -44,6 +44,16 @@ def test_everything():
         do_fixture_test(FIXTURE_NAME, with_privileges=True)
 
 
+def test_partitioning():
+    for FIXTURE_NAME in ["partitioning"]:
+        do_fixture_test(FIXTURE_NAME)
+
+
+def test_collations():
+    for FIXTURE_NAME in ["collations"]:
+        do_fixture_test(FIXTURE_NAME)
+
+
 def test_singleschemea():
     for FIXTURE_NAME in ["singleschema"]:
         do_fixture_test(FIXTURE_NAME, schema="goodschema")
@@ -71,7 +81,9 @@ def do_fixture_test(
         flags += ["--with-privileges"]
     fixture_path = "tests/FIXTURES/{}/".format(fixture_name)
     EXPECTED = io.open(fixture_path + "expected.sql").read().strip()
-    with temporary_database(host='localhost') as d0, temporary_database(host='localhost') as d1:
+    with temporary_database(host="localhost") as d0, temporary_database(
+        host="localhost"
+    ) as d1:
         with S(d0) as s0, S(d1) as s1:
             load_sql_from_file(s0, fixture_path + "a.sql")
             load_sql_from_file(s1, fixture_path + "b.sql")
@@ -95,26 +107,42 @@ def do_fixture_test(
         assert out.getvalue().strip() == EXPECTED
         ADDITIONS = io.open(fixture_path + "additions.sql").read().strip()
         EXPECTED2 = io.open(fixture_path + "expected2.sql").read().strip()
-        if ADDITIONS:
-            with S(d0) as s0, S(d1) as s1:
-                m = Migration(s0, s1)
-                m.inspect_from()
-                m.inspect_target()
-                with raises(AttributeError):
-                    m.changes.nonexist
-                m.set_safety(False)
+
+        with S(d0) as s0, S(d1) as s1:
+            m = Migration(s0, s1, schema=schema)
+            m.inspect_from()
+            m.inspect_target()
+            with raises(AttributeError):
+                m.changes.nonexist
+            m.set_safety(False)
+            if ADDITIONS:
                 m.add_sql(ADDITIONS)
-                m.apply()
+            m.apply()
+
+            if create_extensions_only:
+                m.add_extension_changes(drops=False)
+            else:
                 m.add_all_changes(privileges=with_privileges)
-                assert m.sql.strip() == EXPECTED2  # sql generated OK
-                m.apply()
-                # check for changes again and make sure none are pending
+
+            expected = EXPECTED2 if ADDITIONS else EXPECTED
+
+            assert m.sql.strip() == expected  # sql generated OK
+
+            m.apply()
+            # check for changes again and make sure none are pending
+            if create_extensions_only:
+                m.add_extension_changes(drops=False)
+                assert (
+                    m.changes.i_from.extensions.items()
+                    >= m.changes.i_target.extensions.items()
+                )
+            else:
                 m.add_all_changes(privileges=with_privileges)
                 assert m.changes.i_from == m.changes.i_target
-                assert not m.statements  # no further statements to apply
-                assert m.sql == ""
-                out, err = outs()
-                assert run(args, out=out, err=err) == 0
+            assert not m.statements  # no further statements to apply
+            assert m.sql == ""
+            out, err = outs()
+            assert run(args, out=out, err=err) == 0
         # test alternative parameters
         with S(d0) as s0, S(d1) as s1:
             m = Migration(get_inspector(s0), get_inspector(s1))