Division in Columns within InDB is integer based if both columns are integers (#4057)

Fixes https://www.pivotaltracker.com/story/show/184073099

# Important Notes
- Since now the only operator on columns for division, `/`, returns floats, it may be worth creating an additional `div` operator exposing integer division. But that will be done as a separate task aligning column operator APIs.

distribution/lib/Standard/Database/0.0.0-dev/src/Data/Column.enso

@@ -377,7 +377,7 @@ type Column
        by `other`.  If `other` is a column, the operation is performed pairwise
        between corresponding elements of `self` and `other`.
     / : Column | Any -> Column
-    / self other = self.make_binary_op "/" other
+    / self other = self.make_binary_op "/" other new_type=SQL_Type.double
 
     ## Element-wise modulus.
 

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso

@@ -96,7 +96,7 @@ make_internal_generator_dialect =
     cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]]
     text = [starts_with, contains, ends_with, agg_shortest, agg_longest]+concat_ops+cases
     counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]]
-    arith_extensions = [is_nan]
+    arith_extensions = [is_nan, decimal_div]
     bool = [bool_or]
 
     stddev_pop = ["STDDEV_POP", Base_Generator.make_function "stddev_pop"]
@@ -288,3 +288,7 @@ is_nan = Base_Generator.lift_unary_op "IS_NAN" arg->
 ## PRIVATE
 bool_or = Base_Generator.lift_unary_op "BOOL_OR" arg->
     code "bool_or(" ++ arg ++ ")"
+
+## PRIVATE
+decimal_div = Base_Generator.lift_binary_op "/" x-> y->
+    code "CAST(" ++ x ++ " AS double precision) / CAST(" ++ y ++ " AS double precision)"

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Dialect.enso

@@ -115,9 +115,10 @@ make_internal_generator_dialect =
     text = [starts_with, contains, ends_with]+concat_ops
     counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]]
     stats = [agg_stddev_pop, agg_stddev_samp]
+    arith_extensions = [decimal_div]
 
     bool = [bool_or]
-    my_mappings = text + counts + stats + bool
+    my_mappings = text + counts + stats + arith_extensions + bool
     Base_Generator.base_dialect . extend_with my_mappings
 
 ## PRIVATE
@@ -244,3 +245,7 @@ contains = Base_Generator.lift_binary_op "contains" make_contains_expr
 ## PRIVATE
 bool_or = Base_Generator.lift_unary_op "BOOL_OR" arg->
     code "max(" ++ arg ++ ")"
+
+## PRIVATE
+decimal_div = Base_Generator.lift_binary_op "/" x-> y->
+    code "CAST(" ++ x ++ " AS REAL) / CAST(" ++ y ++ " AS REAL)"

std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/DoubleNumericOp.java

@@ -19,6 +19,10 @@ public abstract class DoubleNumericOp extends MapOperation<Double, DoubleStorage
 
   @Override
   public Storage<Double> runMap(DoubleStorage storage, Object arg) {
+    if (arg == null) {
+      return DoubleStorage.makeEmpty(storage.size());
+    }
+
     double x;
     if (arg instanceof Double) {
       x = (Double) arg;
@@ -27,6 +31,7 @@ public abstract class DoubleNumericOp extends MapOperation<Double, DoubleStorage
     } else {
       throw new UnexpectedTypeException("a Number.");
     }
+
     long[] out = new long[storage.size()];
     for (int i = 0; i < storage.size(); i++) {
       if (!storage.isNa(i)) {

std-bits/table/src/main/java/org/enso/table/data/column/operation/map/numeric/LongNumericOp.java

@@ -28,7 +28,13 @@ public abstract class LongNumericOp extends MapOperation<Long, LongStorage> {
 
   @Override
   public NumericStorage<?> runMap(LongStorage storage, Object arg) {
-    if (!alwaysCastToDouble && arg instanceof Long x) {
+    if (arg == null) {
+      if (alwaysCastToDouble) {
+        return DoubleStorage.makeEmpty(storage.size());
+      } else {
+        return LongStorage.makeEmpty(storage.size());
+      }
+    } else if (!alwaysCastToDouble && arg instanceof Long x) {
       long[] newVals = new long[storage.size()];
       for (int i = 0; i < storage.size(); i++) {
         if (!storage.isNa(i)) {

std-bits/table/src/main/java/org/enso/table/data/column/storage/BoolStorage.java

@@ -32,6 +32,12 @@ public final class BoolStorage extends Storage<Boolean> {
     this.negated = negated;
   }
 
+  public static BoolStorage makeEmpty(int size) {
+    BitSet isMissing = new BitSet(size);
+    isMissing.set(0, size);
+    return new BoolStorage(new BitSet(), isMissing, size, false);
+  }
+
   @Override
   public int size() {
     return size;
@@ -207,7 +213,9 @@ public final class BoolStorage extends Storage<Boolean> {
             new MapOperation<>(Maps.EQ) {
               @Override
               public BoolStorage runMap(BoolStorage storage, Object arg) {
-                if (arg instanceof Boolean v) {
+                if (arg == null) {
+                  return BoolStorage.makeEmpty(storage.size);
+                } else if (arg instanceof Boolean v) {
                   if (v) {
                     return storage;
                   } else {
@@ -239,7 +247,9 @@ public final class BoolStorage extends Storage<Boolean> {
             new MapOperation<>(Maps.AND) {
               @Override
               public BoolStorage runMap(BoolStorage storage, Object arg) {
-                if (arg instanceof Boolean v) {
+                if (arg == null) {
+                  return BoolStorage.makeEmpty(storage.size);
+                } else if (arg instanceof Boolean v) {
                   if (v) {
                     return storage;
                   } else {
@@ -281,7 +291,9 @@ public final class BoolStorage extends Storage<Boolean> {
             new MapOperation<>(Maps.OR) {
               @Override
               public BoolStorage runMap(BoolStorage storage, Object arg) {
-                if (arg instanceof Boolean v) {
+                if (arg == null) {
+                  return BoolStorage.makeEmpty(storage.size);
+                } else if (arg instanceof Boolean v) {
                   if (v) {
                     return new BoolStorage(new BitSet(), storage.isMissing, storage.size, true);
                   } else {

std-bits/table/src/main/java/org/enso/table/data/column/storage/DoubleStorage.java

@@ -32,6 +32,12 @@ public final class DoubleStorage extends NumericStorage<Double> {
     this.size = size;
   }
 
+  public static DoubleStorage makeEmpty(int size) {
+    BitSet isMissing = new BitSet(size);
+    isMissing.set(0, size);
+    return new DoubleStorage(new long[0], size, new BitSet(size));
+  }
+
   /** @inheritDoc */
   @Override
   public int size() {

std-bits/table/src/main/java/org/enso/table/data/column/storage/LongStorage.java

@@ -32,6 +32,12 @@ public final class LongStorage extends NumericStorage<Long> {
     this.size = size;
   }
 
+  public static LongStorage makeEmpty(int size) {
+    BitSet isMissing = new BitSet(size);
+    isMissing.set(0, size);
+    return new LongStorage(new long[0], size, isMissing);
+  }
+
   public LongStorage(long[] data) {
     this(data, data.length, new BitSet());
   }

test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso

@@ -12,8 +12,28 @@ main = run_default_backend spec
 spec setup =
     prefix = setup.prefix
     table_builder = setup.table_builder
-    Test.group prefix+"Column Operations" <|
-        #inmem_todo = if setup.is_database.not then "New Null-equality is not yet implemented in In-memory backend."
+    Test.group prefix+"Basic Column Operations" <|
+        Test.specify "iif" <|
+            t = table_builder [["X", [True, False, Nothing, True]]]
+            t.at "X" . iif 22 33 . to_vector . should_equal [22, 33, Nothing, 22]
+
+        Test.specify "iif on Columns" pending="Not implemented yet." Nothing
+
+    t2 = table_builder [["x", [1, 4, 5, Nothing]], ["y", [2, 3, 5, Nothing]], ["b", [False, False, True, Nothing]]]
+    x = t2.at "x"
+    y = t2.at "y"
+    b = t2.at "b"
+    Test.group prefix+"Column Operations - Equality & Null Handling" <|
+        Test.specify "should provide basic == and != comparisons" pending="TODO figure out proper null handling" <|
+            (x == y).to_vector . should_equal [False, False, True, Nothing]
+            (x != y).to_vector . should_equal [True, True, False, Nothing]
+            (x == 4).to_vector . should_equal [False, True, False, Nothing]
+            (x == Nothing).to_vector . should_equal [Nothing, Nothing, Nothing, Nothing]
+
+        Test.specify "should allow to check which values are null"
+            x.is_missing.to_vector . should_equal [False, False, False, True]
+            (x + Nothing).is_missing.to_vector . should_equal [True, True, True, True]
+
         Test.specify "Column equality should handle nulls correctly" pending="TODO" <|
             a = [2, 3, Nothing, Nothing]
             b = [2, 4, Nothing, 5]
@@ -23,6 +43,17 @@ spec setup =
             t = table_builder [["A", a], ["B", b]]
             ((t.at "A") == (t.at "B")) . to_vector . should_equal r
 
+        Test.specify "equals_ignore_case for ASCII strings" <|
+            # TODO test for nothing too
+            x = ["a", "B", "c", "DEF"]
+            y = ["aa", "b", "c", "dEf"]
+            r = [False, True, True, True]
+
+            x.zip y (.equals_ignore_case) . should_equal r
+
+            t = table_builder [["X", x], ["Y", y]]
+            (t.at "X") . equals_ignore_case (t.at "Y") . to_vector . should_equal r
+
         Test.specify "Text Column equality (including case-insensitive) should handle nulls correctly" pending="TODO" <|
             a = ["Z", "a", "b", Nothing, Nothing]
             b = ["Z", "A", "C", Nothing, "d"]
@@ -36,6 +67,49 @@ spec setup =
             ((t.at "A") == (t.at "B")) . to_vector . should_equal r_sensitive
             ((t.at "A").equals_ignore_case (t.at "B")) . to_vector . should_equal r_insensitive
 
+    Test.group prefix+"Arithmetic and Boolean Column Operations" <|
+        Test.specify "should allow basic operations" <|
+            (x + y).to_vector . should_equal [3, 7, 10, Nothing]
+            (x - y).to_vector . should_equal [-1, 1, 0, Nothing]
+            (x * y).to_vector . should_equal [2, 12, 25, Nothing]
+            (x < y).to_vector . should_equal [True, False, False, Nothing]
+            (x <= y).to_vector . should_equal [True, False, True, Nothing]
+            (x > y).to_vector . should_equal (x <= y).not.to_vector
+            (x >= y).to_vector . should_equal (x < y).not.to_vector
+            #(((x < y) || (x == y)) == (x <= y)).to_vector . should_equal [True, True, True, Nothing]
+            (b || b.not).to_vector . should_equal [True, True, True, Nothing]
+
+        Test.specify "should allow combining a column with a scalar" pending="TODO null handling" <|
+            (x + 100).to_vector . should_equal [101, 104, 105, Nothing]
+            (x * 10).to_vector . should_equal [10, 40, 50, Nothing]
+            (x - 10).to_vector . should_equal [-9, -6, -5, Nothing]
+            (x < 1000).to_vector . should_equal [True, True, True, Nothing]
+            (b || False).to_vector . should_equal [False, False, True, Nothing]
+            (b || True).to_vector . should_equal [True, True, True, True]
+            (b && False).to_vector . should_equal [False, False, False, False]
+
+        Test.specify "division should be aligned with the Enso arithmetic" <|
+            a = [1, 5, 10, 100]
+            b = [2, 2, 4, 5]
+            r = [0.5, 2.5, 2.5, 20.0]
+            a.zip b (/) . should_equal r
+
+            t = table_builder [["A", a], ["B", b]]
+            r2 = (t.at "A") / (t.at "B")
+            r2 . to_vector . should_equal r
+
+            r3 = (t.at "A") / 2
+            r3 . to_vector . should_equal [0.5, 2.5, 5.0, 50.0]
+
+        Test.specify "should return null if one of arguments is missing" pending="TODO null handling" <|
+            nulls = [Nothing, Nothing, Nothing, Nothing]
+            (x + Nothing).to_vector . should_equal nulls
+            (x - Nothing).to_vector . should_equal nulls
+            (x * Nothing).to_vector . should_equal nulls
+            (x / Nothing).to_vector . should_equal nulls
+            (b && Nothing).to_vector . should_equal nulls
+            (b || Nothing).to_vector . should_equal nulls
+
         Test.specify "Between should return null if any of the values are null" pending="TODO" <|
             a = [2, 3, Nothing, 7,       5,       Nothing]
             b = [0, 5, 7,       Nothing, 7,       Nothing]
@@ -45,19 +119,19 @@ spec setup =
             t = table_builder [["A", a], ["B", b], ["C", c]]
             ((t.at "A").between (t.at "B") (t.at "C")) . to_vector . should_equal r
 
-        Test.specify "iif" <|
-            t = table_builder [["X", [True, False, Nothing, True]]]
-            t.at "X" . iif 22 33 . to_vector . should_equal [22, 33, Nothing, 22]
+    Test.group prefix+"Column Operations - Text" <|
+        t3 = table_builder [["s1", ["foobar", "bar", "baz", Nothing]], ["s2", ["foo", "ar", "a", Nothing]]]
+        s1 = t3.at "s1"
+        s2 = t3.at "s2"
+        Test.specify "should handle basic Text operations" <|
+            s1.starts_with s2 . to_vector . should_equal [True, False, False, Nothing]
+            s1.starts_with "foo" . to_vector . should_equal [True, False, False, Nothing]
+            s1.starts_with "ba" . to_vector . should_equal [False, True, True, Nothing]
 
-        Test.specify "iif on Columns" pending="Not implemented yet." Nothing
+            s1.contains s2 . to_vector . should_equal [True, True, True, Nothing]
+            s1.contains "a" . to_vector . should_equal [True, True, True, Nothing]
+            s1.contains "oo" . to_vector . should_equal [True, False, False, Nothing]
 
-        Test.specify "equals_ignore_case for ASCII" <|
-            # TODO test for nothing too
-            x = ["a", "B", "c", "DEF"]
-            y = ["aa", "b", "c", "dEf"]
-            r = [False, True, True, True]
-
-            x.zip y (.equals_ignore_case) . should_equal r
-
-            t = table_builder [["X", x], ["Y", y]]
-            (t.at "X") . equals_ignore_case (t.at "Y") . to_vector . should_equal r
+            s1.ends_with s2 . to_vector . should_equal [False, True, False, Nothing]
+            s1.ends_with "ar" . to_vector . should_equal [True, True, False, Nothing]
+            s1.ends_with "a" . to_vector . should_equal [False, False, False, Nothing]

test/Table_Tests/src/Database/Codegen_Spec.enso

@@ -75,10 +75,10 @@ spec =
             c = t1.at "C"
             arith = (a * 2) + 1
             logic = (c || c.not) && True
-            cmp = (a / a >= b) && (a - b < a)
+            cmp = (a * a >= b) && (a - b < a)
             arith.to_sql.prepare . should_equal ['SELECT (("T1"."A" * ?) + ?) AS "A" FROM "T1" AS "T1"', [[2, int], [1, int]]]
             logic.to_sql.prepare . should_equal ['SELECT (("T1"."C" OR (NOT "T1"."C")) AND ?) AS "C" FROM "T1" AS "T1"', [[True, bool]]]
-            cmp.to_sql.prepare . should_equal ['SELECT ((("T1"."A" / "T1"."A") >= "T1"."B") AND (("T1"."A" - "T1"."B") < "T1"."A")) AS "A" FROM "T1" AS "T1"', []]
+            cmp.to_sql.prepare . should_equal ['SELECT ((("T1"."A" * "T1"."A") >= "T1"."B") AND (("T1"."A" - "T1"."B") < "T1"."A")) AS "A" FROM "T1" AS "T1"', []]
 
         Test.specify "should support simple text operations" <|
             b = t1.at "B"

test/Table_Tests/src/Database/Common_Spec.enso

@@ -36,56 +36,6 @@ spec prefix connection =
             table = upload "Big" original
             table.read.row_count . should_equal n
 
-    Test.group prefix+"Mapping Operations" <|
-        t2 = upload "T2" <| Table.new [["x", [1, 4, 5, Nothing]], ["y", [2, 3, 5, Nothing]], ["b", [False, False, True, Nothing]]]
-        x = t2.at "x"
-        y = t2.at "y"
-        b = t2.at "b"
-        # TODO move these to Column_Operations_Spec
-        Test.specify "should allow combining columns with supported operations" <|
-            (x + y).to_vector . should_equal [3, 7, 10, Nothing]
-            (x - y).to_vector . should_equal [-1, 1, 0, Nothing]
-            (x * y).to_vector . should_equal [2, 12, 25, Nothing]
-            (x / y).to_vector . should_equal [0, 1, 1, Nothing]
-            #(x == y).to_vector . should_equal [False, False, True, Nothing]
-            #(x != y).to_vector . should_equal [True, True, False, Nothing]
-            (x < y).to_vector . should_equal [True, False, False, Nothing]
-            (x <= y).to_vector . should_equal [True, False, True, Nothing]
-            (x > y).to_vector . should_equal (x <= y).not.to_vector
-            (x >= y).to_vector . should_equal (x < y).not.to_vector
-            #(((x < y) || (x == y)) == (x <= y)).to_vector . should_equal [True, True, True, Nothing]
-            (b || b.not).to_vector . should_equal [True, True, True, Nothing]
-
-        Test.specify "should allow casting constants to be applied to the whole column" <|
-            (x + 100).to_vector . should_equal [101, 104, 105, Nothing]
-            (x * 10).to_vector . should_equal [10, 40, 50, Nothing]
-            (x / 2).to_vector . should_equal [0, 2, 2, Nothing]
-            (x - 10).to_vector . should_equal [-9, -6, -5, Nothing]
-            #(x == 4).to_vector . should_equal [False, True, False, Nothing]
-            (x < 1000).to_vector . should_equal [True, True, True, Nothing]
-            (b || False).to_vector . should_equal [False, False, True, Nothing]
-            (b || True).to_vector . should_equal [True, True, True, True]
-            (b && False).to_vector . should_equal [False, False, False, False]
-            (x + Nothing).to_vector . should_equal [Nothing, Nothing, Nothing, Nothing]
-            x.is_missing.to_vector . should_equal [False, False, False, True]
-            #(x == Nothing).to_vector . should_equal [Nothing, Nothing, Nothing, Nothing]
-
-        t3 = upload "T3" <| Table.new [["s1", ["foobar", "bar", "baz", Nothing]], ["s2", ["foo", "ar", "a", Nothing]]]
-        s1 = t3.at "s1"
-        s2 = t3.at "s2"
-        Test.specify "should handle Text operations" <|
-            s1.starts_with s2 . to_vector . should_equal [True, False, False, Nothing]
-            s1.starts_with "foo" . to_vector . should_equal [True, False, False, Nothing]
-            s1.starts_with "ba" . to_vector . should_equal [False, True, True, Nothing]
-
-            s1.contains s2 . to_vector . should_equal [True, True, True, Nothing]
-            s1.contains "a" . to_vector . should_equal [True, True, True, Nothing]
-            s1.contains "oo" . to_vector . should_equal [True, False, False, Nothing]
-
-            s1.ends_with s2 . to_vector . should_equal [False, True, False, Nothing]
-            s1.ends_with "ar" . to_vector . should_equal [True, True, False, Nothing]
-            s1.ends_with "a" . to_vector . should_equal [False, False, False, Nothing]
-
     Test.group prefix+"Masking Tables" <|
         Test.specify "should allow to select rows from a table or column based on an expression" <|
             t2 = t1.filter (t1.at "a" == 1)