diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso index b4a24440d7..be15943666 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Data/Numbers.enso @@ -1248,6 +1248,10 @@ type Integer Integer.parse "20220216" parse text:Text (radix=10:Integer) -> Integer ! Number_Parse_Error = integer_parse text radix + ## PRIVATE + fits_in_long self -> Boolean = + self >= Long.MIN_VALUE && self <= Long.MAX_VALUE + ## A syntax error when parsing a double. @Builtin_Type type Number_Parse_Error diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso index 2153927c32..f2fff0007f 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Column.enso @@ -578,8 +578,9 @@ type DB_Column _ : DB_Column -> other.value_type.is_decimal _ -> False either_is_decimal = self.value_type.is_decimal || other_is_decimal - if either_is_decimal then self.make_binary_op "DECIMAL_DIV" other else - self.make_binary_op "/" other + new_name = self.naming_helper.binary_operation_name "/" self other + if either_is_decimal then self.make_binary_op "DECIMAL_DIV" other new_name else + self.make_binary_op "/" other new_name ## ALIAS modulo, modulus GROUP Standard.Base.Operators diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso index 127889dfc3..61896c6f23 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/DB_Table.enso @@ -1323,9 +1323,10 @@ type DB_Table distinct self columns=self.column_names case_sensitivity:Case_Sensitivity=..Default on_problems:Problem_Behavior=..Report_Warning = key_columns = self.columns_helper.select_columns columns Case_Sensitivity.Default reorder=True error_on_missing_columns=True on_problems=on_problems . catch No_Output_Columns _-> Error.throw No_Input_Columns_Selected - problem_builder = Problem_Builder.new - new_table = self.connection.dialect.prepare_distinct self key_columns case_sensitivity problem_builder - problem_builder.attach_problems_before on_problems new_table + key_columns.if_not_error <| + problem_builder = Problem_Builder.new + new_table = self.connection.dialect.prepare_distinct self key_columns case_sensitivity problem_builder + problem_builder.attach_problems_before on_problems new_table ## GROUP Standard.Base.Selections ICON preparation @@ -2612,16 +2613,8 @@ type DB_Table actual_types = materialized_table.columns.map .value_type expected_types.zip actual_types expected_type-> actual_type-> if expected_type == actual_type then Nothing else - expected_type_kind = Meta.meta expected_type . constructor - actual_type_kind = Meta.meta actual_type . constructor - ## We ignore simple approximations that our in-memory backend does - things like adding default - timezone (because we do not have Date_Time without timezone in-memory), - or changing Float32 to Float64 are silently ignored. - However, bigger changes, like a Binary type column getting coerced to Mixed - _will_ still be reported. - if expected_type_kind == actual_type_kind then Nothing else - # If the reverse was an implicit conversion, undoing it also should not yield warnings: - if self.connection.dialect.get_type_mapping.is_implicit_conversion actual_type expected_type then Nothing else - warnings_builder.append (Inexact_Type_Coercion.Warning expected_type actual_type) + if self.connection.dialect.get_type_mapping.should_warn_on_materialize expected_type actual_type then + warnings_builder.append (Inexact_Type_Coercion.Warning expected_type actual_type) result = max_rows.attach_warning materialized_table Problem_Behavior.Report_Warning.attach_problems_before warnings_builder.to_vector result diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso index 427b22ce86..f6c68b96cd 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Base_Generator.enso @@ -420,25 +420,46 @@ generate_order dialect order_descriptor = ## PRIVATE - Generates SQL code corresponding to a SELECT statement. Arguments: - dialect: The SQL dialect for which the code is being generated. - ctx: A description of the SELECT clause. -generate_select_context : Dialect -> Context -> SQL_Builder -generate_select_context dialect ctx = +generate_select : Dialect -> Vector | Nothing -> Context -> SQL_Builder +generate_select dialect columns ctx = gen_exprs exprs = exprs.map (generate_expression dialect) + gen_column pair = (generate_expression dialect pair.second) ++ alias dialect pair.first + + generated_columns = case columns of + Nothing -> SQL_Builder.code "*" + _ -> SQL_Builder.join ", " (columns.map gen_column) + from_part = generate_from_part dialect ctx.from_spec where_part = (SQL_Builder.join " AND " (gen_exprs ctx.where_filters)) . prefix_if_present " WHERE " group_part = (SQL_Builder.join ", " (gen_exprs ctx.groups)) . prefix_if_present " GROUP BY " + + orders = ctx.orders.map (generate_order dialect) + order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY " + limit_part = case ctx.limit of Nothing -> "" _ : Integer -> " LIMIT " + ctx.limit.to_text - orders = ctx.orders.map (generate_order dialect) - order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY " - (SQL_Builder.code " FROM ") ++ from_part ++ where_part ++ group_part ++ order_part ++ limit_part + extensions = ctx.extensions.map extension-> + part = extension.run_generator (gen_exprs extension.expressions) + [extension.position, part] + + parts = Vector.build builder-> + builder.append [100, SQL_Builder.code "SELECT "] + builder.append [200, generated_columns] + builder.append [300, SQL_Builder.code " FROM " ++ from_part] + builder.append [400, where_part] + builder.append [500, group_part] + builder.append [600, order_part] + builder.append [700, limit_part] + extensions.each builder.append + + SQL_Builder.join "" <| parts.sort on=(.first) . map .second ## PRIVATE @@ -467,18 +488,7 @@ generate_insert_query dialect table_name pairs = generate_query : Dialect -> Query -> SQL_Builder generate_query dialect query = case query of Query.Select columns ctx -> - gen_column pair = (generate_expression dialect pair.second) ++ alias dialect pair.first - cols = case columns of - Nothing -> SQL_Builder.code "*" - _ -> SQL_Builder.join ", " (columns.map gen_column) - prefix = case ctx.distinct_on of - Nothing -> SQL_Builder.code "" - expressions : Vector -> - ## TODO I just realised this does not make sense in other backends than Postgres, - so we should probably fail in such cases; probably rewrite into a generic modifier? or a transform? - generated = SQL_Builder.join ", " (expressions.map (generate_expression dialect)) - SQL_Builder.code "DISTINCT ON (" ++ generated ++ ") " - SQL_Builder.code "SELECT " ++ prefix ++ cols ++ generate_select_context dialect ctx + generate_select dialect columns ctx Query.Insert table_name pairs -> generate_insert_query dialect table_name pairs Query.Create_Table name columns primary_key temporary -> diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Context.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Context.enso index d4a4217fa9..ec2ef44e07 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Context.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/IR/Context.enso @@ -5,6 +5,7 @@ import project.Internal.IR.From_Spec.From_Spec import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.Order_Descriptor.Order_Descriptor import project.Internal.IR.SQL_Expression.SQL_Expression +import project.SQL.SQL_Builder ## PRIVATE @@ -22,7 +23,7 @@ type Context details. for_table : Text -> Text -> Any -> Context for_table table_name alias=table_name internal_temporary_keep_alive_reference=Nothing = - Context.Value (From_Spec.Table table_name alias internal_temporary_keep_alive_reference=internal_temporary_keep_alive_reference) [] [] [] Nothing Nothing + Context.Value (From_Spec.Table table_name alias internal_temporary_keep_alive_reference=internal_temporary_keep_alive_reference) [] [] [] Nothing [] ## PRIVATE @@ -33,7 +34,7 @@ type Context - alias: An alias name to use for table within the query. for_query : Text -> Text -> Context for_query raw_sql alias = - Context.Value (From_Spec.Query raw_sql alias) [] [] [] Nothing Nothing + Context.Value (From_Spec.Query raw_sql alias) [] [] [] Nothing [] ## PRIVATE @@ -43,7 +44,7 @@ type Context - subquery: The subquery to lift into a context. for_subquery : From_Spec -> Context for_subquery subquery = - Context.Value subquery [] [] [] Nothing Nothing + Context.Value subquery [] [] [] Nothing [] ## PRIVATE @@ -66,7 +67,7 @@ type Context grouped-by columns or aggregate expressions. - limit: an optional maximum number of elements that the query should return. - Value (from_spec : From_Spec) (where_filters : Vector SQL_Expression) (orders : Vector Order_Descriptor) (groups : Vector SQL_Expression) (limit : Nothing | Integer) (distinct_on : Nothing | Vector SQL_Expression) + Value (from_spec : From_Spec) (where_filters : Vector SQL_Expression) (orders : Vector Order_Descriptor) (groups : Vector SQL_Expression) (limit : Nothing | Integer) (extensions : Vector Context_Extension) ## PRIVATE @@ -76,7 +77,7 @@ type Context - new_filters: The new filters to set in the query. set_where_filters : Vector SQL_Expression -> Context set_where_filters self new_filters = - Context.Value self.from_spec new_filters self.orders self.groups self.limit self.distinct_on + Context.Value self.from_spec new_filters self.orders self.groups self.limit self.extensions ## PRIVATE @@ -87,7 +88,7 @@ type Context query. add_where_filters : Vector SQL_Expression -> Context add_where_filters self new_filters = - Context.Value self.from_spec (self.where_filters+new_filters) self.orders self.groups self.limit self.distinct_on + Context.Value self.from_spec (self.where_filters+new_filters) self.orders self.groups self.limit self.extensions ## PRIVATE @@ -97,7 +98,7 @@ type Context - new_orders: The new ordering clauses to set in the query. set_orders : Vector Order_Descriptor -> Context set_orders self new_orders = - Context.Value self.from_spec self.where_filters new_orders self.groups self.limit self.distinct_on + Context.Value self.from_spec self.where_filters new_orders self.groups self.limit self.extensions ## PRIVATE @@ -114,7 +115,7 @@ type Context - new_orders: The new ordering clauses to add to the query. add_orders : Vector Order_Descriptor -> Context add_orders self new_orders = - Context.Value self.from_spec self.where_filters new_orders+self.orders self.groups self.limit self.distinct_on + Context.Value self.from_spec self.where_filters new_orders+self.orders self.groups self.limit self.extensions ## PRIVATE @@ -124,7 +125,7 @@ type Context - new_groups: The new grouping clauses to set in the query. set_groups : Vector SQL_Expression -> Context set_groups self new_groups = - Context.Value self.from_spec self.where_filters self.orders new_groups self.limit self.distinct_on + Context.Value self.from_spec self.where_filters self.orders new_groups self.limit self.extensions ## PRIVATE @@ -134,14 +135,13 @@ type Context - new_limit: The new limit clauses to set in the query. set_limit : (Nothing | Integer) -> Context set_limit self new_limit = - Context.Value self.from_spec self.where_filters self.orders self.groups new_limit self.distinct_on + Context.Value self.from_spec self.where_filters self.orders self.groups new_limit self.extensions ## PRIVATE - - Returns a copy of the context with changed `distinct_on` expressions. - set_distinct_on : (Nothing | Vector SQL_Expression) -> Context - set_distinct_on self new_distinct_on = - Context.Value self.from_spec self.where_filters self.orders self.groups self.limit new_distinct_on + Returns a copy of the context with an added extension. + add_extension : Context_Extension -> Context + add_extension self extension = + Context.Value self.from_spec self.where_filters self.orders self.groups self.limit (self.extensions + [extension]) ## PRIVATE @@ -176,3 +176,27 @@ type Context type Subquery_Setup ## PRIVATE Value (subquery : From_Spec) (new_columns : Vector (Vector Internal_Column)) + +## PRIVATE + Describes an extension to a Context that can be used to add additional custom + SQL code as part of the query. +type Context_Extension + ## A recipe for building the extension. + + Arguments: + - position: Determines where the extension code should be inserted. + The positions of common query parts are following: + - 100 - the SELECT keyword + - 200 - the column descriptions + - 300 - the FROM part + - 400 - the WHERE part + - 500 - the GROUP BY part + - 600 - the ORDER BY part + - 700 - the LIMIT part + Setting the position to a value between these will result in it being + sorted in the desired place. + - expressions: Sub-expressions needed for the part. They will be + generated and the results of that will be passed to `run_generator`. + - run_generator: A function that takes the generated expressions and + returns the SQL code that will be inserted at the desired position. + Value (position : Integer) (expressions : Vector SQL_Expression) (run_generator : Vector SQL_Builder -> SQL_Builder) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso index dbae578d35..144d7623b4 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Dialect.enso @@ -19,6 +19,7 @@ import project.Internal.Common.Database_Distinct_Helper import project.Internal.Common.Database_Join_Helper import project.Internal.Error_Mapper.Error_Mapper import project.Internal.IR.Context.Context +import project.Internal.IR.Context.Context_Extension import project.Internal.IR.From_Spec.From_Spec import project.Internal.IR.Internal_Column.Internal_Column import project.Internal.IR.Nulls_Order.Nulls_Order @@ -122,7 +123,7 @@ type Postgres_Dialect distinct_expressions = new_key_columns.map column-> value_type = type_mapping.sql_type_to_value_type column.sql_type_reference.get Database_Distinct_Helper.make_distinct_expression case_sensitivity problem_builder column value_type - new_context = Context.for_subquery setup.subquery . set_distinct_on distinct_expressions + new_context = Context.for_subquery setup.subquery . add_extension (make_distinct_extension distinct_expressions) table.updated_context_and_columns new_context new_columns subquery=True ## PRIVATE @@ -764,6 +765,12 @@ as_int64 expr = as_int32 expr = SQL_Builder.code "(" ++ expr ++ "::int4)" +## PRIVATE +make_distinct_extension expressions = + run_generator sql_expressions = + SQL_Builder.code "DISTINCT ON (" ++ (SQL_Builder.join ", " sql_expressions) ++ ") " + Context_Extension.Value position=120 expressions=expressions run_generator=run_generator + ## PRIVATE The RUNTIME_ERROR operation should allow the query to compile fine and it will not prevent it from running if the branch including this operation is diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso index cbd2da166f..ea43a2049c 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Postgres/Postgres_Type_Mapping.enso @@ -131,6 +131,10 @@ type Postgres_Type_Mapping _ = [source_type, target_type] False + ## PRIVATE + should_warn_on_materialize (db_type : Value_Type) (in_memory_type : Value_Type) -> Boolean = + SQL_Type_Mapping.default_should_warn_on_materialize db_type in_memory_type + ## PRIVATE is_integer_type (value_type : Value_Type) -> Boolean = value_type.is_integer diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso index 9e6ea96d09..1a73e1d231 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQL_Type_Mapping.enso @@ -99,6 +99,13 @@ type SQL_Type_Mapping _ = [source_type, target_type] Unimplemented.throw "This is an interface only." + ## PRIVATE + Specifies if the given type coercion should raise an + `Inexact_Type_Coercion` warning when materializing a table into memory. + should_warn_on_materialize (db_type : Value_Type) (in_memory_type : Value_Type) -> Boolean = + _ = [db_type, in_memory_type] + Unimplemented.throw "This is an interface only." + ## PRIVATE Specifies if this backend recognizes the given type as an integer type. @@ -133,3 +140,11 @@ default_sql_type_to_text sql_type = if sql_type.scale.is_nothing then "(" + sql_type.precision.to_text + ")" else " (" + sql_type.precision.to_text + "," + sql_type.scale.to_text + ")" sql_type.name.trim + suffix + +## PRIVATE +default_should_warn_on_materialize (db_type : Value_Type) (in_memory_type : Value_Type) = + ## We ignore simple approximations that our in-memory backend does - things like adding default + timezone (because we do not have Date_Time without timezone in-memory), + or changing Float32 to Float64 are silently ignored. + However, bigger changes, like a Binary type column getting coerced to Mixed - _will_ still be reported. + (Meta.meta db_type . constructor) != (Meta.meta in_memory_type . constructor) diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso index 5f322c5a39..80489d22fc 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/SQLite/SQLite_Type_Mapping.enso @@ -123,6 +123,10 @@ type SQLite_Type_Mapping _ = [source_type, target_type] False + ## PRIVATE + should_warn_on_materialize (db_type : Value_Type) (in_memory_type : Value_Type) -> Boolean = + SQL_Type_Mapping.default_should_warn_on_materialize db_type in_memory_type + ## PRIVATE is_integer_type (value_type : Value_Type) -> Boolean = value_type.is_integer diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload_Table.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload_Table.enso index 4714305113..067c47208b 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload_Table.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Internal/Upload_Table.enso @@ -4,6 +4,7 @@ import Standard.Base.Errors.Common.Dry_Run_Operation import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.Errors.Illegal_State.Illegal_State import Standard.Base.Runtime.Context +from Standard.Base.Logging import all from Standard.Base.Runtime import assert import Standard.Table.Internal.Problem_Builder.Problem_Builder @@ -517,10 +518,15 @@ dry_run_row_limit = 1000 It is a panic, because it is never expected to happen in user code - if it happens, it is a bug in our code. check_transaction_ddl_support connection = - supports_ddl = connection.jdbc_connection.with_metadata metadata-> - metadata.supportsDataDefinitionAndDataManipulationTransactions - if supports_ddl.not then - Panic.throw (Illegal_State.Error "The connection "+connection.to_text+" does not support transactional DDL statements. Our current implementation of table updates relies on transactional DDL. To support this driver, the logic needs to be amended.") + connection.jdbc_connection.with_metadata metadata-> + supports_ddl = metadata.supportsDataDefinitionAndDataManipulationTransactions && metadata.dataDefinitionIgnoredInTransactions.not + if supports_ddl.not then + Panic.throw (Illegal_State.Error "The connection "+connection.to_text+" does not support transactional DDL statements. Our current implementation of table updates relies on transactional DDL. To support this driver, the logic needs to be amended.") + ddl_causes_commit = metadata.dataDefinitionCausesTransactionCommit + if ddl_causes_commit then + # TODO fix for Snowflake support + #Panic.throw (Illegal_State.Error "The connection "+connection.to_text+" does not fully support DDL statements as part of complex transactions - DDL causes a commit, so we cannot compose it. To support this driver, the logic needs to be amended.") + Nothing ## PRIVATE common_delete_rows target_table key_values_to_delete key_columns allow_duplicate_matches = diff --git a/distribution/lib/Standard/Image/0.0.0-dev/src/Image.enso b/distribution/lib/Standard/Image/0.0.0-dev/src/Image.enso index 565257f947..8c3811884a 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/src/Image.enso +++ b/distribution/lib/Standard/Image/0.0.0-dev/src/Image.enso @@ -107,7 +107,6 @@ type Image _ -> [flags] int_flags = MatOfInt.new (write_flags.flat_map x-> [x.to_integer, x.value]) write_to_local_file file:File = - IO.println "Writing the image to a file: "+file.path Panic.catch JException (Java_Codecs.write file.path self.opencv_mat int_flags) _-> Error.throw (File_Error.IO_Error path.file 'Failed to write to the file') r = if path.is_local then write_to_local_file path.file else diff --git a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso index 1260f1bda9..a1d7439d3f 100644 --- a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso +++ b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Dialect.enso @@ -22,6 +22,7 @@ import Standard.Database.Internal.Common.Database_Join_Helper import Standard.Database.Internal.Error_Mapper.Error_Mapper import Standard.Database.Internal.Internals_Access import Standard.Database.Internal.IR.Context.Context +import Standard.Database.Internal.IR.Context.Context_Extension import Standard.Database.Internal.IR.From_Spec.From_Spec import Standard.Database.Internal.IR.Internal_Column.Internal_Column import Standard.Database.Internal.IR.Nulls_Order.Nulls_Order @@ -125,10 +126,12 @@ type Snowflake_Dialect column_mapping = Dictionary.from_vector <| new_columns.map c-> [c.name, c] new_key_columns = key_columns.map c-> column_mapping.at c.name type_mapping = self.get_type_mapping + # TODO simpler distinct if distinct_expressions are all columns? distinct_expressions = new_key_columns.map column-> value_type = type_mapping.sql_type_to_value_type column.sql_type_reference.get Database_Distinct_Helper.make_distinct_expression case_sensitivity problem_builder column value_type - new_context = Context.for_subquery setup.subquery . set_distinct_on distinct_expressions + new_context = Context.for_subquery setup.subquery + . add_extension (make_distinct_extension distinct_expressions) table.updated_context_and_columns new_context new_columns subquery=True ## PRIVATE @@ -311,7 +314,7 @@ make_dialect_operations = other = [["day_of_year", Base_Generator.make_function "DAYOFYEAR"], ["day_of_week", Base_Generator.make_function "DAYOFWEEKISO"]] operations = [["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]] trivial + fractional + other + operations - other = [["IIF", make_iif], ["RUNTIME_ERROR", make_runtime_error_op]] + other = [["IIF", make_iif], ["RUNTIME_ERROR", make_runtime_error_op], ["ROW_NUMBER_IN_GROUP", make_row_number_in_group]] my_mappings = text + counts + stats + arith_extensions + bool + date_ops + other Base_Generator.base_dialect_operations . extend_with my_mappings @@ -429,6 +432,16 @@ agg_count_distinct_include_null args = replace_null_with_marker expr = SQL_Builder.code "COALESCE(" ++ expr ++ ", {'enso-null-replacement-marker':'"+Random.uuid+"'}::variant)" +## PRIVATE + A helper for `lookup_and_replace`, and perhaps other operation. + It creates an expression that returns a row number within a group. + This is a specialization for Snowflake that adds a dummy ORDER BY clause to satisfy its compiler. +make_row_number_in_group arguments = + if arguments.length == 0 then + Panic.throw <| Illegal_State.Error "The operation ROW_NUMBER_IN_GROUP requires at least one argument." + SQL_Builder.code "ROW_NUMBER() OVER (PARTITION BY " ++ (SQL_Builder.join ", " arguments) ++ " ORDER BY 1)" + + ## PRIVATE starts_with = Base_Generator.lift_binary_sql_function "STARTS_WITH" "STARTSWITH" @@ -675,5 +688,16 @@ make_iif arguments = case arguments.length of _ -> Error.throw <| Illegal_State.Error ("Invalid amount of arguments for operation IIF") +## PRIVATE +make_distinct_extension distinct_expressions = + run_generator sql_expressions = + joined = SQL_Builder.join ", " sql_expressions + ## We could use the ORDER BY here to ensure any previous ordering is preserved by distinct. + But to do so we need to have a robust way of checking such ordering, even if subqueries were taken in the meantime. + This may be related to #10321 + Once fixed, should re-enable the `distinct_returns_first_row_from_group_if_ordered` flag back to `True`. + SQL_Builder.code " QUALIFY ROW_NUMBER() OVER (PARTITION BY " ++ joined ++ " ORDER BY 1) = 1 " + Context_Extension.Value position=550 expressions=distinct_expressions run_generator=run_generator + ## PRIVATE snowflake_dialect_name = "Snowflake" diff --git a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Type_Mapping.enso b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Type_Mapping.enso index b6b4197e45..524624c7a7 100644 --- a/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Type_Mapping.enso +++ b/distribution/lib/Standard/Snowflake/0.0.0-dev/src/Internal/Snowflake_Type_Mapping.enso @@ -137,6 +137,14 @@ type Snowflake_Type_Mapping target_type == integer_value_type _ -> False + ## PRIVATE + should_warn_on_materialize (db_type : Value_Type) (in_memory_type : Value_Type) -> Boolean = + is_decimal_to_integer = db_type.is_decimal && db_type.scale == 0 && in_memory_type.is_integer + # We don't warn about the expected conversion of small integer column to Integer in-memory. + if is_decimal_to_integer then False else + # For other types, we delegate to the default behavior. + SQL_Type_Mapping.default_should_warn_on_materialize db_type in_memory_type + ## PRIVATE is_integer_type (value_type : Value_Type) -> Boolean = case value_type of Value_Type.Integer _ -> True @@ -162,7 +170,9 @@ complex_types_map = Dictionary.from_vector <| effective_size = if sql_type.precision==max_length || (sql_type.precision==9 && sql_type.scale==9) then Nothing else sql_type.precision Value_Type.Char size=effective_size variable_length=True make_char sql_type = - Value_Type.Char size=sql_type.precision variable_length=False + # Even the CHAR type in Snowflake is still variable length - it is just an alias for VARCHAR. + size = sql_type.precision.if_nothing 1 + Value_Type.Char size=size variable_length=True make_binary variable sql_type = Value_Type.Binary size=sql_type.precision variable_length=variable handle_bit sql_type = diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso index b1eb294fc4..8ab438e9fa 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Extensions.enso @@ -486,9 +486,9 @@ Any.should_be_a self typ = import Standard.Examples from Standard.Test import Test - example_should_equal = [1, 2] . should_contain_the_same_elements_as [2, 1] -Any.should_contain_the_same_elements_as : Any -> Integer -> Spec_Result -Any.should_contain_the_same_elements_as self that frames_to_skip=0 = + example_should_equal = [1, 2] . should_equal_ignoring_order [2, 1] +Any.should_equal_ignoring_order : Any -> Integer -> Spec_Result +Any.should_equal_ignoring_order self that frames_to_skip=0 = loc = Meta.get_source_location 1+frames_to_skip that.each element-> if self.contains element . not then @@ -498,6 +498,12 @@ Any.should_contain_the_same_elements_as self that frames_to_skip=0 = if that.contains element . not then msg = "The collection contained an element ("+element.to_text+") which was not expected (at " + loc + ")." Test.fail msg + Test.with_clue "Duplicate elements in either collection should have the same counts - checked by comparing sorted collections: " <| + ## If the vector contains vectors or incomparable values, they may not get sorted correctly. + We normalize by converting to a text representation. + normalize e = + Panic.catch No_Such_Method e.pretty _->e.to_text + (self.sort on=normalize) . should_equal (that.sort on=normalize) frames_to_skip=frames_to_skip+3 Spec_Result.Success ## Asserts that `self` value contains the same elements as `that`. @@ -520,9 +526,9 @@ Any.should_contain_the_same_elements_as self that frames_to_skip=0 = import Standard.Examples from Standard.Test import Test - example_should_equal = [1, 2] . should_contain_the_same_elements_as [2, 1] -Error.should_contain_the_same_elements_as : Any -> Integer -> Spec_Result -Error.should_contain_the_same_elements_as self that frames_to_skip=0 = + example_should_equal = [1, 2] . should_equal_ignoring_order [2, 1] +Error.should_equal_ignoring_order : Any -> Integer -> Spec_Result +Error.should_equal_ignoring_order self that frames_to_skip=0 = _ = [that] Test.fail_match_on_unexpected_error self 1+frames_to_skip diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Main.enso index 7cfd7fe93b..90a7cf39c2 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Main.enso @@ -13,7 +13,7 @@ export project.Extensions.should_be_a export project.Extensions.should_be_false export project.Extensions.should_be_true export project.Extensions.should_contain -export project.Extensions.should_contain_the_same_elements_as +export project.Extensions.should_equal_ignoring_order export project.Extensions.should_end_with export project.Extensions.should_equal export project.Extensions.should_equal_type diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso index 83a4ed9535..abe9121787 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Problems.enso @@ -23,8 +23,11 @@ get_attached_warnings v = the standard testing approach, like `x.should_equal y`. - unwrap_errors: If true, remove any wrapping errors from errors and warnings before checking them. -test_problem_handling : (Problem_Behavior -> Any) -> Vector Any -> (Any -> Nothing) -> Boolean -> Nothing -test_problem_handling action expected_problems result_checker unwrap_errors=True = + - ignore_warning_cardinality: If true, the reported warnings may be + duplicated and that will be accepted. If false (default), the warning count + has to match the counts in `expected_problems`. +test_problem_handling : (Problem_Behavior -> Any) -> Vector Any -> (Any -> Nothing) -> Boolean -> Boolean -> Nothing +test_problem_handling action expected_problems result_checker (unwrap_errors : Boolean = True) (ignore_warning_cardinality : Boolean = False) = unwrap_maybe error = if unwrap_errors then Error.unwrap error else error error_checker error_result = @@ -33,9 +36,10 @@ test_problem_handling action expected_problems result_checker unwrap_errors=True error_result . should_fail_with first_problem_type unwrap_errors=unwrap_errors frames_to_skip=3 (unwrap_maybe error_result.catch) . should_equal first_problem frames_to_skip=3 warnings_checker warnings = - ## TODO [RW] we are not checking if there are no duplicate warnings, because the warnings are in fact duplicated - we should figure out how to handle that and then possibly modify the test Test.with_clue "The warnings were "+warnings.to_text+'.\n' <| - warnings . map unwrap_maybe . should_contain_the_same_elements_as expected_problems frames_to_skip=5 + (if ignore_warning_cardinality then warnings.distinct else warnings) + . map unwrap_maybe + . should_equal_ignoring_order expected_problems frames_to_skip=5 test_advanced_problem_handling action error_checker warnings_checker result_checker frames_to_skip=1 ## UNSTABLE diff --git a/test/AWS_Tests/src/S3_Spec.enso b/test/AWS_Tests/src/S3_Spec.enso index da1369bac9..97db1e8a5e 100644 --- a/test/AWS_Tests/src/S3_Spec.enso +++ b/test/AWS_Tests/src/S3_Spec.enso @@ -429,7 +429,7 @@ add_specs suite_builder = # No new file was created parent_dir = my_file.parent - parent_dir.list . should_contain_the_same_elements_as [my_file, bak_file] + parent_dir.list . should_equal_ignoring_order [my_file, bak_file] # If the original file is deleted and the backup file remains, the original file should _not_ count as existing (this used to fail). my_file.delete diff --git a/test/Base_Tests/src/Data/Dictionary_Spec.enso b/test/Base_Tests/src/Data/Dictionary_Spec.enso index c012ba8665..a8b28e6544 100644 --- a/test/Base_Tests/src/Data/Dictionary_Spec.enso +++ b/test/Base_Tests/src/Data/Dictionary_Spec.enso @@ -429,8 +429,8 @@ add_specs suite_builder = py_dict = py_dict_from_vec ["A", 1, "B", 2] dict = py_dict.insert "C" 3 py_dict.not_empty . should_be_true - py_dict.to_vector . should_contain_the_same_elements_as [["A", 1], ["B", 2]] - dict.to_vector . should_contain_the_same_elements_as [["A", 1], ["B", 2], ["C", 3]] + py_dict.to_vector . should_equal_ignoring_order [["A", 1], ["B", 2]] + dict.to_vector . should_equal_ignoring_order [["A", 1], ["B", 2], ["C", 3]] py_empty_dict.is_empty.should_be_true py_empty_dict.insert "A" 1 . insert "A" 2 . get "A" . should_equal 2 @@ -450,9 +450,9 @@ add_specs suite_builder = group_builder.specify "should treat Enso Dictionaries as Python dicts when passed to Python" pending=pending_python_missing <| dict1 = Dictionary.singleton "A" 1 . insert "B" 2 - py_vec_from_map dict1 . should_contain_the_same_elements_as [["A", 1], ["B", 2]] + py_vec_from_map dict1 . should_equal_ignoring_order [["A", 1], ["B", 2]] dict2 = Dictionary.singleton "A" 1 . insert Nothing 2 - py_vec_from_map dict2 . should_contain_the_same_elements_as [["A", 1], [Nothing, 2]] + py_vec_from_map dict2 . should_equal_ignoring_order [["A", 1], [Nothing, 2]] add_common_specs suite_builder prefix:Text (pending : (Text | Nothing)) (empty_dict_fn : (Nothing -> Dictionary)) = diff --git a/test/Base_Tests/src/Data/Text/Encoding_Spec.enso b/test/Base_Tests/src/Data/Text/Encoding_Spec.enso index 16e14c904d..25d6f174b7 100644 --- a/test/Base_Tests/src/Data/Text/Encoding_Spec.enso +++ b/test/Base_Tests/src/Data/Text/Encoding_Spec.enso @@ -66,7 +66,7 @@ add_specs suite_builder = default_warning = invalid.bytes Encoding.ascii default_warning.should_equal invalid_ascii_out - Problems.get_attached_warnings default_warning . should_contain_the_same_elements_as problems + Problems.get_attached_warnings default_warning . should_equal_ignoring_order problems suite_builder.group "Default Encoding" group_builder-> group_builder.specify "should try reading as UTF-8 by default" <| @@ -199,7 +199,7 @@ add_specs suite_builder = Problems.test_problem_handling action problems tester default_warning = text.utf_8 - Problems.get_attached_warnings default_warning . should_contain_the_same_elements_as problems + Problems.get_attached_warnings default_warning . should_equal_ignoring_order problems group_builder.specify "should convert an array of bytes to text via encoding" <| result = Text.from_bytes kshi_utf_8 Encoding.utf_8 diff --git a/test/Base_Tests/src/Data/Text_Spec.enso b/test/Base_Tests/src/Data/Text_Spec.enso index a2aa195f65..ed0e761767 100644 --- a/test/Base_Tests/src/Data/Text_Spec.enso +++ b/test/Base_Tests/src/Data/Text_Spec.enso @@ -407,7 +407,7 @@ add_specs suite_builder = samples_1 = 0.up_to 10000 . map seed-> "ABCD".take (..Sample 2 seed) - samples_1.should_contain_the_same_elements_as ["AB", "BA", "AC", "CA", "AD", "DA", "BC", "CB", "BD", "DB", "CD", "DC"] + samples_1.distinct.should_equal_ignoring_order ["AB", "BA", "AC", "CA", "AD", "DA", "BC", "CB", "BD", "DB", "CD", "DC"] "ABCDEFGH".drop (..Sample 0) . should_equal "ABCDEFGH" "ABCDEFGH".drop (..Sample 1 seed=42) . should_equal "ABCDEGH" @@ -418,7 +418,7 @@ add_specs suite_builder = samples_2 = 0.up_to 10000 . map seed-> "ABCD".drop (..Sample 2 seed) - samples_2.should_contain_the_same_elements_as ["AB", "AC", "AD", "BC", "CD", "BD"] + samples_2.distinct.should_equal_ignoring_order ["AB", "AC", "AD", "BC", "CD", "BD"] group_builder.specify "should allow taking or dropping many indices or subranges (possibly overlapping)" <| "123"*1000 . take (..By_Index (Vector.new 3000 ix-> 2999-ix)) . should_equal "321"*1000 diff --git a/test/Base_Tests/src/Data/Vector_Spec.enso b/test/Base_Tests/src/Data/Vector_Spec.enso index dec2244662..88169eea93 100644 --- a/test/Base_Tests/src/Data/Vector_Spec.enso +++ b/test/Base_Tests/src/Data/Vector_Spec.enso @@ -912,7 +912,9 @@ type_spec suite_builder name alter = suite_builder.group name group_builder-> _ -> x input = 0.up_to 500 . map gen sorted = input.sort on_problems=..Report_Warning - Warning.get_all sorted . length . should_equal 10 + warnings = Warning.get_all sorted + Test.with_clue "(sorted - warnings = "+warnings.to_display_text+") " <| + warnings.length . should_equal 10 Warning.limit_reached sorted . should_equal True group_builder.specify "an error thrown inside map should be caught as a Map_Error" <| diff --git a/test/Base_Tests/src/Network/Http_Spec.enso b/test/Base_Tests/src/Network/Http_Spec.enso index b4db06920c..b005e8dba6 100644 --- a/test/Base_Tests/src/Network/Http_Spec.enso +++ b/test/Base_Tests/src/Network/Http_Spec.enso @@ -559,30 +559,30 @@ add_specs suite_builder = suite_builder.group "Header resolution" group_builder-> group_builder.specify "Default content type and encoding" <| expected = [Header.content_type "text/plain; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "")) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "")) . should_equal_ignoring_order expected group_builder.specify "Content type specified in body" <| expected = [Header.content_type "application/json; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json")) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected group_builder.specify "Content type specified in header list" <| expected = [Header.content_type "application/json"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "")) . should_equal_ignoring_order expected group_builder.specify "Text encoding specified in body" <| expected = [Header.content_type "text/plain; charset=UTF-16LE"] - resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" encoding=Encoding.utf_16_le)) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [] (Request_Body.Text "" encoding=Encoding.utf_16_le)) . should_equal_ignoring_order expected group_builder.specify "Can't specify content type in both places" <| resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json"] (Request_Body.Text "" content_type="text/plain")) . should_fail_with Illegal_Argument group_builder.specify "Custom header" <| expected = [Header.new "some" "header", Header.content_type "application/json; charset=UTF-8"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json")) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [Header.new "some" "header"] (Request_Body.Text "" content_type="application/json")) . should_equal_ignoring_order expected group_builder.specify "Multiple content types in header list are ok" <| expected = [Header.content_type "application/json", Header.content_type "text/plain"] - resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_contain_the_same_elements_as expected + resolve_headers (Request.new HTTP_Method.Get "" [Header.content_type "application/json", Header.content_type "text/plain"] (Request_Body.Text "")) . should_equal_ignoring_order expected suite_builder.group "Http Error handling" group_builder-> group_builder.specify "should be able to handle request errors" <| diff --git a/test/Base_Tests/src/Random_Spec.enso b/test/Base_Tests/src/Random_Spec.enso index 023dd6180d..ecb2b6ac07 100644 --- a/test/Base_Tests/src/Random_Spec.enso +++ b/test/Base_Tests/src/Random_Spec.enso @@ -49,8 +49,8 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> Random.set_seed seed Random.indices 4 2 - two_out_of_three . should_contain_the_same_elements_as [[0, 1], [0, 2], [1, 2], [1, 0], [2, 0], [2, 1]] - two_out_of_four . should_contain_the_same_elements_as <| + two_out_of_three.distinct.should_equal_ignoring_order [[0, 1], [0, 2], [1, 2], [1, 0], [2, 0], [2, 1]] + two_out_of_four.distinct.should_equal_ignoring_order <| [[0, 1], [0, 2], [1, 2], [1, 0], [2, 0], [2, 1], [0, 3], [1, 3], [2, 3], [3, 0], [3, 1], [3, 2]] permutations = 0.up_to 100 . map _-> @@ -59,8 +59,8 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> Random.indices 3 100 all_permutations = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] - permutations . should_contain_the_same_elements_as all_permutations - permutations_2 . should_contain_the_same_elements_as all_permutations + permutations.distinct.should_equal_ignoring_order all_permutations + permutations_2.distinct.should_equal_ignoring_order all_permutations Random.indices 0 0 . should_equal [] Random.indices 0 100 . should_equal [] @@ -77,7 +77,7 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> all_from_small_range = [Date.new 2023 03 01, Date.new 2023 03 02, Date.new 2023 03 03] dates = 0.up_to 100 . map (_-> Random.date (Date.new 2023 03 01) (Date.new 2023 03 03)) - dates.should_contain_the_same_elements_as all_from_small_range + dates.distinct.should_equal_ignoring_order all_from_small_range group_builder.specify "should allow generating random times" <| Random.set_seed 12345 @@ -94,7 +94,7 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> small_range_end = Time_Of_Day.new 8 12 18 all_from_small_range = [Time_Of_Day.new 8 12 15, Time_Of_Day.new 8 12 16, Time_Of_Day.new 8 12 17, Time_Of_Day.new 8 12 18] times = 0.up_to 100 . map (_-> Random.time small_range_start small_range_end) - times.should_contain_the_same_elements_as all_from_small_range + times.distinct.should_equal_ignoring_order all_from_small_range group_builder.specify "should allow generating random UUIDs" <| Random.uuid . should_only_contain_elements_in "0123456789abcdef-" @@ -104,7 +104,7 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> vector = ["A", "B", "C"] shuffles = 0.up_to 100 . map _-> Random.items vector 2 - shuffles . should_contain_the_same_elements_as [["A", "B"], ["A", "C"], ["B", "A"], ["B", "C"], ["C", "A"], ["C", "B"]] + shuffles.distinct.should_equal_ignoring_order [["A", "B"], ["A", "C"], ["B", "A"], ["B", "C"], ["C", "A"], ["C", "B"]] Random.items ["A", "A", "A"] 2 . should_equal ["A", "A"] Random.items ["A", "A", "A"] 0 . should_equal [] @@ -139,8 +139,8 @@ add_specs suite_builder = suite_builder.group "Random" group_builder-> Random.permute list all_permutations = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] - permutations . should_contain_the_same_elements_as all_permutations - permutations_2 . should_contain_the_same_elements_as all_permutations + permutations.distinct.should_equal_ignoring_order all_permutations + permutations_2.distinct.should_equal_ignoring_order all_permutations group_builder.specify "should not allow using a too-large integer range" <| high = 9223372036854775806999 diff --git a/test/Base_Tests/src/Semantic/Meta_Spec.enso b/test/Base_Tests/src/Semantic/Meta_Spec.enso index e3729bc829..9714a3882d 100644 --- a/test/Base_Tests/src/Semantic/Meta_Spec.enso +++ b/test/Base_Tests/src/Semantic/Meta_Spec.enso @@ -259,10 +259,10 @@ add_specs suite_builder = methods.sort . should_equal ['Value', 'create', 'factory', 'first_method', 'my_method', 'other_method', 'second_method'] group_builder.specify "methods of Integer" <| - Meta.meta Integer . methods . sort . should_equal ['%', '*', '+', '-', '/', '<', '<=', '>', '>=', '^', 'abs', 'bit_and', 'bit_not', 'bit_or', 'bit_shift', 'bit_shift_l', 'bit_shift_r', 'bit_xor', 'ceil', 'div', 'floor', 'negate', 'round', 'to_decimal', 'to_float', 'truncate'] + Meta.meta Integer . methods . sort . should_equal ['%', '*', '+', '-', '/', '<', '<=', '>', '>=', '^', 'abs', 'bit_and', 'bit_not', 'bit_or', 'bit_shift', 'bit_shift_l', 'bit_shift_r', 'bit_xor', 'ceil', 'div', 'fits_in_long', 'floor', 'negate', 'round', 'to_decimal', 'to_float', 'truncate'] group_builder.specify "static methods of Integer" <| - Meta.meta (Meta.type_of Integer) . methods . sort . should_equal ['%', '*', '+', '-', '/', '<', '<=', '>', '>=', '^', 'abs', 'bit_and', 'bit_not', 'bit_or', 'bit_shift', 'bit_shift_l', 'bit_shift_r', 'bit_xor', 'ceil', 'div', 'floor', 'negate', 'parse', 'round', 'to_decimal', 'to_float', 'truncate'] + Meta.meta (Meta.type_of Integer) . methods . sort . should_equal ['%', '*', '+', '-', '/', '<', '<=', '>', '>=', '^', 'abs', 'bit_and', 'bit_not', 'bit_or', 'bit_shift', 'bit_shift_l', 'bit_shift_r', 'bit_xor', 'ceil', 'div', 'fits_in_long', 'floor', 'negate', 'parse', 'round', 'to_decimal', 'to_float', 'truncate'] group_builder.specify "methods of Any" <| Meta.meta Any . methods . should_contain "to_text" diff --git a/test/Base_Tests/src/Semantic/Warnings_Spec.enso b/test/Base_Tests/src/Semantic/Warnings_Spec.enso index 4ca0b0daec..3aeb6c53e9 100644 --- a/test/Base_Tests/src/Semantic/Warnings_Spec.enso +++ b/test/Base_Tests/src/Semantic/Warnings_Spec.enso @@ -186,7 +186,7 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- filtered = warnings.filter x-> x.value % 2 == 0 rewarned = Warning.set warned filtered rewarned.should_equal 'foo' - Warning.get_all rewarned . map .value . should_contain_the_same_elements_as [2,4] + Warning.get_all rewarned . map .value . should_equal_ignoring_order [2,4] group_builder.specify "should allow checking for any warnings" <| Warning.has_warnings "foo" . should_be_false @@ -212,20 +212,23 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- count = Warning.get_all x . length Warning.attach "BAZ" count y . should_equal 0 - Warning.get_all y . map .value . should_contain_the_same_elements_as [4, 3, 2, 1, "BAZ"] + Warning.get_all y . map .value . should_equal_ignoring_order [4, 3, 2, 1, "BAZ"] group_builder.specify "should allow to map the warnings, selectively" <| warned = attach_four_warnings "foo" mapped = map_odd_warnings warned mapped . should_equal 'foo' - Warning.get_all mapped . map .value . should_contain_the_same_elements_as [11, 2, 13, 4] + Warning.get_all mapped . map .value . should_equal_ignoring_order [11, 2, 13, 4] group_builder.specify "should allow to map warnings and errors, selectively" <| warned = attach_four_warnings "foo" mapped = map_odd_warnings_and_errors warned mapped . should_equal 'foo' - Warning.get_all mapped . map .value . should_contain_the_same_elements_as [11, 2, 13, 4] - Warning.get_all mapped . map (w-> w.origin.first.name) . should_contain_the_same_elements_as ["Warnings_Spec.map_odd_warnings_and_errors", "Warnings_Spec.attach_four_warnings"] + Warning.get_all mapped . map .value + . should_equal_ignoring_order [11, 2, 13, 4] + Warning.get_all mapped . map (w-> w.origin.first.name) + . distinct + . should_equal_ignoring_order ["Warnings_Spec.map_odd_warnings_and_errors", "Warnings_Spec.attach_four_warnings"] errored_2 = Error.throw 7 mapped_2 = map_odd_warnings_and_errors errored_2 @@ -311,7 +314,7 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- v = 0.up_to 10 . to_vector r = v.map x->(if x % 2 == 0 then 10*x else Warning.attach 100*x x) r.should_equal [0, 1, 20, 3, 40, 5, 60, 7, 80, 9] - Warning.get_all r wrap_errors=True . map .value . should_contain_the_same_elements_as [Map_Error.Error 1 100, Map_Error.Error 3 300, Map_Error.Error 5 500, Map_Error.Error 7 700, Map_Error.Error 9 900] + Warning.get_all r wrap_errors=True . map .value . should_equal_ignoring_order [Map_Error.Error 1 100, Map_Error.Error 3 300, Map_Error.Error 5 500, Map_Error.Error 7 700, Map_Error.Error 9 900] group_builder.specify "should be preserved after operations on multi-dimensional Vector" <| even x = (Warning.attach x x) % 2 @@ -335,9 +338,9 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- r = a.if_not_error b r.should_equal 2 - Warning.get_all r . map .value . should_contain_the_same_elements_as ["b", "a"] - Warning.get_all a . map .value . should_contain_the_same_elements_as ["a"] - Warning.get_all b . map .value . should_contain_the_same_elements_as ["b"] + Warning.get_all r . map .value . should_equal_ignoring_order ["b", "a"] + Warning.get_all a . map .value . should_equal_ignoring_order ["a"] + Warning.get_all b . map .value . should_equal_ignoring_order ["b"] group_builder.specify "should be preserved around polyglot calls" <| x = Warning.attach "x" 1 @@ -351,11 +354,11 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- r1 = java_id.apply x r1.should_equal 1 - Warning.get_all r1 . map .value . should_contain_the_same_elements_as ["x"] + Warning.get_all r1 . map .value . should_equal_ignoring_order ["x"] r2 = javaized_f.apply x r2.should_equal (Pair.new "A" 11) - Warning.get_all r2 . map .value . should_contain_the_same_elements_as ["f(1)", "x"] + Warning.get_all r2 . map .value . should_equal_ignoring_order ["f(1)", "x"] ## The following will not work, as if the polyglot method expects an `Object` it will get converted to a Java 'primitive' losing the @@ -375,19 +378,19 @@ add_specs suite_builder = suite_builder.group "Dataflow Warnings" group_builder- r1 = CallbackHelper.runCallbackInt f x r1.should_equal (Pair.new "A" 11) - Warning.get_all r1 . map .value . should_contain_the_same_elements_as ["f(1)", "x"] + Warning.get_all r1 . map .value . should_equal_ignoring_order ["f(1)", "x"] r2 = CallbackHelper.runCallbackInt g x r2.should_equal 11 - Warning.get_all r2 . map .value . should_contain_the_same_elements_as ["g(1)", "x"] + Warning.get_all r2 . map .value . should_equal_ignoring_order ["g(1)", "x"] r3 = CallbackHelper.runCallbackInt h x r3.should_equal "{x=1}" - Warning.get_all r3 . map .value . should_contain_the_same_elements_as ["h(1)", "x"] + Warning.get_all r3 . map .value . should_equal_ignoring_order ["h(1)", "x"] r4 = CallbackHelper.runCallbackInt i x r4.should_equal Nothing - Warning.get_all r4 . map .value . should_contain_the_same_elements_as ["i(1)", "x"] + Warning.get_all r4 . map .value . should_equal_ignoring_order ["i(1)", "x"] group_builder.specify "should not affect method dispatch" <| a = My_Fancy_Collection.Value 42 diff --git a/test/Base_Tests/src/System/Reporting_Stream_Encoder_Spec.enso b/test/Base_Tests/src/System/Reporting_Stream_Encoder_Spec.enso index c4c379bf15..0c46dac1e8 100644 --- a/test/Base_Tests/src/System/Reporting_Stream_Encoder_Spec.enso +++ b/test/Base_Tests/src/System/Reporting_Stream_Encoder_Spec.enso @@ -110,7 +110,7 @@ add_specs suite_builder = rse.write "23" result result . should_equal 42 - Problems.get_attached_warnings result . should_contain_the_same_elements_as ["warn:1", "warn:2"] + Problems.get_attached_warnings result . should_equal_ignoring_order ["warn:1", "warn:2"] f.read_text encoding . should_equal "BAZ23" f.delete_if_exists @@ -125,7 +125,7 @@ add_specs suite_builder = rse.write "23" result result . should_equal Nothing - Problems.get_attached_warnings result . should_contain_the_same_elements_as ["warn:1", "warn:2"] + Problems.get_attached_warnings result . should_equal_ignoring_order ["warn:1", "warn:2"] f.read_text encoding . should_equal "BAZ23" f.delete_if_exists diff --git a/test/Geo_Tests/src/Geo_Spec.enso b/test/Geo_Tests/src/Geo_Spec.enso index da24dedf90..2ff417d58c 100644 --- a/test/Geo_Tests/src/Geo_Spec.enso +++ b/test/Geo_Tests/src/Geo_Spec.enso @@ -32,7 +32,7 @@ add_specs suite_builder = group_builder.specify "should allow converting a GeoJSON array of features into a table" <| fields = ['foo', 'bar', 'baz', 'longitude', 'elevation'] t = Geo.geo_json_to_table (geo_json.get "features") fields - t.columns.map .name . should_contain_the_same_elements_as fields + t.columns.map .name . should_equal_ignoring_order fields t.at 'foo' . to_vector . should_equal [1, 2] t.at 'bar' . to_vector . should_equal ['value2', Nothing] t.at 'baz' . to_vector . should_equal [Nothing, 3] @@ -42,7 +42,7 @@ add_specs suite_builder = group_builder.specify "should allow converting a GeoJSON object into a table with provided fields" <| fields = ['foo', 'bar', 'longitude'] t = Geo.geo_json_to_table geo_json fields - t.columns.map .name . should_contain_the_same_elements_as fields + t.columns.map .name . should_equal_ignoring_order fields t.at 'foo' . to_vector . should_equal [1, 2] t.at 'bar' . to_vector . should_equal ['value2', Nothing] t.at 'longitude' . to_vector . should_equal [-118.58, 10.11] @@ -50,7 +50,7 @@ add_specs suite_builder = group_builder.specify "should allow converting a GeoJSON object into a table containing all available fields" <| fields = ['bar', 'baz', 'elevation', 'foo', 'latitude', 'longitude'] t = Geo.geo_json_to_table geo_json - t.columns.map .name . should_contain_the_same_elements_as fields + t.columns.map .name . should_equal_ignoring_order fields t.at 'foo' . to_vector . should_equal [1, 2] t.at 'bar' . to_vector . should_equal ['value2', Nothing] t.at 'baz' . to_vector . should_equal [Nothing, 3] diff --git a/test/Snowflake_Tests/src/Snowflake_Spec.enso b/test/Snowflake_Tests/src/Snowflake_Spec.enso index 5e254d2b68..b4aa1438ea 100644 --- a/test/Snowflake_Tests/src/Snowflake_Spec.enso +++ b/test/Snowflake_Tests/src/Snowflake_Spec.enso @@ -210,7 +210,7 @@ snowflake_specific_spec suite_builder default_connection db_name setup = # Again, when materialized the conversion Decimal->Integer is a feature, so it should not cause warning. Problems.assume_no_problems in_memory1 - in_memory1.at "small_ints" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + in_memory1.at "small_ints" . to_vector . should_equal_ignoring_order [1, 2, 3] t2 = table_builder [["big_ints", [2^100, 2^110, 1]]] t2.at "big_ints" . value_type . should_equal (Value_Type.Decimal 38 0) @@ -226,11 +226,11 @@ snowflake_specific_spec suite_builder default_connection db_name setup = Problems.assume_no_problems in_memory2 # Check correctness of values - in_memory2.at "big_ints" . to_vector . should_contain_the_same_elements_as [2^100, 2^110, 1] + in_memory2.at "big_ints" . to_vector . should_equal_ignoring_order [2^100, 2^110, 1] group_builder.specify "correctly handles Decimal and Float types" <| table_name = Name_Generator.random_name "DecimalFloat" - t1 = default_connection.get.create_table table_name [Column_Description.Value "d1" (Value_Type.Decimal 38 6), Column_Description.Value "d2" (Value_Type.Decimal 10 2), Column_Description.Value "d3" (Value_Type.Decimal 24 -3), Column_Description.Value "f" (Value_Type.Float)] primary_key=[] + t1 = default_connection.get.create_table table_name [Column_Description.Value "d1" (Value_Type.Decimal 38 6), Column_Description.Value "d2" (Value_Type.Decimal 10 2), Column_Description.Value "d3" (Value_Type.Decimal 24 -3), Column_Description.Value "f" (Value_Type.Float)] primary_key=[] temporary=True t1.at "d1" . value_type . should_equal (Value_Type.Decimal 38 6) t1.at "d2" . value_type . should_equal (Value_Type.Decimal 10 2) # Negative scale is not supported so we fallback to defaults: @@ -613,7 +613,7 @@ add_snowflake_specs suite_builder create_connection_fn db_name = Common_Spec.add_specs suite_builder prefix create_connection_fn - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True allows_mixed_type_comparisons=False fixed_length_text_columns=False different_size_integer_types=False removes_trailing_whitespace_casting_from_char_to_varchar=True supports_decimal_type=True supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=False supports_date_time_without_timezone=True supports_nanoseconds_in_time=True is_nan_comparable=True + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True allows_mixed_type_comparisons=False text_length_limited_columns=True fixed_length_text_columns=False different_size_integer_types=False removes_trailing_whitespace_casting_from_char_to_varchar=False supports_decimal_type=True supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=False supports_date_time_without_timezone=True supports_nanoseconds_in_time=True is_nan_comparable=True distinct_returns_first_row_from_group_if_ordered=False aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last=False first_last_row_order=False aggregation_problems=False text_concat=False agg_in_memory_table = ((Project_Description.new enso_dev.Table_Tests).data / "data.csv") . read diff --git a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso index 19063902bd..d551d561cb 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Column_Operations_Spec.enso @@ -18,7 +18,8 @@ from Standard.Test import all import enso_dev.Base_Tests.Data.Round_Spec -from project.Common_Table_Operations.Util import run_default_backend, within_table +import project.Common_Table_Operations.Main.Char_Max_Size_After_Substring_Behavior +from project.Common_Table_Operations.Util import run_default_backend, within_table, is_float_or_decimal main filter=Nothing = run_default_backend add_specs filter @@ -487,16 +488,20 @@ add_specs suite_builder setup = True -> actual.at "col0" . value_type . should_equal (Value_Type.Char size=3 variable_length=True) False -> actual.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) - group_builder.specify "should allow to fill_nothing a fixed width of width 1 correctly expanding ouptut types" <| + group_builder.specify "should allow to fill_nothing a fixed width of width 1 correctly expanding output types" <| t = table_builder [["col0", ["a", Nothing, " "]]] . cast "col0" (Value_Type.Char size=1 variable_length=False) fillBlank = t.fill_nothing ["col0"] "" fillOneSpace = t.fill_nothing ["col0"] " " fillTwoSpaces = t.fill_nothing ["col0"] " " case setup.test_selection.length_restricted_text_columns of - True -> fillBlank.at "col0" . value_type . should_equal (Value_Type.Char size=1 variable_length=True) - fillOneSpace.at "col0" . value_type . should_equal (Value_Type.Char size=1 variable_length=False) + True -> + fillBlank.at "col0" . value_type . should_equal (Value_Type.Char size=1 variable_length=True) + case setup.test_selection.fixed_length_text_columns of + True -> fillOneSpace.at "col0" . value_type . should_equal (Value_Type.Char size=1 variable_length=False) + False -> fillOneSpace.at "col0" . value_type . should_equal (Value_Type.Char size=1 variable_length=True) fillTwoSpaces.at "col0" . value_type . should_equal (Value_Type.Char size=2 variable_length=True) - False -> fillBlank.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) + False -> + fillBlank.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) fillOneSpace.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) fillTwoSpaces.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) @@ -514,8 +519,12 @@ add_specs suite_builder setup = t = table_builder [["col0", [Nothing, "200", Nothing, "400", "500", Nothing]]] . cast "col0" (Value_Type.Char size=3 variable_length=False) actual = t.fill_nothing ["col0"] " " actual.at "col0" . to_vector . should_equal [" ", "200", " ", "400", "500", " "] - case setup.test_selection.fixed_length_text_columns of - True -> actual.at "col0" . value_type . should_equal (Value_Type.Char size=3 variable_length=False) + + case setup.test_selection.length_restricted_text_columns of + True -> + case setup.test_selection.fixed_length_text_columns of + True -> actual.at "col0" . value_type . should_equal (Value_Type.Char size=3 variable_length=False) + False -> actual.at "col0" . value_type . should_equal (Value_Type.Char size=3 variable_length=True) False -> actual.at "col0" . value_type . should_equal (Value_Type.Char variable_length=True) group_builder.specify "should allow to fill_nothing from other columns" <| @@ -788,29 +797,30 @@ add_specs suite_builder setup = setup.expect_integer_type <| (data.x + data.x) (data.x + data.y).value_type . should_be_a (Value_Type.Float ...) setup.expect_integer_type <| (data.x + 2) - (data.x + 1.5).value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal (data.x + 1.5) setup.expect_integer_type <| (data.x - data.x) (data.x - data.y).value_type . should_be_a (Value_Type.Float ...) setup.expect_integer_type <| (data.x - 2) - (data.x - 1.5).value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal (data.x - 1.5) setup.expect_integer_type <| (data.x * data.x) (data.x * data.y).value_type . should_be_a (Value_Type.Float ...) setup.expect_integer_type <| (data.x * 2) - (data.x * 1.5).value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal <| (data.x * 1.5) (data.x ^ data.x).value_type . is_numeric . should_be_true if setup.test_selection.supports_decimal_type then + # TODO in-memory specific tests may be added in In_Memory/Table_Spec as part of #10429 divide_in_memory_pending = if setup.is_database.not then "TODO: https://github.com/enso-org/enso/issues/10429" group_builder.specify "should correctly infer the types (for decimal division)" pending=divide_in_memory_pending <| - (data.x / data.x).value_type . is_floating_point . should_be_true - (data.x / data.y).value_type . is_floating_point . should_be_true - (data.x / data.z).value_type . is_decimal . should_be_true - (data.y / data.z).value_type . is_decimal . should_be_true - (data.x / 2).value_type . is_floating_point . should_be_true - (data.x / 1.5).value_type . is_floating_point . should_be_true + is_float_or_decimal (data.x / data.x) + is_float_or_decimal (data.x / data.y) + is_float_or_decimal (data.x / data.z) + is_float_or_decimal (data.y / data.z) + is_float_or_decimal (data.x / 2) + is_float_or_decimal (data.x / 1.5) group_builder.specify "should check types" <| t = table_builder [["X", [1, 2]], ["Y", ["a", "b"]], ["Z", [True, False]]] @@ -889,11 +899,11 @@ add_specs suite_builder setup = r2 = (t.at "A") / (t.at "B") r2 . to_vector . should_equal r - r2.value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal r2 r3 = (t.at "A") / 2 r3 . to_vector . should_equal [0.5, 2.5, 5.0, 50.0] - r3.value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal r3 a2 = [1.2, 5, 10.2, 100] b2 = [1.2, 2, 2, 5] @@ -906,11 +916,11 @@ add_specs suite_builder setup = r5 = (t2.at "A") / (t2.at "B") r5 . to_vector . should_equal r4 - r5.value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal r5 r6 = (t2.at "A") / 2 r6 . to_vector . should_equal [0.6, 2.5, 5.1, 50.0] - r6.value_type . should_be_a (Value_Type.Float ...) + is_float_or_decimal r6 db_pending = if setup.is_database then "Arithmetic error handling is currently not implemented for the Database backend." group_builder.specify "should allow division by 0 and report warnings" pending=db_pending <| @@ -991,9 +1001,6 @@ add_specs suite_builder setup = group_builder.specify "should allow round on a "+type.to_text+" column (to >0 decimal places)" <| table = table_builder [["x", [0.51, 0.59, 3.51, 3.59, -0.51, -0.59, -3.51, -3.59]]] result = table.at "x" . cast type . round 1 - # TODO why it's becoming an Int? - IO.println (result.value_type) - IO.println (result.cast Value_Type.Char . to_vector) result.to_vector.should_equal [0.5, 0.6, 3.5, 3.6, -0.5, -0.6, -3.5, -3.6] result.name . should_equal "round([x])" @@ -1152,12 +1159,15 @@ add_specs suite_builder setup = resRight.name . should_equal "text_right([strings], 1)" resLeft . to_vector . should_equal ["a", "f", "", Nothing, "c", "I"] resRight . to_vector . should_equal ["a", "r", "", Nothing, "é", "."] - case setup.is_database of - False -> resLeft . value_type . should_equal (Value_Type.Char size=286 variable_length=True) - True -> resLeft . value_type . should_equal (Value_Type.Char variable_length=True) - case setup.is_database of - False -> resRight . value_type . should_equal (Value_Type.Char size=286 variable_length=True) - True -> resRight . value_type . should_equal (Value_Type.Char variable_length=True) + + expected_new_size = case setup.test_selection.char_max_size_after_substring of + Char_Max_Size_After_Substring_Behavior.Reset -> Nothing + Char_Max_Size_After_Substring_Behavior.Kept -> 286 + Char_Max_Size_After_Substring_Behavior.Reduced -> 1 + + Test.with_clue "(Expected behaviour for substring size = "+setup.test_selection.char_max_size_after_substring.to_text+".) " <| + resLeft.value_type . should_equal (Value_Type.Char size=expected_new_size variable_length=True) + resRight.value_type . should_equal (Value_Type.Char size=expected_new_size variable_length=True) group_builder.specify "should handle operation text_left and text_right of grapheme and non-grapheme" <| with_mixed_columns_if_supported [["strings", ["a", "foobar", "", Nothing, "👩‍🔬👩‍🔬V👩‍🔬👩‍🔬", "café", "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of light, it was the season of darkness, it was the spring of hope, it was the winter of despair."]]] t-> @@ -1260,8 +1270,8 @@ add_specs suite_builder setup = s1.is_empty . value_type . should_equal Value_Type.Boolean s1.is_blank . value_type . should_equal Value_Type.Boolean - s1.fill_empty "<>" . value_type . is_text . should_be_true - s1.fill_empty s2 . value_type . is_text . should_be_true + s1.fill_empty "<>" . value_type . should_be_a (Value_Type.Char ...) + s1.fill_empty s2 . value_type . should_be_a (Value_Type.Char ...) group_builder.specify "should support text concatenation with the + operator" <| with_mixed_columns_if_supported [["s1", ["foobar", "bar", "baz", "BAB", Nothing]], ["s2", ["foo", "ar", "a", "b", Nothing]]] t3-> @@ -1269,60 +1279,58 @@ add_specs suite_builder setup = s2 = t3.at "s2" c1 = s1 + s2 c1.to_vector . should_equal ["foobarfoo", "barar", "baza", "BABb", Nothing] - c1.value_type.is_text . should_be_true + c1.value_type.should_be_a (Value_Type.Char ...) c2 = s1 + "_SUF" c2.to_vector . should_equal ["foobar_SUF", "bar_SUF", "baz_SUF", "BAB_SUF", Nothing] - c2.value_type.is_text . should_be_true + c2.value_type.should_be_a (Value_Type.Char ...) c3 = s1 + Nothing c3.to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing] - c3.value_type.is_text . should_be_true + c3.value_type.should_be_a (Value_Type.Char ...) suite_builder.group prefix+"(Column_Operations_Spec) Min/Max Operations" group_builder-> data = Min_Max_Data.setup setup.table_builder group_builder.specify "should allow one or more args and return the correct type" <| c1 = data.a.min 2 c1.to_vector . should_equal [1, 2, 2] - c1.value_type.is_integer . should_be_true + setup.expect_integer_type c1 c2 = data.a.max 2 c2.to_vector . should_equal [2, 2, 3] - c2.value_type.is_integer . should_be_true + setup.expect_integer_type c2 c3 = data.a.min [2.5, 2] c3.to_vector . should_equal [1, 2, 2] - Test.with_clue "c3.value_type="+c3.value_type.to_display_text <| - c3.value_type.is_floating_point . should_be_true + c3.value_type.should_be_a (Value_Type.Float ...) c4 = data.a.max [2.5, 2] c4.to_vector . should_equal [2.5, 2.5, 3] - c4.value_type.is_floating_point . should_be_true + c4.value_type.should_be_a (Value_Type.Float ...) c5 = data.a.min data.b c5.to_vector . should_equal [1, 2, 3] - Test.with_clue "c5.value_type="+c5.value_type.to_display_text+": " <| - c5.value_type.is_floating_point . should_be_true + c5.value_type.should_be_a (Value_Type.Float ...) c6 = data.a.max data.b c6.to_vector . should_equal [4.5, 5.5, 6.5] - c6.value_type.is_floating_point . should_be_true + c6.value_type.should_be_a (Value_Type.Float ...) c7 = data.a.min [data.a, data.b, 1] c7.to_vector . should_equal [1, 1, 1] - c7.value_type.is_floating_point . should_be_true + c7.value_type.should_be_a (Value_Type.Float ...) c8 = data.a.max [data.a, data.b, 1] c8.to_vector . should_equal [4.5, 5.5, 6.5] - c8.value_type.is_floating_point . should_be_true + c8.value_type.should_be_a (Value_Type.Float ...) c9 = (data.t.at "d").min False c9.to_vector . should_equal [False, False, False] - c9.value_type.is_boolean . should_be_true + c9.value_type.should_equal Value_Type.Boolean c10 = (data.t.at "d").max False c10.to_vector . should_equal [True, False, True] - c10.value_type.is_boolean . should_be_true + c10.value_type.should_equal Value_Type.Boolean group_builder.specify "should check types" <| [(.min), (.max)].each op-> @@ -1687,12 +1695,19 @@ add_specs suite_builder setup = group_builder.specify "Should create a column of the correct type on a table with no rows" <| t = table_builder [["x", ["1", "2", "3"]]] empty = t.take 0 - [[False, .is_boolean], [42, .is_integer], ["42", .is_text], ["foo", .is_text], [1.1, .is_floating_point]].map pair-> - value = pair.at 0 - pred = pair.at 1 + + check value expected_type = c = empty.make_constant_column value - pred c.value_type . should_be_true - pred ((empty.set c).at c.name . value_type) . should_be_true + c.value_type.should_be_a expected_type + (empty.set c).at c.name . value_type . should_be_a expected_type + + check False Value_Type.Boolean + check "42" (Value_Type.Char ...) + check "foo" (Value_Type.Char ...) + check 1.1 (Value_Type.Float ...) + + c = empty.make_constant_column 42 + setup.expect_integer_type c nulls_db_pending = if setup.is_database then "Empty NULL columns are unsupported in the database backends" group_builder.specify "Should create a column of the correct type on a table with no rows" pending=nulls_db_pending <| diff --git a/test/Table_Tests/src/Common_Table_Operations/Conversion_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Conversion_Spec.enso index 5fbd61e6bd..1b93d4709a 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Conversion_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Conversion_Spec.enso @@ -20,6 +20,9 @@ type My_Type to_text : Text to_text self = "{{{MY Type [x="+self.x.to_text+"] }}}" +type Lazy_Ref + Value ~get + add_specs suite_builder setup = prefix = setup.prefix materialize = setup.materialize @@ -27,8 +30,10 @@ add_specs suite_builder setup = supports_conversion_failure_reporting = setup.is_database.not table_builder = build_sorted_table setup suite_builder.group prefix+"(Conversion_Spec) Table/Column.cast - to text" group_builder-> + table = Lazy_Ref.Value <| + table_builder [["X", [1, 2, 3000]], ["Y", [True, False, True]], ["Z", [1.5, 0.125, -2.5]], ["W", ["a", "DEF", "a slightly longer text"]]] group_builder.specify "should allow to cast columns of various basic types to text" <| - t = table_builder [["X", [1, 2, 3000]], ["Y", [True, False, True]], ["Z", [1.5, 0.125, -2.5]], ["W", ["a", "DEF", "a slightly longer text"]]] + t = table.get t2 = t.cast t.column_names Value_Type.Char t2.at "X" . value_type . is_text . should_be_true t2.at "Y" . value_type . is_text . should_be_true @@ -42,7 +47,7 @@ add_specs suite_builder setup = t2.at "W" . to_vector . should_equal ["a", "DEF", "a slightly longer text"] group_builder.specify "should allow to cast columns selecting columns by type" <| - t = table_builder [["X", [1, 2, 3000]], ["Y", [True, False, True]], ["Z", [1.5, 0.125, -2.5]], ["W", ["a", "DEF", "a slightly longer text"]]] + t = table.get t2 = t.cast [..By_Type ..Integer] Value_Type.Char t2.at "X" . value_type . should_be_a (Value_Type.Char ...) t2.at "Y" . value_type . is_text . should_be_false @@ -79,30 +84,21 @@ add_specs suite_builder setup = c.to_vector . should_equal ["{{{MY Type [x=42] }}}", "{{{MY Type [x=X] }}}"] group_builder.specify "should allow to cast an integer column to a decimal type" <| - t = table_builder [["X", [1, 2, 3]]] + t = table.get c = t.at "X" . cast Value_Type.Decimal c.value_type.is_decimal . should_be_true - c.to_vector . should_equal [1, 2, 3] + c.to_vector . should_equal [1, 2, 3000] if setup.test_selection.fixed_length_text_columns then group_builder.specify "should allow to cast a text column to fixed-length" <| - t = table_builder [["X", ["a", "DEF", "a slightly longer text"]]] - c = t.at "X" . cast (Value_Type.Char size=3 variable_length=False) + t = table.get + c = t.at "W" . cast (Value_Type.Char size=3 variable_length=False) c.value_type . should_equal (Value_Type.Char size=3 variable_length=False) c.to_vector . should_equal ["a ", "DEF", "a s"] # No Conversion_Failure warning here, because we started with text, so it was expected we will trim it if needed. Problems.assume_no_problems c - group_builder.specify "should allow to cast a text column to variable-length with a max size" <| - t = table_builder [["X", ["a", "DEF", "a slightly longer text"]]] - c = t.at "X" . cast (Value_Type.Char size=3 variable_length=True) - c.value_type . should_equal (Value_Type.Char size=3 variable_length=True) - c.to_vector . should_equal ["a", "DEF", "a s"] - - # No Conversion_Failure warning here, because we started with text, so it was expected we will trim it if needed. - Problems.assume_no_problems c - group_builder.specify "should allow casting a non-text column to fixed-length text" <| t = table_builder [["X", [1, 22, 333]]] c = t.at "X" . cast (Value_Type.Char size=3 variable_length=False) @@ -127,11 +123,23 @@ add_specs suite_builder setup = w2.affected_rows_count . should_equal 4 group_builder.specify "should not allow 0-length Char type" <| - c1 = table_builder [["X", ["a", "", "bcd"]]] . at "X" + c1 = table.get . at "W" r1 = c1.cast (Value_Type.Char size=0 variable_length=False) r1.should_fail_with Illegal_Argument r1.catch.to_display_text . should_contain "positive" + if setup.test_selection.text_length_limited_columns then + group_builder.specify "should allow to cast a text column to variable-length with a max size" <| + t = table_builder [["X", ["a", "DEF", "a slightly longer text"]]] + c = t.at "X" . cast (Value_Type.Char size=3 variable_length=True) + c.value_type . should_equal (Value_Type.Char size=3 variable_length=True) + c.to_vector . should_equal ["a", "DEF", "a s"] + + # No Conversion_Failure warning here, because we started with text, so it was expected we will trim it if needed. + Problems.assume_no_problems c + + group_builder.specify "should not allow 0-length Char type" <| + c1 = table.get . at "W" r2 = c1.cast (Value_Type.Char size=0 variable_length=True) r2.should_fail_with Illegal_Argument @@ -500,7 +508,7 @@ add_specs suite_builder setup = c3.to_vector . should_equal ["1", "2.25", "3", "-45.25", "2.0", "X", "2020-01-01", date_time_str, "12:30:00", "{{{MY Type [x=42] }}}", "True", Nothing, "True"] Problems.assume_no_problems c3 - if setup.test_selection.fixed_length_text_columns then + if setup.test_selection.text_length_limited_columns then c3_2 = t2.cast "super-mix" (Value_Type.Char size=2) . at "super-mix" c3_2.value_type . should_equal (Value_Type.Char size=2) c3_2.to_vector . should_equal ["1", "2.", "3", "-4", "2.", "X", "20", "20", "12", "{{", "Tr", Nothing, "Tr"] diff --git a/test/Table_Tests/src/Common_Table_Operations/Distinct_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Distinct_Spec.enso index 3521e6dcb7..46b04f8dac 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Distinct_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Distinct_Spec.enso @@ -59,7 +59,7 @@ add_specs suite_builder setup = [0.3, 0.4, 0.6].contains (cv.at 1) . should_be_true if setup.test_selection.distinct_returns_first_row_from_group_if_ordered then - group_builder.specify "should allow to select distinct rows based on a subset of columns, returning any first from each group if the table was ordered" <| + group_builder.specify "should allow to select distinct rows based on a subset of columns, returning first from each group if the table was ordered" <| a = ["A", ["a", "a", "a", "a", "a", "a"]] b = ["B", [1, 1, 2, 2, 1, 2]] c = ["C", [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]] diff --git a/test/Table_Tests/src/Common_Table_Operations/Integration_Tests.enso b/test/Table_Tests/src/Common_Table_Operations/Integration_Tests.enso index c355c91a74..e90d0e8120 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Integration_Tests.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Integration_Tests.enso @@ -149,7 +149,7 @@ add_specs suite_builder setup = t4 = t.filter "B" (Filter_Condition.Equal 5) t5 = t4.distinct ["A"] on_problems=..Report_Error r5 = t5 |> materialize - r5.at "A" . to_vector . should_contain_the_same_elements_as ["b", "a"] + r5.at "A" . to_vector . should_equal_ignoring_order ["b", "a"] r5.at "B" . to_vector . should_equal [5, 5] ## This mostly checks that various operations handle all kinds of Integer storage implementations diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Cross_Join_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Cross_Join_Spec.enso index 9749cc998e..bf710577e5 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Cross_Join_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Cross_Join_Spec.enso @@ -45,7 +45,7 @@ add_specs suite_builder setup = expected_rows = [r0, r1, r2, r3] case setup.is_database of - True -> r.should_contain_the_same_elements_as expected_rows + True -> r.should_equal_ignoring_order expected_rows False -> r.should_equal expected_rows group_builder.specify "should work correctly with empty tables" <| @@ -97,7 +97,7 @@ add_specs suite_builder setup = r4 = [1, 4, 'b'] expected_rows = [r1, r3, r1, r3, r2, r4, r2, r4, r1, r3, r1, r3, r1, r3, r1, r3] case setup.is_database of - True -> r.should_contain_the_same_elements_as expected_rows + True -> r.should_equal_ignoring_order expected_rows False -> r.should_equal expected_rows group_builder.specify "should allow self-joins" <| @@ -114,7 +114,7 @@ add_specs suite_builder setup = r3 = [2, 5, 2, 5] expected_rows = [r0, r1, r2, r3] case setup.is_database of - True -> r.should_contain_the_same_elements_as expected_rows + True -> r.should_equal_ignoring_order expected_rows False -> r.should_equal expected_rows group_builder.specify "should rename columns of the right table to avoid duplicates" <| @@ -160,7 +160,7 @@ add_specs suite_builder setup = r5 = [100, 4, 'a', 'x'] expected_rows = [r0, r1, r2, r3, r4, r5] case setup.is_database of - True -> r.should_contain_the_same_elements_as expected_rows + True -> r.should_equal_ignoring_order expected_rows False -> r.should_equal expected_rows group_builder.specify "Cross join is not possible via call to .join" <| diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso index 9bbaaa354d..402cd55553 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Join_Spec.enso @@ -478,7 +478,7 @@ add_specs suite_builder setup = r4.at "Z" . to_vector . should_equal [2.0, 2.0, 2.0, 2.0] r4.at "W" . to_vector . should_equal [1, 3, 1, 3] expected_problems = [Floating_Point_Equality.Error "Z", Floating_Point_Equality.Error "X"] - Problems.get_attached_warnings r3 . should_contain_the_same_elements_as expected_problems + Problems.get_attached_warnings r3 . should_equal_ignoring_order expected_problems group_builder.specify "should correctly handle nulls in equality conditions" <| t1 = table_builder [["X", ["A", Nothing, "a", Nothing, "ą", "b"]], ["Y", [0, 1, 2, 3, 4, 5]]] diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Lookup_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Lookup_Spec.enso index 7e9f51c3d7..663f715b86 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Lookup_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Lookup_Spec.enso @@ -14,28 +14,12 @@ import project.Util main filter=Nothing = run_default_backend add_specs filter -type Data - Value ~connection - - setup create_connection_fn = - Data.Value (create_connection_fn Nothing) - - teardown self = self.connection.close - add_specs suite_builder setup = prefix = setup.prefix - create_connection_fn = setup.create_connection_func materialize = setup.materialize + table_builder = setup.table_builder suite_builder.group prefix+"Table.merge" group_builder-> - data = Data.setup create_connection_fn - - group_builder.teardown <| - data.teardown - - table_builder cols = - setup.table_builder cols connection=data.connection - group_builder.specify "should allow to simply update columns based on a lookup table" <| lookup = table_builder [["Y", ["A", "B", "A"]], ["X", [1, 2, 3]]] my_table = table_builder [["X", [1, 2, 3, 2]], ["Y", ["Z", "ZZ", "ZZZ", "ZZZZ"]], ["Z", [10, 20, 30, 40]]] @@ -292,9 +276,10 @@ add_specs suite_builder setup = r2 = my_table.merge lookup key_columns="X" r2.should_fail_with No_Common_Type - r2.catch.to_display_text . should_contain "Integer" - r2.catch.to_display_text . should_contain "Char" - r2.catch.to_display_text . should_contain "when unifying column [Y]" + txt = r2.catch.to_display_text + ((txt.contains "Integer") || (txt.contains "Decimal")).should_be_true + txt.should_contain "Char" + txt.should_contain "when unifying column [Y]" group_builder.specify "will allow incompatible types if allow_unmatched_rows=False" <| lookup = table_builder [["X", [1, 2, 3]], ["Y", [1, 11, 111]]] @@ -335,7 +320,7 @@ add_specs suite_builder setup = r2 = my_table.merge lookup key_columns=[] add_new_columns=True r2.should_fail_with Illegal_Argument - if setup.is_database.not then group_builder.specify "(in-memory only) will preserve the order of rows from the original table" <| + if setup.is_database.not then group_builder.specify "(in-memory only) will preserve the 279table_builder of rows from the original table" <| lookup = table_builder [["Y", [1, 0]], ["V", ["TRUE", "FALSE"]]] xs = 0.up_to 50 . to_vector ys = xs.map x-> x%2 @@ -386,7 +371,7 @@ add_specs suite_builder setup = r3.row_count . should_equal 3 r3.at "Z" . to_vector . length . should_equal 3 - r3.at "Z" . to_vector . should_contain_the_same_elements_as [10, 20, 30] + r3.at "Z" . to_vector . should_equal_ignoring_order [10, 20, 30] m3.should_fail_with Invariant_Violation r3.at "Z" . to_vector . should_fail_with Invariant_Violation diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso index 8a359d4b86..1044c80ef3 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Union_Spec.enso @@ -1,5 +1,6 @@ from Standard.Base import all import Standard.Base.Errors.Illegal_Argument.Illegal_Argument +import Standard.Base.Runtime.Ref.Ref from Standard.Table import all from Standard.Table.Errors import all @@ -10,6 +11,7 @@ from Standard.Database.Errors import Unsupported_Database_Operation, Integrity_E from Standard.Test import all from project.Common_Table_Operations.Util import all +from project.Common_Table_Operations.Date_Time_Spec import as_local_date_time_repr import project.Util main filter=Nothing = run_default_backend add_specs filter @@ -17,13 +19,9 @@ main filter=Nothing = run_default_backend add_specs filter type My_Type Value x y -type Data - Value ~connection +type Lazy_Ref + Value ~get - setup create_connection_fn = - Data.Value (create_connection_fn Nothing) - - teardown self = self.connection.close # the ... operator used in the calls for Table.from_union and first.union "freezes" the default arguments so that they can be specified later, allowing us to run the full suite of tests call_static_union tables = Table.from_union tables ... @@ -48,94 +46,128 @@ add_specs suite_builder setup = r.catch.to_display_text . should_contain "at least 1" run_union_tests group_builder setup call_union = - create_connection_fn = setup.create_connection_func - data = Data.setup create_connection_fn - group_builder.teardown <| - data.teardown - table_builder cols = - setup.table_builder cols connection=data.connection + materialize = setup.materialize + ## In Database backends we cannot make assumptions about the order of rows in the result, so we sort the table after materializing. + In the in-memory backend, we avoid the sorting on purpose, to test that the order of rows in the result is as expected. + materialize_and_maybe_sort table column_for_sort="rowid" = + if setup.is_database.not then table else + # We also remove the rowid column from the result to avoid interfering with the test. + materialize table . sort column_for_sort . remove_columns "rowid" on_problems=..Ignore + ## A paired builder to `materialize_and_maybe_sort`. + In Database, it adds a rowid column to the table. In in-memory, it just delegates to `table_builder`. + For convenience, to avoid having to specify the starting id on each call, we just use a global counter. + This way, tables created next to each other will have increasing rowids, as we'd expect. + rowid_counter = Ref.new 0 + build_table_with_maybe_rowid structure = + if setup.is_database.not then setup.table_builder structure else + length = structure.first.second.length + start = rowid_counter.modify (+length) + # The rowid column is first to work correctly with By Position matching. + setup.table_builder ([["rowid", start.up_to (start+length) . to_vector]] + structure) + + table_builder = setup.table_builder group_builder.specify "should merge columns from multiple tables" <| t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]], ["C", [True, False, True]]] t2 = table_builder [["A", [4, 5, 6]], ["B", ["d", "e", "f"]], ["C", [False, True, False]]] - t3 = table_builder [["A", [7, 8, 9]], ["B", ["g", "h", "i"]], ["C", [True, False, False]]] t4 = call_union [t1, t2] expect_column_names ["A", "B", "C"] t4 - t4.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - t4.at "B" . to_vector . should_equal ["a", "b", "c", "d", "e", "f"] - t4.at "C" . to_vector . should_equal [True, False, True, False, True, False] - t5 = call_union [t3, t1, t2] - expect_column_names ["A", "B", "C"] t5 - t5.at "A" . to_vector . should_equal [7, 8, 9, 1, 2, 3, 4, 5, 6] - t5.at "B" . to_vector . should_equal ["g", "h", "i", "a", "b", "c", "d", "e", "f"] - t5.at "C" . to_vector . should_equal [True, False, False, True, False, True, False, True, False] + m4 = materialize_and_maybe_sort t4 "A" + m4.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] + m4.at "B" . to_vector . should_equal ["a", "b", "c", "d", "e", "f"] + m4.at "C" . to_vector . should_equal [True, False, True, False, True, False] + # In in-memory we also check one more test with ordering: + if setup.is_database.not then + t3 = table_builder [["A", [7, 8, 9]], ["B", ["g", "h", "i"]], ["C", [True, False, False]]] + t5 = call_union [t3, t1, t2] + expect_column_names ["A", "B", "C"] t5 + t5.at "A" . to_vector . should_equal [7, 8, 9, 1, 2, 3, 4, 5, 6] + t5.at "B" . to_vector . should_equal ["g", "h", "i", "a", "b", "c", "d", "e", "f"] + t5.at "C" . to_vector . should_equal [True, False, False, True, False, True, False, True, False] + + table1 = Lazy_Ref.Value <| + build_table_with_maybe_rowid [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] + table2 = Lazy_Ref.Value <| + build_table_with_maybe_rowid [["C", ["d", "e", "f"]], ["A", [4, 5, 6]]] + table3 = Lazy_Ref.Value <| + build_table_with_maybe_rowid [["D", [Nothing, Nothing, 0]], ["C", ["g", "h", "i"]]] group_builder.specify "should fill unmatched columns (by name matching) with nulls and report a warning by default" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["C", ["d", "e", "f"]], ["A", [4, 5, 6]]] - t3 = table_builder [["D", [Nothing, Nothing, 0]], ["C", ["g", "h", "i"]]] + t1 = table1.get + t2 = table2.get + t3 = table3.get action = call_union [t1, t2, t3] on_problems=_ tester table = - expect_column_names ["A", "B", "C", "D"] table - table.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, Nothing] - table.at "B" . to_vector . should_equal ["a", "b", "c", Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - table.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, "d", "e", "f", "g", "h", "i"] - table.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, 0] + m = materialize_and_maybe_sort table + expect_column_names ["A", "B", "C", "D"] m + m.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, Nothing] + m.at "B" . to_vector . should_equal ["a", "b", "c", Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] + m.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, "d", "e", "f", "g", "h", "i"] + m.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, 0] problems = [Unmatched_Columns.Error ["A", "B", "C", "D"]] Problems.test_problem_handling action problems tester action2 = call_union [t2, t3] on_problems=_ tester2 table = - expect_column_names ["C", "A", "D"] table - table.at "C" . to_vector . should_equal ["d", "e", "f", "g", "h", "i"] - table.at "A" . to_vector . should_equal [4, 5, 6, Nothing, Nothing, Nothing] - table.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, 0] + m = materialize_and_maybe_sort table + expect_column_names ["C", "A", "D"] m + m.at "C" . to_vector . should_equal ["d", "e", "f", "g", "h", "i"] + m.at "A" . to_vector . should_equal [4, 5, 6, Nothing, Nothing, Nothing] + m.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, 0] problems2 = [Unmatched_Columns.Error ["A", "D"]] Problems.test_problem_handling action2 problems2 tester2 group_builder.specify "should fill unmatched columns with nulls with no warning, if In_Any is explicitly chosen" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["C", ["d", "e", "f"]], ["A", [4, 5, 6]]] - t3 = table_builder [["D", [Nothing, Nothing, 0]], ["C", ["g", "h", "i"]]] + t1 = table1.get + t2 = table2.get + t3 = table3.get table = call_union [t1, t2, t3] columns_to_keep=..In_Any on_problems=..Report_Error Problems.assume_no_problems table - expect_column_names ["A", "B", "C", "D"] table - table.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, Nothing] - table.at "B" . to_vector . should_equal ["a", "b", "c", Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] - table.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, "d", "e", "f", "g", "h", "i"] - table.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, 0] + m = materialize_and_maybe_sort table + expect_column_names ["A", "B", "C", "D"] m + m.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, Nothing] + m.at "B" . to_vector . should_equal ["a", "b", "c", Nothing, Nothing, Nothing, Nothing, Nothing, Nothing] + m.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, "d", "e", "f", "g", "h", "i"] + m.at "D" . to_vector . should_equal [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, 0] group_builder.specify "should drop unmatched columns and warn, if In_All is selected" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["C", ["d", "e", "f"]], ["A", [4, 5, 6]]] - t3 = table_builder [["A", [Nothing, Nothing, 0]], ["C", ["g", "h", "i"]]] + t1 = table1.get # A, B + t2 = table2.get # C, A + t3 = table3.get.rename_columns [["D", "A"]] # A, C t4 = call_union [t1, t2, t3] columns_to_keep=..In_All w = Problems.expect_only_warning Unmatched_Columns t4 w.column_names.should_equal ["B", "C"] - expect_column_names ["A"] t4 - t4.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, 0] + m = materialize_and_maybe_sort t4 + expect_column_names ["A"] m + m.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, Nothing, Nothing, 0] group_builder.specify "should fail if asked to drop unmatched columns but the set of common columns is empty" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["C", ["d", "e", "f"]], ["A", [4, 5, 6]]] - t3 = table_builder [["D", [Nothing, Nothing, 0]], ["C", ["g", "h", "i"]]] + # We cannot have the rowid column here as we explicitly want to have no common columns. + t1 = table1.get.remove_columns "rowid" on_problems=..Ignore + t2 = table2.get.remove_columns "rowid" on_problems=..Ignore + t3 = table3.get.remove_columns "rowid" on_problems=..Ignore t4 = call_union [t1, t2, t3] columns_to_keep=..In_All on_problems=..Ignore t4.should_fail_with No_Output_Columns t4.catch.to_display_text . should_equal "No columns in the result, because of another problem: Unmatched columns are set to be dropped, but no common column names were found." + table4 = Lazy_Ref.Value <| + table_builder [["A", [1]], ["X", [2]], ["B", ["a"]], ["Y", [3]]] + table5 = Lazy_Ref.Value <| + table_builder [["A", [4]], ["Z", [5]], ["B", ["b"]], ["X", [6]]] group_builder.specify "should allow to select specified columns for union by In_List, using the ordering from the list" <| - t1 = table_builder [["A", [1]], ["X", [2]], ["B", ["a"]], ["Y", [3]]] - t2 = table_builder [["A", [4]], ["Z", [5]], ["B", ["b"]], ["X", [6]]] + t1 = table4.get + t2 = table5.get t3 = call_union [t1, t2] columns_to_keep=(..In_List ["B", "A"]) expect_column_names ["B", "A"] t3 - t3.at "B" . to_vector . should_equal ["a", "b"] - t3.at "A" . to_vector . should_equal [1, 4] + m3 = materialize_and_maybe_sort t3 "A" + m3.at "B" . to_vector . should_equal ["a", "b"] + m3.at "A" . to_vector . should_equal [1, 4] group_builder.specify "should add a Null column for unmatched columns from In_List" <| t1 = table_builder [["A", [1]], ["X", [2]]] @@ -143,18 +175,19 @@ run_union_tests group_builder setup call_union = t3 = call_union [t1, t2] columns_to_keep=(..In_List ["B", "A"]) expect_column_names ["B", "A"] t3 - t3.at "B" . to_vector . should_equal [Nothing, Nothing] - t3.at "A" . to_vector . should_equal [1, 5] + m3 = materialize_and_maybe_sort t3 "A" + m3.at "B" . to_vector . should_equal [Nothing, Nothing] + m3.at "A" . to_vector . should_equal [1, 5] group_builder.specify "does not allow an empty list in In_List" <| - t1 = table_builder [["A", [1]], ["X", [2]]] - t2 = table_builder [["Z", [4]], ["A", [5]]] + t1 = table1.get + t2 = table2.get r = call_union [t1, t2] columns_to_keep=(..In_List []) r.should_fail_with Illegal_Argument group_builder.specify "does not error if duplicate entries appear in the In_List" <| - t1 = table_builder [["A", [1]], ["X", [2]], ["B", ["a"]], ["Y", [3]]] - t2 = table_builder [["A", [4]], ["Z", [5]], ["B", ["b"]], ["X", [6]]] + t1 = table4.get + t2 = table5.get t3 = call_union [t1, t2] columns_to_keep=(..In_List ["B", "B", "A", "A", "B"]) expect_column_names ["B", "A"] t3 @@ -165,8 +198,9 @@ run_union_tests group_builder setup call_union = t3 = call_union [t1, t2] match_columns=Match_Columns.By_Position expect_column_names ["A", "Y"] t3 - t3.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - t3.at "Y" . to_vector . should_equal ["a", "b", "c", "d", "e", "f"] + m3 = materialize_and_maybe_sort t3 "A" + m3.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] + m3.at "Y" . to_vector . should_equal ["a", "b", "c", "d", "e", "f"] group_builder.specify "should fill extra columns (positional matching) with nulls and report a warning by default" <| t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] @@ -176,9 +210,10 @@ run_union_tests group_builder setup call_union = action = call_union [t1, t2, t3] match_columns=Match_Columns.By_Position on_problems=_ tester table = expect_column_names ["A1", "B1", "C"] table - table.at "A1" . to_vector . should_equal [1, 2, 3, 4, 5, 6, 10, 20, 30] - table.at "B1" . to_vector . should_equal ["a", "b", "c", "d", "e", "f", Nothing, Nothing, Nothing] - table.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, 7, 8, 9, Nothing, Nothing, Nothing] + m = materialize_and_maybe_sort table "A1" + m.at "A1" . to_vector . should_equal [1, 2, 3, 4, 5, 6, 10, 20, 30] + m.at "B1" . to_vector . should_equal ["a", "b", "c", "d", "e", "f", Nothing, Nothing, Nothing] + m.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, 7, 8, 9, Nothing, Nothing, Nothing] problems = [Column_Count_Mismatch.Error 3 1] Problems.test_problem_handling action problems tester @@ -189,7 +224,7 @@ run_union_tests group_builder setup call_union = t4 = call_union [t1, t2, t3] columns_to_keep=..In_All match_columns=..By_Position expect_column_names ["A"] t4 - t4.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6, 10, 20, 30] + t4.at "A" . to_vector . should_equal_ignoring_order [1, 2, 3, 4, 5, 6, 10, 20, 30] w = Problems.expect_only_warning Column_Count_Mismatch t4 w.expected.should_equal 3 w.actual.should_equal 1 @@ -202,13 +237,14 @@ run_union_tests group_builder setup call_union = t4 = call_union [t1, t2, t3] columns_to_keep=..In_Any match_columns=..By_Position on_problems=..Report_Error Problems.assume_no_problems t4 expect_column_names ["A1", "B1", "C"] t4 - t4.at "A1" . to_vector . should_equal [1, 2, 3, 4, 5, 6, 10, 20, 30] - t4.at "B1" . to_vector . should_equal ["a", "b", "c", "d", "e", "f", Nothing, Nothing, Nothing] - t4.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, 7, 8, 9, Nothing, Nothing, Nothing] + m4 = materialize_and_maybe_sort t4 "A1" + m4.at "A1" . to_vector . should_equal [1, 2, 3, 4, 5, 6, 10, 20, 30] + m4.at "B1" . to_vector . should_equal ["a", "b", "c", "d", "e", "f", Nothing, Nothing, Nothing] + m4.at "C" . to_vector . should_equal [Nothing, Nothing, Nothing, 7, 8, 9, Nothing, Nothing, Nothing] group_builder.specify "does not allow In_List with positional matching" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = table_builder [["A1", [4, 5, 6]], ["B1", ["d", "e", "f"]], ["C", [7, 8, 9]]] + t1 = table4.get + t2 = table5.get r = call_union [t1, t2] columns_to_keep=(..In_List ["A", "B"]) match_columns=Match_Columns.By_Position r.should_fail_with Illegal_Argument @@ -218,42 +254,58 @@ run_union_tests group_builder setup call_union = check table = expect_column_names ["X", "A"] table - table.at "X" . to_vector . should_equal [1, 2, 3, 4, 5, 6] - table.at "A" . to_vector . should_equal [Nothing, Nothing, Nothing, "a", "b", "c"] + m = materialize_and_maybe_sort table "X" + m.at "X" . to_vector . should_equal [1, 2, 3, 4, 5, 6] + m.at "A" . to_vector . should_equal [Nothing, Nothing, Nothing, "a", "b", "c"] t3 = call_union [t1, t2] match_columns=Match_Columns.By_Position within_table t3 <| check t3 Problems.get_attached_warnings t3 . should_equal [Column_Count_Mismatch.Error 2 1] - t5 = table_builder [["Y", [7, 8, 9]], ["A", ["d", "e", "f"]], ["Z", [10, 11, 12]]] - t6 = table_builder [["W", [0]]] - t7 = table_builder [["X", [7, 8, 9]], ["Y", ["d", "e", "f"]], ["Z", [10, 11, 12]]] + t5 = setup.light_table_builder [["Y", [7, 8, 9]], ["A", ["d", "e", "f"]], ["Z", [10, 11, 12]]] + t6 = setup.light_table_builder [["W", [0]]] + t7 = setup.light_table_builder [["X", [7, 8, 9]], ["Y", ["d", "e", "f"]], ["Z", [10, 11, 12]]] t8 = call_union [t1, t2, t5, t6, t7] match_columns=Match_Columns.By_Position expect_column_names ["Y", "A", "Z"] t8 group_builder.specify "should allow to merge a table with itself" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] - t2 = call_union [t1, t1, t1] + t1 = table1.get + t2 = call_union [t1, t1, t1] . remove_columns "rowid" on_problems=..Ignore expect_column_names ["A", "B"] t2 - t2.at "A" . to_vector . should_equal [1, 2, 3, 1, 2, 3, 1, 2, 3] - t2.at "B" . to_vector . should_equal ["a", "b", "c", "a", "b", "c", "a", "b", "c"] + case setup.is_database of + False -> + t2.at "A" . to_vector . should_equal [1, 2, 3, 1, 2, 3, 1, 2, 3] + t2.at "B" . to_vector . should_equal ["a", "b", "c", "a", "b", "c", "a", "b", "c"] + True -> + m2 = materialize t2 + m2.at "A" . to_vector . should_equal_ignoring_order [1, 1, 1, 2, 2, 2, 3, 3, 3] + m2.at "B" . to_vector . should_equal_ignoring_order ["a", "a", "a", "b", "b", "b", "c", "c", "c"] group_builder.specify "should not de-duplicate rows" <| + # We don't use build_table_with_maybe_rowid here on purpose, as that would make the rows no longer duplicated. t1 = table_builder [["A", [1, 1, 3]], ["B", ["a", "a", "c"]]] t2 = table_builder [["A", [1, 2, 2]], ["B", ["a", "b", "b"]]] t3 = call_union [t1, t2] expect_column_names ["A", "B"] t3 - t3.at "A" . to_vector . should_equal [1, 1, 3, 1, 2, 2] - t3.at "B" . to_vector . should_equal ["a", "a", "c", "a", "b", "b"] + m3 = materialize t3 + case setup.is_database of + False -> + m3.at "A" . to_vector . should_equal [1, 1, 3, 1, 2, 2] + m3.at "B" . to_vector . should_equal ["a", "a", "c", "a", "b", "b"] + True -> + sorted = m3.sort "A" + sorted.at "A" . to_vector . should_equal [1, 1, 1, 2, 2, 3] + sorted.at "B" . to_vector . should_equal ["a", "a", "a", "b", "b", "c"] group_builder.specify "should gracefully handle the case where no tables to union were provided" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] + t1 = table1.get.remove_columns "rowid" on_problems=..Ignore check_same table = expect_column_names ["A", "B"] table - table.at "A" . to_vector . should_equal [1, 2, 3] - table.at "B" . to_vector . should_equal ["a", "b", "c"] + m = materialize_and_maybe_sort table "A" + m.at "A" . to_vector . should_equal [1, 2, 3] + m.at "B" . to_vector . should_equal ["a", "b", "c"] check_same <| call_union [t1] check_same <| call_union [t1] match_columns=Match_Columns.By_Position @@ -276,47 +328,50 @@ run_union_tests group_builder setup call_union = t3 = call_union [t1, t2] expect_column_names ["A"] t3 Problems.assume_no_problems t3 - t3.at "A" . to_vector . should_equal ["a", "b", "c", "xyz", "abc", "def"] - t3.at "A" . value_type . is_text . should_be_true - Test.with_clue "t3[A].value_type="+(t3.at "A").value_type.to_display_text+": " <| - t3.at "A" . value_type . variable_length . should_be_true + t3.at "A" . to_vector . should_equal_ignoring_order ["a", "b", "c", "xyz", "abc", "def"] + t3.at "A" . value_type . should_be_a (Value_Type.Char ...) + t3.at "A" . value_type . variable_length . should_be_true group_builder.specify "should find a common type that will fit the merged columns (Integer + Float)" <| t1 = table_builder [["A", [0, 1, 2]]] t2 = table_builder [["A", [1.0, 2.0, 2.5]]] setup.expect_integer_type <| t1.at "A" - t2.at "A" . value_type . is_floating_point . should_be_true + t2.at "A" . value_type . should_be_a (Value_Type.Float ...) t3 = call_union [t1, t2] expect_column_names ["A"] t3 Problems.assume_no_problems t3 - t3.at "A" . value_type . is_floating_point . should_be_true - t3.at "A" . to_vector . should_equal [0, 1, 2, 1.0, 2.0, 2.5] + is_float_or_decimal (t3.at "A") + t3.at "A" . to_vector . should_equal_ignoring_order [0, 1, 2, 1.0, 2.0, 2.5] group_builder.specify "should find a common type that will fit the merged columns (numeric + Boolean)" <| - t1 = table_builder [["A", [0, 1, 20]]] - t2 = table_builder [["A", [True, False, True]]] + t1 = build_table_with_maybe_rowid [["A", [0, 1, 20]]] + t2 = build_table_with_maybe_rowid [["A", [True, False, True]]] setup.expect_integer_type <| t1.at "A" t2.at "A" . value_type . should_equal Value_Type.Boolean t3 = call_union [t1, t2] - expect_column_names ["A"] t3 - Problems.assume_no_problems t3 setup.expect_integer_type <| t3.at "A" - t3.at "A" . to_vector . should_equal [0, 1, 20, 1, 0, 1] + Problems.assume_no_problems t3 - t4 = table_builder [["A", [1.5, 0.0, 2.0]]] + m3 = materialize_and_maybe_sort t3 + expect_column_names ["A"] m3 + m3.at "A" . to_vector . should_equal [0, 1, 20, 1, 0, 1] + + t4 = build_table_with_maybe_rowid [["A", [1.5, 0.0, 2.0]]] t5 = call_union [t2, t4] Problems.assume_no_problems t5 - t5.at "A" . value_type . is_floating_point . should_be_true - t5.at "A" . to_vector . should_equal [1.0, 0.0, 1.0, 1.5, 0.0, 2.0] + t5.at "A" . value_type . should_be_a (Value_Type.Float ...) + + m5 = materialize_and_maybe_sort t5 + m5.at "A" . to_vector . should_equal [1.0, 0.0, 1.0, 1.5, 0.0, 2.0] group_builder.specify "should warn about loss of precision when converting large Integer to Float" pending=(if setup.is_database then "Loss_Of_Integer_Precision not yet supported in DB.") <| # 2^70 is not exactly representable as a Float. - t1 = table_builder [["A", [2^70, 2^10, 2]]] - t2 = table_builder [["A", [1.5, 2.0, 2.5]]] + t1 = build_table_with_maybe_rowid [["A", [2^70, 2^10, 2]]] + t2 = build_table_with_maybe_rowid [["A", [1.5, 2.0, 2.5]]] t1.at "A" . value_type . is_decimal . should_be_true t2.at "A" . value_type . is_floating_point . should_be_true @@ -325,8 +380,10 @@ run_union_tests group_builder setup call_union = w = Problems.expect_only_warning Loss_Of_Integer_Precision t3 # TODO should we try to include column name here for context? may be worth it... w.affected_rows_count.should_equal 1 - t3.at "A" . value_type . is_floating_point . should_be_true - t3.at "A" . to_vector . should_equal [(2^70).to_float, 2^10, 2, 1.5, 2.0, 2.5] + + t3.at "A" . value_type . should_be_a (Value_Type.Float ...) + m3 = materialize_and_maybe_sort t3 + m3.at "A" . to_vector . should_equal [(2^70).to_float, 2^10, 2, 1.5, 2.0, 2.5] group_builder.specify "should find a common type (Integer and Char of different sizes)" <| t1 = (table_builder [["X", [0, 1, 2]], ["Y", ['aa', 'bb', 'cc']]]) . cast "X" (Value_Type.Integer Bits.Bits_16) . cast "Y" (Value_Type.Char size=2 variable_length=False) @@ -341,19 +398,29 @@ run_union_tests group_builder setup call_union = t12.at "X" . value_type . should_equal (Value_Type.Integer Bits.Bits_32) t12.at "Y" . value_type . should_equal (Value_Type.Char size=2 variable_length=True) - t12.at "X" . to_vector . should_equal [0, 1, 2, 3, 4, 5] - t12.at "Y" . to_vector . should_equal ['aa', 'bb', 'cc', 'x', 'y', 'z'] + m12 = materialize_and_maybe_sort t12 "X" + m12.at "X" . to_vector . should_equal [0, 1, 2, 3, 4, 5] + m12.at "Y" . to_vector . should_equal ['aa', 'bb', 'cc', 'x', 'y', 'z'] date_time_pending = if setup.test_selection.date_time.not then "Date/Time operations are not supported." group_builder.specify "should warn when converting a Date to Date_Time" pending=date_time_pending <| - t1 = table_builder [["D", [Date_Time.new 2024 5 16 16 48 23]]] - t2 = table_builder [["D", [Date.new 2019 10 23, Date.new 2020]]] + t1 = build_table_with_maybe_rowid [["D", [Date_Time.new 2024 5 16 16 48 23]]] + t2 = build_table_with_maybe_rowid [["D", [Date.new 2019 10 23, Date.new 2020]]] action = call_union [t1, t2] on_problems=_ tester table = - expect_column_names ["D"] table - table.at "D" . value_type . should_equal Value_Type.Date_Time - table.at "D" . to_vector . should_equal_tz_agnostic [Date_Time.new 2024 5 16 16 48 23, Date_Time.new 2019 10 23 0 0 0, Date_Time.new 2020 1 1 0 0 0] + m = materialize_and_maybe_sort table + expect_column_names ["D"] m + m.at "D" . value_type . should_equal Value_Type.Date_Time + ## Some backends will preserve the local time but change the offset (e.g. Snowflake) + and some will preserve the time instant but change the local time in the process of aligning offset (e.g. Postgres). + dt_vec = m.at "D" . to_vector + Test.with_clue <| "(D = "+dt_vec.to_display_text+") " <| + as_local dt = dt.format "YYYY-MM-DD HH:mm:ss" + preserves_local = (dt_vec.map as_local) == ["2024-05-16 16:48:23", "2019-10-23 00:00:00", "2020-01-01 00:00:00"] + at_utc dt = dt.at_zone Time_Zone.utc + preserves_instant = (dt_vec.map at_utc) == ([Date_Time.new 2024 5 16 16 48 23, Date_Time.new 2019 10 23 0 0 0, Date_Time.new 2020 1 1 0 0 0].map at_utc) + (preserves_local || preserves_instant) . should_be_true problems = [Mixing_Date_Time_Types.Date_To_Date_Time "D"] problems.first.to_display_text . should_contain "[D]" Problems.test_problem_handling action problems tester @@ -395,14 +462,15 @@ run_union_tests group_builder setup call_union = action = call_union [t1, t2] on_problems=_ result_checker table = expect_column_names ["A", "B", "C"] table + m = materialize_and_maybe_sort table "A" # If type was matched - the columns are merged as is: - table.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] + m.at "A" . to_vector . should_equal [1, 2, 3, 4, 5, 6] setup.expect_integer_type <| table.at "A" # If mixed, they are converted to text representation: - table.at "B" . to_vector . should_equal ["a", "b", "c", "1", "2", "3"] + m.at "B" . to_vector . should_equal ["a", "b", "c", "1", "2", "3"] table.at "B" . value_type . is_text . should_be_true - v = table.at "C" . to_vector + v = m.at "C" . to_vector # The check needs to be case insensitive because various database backends may represent Booleans with lower or uppercase. v.take 2 . map (t -> t.to_case Case.Lower) . should_equal ["true", "false"] # Nothing is preserved, not converted to text because we want to preserve the meaning of 'missing value': @@ -422,18 +490,21 @@ run_union_tests group_builder setup call_union = types = w.types.map value_type-> Meta.meta value_type . constructor . name Test.with_clue "(should be one of...) " <| - [["Char", "Integer"], ["Boolean", "Char"]].should_contain types + [["Char", "Decimal"], ["Char", "Integer"], ["Boolean", "Char"]].should_contain types Problems.test_advanced_problem_handling action error_checker warnings_checker result_checker group_builder.specify "if no common type can be found, will fall back to converting all types to text and warn (Date+Time)" pending=date_time_pending <| - t1 = table_builder [["D", [Time_Of_Day.new 12, Time_Of_Day.new 13, Time_Of_Day.new 14]]] - t2 = table_builder [["D", [Date.new 2019, Date.new 2020, Date.new 2021]]] + t1 = build_table_with_maybe_rowid [["D", [Time_Of_Day.new 12, Time_Of_Day.new 13, Time_Of_Day.new 14]]] + t2 = build_table_with_maybe_rowid [["D", [Date.new 2019, Date.new 2020, Date.new 2021]]] action = call_union [t1, t2] on_problems=_ tester table = - expect_column_names ["D"] table - table.at "D" . to_vector . should_equal ["12:00:00", "13:00:00", "14:00:00", "2019-01-01", "2020-01-01", "2021-01-01"] - table.at "D" . value_type . is_text . should_be_true + m = materialize_and_maybe_sort table + expect_column_names ["D"] m + # The nanoseconds suffix in time of day is optional - we strip it for the comparison. + strip_ns s = s.replace "\.0+$".to_regex "" + m.at "D" . to_vector . map strip_ns . should_equal ["12:00:00", "13:00:00", "14:00:00", "2019-01-01", "2020-01-01", "2021-01-01"] + m.at "D" . value_type . is_text . should_be_true problems = [No_Common_Type.Warning_Convert_To_Text [Value_Type.Time, Value_Type.Date] "D"] Problems.test_problem_handling action problems tester @@ -442,13 +513,13 @@ run_union_tests group_builder setup call_union = t2 = table_builder [["B", ["a"]]] r1 = call_union [t1, t2] match_columns=Match_Columns.By_Position expect_column_names ["A"] r1 - r1.at "A" . value_type . is_text . should_be_true - r1.at "A" . to_vector . should_equal ["1", "a"] + r1.at "A" . value_type . should_be_a (Value_Type.Char ...) + r1.at "A" . to_vector . should_equal_ignoring_order ["1", "a"] w = Problems.expect_only_warning No_Common_Type r1 w.related_column_name.should_equal "A" group_builder.specify "should gracefully handle tables from different backends" <| - t1 = table_builder [["A", [1, 2, 3]], ["B", ["a", "b", "c"]]] + t1 = table1.get alternative_connection = Database.connect (SQLite.In_Memory) t0 = (Table.new [["A", [1, 2, 4]], ["B", ["10", "20", "30"]]]).select_into_database_table alternative_connection "T0" temporary=True diff --git a/test/Table_Tests/src/Common_Table_Operations/Join/Zip_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Join/Zip_Spec.enso index 877923eece..e1036a91a4 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Join/Zip_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Join/Zip_Spec.enso @@ -190,7 +190,7 @@ add_specs suite_builder setup = t3 = t1.zip t2 expected_problems = [Row_Count_Mismatch.Error 2 1, Duplicate_Output_Column_Names.Error ["Right X"]] - Problems.get_attached_warnings t3 . should_contain_the_same_elements_as expected_problems + Problems.get_attached_warnings t3 . should_equal_ignoring_order expected_problems group_builder.specify "should allow to zip the table with itself" <| ## Even though this does not seem very useful, we should verify that diff --git a/test/Table_Tests/src/Common_Table_Operations/Main.enso b/test/Table_Tests/src/Common_Table_Operations/Main.enso index bc17074ff7..08ddb2f90d 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Main.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Main.enso @@ -118,6 +118,8 @@ type Test_Selection each group. Guaranteed in the in-memory backend, but may not be supported by all databases. - date_time: Specifies if the backend supports date/time operations. + - text_length_limited_columns: Specifies if the backend supports setting + a length limit on text columns. - fixed_length_text_columns: Specifies if the backend supports fixed length text columns. - length_restricted_text_columns: Specifies if the backend supports @@ -125,6 +127,8 @@ type Test_Selection - removes_trailing_whitespace_casting_from_char_to_varchar: if SELECT concat('X', CAST(CAST(' ' AS CHAR(3)) AS VARCHAR(3)), 'X') returns XX then this should be set to True + - char_max_size_after_substring: Specifies how the max size of the char + type behaves after text_left/text_right. - different_size_integer_types: Specifies if the backend supports integer types of various sizes, like 16-bit or 32-bit integers. - supports_8bit_integer: Specifies if the backend supports 8-bit @@ -153,7 +157,7 @@ type Test_Selection - supports_date_time_without_timezone: Specifies if the backend supports date/time operations without a timezone (true for most Database backends). Defaults to `.is_integer`. - Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True is_nan_comparable=False distinct_returns_first_row_from_group_if_ordered=True date_time=True fixed_length_text_columns=False length_restricted_text_columns=True removes_trailing_whitespace_casting_from_char_to_varchar=False different_size_integer_types=True supports_8bit_integer=False supports_decimal_type=False supports_time_duration=False supports_nanoseconds_in_time=False supports_mixed_columns=False supported_replace_params=Nothing run_advanced_edge_case_tests_by_default=True supports_date_time_without_timezone=False + Config supports_case_sensitive_columns=True order_by=True natural_ordering=False case_insensitive_ordering=True order_by_unicode_normalization_by_default=False case_insensitive_ascii_only=False allows_mixed_type_comparisons=True supports_unicode_normalization=False is_nan_and_nothing_distinct=True is_nan_comparable=False distinct_returns_first_row_from_group_if_ordered=True date_time=True text_length_limited_columns=False fixed_length_text_columns=False length_restricted_text_columns=True removes_trailing_whitespace_casting_from_char_to_varchar=False char_max_size_after_substring:Char_Max_Size_After_Substring_Behavior=..Kept different_size_integer_types=True supports_8bit_integer=False supports_decimal_type=False supports_time_duration=False supports_nanoseconds_in_time=False supports_mixed_columns=False supported_replace_params=Nothing run_advanced_edge_case_tests_by_default=True supports_date_time_without_timezone=False ## Specifies if the advanced edge case tests shall be run. @@ -163,6 +167,16 @@ type Test_Selection run_advanced_edge_case_tests self -> Boolean = self.run_advanced_edge_case_tests_by_default || (Environment.get "ENSO_ADVANCED_EDGE_CASE_TESTS" . is_nothing . not) +type Char_Max_Size_After_Substring_Behavior + # The max size is reset to unbounded. + Reset + + # The max size is kept unchanged. + Kept + + # The max size is reduced to the length of the substring. + Reduced + add_specs suite_builder setup = Core_Spec.add_specs suite_builder setup Select_Columns_Spec.add_specs suite_builder setup diff --git a/test/Table_Tests/src/Common_Table_Operations/Map_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Map_Spec.enso index 6ed424ce15..cb113bacba 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Map_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Map_Spec.enso @@ -47,7 +47,7 @@ add_specs suite_builder setup = if (x % 2) == 0 then Warning.attach (Illegal_State.Error "MY WARNING "+x.to_text) (2*x + 1) else 2*x + 1 c1 = t.at "X" . map f warnings = Problems.get_attached_warnings c1 - warnings.map Error.unwrap . map .message . should_contain_the_same_elements_as ["MY WARNING 2", "MY WARNING 4"] + warnings.map Error.unwrap . map .message . should_equal_ignoring_order ["MY WARNING 2", "MY WARNING 4"] c1.to_vector . should_equal [3, 5, 7, 9] group_builder.specify "should respect the expected_value_type" <| diff --git a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso index 1a24b88dde..b3525bea73 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Nothing_Spec.enso @@ -239,7 +239,7 @@ add_specs suite_builder setup = t1 = table . distinct ["x"] v = t1.at "x" . to_vector v . length . should_equal 3 - v . should_contain_the_same_elements_as [value, other_value, Nothing] + v . should_equal_ignoring_order [value, other_value, Nothing] group_builder.specify "Correctly handle Nothing in .distinct for Nothing" <| table = table_builder_typed [["x", [Nothing, Nothing, Nothing, Nothing, Nothing, Nothing]]] Value_Type.Char diff --git a/test/Table_Tests/src/Common_Table_Operations/Select_Columns_Spec.enso b/test/Table_Tests/src/Common_Table_Operations/Select_Columns_Spec.enso index b72453378a..6cc8b4370a 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Select_Columns_Spec.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Select_Columns_Spec.enso @@ -522,7 +522,7 @@ add_specs suite_builder setup = tester = expect_column_names ["FirstColumn", "beta", "gamma", "delta"] err_checker err = err.catch.should_be_a Missing_Input_Columns.Error - err.catch.criteria.should_contain_the_same_elements_as ["omicron", weird_name] + err.catch.criteria.should_equal_ignoring_order ["omicron", weird_name] Problems.test_advanced_problem_handling action err_checker (x-> x) tester err = data.table.rename_columns map @@ -534,7 +534,7 @@ add_specs suite_builder setup = tester = expect_column_names ["FirstColumn", "beta", "gamma", "Another"] err_checker err = err.catch.should_be_a Missing_Input_Columns.Error - err.catch.criteria.should_contain_the_same_elements_as [-200, 100, 300] + err.catch.criteria.should_equal_ignoring_order [-200, 100, 300] Problems.test_advanced_problem_handling action err_checker (x-> x) tester err = data.table.rename_columns map @@ -622,7 +622,7 @@ add_specs suite_builder setup = tester = expect_column_names ["A", "B", "C", "D"] problem_checker problem = problem.should_be_a Too_Many_Column_Names_Provided.Error - problem.column_names.should_contain_the_same_elements_as ["E", "F"] + problem.column_names.should_equal_ignoring_order ["E", "F"] True err_checker err = problem_checker err.catch diff --git a/test/Table_Tests/src/Common_Table_Operations/Util.enso b/test/Table_Tests/src/Common_Table_Operations/Util.enso index abf3e71a4b..c19ffea17a 100644 --- a/test/Table_Tests/src/Common_Table_Operations/Util.enso +++ b/test/Table_Tests/src/Common_Table_Operations/Util.enso @@ -1,5 +1,5 @@ from Standard.Base import all -from Standard.Table import Table +from Standard.Table import Table, Value_Type from Standard.Test import all @@ -75,3 +75,9 @@ build_sorted_table setup table_structure = row_count = table_structure.first.second.length new_structure = table_structure+[["row_id", (0.up_to row_count) . to_vector]] setup.table_builder new_structure . sort "row_id" . remove_columns ["row_id"] + +## PRIVATE +is_float_or_decimal ~column = case column.value_type of + Value_Type.Float _ -> Nothing + Value_Type.Decimal _ _ -> Nothing + other -> Test.fail "Expected "+column.name+" to be a float or decimal, but got "+other.to_display_text diff --git a/test/Table_Tests/src/Database/Common/Names_Length_Limits_Spec.enso b/test/Table_Tests/src/Database/Common/Names_Length_Limits_Spec.enso index 88b3e22b8a..c24af598ec 100644 --- a/test/Table_Tests/src/Database/Common/Names_Length_Limits_Spec.enso +++ b/test/Table_Tests/src/Database/Common/Names_Length_Limits_Spec.enso @@ -112,7 +112,7 @@ add_specs suite_builder prefix create_connection_func = src = Table.new [["X", [1, 2, 3]]] t1 = src.select_into_database_table data.connection long_name temporary=True Problems.assume_no_problems t1 - data.connection.query long_name . at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + data.connection.query long_name . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] longer_name_with_same_prefix = long_name + ("z" * 10) data.connection.query longer_name_with_same_prefix . should_fail_with Table_Not_Found @@ -139,12 +139,12 @@ add_specs suite_builder prefix create_connection_func = t13 = t1.join t3 t1213 = t12.join t13 t1213.row_count . should_equal 2 - t1213.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + t1213.at "X" . to_vector . should_equal_ignoring_order [1, 2] Problems.expect_only_warning Duplicate_Output_Column_Names t1213 t11 = t1.join t1 t11.row_count . should_equal 2 - t11.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + t11.at "X" . to_vector . should_equal_ignoring_order [1, 2] Problems.expect_only_warning Duplicate_Output_Column_Names t1213 Test.with_clue "cross_join: " <| @@ -152,12 +152,12 @@ add_specs suite_builder prefix create_connection_func = t13 = t1.cross_join t3 t1213 = t12.cross_join t13 t1213.row_count . should_equal 16 - t1213.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + t1213.at "X" . to_vector . distinct . should_equal_ignoring_order [1, 2] Problems.expect_only_warning Duplicate_Output_Column_Names t1213 t11 = t1.cross_join t1 t11.row_count . should_equal 4 - t11.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + t11.at "X" . to_vector . should_equal_ignoring_order [1, 2, 1, 2] Problems.expect_only_warning Duplicate_Output_Column_Names t1213 Test.with_clue "union: " <| @@ -166,12 +166,12 @@ add_specs suite_builder prefix create_connection_func = tx = t1.union [t12, t13] # 2 + 2*4 = 10 rows ty = tx.union tx # 10 + 10 = 20 rows ty.row_count . should_equal 20 - ty.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + ty.at "X" . to_vector . distinct . should_equal_ignoring_order [1, 2] Problems.assume_no_problems ty t11 = t1.union t1 t11.row_count . should_equal 4 - t11.at "X" . to_vector . should_contain_the_same_elements_as [1, 2] + t11.at "X" . to_vector . should_equal_ignoring_order [1, 2, 1, 2] Problems.assume_no_problems t11 group_builder.specify "should be fine operating on columns with long names" <| @@ -200,7 +200,7 @@ add_specs suite_builder prefix create_connection_func = t5 = t1.cross_join t4 right_prefix="" last_column = t5.at (-1) - last_column.to_vector . should_contain_the_same_elements_as [4, 10, 18] + last_column.to_vector.distinct.should_equal_ignoring_order [4, 10, 18] if has_maximum_column_name_length.not then group_builder.specify "should allow to create very long column names" <| @@ -385,7 +385,7 @@ add_specs suite_builder prefix create_connection_func = t2 = db_table.aggregate columns=[Aggregate_Column.Maximum name_a, Aggregate_Column.Maximum name_b] w1 = Problems.expect_warning Truncated_Column_Names t2 - w1.original_names . should_contain_the_same_elements_as ["Maximum "+name_a, "Maximum "+name_b] + w1.original_names . should_equal_ignoring_order ["Maximum "+name_a, "Maximum "+name_b] t2.at (w1.find_truncated_name ("Maximum "+name_a)) . to_vector . should_equal [3] t2.at (w1.find_truncated_name ("Maximum "+name_b)) . to_vector . should_equal [6] @@ -399,8 +399,8 @@ add_specs suite_builder prefix create_connection_func = Problems.assume_no_problems db_table2 t3 = db_table2.aggregate columns=(names.map name-> Aggregate_Column.Maximum name) w2 = Problems.expect_warning Truncated_Column_Names t3 - w2.original_names . should_contain_the_same_elements_as (names.map name-> "Maximum " + name) - t3.column_names . should_contain_the_same_elements_as w2.truncated_names + w2.original_names . should_equal_ignoring_order (names.map name-> "Maximum " + name) + t3.column_names . should_equal_ignoring_order w2.truncated_names (0.up_to 15).each i-> t3.at (w2.find_truncated_name ("Maximum " + names.at i)) . to_vector . should_equal [200 + i] @@ -431,10 +431,10 @@ add_specs suite_builder prefix create_connection_func = m2 = t2.read . sort [name_a, name_b] m2.column_names . should_equal [name_a, name_b]+w.truncated_names - m2.at name_a . to_vector . should_contain_the_same_elements_as [1, 2, 3] - m2.at name_b . to_vector . should_contain_the_same_elements_as [4, 5, 6] - m2.at (w.truncated_names.at 0) . to_vector . should_contain_the_same_elements_as [1, 2, 3] - m2.at (w.truncated_names.at 1) . to_vector . should_contain_the_same_elements_as [4, 5, 6] + m2.at name_a . to_vector . distinct . should_equal_ignoring_order [1, 2, 3] + m2.at name_b . to_vector . distinct . should_equal_ignoring_order [4, 5, 6] + m2.at (w.truncated_names.at 0) . to_vector . distinct . should_equal_ignoring_order [1, 2, 3] + m2.at (w.truncated_names.at 1) . to_vector . distinct . should_equal_ignoring_order [4, 5, 6] group_builder.specify "should truncate new column names in other operations" <| name_a = "x" * (max_column_name_length - 1) + "A" diff --git a/test/Table_Tests/src/Database/Postgres_Spec.enso b/test/Table_Tests/src/Database/Postgres_Spec.enso index 701ee02d18..8fad97a454 100644 --- a/test/Table_Tests/src/Database/Postgres_Spec.enso +++ b/test/Table_Tests/src/Database/Postgres_Spec.enso @@ -690,7 +690,7 @@ add_postgres_specs suite_builder create_connection_fn db_name = Common_Spec.add_specs suite_builder prefix create_connection_fn - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True allows_mixed_type_comparisons=False fixed_length_text_columns=True removes_trailing_whitespace_casting_from_char_to_varchar=True supports_decimal_type=True supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=True supports_date_time_without_timezone=True is_nan_comparable=True + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True order_by_unicode_normalization_by_default=True allows_mixed_type_comparisons=False text_length_limited_columns=True fixed_length_text_columns=True removes_trailing_whitespace_casting_from_char_to_varchar=True char_max_size_after_substring=..Reset supports_decimal_type=True supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=True supports_date_time_without_timezone=True is_nan_comparable=True aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last_row_order=False aggregation_problems=False agg_in_memory_table = (enso_project.data / "data.csv") . read diff --git a/test/Table_Tests/src/Database/SQLite_Spec.enso b/test/Table_Tests/src/Database/SQLite_Spec.enso index 34e77d5076..a1764a42db 100644 --- a/test/Table_Tests/src/Database/SQLite_Spec.enso +++ b/test/Table_Tests/src/Database/SQLite_Spec.enso @@ -331,7 +331,7 @@ sqlite_spec suite_builder prefix create_connection_func persistent_connector = Common_Spec.add_specs suite_builder prefix create_connection_func - common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True is_nan_and_nothing_distinct=False date_time=False supported_replace_params=supported_replace_params different_size_integer_types=False length_restricted_text_columns=False run_advanced_edge_case_tests_by_default=True + common_selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=False natural_ordering=False case_insensitive_ordering=True case_insensitive_ascii_only=True is_nan_and_nothing_distinct=False date_time=False supported_replace_params=supported_replace_params different_size_integer_types=False length_restricted_text_columns=False char_max_size_after_substring=..Reset run_advanced_edge_case_tests_by_default=True ## For now `advanced_stats`, `first_last`, `text_shortest_longest` and `multi_distinct` remain disabled, because SQLite does not provide the diff --git a/test/Table_Tests/src/Database/Upload_Spec.enso b/test/Table_Tests/src/Database/Upload_Spec.enso index 010e76e2b7..86581e3fbd 100644 --- a/test/Table_Tests/src/Database/Upload_Spec.enso +++ b/test/Table_Tests/src/Database/Upload_Spec.enso @@ -43,10 +43,6 @@ type Data in_memory_table = Table.new [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] [connection, in_memory_table] - teardown self = - self.connection.close - - in_memory_table_builder args primary_key=[] connection = _ = [primary_key, connection] case args of @@ -69,12 +65,9 @@ database_table_builder name_prefix args primary_key=[] connection = database, so features relying on persistence cannot really be tested. add_specs suite_builder setup make_new_connection persistent_connector=True = prefix = setup.prefix - suite_builder.group prefix+"Creating an empty table" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown + data = Data.setup make_new_connection + suite_builder.group prefix+"(Upload_Spec) Creating an empty table" group_builder-> group_builder.specify "should allow to specify the column names and types" <| t = data.connection.create_table (Name_Generator.random_name "creating-table") structure=[Column_Description.Value "X" Value_Type.Integer, Column_Description.Value "Y" Value_Type.Char] temporary=True t.column_names . should_equal ["X", "Y"] @@ -98,7 +91,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = group_builder.specify "should allow to inherit the structure of an existing Database table" <| t = Table.new [["X", [1, 2]], ["Y", ['a', 'b']]] input_db_table = t.select_into_database_table data.connection (Name_Generator.random_name "input_table") temporary=True - input_db_table.at "X" . to_vector . should_equal [1, 2] + input_db_table.at "X" . to_vector . should_equal_ignoring_order [1, 2] db_table = data.connection.create_table (Name_Generator.random_name "creating-table") structure=input_db_table temporary=True db_table.column_names . should_equal ["X", "Y"] @@ -231,7 +224,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = name = Name_Generator.random_name "primary_key 1" db_table = data.connection.create_table table_name=name structure=[Column_Description.Value "X" Value_Type.Integer, Column_Description.Value "Y" Value_Type.Char, Column_Description.Value "Z" Value_Type.Integer, Column_Description.Value "W" Value_Type.Float] primary_key=["Y", "Z"] temporary=False Panic.with_finalizer (data.connection.drop_table db_table.name) <| - db_table.get_primary_key . should_equal ["Y", "Z"] + db_table.get_primary_key . should_equal_ignoring_order ["Y", "Z"] group_builder.specify "should ensure that primary key columns specified are valid" <| run_with_and_without_output <| @@ -297,17 +290,12 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = t1.at "X" . to_vector . should_fail_with SQL_Error - suite_builder.group prefix+"Uploading an in-memory Table" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Uploading an in-memory Table" group_builder-> group_builder.specify "should create a database table with the same contents as the source" <| db_table = data.in_memory_table.select_into_database_table data.connection (Name_Generator.random_name "creating-table") temporary=True db_table.column_names . should_equal ["X", "Y"] - db_table.at "X" . to_vector . should_equal [1, 2, 3] - db_table.at "Y" . to_vector . should_equal ['a', 'b', 'c'] + db_table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] + db_table.at "Y" . to_vector . should_equal_ignoring_order ['a', 'b', 'c'] setup.expect_integer_type <| db_table.at "X" db_table.at "Y" . value_type . is_text . should_be_true db_table.row_count . should_equal 3 @@ -316,23 +304,23 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = group_builder.specify "should include the created table in the tables directory" <| db_table = data.in_memory_table.select_into_database_table data.connection (Name_Generator.random_name "permanent_table 1") temporary=False Panic.with_finalizer (data.connection.drop_table db_table.name) <| - db_table.at "X" . to_vector . should_equal [1, 2, 3] + db_table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] data.connection.tables.at "Name" . to_vector . should_contain db_table.name - data.connection.query db_table.name . at "X" . to_vector . should_equal [1, 2, 3] + data.connection.query db_table.name . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] group_builder.specify "should include the temporary table in the tables directory" <| db_table = data.in_memory_table.select_into_database_table data.connection (Name_Generator.random_name "temporary_table 1") temporary=True - db_table.at "X" . to_vector . should_equal [1, 2, 3] + db_table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] data.connection.tables.at "Name" . to_vector . should_contain db_table.name - data.connection.query db_table.name . at "X" . to_vector . should_equal [1, 2, 3] + data.connection.query db_table.name . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] if persistent_connector then group_builder.specify "should drop the temporary table after the connection is closed" <| tmp_connection = make_new_connection Nothing db_table = data.in_memory_table.select_into_database_table tmp_connection (Name_Generator.random_name "temporary_table 2") temporary=True name = db_table.name - tmp_connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal [1, 2, 3] + tmp_connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] tmp_connection.close wait_until_temporary_table_is_deleted_after_closing_connection data.connection name @@ -344,9 +332,9 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = db_table = data.in_memory_table.select_into_database_table tmp_connection (Name_Generator.random_name "permanent_table 1") temporary=False name = db_table.name Panic.with_finalizer (data.connection.drop_table name) <| - tmp_connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal [1, 2, 3] + tmp_connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] tmp_connection.close - data.connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal [1, 2, 3] + data.connection.query (SQL_Query.Table_Name name) . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] group_builder.specify "should not create any table if upload fails" <| normal_column = Column.from_vector "Y" ((100+0).up_to (100+1000)).to_vector @@ -354,25 +342,28 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = exploding_table = Table.new [normal_column, exploding_column] name = Name_Generator.random_name "rolling-back-table" data.connection.query (SQL_Query.Table_Name name) . should_fail_with Table_Not_Found - Test.expect_panic_with matcher=ExplodingStoragePayload <| - exploding_table.select_into_database_table data.connection name temporary=False primary_key=Nothing - data.connection.query (SQL_Query.Table_Name name) . should_fail_with Table_Not_Found + + # We ensure that we try to clean-up the table even if the test failed. + Panic.with_finalizer (data.connection.drop_table name if_exists=True) <| + Test.expect_panic_with matcher=ExplodingStoragePayload <| + exploding_table.select_into_database_table data.connection name temporary=False primary_key=Nothing + data.connection.query (SQL_Query.Table_Name name) . should_fail_with Table_Not_Found group_builder.specify "should set a primary key for the table" <| t1 = Table.new [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']], ["Z", [1.0, 2.0, 3.0]]] db_table_1 = t1.select_into_database_table data.connection (Name_Generator.random_name "primary-key-1") primary_key=["Y", "X"] Panic.with_finalizer (data.connection.drop_table db_table_1.name) <| - db_table_1.at "X" . to_vector . should_equal [1, 2, 3] - db_table_1.get_primary_key . should_equal ["Y", "X"] + db_table_1.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] + db_table_1.get_primary_key . should_equal_ignoring_order ["Y", "X"] db_table_2 = t1.select_into_database_table data.connection (Name_Generator.random_name "primary-key-2") Panic.with_finalizer (data.connection.drop_table db_table_2.name) <| - db_table_2.at "X" . to_vector . should_equal [1, 2, 3] + db_table_2.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] db_table_2.get_primary_key . should_equal ["X"] db_table_3 = t1.select_into_database_table data.connection (Name_Generator.random_name "primary-key-3") primary_key=Nothing Panic.with_finalizer (data.connection.drop_table db_table_3.name) <| - db_table_3.at "X" . to_vector . should_equal [1, 2, 3] + db_table_3.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] db_table_3.get_primary_key . should_equal Nothing group_builder.specify "should ensure that primary key columns are valid" <| @@ -396,12 +387,12 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = r2.catch . clashing_example_key_values . should_equal ['b'] r3 = t1.select_into_database_table data.connection (Name_Generator.random_name "primary-key-7") temporary=True primary_key=["X", "Y"] - r3.at "X" . to_vector . should_equal [1, 2, 1] + r3.at "X" . to_vector . should_equal_ignoring_order [1, 2, 1] t2 = Table.new [["X", [1, 2, 1]], ["Y", ['a', 'b', 'a']]] r4 = t2.select_into_database_table data.connection (Name_Generator.random_name "primary-key-7") temporary=True primary_key=["X", "Y"] r4.should_fail_with Non_Unique_Key - r4.catch . clashing_example_key_values . should_equal [1, 'a'] + r4.catch . clashing_example_key_values . should_equal_ignoring_order [1, 'a'] # Will not find clashes if they are not in the first 1000 rows, in Output disabled mode. vec = (0.up_to 1010).to_vector @@ -442,7 +433,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = tmp_name . should_not_equal base_table_name data.connection.query base_table_name . should_fail_with Table_Not_Found - t1.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + t1.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] t2 = (Table.new [["Y", [44]]]).select_into_database_table data.connection base_table_name temporary=False primary_key=[] Problems.expect_only_warning Dry_Run_Operation t2 @@ -457,12 +448,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = # The old dry run table has been invalidated due to overwrite t1.at "X" . to_vector . should_fail_with SQL_Error - suite_builder.group prefix+"Persisting a Database Table (query)" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Persisting a Database Table (query)" group_builder-> group_builder.specify "should be able to create a persistent copy of a DB table" <| t = Table.new [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']], ["Z", [1.0, 2.0, 3.0]]] tmp_connection = make_new_connection Nothing @@ -476,11 +462,11 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = copied_table.at "Y" . value_type . is_text . should_be_true copied_table.at "Z" . value_type . is_floating_point . should_be_true - tmp_connection.query name . at "X" . to_vector . should_equal [1, 2, 3] + tmp_connection.query name . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] tmp_connection.close if persistent_connector then - data.connection.query name . at "X" . to_vector . should_equal [1, 2, 3] + data.connection.query name . at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] group_builder.specify "should be able to persist a complex query with generated columns, joins etc." <| t1 = Table.new [["X", [1, 1, 2]], ["Y", [1, 2, 3]]] @@ -495,11 +481,11 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = copied_table = db_table_4.select_into_database_table data.connection (Name_Generator.random_name "copied-table") temporary=True primary_key=Nothing copied_table.column_names . should_equal ["X", "Y", "C1", "C2", "Right X", "C3"] - copied_table.at "X" . to_vector . should_equal [1, 1, 2] - copied_table.at "C1" . to_vector . should_equal [101, 102, 203] - copied_table.at "C2" . to_vector . should_equal ["constant_text", "constant_text", "constant_text"] - copied_table.at "Right X" . to_vector . should_equal [Nothing, Nothing, 2] - copied_table.at "C3" . to_vector . should_equal [Nothing, Nothing, 5] + copied_table.at "X" . to_vector . should_equal_ignoring_order [1, 1, 2] + copied_table.at "C1" . to_vector . should_equal_ignoring_order [101, 102, 203] + copied_table.at "C2" . to_vector . should_equal_ignoring_order ["constant_text", "constant_text", "constant_text"] + copied_table.at "Right X" . to_vector . should_equal_ignoring_order [Nothing, Nothing, 2] + copied_table.at "C3" . to_vector . should_equal_ignoring_order [Nothing, Nothing, 5] copied_table.is_trivial_query . should_be_true # We check that this is indeed querying a simple DB table and not a complex query like `db_table_4` would be, @@ -520,7 +506,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = copied_table_accessed = tmp_connection.query name copied_table_accessed.column_names . should_equal ["X", "Y", "computed"] - copied_table_accessed.at "computed" . to_vector . should_equal [401, 502, 603] + copied_table_accessed.at "computed" . to_vector . should_equal_ignoring_order [401, 502, 603] tmp_connection.close data.connection.query name . should_fail_with Table_Not_Found @@ -605,12 +591,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = preexisting2.column_names . should_equal ["X"] preexisting2.at "X" . to_vector . should_equal [42] - suite_builder.group prefix+"Appending an in-memory table to a Database table" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Appending an in-memory table to a Database table" group_builder-> test_table_append group_builder data in_memory_table_builder (database_table_builder "target-table") group_builder.specify "will issue a friendly error if using in-memory table as target" <| @@ -620,40 +601,20 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = r1.should_fail_with Illegal_Argument r1.to_display_text.should_contain "in-memory tables are immutable" - suite_builder.group prefix+"Appending a Database table to a Database table" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Appending a Database table to a Database table" group_builder-> test_table_append group_builder data (database_table_builder "source-table") (database_table_builder "target-table") - suite_builder.group prefix+"Deleting rows from a Database table (source=in-memory)" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Deleting rows from a Database table (source=in-memory)" group_builder-> test_table_delete group_builder data in_memory_table_builder (database_table_builder "target-table") - suite_builder.group prefix+"Deleting rows from a Database table (source=Database)" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Deleting rows from a Database table (source=Database)" group_builder-> test_table_delete group_builder data (database_table_builder "source-table") (database_table_builder "target-table") - suite_builder.group prefix+"Deleting rows from a Database table" group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - + suite_builder.group prefix+"(Upload_Spec) Deleting rows from a Database table" group_builder-> group_builder.specify "[ADVANCED] it should be possible to truncate the whole table" <| name = Name_Generator.random_name "table-to-truncate" t = (Table.new [["X", [1, 2, 3]]]).select_into_database_table data.connection name temporary=True primary_key=[] - t.at "X" . to_vector . should_equal [1, 2, 3] + t.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] data.connection.truncate_table name t.at "X" . to_vector . should_equal [] # The table still exists @@ -665,7 +626,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = graduates = table.filter "graduation_year" (Filter_Condition.Less 2023) affected_rows = table.delete_rows graduates # uses the primary key by default affected_rows . should_equal 3 - table.rows.map .to_vector . should_equal [[1, "Alice", 2023], [2, "Bob", 2024]] + table.rows.map .to_vector . should_equal_ignoring_order [[1, "Alice", 2023], [2, "Bob", 2024]] group_builder.specify "will issue a friendly error if using in-memory table as target" <| t1 = Table.new [["X", [1, 2, 3]]] @@ -680,13 +641,8 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = affected_rows . should_equal 3 t1.rows.should_equal [] - execution_context_group_name = prefix+"Output Execution Context for Database operations" + execution_context_group_name = prefix+"(Upload_Spec) Output Execution Context for Database operations" suite_builder.group execution_context_group_name group_builder-> - data = Data.setup make_new_connection - - group_builder.teardown <| - data.teardown - group_builder.specify "should forbid executing updates" <| Context.Output.with_disabled <| r1 = data.connection.execute_update "CREATE TEMPORARY TABLE foo (x INTEGER)" @@ -713,7 +669,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = data.connection.tables . at "Name" . to_vector . should_not_contain dry_run_name data.connection.base_connection.get_tables_advanced include_hidden=True . at "Name" . to_vector . should_contain dry_run_name - table.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] ## This test relies on GC behaviour which may not be fully deterministic. Currently this test seems to work fine, but if there are ever issues @@ -732,7 +688,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = sentinel = Managed_Resource.register "payload" (cleanup_sentinel was_cleanup_performed) table.column_names . should_equal ["X"] data.connection.base_connection.get_tables_advanced include_hidden=True . at "Name" . to_vector . should_contain table.name - table.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] name = table.name payload = sentinel.with x-> "transformed_"+x payload . should_equal "transformed_payload" @@ -768,7 +724,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = # Create a table that has the same name as the dry run table normally would have. pre_existing_table = src.select_into_database_table data.connection dry_run_name temporary=False . should_succeed pre_existing_table.column_names . should_equal ["X"] - pre_existing_table.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + pre_existing_table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] Panic.with_finalizer (data.connection.drop_table pre_existing_table.name if_exists=True) <| new_dry_run_name = Context.Output.with_disabled <| tmp_connection2 = make_new_connection Nothing @@ -784,7 +740,7 @@ add_specs suite_builder setup make_new_connection persistent_connector=True = new_dry_run_name . should_not_equal dry_run_name # The pre-existing table should not have been overwritten. - pre_existing_table.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + pre_existing_table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] tests group_builder data make_new_connection in_memory_table_builder " (from memory)" persistent_connector @@ -804,7 +760,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ expected_rows = [[1, 'a'], [2, 'b'], [3, 'c'], [4, 'd'], [5, 'e'], [6, 'f']] rows1 = result.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should error if new rows clash with existing ones and mode is Insert, target table should remain unchanged" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -817,14 +773,14 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ group_builder.specify "should use the target table primary key for the key by default" <| dest1 = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']], ["Z", [4, 5, 6]]] primary_key=["Y", "Z"] connection=data.connection - default_key_columns dest1 . should_equal ["Y", "Z"] + default_key_columns dest1 . should_equal_ignoring_order ["Y", "Z"] dest2 = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["Y"] connection=data.connection src = source_table_builder [["X", [4, 5]], ["Y", ['b', 'e']]] connection=data.connection # Not specifying `key_columns`, rely on `default_key_columns` inferring Y as default based on the primary key. r1 = dest2.update_rows src rows = r1.rows.to_vector.map .to_vector - rows.should_contain_the_same_elements_as [[1, 'a'], [4, 'b'], [3, 'c'], [5, 'e']] + rows.should_equal_ignoring_order [[1, 'a'], [4, 'b'], [3, 'c'], [5, 'e']] group_builder.specify "should be able to Update existing rows in a table" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] connection=data.connection @@ -835,7 +791,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r1.should_succeed rows = dest.rows.to_vector.map .to_vector - rows.should_contain_the_same_elements_as [[1, 'a'], [2, 'ZZZ'], [3, 'c']] + rows.should_equal_ignoring_order [[1, 'a'], [2, 'ZZZ'], [3, 'c']] group_builder.specify "should fail on unmatched rows in Update mode" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] connection=data.connection @@ -848,7 +804,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ # The table should remain unchanged. rows = dest.rows.to_vector.map .to_vector - rows.should_contain_the_same_elements_as [[1, 'a'], [2, 'b'], [3, 'c']] + rows.should_equal_ignoring_order [[1, 'a'], [2, 'b'], [3, 'c']] group_builder.specify "should upsert by default (update existing rows, insert new rows)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -858,11 +814,11 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r1.column_names . should_equal ["X", "Y"] expected_rows = [[1, 'a'], [2, 'D'], [3, 'c'], [100, 'E']] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows # The original table is updated too. rows2 = dest.rows.to_vector.map .to_vector - rows2.should_contain_the_same_elements_as expected_rows + rows2.should_equal_ignoring_order expected_rows group_builder.specify "should allow to align an existing table with a source (upsert + delete rows missing from source)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -872,9 +828,9 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r1.column_names . should_equal ["X", "Y"] expected_rows = [[2, 'D'], [100, 'E']] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows rows2 = dest.rows.to_vector.map .to_vector - rows2.should_contain_the_same_elements_as expected_rows + rows2.should_equal_ignoring_order expected_rows group_builder.specify "should match columns by name, reordering to destination order if needed (Insert)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -884,7 +840,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ src.column_names . should_equal ["Y", "X"] expected_rows = [[1, 'a'], [2, 'b'], [3, 'c'], [4, 'd'], [5, 'e'], [6, 'f']] rows1 = result.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should match columns by name, reordering to destination order if needed (Upsert)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -894,7 +850,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ src.column_names . should_equal ["Y", "X"] expected_rows = [[1, 'd'], [2, 'b'], [3, 'c'], [5, 'e'], [6, 'f']] rows1 = result.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should match columns by name, reordering to destination order if needed (Update)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -904,7 +860,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ src.column_names . should_equal ["Y", "X"] expected_rows = [[1, 'f'], [2, 'e'], [3, 'd']] rows1 = result.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should match columns by name, reordering to destination order if needed (Align)" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection @@ -914,39 +870,41 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ src.column_names . should_equal ["Y", "X"] expected_rows = [[1, 'e'], [2, 'd'], [6, 'f']] rows1 = result.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should allow to use a transformed table, with computed fields, as a source" <| dest = target_table_builder [["X", [1, 2, 3]], ["Y", ['a', 'b', 'c']]] primary_key=["X"] connection=data.connection t1 = source_table_builder [["Z", [10, 20]], ["Y", ['D', 'E']]] connection=data.connection t2 = source_table_builder [["Z", [20, 10]], ["X", [-99, 10]]] connection=data.connection src = t1.join t2 on=["Z"] join_kind=Join_Kind.Inner . remove_columns "Z" . set (expr "[X] + 100") "X" - src.at "X" . to_vector . should_contain_the_same_elements_as [1, 110] + src.at "X" . to_vector . should_equal_ignoring_order [1, 110] r1 = dest.update_rows src key_columns=["X"] Problems.assume_no_problems r1 r1.column_names . should_equal ["X", "Y"] expected_rows = [[1, 'E'], [110, 'D'], [2, 'b'], [3, 'c']] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows group_builder.specify "should allow specifying no key in Insert mode" <| dest = target_table_builder [["X", [1, 10, 100]]] connection=data.connection src = source_table_builder [["X", [1, 2, 3]]] connection=data.connection result = dest.update_rows src update_action=Update_Action.Insert key_columns=[] - expected = [1, 10, 100, 1, 2, 3] result.column_names . should_equal ["X"] - result.at "X" . to_vector . should_contain_the_same_elements_as expected + result.at "X" . to_vector . should_equal_ignoring_order [1, 10, 100, 1, 2, 3] - r2 = dest.update_rows src update_action=Update_Action.Insert key_columns=Nothing + bump table offset = table.set (table.at "X" + offset) "X" + src2 = bump src 20 # 21, 22, 23 + r2 = dest.update_rows src2 update_action=Update_Action.Insert key_columns=Nothing r2.column_names . should_equal ["X"] - r2.at "X" . to_vector . should_contain_the_same_elements_as expected + r2.at "X" . to_vector . should_equal_ignoring_order [1, 10, 100, 1, 2, 3, 21, 22, 23] + src3 = bump src 30 # 31, 32, 33 default_key_columns dest . should_equal Nothing - r3 = dest.update_rows src update_action=Update_Action.Insert + r3 = dest.update_rows src3 update_action=Update_Action.Insert r3.column_names . should_equal ["X"] - r3.at "X" . to_vector . should_contain_the_same_elements_as expected + r3.at "X" . to_vector . should_equal_ignoring_order [1, 10, 100, 1, 2, 3, 21, 22, 23, 31, 32, 33] group_builder.specify "should fail if no key is specified in other modes" <| dest = target_table_builder [["X", [1, 10, 100]]] connection=data.connection @@ -995,8 +953,8 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r2 = t1.update_rows s1 update_action=Update_Action.Insert key_columns=[] Problems.assume_no_problems r2 m2 = r2.read . sort "Y" - m2.at "Y" . to_vector . should_equal ["a", "b", "c", "x", "y"] - m2.at "X" . to_vector . should_equal [0, 10, 100, 10, Nothing] + m2.at "Y" . to_vector . should_equal_ignoring_order ["a", "b", "c", "x", "y"] + m2.at "X" . to_vector . should_equal_ignoring_order [0, 10, 100, 10, Nothing] group_builder.specify "should fail if the key causes update of multiple values (it's not unique in the target table)" <| dest = target_table_builder [["X", [1, 1, 2]], ["Y", ['a', 'b', 'c']]] connection=data.connection @@ -1044,7 +1002,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r1.column_names . should_equal ["Y", "X"] expected_rows = [[42, 1], [42, 2], [42, 3]] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows data.connection.drop_table dest_name group_builder.specify "should use defaults when inserting new values in upsert, but retain existing values" <| @@ -1059,7 +1017,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ r1.column_names . should_equal ["Y", "X", "Z"] expected_rows = [[1000, 1, 100], [42, 2, 200], [42, 3, 300]] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows data.connection.drop_table dest_name group_builder.specify "should use defaults for missing input columns for newly inserted rows when Aligning the tables, but keep existing values for existing rows" <| @@ -1074,7 +1032,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ # The X=10 stays with Y=100, but the X=2 is inserted with the default Y=42 expected_rows = [[10, 100, -1], [2, 42, -2], [3, 42, -3]] rows1 = r1.rows.to_vector.map .to_vector - rows1.should_contain_the_same_elements_as expected_rows + rows1.should_equal_ignoring_order expected_rows data.connection.drop_table dest_name group_builder.specify "should fail if the source table is missing some columns and the column in the target has no default value" <| @@ -1147,7 +1105,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ Problems.expect_warning Inexact_Type_Coercion r2 # But in dry run the update is not actually performed: - r2.at "X" . to_vector . should_contain_the_same_elements_as [3.25, 4.25, 10.0] + r2.at "X" . to_vector . should_equal_ignoring_order [3.25, 4.25, 10.0] result = dest.update_rows src update_action=Update_Action.Insert key_columns=[] warning = Problems.expect_warning Inexact_Type_Coercion result @@ -1155,7 +1113,7 @@ test_table_append group_builder (data : Data) source_table_builder target_table_ warning.actual_type.is_floating_point . should_be_true result.column_names . should_equal ["X"] - result.at "X" . to_vector . should_contain_the_same_elements_as [3.25, 4.25, 10.0, 1, 2, 0] + result.at "X" . to_vector . should_equal_ignoring_order [3.25, 4.25, 10.0, 1, 2, 0] group_builder.specify "should fail if types of columns are not compatible" <| dest = target_table_builder [["X", ["a", "B", "c"]]] connection=data.connection @@ -1219,7 +1177,7 @@ test_table_delete group_builder (data : Data) source_table_builder target_table_ # key columns should automatically be discovered by the primary key affected_rows = table.delete_rows key_values_to_delete affected_rows . should_equal 2 - table.rows.map .to_vector . should_equal [[1, "Alice", 100], [2, "Bob", 100], [120, "Eve", 120]] + table.rows.map .to_vector . should_equal_ignoring_order [[1, "Alice", 100], [2, "Bob", 100], [120, "Eve", 120]] group_builder.specify "will require key_columns if no default can be used as no primary key is set" <| table = target_table_builder [["X", [1, 2, 3]]] primary_key=[] connection=data.connection @@ -1241,20 +1199,20 @@ test_table_delete group_builder (data : Data) source_table_builder target_table_ r1 = table.delete_rows key_values_to_delete key_columns=["X"] # Error: Unresolved method `delete_rows` r1.should_equal 0 Problems.assume_no_problems r1 - table.at "X" . to_vector . should_equal [1, 2, 3] + table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] key_values_2 = source_table_builder [["X", [4, 3, 5]]] connection=data.connection r2 = table.delete_rows key_values_2 key_columns=["X"] r2.should_equal 1 Problems.assume_no_problems r2 - table.at "X" . to_vector . should_equal [1, 2] + table.at "X" . to_vector . should_equal_ignoring_order [1, 2] group_builder.specify "should allow to use multiple columns as key" <| table = target_table_builder [["X", [1, 2, 2, 3, 4, 4]], ["Y", ['a', 'b', 'c', 'd', 'e', 'f']]] primary_key=[] connection=data.connection keys = source_table_builder [["X", [2, 4]], ["Y", ['b', 'f']]] connection=data.connection affected_rows = table.delete_rows keys key_columns=["X", "Y"] affected_rows . should_equal 2 - table.rows.map .to_vector . should_equal [[1, "a"], [2, "c"], [3, "d"], [4, "e"]] + table.rows.map .to_vector . should_equal_ignoring_order [[1, "a"], [2, "c"], [3, "d"], [4, "e"]] group_builder.specify "should fail if key_columns are missing in source or target tables" <| table = target_table_builder [["X", [1, 2, 3]], ["Y", [4, 5, 6]]] primary_key=[] connection=data.connection @@ -1291,11 +1249,11 @@ test_table_delete group_builder (data : Data) source_table_builder target_table_ r1.catch.example_key . should_equal [2] r1.catch.example_count . should_equal 3 # no changes - table.at "X" . to_vector . should_equal [1, 2, 2, 3, 2] + table.at "X" . to_vector . should_equal_ignoring_order [1, 2, 2, 3, 2] r2 = table.delete_rows keys key_columns=["X"] allow_duplicate_matches=True r2.should_equal 3 - table.rows.map .to_vector . should_equal [[1, "a"], [3, "d"]] + table.rows.map .to_vector . should_equal_ignoring_order [[1, "a"], [3, "d"]] group_builder.specify "should fail if the target table does not exist" <| table = target_table_builder [["X", [1, 2, 3]]] connection=data.connection @@ -1320,21 +1278,21 @@ test_table_delete group_builder (data : Data) source_table_builder target_table_ w1.to_display_text . should_contain "Only the first 1000 distinct rows out of 1999 were used for the dry run" # Target remains unchanged - target.at "X" . to_vector . should_equal [0, 1, 500, 1500, 3500] + target.at "X" . to_vector . should_equal_ignoring_order [0, 1, 500, 1500, 3500] r2 = target.delete_rows source Problems.assume_no_problems r2 # All 3 rows were deleted r2.should_equal 3 - target.at "X" . to_vector . should_equal [0, 3500] + target.at "X" . to_vector . should_equal_ignoring_order [0, 3500] group_builder.specify "will work fine if the target table contains NULL keys" <| t1 = target_table_builder [["X", ["a", "b", Nothing, "c"]], ["Y", [1, 2, 3, Nothing]]] connection=data.connection s1 = source_table_builder [["X", ["b", "c"]]] connection=data.connection t1.delete_rows s1 key_columns=["X"] . should_equal 2 m1 = t1.read . sort "X" - m1.at "X" . to_vector . should_equal [Nothing, "a"] - m1.at "Y" . to_vector . should_equal [3, 1] + m1.at "X" . to_vector . should_equal_ignoring_order [Nothing, "a"] + m1.at "Y" . to_vector . should_equal_ignoring_order [3, 1] group_builder.specify "will raise an error if they source table contains NULL keys" <| t2 = target_table_builder [["X", ["a", "b", "c"]], ["Y", [1, 2, Nothing]]] connection=data.connection @@ -1355,7 +1313,7 @@ tests group_builder (data : Data) make_new_connection source_table_builder (suff r1.column_names . should_equal ["X"] r1.name . should_not_equal name # A small table is uploaded whole. - r1.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + r1.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] r1.row_count . should_equal 3 r1.is_trivial_query . should_be_true @@ -1381,7 +1339,7 @@ tests group_builder (data : Data) make_new_connection source_table_builder (suff # The target table is returned, as usually. r1.name . should_equal dest.name # But the data is not appended due to the dry-run - the table is unmodified. - r1.at "X" . to_vector . should_contain_the_same_elements_as [1, 2, 3] + r1.at "X" . to_vector . should_equal_ignoring_order [1, 2, 3] r1.is_trivial_query . should_be_true group_builder.specify "should return the count of rows that would be deleted for delete_rows, but keep the table unchanged"+suffix <| @@ -1424,7 +1382,7 @@ tests group_builder (data : Data) make_new_connection source_table_builder (suff # Create a table that has the same name as the dry run table normally would have. pre_existing_table = pre_existing_src.select_into_database_table data.connection dry_run_name temporary=False . should_succeed pre_existing_table.column_names . should_equal ["X"] - pre_existing_table.at "X" . to_vector . should_contain_the_same_elements_as [4, 5, 6] + pre_existing_table.at "X" . to_vector . should_equal_ignoring_order [4, 5, 6] Panic.with_finalizer (data.connection.drop_table pre_existing_table.name if_exists=True) <| new_dry_run_name = Context.Output.with_disabled <| tmp_connection2 = make_new_connection Nothing @@ -1433,7 +1391,7 @@ tests group_builder (data : Data) make_new_connection source_table_builder (suff dry_run_table = src3.select_into_database_table tmp_connection2 target_name temporary=True . should_succeed Problems.expect_warning Dry_Run_Operation dry_run_table dry_run_table.column_names . should_equal ["B"] - dry_run_table.at "B" . to_vector . should_contain_the_same_elements_as [7, 8, 9] + dry_run_table.at "B" . to_vector . should_equal_ignoring_order [7, 8, 9] name = Warning.clear dry_run_table.name tmp_connection2.close name @@ -1442,7 +1400,7 @@ tests group_builder (data : Data) make_new_connection source_table_builder (suff new_dry_run_name . should_not_equal dry_run_name # The pre-existing table should not have been overwritten. - pre_existing_table.at "X" . to_vector . should_contain_the_same_elements_as [4, 5, 6] + pre_existing_table.at "X" . to_vector . should_equal_ignoring_order [4, 5, 6] ## PRIVATE diff --git a/test/Table_Tests/src/IO/Delimited_Write_Spec.enso b/test/Table_Tests/src/IO/Delimited_Write_Spec.enso index 855ef66551..597f60a0b4 100644 --- a/test/Table_Tests/src/IO/Delimited_Write_Spec.enso +++ b/test/Table_Tests/src/IO/Delimited_Write_Spec.enso @@ -159,7 +159,7 @@ add_specs suite_builder = escaped. ew1 = Unquoted_Characters_In_Output.Warning 'The Column "Name"' [3] ew2 = Unquoted_Characters_In_Output.Warning "Hello, Column?" [-1, 0, 1, 2, 3] - warnings.should_contain_the_same_elements_as [ew1, ew2] + warnings.should_equal_ignoring_order [ew1, ew2] w1 = warnings.find w-> w.column == 'Hello, Column?' w1.to_display_text . should_equal "The Hello, Column? at rows [the header, 0, 1, 2, 3] contains characters that need quoting, but quoting is disabled. The generated file may be corrupted." expected_text = normalize_lines <| """ diff --git a/test/Table_Tests/src/IO/Excel_Spec.enso b/test/Table_Tests/src/IO/Excel_Spec.enso index 08d54310c1..9c96ef1b43 100644 --- a/test/Table_Tests/src/IO/Excel_Spec.enso +++ b/test/Table_Tests/src/IO/Excel_Spec.enso @@ -81,7 +81,7 @@ spec_fmt suite_builder header file read_method sheet_count=5 = table.row_count . should_equal 25 table.column_names . should_equal ["Sheet Name", "Name", "Quantity", "Price", "A", "B", "C", "D", "E", "Student Name", "Enrolment Date", "Item", "Price 1"] problems = [Empty_Sheet.Error, Duplicate_Output_Column_Names.Error ["Price"]] - Problems.test_problem_handling action problems tester + Problems.test_problem_handling action problems tester ignore_warning_cardinality=True group_builder.specify "should let you read all sheets into a table of tables" <| wb = read_method file @@ -90,7 +90,7 @@ spec_fmt suite_builder header file read_method sheet_count=5 = table.row_count . should_equal 5 table.column_names . should_equal ["Sheet Name", "Table"] problems = [Empty_Sheet.Error, Duplicate_Output_Column_Names.Error ["Price"]] - Problems.test_problem_handling action problems tester + Problems.test_problem_handling action problems tester ignore_warning_cardinality=True group_builder.specify "should let you read some sheets from xlsx" <| wb = read_method file @@ -847,7 +847,7 @@ add_specs suite_builder = workbook.table_types . should_equal ['Worksheet', 'Named Range'] workbook.tables.row_count . should_equal (sheet_names.length + range_names.length) - workbook.tables.at "Name" . to_vector . should_contain_the_same_elements_as (sheet_names + range_names) + workbook.tables.at "Name" . to_vector . should_equal_ignoring_order (sheet_names + range_names) workbook.tables types=["Worksheet"] . row_count . should_equal sheet_names.length workbook.tables types=["Named Range"] . row_count . should_equal range_names.length diff --git a/test/Table_Tests/src/In_Memory/Common_Spec.enso b/test/Table_Tests/src/In_Memory/Common_Spec.enso index de19979750..ac8ab4fdf9 100644 --- a/test/Table_Tests/src/In_Memory/Common_Spec.enso +++ b/test/Table_Tests/src/In_Memory/Common_Spec.enso @@ -16,7 +16,7 @@ type Dummy_Connection Nothing in_memory_setup = - selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True natural_ordering=True case_insensitive_ordering=True order_by_unicode_normalization_by_default=True supports_unicode_normalization=True supports_time_duration=True supports_nanoseconds_in_time=True supports_mixed_columns=True fixed_length_text_columns=True supports_8bit_integer=True + selection = Common_Table_Operations.Main.Test_Selection.Config supports_case_sensitive_columns=True natural_ordering=True case_insensitive_ordering=True order_by_unicode_normalization_by_default=True supports_unicode_normalization=True supports_time_duration=True supports_nanoseconds_in_time=True supports_mixed_columns=True text_length_limited_columns=True fixed_length_text_columns=True supports_8bit_integer=True aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config agg_table_fn _ = (enso_project.data / "data.csv") . read