Enable SQLServer Sort Feature and associated tests (#11379)

* Auto-commit work in progress before clean build on 2024-10-15 12:59:18

* checkpoint

* Clean

* Cleaner

* cleaner

* Cleaner

* Green

* stash

* stash

* Fix sort

* Sub 300 failures

* More passing tests

* Auto-commit work in progress before clean build on 2024-10-22 09:00:41

* SQLServer green 265

* remove dead code

* Add doc

* Add generate_column

* Refactor

* refactor

* Refactor

* Refactor

* Add  is_operation_supported_fn

* Fix

* Fix

* Fix

* Code review changes

* Code review feedback

* typos

* Add comment
This commit is contained in:
AdRiley 2024-10-25 11:16:52 +01:00 committed by GitHub
parent 66c09a96af
commit 1b6a1f990b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 134 additions and 22 deletions

View File

@ -224,6 +224,10 @@ type Redshift_Dialect
_ = value_type
False
## PRIVATE
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
base_gen.default_generate_column self expr name
## PRIVATE
ensure_query_has_no_holes : JDBC_Connection -> Text -> Nothing ! Illegal_Argument
ensure_query_has_no_holes self jdbc:JDBC_Connection raw_sql:Text =

View File

@ -269,6 +269,14 @@ type Dialect
_ = [base_table, key_columns, resolved_aggregates, problem_builder]
Unimplemented.throw "This is an interface only."
## PRIVATE
Generates a column for the given expression for use in the SELECT clause.
Used for databases where the expression syntax is different in the SELECT clause
to the syntax in the WHERE clause
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
_ = [base_gen, expr, name]
Unimplemented.throw "This is an interface only."
## PRIVATE
ensure_query_has_no_holes : JDBC_Connection -> Text -> Nothing ! Illegal_Argument
ensure_query_has_no_holes jdbc:JDBC_Connection raw_sql:Text =

View File

@ -166,6 +166,10 @@ type SQL_Generator
base_expression = self.generate_expression dialect order_descriptor.expression
base_expression ++ collation ++ order_suffix ++ nulls_suffix
## PRIVATE
default_generate_column self dialect expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
self.generate_expression dialect expr ++ alias dialect name
## PRIVATE
Generates SQL code corresponding to a SELECT statement.
@ -175,11 +179,10 @@ type SQL_Generator
generate_select_query_sql : Dialect -> Vector (Pair Text SQL_Expression) -> Context -> SQL_Builder
generate_select_query_sql self dialect columns ctx =
gen_exprs exprs = exprs.map (self.generate_expression dialect)
gen_column pair = (self.generate_expression dialect pair.second) ++ alias dialect pair.first
generated_columns = case columns of
Nothing -> SQL_Builder.code "*"
_ -> SQL_Builder.join ", " (columns.map gen_column)
_ -> SQL_Builder.join ", " (columns.map (c-> dialect.generate_column_for_select self expr=c.second name=c.first))
from_part = self.generate_from_part dialect ctx.from_spec
where_part = (SQL_Builder.join " AND " (gen_exprs ctx.where_filters)) . prefix_if_present " WHERE "

View File

@ -107,14 +107,12 @@ type Context
takes precedence, but if any orderings were already present they are also
taken into account to break ties within the new ordering.
In practice this means, that the old orderings are preserved, but the new
ones are added to the beginning of the list so that they take precedence.
Arguments:
- new_orders: The new ordering clauses to add to the query.
add_orders : Vector Order_Descriptor -> Context
add_orders self new_orders =
Context.Value self.from_spec self.where_filters new_orders+self.orders self.groups self.limit self.extensions
deduped_orders = new_orders+self.orders . distinct .expression
Context.Value self.from_spec self.where_filters deduped_orders self.groups self.limit self.extensions
## PRIVATE

View File

@ -325,6 +325,10 @@ type Postgres_Dialect
_ = value_type
False
## PRIVATE
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
base_gen.default_generate_column self expr name
## PRIVATE
ensure_query_has_no_holes : JDBC_Connection -> Text -> Nothing ! Illegal_Argument
ensure_query_has_no_holes self jdbc:JDBC_Connection raw_sql:Text =

View File

@ -328,6 +328,10 @@ type SQLite_Dialect
_ = value_type
False
## PRIVATE
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
base_gen.default_generate_column self expr name
## PRIVATE
ensure_query_has_no_holes : JDBC_Connection -> Text -> Nothing ! Illegal_Argument
ensure_query_has_no_holes self jdbc:JDBC_Connection raw_sql:Text =

View File

@ -102,6 +102,12 @@ type SQLServer_Dialect
wrap_identifier self identifier =
Base_Generator.wrap_in_quotes identifier
## PRIVATE
Generates a SQL expression for a table literal.
make_table_literal : Vector (Vector Text) -> Vector Text -> Text -> SQL_Builder
make_table_literal self vecs column_names as_name =
Base_Generator.default_make_table_literal self.wrap_identifier vecs column_names as_name
## PRIVATE
Prepares an ordering descriptor.
@ -227,6 +233,7 @@ type SQLServer_Dialect
is_feature_supported self feature:Feature -> Boolean =
case feature of
Feature.Select_Columns -> True
Feature.Sort -> True
_ -> False
## PRIVATE
@ -302,13 +309,81 @@ type SQLServer_Dialect
if raw_sql.contains "#" . not then
jdbc.ensure_query_has_no_holes raw_sql
## PRIVATE
Returns a pair of a SQL_Builder for the given expression and a vector of columns
that have been used in the expression and need to be checked for nulls.
SQL Server needs special handling commpared to ther databases as it does not have a
boolean data type.
This means that you can write
SELECT * FROM MyTable WHERE [Column1] > [Column2]
but you cannot write
SELECT [Column1] > [Column2] FROM MyTable
to write the second query you need to write
SELECT CASE WHEN [Column1] IS NULL OR [Column2] IS NULL WHEN [Column1] > [Column2] THEN 1 ELSE 0 END FROM MyTable
The below function collects all of the fields which are needed to be checked for nulls returning them in a vector
as the second element of the pair.
The first element of the pair is the SQL_Builder for the expression.
generate_expression self base_gen expr = case expr of
SQL_Expression.Column _ _ ->
wrapped_name = base_gen.generate_expression self expr
pair wrapped_name [wrapped_name]
SQL_Expression.Constant value ->
wrapped = case value of
Nothing -> SQL_Builder.code "NULL"
_ -> SQL_Builder.interpolation value
pair wrapped [wrapped]
SQL_Expression.Literal value ->
wrapped = SQL_Builder.code value
pair wrapped [wrapped]
SQL_Expression.Text_Literal _ ->
wrapped_literal = base_gen.generate_expression self expr
pair wrapped_literal []
SQL_Expression.Operation kind arguments metadata ->
op = self.dialect_operations.operations_dict.get kind (Error.throw <| Unsupported_Database_Operation.Error kind)
parsed_args_and_null_checks = arguments.map (c -> self.generate_expression base_gen c)
parsed_args = parsed_args_and_null_checks.map .first
null_checks = parsed_args_and_null_checks.map .second . flatten
expr_result = case kind of
## In the case that we actually want to check for null then we need to generate the null
check sql for all the columns that have been used up to this point and that
becomes the expression.
"IS_NULL" -> _generate_null_check_sql_builder null_checks
_ ->
result = op parsed_args
# If the function expects more arguments, we pass the metadata as the last argument.
case result of
_ : Function -> result metadata
_ -> result
null_checks_result = if kind == "IS_NULL" then [] else null_checks
pair expr_result null_checks_result
query : Query -> pair (base_gen.generate_sub_query self query) []
## PRIVATE
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
expr_null_checks_pair = self.generate_expression base_gen expr
base_expr = expr_null_checks_pair.first
built_expr = case expr of
SQL_Expression.Operation _ _ _ ->
null_check = if expr_null_checks_pair.second.length == 0 then "" else
SQL_Builder.code "WHEN " ++ _generate_null_check_sql_builder expr_null_checks_pair.second ++ " THEN NULL "
SQL_Builder.code "CASE " ++ null_check ++ "WHEN " ++ base_expr ++ " THEN 1 ELSE 0 END"
_ -> base_expr
built_expr ++ Base_Generator.alias self name
## PRIVATE
private _generate_null_check_sql_builder null_checks:Vector -> SQL_Builder =
(null_checks.map it->(it.paren ++ " IS NULL ")) . reduce acc-> i-> acc ++ "OR " ++ i
## PRIVATE
make_dialect_operations =
cases = [["LOWER", Base_Generator.make_function "LOWER"], ["UPPER", Base_Generator.make_function "UPPER"]]
text = [starts_with, contains, ends_with, agg_shortest, agg_longest, make_case_sensitive, ["REPLACE", replace], left, right]+concat_ops+cases+trim_ops
counts = [agg_count_is_null, agg_count_empty, agg_count_not_empty, ["COUNT_DISTINCT", agg_count_distinct], ["COUNT_DISTINCT_INCLUDE_NULL", agg_count_distinct_include_null]]
arith_extensions = [is_nan, is_inf, is_finite, floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]]
bool = [bool_or]
bool = [bool_or, bool_not]
eq = lift_binary_op "==" make_equals
compare = [eq]
@ -319,7 +394,8 @@ make_dialect_operations =
special_overrides = [is_null]
other = [["RUNTIME_ERROR", make_runtime_error_op]]
my_mappings = text + counts + stats + first_last_aggregators + arith_extensions + bool + compare + date_ops + special_overrides + other
Base_Generator.base_dialect_operations . extend_with my_mappings
base = Base_Generator.base_dialect_operations . extend_with my_mappings
Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN")
## PRIVATE
is_null = Base_Generator.lift_unary_op "IS_NULL" arg->
@ -500,6 +576,10 @@ is_finite = Base_Generator.lift_unary_op "IS_FINITE" arg->
bool_or = Base_Generator.lift_unary_op "BOOL_OR" arg->
SQL_Builder.code "bool_or(" ++ arg ++ ")"
## PRIVATE
bool_not = Base_Generator.lift_unary_op "NOT" arg->
SQL_Builder.code "(" ++ arg ++ ")=0"
## PRIVATE
floating_point_div = Base_Generator.lift_binary_op "/" x-> y->
SQL_Builder.code "CAST(" ++ x ++ " AS double precision) / CAST(" ++ y ++ " AS double precision)"

View File

@ -108,7 +108,7 @@ type SQLServer_Type_Mapping
## PRIVATE
The SQLServer_Type_Mapping always relies on the return type determined by
the database backend.
the database backend except for boolean types.
infer_return_type : (SQL_Expression -> SQL_Type_Reference) -> Text -> Vector -> SQL_Expression -> SQL_Type_Reference
infer_return_type infer_from_database_callback op_name arguments expression =
case operations_dict.contains_key op_name of
@ -159,7 +159,7 @@ on_unknown_type sql_type =
## PRIVATE
Maps operation names to functions that infer its result type.
operations_dict : Dictionary Text (Vector -> SQL_Type)
operations_dict = Dictionary.from_vector [["IS_NULL", const (SQL_Type.Value Types.BIT "BIT")],["==", const (SQL_Type.Value Types.BIT "BIT")]]
operations_dict = Dictionary.from_vector [["IS_NULL", const (SQL_Type.Value Types.BIT "BIT")],["==", const (SQL_Type.Value Types.BIT "BIT")],["NOT", const (SQL_Type.Value Types.BIT "BIT")],["BETWEEN", const (SQL_Type.Value Types.BIT "BIT")]]
## PRIVATE
This is the maximum size that JDBC driver reports for 'unbounded' types in

View File

@ -308,6 +308,10 @@ type Snowflake_Dialect
Value_Type.Date_Time _ -> True
_ -> False
## PRIVATE
generate_column_for_select self base_gen expr:(SQL_Expression | Order_Descriptor | Query) name:Text -> SQL_Builder =
base_gen.default_generate_column self expr name
## PRIVATE
ensure_query_has_no_holes : JDBC_Connection -> Text -> Nothing ! Illegal_Argument
ensure_query_has_no_holes self jdbc:JDBC_Connection raw_sql:Text =

View File

@ -85,9 +85,10 @@ add_database_specs suite_builder create_connection_fn =
(agg_in_memory_table.take (..First 0)).select_into_database_table default_connection.get (Name_Generator.random_name "Agg_Empty") primary_key=Nothing temporary=True
is_feature_supported_fn feature:Feature = default_connection.get.dialect.is_feature_supported feature
is_operation_supported_fn operation:Text = default_connection.get.dialect.is_operation_supported operation
flagged_fn = default_connection.get.dialect.flagged
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
Common_Spec.add_specs suite_builder prefix create_connection_fn default_connection setup
add_redshift_specific_specs suite_builder create_connection_fn setup

View File

@ -207,9 +207,10 @@ add_sqlserver_specs suite_builder create_connection_fn =
(agg_in_memory_table.take (..First 0)).select_into_database_table default_connection.get (Name_Generator.random_name "Agg_Empty") primary_key=Nothing temporary=True
is_feature_supported_fn feature:Feature = default_connection.get.dialect.is_feature_supported feature
is_operation_supported_fn operation:Text = default_connection.get.dialect.is_operation_supported operation
flagged_fn = default_connection.get.dialect.flagged
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
Common_Spec.add_specs suite_builder prefix create_connection_fn default_connection setup
Common_Table_Operations.Main.add_specs suite_builder setup

View File

@ -547,9 +547,10 @@ add_snowflake_specs suite_builder create_connection_fn db_name =
(agg_in_memory_table.take (..First 0)).select_into_database_table default_connection.get (Name_Generator.random_name "Agg_Empty") primary_key=Nothing temporary=True
is_feature_supported_fn feature:Feature = default_connection.get.dialect.is_feature_supported feature
is_operation_supported_fn operation:Text = default_connection.get.dialect.is_operation_supported operation
flagged_fn = default_connection.get.dialect.flagged
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_integer_type=is_snowflake_integer is_feature_supported=is_feature_supported_fn flagged=flagged_fn
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_integer_type=is_snowflake_integer is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
Common_Spec.add_specs suite_builder prefix create_connection_fn default_connection setup
snowflake_specific_spec suite_builder default_connection db_name setup

View File

@ -30,7 +30,7 @@ type Data
self.connection.close
add_specs suite_builder setup =
if setup.is_feature_supported Feature.Filter then (add_row_number_specs suite_builder setup) else
if setup.is_feature_supported Feature.Add_Row_Number then (add_row_number_specs suite_builder setup) else
suite_builder.group setup.prefix+"Table.add_row_number" group_builder->
group_builder.specify "add_row_number should report unsupported" <|
table_builder = setup.light_table_builder

View File

@ -73,7 +73,7 @@ type Test_Setup
boolean indicating if the feature is supported by the backend.
- flagged: A function that takes a `Dialect_Flag` and returns a boolean indicating if the
flag is set for the backend.
Config prefix table_fn empty_table_fn (table_builder : (Vector Any -> (Any|Nothing)) -> Any) materialize is_database test_selection aggregate_test_selection create_connection_func light_table_builder is_integer_type=(.is_integer) is_feature_supported flagged
Config prefix table_fn empty_table_fn (table_builder : (Vector Any -> (Any|Nothing)) -> Any) materialize is_database test_selection aggregate_test_selection create_connection_func light_table_builder is_integer_type=(.is_integer) is_feature_supported flagged is_operation_supported
## Specifies if the given Table backend supports custom Enso types.

View File

@ -131,7 +131,7 @@ add_nothing_specs suite_builder setup =
table = setup.light_table_builder [["x", [True, False, Nothing]]]
table.at "x" . not . to_vector . should_equal [False, True, Nothing]
suite_builder.group prefix+"(Nothing_Spec) is_in" group_builder->
if setup.is_operation_supported "IS_IN" then suite_builder.group prefix+"(Nothing_Spec) is_in" group_builder->
values_with_nothing.map triple->
value = triple.at 0
other_value = triple.at 1
@ -176,7 +176,7 @@ add_nothing_specs suite_builder setup =
table = table_builder_typed [["x", [value, Nothing]], ["y", [other_value, Nothing]], ["z", [value, other_value]], ["n", [Nothing, Nothing]]] value_type
table.at "x" . is_in [] . to_vector . should_equal [False, False]
if setup.test_selection.run_advanced_edge_case_tests then suite_builder.group prefix+"(Nothing_Spec) is_in: Boolean+Nothing edge cases" group_builder->
if setup.test_selection.run_advanced_edge_case_tests && setup.is_operation_supported "IS_IN" then suite_builder.group prefix+"(Nothing_Spec) is_in: Boolean+Nothing edge cases" group_builder->
make_containing_values had_null had_true had_false =
null_maybe = if had_null then [Nothing] else []
true_maybe = if had_true then [True] else []
@ -222,7 +222,7 @@ add_nothing_specs suite_builder setup =
c.to_vector . should_equal [output]
group_builder.specify "Boolean is_in: edge cases (Column), "+negation_desc+" "+cs.to_text <|
if setup.is_feature_supported Feature.Distinct then group_builder.specify "Boolean is_in: edge cases (Column), "+negation_desc+" "+cs.to_text <|
input_column = transform_input (Vector.fill containing_values.length input)
t = table_builder_typed [["input", input_column], ["containing", transform_argument containing_values]] Value_Type.Boolean
expected_output = if input_column.is_empty then [] else [output]
@ -232,7 +232,7 @@ add_nothing_specs suite_builder setup =
c.to_vector . length . should_equal input_column.length
c.to_vector.distinct . should_equal expected_output
suite_builder.group prefix+"(Nothing_Spec) distinct" group_builder->
if setup.is_feature_supported Feature.Distinct then suite_builder.group prefix+"(Nothing_Spec) distinct" group_builder->
values_without_nothing.map triple->
value = triple.at 0
other_value = triple.at 1

View File

@ -737,9 +737,10 @@ add_postgres_specs suite_builder create_connection_fn db_name =
(agg_in_memory_table.take (..First 0)).select_into_database_table default_connection.get (Name_Generator.random_name "Agg_Empty") primary_key=Nothing temporary=True
is_feature_supported_fn feature:Feature = default_connection.get.dialect.is_feature_supported feature
is_operation_supported_fn operation:Text = default_connection.get.dialect.is_operation_supported operation
flagged_fn = default_connection.get.dialect.flagged
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_fn light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
Common_Spec.add_specs suite_builder prefix create_connection_fn default_connection setup
postgres_specific_spec suite_builder create_connection_fn db_name setup

View File

@ -352,9 +352,10 @@ sqlite_spec suite_builder prefix create_connection_func persistent_connector =
(agg_in_memory_table.take (..First 0)).select_into_database_table default_connection.get (Name_Generator.random_name "Agg_Empty") primary_key=Nothing temporary=True
is_feature_supported_fn feature:Feature = default_connection.get.dialect.is_feature_supported feature
is_operation_supported_fn operation:Text = default_connection.get.dialect.is_operation_supported operation
flagged_fn = default_connection.get.dialect.flagged
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_func light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn
setup = Common_Table_Operations.Main.Test_Setup.Config prefix agg_table_fn empty_agg_table_fn table_builder materialize is_database=True test_selection=common_selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_func light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
Common_Spec.add_specs suite_builder prefix create_connection_func default_connection setup
sqlite_specific_spec suite_builder prefix create_connection_func setup

View File

@ -37,11 +37,13 @@ in_memory_setup =
Dummy_Connection.Value
is_feature_supported_fn _ =
True
is_operation_supported_fn _ =
True
flagged_fn flag:Dialect_Flag =
case flag of
Dialect_Flag.Supports_Case_Sensitive_Columns -> True
Common_Table_Operations.Main.Test_Setup.Config "[In-Memory] " agg_table_fn empty_table_fn table_builder materialize is_database=False test_selection=selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_func light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn
Common_Table_Operations.Main.Test_Setup.Config "[In-Memory] " agg_table_fn empty_table_fn table_builder materialize is_database=False test_selection=selection aggregate_test_selection=aggregate_selection create_connection_func=create_connection_func light_table_builder=light_table_builder is_feature_supported=is_feature_supported_fn flagged=flagged_fn is_operation_supported=is_operation_supported_fn
add_specs suite_builder =
Common_Table_Operations.Main.add_specs suite_builder in_memory_setup