mirror of
https://github.com/enso-org/enso.git
synced 2025-01-03 14:04:44 +03:00
SQLServer Aggregate Support (#11811)
* 40 red * 18 Red * 31 Red * 20 red * 18 Red * 15 red * 9 Red * 7 * Comment out broken test for now * Green * Cleanup * Changelog * Update check_aggregate_support * Cleanup * Reenable test * Fix tests * Doc comment
This commit is contained in:
parent
e8f781afbf
commit
31772e3565
@ -136,6 +136,7 @@
|
|||||||
- [Enhance Managed_Resource to allow implementation of in-memory caches][11577]
|
- [Enhance Managed_Resource to allow implementation of in-memory caches][11577]
|
||||||
- [Added `add_group_number` to the in-memory database.[11818]
|
- [Added `add_group_number` to the in-memory database.[11818]
|
||||||
- [The reload button clears the HTTP cache.][11673]
|
- [The reload button clears the HTTP cache.][11673]
|
||||||
|
- [SQL Server Support for Aggregate][11811]
|
||||||
|
|
||||||
[11235]: https://github.com/enso-org/enso/pull/11235
|
[11235]: https://github.com/enso-org/enso/pull/11235
|
||||||
[11255]: https://github.com/enso-org/enso/pull/11255
|
[11255]: https://github.com/enso-org/enso/pull/11255
|
||||||
@ -146,6 +147,7 @@
|
|||||||
[11577]: https://github.com/enso-org/enso/pull/11577
|
[11577]: https://github.com/enso-org/enso/pull/11577
|
||||||
[11818]: https://github.com/enso-org/enso/pull/11818
|
[11818]: https://github.com/enso-org/enso/pull/11818
|
||||||
[11673]: https://github.com/enso-org/enso/pull/11673
|
[11673]: https://github.com/enso-org/enso/pull/11673
|
||||||
|
[11811]: https://github.com/enso-org/enso/pull/11811
|
||||||
|
|
||||||
#### Enso Language & Runtime
|
#### Enso Language & Runtime
|
||||||
|
|
||||||
|
@ -157,6 +157,12 @@ type Redshift_Dialect
|
|||||||
_ = [op_kind, args]
|
_ = [op_kind, args]
|
||||||
expression
|
expression
|
||||||
|
|
||||||
|
## PRIVATE
|
||||||
|
Add an extra cast to adjust the output type of aggregate operations.
|
||||||
|
Some DBs do CAST(SUM(x) AS FLOAT) others do SUM(CAST(x AS FLOAT)).
|
||||||
|
cast_aggregate_columns self op_kind:Text columns:(Vector Internal_Column) =
|
||||||
|
self.cast_op_type op_kind columns (SQL_Expression.Operation op_kind (columns.map c->c.expression))
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
||||||
prepare_fetch_types_query self expression context =
|
prepare_fetch_types_query self expression context =
|
||||||
|
@ -35,7 +35,7 @@ from project.Errors import Aggregagtion_Requires_Order
|
|||||||
make_aggregate_column : DB_Table -> Aggregate_Column -> Text -> Dialect -> (Text -> Vector -> SQL_Expression -> SQL_Type_Reference) -> Problem_Builder -> Internal_Column
|
make_aggregate_column : DB_Table -> Aggregate_Column -> Text -> Dialect -> (Text -> Vector -> SQL_Expression -> SQL_Type_Reference) -> Problem_Builder -> Internal_Column
|
||||||
make_aggregate_column table aggregate as dialect infer_return_type problem_builder -> Internal_Column =
|
make_aggregate_column table aggregate as dialect infer_return_type problem_builder -> Internal_Column =
|
||||||
simple_aggregate op_kind columns =
|
simple_aggregate op_kind columns =
|
||||||
expression = dialect.cast_op_type op_kind columns (SQL_Expression.Operation op_kind (columns.map c->c.expression))
|
expression = dialect.cast_aggregate_columns op_kind columns
|
||||||
sql_type_ref = infer_return_type op_kind columns expression
|
sql_type_ref = infer_return_type op_kind columns expression
|
||||||
Internal_Column.Value as sql_type_ref expression
|
Internal_Column.Value as sql_type_ref expression
|
||||||
|
|
||||||
|
@ -179,6 +179,7 @@ type SQL_Generator
|
|||||||
generate_select_query_sql : Dialect -> Vector (Pair Text SQL_Expression) -> Context -> SQL_Builder
|
generate_select_query_sql : Dialect -> Vector (Pair Text SQL_Expression) -> Context -> SQL_Builder
|
||||||
generate_select_query_sql self dialect columns ctx =
|
generate_select_query_sql self dialect columns ctx =
|
||||||
gen_exprs exprs = exprs.map (expr-> dialect.generate_expression self expr for_select=False)
|
gen_exprs exprs = exprs.map (expr-> dialect.generate_expression self expr for_select=False)
|
||||||
|
gen_group_exprs exprs = exprs.map (expr-> dialect.generate_expression self expr for_select=True)
|
||||||
gen_column pair = (dialect.generate_expression self expr=pair.second for_select=True) ++ alias dialect pair.first
|
gen_column pair = (dialect.generate_expression self expr=pair.second for_select=True) ++ alias dialect pair.first
|
||||||
|
|
||||||
generated_columns = case columns of
|
generated_columns = case columns of
|
||||||
@ -187,7 +188,7 @@ type SQL_Generator
|
|||||||
|
|
||||||
from_part = self.generate_from_part dialect ctx.from_spec
|
from_part = self.generate_from_part dialect ctx.from_spec
|
||||||
where_part = (SQL_Builder.join " AND " (gen_exprs ctx.where_filters)) . prefix_if_present " WHERE "
|
where_part = (SQL_Builder.join " AND " (gen_exprs ctx.where_filters)) . prefix_if_present " WHERE "
|
||||||
group_part = (SQL_Builder.join ", " (gen_exprs ctx.groups)) . prefix_if_present " GROUP BY "
|
group_part = (SQL_Builder.join ", " (gen_group_exprs ctx.groups)) . prefix_if_present " GROUP BY "
|
||||||
|
|
||||||
orders = ctx.orders.map (self.generate_order dialect)
|
orders = ctx.orders.map (self.generate_order dialect)
|
||||||
order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY "
|
order_part = (SQL_Builder.join ", " orders) . prefix_if_present " ORDER BY "
|
||||||
@ -663,14 +664,14 @@ preprocess_query (query : Query) -> Query =
|
|||||||
column expression; it should be provided only if `has_quote` is `True` and
|
column expression; it should be provided only if `has_quote` is `True` and
|
||||||
must not be empty then. If the quote character occurs in the expression, it
|
must not be empty then. If the quote character occurs in the expression, it
|
||||||
is escaped by doubling each occurrence.
|
is escaped by doubling each occurrence.
|
||||||
make_concat make_raw_concat_expr make_contains_expr has_quote args =
|
make_concat make_raw_concat_expr make_contains_expr has_quote args append_char="||" =
|
||||||
expected_args = if has_quote then 5 else 4
|
expected_args = if has_quote then 5 else 4
|
||||||
if args.length != expected_args then Error.throw (Illegal_State.Error "Unexpected number of arguments for the concat operation.") else
|
if args.length != expected_args then Error.throw (Illegal_State.Error "Unexpected number of arguments for the concat operation.") else
|
||||||
expr = args.at 0
|
expr = args.at 0
|
||||||
separator = args.at 1
|
separator = args.at 1
|
||||||
prefix = args.at 2
|
prefix = args.at 2
|
||||||
suffix = args.at 3
|
suffix = args.at 3
|
||||||
append = " || "
|
append = " " + append_char + " "
|
||||||
possibly_quoted = case has_quote of
|
possibly_quoted = case has_quote of
|
||||||
True ->
|
True ->
|
||||||
quote = args.at 4
|
quote = args.at 4
|
||||||
|
@ -231,6 +231,12 @@ type Postgres_Dialect
|
|||||||
if cast_to.is_nothing then expression else
|
if cast_to.is_nothing then expression else
|
||||||
SQL_Expression.Operation "CAST" [expression, SQL_Expression.Literal cast_to]
|
SQL_Expression.Operation "CAST" [expression, SQL_Expression.Literal cast_to]
|
||||||
|
|
||||||
|
## PRIVATE
|
||||||
|
Add an extra cast to adjust the output type of aggregate operations.
|
||||||
|
Some DBs do CAST(SUM(x) AS FLOAT) others do SUM(CAST(x AS FLOAT)).
|
||||||
|
cast_aggregate_columns self op_kind:Text columns:(Vector Internal_Column) =
|
||||||
|
self.cast_op_type op_kind columns (SQL_Expression.Operation op_kind (columns.map c->c.expression))
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
||||||
prepare_fetch_types_query self expression context =
|
prepare_fetch_types_query self expression context =
|
||||||
|
@ -215,6 +215,12 @@ type SQLite_Dialect
|
|||||||
_ = [op_kind, args]
|
_ = [op_kind, args]
|
||||||
expression
|
expression
|
||||||
|
|
||||||
|
## PRIVATE
|
||||||
|
Add an extra cast to adjust the output type of aggregate operations.
|
||||||
|
Some DBs do CAST(SUM(x) AS FLOAT) others do SUM(CAST(x AS FLOAT)).
|
||||||
|
cast_aggregate_columns self op_kind:Text columns:(Vector Internal_Column) =
|
||||||
|
self.cast_op_type op_kind columns (SQL_Expression.Operation op_kind (columns.map c->c.expression))
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
||||||
prepare_fetch_types_query self expression context =
|
prepare_fetch_types_query self expression context =
|
||||||
|
@ -212,8 +212,27 @@ type SQLServer_Dialect
|
|||||||
is used only to override the type in cases where the default one that the
|
is used only to override the type in cases where the default one that the
|
||||||
database uses is not what we want.
|
database uses is not what we want.
|
||||||
cast_op_type self (op_kind:Text) (args:(Vector Internal_Column)) (expression:SQL_Expression) =
|
cast_op_type self (op_kind:Text) (args:(Vector Internal_Column)) (expression:SQL_Expression) =
|
||||||
_ = [op_kind, args]
|
is_int ic =
|
||||||
expression
|
typeid = ic.sql_type_reference.get.typeid
|
||||||
|
typeid == Java_Types.SMALLINT || typeid == Java_Types.INTEGER || typeid == Java_Types.BIGINT
|
||||||
|
|
||||||
|
cast_to = case op_kind of
|
||||||
|
"AVG" ->
|
||||||
|
if is_int (args.at 0) then "FLOAT" else Nothing
|
||||||
|
"STDDEV_POP" ->
|
||||||
|
if is_int (args.at 0) then "FLOAT" else Nothing
|
||||||
|
"STDDEV_SAMP" ->
|
||||||
|
if is_int (args.at 0) then "FLOAT" else Nothing
|
||||||
|
_ -> Nothing
|
||||||
|
|
||||||
|
if cast_to.is_nothing then expression else
|
||||||
|
SQL_Expression.Operation "CAST" [expression, SQL_Expression.Literal cast_to]
|
||||||
|
|
||||||
|
## PRIVATE
|
||||||
|
Add an extra cast to adjust the output type of aggregate operations.
|
||||||
|
Some DBs do CAST(SUM(x) AS FLOAT) others do SUM(CAST(x AS FLOAT)).
|
||||||
|
cast_aggregate_columns self op_kind:Text columns:(Vector Internal_Column) =
|
||||||
|
SQL_Expression.Operation op_kind (columns.map c->(self.cast_op_type op_kind columns (Internals_Access.column_expression c)))
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
||||||
@ -224,10 +243,32 @@ type SQLServer_Dialect
|
|||||||
generate_collate self collation_name:Text -> Text = Base_Generator.default_generate_collate collation_name quote_char=""
|
generate_collate self collation_name:Text -> Text = Base_Generator.default_generate_collate collation_name quote_char=""
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
check_aggregate_support : Aggregate_Column -> Boolean ! Unsupported_Database_Operation
|
check_aggregate_support self aggregate:Aggregate_Column -> Boolean ! Unsupported_Database_Operation =
|
||||||
check_aggregate_support self aggregate =
|
unsupported name =
|
||||||
_ = aggregate
|
Error.throw (Unsupported_Database_Operation.Error name)
|
||||||
True
|
case aggregate of
|
||||||
|
Group_By _ _ -> True
|
||||||
|
Count _ -> True
|
||||||
|
Count_Distinct columns _ _ ->
|
||||||
|
if columns.length == 1 then True else
|
||||||
|
unsupported "Count_Distinct on multiple columns"
|
||||||
|
Count_Not_Nothing _ _ -> True
|
||||||
|
Count_Nothing _ _ -> True
|
||||||
|
Count_Not_Empty _ _ -> True
|
||||||
|
Count_Empty _ _ -> True
|
||||||
|
Percentile _ _ _ -> unsupported "Percentile"
|
||||||
|
Mode _ _ -> unsupported "Mode"
|
||||||
|
First _ _ _ _ -> unsupported "First"
|
||||||
|
Last _ _ _ _ -> unsupported "Last"
|
||||||
|
Maximum _ _ -> True
|
||||||
|
Minimum _ _ -> True
|
||||||
|
Shortest _ _ -> unsupported "Shortest"
|
||||||
|
Longest _ _ -> unsupported "Longest"
|
||||||
|
Standard_Deviation _ _ _ -> True
|
||||||
|
Concatenate _ _ _ _ _ _ -> True
|
||||||
|
Sum _ _ -> True
|
||||||
|
Average _ _ -> True
|
||||||
|
Median _ _ -> unsupported "Median"
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
Checks if an operation is supported by the dialect.
|
Checks if an operation is supported by the dialect.
|
||||||
@ -243,6 +284,7 @@ type SQLServer_Dialect
|
|||||||
Feature.Filter -> True
|
Feature.Filter -> True
|
||||||
Feature.Join -> True
|
Feature.Join -> True
|
||||||
Feature.Union -> True
|
Feature.Union -> True
|
||||||
|
Feature.Aggregate -> True
|
||||||
_ -> False
|
_ -> False
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
@ -401,6 +443,7 @@ private _generate_expression dialect base_gen expr expression_kind:Expression_Ki
|
|||||||
|
|
||||||
pair final_expr null_checks_result
|
pair final_expr null_checks_result
|
||||||
query : Query -> pair (base_gen.generate_sub_query dialect query) []
|
query : Query -> pair (base_gen.generate_sub_query dialect query) []
|
||||||
|
descriptor : Order_Descriptor -> pair (base_gen.generate_order dialect descriptor) []
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
type Expression_Kind
|
type Expression_Kind
|
||||||
@ -437,7 +480,7 @@ private _op_return_kind op -> Expression_Kind =
|
|||||||
if return_bool_ops.contains op then Expression_Kind.Boolean_Condition else Expression_Kind.Value
|
if return_bool_ops.contains op then Expression_Kind.Boolean_Condition else Expression_Kind.Value
|
||||||
|
|
||||||
private _op_needs_to_materialize_null_checks op -> Boolean =
|
private _op_needs_to_materialize_null_checks op -> Boolean =
|
||||||
["FILL_NULL", "COALESCE"].contains op
|
["FILL_NULL", "COALESCE", "COUNT_IS_NULL", "COUNT_EMPTY", "COUNT_NOT_EMPTY", "COUNT", "SUM", "AVG", "LONGEST", "SHORTEST", "COUNT_DISTINCT", "COUNT_DISTINCT_INCLUDE_NULL", "STDDEV_POP", "STDDEV_SAMP", "CONCAT", "CONCAT_QUOTE_IF_NEEDED", "MIN", "MAX"].contains op
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
make_dialect_operations =
|
make_dialect_operations =
|
||||||
@ -447,13 +490,13 @@ make_dialect_operations =
|
|||||||
arith_extensions = [floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]]
|
arith_extensions = [floating_point_div, mod_op, decimal_div, decimal_mod, ["ROW_MIN", Base_Generator.make_function "LEAST"], ["ROW_MAX", Base_Generator.make_function "GREATEST"]]
|
||||||
bool = [bool_or]
|
bool = [bool_or]
|
||||||
|
|
||||||
stddev_pop = ["STDDEV_POP", Base_Generator.make_function "stddev_pop"]
|
stddev_pop = ["STDDEV_POP", Base_Generator.make_function "STDEVP"]
|
||||||
stddev_samp = ["STDDEV_SAMP", Base_Generator.make_function "stddev_samp"]
|
stddev_samp = ["STDDEV_SAMP", Base_Generator.make_function "STDEV"]
|
||||||
stats = [agg_median, agg_mode, agg_percentile, stddev_pop, stddev_samp]
|
stats = [stddev_pop, stddev_samp]
|
||||||
date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]]
|
date_ops = [["year", Base_Generator.make_function "year"], make_datepart "quarter", ["month", Base_Generator.make_function "month"], make_datepart "week" "iso_week", ["day", Base_Generator.make_function "day"], make_datepart "hour", make_datepart "minute", make_datepart "day_of_year" "dayofyear", make_day_of_week, make_datepart "second", make_datepart "millisecond", make_extract_microsecond, ["date_add", make_date_add], ["date_diff", make_date_diff], ["date_trunc_to_day", make_date_trunc_to_day]]
|
||||||
special_overrides = [is_empty, ["IIF", _make_iif]]
|
special_overrides = [is_empty, ["IIF", _make_iif]]
|
||||||
other = [["RUNTIME_ERROR", make_runtime_error_op]]
|
other = [["RUNTIME_ERROR", make_runtime_error_op]]
|
||||||
my_mappings = text + counts + stats + first_last_aggregators + arith_extensions + bool + date_ops + special_overrides + other
|
my_mappings = text + counts + arith_extensions + bool + stats + date_ops + special_overrides + other
|
||||||
base = Base_Generator.base_dialect_operations . extend_with my_mappings
|
base = Base_Generator.base_dialect_operations . extend_with my_mappings
|
||||||
Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN")
|
Base_Generator.Dialect_Operations.Value (base.operations_dict.remove "IS_IN")
|
||||||
|
|
||||||
@ -469,68 +512,29 @@ private _make_iif arguments:Vector -> SQL_Builder =
|
|||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_count_is_null = Base_Generator.lift_unary_op "COUNT_IS_NULL" arg->
|
agg_count_is_null = Base_Generator.lift_unary_op "COUNT_IS_NULL" arg->
|
||||||
SQL_Builder.code "SUM(CASE WHEN " ++ arg.paren ++ " IS NULL THEN 1 ELSE 0 END)"
|
SQL_Builder.code "COALESCE(SUM(CASE WHEN " ++ arg.paren ++ " IS NULL THEN 1 ELSE 0 END), 0)"
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_count_empty = Base_Generator.lift_unary_op "COUNT_EMPTY" arg->
|
agg_count_empty = Base_Generator.lift_unary_op "COUNT_EMPTY" arg->
|
||||||
SQL_Builder.code "SUM(CASE WHEN (" ++ arg.paren ++ " IS NULL) OR (" ++ arg.paren ++ " = '') THEN 1 ELSE 0 END)"
|
SQL_Builder.code "COALESCE(SUM(CASE WHEN (" ++ arg.paren ++ " IS NULL) OR (" ++ arg.paren ++ " = '') THEN 1 ELSE 0 END), 0)"
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_count_not_empty = Base_Generator.lift_unary_op "COUNT_NOT_EMPTY" arg->
|
agg_count_not_empty = Base_Generator.lift_unary_op "COUNT_NOT_EMPTY" arg->
|
||||||
SQL_Builder.code "SUM(CASE WHEN (" ++ arg.paren ++ " IS NOT NULL) AND (" ++ arg.paren ++ " != '') THEN 1 ELSE 0 END)"
|
SQL_Builder.code "COALESCE(SUM(CASE WHEN (" ++ arg.paren ++ " IS NOT NULL) AND (" ++ arg.paren ++ " != '') THEN 1 ELSE 0 END), 0)"
|
||||||
|
|
||||||
|
|
||||||
## PRIVATE
|
|
||||||
agg_median = Base_Generator.lift_unary_op "MEDIAN" arg->
|
|
||||||
median = SQL_Builder.code "MEDIAN(" ++ arg ++ ")"
|
|
||||||
has_nan = SQL_Builder.code "BOOLOR_AGG(" ++ arg ++ " = 'NaN'::Double)"
|
|
||||||
SQL_Builder.code "CASE WHEN " ++ has_nan ++ " THEN 'NaN'::Double ELSE " ++ median ++ " END"
|
|
||||||
|
|
||||||
## PRIVATE
|
|
||||||
agg_mode = Base_Generator.lift_unary_op "MODE" arg->
|
|
||||||
SQL_Builder.code "MODE(" ++ arg ++ ")"
|
|
||||||
|
|
||||||
## PRIVATE
|
|
||||||
agg_percentile = Base_Generator.lift_binary_op "PERCENTILE" p-> expr->
|
|
||||||
percentile = SQL_Builder.code "percentile_cont(" ++ p ++ ") WITHIN GROUP (ORDER BY " ++ expr ++ ")"
|
|
||||||
has_nan = SQL_Builder.code "BOOLOR_AGG(" ++ expr ++ " = 'NaN'::Double)"
|
|
||||||
SQL_Builder.code "CASE WHEN " ++ has_nan ++ " THEN 'NaN' ELSE " ++ percentile ++ " END"
|
|
||||||
|
|
||||||
## PRIVATE
|
|
||||||
These are written in a not most-efficient way, but a way that makes them
|
|
||||||
compatible with other group-by aggregations out-of-the-box. In the future, we
|
|
||||||
may want to consider some alternative solutions.
|
|
||||||
first_last_aggregators =
|
|
||||||
first = make_first_aggregator reverse=False ignore_null=False
|
|
||||||
first_not_null = make_first_aggregator reverse=False ignore_null=True
|
|
||||||
last = make_first_aggregator reverse=True ignore_null=False
|
|
||||||
last_not_null = make_first_aggregator reverse=True ignore_null=True
|
|
||||||
[["FIRST", first], ["FIRST_NOT_NULL", first_not_null], ["LAST", last], ["LAST_NOT_NULL", last_not_null]]
|
|
||||||
|
|
||||||
## PRIVATE
|
|
||||||
make_first_aggregator reverse ignore_null args =
|
|
||||||
if args.length < 2 then Error.throw (Illegal_State.Error "Insufficient number of arguments for the operation.") else
|
|
||||||
result_expr = args.first
|
|
||||||
order_bys = args.drop 1
|
|
||||||
|
|
||||||
method_name = if reverse then "LAST_VALUE" else "FIRST_VALUE"
|
|
||||||
filter_clause = if ignore_null then ") IGNORE NULLS OVER" else ") OVER"
|
|
||||||
order_clause = SQL_Builder.code " ORDER BY " ++ SQL_Builder.join "," order_bys
|
|
||||||
SQL_Builder.code (method_name + "(") ++ result_expr ++ filter_clause ++ order_clause
|
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_shortest = Base_Generator.lift_unary_op "SHORTEST" arg->
|
agg_shortest = Base_Generator.lift_unary_op "SHORTEST" arg->
|
||||||
SQL_Builder.code "FIRST_VALUE(" ++ arg ++ ") IGNORE NULLS OVER (ORDER BY LENGTH(" ++ arg ++ "))"
|
SQL_Builder.code "FIRST_VALUE(" ++ arg ++ ") IGNORE NULLS OVER (ORDER BY LEN(" ++ arg ++ "))"
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_longest = Base_Generator.lift_unary_op "LONGEST" arg->
|
agg_longest = Base_Generator.lift_unary_op "LONGEST" arg->
|
||||||
SQL_Builder.code "FIRST_VALUE(" ++ arg ++ ") IGNORE NULLS OVER (ORDER BY LENGTH(" ++ arg ++ ") DESC)"
|
SQL_Builder.code "FIRST_VALUE(" ++ arg ++ ") IGNORE NULLS OVER (ORDER BY LEN(" ++ arg ++ ") DESC)"
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
concat_ops =
|
concat_ops =
|
||||||
make_raw_concat_expr expr separator =
|
make_raw_concat_expr expr separator =
|
||||||
SQL_Builder.code "string_agg(" ++ expr ++ ", " ++ separator ++ ")"
|
SQL_Builder.code "string_agg(" ++ expr ++ ", " ++ separator ++ ")"
|
||||||
concat = Base_Generator.make_concat make_raw_concat_expr make_contains_expr
|
concat = Base_Generator.make_concat make_raw_concat_expr make_contains_expr append_char="+"
|
||||||
[["CONCAT", concat (has_quote=False)], ["CONCAT_QUOTE_IF_NEEDED", concat (has_quote=True)]]
|
[["CONCAT", concat (has_quote=False)], ["CONCAT_QUOTE_IF_NEEDED", concat (has_quote=True)]]
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
@ -554,14 +558,7 @@ agg_count_distinct args = if args.is_empty then (Error.throw (Illegal_Argument.E
|
|||||||
True ->
|
True ->
|
||||||
## A single null value will be skipped.
|
## A single null value will be skipped.
|
||||||
SQL_Builder.code "COUNT(DISTINCT " ++ args.first ++ ")"
|
SQL_Builder.code "COUNT(DISTINCT " ++ args.first ++ ")"
|
||||||
False ->
|
False -> Error.throw (Illegal_Argument.Error "COUNT_DISTINCT supports only single arguments in SQLServer.")
|
||||||
## A tuple of nulls is not a null, so it will not be skipped - but
|
|
||||||
we want to ignore all-null columns. So we manually filter them
|
|
||||||
out.
|
|
||||||
count = SQL_Builder.code "COUNT(DISTINCT (" ++ SQL_Builder.join ", " args ++ "))"
|
|
||||||
are_nulls = args.map arg-> arg.paren ++ " IS NULL"
|
|
||||||
all_nulls_filter = SQL_Builder.code " FILTER (WHERE NOT (" ++ SQL_Builder.join " AND " are_nulls ++ "))"
|
|
||||||
(count ++ all_nulls_filter).paren
|
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
agg_count_distinct_include_null args = case args.length == 1 of
|
agg_count_distinct_include_null args = case args.length == 1 of
|
||||||
@ -595,12 +592,11 @@ ends_with = Base_Generator.lift_binary_op "ENDS_WITH" str-> sub->
|
|||||||
res.paren
|
res.paren
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
contains = Base_Generator.lift_binary_op "CONTAINS" str-> sub->
|
make_contains_expr expr substring =
|
||||||
res = SQL_Builder.code "CHARINDEX(" ++ sub ++ ", " ++ str ++ ") > 0"
|
SQL_Builder.code "CHARINDEX(" ++ substring ++ ", " ++ expr ++ ") > 0"
|
||||||
res.paren
|
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
make_contains_expr expr substring = contains [expr, substring]
|
contains = Base_Generator.lift_binary_op "CONTAINS" make_contains_expr
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
make_case_sensitive = Base_Generator.lift_unary_op "MAKE_CASE_SENSITIVE" arg->
|
make_case_sensitive = Base_Generator.lift_unary_op "MAKE_CASE_SENSITIVE" arg->
|
||||||
|
@ -219,6 +219,12 @@ type Snowflake_Dialect
|
|||||||
_ = [op_kind, args]
|
_ = [op_kind, args]
|
||||||
expression
|
expression
|
||||||
|
|
||||||
|
## PRIVATE
|
||||||
|
Add an extra cast to adjust the output type of aggregate operations.
|
||||||
|
Some DBs do CAST(SUM(x) AS FLOAT) others do SUM(CAST(x AS FLOAT)).
|
||||||
|
cast_aggregate_columns self op_kind:Text columns:(Vector Internal_Column) =
|
||||||
|
self.cast_op_type op_kind columns (SQL_Expression.Operation op_kind (columns.map c->c.expression))
|
||||||
|
|
||||||
## PRIVATE
|
## PRIVATE
|
||||||
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
prepare_fetch_types_query : SQL_Expression -> Context -> SQL_Statement
|
||||||
prepare_fetch_types_query self expression context =
|
prepare_fetch_types_query self expression context =
|
||||||
|
@ -200,8 +200,8 @@ add_sqlserver_specs suite_builder create_connection_fn =
|
|||||||
materialize = .read
|
materialize = .read
|
||||||
|
|
||||||
common_selection = Common_Table_Operations.Main.Test_Selection.Config supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=True
|
common_selection = Common_Table_Operations.Main.Test_Selection.Config supported_replace_params=supported_replace_params run_advanced_edge_case_tests_by_default=True
|
||||||
aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config first_last_row_order=False aggregation_problems=False
|
aggregate_selection = Common_Table_Operations.Aggregate_Spec.Test_Selection.Config advanced_stats=False text_shortest_longest=False first_last=False first_last_row_order=False aggregation_problems=False multi_distinct=False first_last_multi_order=False first_last_ignore_nothing=False text_concat=False
|
||||||
agg_in_memory_table = (enso_project.data / "data.csv") . read
|
agg_in_memory_table = ((Project_Description.new enso_dev.Table_Tests).data / "data.csv") . read
|
||||||
|
|
||||||
agg_table_fn = _->
|
agg_table_fn = _->
|
||||||
agg_in_memory_table.select_into_database_table default_connection.get (Name_Generator.random_name "Agg1") primary_key=Nothing temporary=True
|
agg_in_memory_table.select_into_database_table default_connection.get (Name_Generator.random_name "Agg1") primary_key=Nothing temporary=True
|
||||||
|
@ -1262,7 +1262,7 @@ add_aggregate_specs suite_builder setup =
|
|||||||
loc = Meta.get_source_location 2
|
loc = Meta.get_source_location 2
|
||||||
Test.fail "Expected a Nothing or NaN but got: "+value.to_text+" (at "+loc+")."
|
Test.fail "Expected a Nothing or NaN but got: "+value.to_text+" (at "+loc+")."
|
||||||
|
|
||||||
suite_builder.group prefix+"Table.aggregate should correctly handle infinities" group_builder->
|
if setup.flagged ..Supports_Infinity then suite_builder.group prefix+"Table.aggregate should correctly handle infinities" group_builder->
|
||||||
pos_inf = 1/0
|
pos_inf = 1/0
|
||||||
neg_inf = -1/0
|
neg_inf = -1/0
|
||||||
|
|
||||||
@ -1341,7 +1341,7 @@ add_aggregate_specs suite_builder setup =
|
|||||||
expect_null_or_nan <| m1.columns.first.at 0
|
expect_null_or_nan <| m1.columns.first.at 0
|
||||||
expect_null_or_nan <| m1.columns.second.at 0
|
expect_null_or_nan <| m1.columns.second.at 0
|
||||||
|
|
||||||
suite_builder.group prefix+"Table.aggregate should correctly handle NaN" pending=(resolve_pending test_selection.nan) group_builder->
|
if setup.flagged ..Supports_Separate_NaN then suite_builder.group prefix+"Table.aggregate should correctly handle NaN" pending=(resolve_pending test_selection.nan) group_builder->
|
||||||
nan = 0.log 0
|
nan = 0.log 0
|
||||||
group_builder.specify "on Average" <|
|
group_builder.specify "on Average" <|
|
||||||
t1 = table_builder [["X", [Nothing, nan, 0, 1, 2]]]
|
t1 = table_builder [["X", [Nothing, nan, 0, 1, 2]]]
|
||||||
@ -1790,7 +1790,7 @@ add_aggregate_specs suite_builder setup =
|
|||||||
table = table_builder [["A", [3,2,1]], ["X", [1,2,3]]]
|
table = table_builder [["A", [3,2,1]], ["X", [1,2,3]]]
|
||||||
order = [Sort_Column.Name "A"]
|
order = [Sort_Column.Name "A"]
|
||||||
expect_sum_and_unsupported_errors 2 <|
|
expect_sum_and_unsupported_errors 2 <|
|
||||||
table.aggregate columns=[Sum "X", First ignore_nothing=False "X" (order_by=order), Last ignore_nothing=False "X" (order_by=order)]
|
table.aggregate columns=[Sum "X", First "X" ignore_nothing=False order_by=order, Last "X" ignore_nothing=False order_by=order] on_problems=..Report_Warning
|
||||||
|
|
||||||
if test_selection.first_last_ignore_nothing.not then
|
if test_selection.first_last_ignore_nothing.not then
|
||||||
group_builder.specify "with First and Last with ignore_nothing=True" <|
|
group_builder.specify "with First and Last with ignore_nothing=True" <|
|
||||||
@ -1817,7 +1817,7 @@ add_aggregate_specs suite_builder setup =
|
|||||||
expect_sum_and_unsupported_errors 2 <|
|
expect_sum_and_unsupported_errors 2 <|
|
||||||
table.aggregate columns=[Sum "X", Shortest "Y", Longest "Y"]
|
table.aggregate columns=[Sum "X", Shortest "Y", Longest "Y"]
|
||||||
|
|
||||||
if test_selection.text_concat.not && (setup.prefix.contains "Snowflake" . not) then
|
if test_selection.text_concat.not && (setup.prefix.contains "Snowflake" . not && setup.prefix.contains "SQLServer" . not) then
|
||||||
group_builder.specify "with Concatenate" <|
|
group_builder.specify "with Concatenate" <|
|
||||||
table = table_builder [["X", [1,2,3]], ["Y", ["a", "bb", "ccc"]]]
|
table = table_builder [["X", [1,2,3]], ["Y", ["a", "bb", "ccc"]]]
|
||||||
expect_sum_and_unsupported_errors 1 <|
|
expect_sum_and_unsupported_errors 1 <|
|
||||||
|
Loading…
Reference in New Issue
Block a user