Fix Text_Cleanse tests (#10263)

I hadn't connected the Text_Cleanse tests up properly and as a result they weren't running or working. This fixes that.
This commit is contained in:
AdRiley 2024-06-13 09:25:32 +01:00 committed by GitHub
parent d6d370925a
commit fadb81abe6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 29 additions and 16 deletions

View File

@ -1485,9 +1485,9 @@ type DB_Column
column.text_cleanse [..Leading_Whitespace, ..Trailing_Whitespace]
@remove make_data_cleanse_vector_selector
text_cleanse : Vector Text_Cleanse -> DB_Column ! Unsupported_Database_Operation
text_cleanse : Vector Named_Pattern -> DB_Column ! Unsupported_Database_Operation
text_cleanse self remove =
remove.fold self (current-> tc-> tc.apply current)
remove.map (Text_Cleanse.Value _) . fold self (current-> tc-> tc.apply current)
## GROUP Standard.Base.DateTime
ICON time

View File

@ -2928,7 +2928,7 @@ type DB_Table
table.text_cleanse [..Leading_Whitespace, ..Trailing_Whitespace]
@remove make_data_cleanse_vector_selector
@from (Widget_Helpers.make_column_name_multi_selector add_regex=True add_by_type=True)
text_cleanse : Vector Text_Cleanse -> Vector (Integer | Text | Regex | By_Type) -> DB_Table
text_cleanse : Vector Named_Pattern -> Vector (Integer | Text | Regex | By_Type) -> DB_Table
text_cleanse self remove from:(Vector (Integer | Text | Regex | By_Type)) =
transformer col = col.text_cleanse remove
Table_Helpers.replace_columns_with_transformed_columns self from transformer

View File

@ -2906,7 +2906,7 @@ type Table
table.text_cleanse [..Leading_Whitespace, ..Trailing_Whitespace]
@remove make_data_cleanse_vector_selector
@from (Widget_Helpers.make_column_name_multi_selector add_regex=True add_by_type=True)
text_cleanse : Vector Text_Cleanse -> Vector (Integer | Text | Regex | By_Type) -> Table
text_cleanse : Vector Named_Pattern -> Vector (Integer | Text | Regex | By_Type) -> Table
text_cleanse self remove from:(Vector (Integer | Text | Regex | By_Type)) =
transformer col = col.text_cleanse remove
Table_Helpers.replace_columns_with_transformed_columns self from transformer

View File

@ -29,6 +29,7 @@ import project.Common_Table_Operations.Order_By_Spec
import project.Common_Table_Operations.Select_Columns_Spec
import project.Common_Table_Operations.Take_Drop_Spec
import project.Common_Table_Operations.Temp_Column_Spec
import project.Common_Table_Operations.Text_Cleanse_Spec
import project.Common_Table_Operations.Transpose_Spec
from project.Common_Table_Operations.Util import run_default_backend
@ -152,5 +153,6 @@ add_specs suite_builder setup =
Integration_Tests.add_specs suite_builder setup
Temp_Column_Spec.add_specs suite_builder setup
Nothing_Spec.add_specs suite_builder setup
Text_Cleanse_Spec.add_specs suite_builder setup
main filter=Nothing = run_default_backend add_specs filter

View File

@ -9,6 +9,7 @@ import Standard.Base.Errors.Common.Type_Error
import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
import Standard.Test.Extensions
from Standard.Database.Errors import Unsupported_Database_Operation
from Standard.Table import Column, Table, Value_Type, Auto, Bits
from Standard.Table.Errors import Invalid_Value_Type, Invalid_Column_Names
from project.Common_Table_Operations.Util import run_default_backend
@ -46,27 +47,39 @@ add_specs suite_builder setup =
clean_passenger = ["Passenger", ["Albert Einstein", "Marie Curie ", "Isaac Newton ", "Stephen Hawking", "A d a Lovelace "]]
expected_table = Table.new [clean_flight, clean_passenger, ticket_price]
res = table.text_cleanse [Named_Pattern.Leading_Whitespace] ["Flight", "Passenger"]
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
case res.is_error && setup.is_database of
True ->
res.should_fail_with Unsupported_Database_Operation
False ->
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
group_builder.specify "can select columns with regex" <|
clean_flight = ["Flight", ["BA0123", "BA0123 ", "SG0456 ", "BA 0123", "S G 0 4 5 6 "]]
clean_passenger = ["Passenger", ["Albert Einstein", "Marie Curie ", "Isaac Newton ", "Stephen Hawking", "A d a Lovelace "]]
expected_table = Table.new [clean_flight, clean_passenger, ticket_price]
res = table.text_cleanse [Named_Pattern.Leading_Whitespace] [(regex "Fl.*"), (regex "P.*")]
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
case res.is_error && setup.is_database of
True ->
res.should_fail_with Unsupported_Database_Operation
False ->
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
group_builder.specify "can select columns by type" <|
clean_flight = ["Flight", ["BA0123", "BA0123 ", "SG0456 ", "BA 0123", "S G 0 4 5 6 "]]
clean_passenger = ["Passenger", ["Albert Einstein", "Marie Curie ", "Isaac Newton ", "Stephen Hawking", "A d a Lovelace "]]
expected_table = Table.new [clean_flight, clean_passenger, ticket_price]
res = table.text_cleanse [Named_Pattern.Leading_Whitespace] [..By_Type ..Char]
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
case res.is_error && setup.is_database of
True ->
res.should_fail_with Unsupported_Database_Operation
False ->
r = materialize res . rows . map .to_vector
r.length . should_equal 5
r.should_equal (expected_table . rows . map .to_vector)
group_builder.specify "should error if applied to non-text column" <|
table.text_cleanse [Named_Pattern.Leading_Whitespace] ["Flight", "Passenger", "Ticket Price"] . should_fail_with Invalid_Value_Type
table.text_cleanse [Named_Pattern.Leading_Whitespace] ["Ticket Price"] . should_fail_with Invalid_Value_Type
suite_builder.group "Column Text Cleanse" group_builder->
test_col = Column.from_vector "Test" [" It was", "the best ", "of times", " it was the worst of times "]
group_builder.specify "should remove leading whitespace" <|

View File

@ -20,7 +20,6 @@ import project.In_Memory.Table_Date_Time_Spec
import project.In_Memory.Table_Format_Spec
import project.In_Memory.Table_Time_Of_Day_Spec
import project.In_Memory.Table_Running_Spec
import project.In_Memory.Text_Cleanse_Spec
add_specs suite_builder =
Aggregate_Column_Spec.add_specs suite_builder
@ -41,7 +40,6 @@ add_specs suite_builder =
Table_Spec.add_specs suite_builder
Table_Time_Of_Day_Spec.add_specs suite_builder
Table_Xml_Spec.add_specs suite_builder
Text_Cleanse_Spec.add_specs suite_builder
main filter=Nothing =