Initial template for the Extra Tests workflow (#10636)

- Closes #10618
- adjusts some edge case tests in Snowflake

.github/workflows/extra-nightly-tests.yml

@@ -0,0 +1,79 @@
+# This file is auto-generated. Do not edit it manually!
+# Edit the enso_build::ci_gen module instead and run `cargo run --package enso-build-ci-gen`.
+
+name: Extra Nightly Tests
+on:
+  schedule:
+    - cron: 0 3 * * *
+  workflow_dispatch:
+    inputs:
+      clean_build_required:
+        description: Clean before and after the run.
+        required: false
+        type: boolean
+        default: false
+jobs:
+  enso-build-ci-gen-job-snowflake-tests-linux-x86_64:
+    name: Snowflake Tests (linux, x86_64)
+    runs-on:
+      - self-hosted
+      - Linux
+    steps:
+      - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent')
+        name: Installing wasm-pack
+        uses: jetli/wasm-pack-action@v0.4.0
+        with:
+          version: v0.10.2
+      - name: Expose Artifact API and context information.
+        uses: actions/github-script@v7
+        with:
+          script: "\n    core.exportVariable(\"ACTIONS_RUNTIME_TOKEN\", process.env[\"ACTIONS_RUNTIME_TOKEN\"])\n    core.exportVariable(\"ACTIONS_RUNTIME_URL\", process.env[\"ACTIONS_RUNTIME_URL\"])\n    core.exportVariable(\"GITHUB_RETENTION_DAYS\", process.env[\"GITHUB_RETENTION_DAYS\"])\n    console.log(context)\n    "
+      - name: Checking out the repository
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          submodules: recursive
+      - name: Build Script Setup
+        run: ./run --help || (git clean -ffdx && ./run --help)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: "(contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
+        name: Clean before
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: ./run backend test std-snowflake
+        env:
+          ENSO_SNOWFLAKE_ACCOUNT: ${{ secrets.ENSO_SNOWFLAKE_ACCOUNT }}
+          ENSO_SNOWFLAKE_DATABASE: ${{ secrets.ENSO_SNOWFLAKE_DATABASE }}
+          ENSO_SNOWFLAKE_PASSWORD: ${{ secrets.ENSO_SNOWFLAKE_PASSWORD }}
+          ENSO_SNOWFLAKE_SCHEMA: ${{ secrets.ENSO_SNOWFLAKE_SCHEMA }}
+          ENSO_SNOWFLAKE_USER: ${{ secrets.ENSO_SNOWFLAKE_USER }}
+          ENSO_SNOWFLAKE_WAREHOUSE: ${{ secrets.ENSO_SNOWFLAKE_WAREHOUSE }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - if: (success() || failure()) && github.event.pull_request.head.repo.full_name == github.repository
+        name: Extra Library Test Reporter
+        uses: dorny/test-reporter@v1
+        with:
+          max-annotations: 50
+          name: Extra Library Tests Report (GraalVM CE, linux, x86_64)
+          path: ${{ env.ENSO_TEST_JUNIT_DIR }}/*/*.xml
+          path-replace-backslashes: true
+          reporter: java-junit
+      - if: failure() && runner.os == 'Windows'
+        name: List files if failed (Windows)
+        run: Get-ChildItem -Force -Recurse
+      - if: failure() && runner.os != 'Windows'
+        name: List files if failed (non-Windows)
+        run: ls -lAR
+      - if: "(always()) && (contains(github.event.pull_request.labels.*.name, 'CI: Clean build required') || inputs.clean_build_required)"
+        name: Clean after
+        run: ./run git-clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    env:
+      GRAAL_EDITION: GraalVM CE
+    permissions:
+      checks: write
+env:
+  ENSO_BUILD_SKIP_VERSION_CHECK: "true"

build/build/paths.yaml

@@ -10,6 +10,7 @@
       gui-tests.yml:
       gui.yml:
       engine-nightly.yml:
+      extra-nightly-tests.yml:
       nightly.yml:
       promote.yml:
       release.yml:

build/build/src/ci_gen.rs

@@ -119,6 +119,14 @@ pub mod secret {
     pub const APPLE_NOTARIZATION_PASSWORD: &str = "APPLE_NOTARIZATION_PASSWORD";
     pub const APPLE_NOTARIZATION_TEAM_ID: &str = "APPLE_NOTARIZATION_TEAM_ID";
 
+    // === Snowflake Test Account ===
+    pub const ENSO_SNOWFLAKE_ACCOUNT: &str = "ENSO_SNOWFLAKE_ACCOUNT";
+    pub const ENSO_SNOWFLAKE_USER: &str = "ENSO_SNOWFLAKE_USER";
+    pub const ENSO_SNOWFLAKE_PASSWORD: &str = "ENSO_SNOWFLAKE_PASSWORD";
+    pub const ENSO_SNOWFLAKE_DATABASE: &str = "ENSO_SNOWFLAKE_DATABASE";
+    pub const ENSO_SNOWFLAKE_SCHEMA: &str = "ENSO_SNOWFLAKE_SCHEMA";
+    pub const ENSO_SNOWFLAKE_WAREHOUSE: &str = "ENSO_SNOWFLAKE_WAREHOUSE";
+
     // === Windows Code Signing ===
     /// Name of the GitHub Actions secret that stores path to the Windows code signing certificate
     /// within the runner.
@@ -682,6 +690,22 @@ pub fn engine_nightly() -> Result<Workflow> {
     Ok(workflow)
 }
 
+pub fn extra_nightly_tests() -> Result<Workflow> {
+    let on = Event {
+        // We start at running the tests daily at 3 am, but we may adjust to run it every few days
+        // or only once a week.
+        schedule: vec![Schedule::new("0 3 * * *")?],
+        workflow_dispatch: Some(manual_workflow_dispatch()),
+        ..default()
+    };
+    let mut workflow = Workflow { name: "Extra Nightly Tests".into(), on, ..default() };
+
+    // We run the extra tests only on Linux, as they should not contain any platform-specific
+    // behavior.
+    let target = (OS::Linux, Arch::X86_64);
+    workflow.add(target, job::SnowflakeTests {});
+    Ok(workflow)
+}
 
 pub fn engine_benchmark() -> Result<Workflow> {
     benchmark_workflow("Benchmark Engine", "backend benchmark runtime", Some(4 * 60))
@@ -751,6 +775,7 @@ pub fn generate(
         (repo_root.nightly_yml.to_path_buf(), nightly()?),
         (repo_root.scala_new_yml.to_path_buf(), backend()?),
         (repo_root.engine_nightly_yml.to_path_buf(), engine_nightly()?),
+        (repo_root.extra_nightly_tests_yml.to_path_buf(), extra_nightly_tests()?),
         (repo_root.gui_yml.to_path_buf(), gui()?),
         (repo_root.gui_tests_yml.to_path_buf(), gui_tests()?),
         (repo_root.engine_benchmark_yml.to_path_buf(), engine_benchmark()?),

build/build/src/ci_gen/job.rs

@@ -256,6 +256,57 @@ impl JobArchetype for StandardLibraryTests {
     }
 }
 
+#[derive(Clone, Copy, Debug)]
+pub struct SnowflakeTests {}
+
+const GRAAL_EDITION_FOR_EXTRA_TESTS: graalvm::Edition = graalvm::Edition::Community;
+
+impl JobArchetype for SnowflakeTests {
+    fn job(&self, target: Target) -> Job {
+        let job_name = "Snowflake Tests";
+        let mut job = RunStepsBuilder::new("backend test std-snowflake")
+            .customize(move |step| {
+                let main_step = step
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_ACCOUNT,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_ACCOUNT,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_USER,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_USER,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_PASSWORD,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_PASSWORD,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_DATABASE,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_DATABASE,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_SCHEMA,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_SCHEMA,
+                    )
+                    .with_secret_exposed_as(
+                        secret::ENSO_SNOWFLAKE_WAREHOUSE,
+                        crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_WAREHOUSE,
+                    );
+                vec![
+                    main_step,
+                    step::extra_stdlib_test_reporter(target, GRAAL_EDITION_FOR_EXTRA_TESTS),
+                ]
+            })
+            .build_job(job_name, target)
+            .with_permission(Permission::Checks, Access::Write);
+        job.env(env::GRAAL_EDITION, GRAAL_EDITION_FOR_EXTRA_TESTS);
+        job
+    }
+
+    fn key(&self, (os, arch): Target) -> String {
+        format!("{}-{os}-{arch}", self.id_key_base())
+    }
+}
+
 #[derive(Clone, Copy, Debug)]
 pub struct Lint;
 

build/build/src/ci_gen/step.rs

@@ -42,3 +42,10 @@ pub fn engine_test_reporter((os, arch): Target, graal_edition: graalvm::Edition)
     let path = format!("{}/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR));
     test_reporter(step_name, report_name, path)
 }
+
+pub fn extra_stdlib_test_reporter((os, arch): Target, graal_edition: graalvm::Edition) -> Step {
+    let step_name = "Extra Library Test Reporter";
+    let report_name = format!("Extra Library Tests Report ({graal_edition}, {os}, {arch})");
+    let path = format!("{}/*/*.xml", env_expression(&paths::ENSO_TEST_JUNIT_DIR));
+    test_reporter(step_name, report_name, path)
+}

build/build/src/engine.rs

@@ -97,6 +97,7 @@ pub enum Tests {
     Jvm,
     #[clap(alias = "stdlib")]
     StandardLibrary,
+    StdSnowflake,
 }
 
 impl Benchmarks {
@@ -118,7 +119,7 @@ pub struct BuildConfigurationFlags {
     /// Run JVM tests.
     pub test_jvm: bool,
     /// Whether the Enso standard library should be tested.
-    pub test_standard_library: bool,
+    pub test_standard_library: Option<StandardLibraryTestsSelection>,
     /// Whether benchmarks are compiled.
     ///
     /// Note that this does not run the benchmarks, only ensures that they are buildable.
@@ -147,6 +148,12 @@ pub struct BuildConfigurationFlags {
     pub verify_packages: bool,
 }
 
+#[derive(Clone, Debug)]
+pub enum StandardLibraryTestsSelection {
+    All,
+    Selected(Vec<String>),
+}
+
 impl From<BuildConfigurationFlags> for BuildConfigurationResolved {
     fn from(value: BuildConfigurationFlags) -> Self {
         Self::new(value)
@@ -170,7 +177,7 @@ impl BuildConfigurationResolved {
 
         // Check for components that require Enso Engine runner. Basically everything that needs to
         // run pure Enso code.
-        if config.test_standard_library
+        if config.test_standard_library.is_some()
             || config.execute_benchmarks.contains(&Benchmarks::Enso)
             || config.check_enso_benchmarks
         {
@@ -195,7 +202,7 @@ impl BuildConfigurationFlags {
         self.build_engine_package
             || self.build_launcher_bundle
             || self.build_project_manager_bundle
-            || self.test_standard_library
+            || self.test_standard_library.is_some()
             || self.build_native_runner
     }
 
@@ -206,13 +213,29 @@ impl BuildConfigurationFlags {
     pub fn build_launcher_package(&self) -> bool {
         self.build_launcher_package || self.build_launcher_bundle
     }
+
+    pub fn add_standard_library_test_selection(
+        &mut self,
+        selection: StandardLibraryTestsSelection,
+    ) {
+        use StandardLibraryTestsSelection::*;
+        let combined_selection = match (self.test_standard_library.take(), selection) {
+            (None, selection) => selection,
+            (Some(All), _) | (_, All) => All,
+            (Some(Selected(mut selection)), Selected(new_selection)) => {
+                selection.extend(new_selection);
+                Selected(selection)
+            }
+        };
+        self.test_standard_library = Some(combined_selection);
+    }
 }
 
 impl Default for BuildConfigurationFlags {
     fn default() -> Self {
         Self {
             test_jvm: false,
-            test_standard_library: false,
+            test_standard_library: None,
             build_benchmarks: false,
             check_enso_benchmarks: false,
             execute_benchmarks: default(),

build/build/src/engine/context.rs

@@ -243,7 +243,7 @@ impl RunContext {
 
 
         let _test_results_upload_guard =
-            if self.config.test_jvm || self.config.test_standard_library {
+            if self.config.test_jvm || self.config.test_standard_library.is_some() {
                 // If we run tests, make sure that old and new results won't end up mixed together.
                 let test_results_dir = ENSO_TEST_JUNIT_DIR
                     .get()
@@ -393,14 +393,16 @@ impl RunContext {
             Ok(())
         };
 
-        if self.config.test_standard_library {
-            enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS).await?;
+        match &self.config.test_standard_library {
+            Some(selection) => {
+                enso.run_tests(IrCaches::No, &sbt, PARALLEL_ENSO_TESTS, selection.clone()).await?;
+            }
+            None => {}
         }
 
         perhaps_test_java_generated_from_rust_job.await.transpose()?;
 
         // === Run benchmarks ===
-        debug!("Running benchmarks.");
         let build_benchmark_task = if self.config.build_benchmarks {
             let build_benchmark_task_names = [
                 "runtime-benchmarks/compile",
@@ -422,6 +424,7 @@ impl RunContext {
             build_benchmark_task.as_deref().into_iter().chain(execute_benchmark_tasks);
         let benchmark_command = Sbt::sequential_tasks(build_and_execute_benchmark_task);
         if !benchmark_command.is_empty() {
+            debug!("Running benchmarks.");
             sbt.call_arg(benchmark_command).await?;
         } else {
             debug!("No SBT tasks to run.");

build/build/src/enso.rs

@@ -1,3 +1,4 @@
+use crate::engine::StandardLibraryTestsSelection;
 use crate::prelude::*;
 
 use crate::paths::Paths;
@@ -113,6 +114,7 @@ impl BuiltEnso {
         ir_caches: IrCaches,
         sbt: &crate::engine::sbt::Context,
         async_policy: AsyncPolicy,
+        test_selection: StandardLibraryTestsSelection,
     ) -> Result {
         let paths = &self.paths;
         // Environment for meta-tests. See:
@@ -131,9 +133,22 @@ impl BuiltEnso {
             ide_ci::fs::write(google_api_test_data_dir.join("secret.json"), gdoc_key)?;
         }
 
+        let std_tests = match &test_selection {
+            StandardLibraryTestsSelection::All =>
+                crate::paths::discover_standard_library_tests(&paths.repo_root)?,
+            StandardLibraryTestsSelection::Selected(only) =>
+                only.iter().map(|test| paths.repo_root.test.join(test)).collect(),
+        };
+        let may_need_postgres = match &test_selection {
+            StandardLibraryTestsSelection::All => true,
+            StandardLibraryTestsSelection::Selected(only) =>
+                only.iter().any(|test| test.contains("Postgres_Tests")),
+        };
+
         let _httpbin = crate::httpbin::get_and_spawn_httpbin_on_free_port(sbt).await?;
+
         let _postgres = match TARGET_OS {
-            OS::Linux => {
+            OS::Linux if may_need_postgres => {
                 let runner_context_string = crate::env::ENSO_RUNNER_CONTAINER_NAME
                     .get_raw()
                     .or_else(|_| ide_ci::actions::env::RUNNER_NAME.get())
@@ -156,7 +171,6 @@ impl BuiltEnso {
             _ => None,
         };
 
-        let std_tests = crate::paths::discover_standard_library_tests(&paths.repo_root)?;
         let futures = std_tests.into_iter().map(|test_path| {
             let command = self.run_test(test_path, ir_caches);
             async move { command?.run_ok().await }

build/build/src/libraries_tests.rs

@@ -8,3 +8,17 @@ pub mod s3 {
         }
     }
 }
+
+pub mod snowflake {
+    /// Environment variables used inside of the Snowflake tests.
+    pub mod env {
+        ide_ci::define_env_var! {
+            ENSO_SNOWFLAKE_ACCOUNT, String;
+            ENSO_SNOWFLAKE_USER, String;
+            ENSO_SNOWFLAKE_PASSWORD, String;
+            ENSO_SNOWFLAKE_DATABASE, String;
+            ENSO_SNOWFLAKE_SCHEMA, String;
+            ENSO_SNOWFLAKE_WAREHOUSE, String;
+        }
+    }
+}

build/build/src/rust/parser.rs

@@ -69,7 +69,6 @@ pub async fn run_self_tests(repo_root: &RepoRoot) -> Result {
         .output_ok()
         .await?
         .into_stdout_string()?;
-    trace!("Generated test code:\n{tests_code}");
     ide_ci::fs::tokio::write(&test, tests_code).await?;
 
     Javac

build/cli/src/lib.rs

@@ -36,6 +36,7 @@ use enso_build::config::Config;
 use enso_build::context::BuildContext;
 use enso_build::engine::context::EnginePackageProvider;
 use enso_build::engine::Benchmarks;
+use enso_build::engine::StandardLibraryTestsSelection;
 use enso_build::engine::Tests;
 use enso_build::paths::TargetTriple;
 use enso_build::project;
@@ -413,11 +414,21 @@ impl Processor {
                 let mut config = enso_build::engine::BuildConfigurationFlags::default();
                 for arg in which {
                     match arg {
-                        Tests::Jvm => config.test_jvm = true,
-                        Tests::StandardLibrary => config.test_standard_library = true,
+                        Tests::Jvm => {
+                            config.test_jvm = true;
+                            // We also test the Java parser integration when running the JVM tests.
+                            config.test_java_generated_from_rust = true;
+                        }
+                        Tests::StandardLibrary => config.add_standard_library_test_selection(
+                            StandardLibraryTestsSelection::All,
+                        ),
+                        Tests::StdSnowflake => config.add_standard_library_test_selection(
+                            StandardLibraryTestsSelection::Selected(vec![
+                                "Snowflake_Tests".to_string()
+                            ]),
+                        ),
                     }
                 }
-                config.test_java_generated_from_rust = true;
                 let context = self.prepare_backend_context(config);
                 async move { context.await?.build().void_ok().await }.boxed()
             }

test/Snowflake_Tests/src/Snowflake_Spec.enso

@@ -111,11 +111,11 @@ snowflake_specific_spec suite_builder default_connection db_name setup =
             connection.schemas.find (name-> name.equals_ignore_case "public") . should_succeed
             Meta.is_same_object connection (connection.set_schema "public") . should_be_true
 
-        group_builder.specify "should allow changing schema" pending="TODO?" <|
+        group_builder.specify "should allow changing schema" <|
             connection = default_connection.get
             new_connection = connection.set_schema "information_schema"
             new_schema = new_connection.read (SQL_Query.Raw_SQL "SELECT current_schema()") . at 0 . to_vector . first
-            new_schema . should_equal "information_schema"
+            new_schema . should_equal "INFORMATION_SCHEMA"
 
         group_builder.specify "should allow changing database" <|
             connection = default_connection.get
@@ -331,39 +331,35 @@ snowflake_specific_spec suite_builder default_connection db_name setup =
             r4.catch.to_display_text . should_contain "The name '' is invalid"
 
     suite_builder.group "[Snowflake] Edge Cases" group_builder->
-        group_builder.specify "materialize should respect the overridden type" pending="TODO" <|
+        group_builder.specify "materialize should respect the overridden type" <|
             t0 = table_builder [["x", [False, True, False]], ["A", ["a", "b", "c"]], ["B", ["xyz", "abc", "def"]]]
-            t1 = t0 . cast "A" (Value_Type.Char size=1 variable_length=False) . cast "B" (Value_Type.Char size=3 variable_length=False)
+            t1 = t0 . cast "A" (Value_Type.Char size=1) . cast "B" (Value_Type.Char size=3)
 
             x = t1.at "x"
             a = t1.at "A"
             b = t1.at "B"
-            a.value_type.should_equal (Value_Type.Char size=1 variable_length=False)
-            b.value_type.should_equal (Value_Type.Char size=3 variable_length=False)
+            a.value_type.should_equal (Value_Type.Char size=1)
+            b.value_type.should_equal (Value_Type.Char size=3)
 
             c = x.iif a b
-            c.to_vector.should_equal ["xyz", "b", "def"]
-            Test.with_clue "c.value_type="+c.value_type.to_display_text+": " <|
-                c.value_type.variable_length.should_be_true
+            c.to_vector.should_equal_ignoring_order ["xyz", "b", "def"]
+            # The max length is lost after the IIF
+            c.value_type.should_equal Value_Type.Char
 
             d = materialize c
-            d.to_vector.should_equal ["xyz", "b", "def"]
-            Test.with_clue "d.value_type="+d.value_type.to_display_text+": " <|
-                d.value_type.variable_length.should_be_true
+            d.to_vector.should_equal_ignoring_order ["xyz", "b", "def"]
+            d.value_type.should_equal Value_Type.Char
 
-        group_builder.specify "should be able to round-trip a BigInteger column" pending="TODO" <|
+        group_builder.specify "should be able to round-trip a BigInteger column" <|
             x = 2^70
             m1 = Table.new [["X", [10, x]]]
             m1.at "X" . value_type . should_be_a (Value_Type.Decimal ...)
 
             t1 = m1.select_into_database_table default_connection.get (Name_Generator.random_name "BigInteger") primary_key=[] temporary=True
-            t1.at "X" . value_type . should_be_a (Value_Type.Decimal ...)
-            t1.at "X" . value_type . scale . should_equal 0
-            # TODO revise
-            t1.at "X" . value_type . precision . should_equal 1000
+            t1.at "X" . value_type . should_equal (Value_Type.Decimal 38 0)
             w1 = Problems.expect_only_warning Inexact_Type_Coercion t1
             w1.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=0)
-            w1.actual_type . should_equal (Value_Type.Decimal precision=1000 scale=0)
+            w1.actual_type . should_equal (Value_Type.Decimal precision=38 scale=0)
 
             v1x = t1.at "X" . to_vector
             v1x.should_equal [10, x]
@@ -372,80 +368,53 @@ snowflake_specific_spec suite_builder default_connection db_name setup =
             t2 = t1.set (expr "[X] + 10") "Y"
             t2.at "X" . value_type . should_be_a (Value_Type.Decimal ...)
             t2.at "Y" . value_type . should_be_a (Value_Type.Decimal ...)
-            # TODO revise
-            t2.at "Y" . value_type . scale . should_equal Nothing
-            t2.at "X" . to_vector . should_equal [10, x]
+            t2.at "X" . to_vector . should_equal_ignoring_order [10, x]
+
             # Only works by approximation:
-            t2.at "Y" . to_vector . should_equal [20, x+10]
-            t2.at "Y" . cast Value_Type.Char . to_vector . should_equal ["20", (x+10).to_text]
+            t2.at "Y" . to_vector . should_equal_ignoring_order [20, x+10]
+            t2.at "Y" . cast Value_Type.Char . to_vector . should_equal_ignoring_order ["20", (x+10).to_text]
 
             m2 = t2.remove_warnings.read
             m2.at "X" . value_type . should_be_a (Value_Type.Decimal ...)
-            # As noted above - once operations are performed, the scale=0 may be lost and the column will be approximated as a float.
-            m2.at "Y" . value_type . should_equal Value_Type.Float
-            m2.at "X" . to_vector . should_equal [10, x]
-            m2.at "Y" . to_vector . should_equal [20, x+10]
-            w2 = Problems.expect_only_warning Inexact_Type_Coercion m2
-            w2.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
-            w2.actual_type . should_equal Value_Type.Float
+            m2.at "Y" . value_type . should_be_a (Value_Type.Decimal ...)
+            m2.at "X" . to_vector . should_equal_ignoring_order [10, x]
+            m2.at "Y" . to_vector . should_equal_ignoring_order [20, x+10]
 
             # This has more than 1000 digits.
             super_large = 11^2000
             m3 = Table.new [["X", [super_large]]]
             m3.at "X" . value_type . should_be_a (Value_Type.Decimal ...)
-            t3 = m3.select_into_database_table default_connection.get (Name_Generator.random_name "BigInteger2") primary_key=[] temporary=True
-            t3 . at "X" . value_type . should_be_a (Value_Type.Decimal ...)
-            # If we exceed the 1000 digits precision, we cannot enforce neither scale nor precision anymore.
-            t3 . at "X" . value_type . precision . should_equal Nothing
-            t3 . at "X" . value_type . scale . should_equal Nothing
-            # Works but only relying on imprecise float equality:
-            t3 . at "X" . to_vector . should_equal [super_large]
-            w3 = Problems.expect_only_warning Inexact_Type_Coercion t3
-            w3.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=0)
-            w3.actual_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
+            r3 = m3.select_into_database_table default_connection.get (Name_Generator.random_name "BigInteger2") primary_key=[] temporary=True
+            # Snowflake fails to process such a huge number
+            r3.should_fail_with SQL_Error
+            r3.catch.to_display_text . should_contain "Numeric value"
+            r3.catch.to_display_text . should_contain "is not recognized"
 
-            m4 = t3.remove_warnings.read
-            # Because we no longer have a set scale, we cannot get a BigInteger column back - we'd need BigDecimal, but that is not fully supported yet in Enso - so we get the closest approximation - the imprecise Float.
-            m4 . at "X" . value_type . should_equal Value_Type.Float
-            m4 . at "X" . to_vector . should_equal [super_large]
-            w4 = Problems.expect_only_warning Inexact_Type_Coercion m4
-            w4.requested_type . should_equal (Value_Type.Decimal precision=Nothing scale=Nothing)
-            w4.actual_type . should_equal Value_Type.Float
-
-        group_builder.specify "should round-trip timestamptz column, preserving instant but converting to UTC" pending="TODO" <|
+        group_builder.specify "should round-trip timestamptz column, preserving instant but converting to UTC" <|
             table_name = Name_Generator.random_name "TimestampTZ"
-            table = default_connection.get.create_table table_name [Column_Description.Value "A" (Value_Type.Date_Time with_timezone=True)] primary_key=[]
+            table = default_connection.get.create_table table_name [Column_Description.Value "A" (Value_Type.Date_Time with_timezone=True), Column_Description.Value "rowid" Value_Type.Integer] primary_key=[]
 
-            dt1 = Date_Time.new 2022 05 04 15 30
-            dt2 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.utc)
-            dt3 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.parse "US/Hawaii")
-            dt4 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.parse "Europe/Warsaw")
-            v = [dt1, dt2, dt3, dt4]
+            dt1 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.utc)
+            dt2 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.parse "US/Hawaii")
+            dt3 = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.parse "Europe/Warsaw")
+            v = [dt1, dt2, dt3]
 
             Problems.assume_no_problems <|
-                table.update_rows (Table.new [["A", v]]) update_action=Update_Action.Insert
+                table.update_rows (Table.new [["A", v], ["rowid", (0.up_to v.length).to_vector]]) update_action=Update_Action.Insert
 
-            tz_zero = Time_Zone.parse "Z"
-            v_at_z = v.map dt-> dt.at_zone tz_zero
-            table.at "A" . to_vector . should_equal v_at_z
-            table.at "A" . to_vector . should_equal_tz_agnostic v
+            returned_v =
+                dt1_offset = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.new 0)
+                dt2_offset = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.new -10)
+                dt3_offset = Date_Time.new 2022 05 04 15 30 zone=(Time_Zone.new 2)
+                [dt1_offset, dt2_offset, dt3_offset]
+            table.sort "rowid" . at "A" . to_vector . should_equal returned_v
 
             ## We also check how the timestamp column behaves with interpolations:
-               (See analogous test showing a less nice behaviour without timezones below.)
-            v.each my_dt-> Test.with_clue my_dt.to_text+" " <|
-                # Checks if the two date-times represent the same instant in time.
-                is_same_time_instant dt1 dt2 =
-                    dt1.at_zone tz_zero == dt2.at_zone tz_zero
-                local_equals = v.filter (is_same_time_instant my_dt)
-                # Depending on test runner's timezone, the `my_dt` may be equal to 1 or 2 entries in `v`.
-                [1, 2].should_contain local_equals.length
-
-                Test.with_clue " == "+local_equals.to_text+": " <|
+            v.each my_dt-> Test.with_clue "("+my_dt.to_text+") " <|
                     t2 = table.filter "A" (Filter_Condition.Equal to=my_dt)
-                    t2.row_count . should_equal local_equals.length
-                    t2.at "A" . to_vector . should_equal_tz_agnostic local_equals
+                    t2.row_count . should_equal 1
 
-        group_builder.specify "will round-trip timestamp column without timezone by converting it to UTC" pending="TODO" <|
+        group_builder.specify "will round-trip timestamp column without timezone by converting it to UTC" <|
             table_name = Name_Generator.random_name "Timestamp"
             table = default_connection.get.create_table table_name [Column_Description.Value "A" (Value_Type.Date_Time with_timezone=False)] primary_key=[]
             Problems.assume_no_problems table
@@ -466,7 +435,7 @@ snowflake_specific_spec suite_builder default_connection db_name setup =
 
             # When uploading we want to just strip the timezone information and treat every timestamp as LocalDateTime.
             # This is verified by checking the text representation in the DB: it should show the same local time in all 4 cases, regardless of original timezone.
-            local_dt = "2022-05-04 15:30:00"
+            local_dt = "2022-05-04 15:30:00.000000000"
             table.at "A" . cast Value_Type.Char . to_vector . should_equal [local_dt, local_dt, local_dt, local_dt]
 
             # Then when downloaded, it should be interpreted at the 'system default' timezone.
@@ -477,35 +446,11 @@ snowflake_specific_spec suite_builder default_connection db_name setup =
             Problems.assume_no_problems materialized_table
 
             # We also check how the timestamp column behaves with interpolations:
+            # Given that we lost timezone - all entries match.
             v.each my_dt-> Test.with_clue my_dt.to_text+": " <|
                 t2 = table.filter "A" (Filter_Condition.Equal to=my_dt)
-                is_in_system_tz = my_dt.at_zone Time_Zone.system . time_of_day == my_dt.time_of_day
-                ## Unfortunately, this will work in the following way:
-                   - if the date-time represented as local time in the current system default timezone is 15:30,
-                     then it will match _all_ entries (as they do not have a timezone).
-                   - otherwise, the date-time is converted into UTC before being passed to the Database,
-                     and only then the timezone is stripped - so the local time is actually shifted.
-
-                   This is not ideal - ideally we'd want the local date time to be extracted from the timestamp directly,
-                   before any date conversions happen - this way in _all_ 4 cases we would get 15:30 local time
-                   and all rows would always be matching.
-
-                   That logic is applied when uploading a table. However, in custom queries, we do not currently have
-                   enough metadata to infer that the Date_Time that is being passed to a given `?` hole in the query
-                   should be treated as a local date-time or a zoned date-time.
-                   Thus, we pass it as zoned by default to avoid losing information - and that triggers the conversion
-                   on the Database side. If we want to change that, we would need to add metadata within our operations,
-                   so that an operation like `==` will infer the expected type of the `?` hole based on the type of the
-                   second operand.
-                case is_in_system_tz of
-                    True ->
-                        my_dt.at_zone Time_Zone.system . time_of_day . to_display_text . should_equal "15:30:00"
-                        t2.row_count . should_equal 4
-                        t2.at "A" . to_vector . should_equal [dt1, dt1, dt1, dt1]
-                    False ->
-                        my_dt.at_zone Time_Zone.system . time_of_day . to_display_text . should_not_equal "15:30:00"
-                        t2.row_count . should_equal 0
-                        t2.at "A" . to_vector . should_equal []
+                t2.row_count . should_equal 4
+                t2.at "A" . to_vector . should_equal [dt1, dt1, dt1, dt1]
 
     suite_builder.group "[Snowflake] math functions" group_builder->
         group_builder.specify "round, trunc, ceil, floor" <|

test/Table_Tests/src/Common_Table_Operations/Missing_Values_Spec.enso

@@ -253,7 +253,7 @@ add_specs suite_builder setup =
                 mix_filled.to_vector . should_equal [1, 0, 2, 0]
                 mix_filled.value_type . should_equal Value_Type.Mixed
 
-        group_builder.specify "should correctly unify text columns of various lengths" pending=(if setup.test_selection.fixed_length_text_columns.not then "Fixed-length Char columns are not supported by this backend.") <|
+        group_builder.specify "should correctly unify text columns of various fixed lengths" pending=(if setup.test_selection.fixed_length_text_columns.not then "Fixed-length Char columns are not supported by this backend.") <|
             t0 = table_builder [["A", ["a", Nothing, "c"]], ["B", ["X", "Y", "Z"]], ["C", ["xyz", "abc", "def"]]]
             t1 = t0 . cast "A" (Value_Type.Char size=1 variable_length=False) . cast "B" (Value_Type.Char size=1 variable_length=False) . cast "C" (Value_Type.Char size=3 variable_length=False)
 
@@ -273,6 +273,31 @@ add_specs suite_builder setup =
             e.value_type.should_be_a (Value_Type.Char ...)
             Test.with_clue "e.value_type="+e.value_type.to_display_text+": " <|
                 e.value_type.variable_length.should_be_true
+                max_len = e.value_type.size
+                (max_len.is_nothing || (max_len.integer >= 3)).should_be_true
+
+        group_builder.specify "should correctly unify text columns of various varying lengths" pending=(if setup.test_selection.text_length_limited_columns.not then "Length-limited Char columns are not supported by this backend.") <|
+            t0 = table_builder [["A", ["a", Nothing, "c"]], ["B", ["X", "Y", "Z"]], ["C", ["xyz", "abc", "def"]]]
+            t1 = t0 . cast "A" (Value_Type.Char size=1 variable_length=True) . cast "B" (Value_Type.Char size=1 variable_length=True) . cast "C" (Value_Type.Char size=3 variable_length=True)
+
+            a = t1.at "A"
+            b = t1.at "B"
+            c = t1.at "C"
+            a.value_type.should_equal (Value_Type.Char size=1 variable_length=True)
+            b.value_type.should_equal (Value_Type.Char size=1 variable_length=True)
+            c.value_type.should_equal (Value_Type.Char size=3 variable_length=True)
+
+            d = a.fill_nothing b
+            d.to_vector . should_equal ["a", "Y", "c"]
+            d.value_type . should_equal (Value_Type.Char size=1 variable_length=True)
+
+            e = a.fill_nothing c
+            e.to_vector . should_equal ["a", "abc", "c"]
+            e.value_type.should_be_a (Value_Type.Char ...)
+            Test.with_clue "e.value_type="+e.value_type.to_display_text+": " <|
+                e.value_type.variable_length.should_be_true
+                max_len = e.value_type.size
+                (max_len.is_nothing || (max_len.integer >= 3)).should_be_true
 
         group_builder.specify "should allow setting a default column by reference" <|
             t = table_builder [["A", ["x", "", Nothing]], ["B", ["a", "b", "c"]], ["C", [Nothing, Nothing, "ZZZ"]], ["D", [Nothing, "2", "3"]]]