From 88aaa5134133b3756ac9ce7493aaafebefa9ef5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Wa=C5=9Bko?= Date: Wed, 4 Sep 2024 18:06:54 +0200 Subject: [PATCH] Workaround missing `aws` CLI on self-hosted runners by running the Cloud tests on GH runners (#10977) - Followup of #10964 - I assumed `aws` CLI is available, because it is on GH runners. But we are running on self-hosted by default. - Ideally we should make the CLI available there and switch back. - But for now, trying to run on the GH runner. --- .github/workflows/extra-nightly-tests.yml | 10 ++---- build/build/src/ci_gen.rs | 2 +- build/build/src/ci_gen/job.rs | 42 +++++++++++++++++++---- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.github/workflows/extra-nightly-tests.yml b/.github/workflows/extra-nightly-tests.yml index f74abe4aa32..e0696b442e7 100644 --- a/.github/workflows/extra-nightly-tests.yml +++ b/.github/workflows/extra-nightly-tests.yml @@ -44,11 +44,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./run backend test std-snowflake env: - ENSO_CLOUD_COGNITO_REGION: ${{ secrets.ENSO_CLOUD_COGNITO_REGION }} - ENSO_CLOUD_COGNITO_USER_POOL_ID: ${{ secrets.ENSO_CLOUD_COGNITO_USER_POOL_ID }} - ENSO_CLOUD_COGNITO_USER_POOL_WEB_CLIENT_ID: ${{ secrets.ENSO_CLOUD_COGNITO_USER_POOL_WEB_CLIENT_ID }} - ENSO_CLOUD_TEST_ACCOUNT_PASSWORD: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_PASSWORD }} - ENSO_CLOUD_TEST_ACCOUNT_USERNAME: ${{ secrets.ENSO_CLOUD_TEST_ACCOUNT_USERNAME }} ENSO_SNOWFLAKE_ACCOUNT: ${{ secrets.ENSO_SNOWFLAKE_ACCOUNT }} ENSO_SNOWFLAKE_DATABASE: ${{ secrets.ENSO_SNOWFLAKE_DATABASE }} ENSO_SNOWFLAKE_PASSWORD: ${{ secrets.ENSO_SNOWFLAKE_PASSWORD }} @@ -81,10 +76,9 @@ jobs: permissions: checks: write enso-build-ci-gen-job-standard-library-tests-graal-vm-ce-linux-amd64: - name: Standard Library Tests (GraalVM CE) (linux, amd64) + name: Standard Library Tests (GraalVM CE) (LinuxLatest) runs-on: - - self-hosted - - Linux + - ubuntu-latest steps: - if: startsWith(runner.name, 'GitHub Actions') || startsWith(runner.name, 'Hosted Agent') name: Installing wasm-pack diff --git a/build/build/src/ci_gen.rs b/build/build/src/ci_gen.rs index b7c2132e032..f70ae884940 100644 --- a/build/build/src/ci_gen.rs +++ b/build/build/src/ci_gen.rs @@ -707,7 +707,7 @@ pub fn extra_nightly_tests() -> Result { // We run the extra tests only on Linux, as they should not contain any platform-specific // behavior. - let target = (OS::Linux, Arch::X86_64); + let target = PRIMARY_TARGET; workflow.add(target, job::SnowflakeTests {}); workflow.add(target, job::StandardLibraryTests { graal_edition: graalvm::Edition::Community, diff --git a/build/build/src/ci_gen/job.rs b/build/build/src/ci_gen/job.rs index d03880dfecf..9e273205124 100644 --- a/build/build/src/ci_gen/job.rs +++ b/build/build/src/ci_gen/job.rs @@ -245,8 +245,8 @@ impl JobArchetype for StandardLibraryTests { let graal_edition = self.graal_edition; let should_enable_cloud_tests = self.cloud_tests_enabled; let job_name = format!("Standard Library Tests ({graal_edition})"); - let mut job = RunStepsBuilder::new("backend test standard-library") - .customize(move |step| { + let run_steps_builder = + RunStepsBuilder::new("backend test standard-library").customize(move |step| { let main_step = step .with_secret_exposed_as( secret::ENSO_LIB_S3_AWS_REGION, @@ -268,9 +268,14 @@ impl JobArchetype for StandardLibraryTests { }; vec![updated_main_step, step::stdlib_test_reporter(target, graal_edition)] - }) - .build_job(job_name, target) - .with_permission(Permission::Checks, Access::Write); + }); + let mut job = build_job_ensuring_cloud_tests_run_on_github( + run_steps_builder, + target, + &job_name, + should_enable_cloud_tests, + ) + .with_permission(Permission::Checks, Access::Write); match graal_edition { graalvm::Edition::Community => job.env(env::GRAAL_EDITION, graalvm::Edition::Community), graalvm::Edition::Enterprise => @@ -288,6 +293,26 @@ impl JobArchetype for StandardLibraryTests { } } +/** This is a temporary workaround. + * + * The Cloud tests preparation requires `aws` CLI to be installed on the machine. + * The GitHub hosted runners have it, but our self-hosted runners do not. + * To fix this we either need to modify self-hosted runners to provide the AWS CLI or change the + * way we prepare the Cloud tests to not require it. + */ +fn build_job_ensuring_cloud_tests_run_on_github( + run_steps_builder: RunStepsBuilder, + target: Target, + job_name: &str, + cloud_tests_enabled: bool, +) -> Job { + if cloud_tests_enabled { + run_steps_builder.build_job(job_name, RunnerLabel::LinuxLatest) + } else { + run_steps_builder.build_job(job_name, target) + } +} + #[derive(Clone, Copy, Debug)] pub struct SnowflakeTests {} @@ -324,10 +349,13 @@ impl JobArchetype for SnowflakeTests { crate::libraries_tests::snowflake::env::ENSO_SNOWFLAKE_WAREHOUSE, ); - let updated_main_step = enable_cloud_tests(main_step); + // Temporarily disabled until we can get the Cloud auth fixed. + // Snowflake does not rely on cloud anyway, so it can be disabled. + // But it will rely once we add datalink tests, so this should be fixed soon. + // let updated_main_step = enable_cloud_tests(main_step); vec![ - updated_main_step, + main_step, step::extra_stdlib_test_reporter(target, GRAAL_EDITION_FOR_EXTRA_TESTS), ] })