Merge pull request #21109 from atom-ide-community/windows_tests_4upstream

Parallelize Tests in CI (+ Refactor and Improvements)
2024-09-20 07:28:08 +03:00 · 2020-09-16 13:56:39 +03:00 · 2020-09-16 13:56:39 +03:00 · b94532ed4d
commit b94532ed4d
parent b17a5ab6f6 3dbe763e14
7 changed files with 221 additions and 96 deletions
--- a/script/test
+++ b/script/test
@ -65,7 +65,7 @@ if (process.platform === 'darwin') {
  assertExecutablePaths(executablePaths)
  executablePath = executablePaths[0]
 } else {
-  throw new Error('Running tests on this platform is not supported.')
+  throw new Error('##[error] Running tests on this platform is not supported.')
 }

 function prepareEnv (suiteName) {
@ -83,6 +83,39 @@ function prepareEnv (suiteName) {
  return env
 }

+function spawnTest(executablePath, testArguments, options, callback, testName, finalize = null) {
+  const cp = childProcess.spawn(executablePath, testArguments, options)
+
+  // collect outputs and errors
+	let stderrOutput = ''
+	if (cp.stdout) {
+	  cp.stderr.on('data', data => { stderrOutput += data })
+	  cp.stdout.on('data', data => { stderrOutput += data })
+	}
+
+  // on error
+  cp.on('error', error => {
+    console.log(error, "error")
+    if (finalize) { finalize() } 		// if finalizer provided
+    callback(error)
+  })
+
+	// on close
+  cp.on('close', exitCode => {
+    if (exitCode !== 0) {
+      console.log(`##[error] Tests for ${testName} failed.`.red)
+      console.log(stderrOutput)
+    }
+    if (finalize) { finalize() } 		// if finalizer provided
+    callback(null, {
+      exitCode,
+      step: testName,
+      testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`,
+    })
+  })
+
+}
+
 function runCoreMainProcessTests (callback) {
  const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process')
  const testArguments = [
@ -96,26 +129,35 @@ function runCoreMainProcessTests (callback) {

  const testEnv = Object.assign({}, prepareEnv('core-main-process'), {ATOM_GITHUB_INLINE_GIT_EXEC: 'true'})

-  console.log('Executing core main process tests'.bold.green)
-  const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
-  cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-main-process'}) })
+  console.log('##[command] Executing core main process tests'.bold.green)
+  spawnTest(executablePath, testArguments, {stdio: 'inherit', env: testEnv}, callback, 'core-main-process')
 }

-function runCoreRenderProcessTests (callback) {
-  const testPath = path.join(CONFIG.repositoryRootPath, 'spec')
-  const testArguments = [
-    '--resource-path', resourcePath,
-    '--test', testPath
-  ]
-  const testEnv = prepareEnv('core-render-process')
+function getCoreRenderProcessTestSuites() {

-  console.log('Executing core render process tests'.bold.green)
-  const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
-  cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-render-process'}) })
+// Build an array of functions, each running tests for a different rendering test
+const coreRenderProcessTestSuites = []
+const testPath = path.join(CONFIG.repositoryRootPath, 'spec')
+let testFiles = glob.sync(path.join(testPath, '*-spec.+(js|coffee|ts|jsx|tsx|mjs)'))
+for (let testFile of testFiles) {
+	const testArguments = [
+		'--resource-path', resourcePath,
+		'--test', testFile
+	]
+	// the function which runs by async:
+  coreRenderProcessTestSuites.push( function (callback) {
+    const testEnv = prepareEnv('core-render-process')
+    console.log(`##[command] Executing core render process tests for ${testFile}`.bold.green)
+    spawnTest(executablePath, testArguments, {env: testEnv}, callback, `core-render-process in ${testFile}.`)
+  })
 }

+	return coreRenderProcessTestSuites
+}
+
+
+function getPackageTestSuites() {
+
 // Build an array of functions, each running tests for a different bundled package
 const packageTestSuites = []
 for (let packageName in CONFIG.appMetadata.packageDependencies) {
@ -134,18 +176,20 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) {

  const testFolder = path.join(repositoryPackagePath, testSubdir)

-  packageTestSuites.push(function (callback) {
-    const testArguments = [
-      '--resource-path', resourcePath,
-      '--test', testFolder
-    ]
-    const testEnv = prepareEnv(`bundled-package-${packageName}`)
+	const testArguments = [
+		'--resource-path', resourcePath,
+		'--test', testFolder
+	]

-    const pkgJsonPath = path.join(repositoryPackagePath, 'package.json')
-    const nodeModulesPath = path.join(repositoryPackagePath, 'node_modules')
+	const pkgJsonPath = path.join(repositoryPackagePath, 'package.json')
+	const nodeModulesPath = path.join(repositoryPackagePath, 'node_modules')
+
+	// the function which runs by async:
+  packageTestSuites.push(function (callback) {
+    const testEnv = prepareEnv(`bundled-package-${packageName}`)
    let finalize = () => null
    if (require(pkgJsonPath).atomTestRunner) {
-      console.log(`Installing test runner dependencies for ${packageName}`.bold.green)
+      console.log(`##[command] Installing test runner dependencies for ${packageName}`.bold.green)
      if (fs.existsSync(nodeModulesPath)) {
        const backup = backupNodeModules(repositoryPackagePath)
        finalize = backup.restore
@ -153,91 +197,105 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) {
        finalize = () => fs.removeSync(nodeModulesPath)
      }
      runApmInstall(repositoryPackagePath)
-      console.log(`Executing ${packageName} tests`.green)
+      console.log(`##[command] Executing ${packageName} tests`.green)
    } else {
-      console.log(`Executing ${packageName} tests`.bold.green)
+      console.log(`##[command] Executing ${packageName} tests`.bold.green)
    }
-    const cp = childProcess.spawn(executablePath, testArguments, {env: testEnv})
-
-    let stderrOutput = ''
-    cp.stderr.on('data', data => { stderrOutput += data })
-    cp.stdout.on('data', data => { stderrOutput += data })
-    cp.on('error', error => {
-      console.log(error, "error")
-      finalize()
-      callback(error)
-    })
-    cp.on('close', exitCode => {
-      if (exitCode !== 0) {
-        console.log(`Package tests failed for ${packageName}:`.red)
-        console.log(stderrOutput)
-      }
-      finalize()
-      callback(null, {exitCode, step: `package-${packageName}`})
-    })
+    spawnTest(executablePath, testArguments, {env: testEnv}, callback, `${packageName} package`, finalize)
  })
 }

+	return packageTestSuites
+}
+
+
 function runBenchmarkTests (callback) {
  const benchmarksPath = path.join(CONFIG.repositoryRootPath, 'benchmarks')
  const testArguments = ['--benchmark-test', benchmarksPath]
  const testEnv = prepareEnv('benchmark')

-  console.log('Executing benchmark tests'.bold.green)
+  console.log('##[command] Executing benchmark tests'.bold.green)
  const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
-  cp.on('error', error => { callback(error) })
-  cp.on('close', exitCode => { callback(null, {exitCode, step: 'core-benchmarks'}) })
+  spawnTest(executablePath, testArguments, {stdio: 'inherit', env: testEnv}, callback, `core-benchmarks`)
 }

-let testSuitesToRun = requestedTestSuites() || testSuitesForPlatform(process.platform)
+let testSuitesToRun = requestedTestSuites(process.platform)

-function requestedTestSuites () {
-  const suites = []
-  if (argv.coreMain) {
-    suites.push(runCoreMainProcessTests)
-  }
-  if (argv.coreRenderer) {
-    suites.push(runCoreRenderProcessTests)
-  }
-  if (argv.coreBenchmark) {
-    suites.push(runBenchmarkTests)
-  }
-  if (argv.package) {
-    suites.push(...packageTestSuites)
-  }
-  return suites.length > 0 ? suites : null
-}
+function requestedTestSuites (platform) {
+  // env variable or argv options
+  let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true'
+  let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' || argv.coreMain
+  let coreRenderer = argv.coreRenderer || process.env.ATOM_RUN_CORE_RENDER_TESTS == 'true'
+  let coreRenderer1 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '1'
+  let coreRenderer2 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '2'
+  let packageAll = argv.package || process.env.ATOM_RUN_PACKAGE_TESTS == 'true'
+  let packages1 = process.env.ATOM_RUN_PACKAGE_TESTS === '1'
+  let packages2 = process.env.ATOM_RUN_PACKAGE_TESTS === '2'
+  let benchmark = argv.coreBenchmark
+
+  // Operating system overrides:
+  coreMain = coreMain || (platform === 'linux') || (platform === 'win32' && process.arch === 'x86')
+
+  // split package tests (used for macos in CI)
+  const PACKAGES_TO_TEST_IN_PARALLEL = 23
+  // split core render test (used for windows x64 in CI)
+  const CORE_RENDER_TO_TEST_IN_PARALLEL = 45

-function testSuitesForPlatform (platform) {
  let suites = []
-  switch (platform) {
-    case 'darwin':
-      const PACKAGES_TO_TEST_IN_PARALLEL = 23
+  // Core tess
+  if (coreAll) {
+    suites.push(...[runCoreMainProcessTests, ...getCoreRenderProcessTestSuites()])
+  } else {

-      if (process.env.ATOM_RUN_CORE_TESTS === 'true') {
-        suites = [runCoreMainProcessTests, runCoreRenderProcessTests]
-      } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') {
-        suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL)
-      } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') {
-        suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL)
-      } else {
-        suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites)
+    // Core main tests
+    if (coreMain) {
+      suites.push(runCoreMainProcessTests)
+    }
+
+    // Core renderer tests
+    if (coreRenderer) {
+      suites.push(...getCoreRenderProcessTestSuites())
+    } else {
+      // split
+      if (coreRenderer1) {
+        suites.push(...getCoreRenderProcessTestSuites().slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL))
      }
-      break
-    case 'win32':
-      suites = (process.arch === 'x64') ? [runCoreMainProcessTests, runCoreRenderProcessTests] : [runCoreMainProcessTests]
-      break
-    case 'linux':
-      suites = [runCoreMainProcessTests]
-      break
-    default:
-      console.log(`Unrecognized platform: ${platform}`)
+      if (coreRenderer2) {
+        suites.push(...getCoreRenderProcessTestSuites().slice(CORE_RENDER_TO_TEST_IN_PARALLEL))
+      }
+    }
+
+  }
+
+  // Package tests
+	if (packageAll) {
+	  suites.push(...getPackageTestSuites())
+  } else {
+    // split
+    if (packages1) {
+      suites.push(...getPackageTestSuites().slice(0, PACKAGES_TO_TEST_IN_PARALLEL))
+    }
+    if (packages2) {
+      suites.push(...getPackageTestSuites().slice(PACKAGES_TO_TEST_IN_PARALLEL))
+    }
+  }
+
+  // Benchmark tests
+  if (benchmark) {
+    suites.push(runBenchmarkTests)
  }

  if (argv.skipMainProcessTests) {
    suites = suites.filter(suite => suite !== runCoreMainProcessTests)
  }

+  // Remove duplicates
+  suites = Array.from(new Set(suites))
+
+  if (suites.length == 0) {
+    throw new Error("No tests was requested")
+  }
+
  return suites
 }

@ -248,10 +306,14 @@ async.series(testSuitesToRun, function (err, results) {
  } else {
    const failedSteps = results.filter(({exitCode}) => exitCode !== 0)

-    for (const {step} of failedSteps) {
-      console.error(`Error! The '${step}' test step finished with a non-zero exit code`)
+    if (failedSteps.length > 0) {
+			console.warn("\n \n ##[error] *** Reporting the errors that happened in all of the tests: *** \n \n")
+      for (const {step, testCommand} of failedSteps) {
+        console.error(`##[error] The '${step}' test step finished with a non-zero exit code \n ${testCommand}`)
+      }
+      process.exit(1)
    }

-    process.exit(failedSteps.length === 0 ? 0 : 1)
+    process.exit(0)
  }
 })
--- a/script/vsts/nightly-release.yml
+++ b/script/vsts/nightly-release.yml
@ -16,7 +16,7 @@ jobs:

    dependsOn:
      - GetReleaseVersion
-      - Windows
+      - Windows_test
      - Linux
      - macOS_tests

--- a/script/vsts/platforms/macos.yml
+++ b/script/vsts/platforms/macos.yml
@ -8,6 +8,7 @@ jobs:
      ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ]
      IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ]
      IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ]
+      RunCoreMainTests: true
    pool:
      vmImage: macos-10.14

@ -25,6 +26,9 @@ jobs:

      - template: templates/build.yml

+      # core main tests
+      - template: templates/test.yml
+
      - script: |
          cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory)
        displayName: Stage Artifacts
@ -51,8 +55,8 @@ jobs:
    strategy:
      maxParallel: 3
      matrix:
-        core:
-          RunCoreTests: true
+        renderer:
+          RunCoreRendererTests: true
          RunPackageTests: false
        packages-1:
          RunCoreTests: false
--- a/script/vsts/platforms/templates/test.yml
+++ b/script/vsts/platforms/templates/test.yml
@ -21,6 +21,8 @@ steps:
      ATOM_JASMINE_REPORTER: list
      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
      ATOM_RUN_CORE_TESTS: $(RunCoreTests)
+      ATOM_RUN_CORE_MAIN_TESTS: $(RunCoreMainTests)
+      ATOM_RUN_CORE_RENDER_TESTS: $(RunCoreRendererTests)
      ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests)
    displayName: Run tests
    condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))
--- a/script/vsts/platforms/windows.yml
+++ b/script/vsts/platforms/windows.yml
@ -1,5 +1,5 @@
 jobs:
-  - job: Windows
+  - job: Windows_build
    dependsOn: GetReleaseVersion
    timeoutInMinutes: 180
    strategy:
@ -7,8 +7,10 @@ jobs:
      matrix:
        x64:
          BUILD_ARCH: x64
+          RunCoreMainTests: true
        x86:
          BUILD_ARCH: x86
+          RunCoreMainTests: true

    pool:
      vmImage: vs2017-win2016
@ -54,7 +56,7 @@ jobs:
          artifacts:
            - fileName: atom$(FileID)-windows.zip
              fileDir: $(Build.SourcesDirectory)/out
-              condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+              condition: and( succeeded(), or( eq(variables['BUILD_ARCH'], 'x64'), ne(variables['Build.Reason'], 'PullRequest') ) )
            - fileName: AtomSetup$(FileID).exe
              fileDir: $(Build.SourcesDirectory)/out
              condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true'))
@ -68,3 +70,58 @@ jobs:
            - fileName: RELEASES$(FileID)
              fileDir: $(Build.SourcesDirectory)/out
              condition: and(succeeded(), eq(variables['IsReleaseBranch'], 'true'))
+
+  - job: Windows_tests
+    dependsOn: Windows_build
+    timeoutInMinutes: 180
+    strategy:
+      maxParallel: 2
+      matrix:
+        x64_Renderer_Test1:
+          RunCoreMainTests: false
+          RunCoreRendererTests: 1
+          BUILD_ARCH: x64
+          os: windows-2019
+        x64_Renderer_Test2:
+          RunCoreMainTests: false
+          RunCoreRendererTests: 2
+          BUILD_ARCH: x64
+          os: windows-2019
+
+    pool:
+      vmImage: $(os)
+
+    variables:
+      AppName: $[ dependencies.GetReleaseVersion.outputs['Version.AppName'] ]
+      ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ]
+      IsReleaseBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsReleaseBranch'] ]
+      IsSignedZipBranch: $[ dependencies.GetReleaseVersion.outputs['Version.IsSignedZipBranch'] ]
+
+    steps:
+      - template: templates/preparation.yml
+
+      - template: templates/cache.yml
+        parameters:
+          OS: windows
+
+      - template: templates/bootstrap.yml
+
+      # Downloading the build artifacts
+      - pwsh: |
+          if ($env:BUILD_ARCH -eq "x64") {
+            $env:FileID="-x64"
+          } else {
+            $env:FileID=""
+          }
+          echo "##vso[task.setvariable variable=FileID]$env:FileID" # Azure syntax
+        env:
+          BUILD_ARCH: $(BUILD_ARCH)
+        displayName: Set FileID based on the arch
+
+      - template: templates/download-unzip.yml
+        parameters:
+          artifacts:
+            -  atom$(FileID)-windows.zip
+
+      #  Core renderer tests
+      - template: templates/test.yml
--- a/script/vsts/release-branch-build.yml
+++ b/script/vsts/release-branch-build.yml
@ -19,7 +19,7 @@ jobs:

    dependsOn:
      - GetReleaseVersion
-      - Windows
+      - Windows_tests
      - Linux
      - macOS_tests

--- a/src/main-process/start.js
+++ b/src/main-process/start.js
@ -1,7 +1,7 @@
 const { app } = require('electron');
 const nslog = require('nslog');
 const path = require('path');
-const temp = require('temp').track();
+const temp = require('temp');
 const parseCommandLine = require('./parse-command-line');
 const startCrashReporter = require('../crash-reporter-start');
 const getReleaseChannel = require('../get-release-channel');