githubmirror/atom

script/test
373 строки12 KB
Amin Yahyaabadi
add core main timeout retry trigger
18 сен 2020, 03:51
18 сен 2020, 03:51f3e9434
#!/usr/bin/env node

'use strict'

require('colors')
const argv = require('yargs')
  .option('core-main', {
    describe: 'Run core main process tests',
    boolean: true,
    default: false
  })
  .option('skip-main', {
    describe: 'Skip main process tests if they would otherwise run on your platform',
    boolean: true,
    default: false,
    conflicts: 'core-main'
  })
  .option('core-renderer', {
    describe: 'Run core renderer process tests',
    boolean: true,
    default: false
  })
  .option('core-benchmark', {
    describe: 'Run core benchmarks',
    boolean: true,
    default: false
  })
  .option('package', {
    describe: 'Run bundled package specs',
    boolean: true,
    default: false
  })
  .help()
  .argv

const assert = require('assert')
const async = require('async')
const childProcess = require('child_process')
const fs = require('fs-extra')
const glob = require('glob')
const path = require('path')
const temp = require('temp').track()

const CONFIG = require('./config')
const backupNodeModules = require('./lib/backup-node-modules')
const runApmInstall = require('./lib/run-apm-install')

function assertExecutablePaths(executablePaths) {
	assert(executablePaths.length !== 0, `No atom build found. Please run "script/build" and try again.`)
	assert(executablePaths.length === 1, `More than one application to run tests against was found. ${executablePaths.join(',')}`)
}

const resourcePath = CONFIG.repositoryRootPath
let executablePath
if (process.platform === 'darwin') {
  const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '*.app'))
  assertExecutablePaths(executablePaths)
  executablePath = path.join(executablePaths[0], 'Contents', 'MacOS', path.basename(executablePaths[0], '.app'))
} else if (process.platform === 'linux') {
  const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, 'atom-*', 'atom'))
  assertExecutablePaths(executablePaths)
  executablePath = executablePaths[0]
} else if (process.platform === 'win32') {
  const executablePaths = glob.sync(path.join(CONFIG.buildOutputPath, '**', 'atom*.exe'))
  assertExecutablePaths(executablePaths)
  executablePath = executablePaths[0]
} else {
  throw new Error('##[error] Running tests on this platform is not supported.')
}

function prepareEnv (suiteName) {
  const atomHomeDirPath = temp.mkdirSync(suiteName)
  const env = Object.assign({}, process.env, {ATOM_HOME: atomHomeDirPath})

  if (process.env.TEST_JUNIT_XML_ROOT) {
    // Tell Jasmine to output this suite's results as a JUnit XML file to a subdirectory of the root, so that a
    // CI system can interpret it.
    const fileName = suiteName + '.xml'
    const outputPath = path.join(process.env.TEST_JUNIT_XML_ROOT, fileName)
    env.TEST_JUNIT_XML_PATH = outputPath
  }

  return env
}

function spawnTest(executablePath, testArguments, options, callback, testName, finalize = null) {
  const cp = childProcess.spawn(executablePath, testArguments, options)

  // collect outputs and errors
	let stderrOutput = ''
	if (cp.stdout) {
	  cp.stderr.on('data', data => { stderrOutput += data })
	  cp.stdout.on('data', data => { stderrOutput += data })
	}

  // on error
  cp.on('error', error => {
    console.log(error, "error")
    if (finalize) { finalize() } 		// if finalizer provided
    callback(error)
  })

  // on close
  cp.on('close', exitCode => {
    if (exitCode !== 0) {
      retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, options, callback, testName, finalize)
    } else {
      // successful test
      if (finalize) { finalize() } 		// if finalizer provided
      callback(null, {
        exitCode,
        step: testName,
        testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`,
      })
    }
  })

}

const retryNumber = 6 // the number of times a tests repeats
const retriedTests = new Map() // a cache of retried tests

// Retries the tests if it is timed out for a number of times. Fails the rest of the tests or those that are tried enough times.
function retryOrFailTest(stderrOutput, exitCode, executablePath, testArguments, options, callback, testName, finalize) {
  const testKey = createTestKey(executablePath, testArguments, testName)
  if (isTimedOut(stderrOutput) && shouldTryAgain(testKey)) {
    // retry the timed out test
    let triedNumber = retriedTests.get(testKey) || 0
    retriedTests.set(testKey, triedNumber + 1)
    console.warn(`\n##[warning] Retrying the timed out step: ${testName} \n`)
    spawnTest(executablePath, testArguments, options, callback, testName, finalize)
  } else {
    // fail the test
    if (finalize) { finalize() } 		// if finalizer provided
    console.log(`##[error] Tests for ${testName} failed.`.red)
    console.log(stderrOutput)
    callback(null, {
      exitCode,
      step: testName,
      testCommand: `You can run the test again using: \n\t ${executablePath} ${testArguments.join(' ')}`,
    })
  }
}

// creates a key that is specific to a certain test
function createTestKey(executablePath, testArguments, testName) {
  return `${executablePath} ${testArguments.join(' ')} ${testName}`
}

// check if a test is timed out
function isTimedOut(stderrOutput) {
  if (stderrOutput) {
    return (
			stderrOutput.includes("timeout: timed out after") || // happens in core renderer tests
			stderrOutput.includes("Error: timeout of") || // happens in core main tests
			stderrOutput.includes("Error Downloading Update: Could not get code signature for running application") // happens in github tests
		)
  } else {
    return false
  }
}

// check if a tests should be tried again
function shouldTryAgain(testKey) {
  if (retriedTests.has(testKey)) {
    return (retriedTests.get(testKey) < retryNumber)
  } else {
    return true
  }
}


function runCoreMainProcessTests (callback) {
  const testPath = path.join(CONFIG.repositoryRootPath, 'spec', 'main-process')
  const testArguments = [
    '--resource-path', resourcePath,
    '--test', '--main-process', testPath
  ]

  if(process.env.CI && process.platform === 'linux') {
      testArguments.push('--no-sandbox')
  }

  const testEnv = Object.assign({}, prepareEnv('core-main-process'), {ATOM_GITHUB_INLINE_GIT_EXEC: 'true'})

  console.log('##[command] Executing core main process tests'.bold.green)
  spawnTest(executablePath, testArguments, {stdio: 'inherit', env: testEnv}, callback, 'core-main-process')
}

function getCoreRenderProcessTestSuites() {

// Build an array of functions, each running tests for a different rendering test
const coreRenderProcessTestSuites = []
const testPath = path.join(CONFIG.repositoryRootPath, 'spec')
let testFiles = glob.sync(path.join(testPath, '*-spec.+(js|coffee|ts|jsx|tsx|mjs)'))
for (let testFile of testFiles) {
	const testArguments = [
		'--resource-path', resourcePath,
		'--test', testFile
	]
	// the function which runs by async:
  coreRenderProcessTestSuites.push( function (callback) {
    const testEnv = prepareEnv('core-render-process')
    console.log(`##[command] Executing core render process tests for ${testFile}`.bold.green)
    spawnTest(executablePath, testArguments, {env: testEnv}, callback, `core-render-process in ${testFile}.`)
  })
}

	return coreRenderProcessTestSuites
}


function getPackageTestSuites() {

// Build an array of functions, each running tests for a different bundled package
const packageTestSuites = []
for (let packageName in CONFIG.appMetadata.packageDependencies) {
  if (process.env.ATOM_PACKAGES_TO_TEST) {
    const packagesToTest = process.env.ATOM_PACKAGES_TO_TEST.split(',').map(pkg => pkg.trim())
    if (!packagesToTest.includes(packageName)) continue
  }

  const repositoryPackagePath = path.join(CONFIG.repositoryRootPath, 'node_modules', packageName)
  const testSubdir = ['spec', 'test'].find(subdir => fs.existsSync(path.join(repositoryPackagePath, subdir)))

  if (!testSubdir) {
    console.log(`No test folder found for package: ${packageName}`.yellow)
    continue
  }

  const testFolder = path.join(repositoryPackagePath, testSubdir)

	const testArguments = [
		'--resource-path', resourcePath,
		'--test', testFolder
	]

	const pkgJsonPath = path.join(repositoryPackagePath, 'package.json')
	const nodeModulesPath = path.join(repositoryPackagePath, 'node_modules')

	// the function which runs by async:
  packageTestSuites.push(function (callback) {
    const testEnv = prepareEnv(`bundled-package-${packageName}`)
    let finalize = () => null
    if (require(pkgJsonPath).atomTestRunner) {
      console.log(`##[command] Installing test runner dependencies for ${packageName}`.bold.green)
      if (fs.existsSync(nodeModulesPath)) {
        const backup = backupNodeModules(repositoryPackagePath)
        finalize = backup.restore
      } else {
        finalize = () => fs.removeSync(nodeModulesPath)
      }
      runApmInstall(repositoryPackagePath)
      console.log(`##[command] Executing ${packageName} tests`.green)
    } else {
      console.log(`##[command] Executing ${packageName} tests`.bold.green)
    }
    spawnTest(executablePath, testArguments, {env: testEnv}, callback, `${packageName} package`, finalize)
  })
}

	return packageTestSuites
}


function runBenchmarkTests (callback) {
  const benchmarksPath = path.join(CONFIG.repositoryRootPath, 'benchmarks')
  const testArguments = ['--benchmark-test', benchmarksPath]
  const testEnv = prepareEnv('benchmark')

  console.log('##[command] Executing benchmark tests'.bold.green)
  const cp = childProcess.spawn(executablePath, testArguments, {stdio: 'inherit', env: testEnv})
  spawnTest(executablePath, testArguments, {stdio: 'inherit', env: testEnv}, callback, `core-benchmarks`)
}

let testSuitesToRun = requestedTestSuites(process.platform)

function requestedTestSuites (platform) {
  // env variable or argv options
  let coreAll = process.env.ATOM_RUN_CORE_TESTS === 'true'
  let coreMain = process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true' || argv.coreMain
  let coreRenderer = argv.coreRenderer || process.env.ATOM_RUN_CORE_RENDER_TESTS == 'true'
  let coreRenderer1 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '1'
  let coreRenderer2 = process.env.ATOM_RUN_CORE_RENDER_TESTS === '2'
  let packageAll = argv.package || process.env.ATOM_RUN_PACKAGE_TESTS == 'true'
  let packages1 = process.env.ATOM_RUN_PACKAGE_TESTS === '1'
  let packages2 = process.env.ATOM_RUN_PACKAGE_TESTS === '2'
  let benchmark = argv.coreBenchmark

  // Operating system overrides:
  coreMain = coreMain || (platform === 'linux') || (platform === 'win32' && process.arch === 'x86')

  // split package tests (used for macos in CI)
  const PACKAGES_TO_TEST_IN_PARALLEL = 23
  // split core render test (used for windows x64 in CI)
  const CORE_RENDER_TO_TEST_IN_PARALLEL = 45

  let suites = []
  // Core tess
  if (coreAll) {
    suites.push(...[runCoreMainProcessTests, ...getCoreRenderProcessTestSuites()])
  } else {

    // Core main tests
    if (coreMain) {
      suites.push(runCoreMainProcessTests)
    }

    // Core renderer tests
    if (coreRenderer) {
      suites.push(...getCoreRenderProcessTestSuites())
    } else {
      // split
      if (coreRenderer1) {
        suites.push(...getCoreRenderProcessTestSuites().slice(0, CORE_RENDER_TO_TEST_IN_PARALLEL))
      }
      if (coreRenderer2) {
        suites.push(...getCoreRenderProcessTestSuites().slice(CORE_RENDER_TO_TEST_IN_PARALLEL))
      }
    }

  }

  // Package tests
	if (packageAll) {
	  suites.push(...getPackageTestSuites())
  } else {
    // split
    if (packages1) {
      suites.push(...getPackageTestSuites().slice(0, PACKAGES_TO_TEST_IN_PARALLEL))
    }
    if (packages2) {
      suites.push(...getPackageTestSuites().slice(PACKAGES_TO_TEST_IN_PARALLEL))
    }
  }

  // Benchmark tests
  if (benchmark) {
    suites.push(runBenchmarkTests)
  }

  if (argv.skipMainProcessTests) {
    suites = suites.filter(suite => suite !== runCoreMainProcessTests)
  }

  // Remove duplicates
  suites = Array.from(new Set(suites))

  if (suites.length == 0) {
    throw new Error("No tests was requested")
  }

  return suites
}

async.series(testSuitesToRun, function (err, results) {
  if (err) {
    console.error(err)
    process.exit(1)
  } else {
    const failedSteps = results.filter(({exitCode}) => exitCode !== 0)

    if (failedSteps.length > 0) {
			console.warn("\n \n ##[error] *** Reporting the errors that happened in all of the tests: *** \n \n")
      for (const {step, testCommand} of failedSteps) {
        console.error(`##[error] The '${step}' test step finished with a non-zero exit code \n ${testCommand}`)
      }
      process.exit(1)
    }

    process.exit(0)
  }
})