diff --git a/lib/plugin/junitReporter.js b/lib/plugin/junitReporter.js index 179f28560..f276a81eb 100644 --- a/lib/plugin/junitReporter.js +++ b/lib/plugin/junitReporter.js @@ -7,6 +7,7 @@ import { DOMImplementation, XMLSerializer } from '@xmldom/xmldom' import event from '../event.js' import store from '../store.js' import output from '../output.js' +import container from '../container.js' const defaultConfig = { outputName: 'report.xml', @@ -18,6 +19,7 @@ const defaultConfig = { } const INVALID_XML_CHARS = new RegExp('[\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F\\uFFFE\\uFFFF]', 'g') +const SUITE_HOOK_TITLE = /^"(before all|after all)" hook:/ /** * @@ -66,6 +68,40 @@ export default function (config = {}) { config = Object.assign({}, defaultConfig, config) let written = false + let runnerAttached = false + const hookFailures = [] + const seenHookFailures = new Set() + + const attachRunner = () => { + if (runnerAttached) return + + const mocha = container.mocha() + const runner = mocha && (mocha.runner || mocha.Runner) + if (!runner || typeof runner.on !== 'function') return + + runnerAttached = true + runner.on('fail', (failed, err) => { + if (!failed || failed.type !== 'hook' || !SUITE_HOOK_TITLE.test(failed.title || '')) return + + const suite = failed.parent + const suiteTitle = (suite && suite.title) || '' + const key = `${suiteTitle}::${failed.title}` + if (seenHookFailures.has(key)) return + seenHookFailures.add(key) + + hookFailures.push({ + title: failed.title || 'hook failed', + state: 'failed', + err: err || failed.err || {}, + parent: suite, + file: failed.file || (suite && suite.file), + tags: suiteTitle.match(/@[\\w-]+/g) || [], + meta: {}, + steps: [], + duration: failed.duration || 0, + }) + }) + } const writeReport = result => { if (written) return @@ -76,25 +112,29 @@ export default function (config = {}) { mkdirp.sync(dir) const file = path.join(dir, config.outputName) - fs.writeFileSync(file, buildXml(result, config)) + fs.writeFileSync(file, buildXml(result, config, hookFailures)) output.plugin('junitReporter', `JUnit report saved to ${file}`) } + event.dispatcher.on(event.all.before, attachRunner) + event.dispatcher.on(event.suite.before, attachRunner) + event.dispatcher.on(event.test.before, attachRunner) event.dispatcher.on(event.all.result, writeReport) event.dispatcher.on(event.workers.result, writeReport) } -function buildXml(result, config) { +function buildXml(result, config, hookFailures = []) { const doc = new DOMImplementation().createDocument(null, null, null) - const suites = groupBySuite(result.tests) + const allTests = result.tests.concat(hookFailures) + const suites = groupBySuite(allTests) const root = doc.createElement('testsuites') setAttr(root, 'name', config.testGroupName) - setAttr(root, 'tests', result.tests.length) - setAttr(root, 'failures', countState(result.tests, 'failed')) - setAttr(root, 'skipped', countSkipped(result.tests)) + setAttr(root, 'tests', allTests.length) + setAttr(root, 'failures', countState(allTests, 'failed')) + setAttr(root, 'skipped', countSkipped(allTests)) setAttr(root, 'errors', 0) - setAttr(root, 'time', toSeconds(sumDuration(result.tests))) + setAttr(root, 'time', toSeconds(sumDuration(allTests))) setAttr(root, 'timestamp', toIso(result.stats && result.stats.start)) doc.appendChild(root) diff --git a/test/unit/junitReporter_test.js b/test/unit/junitReporter_test.js index d0c6929ca..c92ade87b 100644 --- a/test/unit/junitReporter_test.js +++ b/test/unit/junitReporter_test.js @@ -8,6 +8,7 @@ import xml2js from 'xml2js' import junitReporter from '../../lib/plugin/junitReporter.js' import event from '../../lib/event.js' import store from '../../lib/store.js' +import container from '../../lib/container.js' import Step from '../../lib/step/base.js' import MetaStep from '../../lib/step/meta.js' @@ -83,6 +84,22 @@ function parseReport(dir) { return new xml2js.Parser().parseStringPromise(fs.readFileSync(path.join(dir, 'report.xml'), 'utf8')) } +function stubMochaRunner() { + const listeners = {} + + container.append({ + mocha: { + runner: { + on(name, fn) { + listeners[name] = fn + }, + }, + }, + }) + + return listeners +} + describe('JUnit Reporter Plugin', () => { let tmpDir let prevOutputDir @@ -91,13 +108,20 @@ describe('JUnit Reporter Plugin', () => { prevOutputDir = store._outputDir tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cjs-junit-')) store.outputDir = tmpDir + event.dispatcher.removeAllListeners(event.all.before) event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.suite.before) + event.dispatcher.removeAllListeners(event.test.before) event.dispatcher.removeAllListeners(event.workers.result) }) afterEach(() => { + event.dispatcher.removeAllListeners(event.all.before) event.dispatcher.removeAllListeners(event.all.result) + event.dispatcher.removeAllListeners(event.suite.before) + event.dispatcher.removeAllListeners(event.test.before) event.dispatcher.removeAllListeners(event.workers.result) + container.append({ mocha: {} }) store._outputDir = prevOutputDir fs.rmSync(tmpDir, { recursive: true, force: true }) }) @@ -198,4 +222,57 @@ describe('JUnit Reporter Plugin', () => { const secondMtime = fs.statSync(path.join(tmpDir, 'report.xml')).mtimeMs expect(secondMtime).to.equal(firstMtime) }) + + it('adds suite-level hook failures as failed testcases', async () => { + const listeners = stubMochaRunner() + junitReporter({}) + event.dispatcher.emit(event.suite.before) + + const suite = { title: 'Suite hooks @smoke', file: '/tests/hooks_test.js', startedAt: Date.now() } + listeners.fail( + { + type: 'hook', + title: '"before all" hook: BeforeSuite for "records suite hook failures"', + parent: suite, + file: suite.file, + duration: 42, + }, + new Error('BeforeSuite failed'), + ) + listeners.fail( + { + type: 'hook', + title: '"before each" hook: Before for "already reported by test"', + parent: suite, + file: suite.file, + duration: 7, + }, + new Error('Before failed'), + ) + listeners.fail( + { + type: 'hook', + title: '"before all" hook: BeforeSuite for "records suite hook failures"', + parent: suite, + file: suite.file, + duration: 42, + }, + new Error('duplicate suite failure'), + ) + + event.dispatcher.emit(event.all.result, { + tests: [], + stats: { start: new Date(), passes: 0, failures: 0, pending: 0, tests: 0 }, + }) + + const parsed = await parseReport(tmpDir) + expect(parsed.testsuites.$.tests).to.equal('1') + expect(parsed.testsuites.$.failures).to.equal('1') + + const suiteEl = parsed.testsuites.testsuite[0] + expect(suiteEl.$.name).to.equal('Suite hooks @smoke') + expect(suiteEl.$.tests).to.equal('1') + expect(suiteEl.testcase[0].$.name).to.contain('"before all" hook') + expect(suiteEl.testcase[0].failure[0].$.message).to.contain('BeforeSuite failed') + }) })