fix(runner): Make exit code configurable when tests are failing (#3116)

andreaspsson · johnjbarton · commit 74da748908bd · 2018-08-30T09:25:42.000-07:00
Fixes #1300
diff --git a/lib/browser_collection.js b/lib/browser_collection.js
@@ -46,7 +46,24 @@ class BrowserCollection {
     return this.browsers.map((browser) => browser.serialize())
   }
 
-  getResults () {
+  calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
+    if (results.disconnected || singleRunBrowserNotCaptured) {
+      return 1
+    }
+    if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
+      return 0
+    }
+    if (results.error) {
+      return 1
+    }
+    if (failOnFailingTestSuite === false) {
+      // Tests executed without infrastructure error, exit with 0 independent of test status.
+      return 0
+    }
+    return results.failed ? 1 : 0
+  }
+
+  getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
     const results = this.browsers.reduce((previous, current) => {
       previous.success += current.lastResult.success
       previous.failed += current.lastResult.failed
@@ -56,7 +73,7 @@ class BrowserCollection {
     }, {success: 0, failed: 0, error: false, disconnected: false, exitCode: 0})
 
     // compute exit status code
-    results.exitCode = results.failed || results.error || results.disconnected ? 1 : 0
+    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
 
     return results
   }
diff --git a/lib/cli.js b/lib/cli.js
@@ -46,6 +46,10 @@ function processArgs (argv, options, fs, path) {
     options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
   }
 
+  if (helper.isString(options.failOnFailingTestSuite)) {
+    options.failOnFailingTestSuite = options.failOnFailingTestSuite === 'true'
+  }
+
   if (helper.isString(options.formatError)) {
     let required
     try {
@@ -196,6 +200,8 @@ function describeStart () {
     .describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
     .describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
     .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
+    .describe('fail-on-failing-test-suite', 'Fail on failing test suite.')
+    .describe('no-fail-on-failing-test-suite', 'Do not fail on failing test suite.')
     .describe('help', 'Print usage and options.')
 }
 
diff --git a/lib/server.js b/lib/server.js
@@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {
 
     const emitRunCompleteIfAllBrowsersDone = () => {
       if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {
-        const results = singleRunBrowsers.getResults()
-        if (singleRunBrowserNotCaptured) {
-          results.exitCode = 1
-        } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
-          results.exitCode = 0
-          this.log.warn('Test suite was empty.')
-        }
-        this.emit('run_complete', singleRunBrowsers, results)
+        this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))
       }
     }
 
diff --git a/test/unit/browser_collection.spec.js b/test/unit/browser_collection.spec.js
@@ -258,4 +258,90 @@ describe('BrowserCollection', () => {
       })
     })
   })
+
+  // ============================================================================
+  // server.calculateExitCode
+  // ============================================================================
+  describe('calculateExitCode', () => {
+    const EXIT_CODE_ERROR = 1
+    const EXIT_CODE_SUCCESS = 0
+
+    describe('no tests', () => {
+      const results = {
+        success: 0,
+        failed: 0,
+        error: true
+      }
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnEmptyTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall fail if failOnEmptyTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnFailingTestSuite is set', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+    })
+    describe('all test passed, no errors', () => {
+      const results = {
+        success: 10,
+        failed: 0,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnFailingTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+      it('shall pass if failOnFailingTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, true, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+    describe('all test passed, with error', () => {
+      const results = {
+        success: 10,
+        failed: 5,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnEmptyTestSuite not is set', () => {
+        const res = collection.calculateExitCode(results, false, false)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnFailingTestSuite is true', () => {
+        const res = collection.calculateExitCode(results, false, true, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnFailingTestSuite is false', () => {
+        const res = collection.calculateExitCode(results, false, true, false)
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+  })
 })

Original file line number	Diff line number	Diff line change
`@@ -247,14 +247,7 @@ class Server extends KarmaEventEmitter {`
`247`	`247`
`248`	`248`	`const emitRunCompleteIfAllBrowsersDone = () => {`
`249`	`249`	`if (Object.keys(singleRunDoneBrowsers).every((key) => singleRunDoneBrowsers[key])) {`
`250`		`- const results = singleRunBrowsers.getResults()`
`251`		`- if (singleRunBrowserNotCaptured) {`
`252`		`- results.exitCode = 1`
`253`		`- } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {`
`254`		`- results.exitCode = 0`
`255`		`- this.log.warn('Test suite was empty.')`
`256`		`- }`
`257`		`- this.emit('run_complete', singleRunBrowsers, results)`
	`250`	`+ this.emit('run_complete', singleRunBrowsers, singleRunBrowsers.getResults(singleRunBrowserNotCaptured, config.failOnEmptyTestSuite, config.failOnFailingTestSuite))`
`258`	`251`	`}`
`259`	`252`	`}`
`260`	`253`