How to re-run only failed JUnit test classes using Gradle?

Question

Inspired by this neat TestNG task, and this SO question I thought I'd whip up something quick for re-running of only failed JUnit tests from Gradle.

But after searching around for awhile, I couldn't find anything analogous which was quite as convenient.

I came up with the following, which seems to work pretty well and adds a <testTaskName>Rerun task for each task of type Test in my project.

import static groovy.io.FileType.FILES

import java.nio.file.Files
import java.nio.file.Paths

// And add a task for each test task to rerun just the failing tests
subprojects {
    afterEvaluate { subproject ->
        // Need to store tasks in static temp collection, else new tasks will be picked up by live collection leading to StackOverflow 
        def testTasks = subproject.tasks.withType(Test)
        testTasks.each { testTask ->
            task "${testTask.name}Rerun"(type: Test) {
                group = 'Verification'
                description = "Re-run ONLY the failing tests from the previous run of ${testTask.name}."

                // Depend on anything the existing test task depended on
                dependsOn testTask.dependsOn 

                // Copy runtime setup from existing test task
                testClassesDirs = testTask.testClassesDirs
                classpath = testTask.classpath

                // Check the output directory for failing tests
                File textXMLDir = subproject.file(testTask.reports.junitXml.destination)
                logger.info("Scanning: $textXMLDir for failed tests.")

                // Find all failed classes
                Set<String> allFailedClasses = [] as Set
                if (textXMLDir.exists()) {
                    textXMLDir.eachFileRecurse(FILES) { f ->
                        // See: http://marxsoftware.blogspot.com/2015/02/determining-file-types-in-java.html
                        String fileType
                        try {
                            fileType = Files.probeContentType(f.toPath())
                        } catch (IOException e) {
                            logger.debug("Exception when probing content type of: $f.")
                            logger.debug(e)

                            // Couldn't determine this to be an XML file.  That's fine, skip this one.
                            return
                        }

                        logger.debug("Filetype of: $f is $fileType.") 

                        if (['text/xml', 'application/xml'].contains(fileType)) {
                            logger.debug("Found testsuite file: $f.")

                            def testSuite = new XmlSlurper().parse(f)
                            def failedTestCases = testSuite.testcase.findAll { testCase ->
                                testCase.children().find { it.name() == 'failure' }
                            }

                            if (!failedTestCases.isEmpty()) {
                                logger.info("Found failures in file: $f.")
                                failedTestCases.each { failedTestCase -> 
                                    def className = failedTestCase['@classname']
                                    logger.info("Failure: $className")
                                    allFailedClasses << className.toString()
                                }
                            }
                        }
                    }
                }

                if (!allFailedClasses.isEmpty()) {
                    // Re-run all tests in any class with any failures
                    allFailedClasses.each { c ->
                        def testPath = c.replaceAll('\\.', '/') + '.class'
                        include testPath
                    }

                    doFirst {
                        logger.warn('Re-running the following tests:')
                        allFailedClasses.each { c ->
                            logger.warn(c)
                        }
                    }
                }

                outputs.upToDateWhen { false } // Always attempt to re-run failing tests
                // Only re-run if there were any failing tests, else just print warning
                onlyIf { 
                    def shouldRun = !allFailedClasses.isEmpty() 
                    if (!shouldRun) {
                        logger.warn("No failed tests found for previous run of task: ${subproject.path}:${testTask.name}.")
                    }

                    return shouldRun
                }
            }
        }
    }
}

Is there any easier way to do this from Gradle? Is there any way to get JUnit to output a consolidated list of failures somehow so I don't have to slurp the XML reports?

I'm using JUnit 4.12 and Gradle 4.5.

Answer 1

Here is one way to do it. The full file will be listed at the end, and is available here.

Part one is to write a small file (called failures) for every failed test:

test {
    // `failures` is defined elsewhere, see below
    afterTest { desc, result -> 
        if ("FAILURE" == result.resultType as String) {
            failures.withWriterAppend { 
                it.write("${desc.className},${desc.name}\n")
            }
        }
    }
}

In part two, we use a test filter (doc here) to restrict the tests to any that are present in the failures file:

def failures = new File("${projectDir}/failures.log")
def failedTests = [] 
if (failures.exists()) {
    failures.eachLine { line ->
        def tokens = line.split(",")
        failedTests << tokens[0] 
    }
}
failures.delete()

test {
    filter {
        failedTests.each { 
            includeTestsMatching "${it}"
        }
    }
    // ...
}

The full file is:

apply plugin: 'java'

repositories {
    jcenter()
}

dependencies {
    testCompile('junit:junit:4.12')
}   

def failures = new File("${projectDir}/failures.log")
def failedTests = [] 
if (failures.exists()) {
    failures.eachLine { line ->
        def tokens = line.split(",")
        failedTests << tokens[0] 
    }
}
failures.delete()

test {
    filter {
        failedTests.each { 
            includeTestsMatching "${it}"
        }
    }

    afterTest { desc, result -> 
        if ("FAILURE" == result.resultType as String) {
            failures.withWriterAppend { 
                it.write("${desc.className},${desc.name}\n")
            }
        }
    }
}