Commit 1921e13d authored by Mark Janes's avatar Mark Janes Committed by Dylan Baker

Filter expected failures from JUnit output

Piglit test results are difficult to analyze with simpler JUnit
visualizers like those provided by Jenkins.  There are hundreds of
failures, but the engineer is typically interested only in NEW
failures.  Jenkins JUnit rules typically expect 100% pass rate, or
some static threshold of failures.

Specifying test names in the [expected-failures] or [expected-crashes]
sections of the config file enables JUnit reporting of just the test
failures which represent new regressions.

Test names are expected in the dotted JUnit format (eg:
"piglit.spec.ARB_fragment_program.fp-formats") and are case
Reviewed-by: default avatarDylan Baker <>
parent 9031648a
......@@ -29,6 +29,7 @@ import abc
import threading
import posixpath
from cStringIO import StringIO
import simplejson as json
except ImportError:
......@@ -38,6 +39,7 @@ try:
except ImportError:
import xml.etree.cElementTree as etree
from framework.core import PIGLIT_CONFIG
import framework.status as status
__all__ = [
......@@ -385,6 +387,18 @@ class JUnitBackend(FSyncMixin, Backend):
self._file = open(os.path.join(dest, 'results.xml'), 'w')
FSyncMixin.__init__(self, **options)
# make dictionaries of all test names expected to crash/fail
# for quick lookup when writing results. Use lower-case to
# provide case insensitive matches.
self._expected_failures = {}
if PIGLIT_CONFIG.has_section("expected-failures"):
for (fail, _) in PIGLIT_CONFIG.items("expected-failures"):
self._expected_failures[fail.lower()] = True
self._expected_crashes = {}
if PIGLIT_CONFIG.has_section("expected-crashes"):
for (fail, _) in PIGLIT_CONFIG.items("expected-crashes"):
self._expected_crashes[fail.lower()] = True
# Write initial headers and other data that etree cannot write for us
self._file.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
......@@ -414,6 +428,21 @@ class JUnitBackend(FSyncMixin, Backend):
# set different root names.
classname = 'piglit.' + classname
expected_result = "pass"
# replace special characters and make case insensitive
lname = (classname + "." + testname).lower()
lname = lname.replace("=", ".")
lname = lname.replace(":", ".")
if lname in self._expected_failures:
expected_result = "failure"
# a test can either fail or crash, but not both
assert( lname not in self._expected_crashes )
if lname in self._expected_crashes:
expected_result = "error"
# Create the root element
element = etree.Element('testcase', name=testname + self._test_suffix,
......@@ -432,10 +461,23 @@ class JUnitBackend(FSyncMixin, Backend):
# one of these statuses
if data['result'] == 'skip':
etree.SubElement(element, 'skipped')
elif data['result'] in ['warn', 'fail', 'dmesg-warn', 'dmesg-fail']:
etree.SubElement(element, 'failure')
if expected_result == "failure":
err.text += "\n\nWARN: passing test as an expected failure"
etree.SubElement(element, 'failure')
elif data['result'] == 'crash':
etree.SubElement(element, 'error')
if expected_result == "error":
err.text += "\n\nWARN: passing test as an expected crash"
etree.SubElement(element, 'error')
elif expected_result != "pass":
err.text += "\n\nERROR: This test passed when it "\
"expected {0}".format(expected_result)
etree.SubElement(element, 'failure')
......@@ -71,3 +71,15 @@ run_test=./%(test_name)s
; Options can be found running piglit run -h and reading the section for
; -b/--backend
; Provide a list of test names that are expected to fail. These tests
; will be listed as passing in JUnit output when they fail. Any
; unexpected result (pass/crash) will be listed as a failure. The
; test name must be written in the JUnit format ('.' instead of '/').
; Special characters for config file format ('=' and ':') must be
; replaced with '.'
; Like expected-failures, but specifies that a test is expected to
; crash.
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment