Commit f16d011d authored by Feceoru, Gabriel's avatar Feceoru, Gabriel Committed by Dylan Baker

framework: Add support for feature readiness.

This adds a new "summary feature" command to piglit which creates a HTML table
with the feature x DUT status (useful for multiple features, multiple DUTs).
Another use case is the feature status for subsequent test results (depending
on the meaning of that test result - DUT or build)
A feature readiness is defined by the piglit regexp which selects the tests
relevant for that feature and the acceptance percentage threshold (pass rate).

It requires an input json file containing the list of features, in the
following format (this is just an example):
{
    "glsl" : {
        "include_tests" : "glsl",
        "exclude_tests" : "",
        "target_rate" : 90
    },
    "arb" : {
        "include_tests" :  "arb_gpu",
        "exclude_tests" : "",
        "target_rate" : 10
    }

}

v3:
Changed json rate to int instead of string
Applied other review comments

v2:
Apply review comments (Dylan, Thomas)
Fixed 2nd round of review comments
Signed-off-by: default avatarGabriel Feceoru <gabriel.feceoru@intel.com>
Reviewed-by: Dylan Baker's avatarDylan Baker <baker.dylan.c@gmail.com>
parent fb07979a
......@@ -35,6 +35,7 @@ __all__ = [
'console',
'csv',
'html',
'feature'
]
......@@ -224,3 +225,39 @@ def aggregate(input_):
print("Aggregated file written to: {}.{}".format(
outfile, backends.compression.get_mode()))
@exceptions.handler
def feature(input_):
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--overwrite",
action="store_true",
help="Overwrite existing directories")
parser.add_argument("featureFile",
metavar="<Feature json file>",
help="Json file containing the features description")
parser.add_argument("summaryDir",
metavar="<Summary Directory>",
help="Directory to put HTML files in")
parser.add_argument("resultsFiles",
metavar="<Results Files>",
nargs="*",
help="Results files to include in HTML")
args = parser.parse_args(input_)
# If args.list and args.resultsFiles are empty, then raise an error
if not args.featureFile and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# If args.list and args.resultsFiles are empty, then raise an error
if not args.resultsFiles or not path.exists(args.featureFile):
raise parser.error("Missing json file")
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
core.checkDir(args.summaryDir, not args.overwrite)
summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
......@@ -25,5 +25,5 @@
# public parts here, so that we have a nice interface to work with.
from __future__ import absolute_import, division, print_function
from .html_ import html
from .html_ import html, feat
from .console_ import console
# Copyright (c) 2015 Intel Corporation
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
from __future__ import print_function, division, absolute_import
import copy
try:
import simplejson as json
except ImportError:
import json
from framework import core, exceptions, profile, status
class FeatResults(object): # pylint: disable=too-few-public-methods
"""Container object for results.
Has the results, feature profiles and feature computed results.
"""
def __init__(self, results, json_file):
with open(json_file) as data:
feature_data = json.load(data)
self.feat_fractions = {}
self.feat_status = {}
self.features = set()
self.results = results
profiles = {}
# we expect all the result sets to be for the same profile
profile_orig = profile.load_test_profile(results[0].options['profile'][0])
for feature in feature_data:
self.features.add(feature)
incl_str = feature_data[feature]["include_tests"]
excl_str = feature_data[feature]["exclude_tests"]
include_filter = [incl_str] if incl_str and not incl_str.isspace() else []
exclude_filter = [excl_str] if excl_str and not excl_str.isspace() else []
opts = core.Options(include_filter=include_filter,
exclude_filter=exclude_filter)
profiles[feature] = copy.deepcopy(profile_orig)
# An empty list will raise PiglitFatalError exception
# But for reporting we need to handle this situation
try:
profiles[feature]._prepare_test_list(opts)
except exceptions.PiglitFatalError:
pass
for results in self.results:
self.feat_fractions[results.name] = {}
self.feat_status[results.name] = {}
for feature in feature_data:
result_set = set(results.tests)
profile_set = set(profiles[feature].test_list)
common_set = profile_set & result_set
passed_list = [x for x in common_set if results.tests[x].result == status.PASS]
total = len(common_set)
passed = len(passed_list)
self.feat_fractions[results.name][feature] = (passed, total)
if total == 0:
self.feat_status[results.name][feature] = status.NOTRUN
else:
if 100 * passed // total >= feature_data[feature]["target_rate"]:
self.feat_status[results.name][feature] = status.PASS
else:
self.feat_status[results.name][feature] = status.FAIL
......@@ -37,9 +37,11 @@ from mako.lookup import TemplateLookup
from framework import backends, exceptions
from .common import Results, escape_filename, escape_pathname
from .feature import FeatResults
__all__ = [
'html',
'feat'
]
_TEMP_DIR = os.path.join(
......@@ -62,8 +64,9 @@ def _copy_static_files(destination):
os.path.join(destination, "result.css"))
def _make_testrun_info(results, destination, exclude):
def _make_testrun_info(results, destination, exclude=None):
"""Create the pages for each results file."""
exclude = exclude or {}
result_css = os.path.join(destination, "result.css")
index = os.path.join(destination, "index.html")
......@@ -146,6 +149,14 @@ def _make_comparison_pages(results, destination, exclude):
page=page, pages=pages))
def _make_feature_info(results, destination):
"""Create the feature readiness page."""
with open(os.path.join(destination, "feature.html"), 'w') as out:
out.write(_TEMPLATES.get_template('feature.mako').render(
results=results))
def html(results, destination, exclude):
"""
Produce HTML summaries.
......@@ -161,3 +172,13 @@ def html(results, destination, exclude):
_copy_static_files(destination)
_make_testrun_info(results, destination, exclude)
_make_comparison_pages(results, destination, exclude)
def feat(results, destination, feat_desc):
"""Produce HTML feature readiness summary."""
feat_res = FeatResults([backends.load(i) for i in results], feat_desc)
_copy_static_files(destination)
_make_testrun_info(feat_res, destination)
_make_feature_info(feat_res, destination)
......@@ -139,6 +139,10 @@ def main():
add_help=False,
help="Aggregate incomplete piglit run.")
aggregate.set_defaults(func=summary.aggregate)
feature = summary_parser.add_parser('feature',
add_help=False,
help="generate feature readiness html report.")
feature.set_defaults(func=summary.feature)
# Parse the known arguments (piglit run or piglit summary html for
# example), and then pass the arguments that this parser doesn't know about
......
<%!
import posixpath # this must be posixpath, since we want /'s not \'s
import re
def feat_result(result):
"""Percentage result string"""
return '{}/{}'.format(result[0], result[1])
def escape_filename(key):
"""Avoid reserved characters in filenames."""
return re.sub(r'[<>:"|?*#]', '_', key)
def escape_pathname(key):
""" Remove / and \\ from names """
return re.sub(r'[/\\]', '_', key)
def normalize_href(href):
"""Force backward slashes in URLs."""
return href.replace('\\', '/')
%>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>Result summary</title>
<link rel="stylesheet" href="index.css" type="text/css" />
</head>
<body>
<h1>Feature readiness</h1>
<table>
<colgroup>
## Name Column
<col />
## Status columns
## Create an additional column for each summary
% for _ in xrange(len(results.results)):
<col />
% endfor
</colgroup>
<tr>
<th/>
% for res in results.results:
<th class="head"><b>${res.name}</b><br />\
(<a href="${normalize_href(posixpath.join(escape_pathname(res.name), 'index.html'))}">info</a>)</th>
% endfor
</tr>
% for feature in results.features:
<tr>
## Add the left most column, the feature name
<td>
<div class="group">
<b>${feature}</b>
</div>
</td>
## add the feature totals
% for res in results.results:
<td class="${results.feat_status[res.name][feature]}">
<b>${feat_result(results.feat_fractions[res.name][feature])}</b>
</td>
% endfor
</tr>
% endfor
</table>
</body>
</html>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment