Commit 4005f4a8 authored by Dylan Baker's avatar Dylan Baker

framework: use TimeAttribute for TestResult and TestrunResult

This makes use of the time TimeAttribute for handling times in results,
both for tests and for runs.

This change requires bumping the json version, and a large chunk of the
changes in this patch are for that change.

This could be split into two patches, but that would require making two
sequential bumps to the json results version, and I felt that the larger
patch was better than making two version bumps.
Reviewed-by: Mark Janes's avatarMark Janes <mark.a.janes@intel.com>
Signed-off-by: default avatarDylan Baker <dylanx.c.baker@intel.com>
parent c22225af
...@@ -42,7 +42,7 @@ __all__ = [ ...@@ -42,7 +42,7 @@ __all__ = [
] ]
# The current version of the JSON results # The current version of the JSON results
CURRENT_JSON_VERSION = 7 CURRENT_JSON_VERSION = 8
# The level to indent a final file # The level to indent a final file
INDENT = 4 INDENT = 4
...@@ -290,6 +290,7 @@ def _update_results(results, filepath): ...@@ -290,6 +290,7 @@ def _update_results(results, filepath):
4: _update_four_to_five, 4: _update_four_to_five,
5: _update_five_to_six, 5: _update_five_to_six,
6: _update_six_to_seven, 6: _update_six_to_seven,
7: _update_seven_to_eight,
} }
while results.results_version < CURRENT_JSON_VERSION: while results.results_version < CURRENT_JSON_VERSION:
...@@ -567,6 +568,26 @@ def _update_six_to_seven(result): ...@@ -567,6 +568,26 @@ def _update_six_to_seven(result):
return result return result
def _update_seven_to_eight(result):
"""Update json results from version 7 to 8.
This update replaces the time attribute float with a TimeAttribute object,
which stores a start time and an end time, and provides methods for getting
total and delta.
This value is used for both TestResult.time and TestrunResult.time_elapsed.
"""
for test in result.tests.viewvalues():
test.time = results.TimeAttribute(end=test.time)
result.time_elapsed = results.TimeAttribute(end=result.time_elapsed)
result.results_version = 8
return result
REGISTRY = Registry( REGISTRY = Registry(
extensions=['', '.json'], extensions=['', '.json'],
backend=JSONBackend, backend=JSONBackend,
......
...@@ -196,7 +196,7 @@ class JUnitBackend(FileBackend): ...@@ -196,7 +196,7 @@ class JUnitBackend(FileBackend):
element = etree.Element('testcase', name=full_test_name, element = etree.Element('testcase', name=full_test_name,
classname=classname, classname=classname,
# Incomplete will not have a time. # Incomplete will not have a time.
time=str(data.time), time=str(data.time.total),
status=str(data.result)) status=str(data.result))
# If this is an incomplete status then none of these values will be # If this is an incomplete status then none of these values will be
...@@ -253,7 +253,7 @@ def _load(results_file): ...@@ -253,7 +253,7 @@ def _load(results_file):
name = name[:-1] name = name[:-1]
result.result = test.attrib['status'] result.result = test.attrib['status']
result.time = float(test.attrib['time']) result.time = results.TimeAttribute(end=float(test.attrib['time']))
result.err = test.find('system-err').text result.err = test.find('system-err').text
# The command is prepended to system-out, so we need to separate those # The command is prepended to system-out, so we need to separate those
...@@ -263,7 +263,7 @@ def _load(results_file): ...@@ -263,7 +263,7 @@ def _load(results_file):
result.out = '\n'.join(out[1:]) result.out = '\n'.join(out[1:])
run_result.tests[name] = result run_result.tests[name] = result
return run_result return run_result
......
...@@ -269,14 +269,13 @@ def run(input_): ...@@ -269,14 +269,13 @@ def run(input_):
profile = framework.profile.merge_test_profiles(args.test_profile) profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path profile.results_dir = args.results_path
time_start = time.time() results.time_elapsed.start = time.time()
# Set the dmesg type # Set the dmesg type
if args.dmesg: if args.dmesg:
profile.dmesg = args.dmesg profile.dmesg = args.dmesg
profile.run(opts, args.log_level, backend) profile.run(opts, args.log_level, backend)
time_end = time.time()
results.time_elapsed = time_end - time_start results.time_elapsed.end = time.time()
backend.finalize({'time_elapsed': results.time_elapsed}) backend.finalize({'time_elapsed': results.time_elapsed})
print('Thank you for running Piglit!\n' print('Thank you for running Piglit!\n'
......
...@@ -151,7 +151,7 @@ class TestResult(object): ...@@ -151,7 +151,7 @@ class TestResult(object):
def __init__(self, result=None): def __init__(self, result=None):
self.returncode = None self.returncode = None
self.time = float() self.time = TimeAttribute()
self.command = str() self.command = str()
self.environment = str() self.environment = str()
self.subtests = Subtests() self.subtests = Subtests()
...@@ -214,9 +214,8 @@ class TestResult(object): ...@@ -214,9 +214,8 @@ class TestResult(object):
# pylint: disable=assigning-non-slot # pylint: disable=assigning-non-slot
inst = cls() inst = cls()
# TODO: There's probably a more clever way to do this for each in ['returncode', 'command', 'exception', 'environment',
for each in ['returncode', 'time', 'command', 'exception', 'time', 'result', 'dmesg']:
'environment', 'result', 'dmesg']:
if each in dict_: if each in dict_:
setattr(inst, each, dict_[each]) setattr(inst, each, dict_[each])
...@@ -284,7 +283,7 @@ class TestrunResult(object): ...@@ -284,7 +283,7 @@ class TestrunResult(object):
self.wglinfo = None self.wglinfo = None
self.clinfo = None self.clinfo = None
self.lspci = None self.lspci = None
self.time_elapsed = None self.time_elapsed = TimeAttribute()
self.tests = {} self.tests = {}
self.totals = collections.defaultdict(Totals) self.totals = collections.defaultdict(Totals)
...@@ -330,15 +329,12 @@ class TestrunResult(object): ...@@ -330,15 +329,12 @@ class TestrunResult(object):
""" """
res = cls() res = cls()
for name in ['name', 'uname', 'options', 'glxinfo', 'wglinfo', 'lspci', for name in ['name', 'uname', 'options', 'glxinfo', 'wglinfo', 'lspci',
'tests', 'totals', 'results_version', 'clinfo']: 'time_elapsed', 'tests', 'totals', 'results_version',
'clinfo']:
value = dict_.get(name) value = dict_.get(name)
if value: if value:
setattr(res, name, value) setattr(res, name, value)
# This could be replaced with a getter/setter property
time = dict_.get('time_elapsed')
res.time_elapsed = None if time is None else float(time)
if not res.totals and not _no_totals: if not res.totals and not _no_totals:
res.calculate_group_totals() res.calculate_group_totals()
......
...@@ -24,7 +24,6 @@ ...@@ -24,7 +24,6 @@
from __future__ import absolute_import, division, print_function from __future__ import absolute_import, division, print_function
import itertools import itertools
import datetime
import re import re
import operator import operator
...@@ -267,13 +266,6 @@ def escape_pathname(key): ...@@ -267,13 +266,6 @@ def escape_pathname(key):
return re.sub(r'[/\\]', '_', key) return re.sub(r'[/\\]', '_', key)
def time_as_delta(time):
"""Convert time to a time delta, or return None."""
if time is not None:
return datetime.timedelta(0, time)
return None
def find_diffs(results, tests, comparator, handler=lambda *a: None): def find_diffs(results, tests, comparator, handler=lambda *a: None):
"""Generate diffs between two or more sets of results. """Generate diffs between two or more sets of results.
......
...@@ -36,7 +36,7 @@ from mako.lookup import TemplateLookup ...@@ -36,7 +36,7 @@ from mako.lookup import TemplateLookup
# the module # the module
from framework import backends, exceptions from framework import backends, exceptions
from .common import Results, escape_filename, escape_pathname, time_as_delta from .common import Results, escape_filename, escape_pathname
__all__ = [ __all__ = [
'html', 'html',
...@@ -85,7 +85,7 @@ def _make_testrun_info(results, destination, exclude): ...@@ -85,7 +85,7 @@ def _make_testrun_info(results, destination, exclude):
out.write(_TEMPLATES.get_template('testrun_info.mako').render( out.write(_TEMPLATES.get_template('testrun_info.mako').render(
name=each.name, name=each.name,
totals=each.totals['root'], totals=each.totals['root'],
time=time_as_delta(each.time_elapsed), time=each.time_elapsed.delta,
options=each.options, options=each.options,
uname=each.uname, uname=each.uname,
glxinfo=each.glxinfo, glxinfo=each.glxinfo,
...@@ -105,9 +105,6 @@ def _make_testrun_info(results, destination, exclude): ...@@ -105,9 +105,6 @@ def _make_testrun_info(results, destination, exclude):
if not os.path.exists(temp_path): if not os.path.exists(temp_path):
os.makedirs(temp_path) os.makedirs(temp_path)
if value.time:
value.time = time_as_delta(value.time)
with open(html_path, 'w') as out: with open(html_path, 'w') as out:
out.write(_TEMPLATES.get_template( out.write(_TEMPLATES.get_template(
'test_result.mako').render( 'test_result.mako').render(
......
...@@ -177,10 +177,10 @@ class Test(object): ...@@ -177,10 +177,10 @@ class Test(object):
# Run the test # Run the test
if self.OPTS.execute: if self.OPTS.execute:
try: try:
time_start = time.time() self.result.time.start = time.time()
dmesg.update_dmesg() dmesg.update_dmesg()
self.run() self.run()
self.result.time = time.time() - time_start self.result.time.end = time.time()
self.result = dmesg.update_result(self.result) self.result = dmesg.update_result(self.result)
# This is a rare case where a bare exception is okay, since we're # This is a rare case where a bare exception is okay, since we're
# using it to log exceptions # using it to log exceptions
......
...@@ -752,3 +752,68 @@ class TestV6toV7(object): ...@@ -752,3 +752,68 @@ class TestV6toV7(object):
"""backends.json.update_results (6 -> 7): Totals are populated""" """backends.json.update_results (6 -> 7): Totals are populated"""
nt.ok_(self.result.totals != {}) nt.ok_(self.result.totals != {})
class TestV7toV8(object):
DATA = {
"results_version": 7,
"name": "test",
"options": {
"profile": ['quick'],
"dmesg": False,
"verbose": False,
"platform": "gbm",
"sync": False,
"valgrind": False,
"filter": [],
"concurrent": "all",
"test_count": 0,
"exclude_tests": [],
"exclude_filter": [],
"env": {
"lspci": "stuff",
"uname": "more stuff",
"glxinfo": "and stuff",
"wglinfo": "stuff"
}
},
"tests": {
'a@test': results.TestResult('pass'),
},
"time_elapsed": 1.2,
}
@classmethod
def setup_class(cls):
"""Class setup. Create a TestrunResult with v4 data."""
cls.DATA['tests']['a@test'] = cls.DATA['tests']['a@test'].to_json()
cls.DATA['tests']['a@test']['time'] = 1.2
with utils.tempfile(
json.dumps(cls.DATA, default=backends.json.piglit_encoder)) as t:
with open(t, 'r') as f:
cls.result = backends.json._update_seven_to_eight(
backends.json._load(f))
def test_time(self):
"""backends.json.update_results (7 -> 8): test time is stored as start and end"""
nt.eq_(self.result.tests['a@test'].time.start, 0.0)
nt.eq_(self.result.tests['a@test'].time.end, 1.2)
def test_time_inst(self):
"""backends.json.update_results (7 -> 8): test time is a TimeAttribute instance"""
nt.ok_(
isinstance(self.result.tests['a@test'].time, results.TimeAttribute),
msg='Testresult.time should have been TimeAttribute, '
'but was "{}"'.format(type(self.result.tests['a@test'].time)))
def test_time_elapsed_inst(self):
"""backends.json.update_results (7 -> 8): total time is stored as TimeAttribute"""
nt.ok_(
isinstance(self.result.time_elapsed, results.TimeAttribute),
msg='TestrunResult.time_elapsed should have been TimeAttribute, '
'but was "{}"'.format(type(self.result.time_elapsed)))
def test_time_elapsed(self):
"""backends.json.update_results (7 -> 8): total time is stored as start and end"""
nt.eq_(self.result.time_elapsed.start, 0.0)
nt.eq_(self.result.time_elapsed.end, 1.2)
...@@ -75,7 +75,7 @@ class TestJsonOutput(utils.StaticDirectory): ...@@ -75,7 +75,7 @@ class TestJsonOutput(utils.StaticDirectory):
backend.initialize(_create_metadata(args, 'test', core.Options())) backend.initialize(_create_metadata(args, 'test', core.Options()))
with backend.write_test('result') as t: with backend.write_test('result') as t:
t(results.TestResult('pass')) t(results.TestResult('pass'))
backend.finalize({'time_elapsed': 1.22}) backend.finalize({'time_elapsed': results.TimeAttribute(end=1.2)})
with open(os.path.join(cls.tdir, 'results.json'), 'r') as f: with open(os.path.join(cls.tdir, 'results.json'), 'r') as f:
cls.json = json.load(f) cls.json = json.load(f)
......
...@@ -89,7 +89,7 @@ class TestJUnitSingleTest(TestJunitNoTests): ...@@ -89,7 +89,7 @@ class TestJUnitSingleTest(TestJunitNoTests):
cls.test_file = os.path.join(cls.tdir, 'results.xml') cls.test_file = os.path.join(cls.tdir, 'results.xml')
result = results.TestResult() result = results.TestResult()
result.time = 1.2345 result.time.end = 1.2345
result.result = 'pass' result.result = 'pass'
result.out = 'this is stdout' result.out = 'this is stdout'
result.err = 'this is stderr' result.err = 'this is stderr'
...@@ -120,7 +120,7 @@ class TestJUnitMultiTest(TestJUnitSingleTest): ...@@ -120,7 +120,7 @@ class TestJUnitMultiTest(TestJUnitSingleTest):
super(TestJUnitMultiTest, cls).setup_class() super(TestJUnitMultiTest, cls).setup_class()
result = results.TestResult() result = results.TestResult()
result.time = 1.2345 result.time.end = 1.2345
result.result = 'pass' result.result = 'pass'
result.out = 'this is stdout' result.out = 'this is stdout'
result.err = 'this is stderr' result.err = 'this is stderr'
...@@ -152,7 +152,7 @@ def test_junit_replace(): ...@@ -152,7 +152,7 @@ def test_junit_replace():
"""backends.junit.JUnitBackend.write_test(): '{separator}' is replaced with '.'""" """backends.junit.JUnitBackend.write_test(): '{separator}' is replaced with '.'"""
with utils.tempdir() as tdir: with utils.tempdir() as tdir:
result = results.TestResult() result = results.TestResult()
result.time = 1.2345 result.time.end = 1.2345
result.result = 'pass' result.result = 'pass'
result.out = 'this is stdout' result.out = 'this is stdout'
result.err = 'this is stderr' result.err = 'this is stderr'
...@@ -175,7 +175,7 @@ def test_junit_skips_bad_tests(): ...@@ -175,7 +175,7 @@ def test_junit_skips_bad_tests():
"""backends.junit.JUnitBackend: skips illformed tests""" """backends.junit.JUnitBackend: skips illformed tests"""
with utils.tempdir() as tdir: with utils.tempdir() as tdir:
result = results.TestResult() result = results.TestResult()
result.time = 1.2345 result.time.end = 1.2345
result.result = 'pass' result.result = 'pass'
result.out = 'this is stdout' result.out = 'this is stdout'
result.err = 'this is stderr' result.err = 'this is stderr'
...@@ -237,8 +237,8 @@ class TestJUnitLoad(utils.StaticDirectory): ...@@ -237,8 +237,8 @@ class TestJUnitLoad(utils.StaticDirectory):
def test_time(self): def test_time(self):
"""backends.junit._load: Time is loaded correctly.""" """backends.junit._load: Time is loaded correctly."""
time = self.xml().tests[self.testname].time time = self.xml().tests[self.testname].time
nt.assert_is_instance(time, float) nt.assert_is_instance(time, results.TimeAttribute)
nt.assert_equal(time, 1.12345) nt.assert_equal(time.total, 1.12345)
def test_command(self): def test_command(self):
"""backends.junit._load: command is loaded correctly.""" """backends.junit._load: command is loaded correctly."""
......
...@@ -469,7 +469,7 @@ class TestTestrunResultToJson(object): ...@@ -469,7 +469,7 @@ class TestTestrunResultToJson(object):
test.clinfo = 'clinfo' test.clinfo = 'clinfo'
test.wglinfo = 'wglinfo' test.wglinfo = 'wglinfo'
test.lspci = 'this is lspci' test.lspci = 'this is lspci'
test.time_elapsed = 1.23 test.time_elapsed.end = 1.23
test.tests = {'a test': results.TestResult('pass')} test.tests = {'a test': results.TestResult('pass')}
cls.test = test.to_json() cls.test = test.to_json()
...@@ -504,7 +504,7 @@ class TestTestrunResultToJson(object): ...@@ -504,7 +504,7 @@ class TestTestrunResultToJson(object):
def test_time(self): def test_time(self):
"""results.TestrunResult.to_json: time_elapsed is properly encoded""" """results.TestrunResult.to_json: time_elapsed is properly encoded"""
nt.eq_(self.test['time_elapsed'], 1.23) nt.eq_(self.test['time_elapsed'].end, 1.23)
def test_tests(self): def test_tests(self):
"""results.TestrunResult.to_json: tests is properly encoded""" """results.TestrunResult.to_json: tests is properly encoded"""
...@@ -530,7 +530,7 @@ class TestTestrunResultFromDict(object): ...@@ -530,7 +530,7 @@ class TestTestrunResultFromDict(object):
test.wglinfo = 'wglinfo' test.wglinfo = 'wglinfo'
test.clinfo = 'clinfo' test.clinfo = 'clinfo'
test.lspci = 'this is lspci' test.lspci = 'this is lspci'
test.time_elapsed = 1.23 test.time_elapsed.end = 1.23
test.tests = { test.tests = {
'a test': results.TestResult('pass'), 'a test': results.TestResult('pass'),
'subtest': subtest, 'subtest': subtest,
...@@ -548,7 +548,7 @@ class TestTestrunResultFromDict(object): ...@@ -548,7 +548,7 @@ class TestTestrunResultFromDict(object):
nt.eq_(baseline, test) nt.eq_(baseline, test)
for attrib in ['name', 'uname', 'glxinfo', 'wglinfo', 'lspci', for attrib in ['name', 'uname', 'glxinfo', 'wglinfo', 'lspci',
'time_elapsed', 'results_version', 'clinfo']: 'results_version', 'clinfo']:
test.description = ('results.TestrunResult.from_dict: ' test.description = ('results.TestrunResult.from_dict: '
'{} is restored correctly'.format(attrib)) '{} is restored correctly'.format(attrib))
yield (test, yield (test,
...@@ -582,6 +582,17 @@ class TestTestrunResultFromDict(object): ...@@ -582,6 +582,17 @@ class TestTestrunResultFromDict(object):
msg='Subtests should be type Status, but was "{}"'.format( msg='Subtests should be type Status, but was "{}"'.format(
type(self.test.tests['subtest'].subtests['foo']))) type(self.test.tests['subtest'].subtests['foo'])))
def test_time_elapsed(self):
"""results.TestrunRresult.from_dict: time_elapsed is restored correctly
"""
nt.eq_(self.baseline.time_elapsed.end, self.test.time_elapsed.end)
def test_time(self):
"""results.TestrunResult.from_dict: time_elapsed is TimeAttribute instance"""
nt.ok_(isinstance(self.test.time_elapsed, results.TimeAttribute),
msg='time_elapsed should be tpe TimeAttribute, '
'but was "{}"'.format(type(self.test.time_elapsed)))
def test_TimeAttribute_to_json(): def test_TimeAttribute_to_json():
"""results.TimeAttribute.to_json(): returns expected dictionary""" """results.TimeAttribute.to_json(): returns expected dictionary"""
......
...@@ -344,19 +344,6 @@ def test_Results_get_results_missing_subtest(): ...@@ -344,19 +344,6 @@ def test_Results_get_results_missing_subtest():
[status.PASS, status.NOTRUN]) [status.PASS, status.NOTRUN])
def test_time_as_delta():
"""summary.time_as_delta: converts a time into a delta"""
input_ = 1.2
expected = datetime.timedelta(0, input_)
nt.eq_(expected, summary.time_as_delta(input_))
def test_time_as_delta_none():
"""summary.time_as_delta: returns None when input is None"""
nt.eq_(None, summary.time_as_delta(None))
def test_escape_filename(): def test_escape_filename():
"""summary.escape_filename: replaces invalid characters with '_'""" """summary.escape_filename: replaces invalid characters with '_'"""
invalid = r'<>:"|?*#' invalid = r'<>:"|?*#'
......
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
</tr> </tr>
<tr> <tr>
<td>Time</td> <td>Time</td>
<td>${value.time}</b> <td>${value.time.delta}</b>
</tr> </tr>
% if value.images: % if value.images:
<tr> <tr>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment