Commit 9b1e7da2 authored by Dylan Baker's avatar Dylan Baker

framework: move executable functions to modules

This moves piglit-run and piglit-resume main() functions to
framework.programs.run as run() and resume() respectively, and moves
piglit-summary-* to framework.programs.summary. It then creates
compatibility wrappers as piglit-*.py for the functions.

This is groundwork for a combined piglit interface. The original files
could not be imported directly since they have dash (-) in their name,
and python considers dash an illegal character for function, class, and
modules names, and python would raise an ImportError when attempting to
import one of them.
Signed-off-by: Dylan Baker's avatarDylan Baker <baker.dylan.c@gmail.com>
Reviewed-by: Jordan Justen's avatarJordan Justen <jordan.l.justen@intel.com>
parent 463b049b
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
from __future__ import print_function
import argparse
import sys
import os
import os.path as path
import time
import framework.core as core
import framework.profile
__all__ = ['run',
'resume']
def run(input_):
parser = argparse.ArgumentParser()
parser.add_argument("-n", "--name",
metavar="<test name>",
default=None,
help="Name of this test run")
parser.add_argument("-d", "--dry-run",
action="store_false",
dest="execute",
help="Do not execute the tests")
parser.add_argument("-t", "--include-tests",
default=[],
action="append",
metavar="<regex>",
help="Run only matching tests "
"(can be used more than once)")
parser.add_argument("-x", "--exclude-tests",
default=[],
action="append",
metavar="<regex>",
help="Exclude matching tests "
"(can be used more than once)")
conc_parser = parser.add_mutually_exclusive_group()
conc_parser.add_argument('-c', '--all-concurrent',
action="store_const",
default="some",
const="all",
dest="concurrency",
help="Run all tests concurrently")
conc_parser.add_argument("-1", "--no-concurrency",
action="store_const",
default="some",
const="none",
dest="concurrency",
help="Disable concurrent test runs")
parser.add_argument("-p", "--platform",
choices=["glx", "x11_egl", "wayland", "gbm"],
help="Name of windows system passed to waffle")
parser.add_argument("-f", "--config",
dest="config_file",
type=argparse.FileType("r"),
help="Optionally specify a piglit config file to use. "
"Default is piglit.conf")
parser.add_argument("--valgrind",
action="store_true",
help="Run tests in valgrind's memcheck")
parser.add_argument("--dmesg",
action="store_true",
help="Capture a difference in dmesg before and "
"after each test. Implies -1/--no-concurrency")
parser.add_argument("-v", "--verbose",
action="store_true",
help="Produce a line of output for each test before "
"and after it runs")
parser.add_argument("test_profile",
metavar="<Path to one or more test profile(s)>",
nargs='+',
help="Path to testfile to run")
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args(input_)
# Set the platform to pass to waffle
if args.platform:
os.environ['PIGLIT_PLATFORM'] = args.platform
# If dmesg is requested we must have serial run, this is becasue dmesg
# isn't reliable with threaded run
if args.dmesg:
args.concurrency = "none"
# Read the config file
if args.config_file:
core.PIGLIT_CONFIG.readfp(args.config_file)
args.config_file.close()
else:
core.PIGLIT_CONFIG.read(os.path.join(os.path.dirname(__file__),
'piglit.conf'))
# Pass arguments into Environment
env = core.Environment(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,
valgrind=args.valgrind,
dmesg=args.dmesg,
verbose=args.verbose)
# Change working directory to the root of the piglit directory
piglit_dir = path.dirname(path.realpath(sys.argv[0]))
os.chdir(piglit_dir)
core.checkDir(args.results_path, False)
results = core.TestrunResult()
# Set results.name
if args.name is not None:
results.name = args.name
else:
results.name = path.basename(args.results_path)
# Begin json.
result_filepath = path.join(args.results_path, 'main')
result_file = open(result_filepath, 'w')
json_writer = core.JSONWriter(result_file)
json_writer.open_dict()
# Write out command line options for use in resuming.
json_writer.write_dict_key('options')
json_writer.open_dict()
json_writer.write_dict_item('profile', args.test_profile)
for key, value in env:
json_writer.write_dict_item(key, value)
if args.platform:
json_writer.write_dict_item('platform', args.platform)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
profile = framework.profile.merge_test_profiles(args.test_profile)
profile.results_dir = args.results_path
json_writer.write_dict_key('tests')
json_writer.open_dict()
time_start = time.time()
# Set the dmesg type
if args.dmesg:
profile.dmesg = args.dmesg
profile.run(env, json_writer)
time_end = time.time()
json_writer.close_dict()
results.time_elapsed = time_end - time_start
json_writer.write_dict_item('time_elapsed', results.time_elapsed)
# End json.
json_writer.close_dict()
json_writer.file.close()
print('Thank you for running Piglit!\n'
'Results have been written to ' + result_filepath)
def resume(input_):
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args(input_)
results = core.load_results(args.results_path)
env = core.Environment(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
verbose=results.options['verbose'])
# attempt to restore a saved platform, if there is no saved platform just
# go on
try:
os.environ['PIGLIT_PLATFORM'] = results.options['platform']
except KeyError:
pass
results_path = path.join(args.results_path, "main")
json_writer = core.JSONWriter(open(results_path, 'w+'))
json_writer.open_dict()
json_writer.write_dict_key("options")
json_writer.open_dict()
for key, value in results.options.iteritems():
json_writer.write_dict_item(key, value)
json_writer.close_dict()
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
if env.dmesg:
profile.dmesg = env.dmesg
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(env, json_writer)
json_writer.close_dict()
json_writer.close_dict()
json_writer.file.close()
print("Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
import argparse
import shutil
import os.path as path
import framework.summary as summary
import framework.status as status
import framework.core as core
import framework.junit
__all__ = ['html',
'junit',
'console']
def html(input_):
# Make a copy of the status text list and add all. This is used as the
# argument list for -e/--exclude
statuses = set(str(s) for s in status.ALL)
statuses.add('all')
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--overwrite",
action="store_true",
help="Overwrite existing directories")
parser.add_argument("-l", "--list",
action="store",
help="Load a newline seperated list of results. These "
"results will be prepended to any Results "
"specified on the command line")
parser.add_argument("-e", "--exclude-details",
default=[],
action="append",
choices=statuses,
help="Optionally exclude the generation of HTML pages "
"for individual test pages with the status(es) "
"given as arguments. This speeds up HTML "
"generation, but reduces the info in the HTML "
"pages. May be used multiple times")
parser.add_argument("summaryDir",
metavar="<Summary Directory>",
help="Directory to put HTML files in")
parser.add_argument("resultsFiles",
metavar="<Results Files>",
nargs="*",
help="Results files to include in HTML")
args = parser.parse_args(input_)
# If args.list and args.resultsFiles are empty, then raise an error
if not args.list and not args.resultsFiles:
raise parser.error("Missing required option -l or <resultsFiles>")
# Convert the exclude_details list to status objects, without this using
# the -e option will except
if args.exclude_details:
# If exclude-results has all, then change it to be all
if 'all' in args.exclude_details:
args.exclude_details = status.ALL
else:
args.exclude_details = frozenset(
status.status_lookup(i) for i in args.exclude_details)
# if overwrite is requested delete the output directory
if path.exists(args.summaryDir) and args.overwrite:
shutil.rmtree(args.summaryDir)
# If the requested directory doesn't exist, create it or throw an error
core.checkDir(args.summaryDir, not args.overwrite)
# Merge args.list and args.resultsFiles
if args.list:
args.resultsFiles.extend(core.parse_listfile(args.list))
# Create the HTML output
output = summary.Summary(args.resultsFiles)
output.generate_html(args.summaryDir, args.exclude_details)
class _Writer:
def __init__(self, filename):
self.report = framework.junit.Report(filename)
self.path = []
def write(self, arg):
testrun = core.load_results(arg)
self.report.start()
self.report.startSuite('piglit')
try:
for name, result in testrun.tests.iteritems():
self.write_test(testrun, name, result)
finally:
self.enter_path([])
self.report.stopSuite()
self.report.stop()
def write_test(self, testrun, test_path, result):
test_path = test_path.split('/')
test_name = test_path.pop()
self.enter_path(test_path)
self.report.startCase(test_name)
duration = None
try:
try:
command = result['command']
except KeyError:
pass
else:
self.report.addStdout(command + '\n')
try:
stdout = result['out']
except KeyError:
pass
else:
if stdout:
self.report.addStdout(stdout + '\n')
try:
stderr = result['err']
except KeyError:
pass
else:
if stderr:
self.report.addStderr(stderr + '\n')
try:
returncode = result['returncode']
except KeyError:
pass
else:
if returncode:
self.report.addStderr('returncode = %s\n' % returncode)
success = result.get('result')
if success in (status.PASS, status.WARN):
pass
elif success == status.SKIP:
self.report.addSkipped()
else:
self.report.addFailure(success.name)
try:
duration = float(result['time'])
except KeyError:
pass
finally:
self.report.stopCase(duration)
def enter_path(self, path):
ancestor = 0
try:
while self.path[ancestor] == path[ancestor]:
ancestor += 1
except IndexError:
pass
for dirname in self.path[ancestor:]:
self.report.stopSuite()
for dirname in path[ancestor:]:
self.report.startSuite(dirname)
self.path = path
def junit(input_):
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output",
metavar="<Output File>",
action="store",
dest="output",
default="piglit.xml",
help="Output filename")
parser.add_argument("testResults",
metavar="<Input Files>",
help="JSON results file to be converted")
args = parser.parse_args(input_)
writer = _Writer(args.output)
writer.write(args.testResults)
def console(input_):
parser = argparse.ArgumentParser()
# Set the -d and -s options as exclusive, since it's silly to call for diff
# and then call for only summary
excGroup1 = parser.add_mutually_exclusive_group()
excGroup1.add_argument("-d", "--diff",
action="store_true",
help="Only display the differences between multiple "
"result files")
excGroup1.add_argument("-s", "--summary",
action="store_true",
help="Only display the summary, not the individual "
"test results")
parser.add_argument("-l", "--list",
action="store",
help="Use test results from a list file")
parser.add_argument("results",
metavar="<Results Path(s)>",
nargs="+",
help="Space seperated paths to at least one results "
"file")
args = parser.parse_args(input_)
# Throw an error if -d/--diff is called, but only one results file is
# provided
if args.diff and len(args.results) < 2:
parser.error('-d/--diff cannot be specified unless two or more '
'results files are specified')
# make list of results
if args.list:
args.results.extend(core.parse_listfile(args.list))
# Generate the output
output = summary.Summary(args.results)
output.generate_text(args.diff, args.summary)
#!/usr/bin/env python
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
from __future__ import print_function
import sys
import os
import os.path as path
import argparse
import framework.core as core
import framework.profile
def main():
parser = argparse.ArgumentParser()
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
# Copyright (c) 2014 Intel Corporation
results = core.load_results(args.results_path)
env = core.Environment(concurrent=results.options['concurrent'],
exclude_filter=results.options['exclude_filter'],
include_filter=results.options['filter'],
execute=results.options['execute'],
valgrind=results.options['valgrind'],
dmesg=results.options['dmesg'],
verbose=results.options['verbose'])
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# Change working directory to the piglit directory
os.chdir(path.dirname(path.realpath(sys.argv[0])))
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# attempt to restore a saved platform, if there is no saved platform just
# go on
try:
os.environ['PIGLIT_PLATFORM'] = results.options['platform']
except KeyError:
pass
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
results_path = path.join(args.results_path, "main")
json_writer = core.JSONWriter(open(results_path, 'w+'))
json_writer.open_dict()
json_writer.write_dict_key("options")
json_writer.open_dict()
for key, value in results.options.iteritems():
json_writer.write_dict_item(key, value)
json_writer.close_dict()
""" Resume an interupted piglit run
json_writer.write_dict_item('name', results.name)
for (key, value) in env.collectData().items():
json_writer.write_dict_item(key, value)
Deprecated compatability wrapper
json_writer.write_dict_key('tests')
json_writer.open_dict()
for key, value in results.tests.iteritems():
json_writer.write_dict_item(key, value)
env.exclude_tests.add(key)
profile = framework.profile.merge_test_profiles(results.options['profile'])
profile.results_dir = args.results_path
if env.dmesg:
profile.dmesg = env.dmesg
"""
# This is resumed, don't bother with time since it wont be accurate anyway
profile.run(env, json_writer)
json_writer.close_dict()
json_writer.close_dict()
json_writer.file.close()
print("Thank you for running Piglit!\n"
"Results have ben wrriten to {0}".format(results_path))
import sys
from framework.programs.run import resume
if __name__ == "__main__":
main()
resume(sys.argv)
#!/usr/bin/env python
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
# Copyright (c) 2014 Intel Corporation
from __future__ import print_function
import argparse
import sys
import os
import os.path as path
import time
sys.path.append(path.dirname(path.realpath(sys.argv[0])))
import framework.core as core
import framework.profile
def main():
parser = argparse.ArgumentParser(sys.argv)
parser.add_argument("-n", "--name",
metavar="<test name>",
default=None,
help="Name of this test run")
parser.add_argument("-d", "--dry-run",
action="store_false",
dest="execute",
help="Do not execute the tests")
parser.add_argument("-t", "--include-tests",
default=[],
action="append",
metavar="<regex>",
help="Run only matching tests "
"(can be used more than once)")
parser.add_argument("-x", "--exclude-tests",
default=[],
action="append",
metavar="<regex>",
help="Exclude matching tests "
"(can be used more than once)")
conc_parser = parser.add_mutually_exclusive_group()
conc_parser.add_argument('-c', '--all-concurrent',
action="store_const",
default="some",
const="all",
dest="concurrency",
help="Run all tests concurrently")
conc_parser.add_argument("-1", "--no-concurrency",
action="store_const",
default="some",
const="none",
dest="concurrency",
help="Disable concurrent test runs")
parser.add_argument("-p", "--platform",
choices=["glx", "x11_egl", "wayland", "gbm"],
help="Name of windows system passed to waffle")
parser.add_argument("-f", "--config",
dest="config_file",
type=argparse.FileType("r"),
help="Optionally specify a piglit config file to use. "
"Default is piglit.conf")
parser.add_argument("--valgrind",
action="store_true",
help="Run tests in valgrind's memcheck")
parser.add_argument("--dmesg",
action="store_true",
help="Capture a difference in dmesg before and "
"after each test. Implies -1/--no-concurrency")
parser.add_argument("-v", "--verbose",
action="store_true",
help="Produce a line of output for each test before "
"and after it runs")
parser.add_argument("test_profile",
metavar="<Path to one or more test profile(s)>",
nargs='+',
help="Path to testfile to run")
parser.add_argument("results_path",
type=path.realpath,
metavar="<Results Path>",
help="Path to results folder")
args = parser.parse_args()
# Set the platform to pass to waffle
if args.platform:
os.environ['PIGLIT_PLATFORM'] = args.platform
# If dmesg is requested we must have serial run, this is becasue dmesg
# isn't reliable with threaded run
if args.dmesg:
args.concurrency = "none"
# Read the config file
if args.config_file:
core.PIGLIT_CONFIG.readfp(args.config_file)
args.config_file.close()
else:
core.PIGLIT_CONFIG.read(os.path.join(os.path.dirname(__file__),
'piglit.conf'))
# Pass arguments into Environment
env = core.Environment(concurrent=args.concurrency,
exclude_filter=args.exclude_tests,
include_filter=args.include_tests,
execute=args.execute,