Commit 400dcd5c authored by Dylan Baker's avatar Dylan Baker

tests: use meta profiles

This is a rather sizeable commit, I'm sorry for that. Basically there
was no clean way I could come up with to make the change from a single
python file to multiple xml files.

The reason we want this is to to avoid duplication amongst profiles (and
very long build times), by allowing a single profile to be included in
multiple other profiles. This means that the all profile is now composed
of the "glslparser", "shader", and "opengl" (everything in all that's
not glslparser, asmparser, or shader tests). Running `piglit run all
foo` still produces the same results. This also allows us to mix native
and non-native tests so that traditional targets like quick_cl still
work.
Tested-by: Rafael Antognolli's avatarRafael Antognolli <rafael.antognolli@intel.com>
parent f387aa9a
...@@ -533,7 +533,7 @@ install ( ...@@ -533,7 +533,7 @@ install (
install ( install (
DIRECTORY tests DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR} DESTINATION ${PIGLIT_INSTALL_LIBDIR}
FILES_MATCHING REGEX ".*\\.(py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$" FILES_MATCHING REGEX ".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
REGEX "CMakeFiles|CMakeLists" EXCLUDE REGEX "CMakeFiles|CMakeLists" EXCLUDE
) )
......
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>opengl</Profile>
<Profile>shader</Profile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
"""Profile that removes all GPU based tests.
This profile is the inverse of gpu.py.
It runs GLSLParserTests, asmparsertests, and ARB_vertex_program and
ARB_fragment_program tests only.
Using driver specific overrides these can be forced to run on arbitrary
hardware.
"""
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from tests.quick import profile as _profile
from framework.test import GLSLParserTest
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
def filter_gpu(name, test):
"""Remove all tests that are run on the GPU."""
if isinstance(test, GLSLParserTest) or name.startswith('asmparsertest'):
return True
return False
profile.filters.append(filter_gpu)
...@@ -3,12 +3,58 @@ ...@@ -3,12 +3,58 @@
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
import os
from framework.test import GLSLParserTest from framework import grouptools
from tests.all import profile as _profile from framework.profile import TestProfile
from framework.test.glsl_parser_test import GLSLParserTest, GLSLParserNoConfigError
from framework.test.piglit_test import ASMParserTest, ROOT_DIR
from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
__all__ = ['profile'] __all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name profile = TestProfile()
profile.filters.append(lambda _, t: isinstance(t, GLSLParserTest)) # Find and add all shader tests.
basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
for dirpath, _, filenames in os.walk(basedir):
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
if ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
try:
test = GLSLParserTest.new(
os.path.join(os.path.relpath(dirpath, basepath), filename))
except GLSLParserNoConfigError:
# In the event that there is no config assume that it is a
# legacy test, and continue
continue
# For glslparser tests you can have multiple tests with the
# same name, but a different stage, so keep the extension.
testname = filename
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
# Collect and add all asmparsertests
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
_basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
for dirpath, _, filenames in os.walk(_basedir):
base_group = grouptools.from_path(os.path.join(
'asmparsertest', os.path.relpath(dirpath, _basedir)))
type_ = os.path.basename(dirpath)
dirname = os.path.relpath(dirpath, ROOT_DIR)
for filename in filenames:
if not os.path.splitext(filename)[1] == '.txt':
continue
group = grouptools.join(base_group, filename)
profile.test_list[group] = ASMParserTest(
type_, os.path.join(dirname, filename))
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>quick_gl</Profile>
<Profile>quick_shader</Profile>
</PiglitMetaProfile>
# -*- coding: utf-8 -*-
# quick.tests minus compiler tests.
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from tests.quick import profile as _profile
from framework.test import GLSLParserTest
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
# Remove all parser tests, as they are compiler test
profile.filters.append(lambda p, t: not isinstance(t, GLSLParserTest))
profile.filters.append(lambda n, _: not n.startswith('asmparsertest'))
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>llvmpipe_gl</Profile>
<Profile>quick_shader</Profile>
</PiglitMetaProfile>
...@@ -8,7 +8,7 @@ import platform ...@@ -8,7 +8,7 @@ import platform
import sys import sys
from framework.grouptools import join from framework.grouptools import join
from tests.gpu import profile as _profile from tests.quick_gl import profile as _profile
__all__ = ['profile'] __all__ = ['profile']
......
...@@ -3,26 +3,27 @@ ...@@ -3,26 +3,27 @@
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
import itertools
import six import six
from tests.gpu import profile as _profile from tests.quick_gl import profile as _profile1
from tests.quick_shader import profile as _profile2
from framework.test import PiglitGLTest from framework.test import PiglitGLTest
from framework.test.shader_test import ShaderTest, MultiShaderTest from framework.test.shader_test import ShaderTest, MultiShaderTest
from framework.profile import TestDict from framework.profile import TestProfile
__all__ = ['profile'] __all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name # Save the filters from the original profiles to a new profile
profile = TestProfile()
# Save the old test_list, but create a new one to contain the modified tests profile.filters = _profile1.filters + _profile2.filters
old_test_list = profile.test_list
profile.test_list = TestDict()
# Add a modified version of each PiglitGLTest as a khr_no_error variant. # Add a modified version of each PiglitGLTest as a khr_no_error variant.
# Shader runner doesn't explitly test for expected errors so we add shader # Shader runner doesn't explitly test for expected errors so we add shader
# tests as is. We actively filter GLSL parser and any other type of tests. # tests as is. We actively filter GLSL parser and any other type of tests.
for name, test in six.iteritems(old_test_list): for name, test in itertools.chain(six.iteritems(_profile1.test_list),
six.iteritems(_profile2.test_list)):
if isinstance(test, (PiglitGLTest, ShaderTest, MultiShaderTest)): if isinstance(test, (PiglitGLTest, ShaderTest, MultiShaderTest)):
profile.test_list['{} khr_no_error'.format(name)] = test profile.test_list['{} khr_no_error'.format(name)] = test
test.command += ['-khr_no_error'] test.command += ['-khr_no_error']
#!/usr/bin/env python # encoding=utf-8
#
# Copyright 2014 Advanced Micro Devices, Inc. # Copyright 2014 Advanced Micro Devices, Inc.
# Copyright © 2018 Intel Corporation
# #
# Permission is hereby granted, free of charge, to any person obtaining a # Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"), # copy of this software and associated documentation files (the "Software"),
...@@ -20,18 +20,15 @@ ...@@ -20,18 +20,15 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE. # SOFTWARE.
#
# Authors: Tom Stellard <thomas.stellard@amd.com>
#
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
from tests.cl import profile as _profile from framework.profile import TestProfile
from framework.test import add_opencv_tests, add_oclconform_tests from framework.test import add_opencv_tests, add_oclconform_tests
profile = _profile.copy() # pylint: disable=invalid-name profile = TestProfile()
add_opencv_tests(profile) add_opencv_tests(profile)
add_oclconform_tests(profile) add_oclconform_tests(profile)
...@@ -4,12 +4,10 @@ ...@@ -4,12 +4,10 @@
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
import collections
import itertools import itertools
import os import os
import platform import platform
import six
from six.moves import range from six.moves import range
from framework import grouptools from framework import grouptools
...@@ -18,18 +16,13 @@ from framework import options ...@@ -18,18 +16,13 @@ from framework import options
from framework import wflinfo from framework import wflinfo
from framework.profile import TestProfile from framework.profile import TestProfile
from framework.driver_classifier import DriverClassifier from framework.driver_classifier import DriverClassifier
from framework.test import (PiglitGLTest, PiglitBaseTest,
GLSLParserTest, GLSLParserNoConfigError)
from framework.test.shader_test import ShaderTest, MultiShaderTest
from framework.test.piglit_test import ( from framework.test.piglit_test import (
ASMParserTest, BuiltInConstantsTest, ROOT_DIR PiglitGLTest, PiglitBaseTest, BuiltInConstantsTest
) )
from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
__all__ = ['profile'] __all__ = ['profile']
PROCESS_ISOLATION = options.OPTIONS.process_isolation
# Disable bad hanging indent errors in pylint # Disable bad hanging indent errors in pylint
# There is a bug in pylint which causes the profile.test_list.group_manager to # There is a bug in pylint which causes the profile.test_list.group_manager to
# be tagged as bad hanging indent, even though it seems to be correct (and # be tagged as bad hanging indent, even though it seems to be correct (and
...@@ -213,8 +206,6 @@ def power_set(s): ...@@ -213,8 +206,6 @@ def power_set(s):
# Collecting all tests # Collecting all tests
profile = TestProfile() # pylint: disable=invalid-name profile = TestProfile() # pylint: disable=invalid-name
shader_tests = collections.defaultdict(list)
wfl_info = wflinfo.WflInfo() wfl_info = wflinfo.WflInfo()
...@@ -295,73 +286,6 @@ def walk_filter_dir_tree(root): ...@@ -295,73 +286,6 @@ def walk_filter_dir_tree(root):
return retval return retval
# Find and add all shader tests.
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
if os.environ.get("PIGLIT_FILTER_DIRECTORIES"):
files = walk_filter_dir_tree(basedir)
else:
files = os.walk(basedir)
for dirpath, _, filenames in files:
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
dirname = os.path.relpath(dirpath, ROOT_DIR)
if ext == '.shader_test':
if PROCESS_ISOLATION:
test = ShaderTest.new(os.path.join(dirname, filename))
else:
shader_tests[groupname].append(os.path.join(dirpath, filename))
continue
elif ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
try:
test = GLSLParserTest.new(os.path.join(dirname, filename))
except GLSLParserNoConfigError:
# In the event that there is no config assume that it is a
# legacy test, and continue
continue
# For glslparser tests you can have multiple tests with the
# same name, but a different stage, so keep the extension.
testname = filename
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
# Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
# this dictionary is constructed, then added to the actual test dictionary.
for group, files in six.iteritems(shader_tests):
assert group not in profile.test_list, 'duplicate group: {}'.format(group)
# If there is only one file in the directory use a normal shader_test.
# Otherwise use a MultiShaderTest
if len(files) == 1:
group = grouptools.join(
group, os.path.basename(os.path.splitext(files[0])[0]))
profile.test_list[group] = ShaderTest.new(files[0])
else:
profile.test_list[group] = MultiShaderTest.new(files)
# Collect and add all asmparsertests
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
_basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
for dirpath, _, filenames in os.walk(_basedir):
base_group = grouptools.from_path(os.path.join(
'asmparsertest', os.path.relpath(dirpath, _basedir)))
type_ = os.path.basename(dirpath)
dirname = os.path.relpath(dirpath, ROOT_DIR)
for filename in filenames:
if not os.path.splitext(filename)[1] == '.txt':
continue
group = grouptools.join(base_group, filename)
profile.test_list[group] = ASMParserTest(
type_, os.path.join(dirname, filename))
# Find and add all apitrace tests. # Find and add all apitrace tests.
classifier = DriverClassifier() classifier = DriverClassifier()
for basedir in [os.path.join(TESTS_DIR, 'apitrace', 'traces')]: for basedir in [os.path.join(TESTS_DIR, 'apitrace', 'traces')]:
......
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>quick_gl</Profile>
<Profile>quick_shader</Profile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>opencl_foreign</Profile>
<Profile>cl</Profile>
</PiglitMetaProfile>
...@@ -13,12 +13,10 @@ don't want that level of exhaustiveness, so this filter removes 80% in a random ...@@ -13,12 +13,10 @@ don't want that level of exhaustiveness, so this filter removes 80% in a random
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
import random
from framework import grouptools from framework import grouptools
from framework.test import PiglitGLTest from framework.test import PiglitGLTest
from framework.test.shader_test import ShaderTest from tests.opengl import profile as _profile
from tests.all import profile as _profile
__all__ = ['profile'] __all__ = ['profile']
...@@ -26,20 +24,6 @@ __all__ = ['profile'] ...@@ -26,20 +24,6 @@ __all__ = ['profile']
# pylint: disable=bad-continuation # pylint: disable=bad-continuation
class FilterVsIn(object):
"""Filter out 80% of the Vertex Attrib 64 vs_in tests."""
def __init__(self):
self.random = random.Random()
self.random.seed(42)
def __call__(self, name, test):
if isinstance(test, ShaderTest) and 'vs_in' in grouptools.split(name):
# 20%
return self.random.random() <= .2
return True
profile = _profile.copy() # pylint: disable=invalid-name profile = _profile.copy() # pylint: disable=invalid-name
# Set the --quick flag on a few image_load_store_tests # Set the --quick flag on a few image_load_store_tests
...@@ -92,7 +76,3 @@ with profile.test_list.group_manager( ...@@ -92,7 +76,3 @@ with profile.test_list.group_manager(
g(['arb_texture_multisample-large-float-texture', '--array', g(['arb_texture_multisample-large-float-texture', '--array',
'--fp16'] + size_arg, '--fp16'] + size_arg,
'large-float-texture-array-fp16', run_concurrent=False) 'large-float-texture-array-fp16', run_concurrent=False)
# These take too long
profile.filters.append(lambda n, _: '-explosion' not in n)
profile.filters.append(FilterVsIn())
# encoding=utf-8
# Copyright © 2018 Intel Coproration
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import random
from framework import grouptools
from .shader import profile as _profile
__all__ = ['profile']
profile = _profile.copy()
# There are too many of theset, it's simply exhaustive to run them all.
class FilterVsIn(object):
"""Filter out 80% of the Vertex Attrib 64 vs_in tests."""
def __init__(self):
self.random = random.Random()
self.random.seed(42)
def __call__(self, name, _):
if 'vs_in' in grouptools.split(name):
# 20%
return self.random.random() <= .2
return True
profile.filters.append(FilterVsIn())
# These take too long
profile.filters.append(lambda n, _: '-explosion' not in n)
...@@ -3,12 +3,54 @@ ...@@ -3,12 +3,54 @@
from __future__ import ( from __future__ import (
absolute_import, division, print_function, unicode_literals absolute_import, division, print_function, unicode_literals
) )
import collections
import os
import six
from framework.options import OPTIONS
from framework import grouptools
from framework.profile import TestProfile
from framework.test.shader_test import ShaderTest, MultiShaderTest from framework.test.shader_test import ShaderTest, MultiShaderTest
from tests.all import profile as _profile from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
__all__ = ['profile'] __all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name profile = TestProfile()
shader_tests = collections.defaultdict(list)
# Find and add all shader tests.
basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
for dirpath, _, filenames in os.walk(basedir):
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
if ext == '.shader_test':
dirname = os.path.relpath(dirpath, basepath)
if OPTIONS.process_isolation:
test = ShaderTest.new(os.path.join(dirname, filename))
else:
shader_tests[groupname].append(os.path.join(dirname, filename))
continue
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
profile.filters.append(lambda _, t: isinstance(t, (ShaderTest, MultiShaderTest))) # Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
# this dictionary is constructed, then added to the actual test dictionary.
for group, files in six.iteritems(shader_tests):
assert group not in profile.test_list, 'duplicate group: {}'.format(group)
# If there is only one file in the directory use a normal shader_test.
# Otherwise use a MultiShaderTest
if len(files) == 1:
group = grouptools.join(
group, os.path.basename(os.path.splitext(files[0])[0]))
profile.test_list[group] = ShaderTest.new(files[0])
else:
profile.test_list[group] = MultiShaderTest.new(files)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment