Commit 400dcd5c authored by Dylan Baker's avatar Dylan Baker

tests: use meta profiles

This is a rather sizeable commit, I'm sorry for that. Basically there
was no clean way I could come up with to make the change from a single
python file to multiple xml files.

The reason we want this is to to avoid duplication amongst profiles (and
very long build times), by allowing a single profile to be included in
multiple other profiles. This means that the all profile is now composed
of the "glslparser", "shader", and "opengl" (everything in all that's
not glslparser, asmparser, or shader tests). Running `piglit run all
foo` still produces the same results. This also allows us to mix native
and non-native tests so that traditional targets like quick_cl still
work.
Tested-by: Rafael Antognolli's avatarRafael Antognolli <rafael.antognolli@intel.com>
parent f387aa9a
......@@ -533,7 +533,7 @@ install (
install (
DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
FILES_MATCHING REGEX ".*\\.(py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
FILES_MATCHING REGEX ".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
REGEX "CMakeFiles|CMakeLists" EXCLUDE
)
......
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>opengl</Profile>
<Profile>shader</Profile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
"""Profile that removes all GPU based tests.
This profile is the inverse of gpu.py.
It runs GLSLParserTests, asmparsertests, and ARB_vertex_program and
ARB_fragment_program tests only.
Using driver specific overrides these can be forced to run on arbitrary
hardware.
"""
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from tests.quick import profile as _profile
from framework.test import GLSLParserTest
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
def filter_gpu(name, test):
"""Remove all tests that are run on the GPU."""
if isinstance(test, GLSLParserTest) or name.startswith('asmparsertest'):
return True
return False
profile.filters.append(filter_gpu)
......@@ -3,12 +3,58 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import os
from framework.test import GLSLParserTest
from tests.all import profile as _profile
from framework import grouptools
from framework.profile import TestProfile
from framework.test.glsl_parser_test import GLSLParserTest, GLSLParserNoConfigError
from framework.test.piglit_test import ASMParserTest, ROOT_DIR
from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
profile = TestProfile()
profile.filters.append(lambda _, t: isinstance(t, GLSLParserTest))
# Find and add all shader tests.
basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
for dirpath, _, filenames in os.walk(basedir):
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
if ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
try:
test = GLSLParserTest.new(
os.path.join(os.path.relpath(dirpath, basepath), filename))
except GLSLParserNoConfigError:
# In the event that there is no config assume that it is a
# legacy test, and continue
continue
# For glslparser tests you can have multiple tests with the
# same name, but a different stage, so keep the extension.
testname = filename
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
# Collect and add all asmparsertests
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
_basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
for dirpath, _, filenames in os.walk(_basedir):
base_group = grouptools.from_path(os.path.join(
'asmparsertest', os.path.relpath(dirpath, _basedir)))
type_ = os.path.basename(dirpath)
dirname = os.path.relpath(dirpath, ROOT_DIR)
for filename in filenames:
if not os.path.splitext(filename)[1] == '.txt':
continue
group = grouptools.join(base_group, filename)
profile.test_list[group] = ASMParserTest(
type_, os.path.join(dirname, filename))
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>quick_gl</Profile>
<Profile>quick_shader</Profile>
</PiglitMetaProfile>
# -*- coding: utf-8 -*-
# quick.tests minus compiler tests.
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from tests.quick import profile as _profile
from framework.test import GLSLParserTest
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
# Remove all parser tests, as they are compiler test
profile.filters.append(lambda p, t: not isinstance(t, GLSLParserTest))
profile.filters.append(lambda n, _: not n.startswith('asmparsertest'))
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>llvmpipe_gl</Profile>
<Profile>quick_shader</Profile>
</PiglitMetaProfile>
......@@ -8,7 +8,7 @@ import platform
import sys
from framework.grouptools import join
from tests.gpu import profile as _profile
from tests.quick_gl import profile as _profile
__all__ = ['profile']
......
......@@ -3,26 +3,27 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import itertools
import six
from tests.gpu import profile as _profile
from tests.quick_gl import profile as _profile1
from tests.quick_shader import profile as _profile2
from framework.test import PiglitGLTest
from framework.test.shader_test import ShaderTest, MultiShaderTest
from framework.profile import TestDict
from framework.profile import TestProfile
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
# Save the old test_list, but create a new one to contain the modified tests
old_test_list = profile.test_list
profile.test_list = TestDict()
# Save the filters from the original profiles to a new profile
profile = TestProfile()
profile.filters = _profile1.filters + _profile2.filters
# Add a modified version of each PiglitGLTest as a khr_no_error variant.
# Shader runner doesn't explitly test for expected errors so we add shader
# tests as is. We actively filter GLSL parser and any other type of tests.
for name, test in six.iteritems(old_test_list):
for name, test in itertools.chain(six.iteritems(_profile1.test_list),
six.iteritems(_profile2.test_list)):
if isinstance(test, (PiglitGLTest, ShaderTest, MultiShaderTest)):
profile.test_list['{} khr_no_error'.format(name)] = test
test.command += ['-khr_no_error']
#!/usr/bin/env python
#
# encoding=utf-8
# Copyright 2014 Advanced Micro Devices, Inc.
# Copyright © 2018 Intel Corporation
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
......@@ -20,18 +20,15 @@
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# Authors: Tom Stellard <thomas.stellard@amd.com>
#
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
from tests.cl import profile as _profile
from framework.profile import TestProfile
from framework.test import add_opencv_tests, add_oclconform_tests
profile = _profile.copy() # pylint: disable=invalid-name
profile = TestProfile()
add_opencv_tests(profile)
add_oclconform_tests(profile)
......@@ -4,12 +4,10 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import collections
import itertools
import os
import platform
import six
from six.moves import range
from framework import grouptools
......@@ -18,18 +16,13 @@ from framework import options
from framework import wflinfo
from framework.profile import TestProfile
from framework.driver_classifier import DriverClassifier
from framework.test import (PiglitGLTest, PiglitBaseTest,
GLSLParserTest, GLSLParserNoConfigError)
from framework.test.shader_test import ShaderTest, MultiShaderTest
from framework.test.piglit_test import (
ASMParserTest, BuiltInConstantsTest, ROOT_DIR
PiglitGLTest, PiglitBaseTest, BuiltInConstantsTest
)
from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
__all__ = ['profile']
PROCESS_ISOLATION = options.OPTIONS.process_isolation
# Disable bad hanging indent errors in pylint
# There is a bug in pylint which causes the profile.test_list.group_manager to
# be tagged as bad hanging indent, even though it seems to be correct (and
......@@ -213,8 +206,6 @@ def power_set(s):
# Collecting all tests
profile = TestProfile() # pylint: disable=invalid-name
shader_tests = collections.defaultdict(list)
wfl_info = wflinfo.WflInfo()
......@@ -295,73 +286,6 @@ def walk_filter_dir_tree(root):
return retval
# Find and add all shader tests.
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
if os.environ.get("PIGLIT_FILTER_DIRECTORIES"):
files = walk_filter_dir_tree(basedir)
else:
files = os.walk(basedir)
for dirpath, _, filenames in files:
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
dirname = os.path.relpath(dirpath, ROOT_DIR)
if ext == '.shader_test':
if PROCESS_ISOLATION:
test = ShaderTest.new(os.path.join(dirname, filename))
else:
shader_tests[groupname].append(os.path.join(dirpath, filename))
continue
elif ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
try:
test = GLSLParserTest.new(os.path.join(dirname, filename))
except GLSLParserNoConfigError:
# In the event that there is no config assume that it is a
# legacy test, and continue
continue
# For glslparser tests you can have multiple tests with the
# same name, but a different stage, so keep the extension.
testname = filename
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
# Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
# this dictionary is constructed, then added to the actual test dictionary.
for group, files in six.iteritems(shader_tests):
assert group not in profile.test_list, 'duplicate group: {}'.format(group)
# If there is only one file in the directory use a normal shader_test.
# Otherwise use a MultiShaderTest
if len(files) == 1:
group = grouptools.join(
group, os.path.basename(os.path.splitext(files[0])[0]))
profile.test_list[group] = ShaderTest.new(files[0])
else:
profile.test_list[group] = MultiShaderTest.new(files)
# Collect and add all asmparsertests
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
_basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
for dirpath, _, filenames in os.walk(_basedir):
base_group = grouptools.from_path(os.path.join(
'asmparsertest', os.path.relpath(dirpath, _basedir)))
type_ = os.path.basename(dirpath)
dirname = os.path.relpath(dirpath, ROOT_DIR)
for filename in filenames:
if not os.path.splitext(filename)[1] == '.txt':
continue
group = grouptools.join(base_group, filename)
profile.test_list[group] = ASMParserTest(
type_, os.path.join(dirname, filename))
# Find and add all apitrace tests.
classifier = DriverClassifier()
for basedir in [os.path.join(TESTS_DIR, 'apitrace', 'traces')]:
......
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>quick_gl</Profile>
<Profile>quick_shader</Profile>
<Profile>glslparser</Profile>
</PiglitMetaProfile>
<?xml version="1.0" encoding="utf-8"?>
<PiglitMetaProfile>
<Profile>opencl_foreign</Profile>
<Profile>cl</Profile>
</PiglitMetaProfile>
......@@ -13,12 +13,10 @@ don't want that level of exhaustiveness, so this filter removes 80% in a random
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import random
from framework import grouptools
from framework.test import PiglitGLTest
from framework.test.shader_test import ShaderTest
from tests.all import profile as _profile
from tests.opengl import profile as _profile
__all__ = ['profile']
......@@ -26,20 +24,6 @@ __all__ = ['profile']
# pylint: disable=bad-continuation
class FilterVsIn(object):
"""Filter out 80% of the Vertex Attrib 64 vs_in tests."""
def __init__(self):
self.random = random.Random()
self.random.seed(42)
def __call__(self, name, test):
if isinstance(test, ShaderTest) and 'vs_in' in grouptools.split(name):
# 20%
return self.random.random() <= .2
return True
profile = _profile.copy() # pylint: disable=invalid-name
# Set the --quick flag on a few image_load_store_tests
......@@ -92,7 +76,3 @@ with profile.test_list.group_manager(
g(['arb_texture_multisample-large-float-texture', '--array',
'--fp16'] + size_arg,
'large-float-texture-array-fp16', run_concurrent=False)
# These take too long
profile.filters.append(lambda n, _: '-explosion' not in n)
profile.filters.append(FilterVsIn())
# encoding=utf-8
# Copyright © 2018 Intel Coproration
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import random
from framework import grouptools
from .shader import profile as _profile
__all__ = ['profile']
profile = _profile.copy()
# There are too many of theset, it's simply exhaustive to run them all.
class FilterVsIn(object):
"""Filter out 80% of the Vertex Attrib 64 vs_in tests."""
def __init__(self):
self.random = random.Random()
self.random.seed(42)
def __call__(self, name, _):
if 'vs_in' in grouptools.split(name):
# 20%
return self.random.random() <= .2
return True
profile.filters.append(FilterVsIn())
# These take too long
profile.filters.append(lambda n, _: '-explosion' not in n)
......@@ -3,12 +3,54 @@
from __future__ import (
absolute_import, division, print_function, unicode_literals
)
import collections
import os
import six
from framework.options import OPTIONS
from framework import grouptools
from framework.profile import TestProfile
from framework.test.shader_test import ShaderTest, MultiShaderTest
from tests.all import profile as _profile
from .py_modules.constants import GENERATED_TESTS_DIR, TESTS_DIR
__all__ = ['profile']
profile = _profile.copy() # pylint: disable=invalid-name
profile = TestProfile()
shader_tests = collections.defaultdict(list)
# Find and add all shader tests.
basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
for dirpath, _, filenames in os.walk(basedir):
groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
for filename in filenames:
testname, ext = os.path.splitext(filename)
if ext == '.shader_test':
dirname = os.path.relpath(dirpath, basepath)
if OPTIONS.process_isolation:
test = ShaderTest.new(os.path.join(dirname, filename))
else:
shader_tests[groupname].append(os.path.join(dirname, filename))
continue
else:
continue
group = grouptools.join(groupname, testname)
assert group not in profile.test_list, group
profile.test_list[group] = test
profile.filters.append(lambda _, t: isinstance(t, (ShaderTest, MultiShaderTest)))
# Because we need to handle duplicate group names in TESTS and GENERATED_TESTS
# this dictionary is constructed, then added to the actual test dictionary.
for group, files in six.iteritems(shader_tests):
assert group not in profile.test_list, 'duplicate group: {}'.format(group)
# If there is only one file in the directory use a normal shader_test.
# Otherwise use a MultiShaderTest
if len(files) == 1:
group = grouptools.join(
group, os.path.basename(os.path.splitext(files[0])[0]))
profile.test_list[group] = ShaderTest.new(files[0])
else:
profile.test_list[group] = MultiShaderTest.new(files)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment