profile.py 18.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# This permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

""" Provides Profiles for test groups

Each set of tests, both native Piglit profiles and external suite integration,
are represented by a TestProfile or a TestProfile derived object.

"""

29 30 31
from __future__ import (
    absolute_import, division, print_function, unicode_literals
)
32
import collections
33
import contextlib
34
import importlib
35
import itertools
36 37 38
import multiprocessing
import multiprocessing.dummy
import os
39

40 41
import six

42
from framework import grouptools, exceptions, options
43
from framework.dmesg import get_dmesg
44
from framework.log import LogManager
45
from framework.monitoring import Monitoring
46
from framework.test.base import Test
47

Dylan Baker's avatar
Dylan Baker committed
48 49
__all__ = [
    'TestProfile',
50
    'load_test_profile',
Dylan Baker's avatar
Dylan Baker committed
51 52 53
    'merge_test_profiles'
]

54

55
class TestDict(collections.MutableMapping):
56
    """A special kind of dict for tests.
57

58 59 60 61
    This dict lowers the names of keys by default.

    This class intentionally doesn't accept keyword arguments. This is
    intentional to avoid breakages.
62 63

    """
64 65 66
    def __init__(self):
        # This is because it had special __setitem__ and __getitem__ protocol
        # methods, and simply passing *args and **kwargs into self.__container
67 68
        # will bypass these methods. It will also break the ordering, since a
        # regular dictionary or keyword arguments are inherintly unordered
69
        #
70 71 72 73
        # This counter is incremented once when the allow_reassignment context
        # manager is opened, and decremented each time it is closed. This
        # allows stacking of the context manager
        self.__allow_reassignment = 0
74
        self.__container = collections.OrderedDict()
75

76 77
    def __setitem__(self, key, value):
        """Enforce types on set operations.
78

79 80 81
        Keys should only be strings, and values should only be more Trees
        or Tests.

82 83 84 85 86 87
        This method makes one additional requirement, it lowers the key before
        adding it. This solves a couple of problems, namely that we want to be
        able to use filesystem heirarchies as groups in some cases, and those
        are assumed to be all lowercase to avoid problems on case insensitive
        filesystems.

88
        """
89
        # keys should be strings
90
        if not isinstance(key, six.text_type):
91 92
            raise exceptions.PiglitFatalError(
                "TestDict keys must be strings, but was {}".format(type(key)))
93 94 95

        # Values should either be more Tests
        if not isinstance(value, Test):
96 97 98
            raise exceptions.PiglitFatalError(
                "TestDict values must be a Test, but was a {}".format(
                    type(value)))
99 100 101 102 103 104

        # This must be lowered before the following test, or the test can pass
        # in error if the key has capitals in it.
        key = key.lower()

        # If there is already a test of that value in the tree it is an error
105 106
        if not self.__allow_reassignment and key in self.__container:
            if self.__container[key] != value:
107 108 109 110
                error = (
                    'Further, the two tests are not the same,\n'
                    'The original test has this command:   "{0}"\n'
                    'The new test has this command:        "{1}"'.format(
111 112
                        ' '.join(self.__container[key].command),
                        ' '.join(value.command))
113 114 115 116
                )
            else:
                error = "and both tests are the same."

117
            raise exceptions.PiglitFatalError(
118
                "A test has already been assigned the name: {}\n{}".format(
119 120
                    key, error))

121
        self.__container[key] = value
122 123 124

    def __getitem__(self, key):
        """Lower the value before returning."""
125
        return self.__container[key.lower()]
126

127 128
    def __delitem__(self, key):
        """Lower the value before returning."""
129 130 131 132 133 134 135
        del self.__container[key.lower()]

    def __len__(self):
        return len(self.__container)

    def __iter__(self):
        return iter(self.__container)
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
    @property
    @contextlib.contextmanager
    def allow_reassignment(self):
        """Context manager that allows keys to be reassigned.

        Normally reassignment happens in error, but sometimes one actually
        wants to do reassignment, say to add extra options in a reduced
        profile. This method allows reassignment, but only within its context,
        making it an explict choice to do so.

        It is safe to nest this contextmanager.

        It is not safe to use this context manager in a threaded application

        """
        self.__allow_reassignment += 1
        yield
        self.__allow_reassignment -= 1

156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
    def filter(self, callable):
        """Filter tests out of the testdict before running.

        This method destructively filters results out of the test test
        dictionary list using the callable provided.

        Arguments:
        callable -- a callable object that returns truthy if the item remains,
                    falsy if it should be removed

        """
        for k, v in list(six.iteritems(self)):
            if not callable((k, v)):
                del self[k]

171 172 173
    def reorder(self, order):
        """Reorder the TestDict to match the order of the provided list."""
        new = collections.OrderedDict()
174 175 176 177 178 179 180 181 182
        try:
            for k in order:
                new[k] = self.__container[k]
        except KeyError:
            # If there is a name in order that isn't available in self there
            # will be a KeyError, this is expected. In this case fail
            # gracefully and report the error to the user.
            raise exceptions.PiglitFatalError(
                'Cannot reorder test: "{}", it is not in the profile.'.format(k))
183 184
        self.__container = new

185

186
class TestProfile(object):
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
    """ Class that holds a list of tests for execution

    This class provides a container for storing tests in either a nested
    dictionary structure (deprecated), or a flat dictionary structure with '/'
    delimited groups.

    Once a TestProfile object is created tests can be added to the test_list
    name as a key/value pair, the key should be a fully qualified name for the
    test, including it's group hierarchy and should be '/' delimited, with no
    leading or trailing '/', the value should be an exectest.Test derived
    object.

    When the test list is filled calling TestProfile.run() will set the
    execution of these tests off, and will flatten the nested group hierarchy
    of self.tests and merge it with self.test_list

    """
204
    def __init__(self):
205
        self.test_list = TestDict()
206
        self.forced_test_list = []
207 208
        self.filters = []
        # Sets a default of a Dummy
209
        self._dmesg = None
210
        self.dmesg = False
211
        self.results_dir = None
212 213
        self._monitoring = None
        self.monitoring = False
214 215 216 217

    @property
    def dmesg(self):
        """ Return dmesg """
218
        return self._dmesg
219 220 221 222 223

    @dmesg.setter
    def dmesg(self, not_dummy):
        """ Set dmesg

224
        Arguments:
225 226 227 228
        not_dummy -- if Truthy dmesg will try to get a PosixDmesg, if Falsy it
                     will get a DummyDmesg

        """
229
        self._dmesg = get_dmesg(not_dummy)
230

231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
    @property
    def monitoring(self):
        """ Return monitoring """
        return self._monitoring

    @monitoring.setter
    def monitoring(self, monitored):
        """ Set monitoring

        Arguments:
        monitored -- if Truthy Monitoring will enable monitoring according the
                     defined rules

        """
        self._monitoring = Monitoring(monitored)

247
    def _prepare_test_list(self):
248 249 250 251 252 253 254
        """ Prepare tests for running

        Flattens the nested group hierarchy into a flat dictionary using '/'
        delimited groups by calling self.flatten_group_hierarchy(), then
        runs it's own filters plus the filters in the self.filters name

        """
255
        def matches_any_regexp(x, re_list):
256
            return any(r.search(x) for r in re_list)
257

258
        # The extra argument is needed to match check_all's API
259 260
        def test_matches(path, test):
            """Filter for user-specified restrictions"""
261 262 263 264
            return ((not options.OPTIONS.include_filter or
                     matches_any_regexp(path, options.OPTIONS.include_filter))
                    and path not in options.OPTIONS.exclude_tests
                    and not matches_any_regexp(path, options.OPTIONS.exclude_filter))
265 266

        filters = self.filters + [test_matches]
Vinson Lee's avatar
Vinson Lee committed
267

268
        def check_all(item):
269
            """ Checks group and test name against all filters """
270 271 272 273 274 275
            path, test = item
            for f in filters:
                if not f(path, test):
                    return False
            return True

276 277 278 279 280 281 282
        if self.forced_test_list:
            # Remove all tests not in the test list, then reorder the tests to
            # match the testlist. This still allows additional filters to be
            # run afterwards.
            self.test_list.filter(lambda i: i[0] in self.forced_test_list)
            self.test_list.reorder(self.forced_test_list)

283
        # Filter out unwanted tests
284
        self.test_list.filter(check_all)
285

286
        if not self.test_list:
287 288
            raise exceptions.PiglitFatalError(
                'There are no tests scheduled to run. Aborting run.')
289

290
    def _pre_run_hook(self):
291 292 293
        """ Hook executed at the start of TestProfile.run

        To make use of this hook one will need to subclass TestProfile, and
294
        set this to do something, as be default it will no-op
295 296 297
        """
        pass

298
    def _post_run_hook(self):
299 300 301
        """ Hook executed at the end of TestProfile.run

        To make use of this hook one will need to subclass TestProfile, and
302
        set this to do something, as be default it will no-op
303 304 305
        """
        pass

306
    def run(self, logger, backend):
307
        """ Runs all tests using Thread pool
308

309
        When called this method will flatten out self.tests into
310 311
        self.test_list, then will prepare a logger, and begin executing tests
        through it's Thread pools.
312

313 314 315
        Based on the value of options.OPTIONS.concurrent it will either run all
        the tests concurrently, all serially, or first the thread safe tests
        then the serial tests.
316 317 318 319

        Finally it will print a final summary of the tests

        Arguments:
320
        backend -- a results.Backend derived instance
321 322

        """
323

324
        self._pre_run_hook()
325

326 327
        chunksize = 1

328
        self._prepare_test_list()
329
        log = LogManager(logger, len(self.test_list))
330

331
        def test(pair, this_pool=None):
332
            """Function to call test.execute from map"""
333
            name, test = pair
334
            with backend.write_test(name) as w:
335
                test.execute(name, log.get(), self.dmesg, self.monitoring)
336
                w(test.result)
337 338
            if self._monitoring.abort_needed:
                this_pool.terminate()
339

340 341
        def run_threads(pool, testlist):
            """ Open a pool, close it, and join it """
342
            pool.imap(lambda pair: test(pair, pool), testlist, chunksize)
343 344 345
            pool.close()
            pool.join()

346 347 348 349 350 351 352
        # Multiprocessing.dummy is a wrapper around Threading that provides a
        # multiprocessing compatible API
        #
        # The default value of pool is the number of virtual processor cores
        single = multiprocessing.dummy.Pool(1)
        multi = multiprocessing.dummy.Pool()

353
        if options.OPTIONS.concurrent == "all":
354
            run_threads(multi, six.iteritems(self.test_list))
355
        elif options.OPTIONS.concurrent == "none":
356
            run_threads(single, six.iteritems(self.test_list))
357 358
        else:
            # Filter and return only thread safe tests to the threaded pool
359
            run_threads(multi, (x for x in six.iteritems(self.test_list)
360
                                if x[1].run_concurrent))
361
            # Filter and return the non thread safe tests to the single pool
362
            run_threads(single, (x for x in six.iteritems(self.test_list)
363
                                 if not x[1].run_concurrent))
364

365
        log.get().summary()
366

367
        self._post_run_hook()
368

369 370 371
        if self._monitoring.abort_needed:
            raise exceptions.PiglitAbort(self._monitoring.error_message)

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
    def filter_tests(self, function):
        """Filter out tests that return false from the supplied function

        Arguments:
        function -- a callable that takes two parameters: path, test and
                    returns whether the test should be included in the test
                    run or not.
        """
        self.filters.append(function)

    def update(self, *profiles):
        """ Updates the contents of this TestProfile instance with another

        This method overwrites key:value pairs in self with those in the
        provided profiles argument. This allows multiple TestProfiles to be
        called in the same run; which could be used to run piglit and external
        suites at the same time.

        Arguments:
        profiles -- one or more TestProfile-like objects to be merged.

        """
        for profile in profiles:
            self.test_list.update(profile.test_list)

397
    @contextlib.contextmanager
398
    def group_manager(self, test_class, group, prefix=None, **default_args):
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
        """A context manager to make working with flat groups simple.

        This provides a simple way to replace add_plain_test,
        add_concurrent_test, etc. Basic usage would be to use the with
        statement to yield and adder instance, and then add tests.

        This does not provide for a couple of cases.
        1) When you need to alter the test after initialization. If you need to
           set instance.env, for example, you will need to do so manually. It
           is recommended to not use this function for that case, but to
           manually assign the test and set env together, for code clearness.
        2) When you need to use a function that modifies profile.

        Arguments:
        test_class -- a Test derived class that. Instances of this class will
                      be added to the profile.
        group -- a string or unicode that will be used as the key for the test
                 in profile.

418 419 420 421 422 423
        Keyword Arguments:
        **default_args -- any additional keyword arguments will be considered
                          default arguments to all tests added by the adder.
                          They will always be overwritten by **kwargs passed to
                          the adder function

424 425 426 427 428 429 430
        >>> from framework.test import PiglitGLTest
        >>> p = TestProfile()
        >>> with p.group_manager(PiglitGLTest, 'a') as g:
        ...     g(['test'])
        ...     g(['power', 'test'], 'powertest')

        """
431
        assert isinstance(group, six.string_types), type(group)
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447

        def adder(args, name=None, **kwargs):
            """Helper function that actually adds the tests.

            Arguments:
            args -- arguments to be passed to the test_class constructor.
                    This must be appropriate for the underlying class

            Keyword Arguments:
            name -- If this is a a truthy value that value will be used as the
                    key for the test. If name is falsy then args will be
                    ' '.join'd and used as name. Default: None
            kwargs -- Any additional args will be passed directly to the test
                      constructor as keyword args.

            """
448 449 450 451 452
            # If there is no name, then either
            # a) join the arguments list together to make the name
            # b) use the argument string as the name
            # The former is used by the Piglit{G,C}LTest classes, the latter by
            # GleanTest
453
            if not name:
454 455 456
                if isinstance(args, list):
                    name = ' '.join(args)
                else:
457
                    assert isinstance(args, six.string_types)
458 459
                    name = args

460
            assert isinstance(name, six.string_types)
461
            lgroup = grouptools.join(group, name)
462

463 464
            self.test_list[lgroup] = test_class(
                args,
465 466
                **dict(itertools.chain(six.iteritems(default_args),
                                       six.iteritems(kwargs))))
467 468 469

        yield adder

470 471 472 473 474 475 476
    @property
    @contextlib.contextmanager
    def allow_reassignment(self):
        """A convenience wrapper around self.test_list.allow_reassignment."""
        with self.test_list.allow_reassignment:
            yield

477

478
def load_test_profile(filename):
479
    """Load a python module and return it's profile attribute.
480 481 482 483 484

    All of the python test files provide a profile attribute which is a
    TestProfile instance. This loads that module and returns it or raises an
    error.

485 486
    This method doesn't care about file extensions as a way to be backwards
    compatible with script wrapping piglit. 'tests/quick', 'tests/quick.tests',
487 488 489 490
    and 'tests/quick.py' are all equally valid for filename.

    This will raise a FatalError if the module doesn't exist, or if the module
    doesn't have a profile attribute.
491 492 493 494

    Arguments:
    filename -- the name of a python module to get a 'profile' from

495 496
    """
    try:
497 498
        mod = importlib.import_module('tests.{0}'.format(
            os.path.splitext(os.path.basename(filename))[0]))
499 500
        return mod.profile
    except AttributeError:
501 502 503
        raise exceptions.PiglitFatalError(
            'There is not profile attribute in module {}.\n'
            'Did you specify the right file?'.format(filename))
504 505 506 507
    except ImportError:
        raise exceptions.PiglitFatalError(
            'There is no test profile called "{}".\n'
            'Check your spelling?'.format(filename))
508 509 510 511 512 513


def merge_test_profiles(profiles):
    """ Helper for loading and merging TestProfile instances

    Takes paths to test profiles as arguments and returns a single merged
514
    TestProfile instance.
515 516 517 518 519

    Arguments:
    profiles -- a list of one or more paths to profile files.

    """
520
    profile = load_test_profile(profiles.pop())
521 522 523
    with profile.allow_reassignment:
        for p in profiles:
            profile.update(load_test_profile(p))
524
    return profile