si-report.py 13.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
#!/usr/bin/env python
# vim: set expandtab tabstop=4 softtabstop=4 shiftwidth=4: */
#
# Copyright 2015 Advanced Micro Devices, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice (including the next
# paragraph) shall be included in all copies or substantial portions of the
# Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

26 27
from collections import defaultdict
import itertools
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
import re
import sys

def format_float(f, suffix = ' %'):
    return "{0:0.2f}{1}".format(f, suffix)

def get_str(value, suffix = ' %'):
    if type(value) == float:
        return format_float(value, suffix)
    else:
        return value

def calculate_percent_change(b, a):
    if b == 0:
        return 0
    return 100 * float(a - b) / float(b)

def cmp_max_unit(current, comp):
    return comp[0] > current[0]

def cmp_min_unit(current, comp):
    return comp[0] < current[0]

def cmp_max_per(current, comp):
    return calculate_percent_change(comp[1], comp[2]) > calculate_percent_change(current[1], current[2])

def cmp_min_per(current, comp):
    return calculate_percent_change(comp[1], comp[2]) < calculate_percent_change(current[1], current[2])

class si_stats:
58 59 60 61 62 63 64 65 66
    metrics = [
        ('sgprs', 'SGPRS', ''),
        ('vgprs', 'VGPRS', ''),
        ('code_size', 'Code Size', 'bytes'),
        ('lds', 'LDS', 'blocks'),
        ('scratch', 'Scratch', 'bytes per wave'),
        ('waitstates', 'Wait states', ''),
    ]

67
    def __init__(self):
68
        self.error = False
69

70 71
        for name in self.get_metrics():
            self.__dict__[name] = 0
72

73 74 75 76 77 78 79 80 81 82 83 84 85
        self._minmax_testname = {}

    def copy(self):
        copy = si_stats()
        copy.error = self.error

        for name in self.get_metrics():
            copy.__dict__[name] = self.__dict__[name]

        copy._minmax_testname = self._minmax_testname.copy()

        return copy

86
    def to_string(self, suffixes = True):
87 88
        strings = []
        for name, printname, suffix in si_stats.metrics:
89 90 91 92 93 94 95 96 97 98
            string = "{}: {}".format(printname, get_str(self.__dict__[name]))

            if suffixes and len(suffix) > 0:
                string += ' ' + suffix

            minmax_testname = self._minmax_testname.get(name)
            if minmax_testname is not None:
                string += ' (in {})'.format(minmax_testname)

            strings.append(string + '\n')
99
        return ''.join(strings)
100

101 102
    def get_metrics(self):
        return [m[0] for m in si_stats.metrics]
103 104 105 106 107

    def __str__(self):
        return self.to_string()

    def add(self, other):
108 109
        for name in self.get_metrics():
            self.__dict__[name] += other.__dict__[name]
110

111
    def update(self, comp, cmp_fn, testname):
112
        for name in self.get_metrics():
113 114 115 116 117
            current = self.__dict__[name]
            if type(current) != tuple:
                current = (0, 0, 0)
            if cmp_fn(current, comp.__dict__[name]):
                self.__dict__[name] = comp.__dict__[name]
118
                self._minmax_testname[name] = testname
119 120

    def update_max(self, comp):
121
        for name in self.get_metrics():
122 123 124 125 126 127 128
            current = self.__dict__[name]
            if type(current) == tuple:
                current = self.__dict__[name][0]
            if comp.__dict__[name][0] > current:
                self.__dict__[name] = comp.__dict__[name]

    def update_min(self, comp):
129
        for name in self.get_metrics():
130 131 132 133 134 135 136
            current = self.__dict__[name]
            if type(current) == tuple:
                current = self.__dict__[name][0]
            if comp.__dict__[name][0] < current:
                self.__dict__[name] = comp.__dict__[name]

    def update_increase(self, comp):
137
        for name in self.get_metrics():
138 139 140 141
            if comp.__dict__[name][0] > 0:
                self.__dict__[name] += 1

    def update_decrease(self, comp):
142
        for name in self.get_metrics():
143 144 145 146
            if comp.__dict__[name][0] < 0:
                self.__dict__[name] += 1

    def is_empty(self):
147
        for name in self.get_metrics():
148 149 150 151 152 153
            x = self.__dict__[name]
            if type(x) == tuple and x[0] is not 0:
                return False
            if type(x) != tuple and x is not 0:
                return False
        return True
154 155


156 157 158 159 160
class si_parser(object):
    re_stats = re.compile(
        r"^Shader Stats: SGPRS: ([0-9]+) VGPRS: ([0-9]+) Code Size: ([0-9]+) "+
        r"LDS: ([0-9]+) Scratch: ([0-9]+)$")
    re_nop = re.compile("^\ts_nop ([0-9]+)")
161

162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
    def __init__(self):
        self._stats = None
        self._in_disasm = False

    def finish(self):
        return self._stats

    def parse(self, msg):
        if not self._in_disasm:
            if msg == "Shader Disassembly Begin":
                old_stats = self._stats
                self._stats = si_stats()
                self._in_disasm = True
                return old_stats

            match = si_parser.re_stats.match(msg)
            if match is not None:
                self._stats.sgprs = int(match.group(1))
                self._stats.vgprs = int(match.group(2))
                self._stats.code_size = int(match.group(3))
                self._stats.lds = int(match.group(4))
                self._stats.scratch = int(match.group(5))
                old_stats = self._stats
                self._stats = None
                return old_stats
187 188 189 190 191 192 193 194 195

            if msg == "LLVM compile failed":
                old_stats = self._stats
                self._stats = None

                if old_stats is None:
                    old_stats = si_stats()
                old_stats.error = True
                return old_stats
196 197 198 199
        else:
            if msg == "Shader Disassembly End":
                self._in_disasm = False
                return None
200

201 202 203 204
            match = si_parser.re_nop.match(msg)
            if match:
                self._stats.waitstates += 1 + int(match.groups()[0])
                return None
205

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
def get_results(filename):
    """
    Returns a dictionary that maps shader_test names to lists of si_stats
    (corresponding to the different shaders within the test's programs).
    """
    results = defaultdict(list)
    parsers = defaultdict(si_parser)

    with open(filename, "r") as file:
        re_line = re.compile(r"^(.+\.shader_test) - (.*)$")

        for line in file:
            match = re_line.match(line)
            if match is None:
                continue

            name = match.group(1)
            message = match.group(2)

            stats = parsers[name].parse(message)
            if stats is not None:
                results[name].append(stats)

    for name, parser in parsers.items():
        stats = parser.finish()
        if stats is not None:
            print "Results for", name, "not fully parsed!"
            results[name].append(stats)
234 235 236 237 238 239

    return results


def compare_stats(before, after):
    result = si_stats()
240
    for name in result.get_metrics():
241 242 243 244 245 246 247
        b = before.__dict__[name]
        a = after.__dict__[name]
        result.__dict__[name] = (a - b, b, a)
    return result

def divide_stats(num, div):
    result = si_stats()
248
    for name in result.get_metrics():
249 250 251 252 253 254 255 256
        if div.__dict__[name] == 0:
            result.__dict__[name] = num.__dict__[name]
        else:
            result.__dict__[name] = 100.0 * float(num.__dict__[name]) / float(div.__dict__[name])
    return result

def print_before_after_stats(before, after, divisor = 1):
    result = si_stats()
257
    for name in result.get_metrics():
258 259 260 261 262 263 264 265 266 267 268
        b = before.__dict__[name] / divisor
        a = after.__dict__[name] / divisor
        if b == 0:
            percent = format_float(0.0)
        else:
            percent = format_float(100 * float(a - b) / float(b))
        result.__dict__[name] = '{} -> {} ({})'.format(get_str(b,''), get_str(a,''), percent)

    print result

def print_cmp_stats(comp):
269
    result = comp.copy()
270
    for name in result.get_metrics():
271
        if type(result.__dict__[name]) != tuple:
272 273 274
            a = 0
            b = 0
        else:
275 276
            b = result.__dict__[name][1]
            a = result.__dict__[name][2]
277 278 279 280 281 282 283 284 285 286 287
        if b == 0:
            percent = format_float(0.0)
        else:
            percent = format_float(100 * float(a - b) / float(b))
        result.__dict__[name] = '{} -> {} ({})'.format(get_str(b,''), get_str(a,''), percent)

    print result


def print_count(stats, divisor):
    result = si_stats()
288
    for name in result.get_metrics():
289 290 291 292 293
        count = stats.__dict__[name]
        percent = float(count) / float(divisor)
        result.__dict__[name] = '{} ({})'.format(get_str(count,''), get_str(percent))
    print result.to_string(False)

294
def compare_results(before_all_results, after_all_results):
295 296 297 298 299 300 301 302 303 304 305 306
    total_before = si_stats()
    total_after = si_stats()
    total_affected_before = si_stats()
    total_affected_after = si_stats()
    increases = si_stats()
    decreases = si_stats()
    max_increase_per = si_stats()
    max_decrease_per = si_stats()
    max_increase_unit = si_stats()
    max_decrease_unit = si_stats()

    num_affected = 0
307 308
    num_tests = 0
    num_shaders = 0
309 310
    num_after_errors = 0
    num_before_errors = 0
311 312 313 314 315 316

    all_names = set(itertools.chain(before_all_results.keys(), after_all_results.keys()))

    only_after_names = []
    only_before_names = []
    count_mismatch_names = []
317
    errors_names = []
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333

    for name in all_names:
        before_test_results = before_all_results.get(name)
        after_test_results = after_all_results.get(name)

        if before_test_results is None:
            only_after_names.append(name)
            continue
        if after_test_results is None:
            only_before_names.append(name)
            continue

        if len(before_test_results) != len(after_test_results):
            count_mismatch_names.append(name)

        num_tests += 1
334
        have_error = False
335 336

        for before, after in zip(before_test_results, after_test_results):
337 338 339 340 341 342 343 344
            if before.error:
                num_before_errors += 1
            if after.error:
                num_after_errors += 1
            if after.error or before.error:
                have_error = True
                continue

345 346 347 348 349 350 351 352 353 354 355
            total_before.add(before)
            total_after.add(after)
            num_shaders += 1

            comp = compare_stats(before, after)
            if not comp.is_empty():
                num_affected += 1
                total_affected_before.add(before)
                total_affected_after.add(after)
                increases.update_increase(comp)
                decreases.update_decrease(comp)
356 357 358 359
                max_increase_per.update(comp, cmp_max_per, name)
                max_decrease_per.update(comp, cmp_min_per, name)
                max_increase_unit.update(comp, cmp_max_unit, name)
                max_decrease_unit.update(comp, cmp_min_unit, name)
360

361 362 363
        if have_error:
            errors_names.append(name)

364
    print '{} shaders in {} tests'.format(num_shaders, num_tests)
365 366 367 368 369
    print "Totals:"
    print_before_after_stats(total_before, total_after)
    print "Totals from affected shaders:"
    print_before_after_stats(total_affected_before, total_affected_after)
    print "Increases:"
370
    print_count(increases, num_shaders)
371
    print "Decreases:"
372
    print_count(decreases, num_shaders)
373 374 375 376 377 378 379 380 381 382 383 384 385

    print "*** BY PERCENTAGE ***\n"
    print "Max Increase:\n"
    print_cmp_stats(max_increase_per)
    print "Max Decrease:\n"
    print_cmp_stats(max_decrease_per)

    print "*** BY UNIT ***\n"
    print "Max Increase:\n"
    print_cmp_stats(max_increase_unit)
    print "Max Decrease:\n"
    print_cmp_stats(max_decrease_unit)

386 387
    def report_ignored(names, what):
        if names:
388
            print "*** {} are ignored:".format(what)
389 390 391 392 393
            s = ', '.join(names[:5])
            if len(names) > 5:
                s += ', and {} more'.format(len(names) - 5)
            print s

394 395 396 397 398 399 400
    report_ignored(only_after_names, "Tests only in 'after' results")
    report_ignored(only_before_names, "Tests only in 'before' results")
    report_ignored(count_mismatch_names, "Tests with different number of shaders")
    report_ignored(errors_names, "Shaders with compilation errors")
    if num_after_errors > 0 or num_before_errors > 0:
        print "*** Compile errors encountered! (before: {}, after: {})".format(
            num_before_errors, num_after_errors)
401 402 403 404 405 406 407 408 409

def main():
    before = sys.argv[1]
    after = sys.argv[2]

    compare_results(get_results(before), get_results(after))

if __name__ == "__main__":
    main()