Commit c584a2fc authored by Marek Olšák's avatar Marek Olšák

si-report.py: add completely new shader statistics reporting

We can remove the old reporting if people are OK with that.
The old reporting has a bug that it reports 0->N changes as 0 % in several
places. (should be inf %)

The new reporting shows:
- VGPR spilling shaders and apps (from the second file only)
- the same for SGPRs
- worst regressions (from the comparison of both files)
- percentage deltas at the end
.. and colors!!!

Example:

 WORST VGPR SPILLS (not deltas)                                        VGPRs SpillVGPR ScratchVGPR
 shaders/private/f1-2015/18.shader_test [0]                               32         0       516
 shaders/private/bioshock-infinite/256.shader_test [0]                    64       176       180
 shaders/private/ue4_lightroom_interior_day/42.shader_test [0]            28         0        76
 shaders/private/dirt-showdown/676.shader_test [0]                        68        49        72
 shaders/private/f1-2015/1102.shader_test [0]                             52         0        72
 shaders/private/bioshock-infinite/814.shader_test [0]                    64        57        60
 shaders/private/ue4_lightroom_interior_day/33.shader_test [0]            24         0        52
 shaders/private/ue4_lightroom_interior_day/37.shader_test [0]            24         0        52
 shaders/private/bioshock-infinite/698.shader_test [0]                    16         0        36
 shaders/private/dirt-showdown/406.shader_test [0]                        64        33        36

 VGPR SPILLING APPS   Shaders SpillVGPR ScratchVGPR
 alien_isolation         2938        12        16
 bioshock-infinite       1769       233       720
 dirt-showdown            541        82       108
 f1-2015                  774         0       624
 tesseract                430         2         4
 ue4_lightroom_inter..     74         0       180

 WORST SGPR SPILLS (not deltas)                                        SGPRs SpillSGPR
 shaders/private/talos_principle/1942.shader_test [1]                     80       168
 shaders/private/ue4_effects_cave/289.shader_test [0]                     80       168
 shaders/private/talos_principle/2052.shader_test [1]                     80       161
 shaders/private/serious_sam_3_bfe/1081.shader_test [1]                   80       148
 shaders/private/borderlands2/5330.shader_test [0]                        80       137
 shaders/private/talos_principle/2040.shader_test [1]                     80       133
 shaders/private/talos_principle/2041.shader_test [1]                     80       133
 shaders/private/talos_principle/2036.shader_test [1]                     80       133
 shaders/private/talos_principle/2035.shader_test [1]                     80       133
 shaders/private/borderlands2/5548.shader_test [0]                        80       131

 SGPR SPILLING APPS   Shaders SpillSGPR AvgPerSh
 alien_isolation         2938     23198      7.9
 batman_arkham_origins    589        30      0.1
 bioshock-infinite       1769        84      0.0
 borderlands2            3968      6449      1.6
 brutal-legend            338       647      1.9
 civilization_beyond..    116       213      1.8
 counter_strike_glob..   1142      4338      3.8
 dirt-showdown            541      1071      2.0
 dolphin                   22        62      2.8
 dota2                   1747       338      0.2
 europa_universalis_4      76        44      0.6
 f1-2015                  774      6245      8.1
 left_4_dead_2           1762     13778      7.8
 metro_2033_redux        2670       547      0.2
 nexuiz                    80       111      1.4
 portal                   474      2211      4.7
 serious_sam_3_bfe        392      6626     16.9
 talos_principle          324      4539     14.0
 team_fortress_2          808      4823      6.0
 thea                     172        41      0.2
 ue4_effects_cave         299       494      1.7
 ue4_elemental            586       355      0.6
 ue4_lightroom_inter..     74        29      0.4
 ue4_realistic_rende..     92        60      0.7
 unigine_heaven           322       174      0.5
 unigine_sanctuary        264       400      1.5
 unigine_tropics          210       328      1.6
 unigine_valley           278       427      1.5
 unity                     72        45      0.6
 warsow                   176        13      0.1
 witcher2                1040         2      0.0

 WORST REGRESSIONS - VGPRS                                            Before     After     Delta Percentage
 shaders/private/metro_2033_redux/1082.shader_test [0]                   136       180        44   32.35 %
 shaders/private/witcher2/20.shader_test [0]                              48        64        16   33.33 %
 shaders/private/witcher2/1097.shader_test [0]                            60        76        16   26.67 %
 shaders/private/unigine_valley/298.shader_test [0]                       44        56        12   27.27 %
 shaders/private/witcher2/158.shader_test [0]                             52        64        12   23.08 %
 shaders/private/witcher2/138.shader_test [0]                             60        72        12   20.00 %
 shaders/private/witcher2/136.shader_test [0]                             60        72        12   20.00 %
 shaders/private/witcher2/187.shader_test [0]                             56        68        12   21.43 %
 shaders/private/left_4_dead_2/1567.shader_test [0]                       32        40         8   25.00 %
 shaders/private/team_fortress_2/4662.shader_test [0]                     20        28         8   40.00 %

 WORST REGRESSIONS - Spilled SGPRs                                    Before     After     Delta Percentage
 shaders/private/serious_sam_3_bfe/883.shader_test [0]                    14        49        35  250.00 %
 shaders/private/talos_principle/1942.shader_test [0]                     14        49        35  250.00 %
 shaders/private/serious_sam_3_bfe/1081.shader_test [0]                   14        49        35  250.00 %
 shaders/private/talos_principle/1941.shader_test [0]                     14        49        35  250.00 %
 shaders/private/talos_principle/2052.shader_test [0]                      9        42        33  366.67 %
 shaders/private/serious_sam_3_bfe/869.shader_test [0]                    35        67        32   91.43 %
 shaders/private/talos_principle/1986.shader_test [0]                     45        76        31   68.89 %
 shaders/private/left_4_dead_2/3764.shader_test [0]                       15        46        31  206.67 %
 shaders/private/talos_principle/2008.shader_test [0]                     45        76        31   68.89 %
 shaders/private/talos_principle/2035.shader_test [0]                     45        76        31   68.89 %

 WORST REGRESSIONS - Spilled VGPRs                                    Before     After     Delta Percentage
 shaders/tesseract/506.shader_test [0]                                     0         2         2     inf %

 WORST REGRESSIONS - Scratch VGPRs                                    Before     After     Delta Percentage
 shaders/tesseract/506.shader_test [0]                                     0         4         4     inf %

 WORST REGRESSIONS - Code Size                                        Before     After     Delta Percentage
 shaders/private/talos_principle/1942.shader_test [0]                   3116      3808       692   22.21 %
 shaders/private/talos_principle/1941.shader_test [0]                   3116      3808       692   22.21 %
 shaders/private/serious_sam_3_bfe/883.shader_test [0]                  3108      3792       684   22.01 %
 shaders/private/serious_sam_3_bfe/1081.shader_test [0]                 3108      3792       684   22.01 %
 shaders/private/serious_sam_3_bfe/869.shader_test [0]                  4124      4744       620   15.03 %
 shaders/private/talos_principle/2052.shader_test [0]                   2904      3512       608   20.94 %
 shaders/private/left_4_dead_2/3764.shader_test [0]                     3344      3928       584   17.46 %
 shaders/private/left_4_dead_2/1260.shader_test [0]                     3344      3928       584   17.46 %
 shaders/private/talos_principle/2061.shader_test [0]                   5312      5892       580   10.92 %
 shaders/private/talos_principle/1994.shader_test [0]                   5312      5892       580   10.92 %

 PERCENTAGE DELTAS    Shaders     SGPRs     VGPRs SpillSGPR SpillVGPR   Scratch  CodeSize  MaxWaves     Waits
 (unknown)                  4     .         .         .         .         .       -0.02 %     .         .
 0ad                        6     .         .         .         .         .         .         .         .
 alien_isolation         2938     .       -5.35 %   -0.77 %     .         .        0.21 %    3.79 %     .
 anholt                    10     .         .         .         .         .         .         .         .
 batman_arkham_origins    589     .       -4.77 %  -96.56 %     .         .       -1.59 %    4.48 %     .
 bioshock-infinite       1769     .       -1.41 %  -86.60 %     .         .       -0.59 %    0.94 %     .
 borderlands2            3968     .       -2.20 %  -36.23 %     .         .       -1.07 %    1.15 %     .
 brutal-legend            338     .       -0.90 %   -6.50 %     .         .       -0.10 %    0.43 %     .
 civilization_beyond..    116     .       -0.97 %    0.47 %     .         .        0.04 %    0.46 %     .
 counter_strike_glob..   1142     .       -2.00 %   -7.58 %     .         .       -0.43 %    0.06 %     .
 dirt-showdown            541     .       -1.37 %  -22.95 %     .       -3.57 %   -0.79 %    1.03 %     .
 dolphin                   22     .         .         .         .         .        0.23 %     .         .
 dota2                   1747     .       -0.07 %     .         .         .        0.03 %    0.04 %     .
 europa_universalis_4      76     .       -0.46 %     .         .         .        0.09 %     .         .
 f1-2015                  774     .       -2.11 %   -1.51 %     .         .       -0.02 %    1.05 %     .
 furmark-0.7.0              4     .         .         .         .         .        0.16 %     .         .
 gimark-0.7.0              10     .         .         .         .         .        0.18 %     .         .
 glamor                    16     .         .         .         .         .       -0.91 %     .         .
 humus-celshading           4     .         .         .         .         .        0.42 %     .         .
 humus-domino               6     .         .         .         .         .         .         .         .
 humus-dynamicbranching    24     .       -0.70 %     .         .         .        0.42 %    0.45 %     .
 humus-hdr                 10     .         .         .         .         .         .         .         .
 humus-portals              2     .         .         .         .         .         .         .         .
 humus-volumetricfog..      6     .         .         .         .         .         .         .         .
 left_4_dead_2           1762     .       -0.62 %   10.13 %     .         .        0.72 %   -0.14 %     .
 metro_2033_redux        2670     .       -4.73 %  -12.20 %     .         .        0.20 %    1.52 %     .
 nexuiz                    80     .         .         .         .         .         .         .         .
 pixmark-julia-fp32         2     .         .         .         .         .         .         .         .
 pixmark-julia-fp64         2     .         .         .         .         .         .         .         .
 pixmark-piano-0.7.0        2     .         .         .         .         .       -0.85 %     .         .
 pixmark-volplosion-..      2     .         .         .         .         .         .         .         .
 plot3d-0.7.0               8     .         .         .         .         .         .         .         .
 portal                   474     .       -2.05 %  206.23 %     .         .        1.69 %    0.58 %     .
 sauerbraten                7     .         .         .         .         .        0.62 %     .         .
 serious_sam_3_bfe        392     .       -8.64 %    0.93 %     .         .       -1.79 %    4.05 %     .
 supertuxkart               4     .         .         .         .         .         .         .         .
 talos_principle          324     .       -7.67 %  -10.07 %     .         .       -1.39 %    2.92 %     .
 team_fortress_2          808     .       -2.62 %  276.21 %     .         .        2.26 %    1.24 %     .
 tesseract                430     .       -1.31 % -100.00 %     inf %     inf %   -0.12 %    0.75 %     .
 tessmark-0.7.0             6     .         .         .         .         .        0.09 %     .         .
 thea                     172     .       -1.98 %     .         .         .        0.34 %    0.91 %     .
 ue4_effects_cave         299     .       -1.87 %   -8.52 %     .         .       -0.18 %    1.87 %     .
 ue4_elemental            586     .       -4.67 %  -42.56 %     .         .       -1.17 %    4.57 %     .
 ue4_lightroom_inter..     74     .       -1.57 %  -12.12 %     .         .        0.12 %    1.00 %     .
 ue4_realistic_rende..     92     .       -0.27 %     .         .         .        0.11 %     .         .
 unigine_heaven           322     .       -0.87 %  -36.73 %     .         .       -0.46 %    0.69 %     .
 unigine_sanctuary        264     .       -1.57 %   -7.41 %     .         .        0.01 %    0.96 %     .
 unigine_tropics          210     .       -1.22 %  -10.87 %     .         .       -0.18 %    0.88 %     .
 unigine_valley           278     .       -2.11 %  -27.87 %     .         .       -0.46 %    1.39 %     .
 unity                     72     .       -1.74 %  -15.09 %     .         .       -0.54 %    0.93 %     .
 warsow                   176     .       -0.10 %     .         .         .         .        0.06 %     .
 warzone2100                4     .       -4.35 %     .         .         .        0.67 %    2.63 %     .
 witcher2                1040     .       -5.93 %  -93.55 %     .         .       -0.59 %    2.54 %     .
 xcom_enemy_within       1236     .       -4.81 % -100.00 %     .         .       -0.03 %    3.22 %     .
 yofrankie                 82     .       -0.41 %     .         .         .        0.80 %    0.14 %     .
 ------------------------------------------------------------------------------------------------------------
 All affected            7287     .       -8.94 %   -5.18 %    0.61 %     .       -0.36 %    6.21 %     .
 ------------------------------------------------------------------------------------------------------------
 Total                  26002     .       -3.09 %   -1.82 %    0.61 %     .       -0.16 %    1.55 %     .
Acked-by: default avatarNicolai Hähnle <nicolai.haehnle@amd.com>
parent f69ed477
......@@ -28,6 +28,11 @@ import itertools
import re
import sys
set_red = "\033[31m"
set_green = "\033[1;32m"
set_yellow = "\033[1;33m"
set_normal = "\033[0m"
def format_float(f, suffix = ' %'):
return "{0:0.2f}{1}".format(f, suffix)
......@@ -42,6 +47,23 @@ def calculate_percent_change(b, a):
return 0 if a == 0 else float("inf")
return 100 * float(a - b) / float(b)
def format_table_cell(n, more_is_better = False, colored = True, is_percent = False):
if is_percent and abs(n) < 0.01:
return " . "
str = ("{:>8.2f} %" if is_percent else "{:>10}").format(n)
if colored:
if n > 0.5:
str = (set_green if more_is_better else set_red) + str + set_normal
elif n < -0.5:
str = (set_red if more_is_better else set_green) + str + set_normal
return str
def format_percent_change(b, a, more_is_better = False, colored = True):
percent = calculate_percent_change(b, a)
return format_table_cell(percent, more_is_better, colored, is_percent = True)
def cmp_max_unit(current, comp):
return comp[0] > current[0]
......@@ -252,6 +274,22 @@ def compare_stats(before, after):
result.__dict__[name] = (a - b, b, a)
return result
def subtract_stats(x, y):
result = si_stats()
for name in result.get_metrics():
result.__dict__[name] = x.__dict__[name] - y.__dict__[name]
return result
def is_regression(before, after):
for field in before.get_metrics():
if field == 'maxwaves':
if before.__dict__[field] > after.__dict__[field]:
return True
else:
if before.__dict__[field] < after.__dict__[field]:
return True
return False
def divide_stats(num, div):
result = si_stats()
for name in result.get_metrics():
......@@ -411,11 +449,224 @@ def compare_results(before_all_results, after_all_results):
print "*** Compile errors encountered! (before: {}, after: {})".format(
num_before_errors, num_after_errors)
class grouped_stats:
def __init__(self):
self.num_shaders = 0
self.before = si_stats()
self.after = si_stats()
self.diff = si_stats()
def add(self, before, after):
self.num_shaders += 1
self.before.add(before)
self.after.add(after)
def set_one_shader(self, before, after):
self.before = before
self.after = after
self.diff = subtract_stats(after, before)
def print_vgpr_spilling_app(self, name):
if (self.after.spilled_vgprs > 0 or
self.after.scratch_vgprs > 0):
print " {:22}{:6}{:10}{:10}".format(
name,
self.num_shaders,
self.after.spilled_vgprs,
self.after.scratch_vgprs)
def print_one_shader_vgpr_spill(self, name):
if (self.after.spilled_vgprs > 0 or
self.after.scratch_vgprs > 0):
print " {:65}{:10}{:10}{:10}".format(
name,
self.after.vgprs,
self.after.spilled_vgprs,
self.after.scratch_vgprs)
def print_sgpr_spilling_app(self, name):
if self.after.spilled_sgprs > 0:
print " {:22}{:6}{:10}{:>9.1f}".format(
name,
self.num_shaders,
self.after.spilled_sgprs,
float(self.after.spilled_sgprs) / float(self.num_shaders))
def print_one_shader_sgpr_spill(self, name):
if self.after.spilled_sgprs > 0:
print " {:65}{:10}{:10}".format(
name,
self.after.sgprs,
self.after.spilled_sgprs)
def print_percentages(self, name):
print " {:22}{:6}{}{}{}{}{}{}{}{}".format(
name,
self.num_shaders,
format_percent_change(self.before.sgprs, self.after.sgprs),
format_percent_change(self.before.vgprs, self.after.vgprs),
format_percent_change(self.before.spilled_sgprs, self.after.spilled_sgprs),
format_percent_change(self.before.spilled_vgprs, self.after.spilled_vgprs),
format_percent_change(self.before.scratch_vgprs, self.after.scratch_vgprs),
format_percent_change(self.before.code_size, self.after.code_size),
format_percent_change(self.before.maxwaves, self.after.maxwaves, more_is_better = True),
format_percent_change(self.before.waitstates, self.after.waitstates))
def print_regression(self, name, field):
print " {:65}{:10}{:10}{}{}".format(
name,
self.before.__dict__[field],
self.after.__dict__[field],
format_table_cell(self.after.__dict__[field] - self.before.__dict__[field]),
format_percent_change(self.before.__dict__[field], self.after.__dict__[field]))
"""
Return "filename [index]", because files can contain multiple shaders.
"""
def get_shader_name(list, orig):
for i in range(10):
# add the index to the name
name = orig + " [{}]".format(i)
if name not in list:
return name
assert False
return "(error)"
def print_yellow(str):
print set_yellow + str + set_normal
def print_tables(before_all_results, after_all_results):
re_app = re.compile(r"^.*/([^/]+)/[^/]+$")
apps = defaultdict(grouped_stats)
shaders = defaultdict(grouped_stats)
total = grouped_stats()
total_affected = grouped_stats()
all_files = set(itertools.chain(before_all_results.keys(),
after_all_results.keys()))
for file in all_files:
# get the application name (inner-most directory)
match_app = re_app.match(file)
if match_app is None:
app = "(unknown)"
else:
app = match_app.group(1)
if len(app) > 22:
app = app[0:19] + ".."
before_test_results = before_all_results.get(file)
after_test_results = after_all_results.get(file)
if before_test_results is None or after_test_results is None:
continue
for before, after in zip(before_test_results, after_test_results):
if after.error or before.error:
continue
apps[app].add(before, after)
total.add(before, after)
if not subtract_stats(before, after).is_empty():
total_affected.add(before, after)
# we don't have to add all shaders, just those that we may need
# to display
if (is_regression(before, after) or
after.scratch_vgprs > 0 or
after.spilled_vgprs > 0 or
after.spilled_sgprs > 0):
name = get_shader_name(shaders, file)
shaders[name].set_one_shader(before, after)
# worst VGPR spills
num = 0
sort_key = lambda v: -v[1].after.scratch_vgprs
for name, stats in sorted(shaders.items(), key = sort_key):
if num == 0:
print_yellow(" WORST VGPR SPILLS (not deltas)" + (" " * 40) +
"VGPRs SpillVGPR ScratchVGPR")
stats.print_one_shader_vgpr_spill(name)
num += 1
if num == 10:
break
if num > 0:
print
# VGPR spilling apps
print_yellow(" VGPR SPILLING APPS Shaders SpillVGPR ScratchVGPR")
for name, stats in sorted(apps.items()):
stats.print_vgpr_spilling_app(name)
print
# worst SGPR spills
num = 0
sort_key = lambda v: -v[1].after.spilled_sgprs
for name, stats in sorted(shaders.items(), key = sort_key):
if num == 0:
print_yellow(" WORST SGPR SPILLS (not deltas)" + (" " * 40) +
"SGPRs SpillSGPR")
stats.print_one_shader_sgpr_spill(name)
num += 1
if num == 10:
break
if num > 0:
print
# SGPR spilling apps
print_yellow(" SGPR SPILLING APPS Shaders SpillSGPR AvgPerSh")
for name, stats in sorted(apps.items()):
stats.print_sgpr_spilling_app(name)
print
# worst regressions
metrics = si_stats().metrics
for i in range(len(metrics)):
# maxwaves regressions aren't reported (see vgprs/sgprs instead)
if metrics[i][0] == 'maxwaves':
continue
field = metrics[i][0]
num = 0
sort_key = lambda v: -v[1].diff.__dict__[field]
for name, stats in sorted(shaders.items(), key = sort_key):
if stats.diff.__dict__[field] <= 0:
continue
if num == 0:
print_yellow(" WORST REGRESSIONS - {:49}".format(metrics[i][1]) +
"Before After Delta Percentage")
stats.print_regression(name, field)
num += 1
if num == 10:
break
if num > 0:
print
# percentages
legend = "Shaders SGPRs VGPRs SpillSGPR SpillVGPR Scratch CodeSize MaxWaves Waits"
print_yellow(" PERCENTAGE DELTAS " + legend)
for name, stats in sorted(apps.items()):
stats.print_percentages(name)
print " " + ("-" * (21 + len(legend)))
total_affected.print_percentages("All affected")
print " " + ("-" * (21 + len(legend)))
total.print_percentages("Total")
print
def main():
before = sys.argv[1]
after = sys.argv[2]
compare_results(get_results(before), get_results(after))
results_before = get_results(before)
results_after = get_results(after)
compare_results(results_before, results_after)
print_tables(results_before, results_after)
if __name__ == "__main__":
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment