Commit 45e1fd8f authored by Alexandros Frantzis's avatar Alexandros Frantzis

Switch to checksum based image checks

Use checksums instead of reference images for image checks to allow
for an easier workflow when an update needs to be made.

Also update the README file with current instructions.
Signed-off-by: Alexandros Frantzis's avatarAlexandros Frantzis <alexandros.frantzis@collabora.com>
parent e237812c
Pipeline #94499 failed with stages
in 18 minutes and 1 second
......@@ -54,7 +54,7 @@ x86_build:
- .debian@container-ifnot-exists
- .container
variables:
DEBIAN_TAG: &x86_build "2019-11-22"
DEBIAN_TAG: &x86_build "2019-12-16"
.use-x86_build:
variables:
......@@ -118,8 +118,6 @@ mesa-build:
.conformance-base:
variables:
TRACES_DB_REPO: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db"
TRACES_DB_COMMIT: "master"
TAG: *x86_build
image: "$CI_REGISTRY_IMAGE/debian/x86_build:$TAG"
stage: test
......@@ -127,8 +125,6 @@ mesa-build:
- mesa-build
before_script:
- tests/test.sh
- git clone --no-checkout "$TRACES_DB_REPO" traces-db
- (cd $CI_PROJECT_DIR/traces-db; git reset "$TRACES_DB_COMMIT" || git reset "origin/$TRACES_DB_COMMIT")
- export DISPLAY=:0
- export MESA_SHA=$(cat $CI_PROJECT_DIR/mesa_sha.txt)
- export LD_LIBRARY_PATH="$CI_PROJECT_DIR/mesa/install/lib/:/usr/local/lib/"
......@@ -139,14 +135,10 @@ mesa-build:
- bash -c "echo DEVICE=$DEVICE_NAME >$CI_PROJECT_DIR/results/summary.txt"
- bash -c "echo MESA_SHA=$MESA_SHA >>$CI_PROJECT_DIR/results/summary.txt"
script:
- cd $CI_PROJECT_DIR/traces-db
# Run renderdoc traces with the surfaceless platform
- EGL_PLATFORM=surfaceless DISPLAY= $CI_PROJECT_DIR/.gitlab-ci/tracie-runner.sh rdc
- EGL_PLATFORM=surfaceless DISPLAY= $CI_PROJECT_DIR/.gitlab-ci/tracie-runner.sh $CI_PROJECT_DIR/.gitlab-ci/traces.yml renderdoc
# Run apitrace traces with xvfb (for now, until we add surfaceless support)
- $CI_PROJECT_DIR/.gitlab-ci/tracie-runner.sh trace
# Disable performance replays, since we don't need them at the moment and
# the replay started failing: https://gitlab.freedesktop.org/gfx-ci/tracie/tracie/issues/1
# - python3 $CI_PROJECT_DIR/scripts/replay_trace_fps.py .
- $CI_PROJECT_DIR/.gitlab-ci/tracie-runner.sh $CI_PROJECT_DIR/.gitlab-ci/traces.yml apitrace
artifacts:
when: always
paths:
......
......@@ -56,6 +56,9 @@ rm -rf "$TMPDIR"
# Sanity check we're using the local mesa $glxinfo | grep $MESA_SHA
apt-get install -y mesa-utils
# Required by our scripts to parse traces definition yaml files
apt-get install -y python3-yaml
# Renderdoc and apitrace
export APITRACE_VERSION=0f541f460bba86a69c8c714076db5a7607cc8d16
export RENDERDOC_VERSION=ba3df46052afd78b8d24a136aa450c08548ef2c9
......
traces-db:
repo: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db"
commit: "master"
traces:
- path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc
expectations:
- device: intel-0x3185
checksum: 1416eeaa9abe052a6090651fc36b4c52
- device: vmware-llvmpipe
checksum: 8867f3a41f180626d0d4b7661ff5c0f4
- path: glmark2/jellyfish.rdc
expectations:
- device: intel-0x3185
checksum: c569e3edbac8a9efcfd590a9d1a2eb12
- device: vmware-llvmpipe
checksum: d82267c25a0decdad7b563c56bb81106
- path: glxgears/glxgears.trace
expectations:
- device: intel-0x3185
checksum: 87b8bcd27cc03a1dbf36aeca66748541
- device: vmware-llvmpipe
checksum: 02aca9b4b4ad6fd60331df6e4f87f2cd
- path: supertuxkart/supertuxkart-antediluvian-abyss.rdc
expectations:
- device: intel-0x3185
checksum: ff827f7eb069afd87cc305a422cba939
- path: supertuxkart/supertuxkart-menu.rdc
expectations:
- device: intel-0x3185
checksum: 0a4095dc7b441643a3336975b61c9e6a
- path: supertuxkart/supertuxkart-ravenbridge-mansion.rdc
expectations:
- device: intel-0x3185
checksum: 00fccc66f7a24446b541daf871d3e84c
#!/usr/bin/env bash
TRACIE_SCRIPT_DIR="$CI_PROJECT_DIR/.gitlab-ci/tracie"
TRACES_YAML="$(readlink -f "$1")"
TRACE_TYPE="$2"
clone_traces_db()
{
local repo="$1"
local commit="$2"
rm -rf traces-db
git clone --no-checkout "$repo" traces-db
(cd traces-db; git reset "$commit" || git reset "origin/$commit")
}
query_traces_yaml()
{
python3 "$TRACIE_SCRIPT_DIR/query_traces_yaml.py" \
--file "$TRACES_YAML" "$@"
}
create_clean_git()
{
rm -rf .clean_git
......@@ -14,58 +33,79 @@ restore_clean_git()
fetch_trace()
{
_trace="${1//,/?}"
local trace="${1//,/?}"
echo -n "[fetch_trace] Fetching $1... "
_output=$(git lfs pull -I "$_trace" 2>&1)
_ret=0
local output=$(git lfs pull -I "$trace" 2>&1)
local ret=0
if [[ $? -ne 0 || ! -f "$1" ]]; then
echo "ERROR"
echo "$_output"
_ret=1
echo "$output"
ret=1
else
echo "OK"
fi
restore_clean_git
return $_ret
return $ret
}
fetch_reference_images()
get_dumped_file()
{
_device_name="$1"
_trace_type="$2"
_wildcard="**/$_device_name/*$_trace_type*.png"
echo -n "[fetch_trace] Fetching reference images '$_wildcard'... "
_output=$(git lfs pull -I "$_wildcard" 2>&1)
_ret=0
if [[ $? -ne 0 ]]; then
echo "ERROR"
echo "$_output"
_ret=1
local trace="$1"
local tracedir="$(dirname "$trace")"
local tracename="$(basename "$trace")"
find "$tracedir/test/$DEVICE_NAME" -name "$tracename*.$2"
}
check_image()
{
local trace="$1"
local image="$2"
checksum=$(python3 "$TRACIE_SCRIPT_DIR/image_checksum.py" "$image")
expected=$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")
if [[ "$checksum" = "$expected" ]]; then
echo "[check_image] Images match for $trace"
return 0
else
echo "OK"
echo "[check_image] Images differ for $trace (expected: $expected, actual: $checksum)"
return 1
fi
restore_clean_git
return $_ret
}
archive_artifact()
{
mkdir -p "$CI_PROJECT_DIR/results"
cp --parents "$1" "$CI_PROJECT_DIR/results"
}
if [[ -n "$(query_traces_yaml traces_db_repo)" ]]; then
clone_traces_db "$(query_traces_yaml traces_db_repo)" \
"$(query_traces_yaml traces_db_commit)"
cd traces-db
else
echo "Warning: No traces-db entry in $TRACES_YAML, assuming traces-db is current directory"
fi
# During git operations various git objects get created which
# may take up significant space. Store a clean .git instance,
# which we restore after various git operations to keep our
# storage consumption low.
create_clean_git
fetch_reference_images "$DEVICE_NAME" "$1" || exit $?
ret=0
TRACIE_SCRIPT_DIR="$CI_PROJECT_DIR/.gitlab-ci/tracie"
for trace in $(git lfs ls-files -n -I \*.$1)
for trace in $(query_traces_yaml traces --device-name "$DEVICE_NAME" --trace-types "$TRACE_TYPE")
do
python3 "$TRACIE_SCRIPT_DIR/trace_has_ref_images.py" --device-name "$DEVICE_NAME" "$trace" || { echo "[fetch_trace] Skipping $trace since it has no reference images"; continue; }
[[ -n "$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")" ]] || { echo "[fetch_trace] Skipping $trace since it has no checksums for $DEVICE_NAME"; continue; }
fetch_trace "$trace" || exit $?
python3 "$TRACIE_SCRIPT_DIR/dump_trace_images.py" --device-name "$DEVICE_NAME" "$trace" || exit $?
python3 "$TRACIE_SCRIPT_DIR/diff_trace_images.py" --device-name "$DEVICE_NAME" --output-dir "$CI_PROJECT_DIR/results" "$trace" || ret=1
image="$(get_dumped_file "$trace" png)"
check_image "$trace" "$image" && check_succeeded=true || { ret=1; check_succeeded=false; }
if [[ "$check_succeeded" = false || "$TRACIE_STORE_IMAGES" = "1" ]]; then
archive_artifact "$image"
fi
archive_artifact "$(get_dumped_file "$trace" log)"
rm "$trace"
done
......
......@@ -27,8 +27,7 @@ import os
import sys
import subprocess
from pathlib import Path
from traceutil import iter_trace_paths, trace_has_images, all_trace_type_names
from traceutil import trace_type_from_name, trace_type_from_filename, TraceType
from traceutil import trace_type_from_filename, TraceType
def log(severity, msg, end='\n'):
print("[dump_trace_images] %s: %s" % (severity, msg), flush=True, end=end)
......@@ -48,57 +47,60 @@ def run_logged_command(cmd, log_path):
logoutput.decode(errors='replace') +
"[dump_traces_images] Process failed with error code: %d" % ret.returncode)
def get_calls_num(trace, device_name):
tracename = str(Path(trace).name)
refdir = str(Path(trace).parent / "references" / device_name)
calls = []
for root, dirs, files in os.walk(refdir):
for file in files:
if file.startswith(tracename) and file.endswith(".png"):
callnum = file[(len(tracename) + 1):-4]
calls.append(callnum)
return calls
def dump_with_apitrace(trace, calls, device_name):
outputdir = str(Path(trace).parent / "test" / device_name)
def get_last_apitrace_frame_call(trace_path):
cmd = ["apitrace", "dump", "--calls=frame", str(trace_path)]
ret = subprocess.run(cmd, stdout=subprocess.PIPE)
for l in reversed(ret.stdout.decode(errors='replace').splitlines()):
s = l.split(None, 1)
if len(s) >= 1 and s[0].isnumeric():
return int(s[0])
return -1
def dump_with_apitrace(trace_path, calls, device_name):
outputdir = str(trace_path.parent / "test" / device_name)
os.makedirs(outputdir, exist_ok=True)
outputprefix = str(Path(outputdir) / Path(trace).name) + "-"
outputprefix = str(Path(outputdir) / trace_path.name) + "-"
if len(calls) == 0:
calls = [str(get_last_apitrace_frame_call(trace_path))]
cmd = ["apitrace", "dump-images", "--calls=" + ','.join(calls),
"-o", outputprefix, trace]
log_path = Path(outputdir) / (Path(trace).name + ".log")
"-o", outputprefix, str(trace_path)]
log_path = Path(outputdir) / (trace_path.name + ".log")
run_logged_command(cmd, log_path)
def dump_with_renderdoc(trace, calls, device_name):
outputdir = str(Path(trace).parent / "test" / device_name)
def dump_with_renderdoc(trace_path, calls, device_name):
outputdir = str(trace_path.parent / "test" / device_name)
script_path = Path(os.path.dirname(os.path.abspath(__file__)))
cmd = [str(script_path / "renderdoc_dump_images.py"), trace, outputdir]
cmd = [str(script_path / "renderdoc_dump_images.py"), str(trace_path), outputdir]
cmd.extend(calls)
log_path = Path(outputdir) / (Path(trace).name + ".log")
log_path = Path(outputdir) / (trace_path.name + ".log")
run_logged_command(cmd, log_path)
def dump_with_testtrace(trace, calls, device_name):
def dump_with_testtrace(trace_path, calls, device_name):
from PIL import Image
outputdir_path = Path(trace).parent / "test" / device_name
outputdir_path = trace_path.parent / "test" / device_name
outputdir_path.mkdir(parents=True, exist_ok=True)
with Path(trace).open() as f:
with trace_path.open() as f:
rgba = f.read()
color = [int(rgba[0:2], 16), int(rgba[2:4], 16),
int(rgba[4:6], 16), int(rgba[6:8], 16)]
if len(calls) == 0: calls = ["0"]
for c in calls:
outputfile = str(outputdir_path / Path(trace).name) + "-" + c + ".png"
outputfile = str(outputdir_path / trace_path.name) + "-" + c + ".png"
log_path = outputdir_path / (trace_path.name + ".log")
with log_path.open(mode='w') as log:
log.write("Writing RGBA: %s to %s" % (rgba, outputfile))
Image.frombytes('RGBA', (32, 32), bytes(color * 32 * 32)).save(outputfile)
def dump_from_trace(trace, device_name):
log("Info", "Dumping trace %s" % trace, end='... ')
calls = get_calls_num(trace, device_name)
trace_type = trace_type_from_filename(trace)
def dump_from_trace(trace_path, calls, device_name):
log("Info", "Dumping trace %s" % trace_path, end='... ')
trace_type = trace_type_from_filename(trace_path.name)
try:
if trace_type == TraceType.APITRACE:
dump_with_apitrace(trace, calls, device_name)
dump_with_apitrace(trace_path, calls, device_name)
elif trace_type == TraceType.RENDERDOC:
dump_with_renderdoc(trace, calls, device_name)
dump_with_renderdoc(trace_path, calls, device_name)
elif trace_type == TraceType.TESTTRACE:
dump_with_testtrace(trace, calls, device_name)
dump_with_testtrace(trace_path, calls, device_name)
else:
raise RuntimeError("Unknown tracefile extension")
log_result("OK")
......@@ -110,43 +112,23 @@ def dump_from_trace(trace, device_name):
log("Debug", "=== Failure log end ===")
return False
def filter_trace_paths_with_ref_images(trace_paths, device_name):
out = []
for trace_path in trace_paths:
if trace_has_images(trace_path, str(Path("references") / device_name)):
out.append(str(trace_path))
else:
log("Warning", "%s has no reference images, skipping" % str(trace_path))
return out
def main():
parser = argparse.ArgumentParser()
parser.add_argument('tracepath', help="single trace or dir to walk recursively")
parser.add_argument('tracepath', help="trace to dump")
parser.add_argument('--device-name', required=True,
help="the name of the graphics device used to produce images")
parser.add_argument('--trace-types', required=False,
default=",".join(all_trace_type_names()),
help="the types of traces to look for in recursive dir walks "
"(by default all types)")
parser.add_argument('--calls', required=False,
help="the call numbers from the trace to dump (default: last frame)")
args = parser.parse_args()
trace_types = [trace_type_from_name(t) for t in args.trace_types.split(",")]
if os.path.isdir(args.tracepath):
all_trace_paths = iter_trace_paths(args.tracepath, trace_types)
elif os.path.isfile(args.tracepath):
all_trace_paths = [Path(args.tracepath)]
traces = filter_trace_paths_with_ref_images(all_trace_paths,
args.device_name)
failed_dump = False
if args.calls is not None:
args.calls = args.calls.split(",")
else:
args.calls = []
for trace in traces:
if not dump_from_trace(trace, args.device_name):
failed_dump = True
success = dump_from_trace(Path(args.tracepath), args.calls, args.device_name)
sys.exit(1 if failed_dump else 0)
sys.exit(0 if success else 1)
if __name__ == "__main__":
main()
#!/usr/bin/python3
#!/usr/bin/env python3
# Copyright (c) 2019 Collabora Ltd
#
......@@ -23,23 +23,17 @@
# SPDX-License-Identifier: MIT
import argparse
import sys
from pathlib import Path
from traceutil import trace_has_images
import hashlib
from PIL import Image
def main():
parser = argparse.ArgumentParser()
parser.add_argument('tracepath', help="single trace")
parser.add_argument('--device-name', required=True,
help="the name of the graphics device used to produce images")
parser.add_argument('imagefile', help='image file to calculate checksum for')
args = parser.parse_args()
has_images = trace_has_images(Path(args.tracepath),
str(Path("references") / args.device_name))
# Use the shell convention of 0 for true/success and 1 for false/failure
return 0 if has_images else 1
md5 = hashlib.md5(Image.open(args.imagefile).tobytes())
print(md5.hexdigest())
if __name__ == "__main__":
sys.exit(main())
main()
#!/usr/bin/env python3
#!/usr/bin/python3
# Copyright (c) 2019 Collabora Ltd
#
......@@ -23,88 +23,85 @@
# SPDX-License-Identifier: MIT
import argparse
import glob
import os
import shutil
import subprocess
import sys
from pathlib import Path
from traceutil import iter_trace_paths, trace_has_images
def log(severity, msg):
print("[diff_trace_images] %s: %s" % (severity, msg))
def find_traces_with_images(directory, device_name):
traces = []
for trace_path in iter_trace_paths(directory):
if trace_has_images(trace_path, str(Path("references") / device_name)):
if trace_has_images(trace_path, str(Path("test") / device_name)):
traces.append(str(trace_path))
else:
log("Warning", "%s has reference but no test images, skipping" % str(trace_path))
return traces
def copy_artifacts_to_outputdir(trace, outputdir, device_name):
trace_path = Path(trace)
tracedir = str(trace_path.parent.name)
tracename = str(trace_path.name)
refprefix = str(trace_path.parent / "references" / device_name / trace_path.name) + "-"
testprefix = str(trace_path.parent / "test" / device_name / trace_path.name) + "-"
refdest = str(Path(outputdir) / tracedir / tracename / "references" / device_name)
testdest = str(Path(outputdir) / tracedir / tracename / "test" / device_name)
os.makedirs(refdest, exist_ok=True)
os.makedirs(testdest, exist_ok=True)
for f in glob.glob(refprefix + '*.png'):
shutil.copy(f, refdest)
for f in glob.glob(testprefix + '*.png'):
shutil.copy(f, testdest)
log_path = trace_path.parent / "test" / device_name / (tracename + ".log")
if log_path.is_file():
shutil.copy(str(log_path), testdest)
return str(Path(outputdir) / tracedir / tracename)
def diff_images(imagedir, device_name):
cmd = ["apitrace", "diff-images", "--output=summary.html",
str(Path("references") / device_name),
str(Path("test") / device_name)]
ret = subprocess.call(cmd, cwd=imagedir)
return ret == 0
import yaml
from traceutil import all_trace_type_names, trace_type_from_name
from traceutil import trace_type_from_filename
def main():
parser = argparse.ArgumentParser()
parser.add_argument('tracepath', help="single trace, or dir to walk recursively")
parser.add_argument('--output-dir', required=True,
help="directory to write results to")
parser.add_argument('--device-name', required=True,
help="the name of the graphics device used to produce images")
def trace_devices(trace):
return [e['device'] for e in trace['expectations']]
args = parser.parse_args()
def cmd_traces_db_repo(args):
with open(args.file, 'r') as f:
y = yaml.safe_load(f)
print(y['traces-db']['repo'])
def cmd_traces_db_commit(args):
with open(args.file, 'r') as f:
y = yaml.safe_load(f)
print(y['traces-db']['commit'])
def cmd_traces(args):
with open(args.file, 'r') as f:
y = yaml.safe_load(f)
traces = y['traces']
traces = filter(lambda t: trace_type_from_filename(t['path']) in args.trace_types,
traces)
if args.device_name:
traces = filter(lambda t: args.device_name in trace_devices(t), traces)
traces = list(traces)
traces = []
if os.path.isdir(args.tracepath):
traces.extend(find_traces_with_images(args.tracepath, args.device_name))
elif os.path.isfile(args.tracepath):
traces.append(args.tracepath)
if len(traces) == 0:
return
failed_diff = False
print('\n'.join((t['path'] for t in traces)))
for trace in traces:
imageout = copy_artifacts_to_outputdir(trace, args.output_dir, args.device_name)
images_match = diff_images(imageout, args.device_name)
if not images_match:
log("Info", "Images differ for %s" % trace)
failed_diff = True
else:
log("Info", "Images match for %s" % trace)
def cmd_checksum(args):
with open(args.file, 'r') as f:
y = yaml.safe_load(f)
traces = y['traces']
trace = next(t for t in traces if t['path'] == args.trace_path)
expectation = next(e for e in trace['expectations'] if e['device'] == args.device_name)
print(expectation['checksum'])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--file', required=True,
help='the name of the yaml file')
subparsers = parser.add_subparsers(help='sub-command help')
parser_traces_db_repo = subparsers.add_parser('traces_db_repo')
parser_traces_db_repo.set_defaults(func=cmd_traces_db_repo)
parser_traces_db_commit = subparsers.add_parser('traces_db_commit')
parser_traces_db_commit.set_defaults(func=cmd_traces_db_commit)
parser_traces = subparsers.add_parser('traces')
parser_traces.add_argument('--device-name', required=False,
help="the name of the graphics device used to "
"produce images")
parser_traces.add_argument('--trace-types', required=False,
default=",".join(all_trace_type_names()),
help="the types of traces to look for in recursive "
"dir walks " "(by default all types)")
parser_traces.set_defaults(func=cmd_traces)
parser_checksum = subparsers.add_parser('checksum')
parser_checksum.add_argument('--device-name', required=True,
help="the name of the graphics device used to "
"produce images")
parser_checksum.add_argument('trace_path')
parser_checksum.set_defaults(func=cmd_checksum)
args = parser.parse_args()
if hasattr(args, 'trace_types'):
args.trace_types = [trace_type_from_name(t) for t in args.trace_types.split(",")]
sys.exit(1 if failed_diff else 0)
args.func(args)
if __name__ == "__main__":
main()
......@@ -88,6 +88,9 @@ def renderdoc_dump_images(filename, eventIds, outputDir):
tracefile = Path(filename).name
if len(eventIds) == 0:
eventIds.append(controller.GetDrawcalls()[-1].eventId)
for eventId in eventIds:
dumpImage(controller, eventId, outputDir, tracefile)
......@@ -95,8 +98,8 @@ def renderdoc_dump_images(filename, eventIds, outputDir):
cap.Shutdown()
if __name__ == "__main__":
if len(sys.argv) < 4:
raise RuntimeError("Usage: renderdoc_dump_images.py <trace> <outputdir> <draw-id>...")
if len(sys.argv) < 3:
raise RuntimeError("Usage: renderdoc_dump_images.py <trace> <outputdir> [<draw-id>...]")
eventIds = [int(e) for e in sys.argv[3:]]
......
#!/usr/bin/python3
# Copyright (c) 2019 Collabora Ltd
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,