Commit 550a4f77 authored by Pablo Saavedra's avatar Pablo Saavedra Committed by Marge Bot

ci: Migrate tracie tests done in shell script to pytest

v2: Verbatim translation from the original shell script
    Make the corrections visible in explicit commits (Andres)
    Remove redundant code (Alexandros)
    Code style nitpick (Rohan)

Reimplementation of the tracie's self-tests using a pythonic test suit
(pytest).

The new tracie/test.py module is almost a direct translation of the
tests defined in the tracie/test.sh. This new implementation of the
test provides a more common framework where define the tests.
Also allows a better introspection for the tests results and/or
resulting errors.

This patch also adds python3-pytest as dependency for the built images
and adapts the tracie-runner scripts to run the self-test using pytest.
Signed-off-by: Pablo Saavedra's avatarPablo Saavedra <psaavedra@igalia.com>
Reviewed-by: Alexandros Frantzis <alexandros.frantzis@collabora.com> [v1]
Reviewed-by: Andres Gomez's avatarAndres Gomez <agomez@igalia.com>
Reviewed-by: Rohan Garg <rohan.garg@collabora.com> [v1]
Part-of: <!4916>
parent 37621da7
......@@ -136,7 +136,7 @@ x86_build:
- .fdo.container-build@debian
- .container
variables:
FDO_DISTRIBUTION_TAG: &x86_build "2020-05-08"
FDO_DISTRIBUTION_TAG: &x86_build "2020-05-18"
.use-x86_build:
variables:
......@@ -149,13 +149,13 @@ x86_build:
x86_test-gl:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-08"
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-05-18"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-08"
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-05-18"
# Debian 9 based x86 build image (old LLVM)
x86_build_old:
......
......@@ -64,6 +64,7 @@ apt-get install -y --no-remove \
python3-mako \
python3-numpy \
python3-pil \
python3-pytest \
python3-requests \
python3-six \
python3-yaml \
......
......@@ -64,6 +64,7 @@ apt-get install -y --no-remove \
python \
python3-distutils \
python3-pil \
python3-pytest \
python3-requests \
python3-yaml \
vulkan-tools \
......
......@@ -26,7 +26,7 @@ RESULTS=`pwd`/results
mkdir -p $RESULTS
# Perform a self-test to ensure tracie is working properly.
"$INSTALL/tracie/tests/test.sh"
python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
if [ "$GALLIUM_DRIVER" = "virpipe" ]; then
# tracie is to use virpipe, and virgl_test_server llvmpipe
......
......@@ -21,7 +21,7 @@ export DXVK_LOG_LEVEL="none"
export DXVK_STATE_CACHE=0
# Perform a self-test to ensure tracie is working properly.
"$INSTALL/tracie/tests/test.sh"
python3 -m pytest -v --pyargs $INSTALL/tracie/tests/test.py
# Sanity check to ensure that our environment is sufficient to make our tests
# run against the Mesa built by CI, rather than any installed distro version.
......
import logging
import pytest
import re
import shutil
from os import environ, chdir
from os.path import dirname, exists, realpath
import tracie
RESULTS_YAML = "results/results.yml"
TRACE_LOG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace.log"
TRACE_LOG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace.log"
TRACE_PNG_TEST1 = "results/trace1/test/gl-test-device/magenta.testtrace-0.png"
TRACE_PNG_TEST2 = "results/trace2/test/vk-test-device/olive.testtrace-0.png"
TRACIE_DIR = dirname(realpath(__file__)) + "/.."
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
def write_to(content, filename):
with open(filename, 'w') as f:
f.write(content)
def read_from(filename):
with open(filename) as f:
content = f.read()
return content
def run_tracie():
'''
Run tests for the .testtrace types, using the "gl-test-device" and
"vk-test-device" device names.
'''
result = tracie.main(["--device-name", "gl-test-device",
"--file", "./tests/traces.yml"])
if not result:
return False
result = tracie.main(["--device-name", "vk-test-device",
"--file", "./tests/traces.yml"])
return result
def prepare_for_run(tmp_path):
'''
Copy all the tracie scripts to the test dir for the unit tests.
This avoids polluting the normal working dir with test result artifacts.
'''
test_dir = str(tmp_path) + "/run"
shutil.copytree(TRACIE_DIR, test_dir)
# Change the working dir to the test_dir
chdir(test_dir)
# Set the traces-db
shutil.move("./tests/test-data", "./traces-db")
# Disable trace storing
environ["TRACIE_STORE_IMAGES"] = "0"
def cleanup(tmp_path):
'''
Performs the clean up of the test dir.
'''
if exists(tmp_path):
shutil.rmtree(tmp_path)
@pytest.fixture(autouse=True)
def run_test(tmp_path):
'''
Wraps the execution of each test as follows:
prepare_for_run()
test()
cleanup()
'''
logger.debug("Working dir: %s", tmp_path)
prepare_for_run(tmp_path)
yield
cleanup(tmp_path)
def check_results_yaml_content(filename, expectations):
'''
Checks the content of the filename with the list of expectations
passed as parameter.
Arguments:
filename (str): The path of the file to check
expectations (list): A list with the content to find in the file
Returns:
bool: The return value. True if the content of the filename satisfies
the expectations, False otherwise.
'''
content = read_from(filename)
for e in expectations:
ocurrencies = re.findall(e, content)
if not len(ocurrencies):
logger.error("Expectation not found in %s: %s", filename, e)
return False
return True
def test_tracie_succeeds_if_all_images_match():
assert run_tracie()
expectations = [
"actual: 5efda83854befe0155ff8517a58d5b51",
"expected: 5efda83854befe0155ff8517a58d5b51",
]
assert check_results_yaml_content(RESULTS_YAML, expectations)
def test_tracie_fails_on_image_mismatch():
filename = "./tests/traces.yml"
content = read_from(filename)
content = content.replace("5efda83854befe0155ff8517a58d5b51",
"8e0a801367e1714463475a824dab363b")
write_to(content, filename)
assert not run_tracie()
expectations = [
"actual: 5efda83854befe0155ff8517a58d5b51",
"expected: 8e0a801367e1714463475a824dab363b",
"trace2/test/vk-test-device/olive.testtrace-0.png"
]
assert check_results_yaml_content(RESULTS_YAML, expectations)
def test_tracie_skips_traces_without_checksum():
filename = "./tests/traces.yml"
content = '''traces:
- path: trace1/red.testtrace
expectations:
- device: bla
checksum: 000000000000000'''
write_to(content, filename)
# red.testtrace should be skipped, since it doesn't
# have any checksums for our device
filename = "./traces-db/trace1/red.testtrace"
content = "ff0000ff"
write_to(content, filename)
assert run_tracie()
def test_tracie_fails_on_dump_image_error():
# "invalid" should fail to parse as rgba and
# cause an error
filename = "./traces-db/trace1/magenta.testtrace"
write_to("invalid\n", filename)
run_tracie()
expectations = [
"actual: error",
"expected: 8e0a801367e1714463475a824dab363b",
"trace1/magenta.testtrace",
]
assert check_results_yaml_content(RESULTS_YAML, expectations)
def test_tracie_stores_only_logs_on_checksum_match():
assert run_tracie()
assert exists(TRACE_LOG_TEST1)
assert exists(TRACE_LOG_TEST2)
assert not exists(TRACE_PNG_TEST1)
assert not exists(TRACE_PNG_TEST2)
def test_tracie_stores_images_on_checksum_mismatch():
filename = "./tests/traces.yml"
content = read_from(filename)
content = content.replace("5efda83854befe0155ff8517a58d5b51",
"8e0a801367e1714463475a824dab363b")
write_to(content, filename)
assert not run_tracie()
assert not exists(TRACE_PNG_TEST1)
assert exists(TRACE_PNG_TEST2)
def test_tracie_stores_images_on_request():
environ["TRACIE_STORE_IMAGES"] = "1"
assert run_tracie()
assert exists(TRACE_PNG_TEST1)
assert exists(TRACE_PNG_TEST2)
#!/bin/sh
TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.."
TEST_DIR=""
TEST_EXIT=0
assert() {
if ! $1; then
echo "Assertion failed: \"$1\""
exit 1
fi
}
run_tracie() {
# Run tests for the .testtrace types, using the "gl-test-device" and "vk-test-device" device names.
python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name gl-test-device && \
python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name vk-test-device
}
cleanup() {
[ "$TEST_DIR" = "/tmp/*" ] && rm -rf "$TEST_DIR"
}
prepare_for_run() {
TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)"
# Copy all the tracie scripts to the test dir for the run-tests.sh script.
# This avoids polluting the normal working dir with test result artifacts.
cp -R "$TRACIE_DIR"/. "$TEST_DIR"
cd "$TEST_DIR"
mkdir traces-db
mv tests/test-data/* traces-db/.
trap cleanup EXIT
# Ensure we have a clean environment.
unset TRACIE_STORE_IMAGES
}
run_test() {
prepare_for_run
log=$(mktemp)
if ($1 > "$log" 2>&1 ;); then
if [ -t 1 ]; then
echo "$1: \e[0;32mSuccess\e[0m"
else
echo "$1: Success"
fi
else
if [ -t 1 ]; then
echo "$1: \e[0;31mFail\e[0m"
else
echo "$1: Fail"
fi
cat "$log"
TEST_EXIT=1
fi
rm "$log"
cleanup
}
assert_results_yaml_contains() {
grep -q "actual: $1" $4
assert "[ $? = 0 ]"
grep -q "expected: $2" $4
assert "[ $? = 0 ]"
if [ $3 != "" ]; then
grep -q $3 $4
fi
assert "[ $? = 0 ]"
}
tracie_succeeds_if_all_images_match() {
run_tracie
assert "[ $? = 0 ]"
assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 5efda83854befe0155ff8517a58d5b51 "" "$PWD/results/results.yml"
}
tracie_fails_on_image_mismatch() {
sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
"$TEST_DIR/tests/traces.yml"
run_tracie
assert "[ $? != 0 ]"
assert_results_yaml_contains 5efda83854befe0155ff8517a58d5b51 8e0a801367e1714463475a824dab363b "trace2/test/vk-test-device/olive.testtrace-0.png" "$PWD/results/results.yml"
}
tracie_skips_traces_without_checksum() {
echo "traces:" > "$TEST_DIR/tests/traces.yml"
echo " - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml"
echo " expectations:" >> "$TEST_DIR/tests/traces.yml"
echo " - device: bla" >> "$TEST_DIR/tests/traces.yml"
echo " checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
# red.testtrace should be skipped, since it doesn't
# have any checksums for our device
echo "ff0000ff" > traces-db/trace1/red.testtrace
run_tracie
assert "[ $? = 0 ]"
}
tracie_fails_on_dump_image_error() {
# "invalid" should fail to parse as rgba and
# cause an error
echo "invalid" > traces-db/trace1/magenta.testtrace
run_tracie
assert "[ $? != 0 ]"
assert_results_yaml_contains "error" "8e0a801367e1714463475a824dab363b" "trace1/magenta.testtrace" "$PWD/results/results.yml"
}
tracie_stores_only_logs_on_checksum_match() {
run_tracie
assert "[ $? = 0 ]"
assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace.log" ]"
assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace.log" ]"
assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
assert "[ ! -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
ls -lR "$TEST_DIR"
}
tracie_stores_images_on_checksum_mismatch() {
sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
"$TEST_DIR/tests/traces.yml"
run_tracie
assert "[ $? != 0 ]"
assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
}
tracie_stores_images_on_request() {
(export TRACIE_STORE_IMAGES=1; run_tracie)
assert "[ $? = 0 ]"
assert "[ -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
ls -lR "$TEST_DIR"
}
run_test tracie_succeeds_if_all_images_match
run_test tracie_fails_on_image_mismatch
run_test tracie_skips_traces_without_checksum
run_test tracie_fails_on_dump_image_error
run_test tracie_stores_only_logs_on_checksum_match
run_test tracie_stores_images_on_checksum_mismatch
run_test tracie_stores_images_on_request
exit $TEST_EXIT
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment