From f44f28d47300e516d4f0b10f40282fbf8932294d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= Date: Wed, 20 Apr 2022 21:25:33 -0400 Subject: [PATCH 1/5] dzn: Clean-up of deqp-runner start script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use the "--fraction" parameter on the right command (it was on the junit call). * Add all deqp-runner options into an array. * Split flaky tests into a different file. Signed-off-by: Louis-Francis Ratté-Boulianne --- .gitlab-ci/windows/deqp_runner_run.ps1 | 32 +++++++++++++++---- .../{warp-fails.txt => dozen-warp-fails.txt} | 1 - src/microsoft/ci/dozen-warp-flakes.txt | 1 + 3 files changed, 26 insertions(+), 8 deletions(-) rename src/microsoft/ci/{warp-fails.txt => dozen-warp-fails.txt} (99%) create mode 100644 src/microsoft/ci/dozen-warp-flakes.txt diff --git a/.gitlab-ci/windows/deqp_runner_run.ps1 b/.gitlab-ci/windows/deqp_runner_run.ps1 index 30d7be243002..6250550e0886 100644 --- a/.gitlab-ci/windows/deqp_runner_run.ps1 +++ b/.gitlab-ci/windows/deqp_runner_run.ps1 @@ -13,19 +13,37 @@ New-Item -Path $hkey_path -force New-ItemProperty -Path $hkey_path -Name $hkey_name -Value 0 -PropertyType DWORD $results = New-Item -ItemType Directory results -$deqp_options = @("--deqp-surface-width", 256, "--deqp-surface-height", 256, "--deqp-surface-type", "pbuffer", "--deqp-gl-config-name", "rgba8888d24s8ms0", "--deqp-visibility", "hidden") -$deqp_module = "C:\deqp\external\vulkancts\modules\vulkan\deqp-vk.exe" -$caselist = "C:\deqp\mustpass\vk-master.txt" -$baseline = ".\_install\warp-fails.txt" -$includes = @("-t", "dEQP-VK.api.*", "-t", "dEQP-VK.info.*", "-t", "dEQP-VK.draw.*", "-t", "dEQP-VK.query_pool.*", "-t", "dEQP-VK.memory.*") + +$deqp_options = @( + "--deqp-surface-width", 256, + "--deqp-surface-height", 256, + "--deqp-surface-type", "pbuffer", + "--deqp-gl-config-name", "rgba8888d24s8ms0", + "--deqp-visibility", "hidden" +) +$deqp_runner_options = @( + "--deqp", "C:\deqp\external\vulkancts\modules\vulkan\deqp-vk.exe", + "--output", $results.FullName, + "--caselist", "C:\deqp\mustpass\vk-master.txt", + "--baseline", ".\_install\dozen-warp-fails.txt", + "--flakes", ".\_install\dozen-warp-flakes.txt", + "--include-tests", "dEQP-VK.api.*", + "--include-tests", "dEQP-VK.info.*", + "--include-tests", "dEQP-VK.draw.*", + "--include-tests", "dEQP-VK.query_pool.*", + "--include-tests", "dEQP-VK.memory.*", + "--testlog-to-xml", "C:\deqp\executor\testlog-to-xml.exe", + "--jobs", 4, + "--fraction", 3 +) $env:DZN_DEBUG = "warp" $env:MESA_VK_IGNORE_CONFORMANCE_WARNING = "true" -deqp-runner run --deqp $($deqp_module) --output $($results) --caselist $($caselist) --baseline $($baseline) $($includes) --testlog-to-xml C:\deqp\executor\testlog-to-xml.exe --jobs 4 -- $($deqp_options) +deqp-runner run $($deqp_runner_options) -- $($deqp_options) $deqpstatus = $? $template = "See https://$($env:CI_PROJECT_ROOT_NAMESPACE).pages.freedesktop.org/-/$($env:CI_PROJECT_NAME)/-/jobs/$($env:CI_JOB_ID)/artifacts/results/{{testcase}}.xml" -deqp-runner junit --testsuite dEQP --results "$($results)/failures.csv" --output "$($results)/junit.xml" --fraction 3 --limit 50 --template $template +deqp-runner junit --testsuite dEQP --results "$($results)/failures.csv" --output "$($results)/junit.xml" --limit 50 --template $template if (!$deqpstatus) { Exit 1 diff --git a/src/microsoft/ci/warp-fails.txt b/src/microsoft/ci/dozen-warp-fails.txt similarity index 99% rename from src/microsoft/ci/warp-fails.txt rename to src/microsoft/ci/dozen-warp-fails.txt index 6922d1f7fc81..2f8d961e35a1 100644 --- a/src/microsoft/ci/warp-fails.txt +++ b/src/microsoft/ci/dozen-warp-fails.txt @@ -883,4 +883,3 @@ dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-highp-output-uint,Fail dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-mediump,Fail dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-mediump-output-uint,Fail dEQP-VK.draw.renderpass.output_location.shuffle.inputs-outputs-mod,Fail -dEQP-VK.query_pool.concurrent_queries.primary_command_buffer,Flake diff --git a/src/microsoft/ci/dozen-warp-flakes.txt b/src/microsoft/ci/dozen-warp-flakes.txt new file mode 100644 index 000000000000..ec20364ff1bd --- /dev/null +++ b/src/microsoft/ci/dozen-warp-flakes.txt @@ -0,0 +1 @@ +dEQP-VK.query_pool.concurrent_queries.primary_command_buffer -- GitLab From f6c1159919b05857af523b5a95555e3cb1975a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= Date: Wed, 20 Apr 2022 23:05:09 -0400 Subject: [PATCH 2/5] dzn: Add some test cases to CI fails --- src/microsoft/ci/dozen-warp-fails.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/microsoft/ci/dozen-warp-fails.txt b/src/microsoft/ci/dozen-warp-fails.txt index 2f8d961e35a1..3e79ebacf62d 100644 --- a/src/microsoft/ci/dozen-warp-fails.txt +++ b/src/microsoft/ci/dozen-warp-fails.txt @@ -883,3 +883,7 @@ dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-highp-output-uint,Fail dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-mediump,Fail dEQP-VK.draw.renderpass.output_location.array.r8g8-uint-mediump-output-uint,Fail dEQP-VK.draw.renderpass.output_location.shuffle.inputs-outputs-mod,Fail +dEQP-VK.draw.renderpass.multiple_interpolation.separate.no_sample_decoration.8_samples,Fail +dEQP-VK.draw.renderpass.multiple_interpolation.structured.no_sample_decoration.16_samples,Fail +dEQP-VK.draw.renderpass.multiple_interpolation.structured.no_sample_decoration.2_samples,Fail +dEQP-VK.draw.renderpass.multiple_interpolation.structured.with_sample_decoration.8_samples,Fail -- GitLab From 40e825595f2fabcd5c359b341f18cde4a7bbd0ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= Date: Wed, 20 Apr 2022 21:39:10 -0400 Subject: [PATCH 3/5] dzn: Use $DEQP_FRACTION variable for deqp-runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Louis-Francis Ratté-Boulianne --- .gitlab-ci/windows/deqp_runner_run.ps1 | 16 +++++++++++++++- src/microsoft/ci/gitlab-ci.yml | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/windows/deqp_runner_run.ps1 b/.gitlab-ci/windows/deqp_runner_run.ps1 index 6250550e0886..ab62806d1b64 100644 --- a/.gitlab-ci/windows/deqp_runner_run.ps1 +++ b/.gitlab-ci/windows/deqp_runner_run.ps1 @@ -14,6 +14,19 @@ New-ItemProperty -Path $hkey_path -Name $hkey_name -Value 0 -PropertyType DWORD $results = New-Item -ItemType Directory results +# Set default values if needed +if ($env:DEQP_FRACTION -eq $null) { + $env:DEQP_FRACTION = 1 +} +if ($env:CI_NODE_INDEX -eq $null) { + $env:CI_NODE_INDEX = 1 +} + +# Adjust fraction if CI parallel mode is enabled +if ($env:CI_NODE_TOTAL -ne $null) { + $env:DEQP_FRACTION *= $env:CI_NODE_TOTAL +} + $deqp_options = @( "--deqp-surface-width", 256, "--deqp-surface-height", 256, @@ -34,7 +47,8 @@ $deqp_runner_options = @( "--include-tests", "dEQP-VK.memory.*", "--testlog-to-xml", "C:\deqp\executor\testlog-to-xml.exe", "--jobs", 4, - "--fraction", 3 + "--fraction-start", $env:CI_NODE_INDEX, + "--fraction", $env:DEQP_FRACTION ) $env:DZN_DEBUG = "warp" diff --git a/src/microsoft/ci/gitlab-ci.yml b/src/microsoft/ci/gitlab-ci.yml index e30f52b25652..097811321fbc 100644 --- a/src/microsoft/ci/gitlab-ci.yml +++ b/src/microsoft/ci/gitlab-ci.yml @@ -32,6 +32,7 @@ test-dozen-deqp: - windows_test_vs2019 variables: GIT_STRATEGY: none # testing doesn't build anything from source + DEQP_FRACTION: 3 script: - . _install/deqp_runner_run.ps1 artifacts: -- GitLab From 21f0792bd33b1bc69af9069e7006482b80b62576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= Date: Thu, 21 Apr 2022 01:29:52 -0400 Subject: [PATCH 4/5] dzn: Use $FDO_CI_CONCURRENT to determine number of deqp-runner jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Louis-Francis Ratté-Boulianne --- .gitlab-ci/windows/deqp_runner_run.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/windows/deqp_runner_run.ps1 b/.gitlab-ci/windows/deqp_runner_run.ps1 index ab62806d1b64..b5e27184e1ac 100644 --- a/.gitlab-ci/windows/deqp_runner_run.ps1 +++ b/.gitlab-ci/windows/deqp_runner_run.ps1 @@ -21,6 +21,9 @@ if ($env:DEQP_FRACTION -eq $null) { if ($env:CI_NODE_INDEX -eq $null) { $env:CI_NODE_INDEX = 1 } +if ($env:FDO_CI_CONCURRENT -eq $null) { + $env:FDO_CI_CONCURRENT = 4 +} # Adjust fraction if CI parallel mode is enabled if ($env:CI_NODE_TOTAL -ne $null) { @@ -46,7 +49,7 @@ $deqp_runner_options = @( "--include-tests", "dEQP-VK.query_pool.*", "--include-tests", "dEQP-VK.memory.*", "--testlog-to-xml", "C:\deqp\executor\testlog-to-xml.exe", - "--jobs", 4, + "--jobs", $env:FDO_CI_CONCURRENT, "--fraction-start", $env:CI_NODE_INDEX, "--fraction", $env:DEQP_FRACTION ) -- GitLab From cf5d65777da83ae353154516c47b68cc2cd87550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Louis-Francis=20Ratt=C3=A9-Boulianne?= Date: Thu, 21 Apr 2022 02:24:44 -0400 Subject: [PATCH 5/5] dzn: Add support for some $DEQP_* vars in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Louis-Francis Ratté-Boulianne --- .gitlab-ci/windows/deqp_runner_run.ps1 | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci/windows/deqp_runner_run.ps1 b/.gitlab-ci/windows/deqp_runner_run.ps1 index b5e27184e1ac..c7a3fbd0a0b7 100644 --- a/.gitlab-ci/windows/deqp_runner_run.ps1 +++ b/.gitlab-ci/windows/deqp_runner_run.ps1 @@ -15,6 +15,18 @@ New-ItemProperty -Path $hkey_path -Name $hkey_name -Value 0 -PropertyType DWORD $results = New-Item -ItemType Directory results # Set default values if needed +if ($env:DEQP_WIDTH -eq $null) { + $env:DEQP_WIDTH = 256 +} +if ($env:DEQP_HEIGHT -eq $null) { + $env:DEQP_HEIGHT = 256 +} +if ($env:DEQP_SURFACE_TYPE -eq $null) { + $env:DEQP_SURFACE_TYPE = "pbuffer" +} +if ($env:DEQP_CONFIG -eq $null) { + $env:DEQP_CONFIG = "rgba8888d24s8ms0" +} if ($env:DEQP_FRACTION -eq $null) { $env:DEQP_FRACTION = 1 } @@ -31,10 +43,10 @@ if ($env:CI_NODE_TOTAL -ne $null) { } $deqp_options = @( - "--deqp-surface-width", 256, - "--deqp-surface-height", 256, - "--deqp-surface-type", "pbuffer", - "--deqp-gl-config-name", "rgba8888d24s8ms0", + "--deqp-surface-width", $env:DEQP_WIDTH, + "--deqp-surface-height", $env:DEQP_HEIGHT, + "--deqp-surface-type", $env:DEQP_SURFACE_TYPE, + "--deqp-gl-config-name", $env:DEQP_CONFIG, "--deqp-visibility", "hidden" ) $deqp_runner_options = @( -- GitLab