ci/bin: Add utilities for CI jobs debugging

Job DAG utility

Use GraphQL API from Gitlab to find jobs dependencies in a pipeline.

Find all dependencies for jobs starting with "iris-"

.gitlab-ci/bin/gitlab_gql.py --sha $(git rev-parse HEAD) --print-dag --regex "iris-.*"

ci_run_n_monitor with automatic dependency lookup

python .gitlab-ci/bin/ci_run_n_monitor.py --rev $(git rev-parse HEAD) --target '.*skqp' --token $(< ~/.gitlab-token)

Merged YAML GraphQL utility

Normally, Mesa devs bump into CI errors and need to replicate the CI job environment locally, but that is not an easy task, since one needs to find the merged YAML CI file from Gitlab interface.

As it can happen often, here is a script that use Gitlab's GraphQL and finds all the variables set by it, alongside with the container image used to run it and the script that it will run.

Example: Expanding kernel+rootfs_amd64 job

python  .gitlab-ci/bin/gitlab_gql.py  -pp "gallo/mesa" --sha $(git rev-parse HEAD) --gitlab-token-file ~/.gitlab-token --print-job-manifest zink

Output

# --------- variables ---------------
export CI_COMMIT_SHA='b059a41b691d89df39a2b73e2cc2069b80450790'
export CI_JOB_JWT_FILE='/minio_jwt'
export CI_PRE_CLONE_SCRIPT='set -o xtrace\nwget -q -O download-git-cache.sh ${CI_PROJECT_URL}/-/raw/b059a41b691d89df39a2b73e2cc2069b80450790/.gitlab-ci/download-git-cache.sh\nbash download-git-cache.sh\nrm download-git-cache.sh\nset +o xtrace'
export CI_PROJECT_NAME='mesa'
export CI_PROJECT_PATH='gallo/mesa'
export CI_REGISTRY_IMAGE='registry.freedesktop.org/gallo/mesa'
export COLLABORA_FARM='online'
export DEBIAN_BASE_TAG='2022-07-01-bb-llvm13'
export DEBIAN_BUILD_MINGW_TAG='2022-07-14-directx-headers'
export DEBIAN_BUILD_TAG='2022-07-14-directx-headers'
export DEBIAN_X86_BUILD_BASE_IMAGE='debian/x86_build-base'
export DEBIAN_X86_BUILD_IMAGE_PATH='debian/x86_build'
export DEBIAN_X86_BUILD_MINGW_IMAGE_PATH='debian/x86_build-mingw'
export DEBIAN_X86_TEST_BASE_IMAGE='debian/x86_test-base'
export DEBIAN_X86_TEST_GL_TAG='2022-07-06-virgl-update'
export DEBIAN_X86_TEST_IMAGE_PATH='debian/x86_test-gl'
export DEBIAN_X86_TEST_VK_TAG='2022-07-18-apitrace-11-1'
export DEQP_SUITE='zink-lvp'
export FDO_BASE_IMAGE='registry.freedesktop.org/gallo/mesa/${MESA_BASE_IMAGE}:2022-07-01-bb-llvm13--290b79e0e78eab67a83766f4e9691be554fc4afd'
export FDO_DISTRIBUTION_TAG='2022-07-06-virgl-update--2022-07-01-bb-llvm13--290b79e0e78eab67a83766f4e9691be554fc4afd'
export FDO_UPSTREAM_REPO='mesa/mesa'
export FD_FARM='online'
export FEDORA_X86_BUILD_TAG='2022-04-24-spirv-tools-5'
export FLAKES_CHANNEL='#zink-ci'
export GALLIVM_PERF='no_quad_lod'
export GIT_STRATEGY='none'
export GPU_VERSION='zink-lvp'
export IGALIA_FARM='online'
export JOB_ARTIFACTS_BASE='minio-packet.freedesktop.org/artifacts/gallo/mesa/${CI_PIPELINE_ID}/${CI_JOB_ID}'
export KERNEL_ROOTFS_TAG='2022-07-19_skqp'
export LIBGL_ALWAYS_SOFTWARE='true'
export LIMA_FARM='online'
export LVP_POISON_MEMORY='1'
export MESA_BASE_TAG='2022-07-01-bb-llvm13'
export MESA_IMAGE='registry.freedesktop.org/gallo/mesa/debian/x86_test-gl:2022-07-06-virgl-update--2022-07-01-bb-llvm13--290b79e0e78eab67a83766f4e9691be554fc4afd'
export MESA_IMAGE_PATH='debian/x86_test-gl'
export MESA_IMAGE_TAG='2022-07-06-virgl-update'
export MESA_LOADER_DRIVER_OVERRIDE='zink'
export MESA_TEMPLATES_COMMIT='290b79e0e78eab67a83766f4e9691be554fc4afd'
export MICROSOFT_FARM='online'
export MINIO_HOST='minio-packet.freedesktop.org'
export PIGLIT_NO_WINDOW=1
export PIGLIT_REPLAY_REFERENCE_IMAGES_BASE='minio-packet.freedesktop.org/mesa-tracie-results/mesa/mesa'
export PIPELINE_ARTIFACTS_BASE='minio-packet.freedesktop.org/artifacts/gallo/mesa/${CI_PIPELINE_ID}'
export WINDOWS_X64_BUILD_PATH='windows/x64_build'
export WINDOWS_X64_BUILD_TAG='2022-06-15-vs-winsdk'
export WINDOWS_X64_TEST_PATH='windows/x64_test'
export WINDOWS_X64_TEST_TAG='2022-06-15-vs-winsdk'
export WINDOWS_X64_VS_PATH='windows/x64_vs'
export WINDOWS_X64_VS_TAG='2022-06-15-vs-winsdk'


# --------- full script ---------------
# before_script
echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"
echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"
unset CI_JOB_JWT
echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"
rm -rf install
tar -xf artifacts/install.tar
echo -e "\e[0Ksection_start:$(date +%s):ldd_section[collapsed=true]\r\e[0KChecking ldd on driver build"
LD_LIBRARY_PATH=install/lib find install/lib -name "*.so" -print -exec ldd {} \;
echo -e "\e[0Ksection_end:$(date +%s):ldd_section\r\e[0K"

# script
xvfb-run --server-args='-noreset' sh -c "GALLIUM_DRIVER=zink VK_DRIVER=lvp install/deqp-runner.sh"

# after_script



# --------- container image ---------------
registry.freedesktop.org/gallo/mesa/debian/x86_test-gl:2022-07-06-virgl-update--2022-07-01-bb-llvm13--290b79e0e78eab67a83766f4e9691be554fc4afd