Commit c1e7e83d authored by Emma Anholt's avatar Emma Anholt Committed by Marge Bot
Browse files

ci: Consistently use -j4 across x86 build jobs and -j8 on ARM.



Our shared runners are set up for concurrent jobs ~= CPUs / 4 (x86) or 8
(ARM).  If you use more build processes than that, then jobs may be
fighting each other for shared system resources, possibly to the point of
failure (we've seen one of the runners OOM on some jobs before, though I'm
not sure if this was the cause).

To try to systematically prevent the problem, we make a ninja wrapper in
the containers that passes the -j flags, and set MAKEFLAGS in the
container builds.  This doesn't cover make in non-container builds, but I
believe we don't have any of those.
Reviewed-by: Michel Dänzer's avatarMichel Dänzer <mdaenzer@redhat.com>
Tested-by: Marge Bot <mesa/mesa!3782>
Part-of: <mesa/mesa!3782>
parent 2f424c83
......@@ -121,7 +121,7 @@ x86_build:
- .fdo.container-build@debian
- .container
variables:
FDO_DISTRIBUTION_TAG: &x86_build "2020-03-13"
FDO_DISTRIBUTION_TAG: &x86_build "2020-03-18-jflags"
.use-x86_build:
variables:
......@@ -134,19 +134,19 @@ x86_build:
x86_test-gl:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-13"
FDO_DISTRIBUTION_TAG: &x86_test-gl "2020-03-18-jflags"
# Debian 10 based x86 test image for VK
x86_test-vk:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-03-27"
FDO_DISTRIBUTION_TAG: &x86_test-vk "2020-04-01-jflags"
# Debian 9 based x86 build image (old LLVM)
x86_build_old:
extends: x86_build
variables:
FDO_DISTRIBUTION_TAG: &x86_build_old "2019-09-18"
FDO_DISTRIBUTION_TAG: &x86_build_old "2019-03-18-jflags"
FDO_DISTRIBUTION_VERSION: stretch-slim
.use-x86_build_old:
......@@ -162,7 +162,7 @@ arm_build:
- .fdo.container-build@debian@arm64v8
- .container
variables:
FDO_DISTRIBUTION_TAG: &arm_build "2020-03-24"
FDO_DISTRIBUTION_TAG: &arm_build "2020-04-01-jflags"
.use-arm_build:
variables:
......@@ -175,7 +175,7 @@ arm_build:
arm_test:
extends: arm_build
variables:
FDO_DISTRIBUTION_TAG: &arm_test "2020-01-30"
FDO_DISTRIBUTION_TAG: &arm_test "2020-03-18-jflags"
.use-arm_test:
variables:
......
......@@ -20,7 +20,7 @@ git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout
pushd /apitrace
git checkout "$APITRACE_VERSION"
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False -DENABLE_WAFFLE=on -DWaffle_DIR=/usr/local/lib/cmake/Waffle/
ninja -C _build -j4
ninja -C _build
mkdir build
cp _build/apitrace build
cp _build/glretrace build
......
......@@ -5,6 +5,6 @@ set -ex
git clone https://gitlab.freedesktop.org/mesa/parallel-deqp-runner.git --depth 1 -b mesa-ci-2019-12-17 /parallel-deqp-runner
pushd /parallel-deqp-runner
meson build/ $EXTRA_MESON_ARGS
ninja -C build -j4 install
ninja -C build install
popd
rm -rf /parallel-deqp-runner
......@@ -23,7 +23,7 @@ cmake -G Ninja \
-DDEQP_TARGET=x11_glx \
-DCMAKE_BUILD_TYPE=Release \
/VK-GL-CTS
ninja -j4
ninja
# Copy out the mustpass list we want.
mkdir /deqp/mustpass
......
......@@ -9,6 +9,6 @@ git submodule update --init
mkdir build
cd build
cmake .. -DCMAKE_BUILD_TYPE=Release -G Ninja
ninja -C . -j4 install
ninja -C . install
cd ../..
rm -rf Fossilize
......@@ -11,7 +11,7 @@ git checkout "$GFXRECONSTRUCT_VERSION"
git submodule update --init
git submodule update
cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release
ninja -C _build -j4 gfxrecon-replay
ninja -C _build gfxrecon-replay
mkdir -p build/bin
install _build/tools/replay/gfxrecon-replay build/bin
strip build/bin/*
......
......@@ -7,7 +7,7 @@ pushd /piglit
git checkout 8771c3860505db2bcf4877216221d774bf90af6b
patch -p1 <$OLDPWD/.gitlab-ci/piglit/disable-vs_in.diff
cmake -G Ninja -DCMAKE_BUILD_TYPE=Release
ninja -j4
ninja
find -name .git -o -name '*ninja*' -o -iname '*cmake*' -o -name '*.[chao]' | xargs rm -rf
rm -rf target_api
popd
......@@ -8,7 +8,7 @@ git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout
pushd /renderdoc
git checkout "$RENDERDOC_VERSION"
cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
ninja -C _build -j4
ninja -C _build
mkdir -p build/lib
cp _build/lib/renderdoc.so build/lib
cp _build/lib/librenderdoc.so build/lib
......
......@@ -19,7 +19,7 @@ cmake -G Ninja -B_build -H. \
-DBUILD_VIA=OFF \
-DBUILD_VKTRACE_REPLAY=OFF \
-C_build/helper.cmake
ninja -C _build -j4 VkLayer_screenshot VkLayer_screenshot-staging-json
ninja -C _build VkLayer_screenshot VkLayer_screenshot-staging-json
mkdir -p build/etc/vulkan/explicit_layer.d
mkdir build/lib
install _build/layersvt/staging-json/VkLayer_screenshot.json build/etc/vulkan/explicit_layer.d
......
......@@ -59,7 +59,7 @@ export LIBDRM_VERSION=libdrm-2.4.100
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -j4 -C build install; cd ..
cd $LIBDRM_VERSION; meson build -D vc4=true -D freedreno=true -D etnaviv=true; ninja -C build install; cd ..
rm -rf $LIBDRM_VERSION
############### Generate cross build file for Meson
......
#!/bin/sh
# Common setup among container builds before we get to building code.
export CCACHE_COMPILERCHECK=content
export CCACHE_COMPRESS=true
export CCACHE_DIR=/cache/mesa/ccache
......@@ -11,3 +13,17 @@ export CC="/usr/lib/ccache/gcc"
export CXX="/usr/lib/ccache/g++"
ccache --show-stats
if uname -m | grep -q arm || uname -m | grep -q aarch64; then
export JFLAGS=-j8
else
export JFLAGS=-j4
fi
# Make a wrapper script for ninja to always include the -j flags
echo /usr/bin/ninja $JFLAGS '"$@"' > /usr/local/bin/ninja
chmod +x /usr/local/bin/ninja
# Set MAKEFLAGS so that all make invocations in container builds include the
# flags (doesn't apply to non-container builds, but we don't run make there)
export MAKEFLAGS=$JFLAGS
......@@ -45,7 +45,7 @@ mkdir -p kernel
wget -qO- ${KERNEL_URL} | tar -xz --strip-components=1 -C kernel
pushd kernel
./scripts/kconfig/merge_config.sh ${DEFCONFIG} ../.gitlab-ci/${KERNEL_ARCH}.config
make -j12 ${KERNEL_IMAGE_NAME} dtbs
make ${KERNEL_IMAGE_NAME} dtbs
cp arch/${KERNEL_ARCH}/boot/${KERNEL_IMAGE_NAME} /lava-files/.
cp ${DEVICE_TREES} /lava-files/.
popd
......
......@@ -170,36 +170,36 @@ export WAYLAND_PROTOCOLS_VERSION=wayland-protocols-1.12
wget $XORG_RELEASES/util/$XORGMACROS_VERSION.tar.bz2
tar -xvf $XORGMACROS_VERSION.tar.bz2 && rm $XORGMACROS_VERSION.tar.bz2
cd $XORGMACROS_VERSION; ./configure; make -j4 install; cd ..
cd $XORGMACROS_VERSION; ./configure; make install; cd ..
rm -rf $XORGMACROS_VERSION
wget $XCB_RELEASES/$XCBPROTO_VERSION.tar.bz2
tar -xvf $XCBPROTO_VERSION.tar.bz2 && rm $XCBPROTO_VERSION.tar.bz2
cd $XCBPROTO_VERSION; ./configure; make -j4 install; cd ..
cd $XCBPROTO_VERSION; ./configure; make install; cd ..
rm -rf $XCBPROTO_VERSION
wget $XCB_RELEASES/$LIBXCB_VERSION.tar.bz2
tar -xvf $LIBXCB_VERSION.tar.bz2 && rm $LIBXCB_VERSION.tar.bz2
cd $LIBXCB_VERSION; ./configure; make -j4 install; cd ..
cd $LIBXCB_VERSION; ./configure; make install; cd ..
rm -rf $LIBXCB_VERSION
wget https://dri.freedesktop.org/libdrm/$LIBDRM_VERSION.tar.bz2
tar -xvf $LIBDRM_VERSION.tar.bz2 && rm $LIBDRM_VERSION.tar.bz2
cd $LIBDRM_VERSION
meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -j4 -C build install
rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -j4 -C build install
rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -j4 -C build install
meson build -D vc4=true -D freedreno=true -D etnaviv=true -D libdir=lib/x86_64-linux-gnu; ninja -C build install
rm -rf build; meson --cross-file=/cross_file-ppc64el.txt build -D libdir=lib/powerpc64le-linux-gnu; ninja -C build install
rm -rf build; meson --cross-file=/cross_file-i386.txt build -D libdir=lib/i386-linux-gnu; ninja -C build install
cd ..
rm -rf $LIBDRM_VERSION
wget $WAYLAND_RELEASES/$LIBWAYLAND_VERSION.tar.xz
tar -xvf $LIBWAYLAND_VERSION.tar.xz && rm $LIBWAYLAND_VERSION.tar.xz
cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make -j4 install; cd ..
cd $LIBWAYLAND_VERSION; ./configure --enable-libraries --without-host-scanner --disable-documentation --disable-dtd-validation; make install; cd ..
rm -rf $LIBWAYLAND_VERSION
wget $WAYLAND_RELEASES/$WAYLAND_PROTOCOLS_VERSION.tar.xz
tar -xvf $WAYLAND_PROTOCOLS_VERSION.tar.xz && rm $WAYLAND_PROTOCOLS_VERSION.tar.xz
cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make -j4 install; cd ..
cd $WAYLAND_PROTOCOLS_VERSION; ./configure; make install; cd ..
rm -rf $WAYLAND_PROTOCOLS_VERSION
......@@ -209,7 +209,7 @@ rm -rf $WAYLAND_PROTOCOLS_VERSION
GLVND_VERSION=1.2.0
wget https://gitlab.freedesktop.org/glvnd/libglvnd/-/archive/v$GLVND_VERSION/libglvnd-v$GLVND_VERSION.tar.gz
tar -xvf libglvnd-v$GLVND_VERSION.tar.gz && rm libglvnd-v$GLVND_VERSION.tar.gz
pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make -j4 install; popd
pushd libglvnd-v$GLVND_VERSION; ./autogen.sh; ./configure; make install; popd
rm -rf libglvnd-v$GLVND_VERSION
......@@ -217,7 +217,7 @@ pushd /usr/local
git clone https://gitlab.freedesktop.org/mesa/shader-db.git --depth 1
rm -rf shader-db/.git
cd shader-db
make -j4
make
popd
......
......@@ -57,7 +57,7 @@ meson _build --native-file=native.file \
${EXTRA_OPTION}
cd _build
meson configure
ninja -j4
ninja
LC_ALL=C.UTF-8 ninja test
ninja install
cd ..
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment