Commit 98e3c7d8 authored by Nirbheek Chauhan's avatar Nirbheek Chauhan 🐜
Browse files

cerbero ci: Refactor cerbero_before_script

We were doing a few things suboptimally:

1. Making a symlink from the current directory to
   ../../gstreamer/cerbero so that build-tools kept working, which
   should not be necessary since cerbero relocates all binaries inside
   the cerbero homedir after extracting the cache.

2. Hard-coding the cerbero builddir when building on Windows because
   `pwd` was outputting an MSYS-style path which Python can't read

3. rsync-ing all the contents of $CERBERO_HOST_DIR, which is
   bad because we should be building on top of the artifact cache.
   The only thing we want from $CERBERO_HOST_DIR is cerbero-sources
   (the network cache), and we can setup the toolchain during
   bootstrap with the new --system and --toolchains arguments which
   were split out of --system-only.

Part-of: <!350>
parent ef981baf
......@@ -20,18 +20,18 @@ RUN git clone -b ${DEFAULT_BRANCH}
cd cerbero && \
echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \
echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \
./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no && \
./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --build-tools=no --toolchains=no && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --build-tools=no --toolchains=no && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --build-tools=no --toolchains=no && \
cd .. && \
rm /cerbero/cerbero-sources/android-ndk-*.zip && \
rm /cerbero/cerbero-sources/mingw-*.tar.xz && \
......@@ -18,8 +18,8 @@ echo 'vs_install_version = "vs15"' >> localconf.cbc
# Fetch all package requirements for a visualstudio gstreamer build
./cerbero-uninstalled -t -v visualstudio -c localconf.cbc -c config/win64.cbc fetch-package gstreamer-1.0
# Extract the toolchain and fixup the MSYS installation
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --system-only --offline
# Fixup the MSYS installation
./cerbero-uninstalled -t -c localconf.cbc -c config/win64.cbc bootstrap -y --build-tools=no --toolchains=no --offline
# Delete mingw toolchain binary tarball
rm /c/cerbero/cerbero-sources/mingw-*.tar.xz
......@@ -8,16 +8,60 @@ show_ccache_sum() {
# Produces runtime and devel tarball packages for linux/android or .pkg for macos
cerbero_package_and_check() {
# FIXME: mingw translates $(pwd) in a broken way
if [[ $CONFIG == win??.cbc ]]; then
# XXX: This is copied and modified from the cerbero-uninstalled script
# Use `mount` to get a list of MSYS mount points that the MSYS shell uses.
# That's our reference point for translating from MSYS paths to Win32 paths.
# We assume that the MSYS mount point directories are only in the filesystem
# root. This will break if people add their own custom mount points beyond what
# MSYS automatically creates, which is highly unlikely.
# /d -> d:/
# /c -> c:/
# /d/projects/cerbero -> d:/projects/cerbero/
# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/
# /mingw -> C:\\MinGW/
# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/
# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/
msys_dir_to_win32() {
set -e
local msys_path stripped_path mount_point path mounted_path
# If the path is already a native path, just return that
if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then
echo $1
# Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary
# anchor to split the path into components
# Strip leading slash
# Get the first path component, which may be a mount point
# Get the path inside the mountp oint
mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")"
# If it's not a mounted path (like /c or /tmp or /mingw), then it's in the
# general MSYS root mount
if [[ -z $mounted_path ]]; then
mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")"
echo ${mounted_path}${path%/}
# Print the working directory in the native OS path format, but with forward
# slashes
pwd_native() {
if [[ -n "$MSYSTEM" ]]; then
msys_dir_to_win32 "$(pwd)"
$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$PACKAGE_PATH" gstreamer-1.0
# Produces runtime and devel tarball packages for linux/android or .pkg for macos
cerbero_package_and_check() {
$CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0
# Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
if [[ $CONFIG != *universal* ]]; then
......@@ -29,52 +73,30 @@ cerbero_package_and_check() {
cerbero_before_script() {
# FIXME Wrong namespace
# Workaround build-tools having hardcoded internal path
mkdir -p "../../gstreamer"
time ln -sf "$(pwd)" "../../gstreamer/cerbero"
# Don't try to symlink twice because on MSYS `ln` does a `cp` since it
# doesn't support the new NTFS symlink feature.
if [[ ${CI_PROJECT_NAMESPACE} != gstreamer ]]; then
mkdir -p "../../${CI_PROJECT_NAMESPACE}"
time ln -sf "$(pwd)" "../../${CI_PROJECT_NAMESPACE}/cerbero"
ls -lh
# Make sure there isn't a pre-existing config hanging around
rm -v -f localconf.cbc
rm -v -f ${CERBERO_HOST_DIR}/localconf.cbc
# Copy cerbero git repo stored on the image
cp -a "${CERBERO_HOST_DIR}/.git" .
git checkout .
git status
if [[ $CONFIG == win??.cbc ]]; then
# For windows hardcode the path so it doesn't get
# mangled by msys path handling
# FIXME: make the sources point to pwd/$CI_PROJECT_DIR like the rest
echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc
echo 'home_dir="C:/cerbero/cerbero-build"' >> localconf.cbc
# If there's no cerbero-sources directory in the runner cache, copy it from
# the image cache
if ! [[ -d ${CERBERO_SOURCES} ]]; then
du -sch "${CERBERO_SOURCES}"
echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc
echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc
if [[ $CONFIG == win??.cbc ]]; then
# Visual Studio 2017 build tools install path
echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
echo 'vs_install_version = "vs15"' >> localconf.cbc
echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" > localconf.cbc
echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
cat localconf.cbc
time rsync -aH "${CERBERO_HOST_DIR}" .
cat localconf.cbc
# FIXME: if you comment out this line it fails like so, no clue why. Its not windows defender either.
# From
# b02080cb..d6923e42 master -> origin/master
# Fetching origin
# error: unable to create file cerbero-uninstalled: Permission denied
# fatal: Could not reset index file to revision 'd6923e4216c8a17759527a3db070d15cf7ff10a0'.
# ERROR: Failed to proceed with self update Command Error: Running ['git', 'reset', '--hard', 'd6923e4216c8a17759527a3db070d15cf7ff10a0'] returned 128
git status
time ./cerbero-uninstalled --self-update manifest.xml
......@@ -82,8 +104,10 @@ cerbero_script() {
$CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
$CERBERO $CERBERO_ARGS fetch-bootstrap
$CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
du -sch "${CERBERO_SOURCES}"
if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then
......@@ -92,7 +116,7 @@ cerbero_script() {
$CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
......@@ -100,9 +124,9 @@ cerbero_deps_script() {
$CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
$CERBERO $CERBERO_ARGS fetch-bootstrap
$CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
$CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
$CERBERO $CERBERO_ARGS bootstrap --offline --system=no
$CERBERO $CERBERO_ARGS build-deps --offline \
gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \
gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \
......@@ -683,7 +683,7 @@ valgrind ges:
# used by macos packages as we only ever install to a fixed directory
# location where cerbero is cached on the host
# location where the cerbero git repo is stored on the image
CERBERO_HOST_DIR: "/cerbero/"
- echo "Fetching ${CERBERO_SCRIPT_URL}"
......@@ -1196,10 +1196,7 @@ build gst-omx tizonia fedora x86_64:
CONFIG: 'win64.cbc'
ARCH: 'msvc_x86_64'
# rsync doesn't like windows paths like `C:/cerbero`
CERBERO_HOST_DIR: "/c/cerbero/"
CERBERO_HOME: "C:/cerbero-build"
CERBERO_SOURCES: "C:/cerbero-sources"
CERBERO_HOST_DIR: "C:/cerbero"
CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS} -v visualstudio -v nowerror"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment