Commit 71a467f5 authored by Thibault Saunier's avatar Thibault Saunier 🌵

Merging gst-ci

parents 420940f1 aa18f639
stages:
- "build docker"
- "preparation"
- "pre-build"
- "build"
- "test"
# use the binaries in some way
- "integrate"
test manifest:
variables:
GIT_STRATEGY: fetch
image: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/test-manifest:2019-10-23-793476'
stage: "preparation"
script:
- pytest-3 --junitxml=junit.xml --cov=build_manifest gitlab/build_manifest.py
coverage: '/TOTAL.*\s+(\d+%)$/'
artifacts:
reports:
junit:
- "junit.xml"
.base:
image: "docker:stable"
services:
- docker:dind
when: 'manual'
variables:
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
DOCKER_DRIVER: "overlay2"
GIT_STRATEGY: fetch
script:
- export DATE=$(date +"%Y-%m-%d")
- export IMAGE="${CI_REGISTRY_IMAGE}/${ARCH}/${TAG}"
- export _UID="${IMAGE}:${DATE}-${CI_JOB_ID}"
- export LATEST="${IMAGE}:latest"
- docker build --pull -f ${DOCKERFILE} -t ${LATEST} ${CONTEXT_DIR}
# If we are in a fork, push the image to the reigstry regardless the branch
- |
if [ "$CI_PROJECT_NAMESPACE" != "gstreamer" ]; then
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
docker push ${LATEST}
fi
# Push the images to the upstream registry
- |
if [ "$CI_COMMIT_REF_NAME" = "master" -a "$CI_PROJECT_NAMESPACE" = "gstreamer" ]; then
# Tag the image with the git ref
docker image tag ${LATEST} ${_UID}
# Push the tags
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
docker push ${_UID}
docker push ${LATEST}
fi
alpine amd64 manifest builder docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "build-manifest"
CONTEXT_DIR: "docker/build_manifest/"
DOCKERFILE: "docker/build_manifest/Dockerfile"
extends: .base
test manifest amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "test-manifest"
CONTEXT_DIR: "docker/test_manifest/"
DOCKERFILE: "docker/test_manifest/Dockerfile"
extends: .base
gst-indent amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "gst-indent"
CONTEXT_DIR: "docker/indent/"
DOCKERFILE: "docker/indent/Dockerfile"
extends: .base
fedora amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "fedora"
CONTEXT_DIR: "docker/fedora/"
DOCKERFILE: "docker/fedora/Dockerfile"
extends: .base
cerbero fedora amd64 docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "cerbero-fedora"
CONTEXT_DIR: "docker/cerbero/"
DOCKERFILE: "docker/cerbero/Dockerfile-fedora"
extends: .base
android docker:
stage: "build docker"
variables:
ARCH: "amd64"
TAG: "android"
CONTEXT_DIR: "docker/android/"
DOCKERFILE: "docker/android/Dockerfile"
extends: .base
include: "gitlab/ci_template.yml"
.build local:
extends: '.build'
when: 'manual'
needs:
- 'manifest'
# Test the build job against the latest build image tag and the local manifest
build fedora x86_64 local:
extends: '.build local'
stage: 'build'
image: "${CI_REGISTRY_IMAGE}/amd64/fedora:latest"
artifacts:
expire_in: '5 days'
when: always
paths:
- "manifest.xml"
- "gst-build/"
except:
refs:
- "master@gstreamer/gst-ci"
build cerbero fedora x86_64 local:
extends: '.cerbero fedora x86_64'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
build cerbero cross-android universal local:
extends: '.cerbero cross-android universal'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
build cerbero cross win32 local:
extends: '.cerbero cross win32'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
build cerbero cross win64 local:
extends: '.cerbero cross win64'
image: "${CI_REGISTRY_IMAGE}/amd64/cerbero-fedora:latest"
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
.test fedora x86_64 local:
image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest'
extends: '.test'
dependencies:
- 'build fedora x86_64 local'
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
check fedora local:
extends: '.test fedora x86_64 local'
variables:
TEST_SUITE: 'check.gst*'
cross-android universal examples local:
extends: '.cross-android universal examples'
image: "${CI_REGISTRY_IMAGE}/amd64/android:latest"
dependencies:
- "build cerbero cross-android universal local"
when: 'manual'
except:
refs:
- "master@gstreamer/gst-ci"
integration testsuites fedora local:
extends: '.test fedora x86_64 local'
before_script:
- rm -f gst-build/build/subprojects/gstreamer-vaapi/gst/vaapi/libgstvaapi.so
variables:
EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures"
TEST_SUITE: "validate ges"
GStreamer Continuous Integration
================================
This repository contains all material relevant to the GStreamer
Continuous Integration system.
* Jenkins scripts
* Docker images
* Build scripts and code
GStreamer Docker images
FROM registry.fedoraproject.org/fedora:29
ENV ANDROID_HOME=/android/sdk
ENV ANDROID_NDK_HOME=/android/ndk
COPY prepare.sh cleanup.sh /root/
RUN /usr/bin/sh /root/prepare.sh && \
/usr/bin/sh /root/cleanup.sh
set -eux
echo "Removing DNF cache"
dnf clean all
rm -R /root/*
rm -rf /var/cache/dnf /var/log/dnf*
set -eux
dnf install -y \
file \
git-core \
java-1.8.0-openjdk-devel \
lbzip2 \
make \
pkg-config \
unzip \
which \
xz
mkdir -p /android/sources
curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r18b-linux-x86_64.zip
unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/
# remove the intermediate versioned directory
mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/
curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip
unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/
mkdir -p ${ANDROID_HOME}/licenses
# Accept licenses. Values taken from:
# ANDROID_HOME=/path/to/android/sdk-tools $ANDROID_HOME/tools/bin/sdkmanager --licenses
echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license
echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license
echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license
echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license
echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license
# pre-cache deps
export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android
curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.0/gstreamer-1.0-android-universal-1.16.0.tar.xz
mkdir $GSTREAMER_ROOT_ANDROID
tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID
ls $GSTREAMER_ROOT_ANDROID
git clone https://gitlab.freedesktop.org/gstreamer/gst-examples.git /android/sources/gst-examples
chmod +x /android/sources/gst-examples/playback/player/android/gradlew
/android/sources/gst-examples/playback/player/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-examples/playback/player/android dependencies --refresh-dependencies
git clone https://gitlab.freedesktop.org/gstreamer/gst-docs.git /android/sources/gst-docs
chmod +x /android/sources/gst-docs/examples/tutorials/android/gradlew
/android/sources/gst-docs/examples/tutorials/android/gradlew --no-search-upward --no-daemon --project-dir /android/sources/gst-docs/examples/tutorials/android dependencies --refresh-dependencies
unset GSTREAMER_ROOT_ANDROID
rm -rf /android/sources
FROM base/devel:latest
ENV LANG=en_US.UTF-8
ENV GST_BUILD_PATH="/gst-build/"
ENV GST_VALIDATE_OUTPUT="/validate-output"
ENV GST_VALIDATE_TESTSUITES="$GST_VALIDATE_OUTPUT/gst-integration-testsuites"
ENV GST_BUILD_MESON_ARGS="-Dpython=disabled -Dlibav=enabled -Dugly=enabled -Dbad=enabled -Ddevtools=enabled -Dges=enabled -Drtsp_server=enabled -Domx=disabled -Dvaapi=disabled -Dsharp=enabled"
ENV PATH="/usr/lib/ccache/bin/:${PATH}"
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
RUN pacman -Sy --noconfirm python3 libxml2 libxslt cmake libyaml git make nodejs ninja clang llvm flex python-gobject gstreamer gst-plugins-base gst-plugins-bad gst-plugins-ugly gst-libav gst-editing-services python-pip npm json-glib gobject-introspection wget mono unzip glib2-docs xorg-xset xorg-server-xvfb ccache
RUN pip install --upgrade git+https://github.com/hotdoc/hotdoc.git@gst git+https://github.com/hotdoc/hotdoc_c_extension.git@gst meson
RUN mkdir $GST_VALIDATE_OUTPUT && \
git clone https://anongit.freedesktop.org/git/gstreamer/gst-integration-testsuites.git $GST_VALIDATE_TESTSUITES && \
cd $GST_VALIDATE_TESTSUITES/medias/ && \
python get_files.py defaults/
# get gst-build and make all subprojects available
RUN git clone git://anongit.freedesktop.org/gstreamer/gst-build $GST_BUILD_PATH && \
cd $GST_BUILD_PATH && \
meson build/ $GST_BUILD_MESON_ARGS && rm -Rf build/
RUN rm -Rf /var/cache/pacman/
\ No newline at end of file
#!/bin/sh
echo Building gstreamer/build-base-fedora:latest
docker build -t gstreamer/build-base-fedora:latest . -f Dockerfile-fedora
echo Building gstreamer/build-base-ubuntu:latest
docker build -t gstreamer/build-base-ubuntu:latest . -f Dockerfile-ubuntu
FROM python:3.7.1-alpine
RUN pip install requests
RUN apk add git
RUN cd / && git clone https://gitlab.freedesktop.org/gstreamer/gst-ci.git
\ No newline at end of file
FROM registry.fedoraproject.org/fedora:30
# System setup
RUN echo "fastestmirror=true" >> /etc/dnf/dnf.conf && \
dnf upgrade -y && \
dnf install -y sudo git lbzip2 rsync nasm
# Configure git for various usage
RUN git config --global user.email "cerbero@gstreamer.freedesktop.org" && \
git config --global user.name "Cerbero Build System"
RUN git clone https://gitlab.freedesktop.org/gstreamer/cerbero.git && \
mkdir $HOME/.cerbero && \
echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc && \
echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc && \
cd cerbero && \
echo "local_sources=\"/cerbero/cerbero-sources\"" >> localconf.cbc && \
echo "home_dir=\"/cerbero/cerbero-build\"" >> localconf.cbc && \
./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc bootstrap -y --system-only && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package gstreamer-1.0 && \
./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc bootstrap -y --system-only && \
cd .. && \
rm /cerbero/cerbero-sources/android-ndk-*.zip && \
rm /cerbero/cerbero-sources/mingw-*.tar.xz && \
rm -rf /cerbero/cerbero-build/{dist,logs,sources} && \
rm -f /cerbero/cerbero-build/{linux,windows,android}*.cache && \
dnf clean all
FROM registry.fedoraproject.org/fedora:30
COPY prepare.sh cleanup.sh /root/
RUN /usr/bin/sh /root/prepare.sh && \
/usr/bin/sh /root/cleanup.sh
set -eux
echo "Removing DNF cache"
dnf clean all
rm -R /root/*
rm -rf /var/cache/dnf /var/log/dnf*
set -eux
dnf install -y git-core ninja-build dnf-plugins-core
# Configure git for various usage
git config --global user.email "gst-build@gstreamer.net"
git config --global user.name "Gstbuild Runner"
# Add rpm fusion repositories in order to access all of the gst plugins
sudo dnf install -y \
"https://download1.rpmfusion.org/free/fedora/rpmfusion-free-release-$(rpm -E %fedora).noarch.rpm" \
"https://download1.rpmfusion.org/nonfree/fedora/rpmfusion-nonfree-release-$(rpm -E %fedora).noarch.rpm"
dnf upgrade -y
# install rest of the extra deps
dnf install -y \
aom \
aom-extra-tools \
bat \
libaom \
libaom-devel \
ccache \
cmake \
clang-devel \
elfutils \
elfutils-libs \
elfutils-devel \
gcc \
gcc-c++ \
gdb \
git-lfs \
glslc \
gtk3 \
gtk3-devel \
graphene \
graphene-devel \
gsl \
gsl-devel \
ffmpeg \
ffmpeg-libs \
ffmpeg-devel \
flex \
flite \
flite-devel \
mono-devel \
procps-ng \
patch \
redhat-rpm-config \
json-glib \
json-glib-devel \
libnice \
libnice-devel \
libunwind \
libunwind-devel \
libyaml-devel \
libxml2-devel \
libxslt-devel \
llvm-devel \
make \
neon \
neon-devel \
nunit \
npm \
opencv \
opencv-devel \
openjpeg2 \
openjpeg2-devel \
SDL2 \
SDL2-devel \
sbc \
sbc-devel \
x264 \
x264-libs \
x264-devel \
python3 \
python3-devel \
python3-libs \
python3-gobject \
python3-cairo \
python3-cairo-devel \
valgrind \
vulkan \
vulkan-devel \
mesa-omx-drivers \
mesa-libGL \
mesa-libGL-devel \
mesa-libGLU \
mesa-libGLU-devel \
mesa-libGLES \
mesa-libGLES-devel \
mesa-libOpenCL \
mesa-libOpenCL-devel \
mesa-libgbm \
mesa-libgbm-devel \
mesa-libd3d \
mesa-libd3d-devel \
mesa-libOSMesa \
mesa-libOSMesa-devel \
mesa-vulkan-drivers \
wpewebkit \
wpewebkit-devel \
wpebackend-fdo \
wpebackend-fdo-devel \
xorg-x11-server-utils \
xorg-x11-server-Xvfb
# Install common debug symbols
dnf debuginfo-install -y gtk3 \
glib2 \
glibc \
freetype \
openjpeg \
gobject-introspection \
python3 \
python3-libs \
python3-gobject \
libjpeg-turbo \
glib-networking \
libcurl \
libsoup \
nss \
nss-softokn \
nss-softokn-freebl \
nss-sysinit \
nss-util \
openssl \
openssl-libs \
openssl-pkcs11 \
brotli \
bzip2-libs \
gpm-libs \
harfbuzz \
harfbuzz-icu \
json-c \
json-glib \
libbabeltrace \
libffi \
libsrtp \
libunwind \
mpg123-libs \
neon \
orc-compiler \
orc \
pixman \
pulseaudio-libs \
pulseaudio-libs-glib2 \
wavpack \
webrtc-audio-processing \
ffmpeg \
ffmpeg-libs \
faad2-libs \
libavdevice \
libmpeg2 \
faac \
fdk-aac \
x264 \
x264-libs \
x265 \
x265-libs \
xz \
xz-libs \
zip \
zlib
pip3 install meson==0.50.1 hotdoc
# Install the dependencies of gstreamer
dnf builddep -y gstreamer1 \
gstreamer1-plugins-base \
gstreamer1-plugins-good \
gstreamer1-plugins-good-extras \
gstreamer1-plugins-ugly \
gstreamer1-plugins-ugly-free \
gstreamer1-plugins-bad-nonfree \
gstreamer1-plugins-bad-free \
gstreamer1-plugins-bad-free-extras \
gstreamer1-plugins-bad-freeworld \
gstreamer1-libav \
gstreamer1-rtsp-server \
gstreamer1-vaapi \
python3-gstreamer1 \
-x meson
# Remove gst-devel packages installed by builddep above
dnf remove -y "gstreamer1*devel"
# Remove Qt5 devel packages as we haven't tested building it and
# it leads to build issues in examples.
dnf remove -y "qt5-qtbase-devel"
# FIXME: Why does installing directly with dnf doesn't actually install
# the documentation files?
dnf download glib2-doc gdk-pixbuf2-devel*x86_64* gtk3-devel-docs
rpm -i --reinstall *.rpm
rm -f *.rpm
# get gst-build and make all subprojects available
git clone git://anongit.freedesktop.org/gstreamer/gst-build /gst-build/
cd /gst-build
meson subprojects download
FROM debian:stretch-slim
RUN apt update -yqq \
&& apt install -y curl indent git findutils \
&& rm -R /var/lib/apt/ /var/log/apt/
\ No newline at end of file
FROM anapsix/alpine-java:latest
# Get the allure test report framework
RUN export ALLURE_VERSION=2.7.0 && \
wget "https://bintray.com/qameta/generic/download_file?file_path=io%2Fqameta%2Fallure%2Fallure%2F2.7.0%2Fallure-${ALLURE_VERSION}.zip" -O allure.zip
unzip allure.zip && \
mv allure-${ALLURE_VERSION} /opt/allure && \
chmod +x /opt/allure/bin/allure && \
rm allure.zip
\ No newline at end of file
#!/bin/sh
echo Building gstreamer/base-fedora:latest
docker build -t gstreamer/base-fedora:latest . -f Dockerfile-fedora
echo Building gstreamer/base-ubuntu:latest
docker build -t gstreamer/base-ubuntu:latest . -f Dockerfile-ubuntu
FROM registry.fedoraproject.org/fedora:29
RUN dnf install -y \
python3-pytest \
python3-requests \
python3-pytest-cov \
git-core \
&& rm -rf /var/cache/dnf /var/log/dnf*
\ No newline at end of file
# escape=`
FROM 'mcr.microsoft.com/windows/servercore:1809'
# Make sure any failure in PowerShell scripts is fatal
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]
ENV ErrorActionPreference='Stop'
COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\
RUN C:\install_toolchain.ps1
RUN C:\prepare_gst_env.ps1
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
$python_dl_url = 'https://www.python.org/ftp/python/3.7.4/python-3.7.4.exe'
$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.24.0.windows.2/MinGit-2.24.0.2-64-bit.zip'
$zip_url = 'https://www.7-zip.org/a/7z1900-x64.exe'
$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz'
Write-Host "Installing VisualStudio"
Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
Remove-Item C:\vs_buildtools.exe -Force
Write-Host "Installing Python"
Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe
Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
Remove-Item C:\python3.exe -Force
Write-Host "Installing Git"
Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip
Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
Remove-Item C:\mingit.zip -Force
$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
Write-Host "Installing 7zip"
Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe
Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
Remove-Item C:\7z-x64.exe -Force
Write-Host "Installing MSYS2"
Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz
C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
Remove-Item C:\msys2-x86_64.tar.xz -Force
Remove-Item C:\msys2-x86_64.tar -Force
Remove-Item C:\7zip -Recurse -Force
Write-Host "Installing Choco"
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
refreshenv
Write-Host "Installing git-lfs"
choco install -y git-lfs
refreshenv
Write-Host "Installing Meson"
pip install meson
Write-Host "Complete"
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;