Commits (27)
- "build docker"
- "preparation"
# Test just one basic build, if succeds procced to test the rest
- "pre-build"
- "build"
- "test"
# Run multiple builds and tests, multi-distro, multi-arch
- "full builds"
- "full tests"
# build some apps to check that cross-platform binaries are usable
- "apps"
# use the binaries in some way
- "integrate"
test manifest:
......@@ -118,6 +115,8 @@ include: "gitlab/ci_template.yml"
.build local:
extends: '.build'
when: 'manual'
- 'manifest'
# Test the build job against the latest build image tag and the local manifest
build fedora x86_64 local:
......@@ -164,7 +163,7 @@ build cerbero cross win64 local:
image: '${CI_REGISTRY_IMAGE}/amd64/fedora:latest'
extends: '.test'
- build fedora x86_64 local
- 'build fedora x86_64 local'
when: 'manual'
......@@ -151,7 +151,7 @@ dnf debuginfo-install -y gtk3 \
zip \
pip3 install meson==0.49.2 git+https://github.com/axiros/terminal_markdown_viewer
pip3 install meson==0.50.1 git+https://github.com/axiros/terminal_markdown_viewer
# Install the dependencies of gstreamer
dnf builddep -y gstreamer1 \
# escape=`
FROM 'mcr.microsoft.com/windows/servercore:1607'
# Make sure any failure in PowerShell scripts is fatal
SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop';"]
COPY install_toolchain.ps1 prepare_gst_env.ps1 C:\
RUN C:\install_toolchain.ps1
RUN C:\prepare_gst_env.ps1
\ No newline at end of file
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
$python_dl_url = 'https://www.python.org/ftp/python/3.7.3/python-3.7.3.exe'
$msvc_2017_url = 'https://aka.ms/vs/15/release/vs_buildtools.exe'
$git_url = 'https://github.com/git-for-windows/git/releases/download/v2.19.1.windows.1/MinGit-2.19.1-64-bit.zip'
$zip_url = 'https://www.7-zip.org/a/7z1805-x64.exe'
$msys_url = 'https://ayera.dl.sourceforge.net/project/msys2/Base/x86_64/msys2-base-x86_64-20180531.tar.xz'
Write-Host "Installing VisualStudio"
Invoke-WebRequest -Uri $msvc_2017_url -OutFile C:\vs_buildtools.exe
Start-Process C:\vs_buildtools.exe -ArgumentList '--quiet --wait --norestart --nocache --installPath C:\BuildTools --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended' -Wait
Remove-Item C:\vs_buildtools.exe -Force
Write-Host "Installing Python"
Invoke-WebRequest -Uri $python_dl_url -OutFile C:\python3.exe
Start-Process C:\python3.exe -ArgumentList '/quiet InstallAllUsers=1 PrependPath=1' -Wait
Remove-Item C:\python3.exe -Force
Write-Host "Installing Git"
Invoke-WebRequest -Uri $git_url -OutFile C:\mingit.zip
Expand-Archive C:\mingit.zip -DestinationPath c:\mingit
Remove-Item C:\mingit.zip -Force
$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';' + 'c:\mingit\cmd'
[Environment]::SetEnvironmentVariable('PATH', $env:PATH, [EnvironmentVariableTarget]::Machine)
Write-Host "Installing 7zip"
Invoke-WebRequest -Uri $zip_url -OutFile C:\7z-x64.exe
Start-Process C:\7z-x64.exe -ArgumentList '/S /D=C:\7zip\' -Wait
Remove-Item C:\7z-x64.exe -Force
Write-Host "Installing MSYS2"
Invoke-WebRequest -Uri $msys_url -OutFile C:\msys2-x86_64.tar.xz
C:\7zip\7z e C:\msys2-x86_64.tar.xz -Wait
C:\7zip\7z x C:\msys2-x86_64.tar -o"C:\\"
Remove-Item C:\msys2-x86_64.tar.xz -Force
Remove-Item C:\msys2-x86_64.tar -Force
Remove-Item C:\7zip -Recurse -Force
Write-Host "Installing Choco"
iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
Write-Host "Installing git-lfs"
choco install -y git-lfs
$env:PATH += ";C:\msys64\usr\bin;C:\msys64\mingw64/bin;C:\msys64\mingw32/bin"
C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys"
C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm"
C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"
pip install meson
\ No newline at end of file
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;
git config --global user.email "gst-build@gstreamer.net"
git config --global user.name "Gstbuild Runner"
# Download gst-build and all its subprojects
git clone https://gitlab.freedesktop.org/gstreamer/gst-build.git C:\gst-build
# download the subprojects to try and cache them
meson subprojects download --sourcedir C:\gst-build
# Remove files that will conflict with a fresh clone on the runner side
Remove-Item -Force 'C:/gst-build/subprojects/*.wrap'
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-nasm'
Remove-Item -Recurse -Force 'C:/gst-build/subprojects/win-flex-bison-binaries'
Move-Item C:\gst-build\subprojects C:\subprojects
Remove-Item -Recurse -Force C:\gst-build
......@@ -2,12 +2,14 @@
import argparse
import os
import requests
import sys
import subprocess
import urllib.error
import urllib.parse
import urllib.request
import json
from typing import Dict, Tuple, List
from urllib.parse import urlparse
# from pprint import pprint
......@@ -26,6 +28,7 @@ GSTREAMER_MODULES: List[str] = [
......@@ -44,42 +47,68 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
def get_cerbero_last_build_info (namespace : str, branch : str):
base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs"
url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log"
deps = [{'commit': None}]
def get_branches_info(module: str, namespace: str, branches: List[str]) -> Tuple[str, str]:
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', *branches)
# The logs are only available if all jobs have passed so it does not
# matter which distro/arch is picked.
values = { 'job': "cerbero deps fedora x86_64" }
data = urllib.parse.urlencode(values)
req = urllib.request.Request(f"{url}?{data}")
resp = urllib.request.urlopen(req);
deps = json.loads(resp.read())
except urllib.error.URLError:
return None
return deps[0]['commit']
def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]:
res = git('ls-remote', f'https://gitlab.freedesktop.org/{namespace}/{module}.git', branch)
except subprocess.CalledProcessError:
return None, None
if not res:
return None, None
# Special case cerbero to avoid cache misses
if module == 'cerbero':
sha = get_cerbero_last_build_info(namespace, branch)
if sha is not None:
return sha, sha
lines = res.split('\n')
for branch in branches:
for line in lines:
if line.endswith('/' + branch):
sha, refname = line.split('\t')
except ValueError:
return refname.strip(), sha
for line in lines:
if line.endswith('/' + branch):
sha, refname = line.split('\t')
except ValueError:
return refname.strip(), sha
return None, None
def find_repository_sha(module: str, branchname: str) -> Tuple[str, str, str]:
namespace: str = os.environ["CI_PROJECT_NAMESPACE"]
ups_branch: str = os.getenv('GST_UPSTREAM_BRANCH', default='master')
if module == "orc":
ups_branch = os.getenv('ORC_UPSTREAM_BRANCH', default='master')
if module == os.environ['CI_PROJECT_NAME']:
return 'user', branchname, os.environ['CI_COMMIT_SHA']
if branchname != "master":
remote_refname, sha = get_branches_info(module, namespace, [branchname])
if branchname != ups_branch:
remote_refname, sha = get_branch_info(module, namespace, branchname)
if sha is not None:
return 'user', remote_refname, sha
# Check upstream project for a branch
remote_refname, sha = get_branches_info(module, 'gstreamer', [branchname, 'master'])
remote_refname, sha = get_branch_info(module, 'gstreamer', ups_branch)
if sha is not None:
return 'origin', remote_refname, sha
......@@ -125,6 +154,7 @@ def test_find_repository_sha():
os.environ["CI_PROJECT_NAME"] = "some-random-project"
os.environ["CI_PROJECT_URL"] = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-good"
os.environ["CI_PROJECT_NAMESPACE"] = "alatiera"
os.environ["GST_UPSTREAM_BRANCH"] = "master"
del os.environ["READ_PROJECTS_TOKEN"]
# This should find the repository in the user namespace
......@@ -155,11 +185,13 @@ def test_get_project_branch():
os.environ["CI_PROJECT_NAMESPACE"] = "nowaythisnamespaceexists_"
del os.environ["READ_PROJECTS_TOKEN"]
os.environ['GST_UPSTREAM_BRANCH'] = '1.12'
remote, refname, twelve = find_repository_sha('gst-plugins-good', '1.12')
assert twelve is not None
assert remote == 'origin'
assert refname == "refs/heads/1.12"
os.environ['GST_UPSTREAM_BRANCH'] = '1.14'
remote, refname, fourteen = find_repository_sha('gst-plugins-good', '1.14')
assert fourteen is not None
assert remote == 'origin'
This diff is collapsed.
......@@ -2,7 +2,7 @@
set -e
WORK_DIR=`mktemp -d -p "$DIR"`
......@@ -33,4 +33,4 @@ rsync -rvaz --links --delete documentation/ $DOC_BASE || /bin/true
chmod -R g+w $DOC_BASE; chgrp -R gstreamer $DOC_BASE
echo "Done updating documentation"
echo ""
\ No newline at end of file
echo ""