Commit 2b8cc139 authored by Benjamin Tissoires's avatar Benjamin Tissoires
Browse files

generate the CI templates based on a template



It was quite cumbersome to do a change in one template
and then reproduce it everywhere.

Use Jinja2 to produce each template so we have only one place
to edit.

Note: a further commit will ensure the files are correctly
generated.
Signed-off-by: Benjamin Tissoires's avatarBenjamin Tissoires <benjamin.tissoires@gmail.com>
parent 9d4859ef
alpine:
distribution: alpine
package_type: pkgs
version: latest
upgrade:
- apk update
- apk upgrade
# also, default image doesn't have bash
- apk add bash
install:
- apk add
clean:
- rm -f /var/cache/apk/APKINDEX.*
arch:
distribution: arch
image: archlinux/base
package_type: pkgs
version: rolling
upgrade:
- pacman -S --refresh
- pacman -S --sysupgrade --noconfirm
install:
- pacman -S --noconfirm
clean:
# in case it doesn't exist yet, otherwise pacman clean fails
- mkdir -p /var/cache/pacman/pkg
- pacman -S --clean --noconfirm
aarch64: false
centos:
distribution: centos
package_type: rpms
version_examples: 7.7, 8.0
upgrade:
- dnf --help >/dev/null 2>&1 && DNF=dnf || DNF=yum
- $DNF upgrade -y
install:
- $DNF install -y
clean:
- $DNF clean all
aarch64: false
debian:
distribution: debian
package_type: debs
version_examples: stretch, sid
prepare:
- cat /etc/apt/sources.list
- echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/locale/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf
- echo '#!/bin/sh' > /usr/sbin/policy-rc.d
- echo 'exit 101' >> /usr/sbin/policy-rc.d
- chmod +x /usr/sbin/policy-rc.d
upgrade:
- env DEBIAN_FRONTEND=noninteractive apt-get update
- env DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
install:
- env DEBIAN_FRONTEND=noninteractive apt-get -y install
clean:
- env DEBIAN_FRONTEND=noninteractive apt-get clean
- rm -f /var/lib/apt/lists/*.lz4
fedora:
distribution: fedora
package_type: rpms
version_examples: 28, 29, rawhide
upgrade:
- dnf upgrade -y
install:
- dnf install -y
clean:
- dnf clean all
#!/bin/env python3
# vim: set expandtab shiftwidth=4 tabstop=4 textwidth=0:
import jinja2
import os
import yaml
from pathlib import Path
if __name__ == '__main__':
config_data = {}
# load the various definitions in the provided YAML files
src_folder = Path('src')
for distrib in [x for x in src_folder.iterdir()
if x.name.endswith('.yml')]:
with open(distrib) as fd:
data = yaml.load(fd, Loader=yaml.Loader)
# add some generated variables to keep the template simple
for config in data.values():
config['DISTRIBUTION'] = config['distribution'].upper()
if 'image' in config:
config['docker_repo'] = config['image'].split('/')[0]
else:
config['docker_repo'] = config['distribution']
if 'version' in config:
config['VERSION'] = config['version']
if 'image' not in config:
config['image'] = f'{config["distribution"]}:{config["version"]}'
else:
config['VERSION'] = f'${config["DISTRIBUTION"]}_VERSION'
if 'image' not in config:
config['image'] = '$DISTRO:$DISTRO_VERSION'
config['PACKAGES'] = f'{config["DISTRIBUTION"]}_{config["package_type"]}'.upper()
if 'aarch64' not in config:
config['aarch64'] = True
config_data.update(data)
env = jinja2.Environment(loader = jinja2.FileSystemLoader('./src'),
trim_blocks=True,
lstrip_blocks=True)
template = env.get_template('template.tmpl')
out_folder = Path('templates')
if not out_folder.exists():
os.mkdir(out_folder)
for distrib, config in config_data.items():
dest = out_folder / f'{distrib}.yml'
print(f'generating {dest}')
with open(dest, 'w') as out_stream:
template.stream(config).dump(out_stream)
# vim: set expandtab shiftwidth=2 tabstop=8 textwidth=0:
# This template will create a {{distribution}} image based on the following variables:
#
{% if not version %}
# - {{DISTRIBUTION}}_VERSION: the {{distribution}} version ({{version_examples}}, etc...)
{% endif %}
# - {{PACKAGES}}: if set, list of packages that needs to be installed
# - {{DISTRIBUTION}}_EXEC: if set, this command will be run once the packages have
# been installed
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../{{docker_repo}}/".
# If this variable isn't defined, "{{VERSION}}" is used for
# the suffix.
# - {{DISTRIBUTION}}_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
#
# The resulting image will be pushed in the local registry, under:
# $CI_REGISTRY_IMAGE/{{docker_repo}}/$REPO_SUFFIX:${{DISTRIBUTION}}_TAG
#
# Two flavors of templates are available:
# - `.{{distribution}}@container-build`: this will force rebuild a new container
# and tag it with ${{DISTRIBUTION}}_TAG without checks
# - `.{{distribution}}@container-ifnot-exists`: this will rebuild a new container
# only if ${{DISTRIBUTION}}_TAG is not available in the local registry or
# in the $UPSTREAM_REPO registry
# we can not reuse exported variables in after_script,
# so have a common definition
.{{distribution}}_vars: &distro_vars |
# exporting templates variables
# https://gitlab.com/gitlab-com/support-forum/issues/4349
export BUILDAH_FORMAT=docker
export DISTRO={{docker_repo}}
export DISTRO_TAG=${{DISTRIBUTION}}_TAG
export DISTRO_VERSION={{VERSION}}
export DISTRO_EXEC=${{DISTRIBUTION}}_EXEC
if [ x"$REPO_SUFFIX" == x"" ] ;
then
export REPO_SUFFIX=$DISTRO_VERSION
fi
export BUILDAH_RUN="buildah run --isolation chroot"
export BUILDAH_COMMIT="buildah commit --format docker"
.{{distribution}}@container-build:
image: $CI_REGISTRY/wayland/ci-templates/buildah:latest
stage: build
before_script:
# log in to the registry
- podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- *distro_vars
script:
- *distro_vars
- if [[ x"$DISTRO_TAG" == x"" ]] ;
then
echo $DISTRO tag missing;
exit 1;
fi
- echo Building $DISTRO/$REPO_SUFFIX:$DISTRO_TAG from {{ image }}
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from {{ image }})
- buildmnt=$(buildah mount $buildcntr)
{% if prepare %}
{% for command in prepare %}
- {{command.replace(" /", " $buildmnt/")}}
{% endfor %}
{% endif %}
{% for command in upgrade %}
- $BUILDAH_RUN $buildcntr {{command}}
{% endfor %}
- if [[ x"${{PACKAGES}}" != x"" ]] ;
then
{% for command in install %}
$BUILDAH_RUN $buildcntr {{command}} ${{PACKAGES}} ;
{% endfor %}
fi
# check if there is an optional post install script and run it
- if [[ x"$DISTRO_EXEC" != x"" ]] ;
then
echo Running $DISTRO_EXEC ;
set -x ;
mkdir $buildmnt/tmp/clone ;
pushd $buildmnt/tmp/clone ;
git init ;
git remote add origin $CI_REPOSITORY_URL ;
git fetch --depth 1 origin $CI_COMMIT_SHA ;
git checkout FETCH_HEAD > /dev/null;
buildah config --workingdir /tmp/clone $buildcntr ;
$BUILDAH_RUN $buildcntr bash -c "set -x ; $DISTRO_EXEC" ;
popd ;
rm -rf $buildmnt/tmp/clone ;
set +x ;
fi
# do not store the packages database, it's pointless
{% for command in clean %}
- $BUILDAH_RUN $buildcntr {{command}}
{% endfor %}
# set up the working directory
- mkdir $buildmnt/app
- buildah config --workingdir /app $buildcntr
# umount the container, not required, but, heh
- buildah unmount $buildcntr
# tag the current container
- $BUILDAH_COMMIT $buildcntr $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
# clean up the working container
- buildah rm $buildcntr
# push the container image to the registry
# There is a bug when pushing 2 tags in the same repo with the same base:
# this may fail. Just retry it after.
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG || true
- sleep 2
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
{% if aarch64 %}
.{{distribution}}@container-build@arm64v8:
extends: .{{distribution}}@container-build
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
tags:
- aarch64
{% endif %}
.before_script_ifnot_exists: &before_script_ifnot_exists
before_script:
# log in to the registry
- podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- *distro_vars
# check if our image is already in the current registry
- skopeo inspect docker://$CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG > /dev/null && exit 0 || true
# copy the original image into the current project registry namespace
- skopeo copy docker://$CI_REGISTRY/$UPSTREAM_REPO/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
docker://$CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG && exit 0 || true
.{{distribution}}@container-ifnot-exists:
extends: .{{distribution}}@container-build
<<: *before_script_ifnot_exists
{% if aarch64 %}
.{{distribution}}@container-ifnot-exists@arm64v8:
extends: .{{distribution}}@container-ifnot-exists
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
tags:
- aarch64
{% endif %}
ubuntu:
distribution: ubuntu
package_type: debs
version_examples: 18.04, 16.04
prepare:
- echo 'path-exclude=/usr/share/doc/*' > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/locale/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/man/*' >> /etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf
- echo '#!/bin/sh' > /usr/sbin/policy-rc.d
- echo 'exit 101' >> /usr/sbin/policy-rc.d
- chmod +x /usr/sbin/policy-rc.d
upgrade:
- env DEBIAN_FRONTEND=noninteractive apt-get update
- env DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
install:
- env DEBIAN_FRONTEND=noninteractive apt-get install -y
clean:
- env DEBIAN_FRONTEND=noninteractive apt-get clean
- rm -f /var/lib/apt/lists/*.lz4
......@@ -5,11 +5,11 @@
# - ALPINE_PKGS: if set, list of packages that needs to be installed
# - ALPINE_EXEC: if set, this command will be run once the packages have
# been installed
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../alpine/".
# If this variable isn't defined, "latest" is used for
# the suffix.
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - ALPINE_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
#
......@@ -20,7 +20,7 @@
# - `.alpine@container-build`: this will force rebuild a new container
# and tag it with $ALPINE_TAG without checks
# - `.alpine@container-ifnot-exists`: this will rebuild a new container
# only if $ALPINE is not available in the local registry or
# only if $ALPINE_TAG is not available in the local registry or
# in the $UPSTREAM_REPO registry
# we can not reuse exported variables in after_script,
......@@ -61,18 +61,19 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from alpine:latest)
- buildmnt=$(buildah mount $buildcntr)
- $BUILDAH_RUN $buildcntr apk update
- $BUILDAH_RUN $buildcntr apk upgrade
- $BUILDAH_RUN $buildcntr apk add bash
- if [[ x"$ALPINE_PKGS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr apk add $ALPINE_PKGS ;
fi
# check if there is an optional post install script and run it.
# also, default image doesn't have bash
# check if there is an optional post install script and run it
- if [[ x"$DISTRO_EXEC" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr apk add bash ;
echo Running $DISTRO_EXEC ;
set -x ;
mkdir $buildmnt/tmp/clone ;
......@@ -89,7 +90,7 @@
fi
# do not store the packages database, it's pointless
- rm -f $buildmnt/var/cache/apk/APKINDEX.*
- $BUILDAH_RUN $buildcntr rm -f /var/cache/apk/APKINDEX.*
# set up the working directory
- mkdir $buildmnt/app
......@@ -108,7 +109,6 @@
- sleep 2
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
.alpine@container-build@arm64v8:
extends: .alpine@container-build
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
......
......@@ -11,7 +11,7 @@
# If this variable isn't defined, "rolling" is used for
# the suffix.
# - ARCH_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
# tag does not exist, create a new build and tag it
#
# The resulting image will be pushed in the local registry, under:
# $CI_REGISTRY_IMAGE/archlinux/$REPO_SUFFIX:$ARCH_TAG
......@@ -20,7 +20,7 @@
# - `.arch@container-build`: this will force rebuild a new container
# and tag it with $ARCH_TAG without checks
# - `.arch@container-ifnot-exists`: this will rebuild a new container
# only if $ARCH is not available in the local registry or
# only if $ARCH_TAG is not available in the local registry or
# in the $UPSTREAM_REPO registry
# we can not reuse exported variables in after_script,
......@@ -61,8 +61,10 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from archlinux/base)
- buildmnt=$(buildah mount $buildcntr)
- $BUILDAH_RUN $buildcntr pacman -S --refresh
- $BUILDAH_RUN $buildcntr pacman -S --sysupgrade --noconfirm
- if [[ x"$ARCH_PKGS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr pacman -S --noconfirm $ARCH_PKGS ;
......@@ -86,9 +88,8 @@
set +x ;
fi
# in case it doesn't exist yet, otherwise pacman clean fails
- $BUILDAH_RUN $buildcntr mkdir -p /var/cache/pacman/pkg
# do not store the packages database, it's pointless
- $BUILDAH_RUN $buildcntr mkdir -p /var/cache/pacman/pkg
- $BUILDAH_RUN $buildcntr pacman -S --clean --noconfirm
# set up the working directory
......@@ -127,4 +128,3 @@
.arch@container-ifnot-exists:
extends: .arch@container-build
<<: *before_script_ifnot_exists
......@@ -9,7 +9,7 @@
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../centos/".
# If this variable isn't defined, $CENTOS_VERSION is used for
# If this variable isn't defined, "$CENTOS_VERSION" is used for
# the suffix.
# - CENTOS_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
......@@ -21,7 +21,7 @@
# - `.centos@container-build`: this will force rebuild a new container
# and tag it with $CENTOS_TAG without checks
# - `.centos@container-ifnot-exists`: this will rebuild a new container
# only if $CENTOS is not available in the local registry or
# only if $CENTOS_TAG is not available in the local registry or
# in the $UPSTREAM_REPO registry
# we can not reuse exported variables in after_script,
......@@ -62,8 +62,10 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from $DISTRO:$DISTRO_VERSION)
- buildmnt=$(buildah mount $buildcntr)
- $BUILDAH_RUN $buildcntr dnf --help >/dev/null 2>&1 && DNF=dnf || DNF=yum
- $BUILDAH_RUN $buildcntr $DNF upgrade -y
- if [[ x"$CENTOS_RPMS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr $DNF install -y $CENTOS_RPMS ;
......@@ -126,4 +128,3 @@
.centos@container-ifnot-exists:
extends: .centos@container-build
<<: *before_script_ifnot_exists
......@@ -9,7 +9,7 @@
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../debian/".
# If this variable isn't defined, $DEBIAN_VERSION is used for
# If this variable isn't defined, "$DEBIAN_VERSION" is used for
# the suffix.
# - DEBIAN_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
......@@ -62,7 +62,8 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from $DISTRO:$DISTRO_VERSION)
- buildmnt=$(buildah mount $buildcntr)
- $BUILDAH_RUN $buildcntr cat /etc/apt/sources.list
- cat $buildmnt/etc/apt/sources.list
- echo 'path-exclude=/usr/share/doc/*' > $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/locale/*' >> $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/man/*' >> $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
......@@ -73,6 +74,7 @@
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get update
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
- if [[ x"$DEBIAN_DEBS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get -y install $DEBIAN_DEBS ;
......@@ -98,7 +100,7 @@
# do not store the packages database, it's pointless
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get clean
- rm -f $buildmnt/var/lib/apt/lists/*.lz4
- $BUILDAH_RUN $buildcntr rm -f /var/lib/apt/lists/*.lz4
# set up the working directory
- mkdir $buildmnt/app
......@@ -117,7 +119,6 @@
- sleep 2
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
.debian@container-build@arm64v8:
extends: .debian@container-build
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
......
......@@ -9,7 +9,7 @@
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../fedora/".
# If this variable isn't defined, $FEDORA_VERSION is used for
# If this variable isn't defined, "$FEDORA_VERSION" is used for
# the suffix.
# - FEDORA_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
......@@ -21,7 +21,7 @@
# - `.fedora@container-build`: this will force rebuild a new container
# and tag it with $FEDORA_TAG without checks
# - `.fedora@container-ifnot-exists`: this will rebuild a new container
# only if $FEDORA is not available in the local registry or
# only if $FEDORA_TAG is not available in the local registry or
# in the $UPSTREAM_REPO registry
# we can not reuse exported variables in after_script,
......@@ -62,7 +62,9 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from $DISTRO:$DISTRO_VERSION)
- buildmnt=$(buildah mount $buildcntr)
- $BUILDAH_RUN $buildcntr dnf upgrade -y
- if [[ x"$FEDORA_RPMS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr dnf install -y $FEDORA_RPMS ;
......@@ -106,7 +108,6 @@
- sleep 2
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
.fedora@container-build@arm64v8:
extends: .fedora@container-build
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
......
......@@ -9,7 +9,7 @@
# - UPSTREAM_REPO: the upstream project on this gitlab instance where we might
# find the given tag (for example: `wayland/weston`)
# - REPO_SUFFIX: The repository name suffix after ".../ubuntu/".
# If this variable isn't defined, $UBUNTU_VERSION is used for
# If this variable isn't defined, "$UBUNTU_VERSION" is used for
# the suffix.
# - UBUNTU_TAG: tag to copy the image from the upstream registry. If the
# tag does not exist, create a new build and tag it
......@@ -62,6 +62,7 @@
# initial set up: take the base image, update it and install the packages
- buildcntr=$(buildah from $DISTRO:$DISTRO_VERSION)
- buildmnt=$(buildah mount $buildcntr)
- echo 'path-exclude=/usr/share/doc/*' > $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/locale/*' >> $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
- echo 'path-exclude=/usr/share/man/*' >> $buildmnt/etc/dpkg/dpkg.cfg.d/99-exclude-cruft
......@@ -72,6 +73,7 @@
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get update
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade
- if [[ x"$UBUNTU_DEBS" != x"" ]] ;
then
$BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get install -y $UBUNTU_DEBS ;
......@@ -97,7 +99,7 @@
# do not store the packages database, it's pointless
- $BUILDAH_RUN $buildcntr env DEBIAN_FRONTEND=noninteractive apt-get clean
- rm -f $buildmnt/var/lib/apt/lists/*.lz4
- $BUILDAH_RUN $buildcntr rm -f /var/lib/apt/lists/*.lz4
# set up the working directory
- mkdir $buildmnt/app
......@@ -116,7 +118,6 @@
- sleep 2
- podman push $CI_REGISTRY_IMAGE/$DISTRO/$REPO_SUFFIX:$DISTRO_TAG
.ubuntu@container-build@arm64v8:
extends: .ubuntu@container-build
image: $CI_REGISTRY/wayland/ci-templates/arm64v8/buildah:latest
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment