# # THIS FILE IS GENERATED, DO NOT EDIT # # we cannot reuse exported variables in after_script, # so let's have a common definition .fdo.fedora_vars: &fdo_distro_vars # exporting templates variables # https://gitlab.com/gitlab-com/support-forum/issues/4349 - export BUILDAH_FORMAT=docker - export DISTRO=fedora - export DISTRO_TAG=$FDO_DISTRIBUTION_TAG - export DISTRO_VERSION="$FDO_DISTRIBUTION_VERSION" - export DISTRO_EXEC=$FDO_DISTRIBUTION_EXEC - if [ x"$FDO_REPO_SUFFIX" == x"" ] ; then export FDO_REPO_SUFFIX=fedora/$DISTRO_VERSION ; fi - export BUILDAH_RUN="buildah run --isolation chroot" - export BUILDAH_COMMIT="buildah commit --format docker" - export BUILDAH_FROM="buildah from --isolation=chroot" - export CACHE_DIR=${FDO_CACHE_DIR:-/cache} # to be able to test the following script in the CI of the ci-templates # project itself, we need to put a special case here to have a # different image to pull if it already exists - export REPO_SUFFIX_LOCAL=$FDO_REPO_SUFFIX - if [[ x"$REPO_SUFFIX_LOCAL" == x"fedora/ci_templates_test_upstream" ]] ; then export FDO_REPO_SUFFIX=${DISTRO}/${DISTRO_VERSION} ; fi # have a special definition for the test if the image already exists .fdo.fedora_exists: &fdo_distro_exists | # if-not-exists steps set -x if [[ -z "$FDO_FORCE_REBUILD" ]] then # disable exit on failure set +e # check if our image is already in the current registry # we store the sha of the digest and the layers skopeo inspect docker://$CI_REGISTRY_IMAGE/$REPO_SUFFIX_LOCAL:$DISTRO_TAG | jq '[.Digest, .Layers]' > local_sha # check if our image is already in the upstream registry if [[ -z "$FDO_UPSTREAM_REPO" ]] then echo "WARNING! Variable \$FDO_UPSTREAM_REPO is undefined, cannot check for images" else UPSTREAM_REGISTRY_PATH="$(echo $FDO_UPSTREAM_REPO | tr '[:upper:]' '[:lower:]')" ; skopeo inspect docker://$CI_REGISTRY/$UPSTREAM_REGISTRY_PATH/$FDO_REPO_SUFFIX:$DISTRO_TAG | jq '[.Digest, .Layers]' > upstream_sha fi # reenable exit on failure set -e # if the upstream repo has an image, ensure we use the same if [ -s upstream_sha ] then # ensure we use the same image from upstream diff upstream_sha local_sha && exit 0 || true # copy the original image into the current project registry namespace # we do 2 attempts with skopeo copy at most skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ docker://$CI_REGISTRY/$UPSTREAM_REGISTRY_PATH/$FDO_REPO_SUFFIX:$DISTRO_TAG \ docker://$CI_REGISTRY_IMAGE/$REPO_SUFFIX_LOCAL:$DISTRO_TAG || \ skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \ docker://$CI_REGISTRY/$UPSTREAM_REGISTRY_PATH/$FDO_REPO_SUFFIX:$DISTRO_TAG \ docker://$CI_REGISTRY_IMAGE/$REPO_SUFFIX_LOCAL:$DISTRO_TAG exit 0 fi # if we have a local image but none in the upstream repo, use our if [ -s local_sha ] then exit 0 fi fi .fdo.fedora: variables: FDO_DISTRIBUTION_NAME: "fedora" .fdo.container-build-common@fedora: extends: .fdo.fedora stage: build script: # check that the container registry is enabled - | if [[ -z "$CI_REGISTRY_IMAGE" ]] then # the container registry is not enabled in the project, no point to # continue; let the user know so they can fix their project settings echo The container registry is not enabled in $CI_PROJECT_PATH, \ enable it in the project general settings panel ; cat <<EOF > container-build-report.xml ; <?xml version="1.0" encoding="utf-8"?> <testsuites tests="1" errors="0" failures="1"> <testsuite name="registry-not-enabled" tests="1" errors="0" failures="1" skipped="0"> <testcase name="registry-not-enabled"> <failure message="The container registry is not enabled in $CI_PROJECT_PATH, enable it in the project general settings panel"/> </testcase> </testsuite> </testsuites> EOF exit 1 ; fi # log in to the registry - podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - skopeo login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - *fdo_distro_vars - if [[ x"$DISTRO_TAG" == x"" ]] ; then echo $DISTRO tag missing; exit 1; fi - *fdo_distro_exists - if [[ x"$FDO_BASE_IMAGE" != x"" ]] ; then BASE_IMAGE="$FDO_BASE_IMAGE"; else BASE_IMAGE="$DISTRO:$DISTRO_VERSION"; fi - echo Building $FDO_REPO_SUFFIX:$DISTRO_TAG from $BASE_IMAGE # initial set up: take the base image, update it and install the packages - buildcntr=$($BUILDAH_FROM $BASE_IMAGE) - buildmnt=$(buildah mount $buildcntr) - $BUILDAH_RUN $buildcntr dnf upgrade -y --setopt=install_weak_deps=False --nogpgcheck fedora-release 'fedora-repos*' - $BUILDAH_RUN $buildcntr dnf upgrade -y --setopt=install_weak_deps=False - if [[ x"$FDO_DISTRIBUTION_PACKAGES" != x"" ]] ; then $BUILDAH_RUN $buildcntr dnf install -y $FDO_DISTRIBUTION_PACKAGES ; fi # check if there is an optional post install script and run it - if [[ x"$DISTRO_EXEC" != x"" ]] ; then echo Running $DISTRO_EXEC ; set -x ; mkdir $buildmnt/tmp/clone ; pushd $buildmnt/tmp/clone ; git init ; git remote add origin $CI_REPOSITORY_URL ; git fetch --depth 1 origin $CI_COMMIT_SHA ; git checkout FETCH_HEAD > /dev/null; buildah config --workingdir /tmp/clone --env HOME="$HOME" $buildcntr ; if [ -e $CACHE_DIR ] ; then CACHE="-v $CACHE_DIR:/cache:rw,shared,z" ; fi ; $BUILDAH_RUN $CACHE $buildcntr bash -c "set -x ; $DISTRO_EXEC" ; popd ; buildah config --workingdir / --env HOME="$HOME" $buildcntr ; rm -rf $buildmnt/tmp/clone ; set +x ; fi # do not store the packages database, it's pointless - $BUILDAH_RUN $buildcntr dnf clean all # set up the working directory - mkdir -p $buildmnt/app - buildah config --workingdir /app $buildcntr # umount the container, not required, but, heh - buildah unmount $buildcntr - if [[ x"$FDO_EXPIRES_AFTER" != x"" ]] ; then buildah config -l fdo.expires-after=$FDO_EXPIRES_AFTER $buildcntr; fi - if [[ x"$FDO_UPSTREAM_REPO" != x"" ]] ; then buildah config -l fdo.upstream-repo=$FDO_UPSTREAM_REPO $buildcntr; fi - buildah config -l fdo.pipeline_id="$CI_PIPELINE_ID" $buildcntr; - buildah config -l fdo.job_id="$CI_JOB_ID" $buildcntr; - buildah config -l fdo.project="$CI_PROJECT_PATH" $buildcntr; - buildah config -l fdo.commit="$CI_COMMIT_SHA" $buildcntr; # tag the current container - $BUILDAH_COMMIT $buildcntr $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG # clean up the working container - buildah rm $buildcntr # push the container image to the registry # There is a bug when pushing 2 tags in the same repo with the same base: # this may fail. Just retry it after. - set -x - podman push $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG || true - sleep 2 - podman push $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG - set +x artifacts: paths: - container-build-report.xml reports: junit: container-build-report.xml ### # Checks for a pre-existing fedora container image and builds it if # it does not yet exist. # # If an image with the same version or suffix exists in the upstream project's registry, # the image is copied into this project's registry. # If no such image exists, the image is built and pushed to the local registry. # # **Example:** # # .. code-block:: yaml # # my-fedora-image: # extends: .fdo.container-build@fedora@x86_64 # variables: # FDO_DISTRIBUTION_PACKAGES: 'curl wget gcc valgrind' # FDO_DISTRIBUTION_VERSION: '33' # FDO_DISTRIBUTION_TAG: '2020-03-20' # # # # **Reserved by this template:** # # - ``image:`` do not override # - ``script:`` do not override # # **Variables:** # # .. attribute:: FDO_DISTRIBUTION_VERSION # # **This variable is required** # # The fedora version to build, e.g. '33', '32' # # .. attribute:: FDO_DISTRIBUTION_TAG # # **This variable is required** # # String to identify the image in the registry. # # .. attribute:: FDO_UPSTREAM_REPO # # The GitLab project path to the upstream project # # .. attribute:: FDO_REPO_SUFFIX # # The repository name suffix to use, see below. # # .. attribute:: FDO_DISTRIBUTION_PACKAGES # # Packages to install as a space-separated single string, e.g. "curl wget" # # .. attribute:: FDO_DISTRIBUTION_EXEC # # An executable run after the installation of the :attr:`FDO_DISTRIBUTION_PACKAGES` # # .. attribute:: FDO_FORCE_REBUILD # # If set, the image will be built even if it exists in the registry already # # .. attribute:: FDO_BASE_IMAGE # # By default, the base image to start with is # ``fedora:$FDO_DISTRIBUTION_VERSION`` # and all dependencies are installed on top of that base image. If # ``FDO_BASE_IMAGE`` is given, it references a different base image to start with. # This image usually requires the full registry path, e.g. # ``registry.freedesktop.org/projectGroup/projectName/repo_suffix:tag-name`` # # .. attribute:: FDO_EXPIRES_AFTER # # If set, enables an expiration time on the image to # aid the garbage collector in deciding when an image can be removed. This # should be set for temporary images that are not intended to be kept # forever. Allowed values are e.g. ``1h`` (one hour), ``2d`` (two days) or # ``3w`` (three weeks). # # .. attribute:: FDO_CACHE_DIR # # If set, the given directory is mounted as ``/cache`` # when ``FDO_DISTRIBUTION_EXEC`` is run. This can allow for passing of # cache values between build jobs (if run on the same runner). You should # not usually need to set this, it defaults to ``/cache`` from the host # and thus enables cache sharing by default. # # The resulting image will be pushed to the local registry. # # If :attr:`FDO_REPO_SUFFIX` was specified, the image path is # ``$CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$FDO_DISTRIBUTION_TAG``. # Use the ``.fdo.suffixed-image@fedora`` template to access or use this image. # # If :attr:`FDO_REPO_SUFFIX` was **not** specified, the image path is # ``$CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG``. # Use the ``.fdo.distribution-image@fedora`` template to access or use this image. # .fdo.container-build@fedora@x86_64: extends: .fdo.container-build-common@fedora image: quay.io/freedesktop.org/ci-templates:buildah-x86_64-2020-12-18.2 ### # Alias to ``.fdo.container-build@fedora@x86_64``. .fdo.container-build@fedora: extends: .fdo.container-build@fedora@x86_64 ### # Checks for a pre-existing fedora container image for the # ``aarch64`` processor architecture and builds it if it does not yet exist. # # This template requires runners with the ``aarch64`` tag. # # See ``.fdo.container-build@fedora@x86_64`` for details. .fdo.container-build@fedora@aarch64: extends: .fdo.container-build-common@fedora image: quay.io/freedesktop.org/ci-templates:buildah-aarch64-2020-12-18.2 tags: - aarch64 ### # Checks for a pre-existing fedora container image usable on # KVM-enabled runners and builds it if it does not yet exist. # # This template requires runners with the ``kvm`` tag. # # See ``.fdo.container-build@fedora@x86_64`` for details. .fdo.qemu-build@fedora@x86_64: extends: .fdo.container-build@fedora@x86_64 tags: - kvm image: quay.io/freedesktop.org/ci-templates:qemu-mkosi-base-x86_64-2020-12-18.2 script: # log in to the registry - podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - skopeo login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - *fdo_distro_vars - *fdo_distro_exists # start our current base mkosi image - /bin/bash /app/vmctl start -cdrom /app/my-seed.iso - QEMU_VERSION=$FDO_DISTRIBUTION_VERSION - | cat <<EOF > mkosi.default [Distribution] Distribution=fedora Release=$QEMU_VERSION [Output] Format=gpt_ext4 Bootable=yes BootProtocols=bios WithUnifiedKernelImages=no Password=root KernelCommandLine=!* selinux=0 audit=0 rw console=tty0 console=ttyS0 systemd.journald.forward_to_console=1 [Partitions] RootSize=2G [Packages] # The packages to appear in both the build and the final image Packages= openssh-clients openssh-server NetworkManager iproute iputils git-core EOF - echo $FDO_DISTRIBUTION_PACKAGES | tr ' ' '\n' | sed -e 's/^/ /' >> mkosi.default # create a new ssh key - ssh-keygen -t ed25519 -f /root/.ssh/id_ed25519_target -N '' - mkdir -p mkosi.extra/root/.ssh - chmod 700 mkosi.extra/root/.ssh - cp /root/.ssh/id_ed25519_target.pub mkosi.extra/root/.ssh/authorized_keys - chmod 600 mkosi.extra/root/.ssh/authorized_keys # enable sshd on the target - mkdir -p mkosi.extra/etc/systemd/system/multi-user.target.wants # send the mkosi files to the VM - scp mkosi.default vm:/root/mkosi.default - scp mkosi.default vm:/root/mkosi.default - scp -r mkosi.extra vm:/root/ # enable sshd on the target - /app/vmctl exec ln -s /usr/lib/systemd/system/sshd.service mkosi.extra/etc/systemd/system/multi-user.target.wants/sshd.service # create a cache folder (useful only when manually testing this script) - /app/vmctl exec mkdir mkosi.cache # run mkosi in the VM! - /app/vmctl exec mkosi # mount the root partition locally to extract the kernel and initramfs - /app/vmctl exec mkdir loop - offset=$(/app/vmctl exec fdisk -l image.raw | grep image.raw2 | cut -d ' ' -f 3) - /app/vmctl exec mount -o ro,loop,offset=$(($offset * 512)) image.raw loop/ # fetch kernel and initramfs - /app/vmctl exec ls loop/boot/ - /app/vmctl exec "cp loop/boot/vmlinuz* loop/boot/initr* ." - /app/vmctl exec umount loop/ # now compress the image (we wanted to extract first the kernel and initrd) - /app/vmctl exec xz -T0 image.raw # fetch the image and kernel - scp vm:image.raw.xz dest-image.raw.xz - scp vm:vmlinuz\* vm:initr\* . # terminate qemu - kill $(pgrep qemu) || true # building the final image - | cat > /etc/containers/storage.conf <<EOF [storage] driver = "vfs" runroot = "/var/run/containers/storage" graphroot = "/var/lib/containers/storage" EOF - QEMU_BASE_IMAGE=${QEMU_BASE_IMAGE:-quay.io/freedesktop.org/ci-templates:qemu-base-x86_64-2020-12-18.2} - echo Building $FDO_REPO_SUFFIX:$DISTRO_TAG from $QEMU_BASE_IMAGE - buildcntr=$($BUILDAH_FROM $QEMU_BASE_IMAGE) - buildmnt=$(buildah mount $buildcntr) # insert our final VM image we just built - mkdir -p $buildmnt/app - mv dest-image.raw.xz $buildmnt/app/image.raw.xz - mv vmlinuz* initr* $buildmnt/app/ - mkdir $buildmnt/root/.ssh - chmod 700 $buildmnt/root/.ssh - cp /root/.ssh/id_ed25519_target $buildmnt/root/.ssh/id_ed25519 - cp /root/.ssh/id_ed25519_target.pub $buildmnt/root/.ssh/id_ed25519.pub # umount the container, not required, but, heh - buildah unmount $buildcntr - if [[ x"$FDO_EXPIRES_AFTER" != x"" ]] ; then buildah config -l fdo.expires-after=$FDO_EXPIRES_AFTER $buildcntr; fi - if [[ x"$FDO_UPSTREAM_REPO" != x"" ]] ; then buildah config -l fdo.upstream-repo=$FDO_UPSTREAM_REPO $buildcntr; fi # tag the current container - $BUILDAH_COMMIT $buildcntr $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG # clean up the working container - buildah rm $buildcntr # push the container image to the registry # There is a bug when pushing 2 tags in the same repo with the same base: # this may fail. Just retry it after. - set -x - podman push $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG || true - sleep 2 - podman push $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$DISTRO_TAG - set +x # Alias to ``.fdo.qemu-build@fedora@x86_64`` .fdo.qemu-build@fedora: extends: .fdo.qemu-build@fedora@x86_64 ### # fedora template that pulls the fedora image from the # registry based on ``FDO_DISTRIBUTION_VERSION`` and ``FDO_DISTRIBUTION_TAG``. # This template must be provided the same variable values as supplied in # ``.fdo.container-build@fedora@x86_64``. # # This template sets ``image:`` to the generated image. You may override this. # # **Example:** # # .. code-block:: yaml # # my-fedora-test: # extends: .fdo.distribution-image@fedora # variables: # FDO_DISTRIBUTION_VERSION: '33' # FDO_DISTRIBUTION_TAG: '2020-03-20' # script: # - meson builddir # - ninja -C builddir test # # **Variables:** # # .. attribute:: FDO_DISTRIBUTION_VERSION # # **This variable is required** # # The fedora version to build, e.g. '33', '32' # # The value supplied must be the same as supplied in # ``.fdo.container-build@fedora@x86_64``. # # .. attribute:: FDO_DISTRIBUTION_TAG # # **This variable is required** # # String to identify the image in the registry. # # The value supplied must be the same as supplied in # ``.fdo.container-build@fedora@x86_64``. # # .. attribute:: FDO_DISTRIBUTION_IMAGE # # **This variable is set by this template and should be treated as read-only** # # Path to the registry image # # .. attribute:: FDO_DISTRIBUTION_NAME # # **This variable is set by this template and should be treated as read-only** # # Set to the string "fedora" # # .. note:: If you used ``FDO_REPO_SUFFIX`` when building the container, use # ``.fdo.suffixed-image@fedora`` instead. .fdo.distribution-image@fedora: extends: .fdo.fedora image: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG variables: FDO_DISTRIBUTION_IMAGE: $CI_REGISTRY_IMAGE/fedora/$FDO_DISTRIBUTION_VERSION:$FDO_DISTRIBUTION_TAG ### # fedora template that pulls the fedora image from the # registry based on ``FDO_REPO_SUFFIX``. # This template must be provided the same variable values as supplied in # ``.fdo.container-build@fedora@x86_64``. # # This template sets ``image:`` to the generated image. You may override this. # # **Example:** # # .. code-block:: yaml # # my-fedora-test@x86_64: # extends: .fdo.distribution-image@fedora # variables: # FDO_REPO_SUFFIX: 'some/path' # FDO_DISTRIBUTION_TAG: '2020-03-20' # script: # - meson builddir # - ninja -C builddir test # # # **Variables:** # # .. attribute:: FDO_REPO_SUFFIX # # **This variable is required** # # The repository name suffix. # # The value supplied must be the same as supplied in # ``.fdo.container-build@fedora@x86_64``. # # .. attribute:: FDO_DISTRIBUTION_TAG # # **This variable is required** # # String to identify the image in the registry. # # The value supplied must be the same as supplied in # ``.fdo.container-build@fedora@x86_64``. # # .. attribute:: FDO_DISTRIBUTION_IMAGE # # **This variable is set by this template and should be treated as read-only** # # Path to the registry image # # .. attribute:: FDO_DISTRIBUTION_NAME # # **This variable is set by this template and should be treated as read-only** # # Set to the string "fedora" # # # Variables provided by this template should be considered read-only. # # .. note:: If you did not use ``FDO_REPO_SUFFIX`` when building the container, use # ``.fdo.distribution-image@fedora`` instead. .fdo.suffixed-image@fedora: extends: .fdo.fedora image: $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$FDO_DISTRIBUTION_TAG variables: FDO_DISTRIBUTION_IMAGE: $CI_REGISTRY_IMAGE/$FDO_REPO_SUFFIX:$FDO_DISTRIBUTION_TAG