Commit acda94e1 authored by Peter Hutterer's avatar Peter Hutterer
Browse files

Provide a vmctl start|start-kernel|stop|exec command



This wraps the common things we need to do - start the vm, stop it later and
in between exec a few commands on the VM itself.

exec is just a wrapper around ssh and vmctl start installs a ssh_config.d
entry so we can acces the vm as host "vm". This simplifies our scp calls a lot
too since we don't need to know about the port anymore and calling it "vm" is
less confusing than "localhost".

FTR, I did have a vmctl copy command in the test branch but it ended up being
worse than just calling scp - the argument for exec is weak enough as it is.
Signed-off-by: Peter Hutterer's avatarPeter Hutterer <peter.hutterer@who-t.net>
parent 4b299728
......@@ -67,8 +67,8 @@ include:
- local: '/.gitlab-ci/ubuntu-ci.yml'
variables:
BOOTSTRAP_TAG: '2020-03-17'
QEMU_TAG: '2020-03-18'
BOOTSTRAP_TAG: '2020-03-24'
QEMU_TAG: '2020-03-24'
stages:
......
......@@ -95,21 +95,21 @@
script:
- pushd /app
# start the VM
- bash /app/start_vm.sh
- bash /app/vmctl start
# run both curl and wget because one of those two is installed and one is
# in the base image, but it depends on the distro which one
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
# start the VM, with the kernel parameters
- bash /app/start_vm_kernel.sh
- bash /app/vmctl start-kernel
# make sure we can still use curl/wget
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
artifacts:
name: logs-$CI_PIPELINE_ID
when: always
......
......@@ -95,21 +95,21 @@
script:
- pushd /app
# start the VM
- bash /app/start_vm.sh
- bash /app/vmctl start
# run both curl and wget because one of those two is installed and one is
# in the base image, but it depends on the distro which one
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
# start the VM, with the kernel parameters
- bash /app/start_vm_kernel.sh
- bash /app/vmctl start-kernel
# make sure we can still use curl/wget
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
artifacts:
name: logs-$CI_PIPELINE_ID
when: always
......
......@@ -95,21 +95,21 @@
script:
- pushd /app
# start the VM
- bash /app/start_vm.sh
- bash /app/vmctl start
# run both curl and wget because one of those two is installed and one is
# in the base image, but it depends on the distro which one
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
# start the VM, with the kernel parameters
- bash /app/start_vm_kernel.sh
- bash /app/vmctl start-kernel
# make sure we can still use curl/wget
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
artifacts:
name: logs-$CI_PIPELINE_ID
when: always
......
......@@ -66,7 +66,7 @@
FDO_DISTRIBUTION_TAG: $QEMU_TAG
FDO_REPO_SUFFIX: qemu-base
FDO_DISTRIBUTION_PACKAGES: 'buildah-1.11.3-2.fc31 podman-2:1.6.2-2.fc31 skopeo jq git qemu genisoimage usbutils'
FDO_DISTRIBUTION_EXEC: 'bootstrap/prep_qemu.sh'
FDO_DISTRIBUTION_EXEC: 'mkdir -p /app && cp bootstrap/vmctl.sh /app/vmctl'
# qemu container capable of creating an other VM image
#
......@@ -76,4 +76,4 @@
- kvm
variables:
FDO_REPO_SUFFIX: qemu-mkosi-base
FDO_DISTRIBUTION_EXEC: 'bootstrap/prep_qemu.sh && bootstrap/prep_mkosi.sh'
FDO_DISTRIBUTION_EXEC: 'mkdir -p /app && cp bootstrap/vmctl.sh /app/vmctl && bootstrap/prep_mkosi.sh'
......@@ -61,19 +61,19 @@ genisoimage -output /app/my-seed.iso \
-rock /app/user-data /app/meta-data
# do some initial preparation in the target VM so it is mkosi capable
/app/start_vm.sh -cdrom /app/my-seed.iso
/app/vmctl start -cdrom /app/my-seed.iso
# install mkosi dependencies to build up our final VM image
ssh localhost -p 5555 dnf install -y mkosi systemd-container
/app/vmctl exec dnf install -y mkosi systemd-container
# use the upstream mkosi code
ssh localhost -p 5555 git clone https://github.com/systemd/mkosi.git
/app/vmctl exec git clone https://github.com/systemd/mkosi.git
# pacman needs 5.2.0 to work, fedora is still on 5.0.0
# https://bugzilla.redhat.com/show_bug.cgi?id=1582967
# arch / qemu is disabled until it gets fixed
# stop the vm and compress the image file
/app/stop_vm.sh
/app/vmctl stop
# manually compress the image with `-T0` to use multithreading
xz -T0 /app/image.raw
......
#!/bin/bash
set -e
set -x
mkdir -p /app
cat > /app/start_vm.sh <<EOF
#!/bin/bash
set -x
if [[ ! -e /app/image.raw ]]
then
xz -d -T0 /app/image.raw.xz || (echo "Failed to unpack image" && exit 1)
fi
qemu-system-x86_64 -machine accel=kvm \\
-smp 2 -m 1024 \\
-drive format=raw,file=/app/image.raw \\
-device virtio-net-pci,netdev=net0 \\
-netdev user,id=net0,hostfwd=tcp::5555-:22 \\
-display none \\
-daemonize \\
"\$@" \\
-serial file:\$CI_BUILDS_DIR/\$CI_PROJECT_PATH/console.out
exit_code=\$?
# Connect once to store the host key locally
if [[ \$exit_code -eq 0 ]]; then
ssh -o StrictHostKeyChecking=accept-new localhost -p 5555 uname -a
exit_code=\$?
fi
if [[ \$exit_code -ne 0 ]]; then
echo "***********************************************************"
echo "* *"
echo "* WARNING: failed to start or connect to VM *"
echo "* *"
echo "***********************************************************"
fi
exit \$exit_code
EOF
chmod +x /app/start_vm.sh
cat > /app/stop_vm.sh <<EOF
#!/bin/bash
set -x
ssh localhost -p 5555 halt
sleep 2
pkill qemu
EOF
chmod +x /app/stop_vm.sh
#!/bin/bash
usage() {
echo "Usage: vmctl {start|start-kernel|stop|exec} [command]"
echo ""
echo "Options: "
echo " start ... start the virtual machine"
echo " start-kernel ... start the virtual machine with a custom kernel"
echo " stop ... stop the virtual machine"
echo " exec ... exec command on the virtual machine"
}
do_start() {
set -ex
if [[ ! -e /app/image.raw ]]
then
xz -d -T0 /app/image.raw.xz || (echo "Failed to unpack image" && exit 1)
fi
qemu-system-x86_64 -machine accel=kvm \
-smp 2 -m 1024 \
-drive format=raw,file=/app/image.raw \
-device virtio-net-pci,netdev=net0 \
-netdev user,id=net0,hostfwd=tcp::5555-:22 \
-display none \
-daemonize \
"$@" \
-serial file:$CI_BUILDS_DIR/$CI_PROJECT_PATH/console.out
exit_code=$?
if [[ ! -e /etc/ssh/ssh_config.d/99-vm.conf ]]; then
cat >/etc/ssh/ssh_config.d/99-vm.conf <<EOF
Host vm
HostName localhost
Port 5555
EOF
fi
# Connect once to store the host key locally
if [[ $exit_code -eq 0 ]]; then
ssh -o StrictHostKeyChecking=accept-new vm uname -a
exit_code=$?
fi
if [[ $exit_code -ne 0 ]]; then
echo "***********************************************************"
echo "* *"
echo "* WARNING: failed to start or connect to VM *"
echo "* *"
echo "***********************************************************"
fi
exit $exit_code
}
do_start_kernel() {
set -x
KERNEL=$1
shift
set -e
# default to the latest kernel
if [[ x"$KERNEL" == x"" ]]
then
KERNEL=$(ls /app/vmlinuz* | sort | tail -1)
fi
INITRD=$(ls /app/initr* | sort | tail -1)
do_start -kernel $KERNEL \
-initrd $INITRD \
-append "root=/dev/sda2 selinux=0 audit=0 rw console=tty0 console=ttyS0" \
"$@"
}
do_stop() {
set -x
do_exec halt
sleep 2
pkill qemu
rm -f /etc/ssh/ssh_config.d/99-vm.conf
exit 0
}
do_exec() {
set -x
ssh -oControlMaster=auto -oControlPersist=5 vm "$@"
}
case $1 in
start)
shift
do_start "$@"
;;
start-kernel)
shift
do_start_kernel "$@"
;;
stop)
shift
do_stop "$@"
;;
exec)
shift
do_exec "$@"
;;
*)
usage
exit 1
;;
esac
......@@ -265,9 +265,10 @@ instead of ``container``. The above example thus becomes:
# Packages to install on the vm
FDO_DISTRIBUTION_PACKAGES: "valgrind gcc"
Once built, the container image provides the script ``/app/start_vm.sh``
Once built, the container image provides the script ``/app/vmctl start``
to start the virtual machine. The VM is configured to accept ssh connection
on port ``5555``.
and aliased as host ``vm``. Commands can be run on the virtual machine with
the ``/app/vmctl exec`` helper.
.. code-block:: yaml
......@@ -279,22 +280,23 @@ on port ``5555``.
- .myproject.fedora:30
stage: test
script:
# start the VM
- /app/start_vm.sh
# start the VM. This also sets up ssh/scp to connect to "vm"
# correctly.
- /app/vmctl start
# copy our workspace to the VM
# The quotes are required to stop the ':' from parsing as yaml
- "scp -P 5555 -r $PWD localhost:"
- scp -r $PWD "vm:"
# We don't want any failed commands to exit our script until VM
# cleanup has been completed.
- set +e
# run test-command on the VM and create the .success file if it
# succeeds
- ssh localhost -p 5555 "cd $CI_PROJECT_NAME ; test-command" && touch .success
- /app/vmctl exec "cd $CI_PROJECT_NAME ; test-command" && touch .success
# copy any test results from the VM to our container so we can
# save them as artifacts
- scp -P 5555 -r localhost:$CI_PROJECT_NAME/test-results.xml .
- scp -r vm:$CI_PROJECT_NAME/test-results.xml .
# shut down the VM
- /app/stop_vm.sh
- /app/vmctl stop
# VM cleanup is complete, any command failures now should result in
# a CI failed job
- set -e
......
......@@ -133,7 +133,7 @@ globals:
- 'sh test/script.sh'
## @bootstrap_tag: the current image tag associated with this bootstrap image
bootstrap_tag: '2020-03-17'
bootstrap_tag: '2020-03-24'
## @qemu_tag: the current image tag associated with this qemu bootstrap image
qemu_tag: '2020-03-18'
qemu_tag: '2020-03-24'
......@@ -102,21 +102,21 @@
script:
- pushd /app
# start the VM
- bash /app/start_vm.sh
- bash /app/vmctl start
# run both curl and wget because one of those two is installed and one is
# in the base image, but it depends on the distro which one
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
# start the VM, with the kernel parameters
- bash /app/start_vm_kernel.sh
- bash /app/vmctl start-kernel
# make sure we can still use curl/wget
- ssh -p 5555 localhost curl --insecure https://gitlab.freedesktop.org
- ssh -p 5555 localhost wget --no-check-certificate https://gitlab.freedesktop.org
- /app/vmctl exec curl --insecure https://gitlab.freedesktop.org
- /app/vmctl exec wget --no-check-certificate https://gitlab.freedesktop.org
# terminate the VM
- bash /app/stop_vm.sh
- bash /app/vmctl stop
artifacts:
name: logs-$CI_PIPELINE_ID
when: always
......
......@@ -327,7 +327,7 @@
- *fdo_distro_exists
# start our current base mkosi image
- /bin/bash /app/start_vm.sh -cdrom /app/my-seed.iso
- /bin/bash /app/vmctl start -cdrom /app/my-seed.iso
- QEMU_VERSION=$FDO_DISTRIBUTION_VERSION
{% if ubuntu_versions %}
......@@ -388,37 +388,38 @@
- mkdir -p mkosi.extra/etc/systemd/system/multi-user.target.wants
# send the mkosi files to the VM
- scp -P 5555 mkosi.default localhost:/root/mkosi.default
- scp -P 5555 -r mkosi.extra localhost:/root/
- scp mkosi.default vm:/root/mkosi.default
- scp mkosi.default vm:/root/mkosi.default
- scp -r mkosi.extra vm:/root/
# enable sshd on the target
- ssh localhost -p 5555 ln -s /usr/lib/systemd/system/sshd.service
- /app/vmctl exec ln -s /usr/lib/systemd/system/sshd.service
mkosi.extra/etc/systemd/system/multi-user.target.wants/sshd.service
# create a cache folder (useful only when manually testing this script)
- ssh localhost -p 5555 mkdir mkosi.cache
- /app/vmctl exec mkdir mkosi.cache
# run mkosi in the VM!
- ssh localhost -p 5555 mkosi/mkosi
- /app/vmctl exec mkosi/mkosi
# mount the root partition locally to extract the kernel and initramfs
- ssh localhost -p 5555 mkdir loop
- offset=$(ssh localhost -p 5555 fdisk -l image.raw | grep image.raw2 | cut -d ' ' -f 3)
- ssh localhost -p 5555 mount -o ro,loop,offset=$(($offset * 512)) image.raw loop/
- /app/vmctl exec mkdir loop
- offset=$(/app/vmctl exec fdisk -l image.raw | grep image.raw2 | cut -d ' ' -f 3)
- /app/vmctl exec mount -o ro,loop,offset=$(($offset * 512)) image.raw loop/
# fetch kernel and initramfs
- ssh localhost -p 5555 ls loop/boot/
- ssh localhost -p 5555 "cp loop/boot/vmlinuz* loop/boot/initr* ."
- /app/vmctl exec ls loop/boot/
- /app/vmctl exec "cp loop/boot/vmlinuz* loop/boot/initr* ."
- ssh localhost -p 5555 umount loop/
- /app/vmctl exec umount loop/
# now compress the image (we wanted to extract first the kernel and initrd)
- ssh localhost -p 5555 xz -T0 image.raw
- /app/vmctl exec xz -T0 image.raw
# fetch the image and kernel
- scp -P 5555 localhost:image.raw.xz dest-image.raw.xz
- scp vm:image.raw.xz dest-image.raw.xz
- scp -P 5555 localhost:vmlinuz\* localhost:initr\* .
- scp vm:vmlinuz\* vm:initr\* .
# terminate qemu
- kill $(pgrep qemu) || true
......@@ -448,32 +449,6 @@
- cp /root/.ssh/id_rsa_target $buildmnt/root/.ssh/id_rsa
- cp /root/.ssh/id_rsa_target.pub $buildmnt/root/.ssh/id_rsa.pub
- |
cat > $buildmnt/app/start_vm_kernel.sh <<EOF
#!/bin/bash
set -x
KERNEL=\$1
shift
set -e
if [[ x"\$KERNEL" == x"" ]]
then
KERNEL=\$(ls /app/vmlinuz* | sort | tail -1)
fi
INITRD=\$(ls /app/initr* | sort | tail -1)
bash /app/start_vm.sh -kernel \$KERNEL \\
-initrd \$INITRD \\
-append "root=/dev/sda2 selinux=0 audit=0 rw console=tty0 console=ttyS0" \\
"\$@"
EOF
- chmod +x $buildmnt/app/start_vm_kernel.sh
# umount the container, not required, but, heh
- buildah unmount $buildcntr
......
......@@ -171,7 +171,7 @@
#
.fdo.container-build@alpine:
extends: .fdo.alpine
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24
stage: build
script:
# log in to the registry
......@@ -269,7 +269,7 @@
# See ``.fdo.container-build@alpine`` for details.
.fdo.container-build@alpine@arm64v8:
extends: .fdo.container-build@alpine
image: $CI_REGISTRY/freedesktop/ci-templates/arm64v8/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/arm64v8/buildah:2020-03-24
tags:
- aarch64
......
......@@ -171,7 +171,7 @@
#
.fdo.container-build@arch:
extends: .fdo.arch
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24
stage: build
script:
# log in to the registry
......
......@@ -177,7 +177,7 @@
#
.fdo.container-build@centos:
extends: .fdo.centos
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24
stage: build
script:
# log in to the registry
......
......@@ -177,7 +177,7 @@
#
.fdo.container-build@debian:
extends: .fdo.debian
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/buildah:2020-03-24
stage: build
script:
# log in to the registry
......@@ -284,7 +284,7 @@
# See ``.fdo.container-build@debian`` for details.
.fdo.container-build@debian@arm64v8:
extends: .fdo.container-build@debian
image: $CI_REGISTRY/freedesktop/ci-templates/arm64v8/buildah:2020-03-17
image: $CI_REGISTRY/freedesktop/ci-templates/arm64v8/buildah:2020-03-24
tags:
- aarch64
......@@ -300,7 +300,7 @@
extends: .fdo.container-build@debian
tags:
- kvm
image: $CI_REGISTRY/freedesktop/ci-templates/qemu-mkosi-base:2020-03-18
image: $CI_REGISTRY/freedesktop/ci-templates/qemu-mkosi-base:2020-03-24
script:
# log in to the registry
- podman login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
......@@ -310,7 +310,7 @@
- *fdo_distro_exists
# start our current base mkosi image
- /bin/bash /app/start_vm.sh -cdrom /app/my-seed.iso
- /bin/bash /app/vmctl start -cdrom /app/my-seed.iso
- QEMU_VERSION=$FDO_DISTRIBUTION_VERSION
......@@ -353,37 +353,38 @@
- mkdir -p mkosi.extra/etc/systemd/system/multi-user.target.wants
# send the mkosi files to the VM
- scp -P 5555 mkosi.default localhost:/root/mkosi.default
- scp -P 5555 -r mkosi.extra localhost:/root/
- scp mkosi.default vm:/root/mkosi.default
- scp mkosi.default vm:/root/mkosi.default
- scp -r mkosi.extra vm:/root/
# enable sshd on the target
- ssh localhost -p 5555 ln -s /usr/lib/systemd/system/sshd.service
- /app/vmctl exec ln -s /usr/lib/systemd/system/sshd.service
mkosi.extra/etc/systemd/system/multi-user.target.wants/sshd.service
# create a cache folder (useful only when manually testing this script)
- ssh localhost -p 5555 mkdir mkosi.cache
- /app/vmctl exec mkdir mkosi.cache
# run mkosi in the VM!
- ssh localhost -p 5555 mkosi/mkosi
- /app/vmctl exec mkosi/mkosi
# mount the root partition locally to extract the kernel and initramfs
- ssh localhost -p 5555 mkdir loop
- offset=$(ssh localhost -p 5555 fdisk -l image.raw | grep image.raw2 | cut -d ' ' -f 3)
- ssh localhost -p 5555 mount -o ro,loop,offset=$(($offset * 512)) image.raw loop/
- /app/vmctl exec mkdir loop
- offset=$(/app/vmctl exec fdisk -l image.raw | grep image.raw2 | cut -d ' ' -f 3)
- /app/vmctl exec mount -o ro,loop,offset=$(($offset * 512)) image.raw loop/
# fetch kernel and initramfs
- ssh localhost -p 5555 ls loop/boot/
- ssh localhost -p 5555 "cp loop/boot/vmlinuz* loop/boot/initr* ."
- /app/vmctl exec ls loop/boot/
- /app/vmctl exec "cp loop/boot/vmlinuz* loop/boot/initr* ."
- ssh localhost -p 5555 umount loop/
- /app/vmctl exec umount loop/
# now compress the image (we wanted to extract first the kernel and initrd)
- ssh localhost -p 5555 xz -T0 image.raw
- /app/vmctl exec xz -T0 image.raw
# fetch the image and kernel
- scp -P 5555 localhost:image.raw.xz dest-image.raw.xz
- scp vm:image.raw.xz dest-image.raw.xz
- scp -P 5555 localhost:vmlinuz\* localhost:initr\* .
- scp vm:vmlinuz\* vm:initr\* .
# terminate qemu
- kill $(pgrep qemu) || true
......@@ -397,7 +398,7 @@
graphroot = "/var/lib/containers/storage"
EOF
- QEMU_BASE_IMAGE=${QEMU_BASE_IMAGE:-$CI_REGISTRY/freedesktop/ci-templates/qemu-base:2020-03-18}
- QEMU_BASE_IMAGE=${QEMU_BASE_IMAGE:-$CI_REGISTRY/freedesktop/ci-templates/qemu-base:2020-03-24}
- echo Building $FDO_REPO_SUFFIX:$DISTRO_TAG from $QEMU_BASE_IMAGE
......@@ -413,32 +414,6 @@