.gitlab-ci.yml 8.22 KB
Newer Older
1
stages:
2
  - sanity check
3
  - bootstrapping
4
  - distributions
5
  - post children pipelines
Peter Hutterer's avatar
Peter Hutterer committed
6
  - ci-fairy
7
  - deploy
8
  - publish
9
  - test published images
10
11


12
#
13
14
# We want those to fail as early as possible, so we use the most minimal image
# capable of running a python script.
15
#
16
17

.pip_install:
18
  stage: sanity check
19
  image: python:alpine
20
  before_script:
21
    - apk add git
22
23
24

sanity check:
  extends: .pip_install
25
  script:
26
27
    - pip3 install --user PyYAML
    - pip3 install --user .
28
29
30
31
32
33
    - python3 ./src/generate_templates.py

    - git diff --exit-code && exit 0 || true

    - echo "some files were not generated through 'src/generate_templates.py' or
      have not been committed. Please edit the files under 'src', run
Bastien Nocera's avatar
Bastien Nocera committed
34
      'src/generate_templates.py' and then commit the result"
35
36
    - exit 1

37
38

check commits:
39
  extends: .pip_install
40
  script:
41
42
    - pip3 install .
    - ci-fairy check-commits --signed-off-by --junit-xml=results.xml
43
  except:
44
    - master@freedesktop/ci-templates
45
46
  variables:
    GIT_DEPTH: 100
47
    GIT_STRATEGY: clone
48
49
50
51
  artifacts:
    reports:
      junit: results.xml

52
53
54
55
56
57
58
59
60
61
62
63

pytest ci-fairy:
  extends: .pip_install
  script:
    - pip3 install pytest
    - pip3 install .
    - pytest --junitxml=results.xml
  artifacts:
    reports:
      junit: results.xml


64
65
66
67
68
69
70
71
72
73
flake8 ci-fairy:
  extends: .pip_install
  script:
    - pip3 install flake8
    # 501: line too long
    # 504: line break after binary operator
    # 741: ambiguous variable name
    - flake8 --ignore=W501,E501,W504,W741,E741


74
75
76
#
# Build the base images that the various distribution templates rely on
#
77
78
79
80
81
82
83
bootstrapping:
  stage: bootstrapping
  trigger:
    include:
      - local: '/.gitlab-ci/bootstrap-ci.yml'
    strategy: depend

84
85
86
#
# Test the templates for each distribution
#
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
alpine:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/alpine-ci.yml'
    strategy: depend

arch:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/arch-ci.yml'
    strategy: depend

centos:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/centos-ci.yml'
    strategy: depend

debian:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/debian-ci.yml'
    strategy: depend

fedora:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/fedora-ci.yml'
    strategy: depend

ubuntu:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/ubuntu-ci.yml'
    strategy: depend

129

130
#
131
132
133
134
135
136
137
138
139
140
# This is a dummy job to work around a gitlab bug.
# All distribution jobs above run in child pipelines. If a job in a child
# pipeline fails, that pipeline fails and our parent pipeline is stuck.
# Re-triggering the failed job does not continue the parent pipeline even when
# that job succeeds on the second attempt.
#
# The dummy job here checks for the results of the child pipelines via the API.
# If they are successful, this job succeeds and our pipeline continues.
#
# This job needs to be re-run manually after all child pipelines are fixed.
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
#
check children pipelines:
  stage: post children pipelines
  image: alpine:latest
  before_script:
    - apk update
    - apk add jq curl
  script:
    - URL=https://gitlab.freedesktop.org/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges
    - DATA=$(curl $URL)
    - echo $DATA | jq
    - CHILD_STATUSES=$(echo $DATA | jq -r '.[] | .downstream_pipeline.status' | sort -u | grep -v null)
    - echo $CHILD_STATUSES
    - '[ "$CHILD_STATUSES" == "success" ]'
  when: always

157
158
159
#
# Build the .fdo.ci-fairy base image
#
Peter Hutterer's avatar
Peter Hutterer committed
160
161
162
163
164
165
166
167
168
ci-fairy images:
  stage: ci-fairy
  trigger:
    include:
      - local: '/.gitlab-ci/ci-fairy-ci.yml'
    strategy: depend

.ci-fairy-tag:
  variables:
169
    FDO_DISTRIBUTION_TAG: sha256-b76681d87ea9b06907f7828206a1f99d06a9821dc86c797768736b28509e32b5
Peter Hutterer's avatar
Peter Hutterer committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195

.ci-fairy-local-image:
  extends:
    - .ci-fairy-tag
  image: $CI_REGISTRY_IMAGE/ci-fairy:$FDO_DISTRIBUTION_TAG


check merge request:
  extends:
    - .ci-fairy-local-image
  stage: deploy
  script:
    - ci-fairy check-merge-request --require-allow-collaboration --junit-xml=check-merge-request.xml
  artifacts:
    expire_in: 1 week
    when: on_failure
    paths:
      - check-merge-request.xml
    reports:
      junit: check-merge-request.xml
  variables:
    FDO_UPSTREAM_REPO: freedesktop/ci-templates
  # We allow this to fail because no MR may have been filed yet
  allow_failure: true


Peter Hutterer's avatar
Peter Hutterer committed
196
197
pages:
  extends: .pip_install
198
  stage: deploy
Peter Hutterer's avatar
Peter Hutterer committed
199
  script:
200
201
202
203
  - pip3 install sphinx sphinx-rtd-theme
  # Upstream bug in the HTML rendering for YAML nodes so let's used the fixed version
  # until this gets merged
  - pip3 install git+https://github.com/whot/sphinxcontrib-autoyaml.git@wip/fix-definition-rendering
204
  - sh -x doc/build-docs.sh
205
  - mv build public
Peter Hutterer's avatar
Peter Hutterer committed
206
207
208
  artifacts:
    paths:
    - public
209

210
211
212
213
214
215
216

#
# Template to publish to quay.io
# Requires: LOCAL_IMAGE_SPEC and DISTANT_IMAGE_SPEC to be set to the respective image paths
#
.publish.template:
  stage: publish
217
  image: $CI_REGISTRY_IMAGE/x86_64/buildah:2020-12-18.2
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
  script:
    - skopeo login --username "$QUAY_USER" --password "$(cat $QUAY_TOKEN)" quay.io
    - set -x
    - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$LOCAL_IMAGE_SPEC"
    - export DISTANT_IMAGE="quay.io/freedesktop.org/ci-templates:$DISTANT_IMAGE_SPEC"

    # fetch the local and distant digest and layers to
    # ensure the image is not already on the remote

    # disable exit on failure
    - set +e

    # we store the labels and the sha of the layers
    # we can not rely on the digest: the Name is different
    # but the labels are set with the commit ID and the pipeline/job
    # ids, which should ensure a correct match
    - skopeo inspect docker://$LOCAL_IMAGE | jq '[.Labels, .Layers]' > local_sha

    # we store the same information from the distant registry
    - skopeo inspect docker://$DISTANT_IMAGE | jq '[.Labels, .Layers]' > distant_sha

    # reenable exit on failure
    - set -e

    # if the distant repo has an image, ensure we use the same
    # and finish the job if so
    - diff distant_sha local_sha && exit 0 || true

    # copy the original image into the distant registry
    - skopeo copy docker://$LOCAL_IMAGE docker://$DISTANT_IMAGE

    - set +x
    - skopeo logout quay.io


253
254
255
#
# Publish the various images to the remote registry
#
256
257
258
259
260
261
262
263
264
265
266
267
268
publish to quay.io:
  extends:
  - .publish.template
  parallel:
    matrix:
      - FDO_DISTRIBUTION_IMAGE:
          - buildah
        SUFFIX:
          - ''
        FDO_ARCHITECTURE:
          - x86_64
          - aarch64
        FDO_DISTRIBUTION_TAG:
269
          - '2020-12-18.2'
270
271
272
273
274
275
276
277
      - FDO_DISTRIBUTION_IMAGE:
          - qemu
          - qemu-mkosi
        SUFFIX:
          - '-base'
        FDO_ARCHITECTURE:
          - x86_64
        FDO_DISTRIBUTION_TAG:
278
          - '2021-02-15.0'
279
280
281
282
283
284
285
286
  variables:
    # local format,   e.g. x86_64/mkosi-base-12324
    # distant format, e.g. mkosi-base-x86_64-12324
    LOCAL_IMAGE_SPEC:   $FDO_ARCHITECTURE/${FDO_DISTRIBUTION_IMAGE}${SUFFIX}:${FDO_DISTRIBUTION_TAG}
    DISTANT_IMAGE_SPEC: $FDO_DISTRIBUTION_IMAGE${SUFFIX}-$FDO_ARCHITECTURE-${FDO_DISTRIBUTION_TAG}
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'

Peter Hutterer's avatar
Peter Hutterer committed
287
288
289
290
291
292
293
294
295
296
297
298
#
# Publish our ci-fairy image on quay.io
#
publish ci-fairy to quay.io:
  extends:
    - .ci-fairy-tag
    - .publish.template
  variables:
    LOCAL_IMAGE_SPEC:   ci-fairy:$FDO_DISTRIBUTION_TAG
    DISTANT_IMAGE_SPEC: ci-fairy-$FDO_DISTRIBUTION_TAG
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'
299

300
301
302
303
304
#
# Verify that all quay.io images directly referenced by our templates
# exist
#
test published images:
305
  image: $CI_REGISTRY_IMAGE/x86_64/buildah:2020-12-18.2
306
307
  stage: test published images
  script:
308
309
    - skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-aarch64-2020-12-18.2
    - skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-x86_64-2020-12-18.2
310
    - skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-b76681d87ea9b06907f7828206a1f99d06a9821dc86c797768736b28509e32b5
311
312
    - skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-base-x86_64-2021-02-15.0
    - skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-mkosi-base-x86_64-2021-02-15.0
313
  rules:
314
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'