gitlab-ci.tmpl 7.36 KB
Newer Older
1
2
3
stages:
  - sanity check
  - bootstrapping
4
  - distributions
5
  - post children pipelines
Peter Hutterer's avatar
Peter Hutterer committed
6
  - ci-fairy
7
  - deploy
8
  - publish
9
  - test published images
10
11
12


#
13
14
# We want those to fail as early as possible, so we use the most minimal image
# capable of running a python script.
15
#
16
17

.pip_install:
18
  stage: sanity check
19
  image: python:alpine
20
  before_script:
21
    - apk add git
22
23
24

sanity check:
  extends: .pip_install
25
  script:
26
27
    - pip3 install --user PyYAML
    - pip3 install --user .
28
29
30
31
32
33
    - python3 ./src/generate_templates.py

    - git diff --exit-code && exit 0 || true

    - echo "some files were not generated through 'src/generate_templates.py' or
      have not been committed. Please edit the files under 'src', run
Bastien Nocera's avatar
Bastien Nocera committed
34
      'src/generate_templates.py' and then commit the result"
35
36
37
38
    - exit 1


check commits:
39
  extends: .pip_install
40
  script:
41
42
    - pip3 install .
    - ci-fairy check-commits --signed-off-by --junit-xml=results.xml
43
  except:
44
    - master@freedesktop/ci-templates
45
46
  variables:
    GIT_DEPTH: 100
47
    GIT_STRATEGY: clone
48
49
50
51
  artifacts:
    reports:
      junit: results.xml

52
53
54
55
56
57
58
59
60
61
62
63

pytest ci-fairy:
  extends: .pip_install
  script:
    - pip3 install pytest
    - pip3 install .
    - pytest --junitxml=results.xml
  artifacts:
    reports:
      junit: results.xml


64
65
66
67
68
69
70
71
72
73
flake8 ci-fairy:
  extends: .pip_install
  script:
    - pip3 install flake8
    # 501: line too long
    # 504: line break after binary operator
    # 741: ambiguous variable name
    - flake8 --ignore=W501,E501,W504,W741,E741


74
75
76
#
# Build the base images that the various distribution templates rely on
#
77
78
79
80
81
82
83
bootstrapping:
  stage: bootstrapping
  trigger:
    include:
      - local: '/.gitlab-ci/bootstrap-ci.yml'
    strategy: depend

84
85
86
#
# Test the templates for each distribution
#
87
88
89
90
91
92
93
94
95
{% for distribution in distribs %}
{{distribution}}:
  stage: distributions
  trigger:
    include:
      - local: '/.gitlab-ci/{{distribution}}-ci.yml'
    strategy: depend

{% endfor %}
96

97
#
98
99
100
101
102
103
104
105
106
107
# This is a dummy job to work around a gitlab bug.
# All distribution jobs above run in child pipelines. If a job in a child
# pipeline fails, that pipeline fails and our parent pipeline is stuck.
# Re-triggering the failed job does not continue the parent pipeline even when
# that job succeeds on the second attempt.
#
# The dummy job here checks for the results of the child pipelines via the API.
# If they are successful, this job succeeds and our pipeline continues.
#
# This job needs to be re-run manually after all child pipelines are fixed.
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
#
check children pipelines:
  stage: post children pipelines
  image: alpine:latest
  before_script:
    - apk update
    - apk add jq curl
  script:
    - URL=https://gitlab.freedesktop.org/api/v4/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges
    - DATA=$(curl $URL)
    - echo $DATA | jq
    - CHILD_STATUSES=$(echo $DATA | jq -r '.[] | .downstream_pipeline.status' | sort -u | grep -v null)
    - echo $CHILD_STATUSES
    - '[ "$CHILD_STATUSES" == "success" ]'
  when: always

124
125
126
#
# Build the .fdo.ci-fairy base image
#
Peter Hutterer's avatar
Peter Hutterer committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
ci-fairy images:
  stage: ci-fairy
  trigger:
    include:
      - local: '/.gitlab-ci/ci-fairy-ci.yml'
    strategy: depend

.ci-fairy-tag:
  variables:
    FDO_DISTRIBUTION_TAG: {{ci_fairy.sha256sum(sha_source)}}

.ci-fairy-local-image:
  extends:
    - .ci-fairy-tag
  image: $CI_REGISTRY_IMAGE/ci-fairy:$FDO_DISTRIBUTION_TAG


check merge request:
  extends:
    - .ci-fairy-local-image
  stage: deploy
  script:
    - ci-fairy check-merge-request --require-allow-collaboration --junit-xml=check-merge-request.xml
  artifacts:
    expire_in: 1 week
    when: on_failure
    paths:
      - check-merge-request.xml
    reports:
      junit: check-merge-request.xml
  variables:
    FDO_UPSTREAM_REPO: freedesktop/ci-templates
  # We allow this to fail because no MR may have been filed yet
  allow_failure: true


Peter Hutterer's avatar
Peter Hutterer committed
163
164
pages:
  extends: .pip_install
165
  stage: deploy
Peter Hutterer's avatar
Peter Hutterer committed
166
  script:
167
168
169
170
  - pip3 install sphinx sphinx-rtd-theme
  # Upstream bug in the HTML rendering for YAML nodes so let's used the fixed version
  # until this gets merged
  - pip3 install git+https://github.com/whot/sphinxcontrib-autoyaml.git@wip/fix-definition-rendering
171
  - sh -x doc/build-docs.sh
172
  - mv build public
Peter Hutterer's avatar
Peter Hutterer committed
173
174
175
  artifacts:
    paths:
    - public
176

177
178
179
180
181
182
183

#
# Template to publish to quay.io
# Requires: LOCAL_IMAGE_SPEC and DISTANT_IMAGE_SPEC to be set to the respective image paths
#
.publish.template:
  stage: publish
184
  image: $CI_REGISTRY_IMAGE/buildah:{{bootstrap_tag}}
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
  script:
    - skopeo login --username "$QUAY_USER" --password "$(cat $QUAY_TOKEN)" {{ci_templates_registry}}
    - set -x
    - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$LOCAL_IMAGE_SPEC"
    - export DISTANT_IMAGE="{{ci_templates_registry}}{{ci_templates_registry_path}}:$DISTANT_IMAGE_SPEC"

    # fetch the local and distant digest and layers to
    # ensure the image is not already on the remote

    # disable exit on failure
    - set +e

    # we store the labels and the sha of the layers
    # we can not rely on the digest: the Name is different
    # but the labels are set with the commit ID and the pipeline/job
    # ids, which should ensure a correct match
    - skopeo inspect docker://$LOCAL_IMAGE | jq '[.Labels, .Layers]' > local_sha

    # we store the same information from the distant registry
    - skopeo inspect docker://$DISTANT_IMAGE | jq '[.Labels, .Layers]' > distant_sha

    # reenable exit on failure
    - set -e

    # if the distant repo has an image, ensure we use the same
    # and finish the job if so
    - diff distant_sha local_sha && exit 0 || true

    # copy the original image into the distant registry
    - skopeo copy docker://$LOCAL_IMAGE docker://$DISTANT_IMAGE

    - set +x
    - skopeo logout {{ci_templates_registry}}


220
221
222
#
# Publish the various images to the remote registry
#
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
publish to {{ci_templates_registry}}:
  extends:
  - .publish.template
  parallel:
    matrix:
      - FDO_DISTRIBUTION_IMAGE:
          - buildah
        SUFFIX:
          - ''
        FDO_ARCHITECTURE:
          - x86_64
          - aarch64
        FDO_DISTRIBUTION_TAG:
          - '{{bootstrap_tag}}'
      - FDO_DISTRIBUTION_IMAGE:
          - qemu
          - qemu-mkosi
        SUFFIX:
          - '-base'
        FDO_ARCHITECTURE:
          - x86_64
        FDO_DISTRIBUTION_TAG:
          - '{{qemu_tag}}'
  variables:
    # local format,   e.g. x86_64/mkosi-base-12324
    # distant format, e.g. mkosi-base-x86_64-12324
    LOCAL_IMAGE_SPEC:   $FDO_ARCHITECTURE/${FDO_DISTRIBUTION_IMAGE}${SUFFIX}:${FDO_DISTRIBUTION_TAG}
    DISTANT_IMAGE_SPEC: $FDO_DISTRIBUTION_IMAGE${SUFFIX}-$FDO_ARCHITECTURE-${FDO_DISTRIBUTION_TAG}
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'

Peter Hutterer's avatar
Peter Hutterer committed
254
255
256
257
258
259
260
261
262
263
264
265
#
# Publish our ci-fairy image on quay.io
#
publish ci-fairy to {{ci_templates_registry}}:
  extends:
    - .ci-fairy-tag
    - .publish.template
  variables:
    LOCAL_IMAGE_SPEC:   ci-fairy:$FDO_DISTRIBUTION_TAG
    DISTANT_IMAGE_SPEC: ci-fairy-$FDO_DISTRIBUTION_TAG
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'
266

267
268
269
270
271
#
# Verify that all {{ci_templates_registry}} images directly referenced by our templates
# exist
#
test published images:
272
  image: $CI_REGISTRY_IMAGE/buildah:{{bootstrap_tag}}
273
274
275
276
277
278
279
  stage: test published images
  script:
    {% for remote_image in remote_images %}
    - skopeo inspect docker://{{remote_image}}
    {% endfor %}
  rules:
    - if: '$CI_COMMIT_BRANCH == "master" && $CI_PROJECT_PATH == "freedesktop/ci-templates"'