.gitlab-ci.yml 15.3 KB
Newer Older
Guillaume Desmottes's avatar
Guillaume Desmottes committed
1
2
3
# We use https://gitlab.freedesktop.org/freedesktop/ci-templates
# to build the images used by the ci.
#
4
# Here is how to properly update those images:
5
# - new Rust stable version: update GST_RS_IMG_TAG and update Rust version
6
7
# - add dependencies: update FDO_DISTRIBUTION_PACKAGES and update GST_RS_IMG_TAG
# - update GStreamer version: update the tag in ci/install-gst.sh and update GST_RS_IMG_TAG
Guillaume Desmottes's avatar
Guillaume Desmottes committed
8
#
9
10
# GST_RS_IMG_TAG is defined in ci/images_template.yml and should be updated
# either by:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
11
12
13
# - setting it to the current date and the version suffix to 0
# - incrementing the version suffix
#
14
# After each update commit your changes and push to your personal repo.
Guillaume Desmottes's avatar
Guillaume Desmottes committed
15
16
17
18
19
# After review and ci approval merge the branch as usual.
#
# Updating the nightly image should be done by simply running a scheduled ci
# pipeline on the upstream repo with the $UPDATE_NIGHTLY variable defined.

20
.templates_sha: &templates_sha 567700e483aabed992d0a4fea84994a0472deff6
Guillaume Desmottes's avatar
Guillaume Desmottes committed
21
22
23
24
25
26

include:
  - project: 'freedesktop/ci-templates'
    ref: *templates_sha
    file: '/templates/debian.yml'

27
28
  - local: "ci/images_template.yml"

29
30
31
32
33
34
35
36
37
38
workflow:
  rules:
    - if: $CI_MERGE_REQUEST_IID
    # don't create a pipeline if its a commit pipeline, on a branch and that branch has
    # open merge requests (bc we will get a MR build instead)
    - if: $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

Guillaume Desmottes's avatar
Guillaume Desmottes committed
39
40
variables:
  FDO_UPSTREAM_REPO: gstreamer/gstreamer-rs
Sophie Herold's avatar
Sophie Herold committed
41
  RUST_DOCS_FLAGS: "--extern-html-root-url=muldiv=https://docs.rs/muldiv/1.0.0/muldiv/ -Z unstable-options"
42
  NAMESPACE: gstreamer
Sophie Herold's avatar
Sophie Herold committed
43
44
45
46
47
  # format is <branch>=<name>
  # the name is used in the URL
  # latest release must be at the top
  # (only relevant on master branch)
  RELEASES:
Sebastian Dröge's avatar
Sebastian Dröge committed
48
    0.18=0.18
49
    0.17=0.17
Sophie Herold's avatar
Sophie Herold committed
50
    0.16=0.16
Guillaume Desmottes's avatar
Guillaume Desmottes committed
51

52
stages:
53
54
  - "container-base"
  - "container-final"
Jordan Petridіs's avatar
Jordan Petridіs committed
55
  - "lint"
56
  - "test"
Jordan Petridіs's avatar
Jordan Petridіs committed
57
  - "extras"
58
  - "deploy"
59

60

61
.debian:11:
62
  variables:
63
    FDO_DISTRIBUTION_VERSION: 'bullseye-slim'
64
  before_script:
65
66
67
68
69
70
    - source ./ci/env.sh
    - mkdir .cargo && echo -e "[net]\ngit-fetch-with-cli = true" > .cargo/config
    # If cargo exists assume we probably will want to update
    # the lockfile
    - |
      if command -v cargo; then
71
72
        cargo generate-lockfile --color=always
        cargo update --color=always
73
      fi
Guillaume Desmottes's avatar
Guillaume Desmottes committed
74

75
76
.debian:11-base:
  extends: .debian:11
77
78
79
  variables:
    FDO_DISTRIBUTION_TAG: 'base-$GST_RS_IMG_TAG'

80
81
.debian:11-stable:
  extends: .debian:11
Guillaume Desmottes's avatar
Guillaume Desmottes committed
82
  variables:
83
    RUST_IMAGE_FULL: "1"
84
85
    FDO_DISTRIBUTION_TAG: '$GST_RS_STABLE-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_STABLE $RUST_IMAGE_FULL'
86

87
88
.debian:11-msrv:
  extends: .debian:11
Guillaume Desmottes's avatar
Guillaume Desmottes committed
89
  variables:
90
91
    FDO_DISTRIBUTION_TAG: '$GST_RS_MSRV-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh $GST_RS_MSRV $RUST_IMAGE_FULL'
Guillaume Desmottes's avatar
Guillaume Desmottes committed
92

93
94
.debian:11-nightly:
  extends: .debian:11
Guillaume Desmottes's avatar
Guillaume Desmottes committed
95
  variables:
96
97
    FDO_DISTRIBUTION_TAG: 'nightly-$GST_RS_IMG_TAG'
    FDO_DISTRIBUTION_EXEC: 'bash ci/install-rust.sh nightly $RUST_IMAGE_FULL'
98

99
.build-base-image:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
100
101
  extends:
  - .fdo.container-build@debian
102
  stage: container-base
Guillaume Desmottes's avatar
Guillaume Desmottes committed
103
  variables:
Jordan Petridіs's avatar
Jordan Petridіs committed
104
105
106
107
108
    FDO_DISTRIBUTION_PACKAGES: "build-essential curl python3-setuptools liborc-0.4-dev libglib2.0-dev libxml2-dev libgtk-3-dev libegl1-mesa libgles2-mesa libgl1-mesa-dri libgl1-mesa-glx libwayland-egl1-mesa xz-utils libssl-dev git wget ca-certificates ninja-build python3-pip flex bison libglib2.0-dev libx11-dev libx11-xcb-dev libsoup2.4-dev libvorbis-dev libogg-dev libtheora-dev libmatroska-dev libvpx-dev libopus-dev libgraphene-1.0-dev"
    FDO_DISTRIBUTION_EXEC: >-
      bash ci/install-gst.sh &&
      bash ci/install-gtk4.sh &&
      pip3 install git+http://gitlab.freedesktop.org/freedesktop/ci-templates
109
110
111
112
113
114

.build-final-image:
  extends:
  - .fdo.container-build@debian
  stage: container-final
  variables:
115
    FDO_BASE_IMAGE: '$CI_REGISTRY_IMAGE/debian/bullseye-slim:base-$GST_RS_IMG_TAG'
116
117
118
119

build-base:
  extends:
    - .build-base-image
120
    - .debian:11-base
Guillaume Desmottes's avatar
Guillaume Desmottes committed
121
122
123

build-stable:
  extends:
124
    - .build-final-image
125
    - .debian:11-stable
Guillaume Desmottes's avatar
Guillaume Desmottes committed
126

127
build-msrv:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
128
  extends:
129
    - .build-final-image
130
    - .debian:11-msrv
Guillaume Desmottes's avatar
Guillaume Desmottes committed
131
132
133

build-nightly:
  extends:
134
    - .build-final-image
135
    - .debian:11-nightly
Guillaume Desmottes's avatar
Guillaume Desmottes committed
136
137
138
139
140
141
142
143

update-nightly:
  extends: build-nightly
  only:
    variables:
      - $UPDATE_NIGHTLY == "1"
  variables:
      FDO_FORCE_REBUILD: 1
144

Guillaume Desmottes's avatar
Guillaume Desmottes committed
145
146
147
.dist-debian-container:
  extends:
  - .fdo.distribution-image@debian
148
149
  after_script:
    - rm -rf target
150

Guillaume Desmottes's avatar
Guillaume Desmottes committed
151
152
.img-stable:
  extends:
153
    - .debian:11-stable
154
    - .dist-debian-container
Guillaume Desmottes's avatar
Guillaume Desmottes committed
155

156
.img-msrv:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
157
  extends:
158
    - .debian:11-msrv
159
    - .dist-debian-container
Guillaume Desmottes's avatar
Guillaume Desmottes committed
160
161
162

.img-nightly:
  extends:
163
    - .debian:11-nightly
164
    - .dist-debian-container
Guillaume Desmottes's avatar
Guillaume Desmottes committed
165

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
# GST_PLUGINS_RS_TOKEN is a variable of type 'Var' defined in gstreamer-rs CI
# settings and containing a gst-plugins-rs pipeline trigger token
.plugins-update:
  stage: deploy
  script:
    - |
    # FDO_DISTRIBUTION_IMAGE still has indirections
    - echo $FDO_DISTRIBUTION_IMAGE
    - DISTRO_IMAGE=$(eval echo ${FDO_DISTRIBUTION_IMAGE})
    - echo $DISTRO_IMAGE
    # retrieve the infos from the registry
    - JSON_IMAGE=$(skopeo inspect docker://$DISTRO_IMAGE)
    - IMAGE_PIPELINE_ID=$(echo $JSON_IMAGE | jq -r '.Labels["fdo.pipeline_id"]')
    - echo $IMAGE_PIPELINE_ID
    - echo $CI_PIPELINE_ID
    - |
      if [[ x"$IMAGE_PIPELINE_ID" == x"$CI_PIPELINE_ID" ]]; then
        echo "Image has been updated, notify gst-plugins-rs"
184
        curl -X POST -F "token=$GST_PLUGINS_RS_TOKEN" -F "ref=master" -F "variables[UPDATE_IMG]=$UPDATE_IMG" https://gitlab.freedesktop.org/api/v4/projects/1400/trigger/pipeline
185
186
187
188
189
190
191
192
193
194
195
196
      else
        echo "Image has not been updated, ignore"
      fi
  rules:
    - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PROJECT_PATH == "gstreamer/gstreamer-rs"'

# Those jobs need to use another image as ours doesn't have 'skopeo'
# and it's not easily installable in Debian stable for now.
plugins-update-stable:
  extends:
    - .plugins-update
    - .img-stable
197
  image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
198
199
200
  variables:
    UPDATE_IMG: "stable"

201
plugins-update-msrv:
202
203
  extends:
    - .plugins-update
204
    - .img-msrv
205
  image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
206
  variables:
207
    UPDATE_IMG: "msrv"
208
209
210
211
212

plugins-update-nightly:
  extends:
    - .plugins-update
    - .img-nightly
213
  image: quay.io/freedesktop.org/ci-templates:container-build-base-2021-07-29.0
214
215
216
  variables:
    UPDATE_IMG: "nightly"

Guillaume Desmottes's avatar
Guillaume Desmottes committed
217
.cargo_test_var: &cargo_test
218
    - rustc --version
219
220
221
      # First build and test all the crates with their relevant features
      # Keep features in sync with below
    - |
222
223
224
      get_features() {
        crate=$1
        if [ "$crate" = "gstreamer" ]; then
225
          echo "--features=ser_de,v1_20"
226
        else
227
          echo "--features=v1_20"
228
229
        fi
      }
230
      for crate in gstreamer* gstreamer-gl/{egl,wayland,x11}; do
231
232
        if [ -e $crate/Cargo.toml ]; then
          if [ -n "$ALL_FEATURES" ]; then
233
            FEATURES="$(get_features $crate)"
234
          else
235
            FEATURES=""
236
          fi
237
238
239
240
241

          echo "Building and testing $crate with $FEATURES"

          cargo build --locked --color=always --manifest-path $crate/Cargo.toml $FEATURES
          G_DEBUG=fatal_warnings cargo test --color=always --manifest-path $crate/Cargo.toml $FEATURES
242
243
        fi
      done
244
245

    - |
246
      if [ -n "$EXAMPLES_TUTORIALS" ]; then
247
248
        cargo build --locked --color=always --manifest-path examples/Cargo.toml --bins --examples --all-features
        cargo build --locked --color=always --manifest-path tutorials/Cargo.toml --bins --examples --all-features
249
      fi
250

Guillaume Desmottes's avatar
Guillaume Desmottes committed
251
252
253
254
255
.cargo test:
  stage: "test"
  script:
    - *cargo_test

256
test msrv:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
257
258
  extends:
    - '.cargo test'
259
    - .img-msrv
260
261

test stable:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
262
263
264
  extends:
    - '.cargo test'
    - .img-stable
265
266

test stable all-features:
267
268
  variables:
    ALL_FEATURES: 'yes'
269
    EXAMPLES_TUTORIALS: 'yes'
Guillaume Desmottes's avatar
Guillaume Desmottes committed
270
271
272
  extends:
    - '.cargo test'
    - .img-stable
273
274

test nightly:
275
  allow_failure: true
Guillaume Desmottes's avatar
Guillaume Desmottes committed
276
277
278
  extends:
    - '.cargo test'
    - .img-nightly
279
280

test nightly all-features:
281
  allow_failure: true
282
  variables:
283
    ALL_FEATURES: 'yes'
284
    EXAMPLES_TUTORIALS: 'yes'
Guillaume Desmottes's avatar
Guillaume Desmottes committed
285
286
287
  extends:
    - '.cargo test'
    - .img-nightly
Jordan Petridіs's avatar
Jordan Petridіs committed
288

289
290
291
292
.cargo test sys:
  stage: "test"
  script:
    - rustc --version
293
294
295
    - |
      get_features() {
        module=${1%%/sys}
296
        echo "--features=v1_20"
297
      }
298
299
      # First build and test all the crates with their relevant features
      # Keep features in sync with below
300
      for crate in gstreamer*/sys gstreamer-gl/*/sys; do
301
        if [ -e $crate/Cargo.toml ]; then
302
          echo "Building $crate with $(get_features $crate)"
303
          cargo build --locked --color=always --manifest-path $crate/Cargo.toml $(get_features $crate)
304
305
        fi
      done
306
307
308
309
310
311
312
313
      # Run tests for crates we can currently run.
      # Other tests are broken currently.
      for crate in gstreamer/sys \
                   gstreamer-app/sys \
                   gstreamer-audio/sys \
                   gstreamer-base/sys \
                   gstreamer-check/sys \
                   gstreamer-controller/sys \
314
315
316
317
                   gstreamer-gl/sys \
                   gstreamer-gl/egl/sys \
                   gstreamer-gl/wayland/sys \
                   gstreamer-gl/x11/sys \
318
319
320
321
322
323
324
325
326
327
328
                   gstreamer-mpegts/sys \
                   gstreamer-net/sys \
                   gstreamer-pbutils/sys \
                   gstreamer-player/sys \
                   gstreamer-rtsp-server/sys \
                   gstreamer-rtsp/sys \
                   gstreamer-sdp/sys \
                   gstreamer-tag/sys \
                   gstreamer-video/sys \
                   gstreamer-webrtc/sys; do
        echo "Testing $crate with $(get_features $crate)"
329
        cargo test --locked --color=always --manifest-path $crate/Cargo.toml $(get_features $crate)
330
      done
331
332
333
334
335
336

test stable sys:
  extends:
    - '.cargo test sys'
    - .img-stable

337
test msrv sys:
338
339
  extends:
    - '.cargo test sys'
340
    - .img-msrv
341
342
343
344
345
346

test nightly sys:
  extends:
    - '.cargo test sys'
    - .img-nightly

Jordan Petridіs's avatar
Jordan Petridіs committed
347
rustfmt:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
348
  extends: .img-stable
Jordan Petridіs's avatar
Jordan Petridіs committed
349
350
351
352
  stage: "lint"
  script:
    - cargo fmt --version
    - cargo fmt -- --color=always --check
Jordan Petridіs's avatar
Jordan Petridіs committed
353

354
355
356
357
358
359
check commits:
  extends: .img-stable
  stage: "lint"
  script:
    - ci-fairy check-commits --textwidth 0 --no-signed-off-by

Jordan Petridіs's avatar
Jordan Petridіs committed
360
clippy:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
361
  extends: .img-stable
Jordan Petridіs's avatar
Jordan Petridіs committed
362
  stage: 'extras'
363
  variables:
Guillaume Gomez's avatar
Guillaume Gomez committed
364
    CLIPPY_LINTS: -D warnings
Jordan Petridіs's avatar
Jordan Petridіs committed
365
366
  script:
    - cargo clippy --version
367
368
      # Keep features in sync with above
    - |
369
370
371
      get_features() {
        crate=$1
        if [ "$crate" = "gstreamer" ]; then
372
          echo "--features=ser_de,v1_20"
373
        else
374
          echo "--features=v1_20"
375
376
        fi
      }
377
      for crate in gstreamer* gstreamer-gl/{egl,wayland,x11}; do
378
        if [ -e $crate/Cargo.toml ]; then
379
380
381
          FEATURES=$(get_features $crate)

          echo "Running clippy on $crate with $FEATURES"
382

383
          cargo clippy --locked --color=always --manifest-path $crate/Cargo.toml $FEATURES --all-targets -- $CLIPPY_LINTS
384
        fi
385
      done
386
387
    # And also run over all the examples/tutorials
    - |
388
389
      cargo clippy --locked --color=always --manifest-path examples/Cargo.toml --all-targets --all-features -- $CLIPPY_LINTS
      cargo clippy --locked --color=always --manifest-path tutorials/Cargo.toml --all-targets --all-features -- $CLIPPY_LINTS
Guillaume Desmottes's avatar
Guillaume Desmottes committed
390

391
deny:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
392
  extends: .img-stable
Guillaume Desmottes's avatar
Guillaume Desmottes committed
393
394
395
396
  stage: 'extras'
  only:
    - schedules
  script:
397
    - cargo deny --color=always check
398

399
gir-checks:
400
401
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
Guillaume Desmottes's avatar
Guillaume Desmottes committed
402
  extends: .img-stable
403
404
  stage: 'extras'
  script:
405
    - git diff --exit-code
406
    - git clone --depth 1 https://github.com/gtk-rs/checker
407
    - git diff --exit-code
408
    - cd checker && echo '[workspace]' >> Cargo.toml
409
    - cd .. && git diff --exit-code && cd checker
410
    - cargo build --locked --color=always --release
411
    - cd .. && git diff --exit-code && cd checker
412
    - |
Guillaume Gomez's avatar
Guillaume Gomez committed
413
      cargo run --color=always --release -- ../gstreamer* ../gstreamer-gl/{egl,wayland,x11}
414
    - cd .. && git diff --exit-code && cd checker
Guillaume Gomez's avatar
Guillaume Gomez committed
415
416
417
418
419
420
    # Check doc aliases
    - |
      for crate in ../gstreamer* ../gstreamer-gl/{egl,wayland,x11}; do
        echo '--> Checking doc aliases in ' $crate
        python3 doc_aliases.py $crate
      done
421
    - cd ..
Guillaume Gomez's avatar
Guillaume Gomez committed
422
423
    # To ensure that there was no missing #[doc(alias = "...")]
    - git diff --exit-code
424
    - |
425
      for crate in gstreamer* gstreamer-gl/{egl,wayland,x11}; do
426
        echo '-->' $crate
427
        ./checker/check_init_asserts $crate
428
      done
429
    - git diff --exit-code
430

431
outdated:
Guillaume Desmottes's avatar
Guillaume Desmottes committed
432
  extends: .img-stable
433
434
435
436
  stage: 'extras'
  only:
    - schedules
  script:
437
    - cargo outdated --color=always --root-deps-only --exit-code 1 -v
438

Guillaume Desmottes's avatar
Guillaume Desmottes committed
439
coverage:
440
  allow_failure: true
Guillaume Desmottes's avatar
Guillaume Desmottes committed
441
442
443
444
445
446
447
448
449
450
451
  extends:
    - '.cargo test'
    - .img-nightly
  stage: 'extras'
  variables:
    ALL_FEATURES: 'yes'
    RUSTFLAGS: "-Zinstrument-coverage"
    LLVM_PROFILE_FILE: "gstreamer-rs-%p-%m.profraw"
  script:
    - *cargo_test
    # generate html report
452
    - grcov . --binary-path ./target/debug/ -s . -t html --branch --ignore-not-existing --ignore "*target*" --ignore "*/sys/*" --ignore "examples/*" --ignore "tutorials/*" --ignore "*/build.rs" -o ./coverage/
Guillaume Desmottes's avatar
Guillaume Desmottes committed
453
    # generate cobertura report for gitlab integration
454
    - grcov . --binary-path ./target/debug/ -s . -t cobertura --branch --ignore-not-existing --ignore "*target*" --ignore "*/sys/*" --ignore "examples/*" --ignore "tutorials/*" --ignore "*/build.rs" -o coverage.xml
455
456
457
    # output coverage summary for gitlab parsing.
    # TODO: use grcov once https://github.com/mozilla/grcov/issues/556 is fixed
    - grep % coverage/index.html | head -1 ; true
Guillaume Desmottes's avatar
Guillaume Desmottes committed
458
459
460
461
462
463
  artifacts:
    paths:
      - 'coverage'
    reports:
      cobertura: coverage.xml

464
465
466
467
468
469
doc-stripping:
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
  extends: .img-nightly
  stage: 'extras'
  script:
470
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --strip-docs --no-fmt
471
472
    - git diff --quiet || (echo 'Files changed after running `rustdoc-stripper -s`, make sure all documentation is protected with `// rustdoc-stripper-ignore-next`!'; git diff; false)

473
docs:
474
475
  variables:
    GIT_SUBMODULE_STRATEGY: recursive
476
477
478
  extends: .img-nightly
  stage: 'extras'
  script:
479
480
481
    - curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
        https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
    - chmod +x gir-rustdoc.py
482
    - PATH=~/.cargo/bin/:$PATH ./generator.py --gir-files-directories gir-files gst-gir-files --embed-docs --no-fmt
483
    - |
Sophie Herold's avatar
Sophie Herold committed
484
      RUSTDOCFLAGS="$RUST_DOCS_FLAGS"
485
      eval $(./gir-rustdoc.py pre-docs)
Sophie Herold's avatar
Sophie Herold committed
486
487
      cargo +nightly doc --workspace --exclude examples --exclude tutorials --color=always --features=dox --no-deps
    - mv target/doc docs
488
489
  artifacts:
    paths:
Sophie Herold's avatar
Sophie Herold committed
490
      - 'docs'
491

492
493
494
495
# https://docs.gitlab.com/ee/user/project/pages/#how-it-works
# GitLab automatically deploys the `public/` folder from an
# artifact generated by the job named `pages`.  This step
# re-uses the docs from the build-test `docs` step above.
496
pages:
Sophie Herold's avatar
Sophie Herold committed
497
  extends: .img-nightly
498
  stage: 'deploy'
499
500
501
  dependencies:
    - docs
  script:
502
503
504
505
    - curl --proto '=https' --tlsv1.2 -sSf -o gir-rustdoc.py
        https://gitlab.gnome.org/World/Rust/gir-rustdoc/-/raw/main/gir-rustdoc.py
    - chmod +x gir-rustdoc.py
    - ./gir-rustdoc.py html-index
Sophie Herold's avatar
Sophie Herold committed
506
507
508
509
    # development docs
    - mkdir public/git
    - mv docs public/git/docs
    # stable docs
510
    - ./gir-rustdoc.py docs-from-artifacts
511
    - ls public/
512
513
514
  artifacts:
    paths:
      - 'public'
Sophie Herold's avatar
Sophie Herold committed
515
  rules:
516
    - if: ($CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH) && ($CI_PROJECT_NAMESPACE == $NAMESPACE)
Sophie Herold's avatar
Sophie Herold committed
517
      when: 'manual'