ci_template.yml 42.7 KB
Newer Older
1
stages:
2
  - 'build docker'
3
  - 'preparation'
Matthew Waters's avatar
Matthew Waters committed
4
  - 'pre-build'
5 6
  - 'build'
  - 'test'
Matthew Waters's avatar
Matthew Waters committed
7 8
  # Use the resulting binaries
  - 'integrate'
9 10

variables:
11
  # Branch to track for modules that have no ref specified in the manifest
12
  GST_UPSTREAM_BRANCH: '1.18'
13
  ORC_UPSTREAM_BRANCH: 'master'
14 15 16 17 18 19 20

  ###
  # IMPORTANT
  # These are the version tags for the docker images the CI runs against.
  # If you are hacking on them or need a them to rebuild, its enough
  # to change any part of the string of the image you want.
  ###
21
  FEDORA_TAG:  '2020-10-30.0'
22 23 24 25 26
  CERBERO_TAG: '2020-10-22.0'
  ANDROID_TAG: '2020-10-22.0'
  MANIFEST_TAG: '2020-10-22.0'
  TEST_MANIFEST_TAG: '2020-10-22.0'
  INDENT_TAG: '2020-10-22.0'
27 28 29 30 31 32 33 34

  GST_UPSTREAM_REPO: 'gstreamer/gst-ci'

  FEDORA_AMD64_SUFFIX:  'amd64/fedora'
  CERBERO_AMD64_SUFFIX: 'amd64/cerbero-fedora'
  ANDROID_AMD64_SUFFIX: 'amd64/android-fedora'
  MANIFEST_AMD64_SUFFIX: 'amd64/build-manifest'
  TEST_MANIFEST_AMD64_SUFFIX: 'amd64/test-manifest'
35
  INDENT_AMD64_SUFFIX: 'amd64/gst-indent'
36 37

  FEDORA_IMAGE:   "$CI_REGISTRY_IMAGE/$FEDORA_AMD64_SUFFIX:$FEDORA_TAG-$GST_UPSTREAM_BRANCH"
38
  FEDORA_DOCS_IMAGE: "registry.freedesktop.org/gstreamer/gst-ci/amd64/fedora:2020-07-03.0-master"
39 40 41 42
  CERBERO_IMAGE:  "$CI_REGISTRY_IMAGE/$CERBERO_AMD64_SUFFIX:$CERBERO_TAG-$GST_UPSTREAM_BRANCH"
  ANDROID_IMAGE:  "$CI_REGISTRY_IMAGE/$ANDROID_AMD64_SUFFIX:$ANDROID_TAG-$GST_UPSTREAM_BRANCH"
  MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$MANIFEST_AMD64_SUFFIX:$MANIFEST_TAG-$GST_UPSTREAM_BRANCH"
  TEST_MANIFEST_IMAGE: "$CI_REGISTRY_IMAGE/$TEST_MANIFEST_AMD64_SUFFIX:$TEST_MANIFEST_TAG-$GST_UPSTREAM_BRANCH"
43
  INDENT_IMAGE:   "$CI_REGISTRY_IMAGE/$INDENT_AMD64_SUFFIX:$INDENT_TAG-$GST_UPSTREAM_BRANCH"
44

45
  WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gst-ci/amd64/windows:v14-1.18'
46

47
  GIT_STRATEGY: none
48
  MESON_BUILDTYPE_ARGS: --default-library=both
Xavier Claessens's avatar
Xavier Claessens committed
49
  DEFAULT_MESON_ARGS: >
50 51 52 53
    -Dlibnice:tests=disabled
    -Dlibnice:examples=disabled
    -Dopenh264:tests=disabled
    -Dpygobject:tests=false
Xavier Claessens's avatar
Xavier Claessens committed
54 55 56 57 58 59 60 61 62
    -Dpython=enabled
    -Dlibav=enabled
    -Dugly=enabled
    -Dbad=enabled
    -Ddevtools=enabled
    -Dges=enabled
    -Drtsp_server=enabled
    -Dvaapi=enabled
    -Dsharp=disabled
63

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
  MESON_GST_WERROR: >
    -Dgstreamer:werror=true
    -Dgst-plugins-base:werror=true
    -Dgst-plugins-good:werror=true
    -Dgst-plugins-ugly:werror=true
    -Dgst-plugins-bad:werror=true
    -Dgst-rtsp-server:werror=true
    -Dgst-libav:werror=true
    -Dgst-examples:werror=true
    -Dgst-editing-services:werror=true
    -Dgst-docs:werror=true
    -Dgst-omx:werror=true
    -Dgst-devtools:werror=true
    -Dgst-python:werror=true
    -Dgstreamer-vaapi:werror=true
    -Dgstreamer-sharp:werror=true

81
  DEFAULT_CERBERO_ARGS: >
82 83
    --variants werror
    --timestamps
84

85 86 87 88 89
workflow:
  rules:
    - if: $CI_MERGE_REQUEST_IID
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH
90 91 92
#
# Global CI policy
#
Nicolas Dufresne's avatar
Nicolas Dufresne committed
93
# This can be used to configure global behaviour our our jobs.
94
#
95
default:
96 97 98 99
  retry:
    max: 2
    when:
      - 'runner_system_failure'
100 101
      - 'stuck_or_timeout_failure'
      - 'scheduler_failure'
102
      - 'api_failure'
103
  interruptible: true
104

105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123

# Script to check if a docker image exists in the upstream registry
# and if so copy it to the forked registry so we can use it
#
# This is copied/adapted from citemplates/templates/fedora.yml
# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml
.check_image: &check_image_exists |
  # if-not-exists steps
  set -x

  if [[ -z "$GST_FORCE_REBUILD" ]]
  then
    # disable exit on failure
    set +e

    # check if our image is already in the current registry
    # we store the sha of the digest and the layers
    skopeo inspect docker://$LOCAL_IMAGE | jq '[.Digest, .Layers]' > local_sha

124 125 126
    # check if our image is already in our gst-ci registry fork
    skopeo inspect docker://$LOCAL_GST_CI_IMAGE | jq '[.Digest, .Layers]' > local_gst_ci_sha

127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
    # check if our image is already in the upstream registry
    if [[ -z "$GST_UPSTREAM_REPO" ]]
    then
      echo "WARNING! Variable \$GST_UPSTREAM_REPO is undefined, cannot check for images"
    else
      skopeo inspect docker://$GST_UPSTREAM_IMAGE | jq '[.Digest, .Layers]' > upstream_sha
    fi

    # reenable exit on failure
    set -e

    # if the upstream repo has an image, ensure we use the same
    if [ -s upstream_sha ]
    then

142
      echo "Checking if $LOCAL_IMAGE is up to date"
143 144 145
      # ensure we use the same image from upstream
      diff upstream_sha local_sha && exit 0 || true

146
      echo "Copying image from gstreamer/gst-ci to local registry"
147 148 149 150 151 152 153 154 155 156 157 158
      # copy the original image into the current project registry namespace
      # we do 2 attempts with skopeo copy at most
      skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
                  docker://$GST_UPSTREAM_IMAGE \
                  docker://$LOCAL_IMAGE || \
      skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
                  docker://$GST_UPSTREAM_IMAGE \
                  docker://$LOCAL_IMAGE

      exit 0
    fi

159 160
    set +x

161 162 163
    # if the local ci fork repo has an image, ensure we use the same
    if [ -s local_gst_ci_sha ]
    then
164
      echo "Checking if $LOCAL_GST_CI_IMAGE is up to date"
165 166 167
      # ensure we use the same image from upstream
      diff local_gst_ci_sha local_sha && exit 0 || true

168
      echo "Copying image from $CI_PROJECT_NAMESPACE/gst-ci to local registry"
169 170 171 172 173 174 175 176 177 178 179 180
      # copy the original image into the current project registry namespace
      # we do 2 attempts with skopeo copy at most
      skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
                  docker://$LOCAL_GST_CI_IMAGE \
                  docker://$LOCAL_IMAGE || \
      skopeo copy --dest-creds $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD \
                  docker://$LOCAL_GST_CI_IMAGE \
                  docker://$LOCAL_IMAGE

      exit 0
    fi

181 182 183
    # if we have a local image but none in the upstream repo, use our
    if [ -s local_sha ]
    then
184
      echo "Using $LOCAL_IMAGE"
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
      exit 0
    fi
  fi


# Build docker images from Dockerfiles
#
# This is copied/adapted from citemplates/templates/fedora.yml
# https://gitlab.freedesktop.org/freedesktop/ci-templates/-/blob/96b621fe9f57ec2464f8d1a0940446afbf6c8f59/templates/fedora.yml
#
# CITemplates builds uses buildah run/commit workflow to build the images which doesn't
# fit us atm and our images need further adjustment to make use of it. Porting to
# ci-templates is the next step though.
#
# All the documentation from citemplates should be applicable here, so please refer there
.base:
201
  image: "$CI_REGISTRY/freedesktop/ci-templates/buildah:2020-07-20.1"
202 203 204 205 206 207 208 209
  variables:
    STORAGE_DRIVER: 'vfs'
    BUILDAH_FORMAT: 'docker'
    BUILDAH_ISOLATION: 'chroot'

    GIT_STRATEGY: fetch
  script:
    - export LOCAL_IMAGE="$CI_REGISTRY_IMAGE/$REPO_SUFFIX:$TAG"
210
    - export LOCAL_GST_CI_IMAGE="$CI_REGISTRY/$CI_PROJECT_NAMESPACE/gst-ci/$REPO_SUFFIX:$TAG"
211 212
    - export GST_UPSTREAM_IMAGE="$CI_REGISTRY/$GST_UPSTREAM_REPO/$REPO_SUFFIX:$TAG"

213 214 215 216 217 218 219 220 221 222
    # check if the docker registry is enabled, else the variable will be missing
    - |
      if [[ -z "$CI_REGISTRY_IMAGE" ]]
      then
        echo "ERROR! Looks like your repository/fork has disabled Docker Registries."
        echo "Pleae enable them by following the documentation from here:"
        echo "https://docs.gitlab.com/ee/user/packages/container_registry/#enable-the-container-registry-for-your-project"
        exit 1
      fi

223 224 225 226
    # Newer versions of podman/buildah try to set overlayfs mount options when
    # using the vfs driver, and this causes errors.
    - sed -i '/^mountopt =.*/d' /etc/containers/storage.conf

227
    - *check_image_exists
228

229 230 231 232 233 234 235 236 237
    # This part of the job should only ever be possible to be reach in a gst-ci fork
    - |
      if [[ "$CI_PROJECT_NAME" != 'gst-ci' ]]
      then
        echo "ERROR! Something is wrong!"
        echo "This part of the job is supposed to be executed only in gst-ci forks, and its probably going to fail else"
        exit 1
      fi

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
    - echo "Building image $LOCAL_IMAGE"

    - >
      buildah bud
      --build-arg DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH
      --label ci.job_id=$CI_JOB_ID
      --label pipeline.url=$CI_PIPELINE_URL
      --label git.ref_name=$CI_COMMIT_REF_NAME
      --label git.sha=$CI_COMMIT_SHA
      --label gitlab.project_path=$CI_PROJECT_PATH
      --label gitlab.project_url=$CI_PROJECT_URL
      --label fdo.upstream-repo=$GST_UPSTREAM_REPO
      -f $DOCKERFILE
      -t $LOCAL_IMAGE
      $CONTEXT_DIR

    - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY

    # Sanity check
    - |
      if [ "$CI_PROJECT_PATH" = "gstreamer/gst-ci" ]; then
        if [ "$LOCAL_IMAGE" != "$GST_UPSTREAM_IMAGE" ]; then
          echo "ERROR!!! AAAAA! THE IMAGE URI DOES NOT MATCH THE EXPECTED UPSTREAM ONE"
          echo $LOCAL_IMAGE
          echo $GST_UPSTREAM_IMAGE
          exit 1
        fi
      fi

    -  buildah push $LOCAL_IMAGE

fedora amd64 docker:
  stage: "build docker"
  variables:
    REPO_SUFFIX: "$FEDORA_AMD64_SUFFIX"
    TAG: "$FEDORA_TAG-$GST_UPSTREAM_BRANCH"

    CONTEXT_DIR: "./docker/fedora/"
    DOCKERFILE: "docker/fedora/Dockerfile"
  extends: .base

cerbero fedora amd64 docker:
  stage: "build docker"
  variables:
    REPO_SUFFIX: "$CERBERO_AMD64_SUFFIX"
    TAG: "$CERBERO_TAG-$GST_UPSTREAM_BRANCH"

    CONTEXT_DIR: "docker/cerbero/"
    DOCKERFILE: "docker/cerbero/Dockerfile-fedora"
  extends: .base

android docker:
  stage: "build docker"
  variables:
    REPO_SUFFIX: "$ANDROID_AMD64_SUFFIX"
    TAG: "$ANDROID_TAG-$GST_UPSTREAM_BRANCH"

    CONTEXT_DIR: "docker/android/"
    DOCKERFILE: "docker/android/Dockerfile"
  extends: .base

alpine amd64 manifest builder docker:
  stage: "build docker"
  variables:
    REPO_SUFFIX: "$MANIFEST_AMD64_SUFFIX"
    TAG: '$MANIFEST_TAG-$GST_UPSTREAM_BRANCH'

    CONTEXT_DIR: "docker/build_manifest/"
    DOCKERFILE: "docker/build_manifest/Dockerfile"
  extends: .base

gst-indent amd64 docker:
  stage: "build docker"
  variables:
    REPO_SUFFIX: "$INDENT_AMD64_SUFFIX"
313
    TAG: "$INDENT_TAG-$GST_UPSTREAM_BRANCH"
314 315 316 317 318 319 320 321

    CONTEXT_DIR: "docker/indent/"
    DOCKERFILE: "docker/indent/Dockerfile"
  extends: .base

#
# Job to create the manifest.xml to used for our builds
#
322
manifest:
323
  image: $MANIFEST_IMAGE
324
  rules:
325 326
    # Always have automatic pipeline for branchs in cerbero and docs
    # cause they need to update artifacts, like the docs site or cerbero deps
327
    - if: '$CI_PROJECT_PATH == "gstreamer/cerbero"'
328
    - if: '$CI_PROJECT_PATH == "gstreamer/gst-docs"'
329

330 331 332
    # If the MR is assigned to the Merge bot, trigger the pipeline automatically
    - if: '$CI_MERGE_REQUEST_ASSIGNEES == "gstreamer-merge-bot"'
    # When the assignee isn't the merge bot, require an explicit action to trigger the pipeline
333
    # to avoid wasting CI resources
334
    - if: '$CI_MERGE_REQUEST_ASSIGNEES != "gstreamer-merge-bot"'
335 336 337
      when: 'manual'
    # If this matches, it means the pipeline is running against either the main
    # or a stable branch, so make it manual
338
    - if: '$CI_COMMIT_BRANCH == $GST_UPSTREAM_BRANCH'
339 340
      when: 'manual'

341
  stage: 'preparation'
342
  script:
343 344 345 346
    - cd /gst-ci
    - gitlab/build_manifest.py --self-update
    - gitlab/build_manifest.py ${CI_PROJECT_DIR}/manifest.xml
    - cat ${CI_PROJECT_DIR}/manifest.xml
347
  artifacts:
348
    expire_in: "7 days"
349 350 351
    paths:
      - "manifest.xml"

352 353 354
#
# gst-indent!!
#
355
gst indent:
356
  image: $INDENT_IMAGE
357 358 359
  stage: 'preparation'
  variables:
    GIT_STRATEGY: 'fetch'
Nicolas Dufresne's avatar
Nicolas Dufresne committed
360 361
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(gstreamer-sharp|gst-integration-testsuites|cerbero|gst-docs)$/'
362 363 364
  script:
    # man indent. grep RETURN VALUE, grab a beer on my behalf...
    - indent --version || true
365
    - |
366
      filter_cmd=("cat")
367
      if test -f ".indentignore"; then
368
          filter_args=()
369 370
          while read -r line; do
              if test -n "$line"; then
371
                  filter_args+=("-e" "$line")
372 373
              fi
          done < ".indentignore"
374 375
          if [[ ${#filter_args[@]} -gt 0 ]]; then
              filter_cmd=("grep" "-v" "${filter_args[@]}")
376 377
          fi
      fi
378
      find . -name '*.c' | "${filter_cmd[@]}" | xargs -d '\n' gst-indent
379
    - |
380 381 382
      if git diff --quiet; then
          echo "Code is properly formatted"
      else
383 384 385 386 387
          git diff --color=always
          echo 'style diverges, please run gst-indent first'
          exit 1
      fi

388 389 390
#
# gst-build setup templates
#
391 392
.gst_build_template: &gst_build
  - echo $MESON_ARGS
393 394 395 396 397 398
  # Sometimes, gitlab-runner want to reuse
  # existing docker volumes without cleaning them up...
  # Make sure the docker volume is clean
  - rm -rf gst-build || true
  - rm -rf meson-logs || true
  - rm -rf validate-logs || true
399 400 401 402 403 404

  - curl -L -o clone_manifest_ref.py "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/${GST_UPSTREAM_BRANCH}/gitlab/clone_manifest_ref.py"
  - python3 clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination gst-build
  - cd gst-build/
  # only copy immediate directories. We don't want to modify .wrap files
  - find /gst-build/subprojects/ -mindepth 1 -maxdepth 1 -type d -exec cp -r \{\} subprojects/ \;
405 406 407

  # Reset the state of the subprojects to what gst-build expects
  - meson subprojects update --reset
408
  - ./git-update --no-interaction --manifest="${CI_PROJECT_DIR}/manifest.xml"
409

410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
  - meson build/ $MESON_ARGS
  - ninja -C build/
  - ccache --show-stats

.gst_build_ccache_vars:
  variables:
    CCACHE_COMPILERCHECK: 'content'
    CCACHE_COMPRESS: 'true'
    CCACHE_BASEDIR: '/cache/gstreamer/gst-build'
    CCACHE_DIR: '/cache/gstreamer/gst-build/ccache/'
    # shared across everything really
    CCACHE_MAXSIZE: '10G'
    CARGO_HOME: '/cache/gstreamer/cargo'

.simple_fedora_build: &simple_build >-
  ${DEFAULT_MESON_ARGS}
  -Dsharp=enabled
  -Domx=enabled
  -Dgst-omx:target=generic
429
  -Ddoc=disabled
430 431
  -Drs=disabled
  ${MESON_BUILDTYPE_ARGS}
432
  ${MESON_GST_WERROR}
433

434
.build:
Matthew Waters's avatar
Matthew Waters committed
435
  stage: 'build'
436
  extends:
437
    - '.gst_build_ccache_vars'
438 439
  needs:
    - "manifest"
440
  # Taking into account the slowest shared runner + time needed to upload the binaries to artifacts
441 442
  # Also need to take into account I/O of pulling docker images and uploading artifacts
  timeout: '45min'
443
  variables:
444
    MESON_ARGS: "${DEFAULT_MESON_ARGS} ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
445 446
  rules:
    - if: '$CI_PROJECT_NAME != "cerbero"'
447
  script:
448
    *gst_build
449
  after_script:
450 451 452 453
    - mv gst-build/build/meson-logs/ meson-logs
    # Cleanup everything else to reduce the size
    # of the docker volume we leave behind
    - rm -rf gst-build
454 455 456 457
  artifacts:
    expire_in: "7 days"
    when: "always"
    paths:
458
      - 'meson-logs/'
459
      - 'manifest.xml'
460

461
.build fedora x86_64:
462
  extends: '.build'
463
  image: $FEDORA_IMAGE
Thibault Saunier's avatar
Thibault Saunier committed
464
  variables:
465
    MESON_ARGS: *simple_build
Nicolas Dufresne's avatar
Nicolas Dufresne committed
466 467
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-omx|gst-plugins-rs)$/'
468

469
build nodebug fedora x86_64:
470
  extends: '.build'
Thibault Saunier's avatar
Thibault Saunier committed
471
  image: $FEDORA_IMAGE
472
  variables:
473
    MESON_ARGS: "${DEFAULT_MESON_ARGS} -Dsharp=enabled -Dgstreamer:gst_debug=false -Domx=enabled -Dgst-omx:target=generic ${MESON_BUILDTYPE_ARGS} ${MESON_GST_WERROR}"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
474 475
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/'
476 477

build static fedora x86_64:
478
  extends: '.build fedora x86_64'
479 480
  variables:
    MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
481 482
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/'
483 484 485 486 487

build static nodebug fedora x86_64:
  extends: 'build nodebug fedora x86_64'
  variables:
    MESON_BUILDTYPE_ARGS: "--default-library=static -Dintrospection=disabled -Ddoc=disabled"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
488 489
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/'
490

491
build clang fedora x86_64:
492
  extends: '.build fedora x86_64'
493 494 495
  variables:
    CC: 'ccache clang'
    CXX: 'ccache clang++'
Nicolas Dufresne's avatar
Nicolas Dufresne committed
496 497
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/'
498

499 500
.test:
  stage: 'test'
501
  extends:
502 503 504
    - '.gst_build_ccache_vars'
  needs:
    - "manifest"
505
  variables:
506 507
    MESON_ARGS: *simple_build

508
    # Disable colored output to avoid weird rendering issues
509 510 511
    GST_DEBUG_NO_COLOR: "true"
    CI_ARTIFACTS_URL: "${CI_PROJECT_URL}/-/jobs/${CI_JOB_ID}/artifacts/raw/validate-logs/"
    GST_VALIDATE_LAUNCHER_FORCE_COLORS: "true"
512
    TIMEOUT_FACTOR: "2"
513
    CARGO_HOME: "/cache/gstreamer/cargo"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
514 515
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-examples|gst-docs)$/'
516
  script:
517 518
    - *gst_build

519
    - echo "-> Running ${TEST_SUITE}"
520 521 522
    - >
      ./gst-uninstalled.py
      gst-validate-launcher ${TEST_SUITE}
523
      --dump-on-failure
524 525 526 527
      --mute
      --shuffle
      --no-display
      --meson-no-rebuild
528
      --timeout-factor "${TIMEOUT_FACTOR}"
529 530 531
      --fail-on-testlist-change
      -l "${CI_PROJECT_DIR}/validate-logs/"
      --xunit-file "${CI_PROJECT_DIR}/validate-logs/xunit.xml"
532
      ${EXTRA_VALIDATE_ARGS}
533 534 535 536 537
  after_script:
    - mv gst-build/build/meson-logs/ meson-logs
    # Cleanup everything else to reduce the size
    # of the docker volume we leave behind
    - rm -rf gst-build
538
  artifacts:
539
    expire_in: '14 days'
540 541
    when: always
    paths:
542
      - 'meson-logs/'
543
      - 'validate-logs'
544
      - 'manifest.xml'
545 546
    reports:
      junit:
547
        - "validate-logs/*.xml"
548 549

.test fedora x86_64:
550
  image: $FEDORA_IMAGE
551
  extends: '.test'
552
  tags: ['gstreamer']
553 554 555 556 557

check fedora:
  extends: '.test fedora x86_64'
  variables:
    TEST_SUITE: "check.gst*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
558
  rules:
559
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs)$/'
560

561 562
integration testsuites fedora:
  extends: '.test fedora x86_64'
563
  parallel: 4
564
  variables:
565
    EXTRA_VALIDATE_ARGS: "--timeout-factor=2 --retry-on-failures --check-bugs --parts=${CI_NODE_TOTAL} --part-index=${CI_NODE_INDEX}"
566
    TEST_SUITE: "validate ges"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
567
  rules:
568
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-examples|gst-omx|gstreamer-sharp|gst-plugins-rs)$/'
569

570 571 572 573
gstreamer-full:
  extends: 'build static fedora x86_64'
  stage: integrate
  variables:
574
    MESON_ARGS: >
575
      --default-library=static
576 577
      -Dauto_features=disabled
      -Dgst-plugins-good:alpha=enabled
578
      $MESON_GST_WERROR
Nicolas Dufresne's avatar
Nicolas Dufresne committed
579 580 581
  rules:
    - if: '$CI_PROJECT_NAME =~ /^(gst-build|gst-ci)$/'

582 583 584 585 586 587 588 589 590
  after_script:
  - cd gst-build/
  - ninja -C build install
  - export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib64/gstreamer-1.0/pkgconfig
  - export LD_LIBRARY_PATH=/usr/local/lib64
  - cd examples/static-plugins
  - meson _build
  - ninja -C _build
  - meson test -C _build -v
591
  artifacts:
592 593
    expire_in: "7 days"
    when: "always"
594 595 596
    paths:
      - 'gst-build/build/meson-logs/'
      - 'gst-build/build/gstinitstaticplugins.c'
597

598 599 600
# Valgrind
.valgrind fedora x86_64:
  extends: '.test fedora x86_64'
Matthew Waters's avatar
Matthew Waters committed
601
  stage: 'test'
602
  variables:
603
    EXTRA_VALIDATE_ARGS: "--valgrind"
604 605 606 607 608

valgrind core:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gstreamer\\..*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
609
  rules:
610
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-ci|gst-devtools)$/'
611 612 613 614 615

valgrind base:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-base\\..*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
616
  rules:
617
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-ci|gst-devtools)$/'
618 619 620 621 622

valgrind good:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-good\\..*"
623 624
    # take longer time due to splitmux unit test
    TIMEOUT_FACTOR: "4"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
625
  rules:
626
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-ci|gst-devtools)$/'
627 628 629 630 631

valgrind ugly:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-ugly\\..*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
632
  rules:
633
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-ugly|gst-ci|gst-devtools)$/'
634 635 636 637 638

valgrind bad:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-plugins-bad\\..*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
639
  rules:
640
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-plugins-bad|gst-ci|gst-devtools)$/'
641

642 643 644 645
valgrind ges:
  extends: '.valgrind fedora x86_64'
  variables:
    TEST_SUITE: "check.gst-editing-services\\..*"
Nicolas Dufresne's avatar
Nicolas Dufresne committed
646 647
  rules:
    - if: '$CI_PROJECT_NAME =~ /^(gstreamer|gst-plugins-base|gst-plugins-good|gst-editing-services|gst-ci|gst-devtools)$/'
648

649 650 651 652
# Template for Cerbero GStreamer Build
#
# Parameters:
# CONFIG: The name of the configuration file to use
Matthew Waters's avatar
Matthew Waters committed
653
# ARCH: The cerbero <os>_<cpu> (used in cache key)
654
#
Matthew Waters's avatar
Matthew Waters committed
655
# Produces runtime and devel tarball packages for linux/android or .pkg for macos
656 657 658 659 660 661 662 663 664
.cerbero template: &cerbero_package_and_check
  - $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o $(pwd) gstreamer-1.0
  # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
  - |-
      [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version
  - |-
      [[ $CONFIG == *universal* ]] || $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX
  - test "x${HAVE_CCACHE}" = "xyes" && ccache -s || true    # eat the return value from the failing test

665
.cerbero:
Matthew Waters's avatar
Matthew Waters committed
666
  stage: "build"
667
  image: $CERBERO_IMAGE
Nicolas Dufresne's avatar
Nicolas Dufresne committed
668 669
  rules:
    - if: '$CI_PROJECT_NAME == "cerbero"'
670 671
  needs:
    - "manifest"
672 673 674
  # Ensure that the runners it will be executed on
  # will have plenty of space for the cache
  tags: ['gstreamer']
675
  timeout: '3h'
676
  variables:
677 678 679 680 681 682 683
    CCACHE_COMPILERCHECK: "content"
    CCACHE_COMPRESS: "true"
    CCACHE_BASEDIR: "/cache/gstreamer/cerbero/"
    CCACHE_DIR: "/cache/gstreamer/cerbero/ccache/"
    # shared across everything really
    CCACHE_MAXSIZE: "50G"

Matthew Waters's avatar
Matthew Waters committed
684 685
    CERBERO_HOME: "cerbero-build"
    CERBERO_SOURCES: "cerbero-sources"
686 687
    CERBERO_ARGS: "${DEFAULT_CERBERO_ARGS}"
    CERBERO: "./cerbero-uninstalled -c config/${CONFIG} -c localconf.cbc -m manifest.xml"
Matthew Waters's avatar
Matthew Waters committed
688
    CERBERO_PACKAGE_ARGS: "-t"
689 690
    CERBERO_RUN_WRAPPER: "" # 'wine' on cross-winXX
    CERBERO_RUN_SUFFIX: "" # '.exe' on cross-winXX
Matthew Waters's avatar
Matthew Waters committed
691 692 693
    HAVE_CCACHE: "yes"
    # used by macos packages as we only ever install to a fixed directory
    CERBERO_OVERRIDDEN_DIST_DIR: ""
694 695
    # location where cerbero is cached on the host
    CERBERO_HOST_DIR: "/cerbero/"
696 697
  before_script:
    # FIXME Wrong namespace
698
    # Workaround build-tools having hardcoded internal path
Matthew Waters's avatar
Matthew Waters committed
699 700
    - pwd
    - mkdir -p ../../gstreamer
701
    - test -e ../../gstreamer/cerbero || ln -sf $(pwd) ../../gstreamer/cerbero
Matthew Waters's avatar
Matthew Waters committed
702
    - mkdir -p ../../${CI_PROJECT_NAMESPACE}
703
    - test -e ../../${CI_PROJECT_NAMESPACE}/cerbero || ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero
704
    - rsync -aH ${CERBERO_HOST_DIR} .
Matthew Waters's avatar
Matthew Waters committed
705 706
    - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc
    - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
707 708
    - ./cerbero-uninstalled --self-update manifest.xml
  script:
709
    - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats
710 711 712
    - $CERBERO $CERBERO_ARGS show-config
    - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
    - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
713
    - $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
714 715 716 717
    - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
          && test -d ${CERBERO_HOME}/dist/${ARCH}
          && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR}
          && rsync -aH ${CERBERO_HOME}/dist/${ARCH}/ ${CERBERO_OVERRIDDEN_DIST_DIR}
718
    - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
719
    - *cerbero_package_and_check
720
  cache:
721
    key: "${CI_JOB_NAME}"
722 723 724 725
    paths:
      - "${CERBERO_SOURCES}"
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
726
    expire_in: '5 days'
727 728 729
    when: 'always'
    paths:
      - "manifest.xml"
Matthew Waters's avatar
Matthew Waters committed
730
      - "${CERBERO_HOME}/logs"
731
      - "*.tar.*"
732

733
.build windows:
734
  image: $WINDOWS_IMAGE
735
  stage: 'build'
736 737
  needs:
    - 'manifest'
738 739 740
  tags:
    - 'docker'
    - 'windows'
741 742
    - '1809'
  timeout: '45min'
743 744 745 746 747 748 749
  variables:
    MESON_ARGS: >
      ${DEFAULT_MESON_ARGS}
      -Dpython=disabled
      -Dlibav=disabled
      -Dvaapi=disabled
      -Ddevtools=disabled
Nicolas Dufresne's avatar
Nicolas Dufresne committed
750 751
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/'
752
  script:
753 754 755 756 757 758
    # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or
    # from a previous job due to some gitlab bug or implicit behavior?
    # So let's always check and clear it out if its there
    # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137
    - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build }

759 760 761 762
    # Make sure powershell exists on errors
    # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6
    - $ErrorActionPreference = "Stop"

763 764
    - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py
    - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build
765
    - cd $env:CI_PROJECT_DIR/gst-build
766

767
    - cp -r C:/subprojects/* subprojects/
768 769
    # Reset the subprojects to invalidate the cache if needed
    - meson subprojects update --reset
770
    # Run the git-update script and feed it the manifest to setup the environment
771 772
    - python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml

773 774 775 776 777 778 779 780 781
    # For some reason, options are separated by newline instead of space, so we
    # have to replace them first.
    - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
    # Gitlab executes PowerShell in docker, but VsDevCmd.bat is a batch script.
    # Environment variables substitutions is done by PowerShell before calling
    # cmd.exe, that's why we use $env:FOO instead of %FOO%
    - cmd.exe /C "C:\BuildTools\Common7\Tools\VsDevCmd.bat -host_arch=amd64 -arch=$env:ARCH &&
        meson build $env:MESON_ARGS &&
        ninja -C build"
782 783 784 785 786
  artifacts:
    expire_in: "7 days"
    when: "always"
    paths:
      - 'gst-build/build/meson-logs/'
787 788 789 790 791 792 793 794 795 796 797 798 799

build vs2017 amd64:
  extends: '.build windows'
  variables:
    ARCH: 'amd64'

build vs2017 x86:
  extends: '.build windows'
  variables:
    ARCH: 'x86'

build msys2 :
  extends: '.build windows'
800
  timeout: '60min'
801 802 803 804
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-docs|gst-integration-testsuites|gst-omx|gst-plugins-rs|gstreamer-vaapi)$/'
      allow_failure: true
      when: 'manual'
805
  script:
806 807 808 809 810 811
    # Sometimes there's seems to be an existing gst-build clone that comes either from the ether or
    # from a previous job due to some gitlab bug or implicit behavior?
    # So let's always check and clear it out if its there
    # https://gitlab.freedesktop.org/tpm/gstreamer-sharp/-/jobs/1672137
    - if (Test-Path $env:CI_PROJECT_DIR/gst-build) { Remove-Item -Recurse -Force $env:CI_PROJECT_DIR/gst-build }

812 813 814 815
    # Make sure powershell exists on errors
    # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_preference_variables?view=powershell-6
    - $ErrorActionPreference = "Stop"

816 817
    # For some reason docker build hangs if this is included in the image, needs more troubleshooting
    - $env:PATH += ';C:\msys64\usr\bin;C:\msys64\mingw64\bin;C:\msys64\mingw32\bin'
Nirbheek Chauhan's avatar
Nirbheek Chauhan committed
818
    # Copied from https://github.com/msys2/setup-msys2/blob/master/main.js#L98
819
    - C:\msys64\usr\bin\bash -c "pacman-key --init && pacman-key --populate msys2 && pacman-key --refresh-keys || true"
Nirbheek Chauhan's avatar
Nirbheek Chauhan committed
820 821 822 823 824 825 826 827
    - C:\msys64\usr\bin\bash -c "sed -i 's/^CheckSpace/#CheckSpace/g' /etc/pacman.conf"
    - echo "Updating MSYS2"
    - C:\msys64\usr\bin\bash -c "pacman -Syuu --noconfirm || echo Update failed, ignoring"
    - echo "Killing all MSYS2 processes"
    - taskkill /F /FI "MODULES eq msys-2.0.dll"
    - echo "Completing MSYS2 update"
    - C:\msys64\usr\bin\bash -c "pacman -Suu --noconfirm"
    - echo "Installing needed MSYS2 packages"
828 829
    - C:\msys64\usr\bin\bash -c "pacman -Sy --noconfirm --needed mingw-w64-x86_64-toolchain ninja"

830 831 832 833
    - Invoke-WebRequest -Uri "https://gitlab.freedesktop.org/gstreamer/gst-ci/raw/$env:GST_UPSTREAM_BRANCH/gitlab/clone_manifest_ref.py" -OutFile $env:CI_PROJECT_DIR/clone_manifest_ref.py
    - python clone_manifest_ref.py --manifest manifest.xml --project gst-build --destination $env:CI_PROJECT_DIR/gst-build
    - cd $env:CI_PROJECT_DIR/gst-build

834 835 836
    - cp -r C:\subprojects\* subprojects\

    # Run the git-update script and feed it the manifest to setup the environment
837
    - cd $env:CI_PROJECT_DIR/gst-build; python git-update --no-interaction --manifest=$env:CI_PROJECT_DIR/manifest.xml
838 839 840
    # For some reason, options are separated by newline instead of space, so we
    # have to replace them first.
    - $env:MESON_ARGS = $env:MESON_ARGS.replace("`n"," ")
841 842 843
    # Replace forward slashes with backwards so bash doesn't complain
    - $env:_PROJECT_DIR = $env:CI_PROJECT_DIR.replace('\','/')
    - C:\msys64\usr\bin\bash -c "cd $env:_PROJECT_DIR/gst-build &&
844 845 846 847
        meson build $env:MESON_ARGS &&
        ninja -C build"


848 849 850
# Template for Cerbero GStreamer Deps
#
# This template is used by cerbero/ project to pre-built the GStreamer
Thibault Saunier's avatar
Thibault Saunier committed
851
# dependencies. When available, the .cerbero jobs will download this artifact
852 853 854 855
# in order to speed up the build.
#
# Parameters:
# CONFIG: The name of the configuration file to use
856
# ARCH: The cerbero <os>_<cpu> (used in cache key)
857
#
858
# Produce an artifact with the dist/ and .cache along
859 860 861
# with the associated build-tools.
.cerbero deps:
  extends: .cerbero
Nicolas Dufresne's avatar
Nicolas Dufresne committed
862 863
  rules:
    - if: '$CI_PROJECT_NAME == "cerbero"'
864
  script:
865
    - test "x${HAVE_CCACHE}" = "xyes" && ccache --show-stats
866 867 868 869 870
    - $CERBERO $CERBERO_ARGS show-config
    - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
    - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
    - $CERBERO $CERBERO_ARGS bootstrap --offline --build-tools-only
    - $CERBERO $CERBERO_ARGS build-deps --offline
871 872
          gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0
          gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0
873
          gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice
Matthew Waters's avatar
Matthew Waters committed
874 875 876
    - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
          && mkdir -p ${CERBERO_HOME}/dist/${ARCH}
          && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH}
877 878 879
    - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH}
    - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x"
          || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH}
880
    - *cerbero_package_and_check
881

882 883 884
#
# Cerbero Linux X86_64 build
#
885 886
cerbero deps fedora x86_64:
  extends: '.cerbero deps'
887 888
  variables:
    CONFIG: "linux.config"
889
    ARCH: "linux_x86_64"
890

891
build cerbero fedora x86_64:
892
  extends: '.cerbero'
Nicolas Dufresne's avatar
Nicolas Dufresne committed
893 894
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/'
895 896
  variables:
    CONFIG: "linux.config"
897

898 899 900
#
# Cerbero Android Universal build
#
901
.cerbero cross-android universal:
902 903
  variables:
    CONFIG: "cross-android-universal.cbc"
904
    ARCH: "android_universal"
905 906 907 908 909 910 911
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
    expire_in: '5 days'
    when: 'always'
    paths:
      - "manifest.xml"
      - "${CERBERO_HOME}/logs"
912
      - "*[0-9].tar.*"
913

914 915 916 917
cerbero deps cross-android universal:
  extends:
    - '.cerbero deps'
    - '.cerbero cross-android universal'
918

919
build cerbero cross-android universal:
920 921 922
  extends:
    - '.cerbero'
    - '.cerbero cross-android universal'
Nicolas Dufresne's avatar
Nicolas Dufresne committed
923
  rules:
924
    - if: '$CI_MERGE_REQUEST_IID && $CI_PROJECT_NAME == "gst-docs"'
925
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-integration-testsuites|gst-plugins-rs)$/'
926

927 928 929
#
# Cerbero Cross Windows builds
#
930
cerbero deps cross-windows x86:
931
  extends: '.cerbero deps'
932 933
  variables:
    CONFIG: "cross-win32.cbc"
934
    ARCH: "mingw_x86"
935

936
build cerbero cross win32:
937
  extends: '.cerbero'
938 939
  rules:
    - if: '$CI_PROJECT_NAME !~ /^(cerbero|gst-build|gst-docs|gst-omx|gstreamer-vaapi|gst-docs|gst-integration-testsuites|gst-plugins-rs)$/'
940 941
  variables:
    CONFIG: "cross-win32.cbc"
942 943
    CERBERO_RUN_WRAPPER: "wine"
    CERBERO_RUN_SUFFIX: ".exe"
944

945

946
cerbero deps cross-windows x86_64:
947
  extends: '.cerbero deps'
948 949
  variables:
    CONFIG: "cross-win64.cbc"
950
    ARCH: "mingw_x86_64"
951

952
build cerbero cross win64:
953
  extends: '.cerbero'