ci_fairy.py 45.1 KB
Newer Older
1 2
#!/usr/bin/env python3

3
import boto3
4
import botocore
5 6
import click
import colored
7
import functools
8
import gitdb
9 10
import gitlab
from gitlab import Gitlab
11
import io
12
import itertools
13
import jinja2
14
import json
15 16 17
import fnmatch
import logging
import os
18
import re
19
import shutil
20
import sys
21
import time
22
import urllib.parse
23
import yaml
24
from botocore.client import Config
25
from pathlib import Path
26 27


28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
class _env_ci_fairy:
    @staticmethod
    def hashfiles(*argv):
        '''
        Hashes the filenames (@argv) and the content of the files in some
        unspecified (but stable) way and returns a 64 characters (hex) digest.
        '''

        import hashlib

        argv = list(argv)

        if not argv:
            raise ValueError("must specify at least one filename to hash")

        h = hashlib.new("sha256")
        h.update(f"{len(argv)}\n".encode("utf-8"))
        for filename in argv:
            with open(filename, "rb") as f:
                d = f.read()
                h.update(f"{len(filename)} {filename}\n{len(d)}\n".encode("utf-8"))
                h.update(d)
        return h.hexdigest()

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67
    @staticmethod
    def sha256sum(path, prefix=True):
        '''
        Hashes content of the files with sha256 and produces a string in the
        form "sha256-<hexdigest>".

        If prefix is False, the returned string is only the hexdigest.
        '''

        import hashlib
        h = hashlib.new("sha256")
        with open(path, "rb") as f:
            d = f.read()
            h.update(d)
        return f'{"sha256-" if prefix else ""}{h.hexdigest()}'

68

69
def gitlab_project(gitlab, project=None, fallback='__default__'):
70 71 72 73 74
    '''
    Returns the Gitlab Project object for the given project name. Where name
    is None, it is taken from the CI environment.
    '''
    if project is None:
75 76 77 78 79 80 81
        # we can't default fallback to os.getenv() without breaking the
        # tests (which rely on putenv), so let's use a special value here.
        if fallback == '__default__':
            fallback = os.getenv('CI_PROJECT_ID')
        if fallback is None:
            return None
        project_id = urllib.parse.quote(fallback, safe='')
82 83 84 85
        if project_id is None:
            return None
        p = gitlab.projects.get(project_id)
    else:
86
        p = gitlab.projects.list(search=project, search_namespaces=True)
87 88 89
        # gitlab does substring matches, so foo/bar matches foo/barbaz as
        # well
        if len(p) > 1:
90
            p = [x for x in p if x.path == project or x.path_with_namespace == project]
91
        p = p[0] if len(p) == 1 else None
92 93 94 95

    return p


96 97 98 99 100 101 102 103 104
class ColorFormatter(logging.Formatter):
    def format(self, record):
        COLORS = {
            'DEBUG': colored.fg('orchid'),
            'INFO': colored.attr('reset'),
            'WARNING': colored.fg('salmon_1'),
            'ERROR': colored.fg('red'),
            'CRITICAL': colored.fg('red') + colored.bg('yellow'),
        }
105
        return COLORS[record.levelname] + super().format(record) + colored.attr('reset')
106 107 108 109 110 111 112 113 114 115 116 117 118


log_format = '%(levelname)s: %(message)s'
logger_handler = logging.StreamHandler()
if os.isatty(sys.stderr.fileno()):
    logger_handler.setFormatter(ColorFormatter(log_format))
else:
    logger_handler.setFormatter(logging.Formatter(log_format))
logger = logging.getLogger('ci-fairy')
logger.addHandler(logger_handler)
logger.setLevel(logging.ERROR)


119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
def make_junit_xml(filename, test_suite_id, test_suite_name, failures):
    '''
    Write a JUNIT XML compatible file to filename based on the failures
    dictionary where each failure is a tuple of (id, short-message,
    full-message).
    '''
    import xml.etree.cElementTree as ET

    root = ET.Element('testsuites')
    suite = ET.SubElement(root, 'testsuite',
                          id=test_suite_id,
                          name=test_suite_name,
                          tests=str(len(failures)),
                          errors=str(len(failures)))
    for f in failures:
        tcaseid, name, message = f
        tcaseid = f'{test_suite_id}.{tcaseid}'
        tcase = ET.SubElement(suite, 'testcase', id=tcaseid, name=name)
        failure = ET.SubElement(tcase, 'failure', message=name, type='ERROR')
        failure.text = message

    ET.ElementTree(root).write(filename, encoding='UTF-8', xml_declaration=True)


143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
class LazyGitlab(object):
    def __init__(self):
        self.initialized = False
        self.url = None
        self.job_token = None
        self.private_token = None
        self.gitlab = None

    def __getattribute__(self, name):
        if name == 'gitlab' and not self.initialized:
            tokens = {}
            if self.private_token is not None:
                logger.debug('Using authfile')
                tokens['private_token'] = self.private_token
            elif self.job_token is not None:
                logger.debug('Using $CI_JOB_TOKEN')
                tokens['job_token'] = self.job_token
            else:
                logger.debug('Connecting without authentication')

            if not self.url:
164
                raise click.UsageError('Missing gitlab URL')
165 166 167 168 169 170 171 172

            self.gitlab = Gitlab(self.url, **tokens)
            self.initialized = True
            return self.gitlab

        return object.__getattribute__(self, name)


173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
class GitCommitValidator(object):
    def __init__(self, commit):
        self.commit = commit
        self.errors = []

    def _check_author_email(self):
        # The full address is @users.noreply.gitlab.freedesktop.org but
        # let's make this more generic
        if '@users.noreply' in self.commit.author.email:
            self.__error('git author email invalid\n'
                         'Please set your name and email with the commands\n'
                         '    git config --global user.name Your Name\n'
                         '    git config --global user.email your.email@provider.com\n')

    def _check_fixup_squash(self):
        if (self.commit.message.startswith('fixup!') or
                self.commit.message.startswith('squash!')):
            self.__error('Leftover "fixup!" or "squash!" commit found, please squash')

    def _check_second_line(self):
        try:
            second_line = self.commit.message.split('\n')[1]
            if second_line != '':
                self.__error('Second line in commit message must be empty')
        except IndexError:
            pass

    def check(self):
        self._check_author_email()
        self._check_fixup_squash()
        self._check_second_line()

    def check_sob(self, must_have_sob):
        lines = self.commit.message.split('\n')
        sob = [l for l in lines if l.startswith('Signed-off-by:')]

        if not must_have_sob:
            if sob:
                self.__error('Do not use Signed-off-by in commits')
        else:
            if not sob:
                self.__error('Missing "Signed-off-by: author information"')

    def check_text_width(self, tw):
        lines = self.commit.message.split('\n')
218 219
        shortlog = lines[0]
        if len(shortlog) >= tw:
220 221 222
            is_revert = shortlog.startswith('Revert "') and "This reverts commit" in self.commit.message
            if not is_revert:
                self.__error(f'Commit message subject must not exceed {tw} characters')
223

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
    def _match_pattern_rule(self, subject, rule):
        if rule.get('where', 'message') == 'subject':
            msg = subject
        else:
            msg = self.commit.message

        regex = str(rule['regex'])
        return re.search(regex, msg, re.MULTILINE)

    def check_patterns(self, rules):
        subject = self.commit.message.split('\n')[0]
        patterns = rules.get('patterns', {})

        for rule in patterns.get('require', []):
            if self._match_pattern_rule(subject, rule) is None:
                self.__error(rule.get('message', f'Required pattern not found: {rule["regex"]}'))

        for rule in patterns.get('deny', []):
            if self._match_pattern_rule(subject, rule) is not None:
                self.__error(rule.get('message', f'Disallowed pattern found: {rule["regex"]}'))

245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
    @property
    def failed(self):
        return bool(self.errors)

    def __error(self, errormsg):
        self.errors.append(errormsg)

    @property
    def error(self):
        if not self.failed:
            return None

        msg = (f'Commit message check failed\n\n'
               f'  commit: {str(self.commit)}\n'
               f'  author: {self.commit.author.name} <{self.commit.author.email}>\n'
               f'\n'
               f'  {self.commit.summary}\n'
               f'\n'
               f'\n'
               f'After correcting the issues below, force-push to the same branch.\n'
               f'This will re-trigger the CI.\n\n'
               f'---\n')
        for idx, err in enumerate(self.errors):
            msg += f'{idx+1}. {err}\n---\n'
        return msg

    @property
    def junit_tuple(self):
        '''
        A tuple of (id, short message, full description)
        '''
        if not self.failed:
            return None

        msg = '\n\n'.join([f'{idx + 1}. {err}' for idx, err in enumerate(self.errors)])

        return (str(self.commit),
282
                f'{str(self.commit)[:8]} failed {len(self.errors)} commit message checks',
283 284 285
                msg)


286 287 288 289 290 291 292 293 294 295 296 297 298
class S3(object):
    '''
    An abstract object providing a common API
    for AWS S3 server (remotes) or local file system
    '''
    def __init__(self):
        pass

    @classmethod
    def s3(cls, full_path, credentials=None):
        '''
        Factory method to get an S3 object based on
        its path.
299
        :param full_path: the full path of the object (local posix path or "minio://host/bucket/key")
300 301 302 303 304 305 306 307 308 309 310
        :type full_path: str
        :param credentials: a file path with the appropriate credentials (access key, secret key and session token)
        :type credentials: str

        :returns: An S3Object
        '''
        prefix = 'minio://'

        if not full_path.startswith(prefix):
            return S3Object(full_path)

311
        # full_path should now be in the form `minio://host/bucket/key`
312 313 314

        path_str = full_path[len(prefix):]

315
        s3_remote = S3Remote(path_str, credentials)
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368

        return s3_remote.get(path_str)


class S3Object(object):
    '''
    Wrapper around the S3 API for local or remote objects.
    A plain S3Object wraps a local file system.

    Subclasses wrap the remote API.
    '''
    def __init__(self, key):
        self.key = key
        self.path = Path(key)

    @property
    def exists(self):
        return self.path.exists()

    @property
    def is_dir(self):
        if not self.exists:
            return self.key.endswith('/')

        return self.path.is_dir()

    @property
    def is_local(self):
        return True

    @property
    def name(self):
        return self.path.name

    def copy_from(self, other):
        if other.is_local:
            try:
                shutil.copy(other.key, self.path)
            except IsADirectoryError:
                raise IsADirectoryError(f"Error: cannot do recursive cp of directory '{other.key}'")
        else:
            other.download_file(self)

    @property
    def children(self):
        return [S3Object(p) for p in self.path.glob('*')]


class S3Remote(S3Object):
    '''
    A remote S3 server.
    This contains the list of buckets.
    '''
369
    def __init__(self, full_path, credentials):
370 371 372
        super().__init__('/')

        with open(credentials) as credfile:
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
            data = json.load(credfile)

        host = full_path

        if '/' in full_path:
            host, _ = full_path.split('/', 1)

        creds = None

        for h, c in data.items():
            if h == host:
                creds = c
                break

        if not host:
            raise KeyError(f"missing host information in 'minio://{full_path}'")

        if creds is None:
            raise KeyError(f"host '{host}' not found in credentials, please use 'ci-fairy minio login' first")
392

393 394 395 396 397 398 399
        self._s3 = boto3.resource('s3',
                                  endpoint_url=creds['endpoint_url'],
                                  aws_access_key_id=creds['AccessKeyId'],
                                  aws_secret_access_key=creds['SecretAccessKey'],
                                  aws_session_token=creds['SessionToken'],
                                  config=Config(signature_version='s3v4'),
                                  region_name='us-east-1')
400
        self._name = host
401 402 403 404 405 406 407 408 409

        self._buckets = None

    @property
    def buckets(self):
        if self._buckets is None:
            bucket_names = [b.name for b in self._s3.buckets.all()]
            self._buckets = {b: self._s3.Bucket(b) for b in bucket_names}
        return self._buckets
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431

    @property
    def name(self):
        return self._name

    @property
    def is_local(self):
        return False

    @property
    def exists(self):
        return True

    @property
    def is_dir(self):
        return True

    def copy_from(self, other):
        raise ValueError('No destination bucket provided')

    @property
    def children(self):
432
        return [S3Bucket(b, self) for b in self.buckets.values()]
433 434

    def get(self, path):
435 436 437
        # Remove the host from the URL
        path = path[len(self.name):].lstrip('/')

438
        if not path:
439
            # minio://host/
440 441 442 443 444 445 446 447
            return self

        bucket_name = path
        key = ''

        if '/' in path:
            bucket_name, key = path.split('/', 1)

448
        return S3Bucket(self._s3.Bucket(bucket_name), self).get(key)
449 450 451 452 453 454


class S3Bucket(S3Object):
    '''
    A remote S3 bucket
    '''
455
    def __init__(self, bucket, remote):
456 457
        super().__init__('/')
        self._bucket = bucket
458
        self._remote = remote
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473

    @property
    def exists(self):
        return True

    @property
    def is_dir(self):
        return True

    @property
    def name(self):
        return self._bucket.name

    @property
    def children(self):
474 475 476 477
        if self.name not in self._remote.buckets:
            # minio://host/bucket_that_doesn_t_exist
            raise FileNotFoundError(f"bucket '{self.name}' doesn't exist on {self._remote.name}")

478 479 480 481 482
        objs = [o.key for o in self._bucket.objects.all()]
        children = []

        for o in objs:
            if '/' in o:
483
                # minio://host/bucket/some/path/some/file
484 485 486 487 488 489
                # o is now: some/path/some/file
                root, _ = o.split('/', 1)
                if root not in children:
                    # root is "some" and is not in the children list
                    children.append(root)
            else:
490
                # minio://host/bucket/some_file
491 492 493 494 495 496 497 498 499 500
                children.append(o)

        return [S3RemoteObject(self, c) for c in children]

    def copy_from(self, other):
        dst = other.name
        self.upload_file(other, dst)

    def get(self, key):
        if not key:
501 502
            # - minio://host/bucket
            # - minio://host/bucket/
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
            return self

        return S3RemoteObject(self, key)

    @property
    def objects(self):
        return self._bucket.objects.all()

    def upload_file(self, local_obj, remote_obj):
        if not local_obj.is_local:
            raise ValueError('at least one argument must be a local path')

        return self._bucket.upload_file(str(local_obj.path), str(remote_obj))

    def _download_file(self, remote_obj, local_obj):
        if remote_obj.is_dir:
            raise IsADirectoryError('cannot do recursive cp')

        local_path = local_obj.path

        if local_obj.is_dir:
            if not local_obj.exists:
                raise IsADirectoryError(f"directory '{local_path}' does not exist")

            local_path = local_path / remote_obj.name

        return self._bucket.download_file(str(remote_obj.key), str(local_path))

    def download_file(self, local_dest):
        return self._download_file(self, local_dest)


class S3RemoteObject(S3Object):
    '''
    A remote S3 object.
    The key is the full path of the file or directory within
    its bucket.
    '''
    def __init__(self, bucket, key):
        super().__init__(key)
        self._bucket = bucket
544 545 546
        self._is_dir = key.endswith('/')
        self._children = None
        self._exists = False
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561

    @property
    def is_dir(self):
        return self._is_dir

    @property
    def is_local(self):
        return False

    @property
    def exists(self):
        return self._exists

    @property
    def children(self):
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
        if self._children is None:
            self._children = []
            objs = [o.key for o in self._bucket.objects if o.key.startswith(self.key)]

            # find if the object exists already (full key matches)
            self._exists = self.key in objs

            # special case for directories
            if not objs:
                # nothing in the remote matches, check if we
                # have a terminating '/'
                self._is_dir = self.key.endswith('/')

            elif not self._exists:
                # at least one remote object starts with our key,
                # check if we have a parent of a remote object, or
                # just if the path and name starts with the same key

                # build the list of all parents of all objects
                parents = [p for o in objs for p in Path(o).parents]

                self._is_dir = self._exists = self.path in parents

                # compute the list of files or dir immediately below the
                # current dir
                if self._is_dir:
                    for o in objs:
                        path = Path(o)

                        for parent in path.parents:
                            if parent == self.path:
                                self._children.append(path)
                                break

                            path = parent
597 598 599 600 601 602 603 604 605 606 607 608 609 610
        return self._children

    def copy_from(self, other):
        dst = self.path

        if self.is_dir:
            dst = self.path / other.name

        self._bucket.upload_file(other, dst)

    def download_file(self, local_dest):
        return self._bucket._download_file(self, local_dest)


611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
@click.group()
@click.option('-v', '--verbose', count=True, help='increase verbosity')
@click.option('--gitlab-url', help='GitLab URL with transport protocol, e.g.  http://gitlab.freedesktop.org')
@click.option('--authfile', help='Path to a file containing the gitlab auth token string')
@click.pass_context
def ci_fairy(ctx, verbose, gitlab_url, authfile):
    verbose_levels = {
        0: logging.ERROR,
        1: logging.INFO,
        2: logging.DEBUG,
    }
    logger.setLevel(verbose_levels.get(verbose, 0))

    if gitlab_url is None:
        gitlab_url = os.getenv('CI_SERVER_URL')
626
        if not gitlab_url:
627 628
            import git

629
            try:
630 631
                from urllib.parse import urlparse

632
                repo = git.Repo(search_parent_directories=True)
633 634 635 636 637 638 639 640 641 642 643 644 645
                url = repo.remotes.origin.url
                # urlparse doesn't work with ssh specifiers which are in the
                # form user@host:path. Convert those into ssh://user@host so
                # urlparse works.
                if '//' not in url[:10]:
                    url = 'ssh://{}' + url

                # split off the user@ component if it's there
                server = urlparse(url).netloc.split('@')[-1]
                # split off an ssh-like path component if it's there
                server = server.split(':')[0]
                # Force https because what else could it be
                gitlab_url = 'https://' + server
646 647 648 649
            except git.exc.InvalidGitRepositoryError:
                pass
            except AttributeError:  # origin does not exist
                pass
650

651 652
    ctx.ensure_object(LazyGitlab)
    ctx.obj.url = gitlab_url
653 654 655

    if authfile is not None:
        token = open(authfile).read().strip()
656
        ctx.obj.private_token = token
657 658 659
    else:
        token = os.getenv('CI_JOB_TOKEN')
        if token:
660
            ctx.obj.job_token = token
661 662


663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
def credentials_option(required=True):
    def inner_function(func):
        @click.option('--credentials',
                      default='.minio_credentials',
                      help='the file to store the credentials (default to $PWD/.minio_credentials)',
                      type=click.Path(exists=required,
                                      file_okay=True,
                                      dir_okay=False,
                                      writable=not required,  # we write the file if required is false (login case)
                                      readable=True,
                                      allow_dash=False))
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        return wrapper
    return inner_function


@ci_fairy.group()
def minio():
    pass


@minio.command()
@credentials_option(required=False)
@click.option('--endpoint-url',
              default='https://minio-packet.freedesktop.org',
              help='The minio instance to contact')
@click.argument('token')
def login(credentials, endpoint_url, token):
    '''Login to the minio server'''
694 695 696 697 698 699 700 701 702
    from urllib.parse import urlparse

    credentials = Path(credentials)

    data = {}
    if credentials.exists():
        with open(credentials) as infile:
            data = json.load(infile)

703 704 705 706 707 708 709 710 711 712 713 714
    session = boto3.Session()

    sts = session.client('sts',
                         endpoint_url=endpoint_url,
                         config=Config(signature_version='s3v4'),
                         region_name='us-east-1')

    roleArn = 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole'
    ret = sts.assume_role_with_web_identity(DurationSeconds=900,
                                            WebIdentityToken=token,
                                            RoleArn=roleArn,
                                            RoleSessionName='session_name')
715 716 717

    server = urlparse(endpoint_url).netloc
    server = server.strip('/')
718 719 720
    creds = ret['Credentials']
    creds['endpoint_url'] = endpoint_url
    creds['Expiration'] = creds['Expiration'].isoformat()
721
    data[server] = creds
722
    with open(credentials, 'w') as outfile:
723
        json.dump(data, outfile)
724 725 726 727 728 729 730 731 732


@minio.command()
@credentials_option()
@click.argument('path', default='.')
@click.pass_context
def ls(ctx, credentials, path):
    try:
        s3_obj = S3.s3(path, credentials)
733 734
    except KeyError as e:
        ctx.fail(e)
735 736 737 738 739 740 741 742 743 744 745
    except botocore.exceptions.ClientError as e:
        ctx.fail(e)

    # for ls, we need to actually query the object to check if it exists
    # we can not rely on the assumption it does exist
    # calling children lazily evaluate the object and we sync ourself
    # with the remote
    try:
        children = s3_obj.children
    except FileNotFoundError as e:
        ctx.fail(e)
746 747 748 749 750 751 752

    if not s3_obj.exists:
        ctx.fail(f"file '{path}' does not exist")

    if not s3_obj.is_dir:
        print(s3_obj.name)
    else:
753
        for o in children:
754 755 756 757 758 759 760 761 762 763 764
            print(o.name)


@minio.command()
@credentials_option()
@click.argument('src')
@click.argument('dst')
@click.pass_context
def cp(ctx, credentials, src, dst):
    try:
        src = S3.s3(src, credentials)
765 766
    except KeyError as e:
        ctx.fail(e)
767 768

    # src doesn't exist
769
    if not src.exists and src.is_local:
770 771 772 773
        ctx.fail(f"source file '{src.path}' does not exist")

    try:
        dst = S3.s3(dst, credentials)
774 775
    except KeyError as e:
        ctx.fail(e)
776 777 778 779 780 781 782

    try:
        dst.copy_from(src)
    except ValueError as e:
        ctx.fail(e)
    except IsADirectoryError as e:
        ctx.fail(e)
783 784 785 786
    except boto3.exceptions.S3UploadFailedError as e:
        ctx.fail(e)
    except botocore.exceptions.ClientError as e:
        ctx.fail(e)
787 788


789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
@ci_fairy.command()
@click.option('--repository', help='The registry repository to work on, e.g.  fedora/latest')
@click.option('--project', help='Project name, e.g.  freedesktop/ci-templates')
@click.option('--all', is_flag=True, help='delete all images')
@click.option('--exclude-tag', help='fnmatch pattern to exclude, i.e. to not delete')
@click.option('--tag', help='fnmatch pattern to delete')
@click.option('--dry-run', is_flag=True, help='Don\'t actually delete anything')
@click.pass_context
def delete_image(ctx, repository, project, all, exclude_tag, tag, dry_run):
    '''
    Delete images from the container registry.

    One of --tag, --exclude-tag or --all is required.

    Use --tag to delete a single tag only, use --exclude-tag to delete all
    but the given tag. Use --all to delete all images.

    Use with --repository to limit to one repository only.

    The tool is designed to use the GitLab environment variables for user,
    passwords, projects, etc. where possible.

    Where other authentication is needed, use a --authfile containing the
    string that is your gitlab private token value with 'api' access.
    This tool will strip any whitespaces from that file and use the rest of
    the file as token value.
    '''
    if not tag and not exclude_tag and not all:
817
        raise click.UsageError('One of --tag, --exclude-tag, or --all is required.')
818

819 820
    p = gitlab_project(ctx.obj.gitlab, project)
    if not p:
821 822
        raise click.BadParameter('Unable to find or identify project')

823 824
    repos = [r for r in p.repositories.list() if repository is None or repository == r.name]
    for repo in repos:
825
        logger.debug(f'Repository {repo.name}')
826 827 828 829 830 831 832 833 834 835 836 837 838
        for t in repo.tags.list(all=True):
            if all:
                do_delete = True
            elif fnmatch.fnmatchcase(t.name, exclude_tag or '\0'):
                do_delete = False
            elif fnmatch.fnmatchcase(t.name, tag or '\0'):
                do_delete = True
            elif exclude_tag:
                do_delete = True
            else:
                do_delete = False

            if do_delete:
839
                logger.info(f'Deleting tag {repo.name}:{t.name}')
840 841 842 843 844
                if not dry_run:
                    try:
                        t.delete()
                    except gitlab.exceptions.GitlabDeleteError as e:
                        if e.response_code == 403:
845
                            raise click.BadParameter(f'Insufficient permissions to delete tag {repo.name}:{t.name}')
846 847 848 849
                        else:
                            raise e


850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874
@ci_fairy.command()
@click.option('--filename', help='The file to run through the linter (default: .gitlab-ci.yml)')
@click.pass_context
def lint(ctx, filename):
    '''
    Run the .gitlab-ci.yml through the gitlab instance's linter.

    If no filename is given, this tool searches for a .gitlab-ci.yml in the
    current path. If none exists, it will search the parent directories for
    one.
    '''
    if not filename:
        f = Path('.gitlab-ci.yml').resolve()
        if not f.exists():
            for p in f.parents:
                f = Path(p / '.gitlab-ci.yml')
                if f.exists():
                    filename = f
                    break
                # Search upwards but only until within the current git tree
                # (if any)
                if Path(f.parent / '.git').exists():
                    break

            if not filename:
875
                raise click.UsageError('Unable to find .gitlab-ci.yml')
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
        else:
            filename = f
    else:
        filename = Path(filename)

    logger.debug(f'linting file {filename}')

    with open(filename) as fd:
        status, errors = ctx.obj.gitlab.lint(fd.read())
        if status:
            logger.info(f'{filename} has no linting errors')
        else:
            logger.error(f'{filename} failed linting:')
            for e in errors:
                logger.error(e)


893
@ci_fairy.command()
894 895
@click.option('--config', help='YAML configuration file', required=False, multiple=True)
@click.option('--root', help='YAML node to use as root node', type=str, multiple=True)
896
@click.option('--output-file', '-o', help='output file to write to (default: stdout)')
897
@click.option('--verify', is_flag=True, help='Compare the generated file against the existing one and fail if they differ')
898
@click.argument('template', required=False)
899
@click.pass_context
900
def generate_template(ctx, config, root, template, output_file, verify):
901 902 903 904
    '''
    Generate a file based on the given Jinja2 template and the YAML
    configuration file.

905 906 907 908 909 910 911
    If called without any arguments, this command reads the
    .gitlab-ci/config.yml and .gitlab-ci/ci.template files and (over)writes
    the .gitlab-ci.yml file.

    Otherwise, both --config and the template must be provided on the
    commandline.

912 913 914
    Where the --root option is specified, the children of that root node are
    the data to be used in the template. For nested root nodes, use
    /path/to/node.
915 916 917 918

    Where the --verify option is given, the output file is not overwritten.
    Instead, ci-fairy compares the newly generated to the existing output
    file and exits with an error if the two differs.
919
    '''
920
    if len(root) > 0 and len(root) != len(config):
921
        raise click.UsageError('--root must be given for each --config')
922 923

    if template is None:
924 925
        if not config:
            config = ['.gitlab-ci/config.yml']
926 927 928 929
            template = '.gitlab-ci/ci.template'
            if output_file is None:
                output_file = '.gitlab-ci.yml'
        else:
930
            raise click.UsageError('--config and template are required')
931
    elif not config:
932
        raise click.UsageError('--config and template are required')
933

934 935 936
    for c in config:
        configfile = Path(c)
        if configfile.name != '-' and not configfile.is_file():
937
            raise click.BadParameter(f'config file {configfile} does not exist or is not a file')
938
        elif configfile.name == '-' and len(config) > 1:
939
            raise click.UsageError('stdin is only supported for a single --config option')
940 941 942

    templatefile = Path(template)
    if not templatefile.is_file():
943
        raise click.BadParameter(f'template file {templatefile} does not exist or is not a file')
944

945 946 947 948 949 950
    data = {}
    for cfile, rootnode in itertools.zip_longest(config, root):
        if cfile == '-':
            fd = sys.stdin
        else:
            fd = open(cfile)
951

952 953 954 955 956 957 958 959 960
        cdata = yaml.safe_load(fd)
        if rootnode:
            try:
                if rootnode.startswith('/'):
                    for r in rootnode[1:].split('/'):
                        cdata = cdata[r]
                else:
                    cdata = cdata[rootnode]
            except KeyError:
961
                raise click.BadParameter(f'Node {r} not found')
962 963

        data.update(cdata)
964 965

    templatedir = templatefile.parent
966
    env = jinja2.Environment(loader=jinja2.FileSystemLoader(os.fspath(templatedir)),
967 968
                             trim_blocks=True, lstrip_blocks=True,
                             extensions=['jinja2.ext.do'])
969
    env.globals["ci_fairy"] = _env_ci_fairy
970
    template = env.get_template(templatefile.name)
971 972 973 974 975 976 977

    if output_file is None:
        outfile = sys.stdout
        output_file = 'stdout'
    else:
        outfile = open(Path(output_file), 'r' if verify else 'w')

978 979
    if verify:
        import difflib
980 981 982 983 984 985 986 987
        newfile = template.render(data)
        oldfile = outfile.read()

        # difflib requires a weird format and we may insert line endings
        # converting to that. Jinja2 does not insert trailing
        # newlines for the last line, so even where we differ on the last
        # newline, we'll assume the files are identical.
        #
988
        # both need to be in format [ 'line1\n', 'line2\n', ...]
989 990 991
        newfile = [f'{l}\n' for l in newfile.split('\n')]
        oldfile = [f'{l}\n' for l in oldfile.split('\n')]

992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
        diff = difflib.unified_diff(oldfile,
                                    newfile,
                                    tofile=f'{output_file} (generated)',
                                    fromfile=output_file)

        red = ''
        green = ''
        reset = ''
        try:
            if os.isatty(sys.stdout.fileno()):
                red = colored.fg('red')
                green = colored.fg('green')
                reset = colored.attr('reset')
        # pytest replaces sys.stdout and throws an exception on fileno()
        except io.UnsupportedOperation:
            pass

1009 1010
        # diff evaluates to True even where it's empty...
        have_diff = False
1011
        for l in diff:
1012
            have_diff = True
1013 1014 1015 1016 1017 1018 1019 1020
            if l[0] == '-':
                prefix = red
            elif l[0] == '+':
                prefix = green
            else:
                prefix = ''
            print(f'{prefix}{l}{reset}', end='')

1021
        if have_diff:
1022 1023 1024
            sys.exit(1)
    else:
        template.stream(data).dump(outfile)
1025 1026


1027 1028 1029 1030 1031 1032 1033 1034
@ci_fairy.command()
@click.argument('commit-range', type=str, required=False, default='HEAD')
@click.option('--branch', type=str,
              help='Check commits compared to this branch '
                   '(default: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME if present or master')
@click.option('--signed-off-by/--no-signed-off-by', default=None,
              help='Require a Signed-off-by tag to be (or not be) present')
@click.option('--textwidth', type=int, default=80,
1035
              help='Require the commit message subject to be less than N characters wide.')
1036 1037
@click.option('--rules-file', type=click.File(), required=False,
              help='Custom rules definitions in YAML format')
1038 1039
@click.option('--junit-xml', help='junit output file to write to')
@click.pass_context
1040
def check_commits(ctx, commit_range, branch, signed_off_by, textwidth, rules_file, junit_xml):
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
    '''
    Check a commit range for some properties. Some checks are always
    performed, others can be changed with commandline arguments.

    Where a commit range is given, that range is validated. Otherwise, the
    commit range is "branch..HEAD", where branch is either the given one, or
    $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, or master. When run in the GitLab
    CI environment, the FDO_UPSTREAM_REPO is added and the commit range
    defaults to "upstream-repo/branchname..HEAD".
    '''
1051
    import git
1052 1053 1054 1055

    try:
        repo = git.Repo('.', search_parent_directories=True)
    except git.exc.InvalidGitRepositoryError:
1056
        raise click.UsageError('This must be run from within the git repository')
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087

    # if we're running in the CI we add the upstream project
    # as remote 'cifairy' so we can build the commit range appropriately.
    # We don't do this when running locally because we don't want to mess
    # with a user's git repo.
    if os.getenv('CI'):
        upstream = os.getenv('FDO_UPSTREAM_REPO')
        if not upstream:
            logger.warning('$FDO_UPSTREAM_REPO not set, using local branches to compare')
            upstream = os.getenv('CI_PROJECT_PATH')
        host = os.getenv('CI_SERVER_HOST')
        url = f'https://{host}/{upstream}'

        remote = 'cifairy'
        if remote not in repo.remotes:
            upstream = repo.create_remote(remote, url)
        else:
            upstream = repo.remotes[remote]
        upstream.fetch()
        remote_prefix = f'{remote}/'
    else:
        remote_prefix = ''

    # a user-specified range takes precedence. if it's just a single commit
    # sha fall back to the target_branch..$sha range
    if '..' not in commit_range:
        sha = commit_range
        target_branch = branch or os.getenv('CI_MERGE_REQUEST_TARGET_BRANCH_NAME') or 'master'
        commit_range = f'{remote_prefix}{target_branch}..{sha}'
    logger.debug(f'Using commit range {commit_range}')

1088
    if rules_file is None:
1089
        default_rules = Path('.gitlab-ci/commit-rules.yml')
1090 1091 1092 1093 1094 1095 1096 1097 1098
        if default_rules.exists():
            rules_file = default_rules.open()

    if rules_file is not None:
        yml = os.path.expandvars(rules_file.read())
        rules = yaml.safe_load(yml)
    else:
        rules = None

1099 1100 1101 1102 1103 1104 1105 1106
    failures = []
    for commit in repo.iter_commits(commit_range):
        validator = GitCommitValidator(commit)
        validator.check()
        if signed_off_by is not None:
            validator.check_sob(signed_off_by)
        if textwidth > 0:
            validator.check_text_width(textwidth)
1107 1108
        if rules is not None:
            validator.check_patterns(rules)
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121

        if validator.failed:
            failures.append(validator)
            logger.error(validator.error)

    if junit_xml and failures:
        sha = str(next(repo.iter_commits('HEAD')))[:8]
        make_junit_xml(junit_xml, f'commitmsg.{sha}', f'Commit message check for {sha}',
                       [f.junit_tuple for f in failures])

    sys.exit(1 if failures else 0)


1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
@ci_fairy.command()
@click.option('--project', help='Project the merge request is filed against, e.g. freedesktop/ci-templates')
@click.option('--merge-request-iid', help='The merge request IID to check')
@click.option('--junit-xml', help='junit output file to write to')
@click.option('--require-allow-collaboration', is_flag=True, help='Check that allow_collaboration is set')
@click.pass_context
def check_merge_request(ctx, project, merge_request_iid, junit_xml, require_allow_collaboration):
    '''
    Checks the given merge request in the project for various settings.
    Currently supported:

Jonas Ådahl's avatar
Jonas Ådahl committed
1133
    --require-allow-collaboration: checks that the ``allow_collaboration``
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
        boolean flag is set (the one that allows maintainers to rebase an
        MR).

    If the requirements are not set, this commands exit with status code 1.
    If a junit output is given, this command writes a junit XML file with an
    appropriate error message.

    This command defaults to CI_MERGE_REQUEST_PROJECT_ID and CI_MERGE_REQUEST_IID,
    see https://docs.gitlab.com/ce/ci/merge_request_pipelines/index.html

    Where not present, it uses FDO_UPSTREAM_REPO to find the upstream
    repository and finds the merge request associated with this commit. This
    relies on CI_COMMIT_SHA.

1148
    This command exits with status 3 if no suitable merge request could be
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
    found.
    '''
    fallback = None
    if not project:
        fallback = \
            os.getenv('CI_MERGE_REQUEST_PROJECT_ID') or \
            os.getenv('FDO_UPSTREAM_REPO') or \
            os.getenv('CI_PROJECT_ID')

    p = gitlab_project(ctx.obj.gitlab, project, fallback=fallback)
    if not p:
1160
        raise click.BadParameter('Unable to find or identify project')
1161 1162

    if not require_allow_collaboration:
1163
        raise click.UsageError('At least one check must be specified')
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175

    failures = []
    exit_status = 0

    # First check: CI_MERGE_REQUEST_IID. If we have this one we can use it
    # directly but since that requires a detached pipeline (and rules: in
    # the gitlab-ci.yml file) most projects won't bother, so...
    if merge_request_iid is None:
        merge_request_iid = os.getenv('CI_MERGE_REQUEST_IID')
    if merge_request_iid is not None:
        mr = p.mergerequests.get(merge_request_iid)
        if not mr:
1176 1177
            raise click.BadParameter(f'Merge request IID {merge_request_iid} does not exist')

1178 1179 1180 1181 1182 1183 1184 1185
    # ... the fallback is to check FDO_UPSTREAM_REPO for all open merge requests
    # and compare their sha with the sha of our current pipeline (or HEAD if
    # run locally). But a branch may not have an MR, so we exit with a
    # different exit code in that case.
    else:
        logger.debug('This is not a merge pipeline, searching for MR')
        sha = os.getenv('CI_COMMIT_SHA')
        if not sha:
1186 1187
            import git

1188 1189 1190 1191 1192 1193
            try:
                repo = git.Repo('.')
                sha = repo.commit('HEAD').hexsha
            except git.exc.InvalidGitRepositoryError:
                pass
        if not sha:
1194
            raise click.BadParameter('Cannot find git sha, unable to search for merge request')
1195 1196

        mr = next(iter([m for m in p.mergerequests.list(state='opened', per_page=100) if m.sha == sha]), None)
1197 1198 1199 1200 1201 1202 1203 1204 1205
        # No open MR with our sha? In the pipeline run after merging
        # the MR is already in 'merged' state - too late for our checks.
        if not mr:
            merged = [m for m in p.mergerequests.list(state='merged', order_by='updated_at', per_page=100) if m.sha == sha]
            mr = next(iter(merged), None)
            if mr:
                logger.info(f'Merge request !{mr.id} is already merged, skipping checks')
                sys.exit(0)

1206 1207 1208
        if not mr:
            # Not having a merge request *may* be fine so let's use a
            # special exit code.
1209
            exit_status = 3
1210 1211 1212 1213 1214 1215 1216 1217
            tcaseid = 'mr_is_filed'
            tcasename = 'Check a merge request is filed'
            message = f'No open merge request against {p.path_with_namespace} with sha {sha}'
            failures.append((tcaseid, tcasename, message))
            logger.error(message)

    if exit_status == 0:
        if require_allow_collaboration:
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
            try:
                if not mr.allow_collaboration:
                    exit_status = 1
                    tcaseid = 'allow_collaboration'
                    tcasename = 'Check allow_collaboration checkbox is set'
                    message = '''
    Error: This merge request does not allow edits from maintainers.

    Please edit the merge request and set the checkbox to
    "Allow commits from members who can merge to the target branch"
    See https://docs.gitlab.com/ce/user/project/merge_requests/allow_collaboration.html'''
                    failures.append((tcaseid, tcasename, message))
                    logger.error(message)
1231
            except AttributeError:
1232 1233 1234
                # filing an MR against master in your own personal repo
                # doesn't have that checkbox
                pass
1235 1236 1237

    if failures:
        if junit_xml is not None:
1238 1239
            suite_id = f'{p.name}.merge_request.{merge_request_iid}'
            suite_name = f'Merge request check ({p.name} !{merge_request_iid})'
1240 1241 1242 1243
            make_junit_xml(junit_xml, suite_id, suite_name, failures)
        sys.exit(exit_status)


1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
@ci_fairy.command()
@click.option('--project', help='Project to check, e.g. freedesktop/ci-templates')
@click.option('--latest', is_flag=True, help='Use the most recent pipeline instead of the sha')
@click.option('--sha', help='Use a specific sha instead of HEAD', default='HEAD')
@click.option('--interval', help='The polling interval in seconds', type=int, default=30)
@click.pass_context
def wait_for_pipeline(ctx, project, sha, latest, interval):
    '''
    Waits for a pipeline to complete, printing the status of various jobs on
    the terminal. This tool must be called from within the git repository to
    automatically resolve the pipelines.

    The project defaults to $GITLAB_USER_ID/$basename or $USER/$basename if
    the former isn't set.
    '''
    if not project:
        userid = os.environ.get('GITLAB_USER_ID') or os.environ.get('USER')
        if not userid:
1262
            raise click.BadParameter('Unable to find or identify project')
1263 1264 1265 1266 1267

        project = f'{userid}/{Path.cwd().name}'

    p = gitlab_project(ctx.obj.gitlab, project, fallback=None)
    if not p:
1268
        raise click.BadParameter('Unable to find or identify project')
1269 1270 1271 1272

    if latest:
        pipelines = p.pipelines.list