Commit 08d0abcd authored by Peter Hutterer's avatar Peter Hutterer

ci-fairy: add a wait-for-pipeline command

Sometimes, running a CI pipeline is a game of whack-a-mole with the various
configurations and different build targets. Let's add a ci-fairy command that
can be run after pushing to wait for the current pipeline(s) to finish.

Usage: ci-fairy wait-for-pipeline

There are a few optional arguments, but most of the time it should find the
information itself. It picks HEAD from cwd by default, figures out the repo
etc. and then monitors the running pipeline.

Note that the repo lookup is *not* using the git config because we cannot know
which repository is the intended one. We cannot use the current branch's
upstream either because there's a reasonable chance it's origin/master.

So we just assume that the pipeline after pushing runs in the user fork,
defined by $GITLAB_USER_ID (or as fallback $USER). Where this isn't
sufficient, specify --project.
Signed-off-by: Peter Hutterer's avatarPeter Hutterer <peter.hutterer@who-t.net>
parent 3f4e4de1
......@@ -322,3 +322,28 @@ is almost never what you really want. To work around this, the above example
defines global rules in ``workflow``, effectively providing a ``rules``
definition for all jobs. Ensure that this is compatible with your project's
CI.
.. _ci-fairy-wait-for-pipeline:
Waiting for a pipeline
----------------------
``ci-fairy`` can be used to wait for the completion of a pipeline. ::
$ ci-fairy wait-for-pipeline
In the default invocation with no arguments ``ci-fairy`` will guess the
gitlab project path based on the ``$GITLAB_USER_ID`` if set, or ``$USER``
otherwise, combined with the basename of the current directory. The pipline
to wait for defaults to the pipeline matching the current git sha::
$ whoami
bob
$ cd myproject
$ git push -q gitlab mybranch
$ ci-fairy wait-for-pipeline
Pipeline https://gitlab.freedesktop.org/bob/myproject/-/pipelines/12345
status: success | 90/90 | created: 0 | pending: 0 | running: 0 | failed: 1 | success: 89
Please see the ``--help`` output for more details on controlling which
pipeline to wait for.
......@@ -6,6 +6,7 @@ import click
import colored
import functools
import git
import gitdb
import gitlab
from gitlab import Gitlab
import jinja2
......@@ -15,6 +16,7 @@ import logging
import os
import shutil
import sys
import time
import urllib.parse
import yaml
from botocore.client import Config
......@@ -1101,6 +1103,119 @@ def check_merge_request(ctx, project, merge_request_iid, junit_xml, require_allo
sys.exit(exit_status)
@ci_fairy.command()
@click.option('--project', help='Project to check, e.g. freedesktop/ci-templates')
@click.option('--latest', is_flag=True, help='Use the most recent pipeline instead of the sha')
@click.option('--sha', help='Use a specific sha instead of HEAD', default='HEAD')
@click.option('--interval', help='The polling interval in seconds', type=int, default=30)
@click.pass_context
def wait_for_pipeline(ctx, project, sha, latest, interval):
'''
Waits for a pipeline to complete, printing the status of various jobs on
the terminal. This tool must be called from within the git repository to
automatically resolve the pipelines.
The project defaults to $GITLAB_USER_ID/$basename or $USER/$basename if
the former isn't set.
'''
if not project:
userid = os.environ.get('GITLAB_USER_ID') or os.environ.get('USER')
if not userid:
logger.error('Unable to find or identify project')
sys.exit(1)
project = f'{userid}/{Path.cwd().name}'
p = gitlab_project(ctx.obj.gitlab, project, fallback=None)
if not p:
logger.error('Unable to find or identify project')
sys.exit(1)
if latest:
pipelines = p.pipelines.list(per_page=1)
else:
try:
# If called from within a git repostory, try to resolve the
# symbolic ref correctly
repo = git.Repo(search_parent_directories=True)
sha = repo.rev_parse(sha)
except gitdb.exc.BadName:
logger.error(f'Unable to parse sha "{sha}"')
sys.exit(1)
except git.exc.InvalidGitRepositoryError:
pass
pipelines = p.pipelines.list(sha=sha)
if not pipelines:
logger.info('No matching pipeline found')
sys.exit(1)
if len(pipelines) > 1:
logger.warning('Multiple pipelines found, using most recent one')
pipeline = pipelines[0]
# Rough grouping of the various statuses we have
pipeline_statuses = {
'active': ['created', 'waiting_for_resource', 'preparing', 'pending', 'running'],
'failed': ['failed', 'canceled'],
'complete': ['success', 'skipped', 'manual', 'scheduled']
}
job_statuses = {
'active': ['created', 'pending', 'running'],
'failed': ['failed', 'canceled'],
'complete': ['skipped', 'manual', 'success'],
}
print(f'Pipeline {pipeline.web_url}')
exit_code = 0
pulse = 0
while True:
pipeline = p.pipelines.get(pipeline.id)
# Minor race condition: the pipeline may complete while we fetch the
# jobs below and we'd be printing the wrong status. Too niche to
# worry about.
if pipeline.status in pipeline_statuses['failed']:
exit_code = 1
jobs = pipeline.jobs.list(per_page=100)
active = [j for j in jobs if j.status in job_statuses['active']]
njobs = len(jobs)
ndone = njobs - len(active)
# We use this dict as histogram for each job status, ignoring
# skipped, canceled and manual since we don't usually need to
# monitor these.
statuses = {name: 0 for name in ['created', 'pending', 'running', 'failed', 'success']}
for j in jobs:
try:
statuses[j.status] += 1
except KeyError:
pass # not all job statuses are monitored
# Now assemble the output line
# \x1B[2K == "delete line"
output = f'\r\x1B[2K status: {pipeline.status[:9]:9s} | {ndone}/{njobs} '
for s, count in statuses.items():
output += f'| {s}: {count:2d} '
# The pulse is just there to make sure something is visibly moving
output += '.' * (pulse % 5)
pulse += 1
print(output, end='', flush=True)
if not active:
break
time.sleep(interval)
print('')
sys.exit(exit_code)
def main():
ci_fairy()
......
......@@ -84,6 +84,27 @@ def mock_gitlab(gitlab):
project.mergerequests.list = MagicMock(side_effect=mrlist)
project.mergerequests.get = MagicMock(side_effect=mrget)
pipelines = []
for i in range(5):
p = MagicMock()
p.status = {0: 'running',
1: 'failed',
2: 'success'}[i % 3]
p.id = i
p.sha = f'dead{i:08x}'
p.web_url = f'{GITLAB_TEST_URL}/-/pipelines/{p.id}'
pipelines.append(p)
def plget(pid):
return [p for p in pipelines if p.id == int(pid)][0]
def pllist(sha=None, per_page=None, order_by='created_at'):
# order_by is ignored for our tests
return [p for p in pipelines if sha is None or p.sha == sha]
project.pipelines.list = MagicMock(side_effect=pllist)
project.pipelines.get = MagicMock(side_effect=plget)
ctx = gitlab(GITLAB_TEST_URL)
ctx.projects.list = MagicMock(return_value=[project])
# This always returns our project, even where the ID doesn't match. Good
......@@ -802,6 +823,95 @@ def test_merge_request_already_merged(gitlab, caplog, gitlab_default_env):
assert 'Merge request !3 is already merged, skipping checks' in caplog.text
# fun fact: @patch is a stack, so the order of arguments matters
@patch('ci_fairy.Path.cwd', MagicMock(return_value=Path(GITLAB_TEST_PROJECT_PATH)))
@patch('ci_fairy.git')
@patch('ci_fairy.Gitlab')
def test_wait_for_pipeline(gitlab, git, caplog, gitlab_default_env):
args = ['-vv', 'wait-for-pipeline']
env = gitlab_default_env
env['USER'] = 'project12'
# a minimal git repository, we need this for the sha lookup
def mock_gitrepo(git, sha):
repo = MagicMock()
repo.rev_parse = MagicMock(return_value=sha)
git.Repo = MagicMock(return_value=repo)
# See mock pipelines setup, they're in order
# running/failed/success for each
mock_gitrepo(git, 'dead00000000')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
assert f'status: {"running":9s}' in result.output
args = ['-vv', 'wait-for-pipeline', '--sha=dead00000001']
mock_gitrepo(git, 'dead00000001')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
assert f'status: {"failed":9s}' in result.output
args = ['-vv', 'wait-for-pipeline', '--sha=dead00000002']
mock_gitrepo(git, 'dead00000002')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
assert f'status: {"success":9s}' in result.output
args = ['-vv', 'wait-for-pipeline', '--latest']
mock_gitrepo(git, 'dead00000003')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
assert f'status: {"running":9s}' in result.output
# Basically just an argparse check - because we don't set up the jobs in
# the pipeline we never actually invoke the interval. Something for
# later maybe
args = ['-vv', 'wait-for-pipeline', '--interval=5']
mock_gitrepo(git, 'dead00000003')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
assert f'status: {"running":9s}' in result.output
# Fallback to GITLAB_USER_ID
# we can't remove USER because it's in the os.env, but setting it to the
# empty string is enough for it to be False
env['USER'] = ''
env['GITLAB_USER_ID'] = 'project12'
args = ['-vv', 'wait-for-pipeline']
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
# we can't remove GITLAB_USER_ID because it'll be set in the pipeline,
# but setting it to the empty string is enough for it to be False
env['USER'] = ''
env['GITLAB_USER_ID'] = ''
args = ['-vv', 'wait-for-pipeline']
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
assert 'Unable to find or identify project' in caplog.text
args = ['-vv', 'wait-for-pipeline', f'--project={GITLAB_TEST_PROJECT_PATH}']
mock_gitrepo(git, 'dead00000003')
gitlab, project, _ = mock_gitlab(gitlab)
runner = CliRunner(env=env)
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
assert f'status: {"running":9s}' in result.output
def mock_s3_session(session):
client = MagicMock(name='client')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment