Commit 43ac932d authored by Peter Hutterer's avatar Peter Hutterer Committed by Benjamin Tissoires

ci-fairy: add a --verify command to generate-template

A number of projects have a job that runs generate-template followed by a diff
to make sure the generated file is the one checked in. Let's remove the manual
diff step from that and provide it as built-in functionality from ci-fairy.
Signed-off-by: Peter Hutterer's avatarPeter Hutterer <peter.hutterer@who-t.net>
[re-run generate_templates.py after rebase]
Signed-off-by: Benjamin Tissoires's avatarBenjamin Tissoires <benjamin.tissoires@gmail.com>
parent e3d23bc9
......@@ -206,7 +206,7 @@ test published images:
script:
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-aarch64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-x86_64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-8dfd966d36784771d67cdfca46bfe23ae7a320cbed3fb5c6f883508bd8d157c0
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f35ed97d970ffc3c1447bf388756ed6d54cc8e662f01718cb71aef2f650acece
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-base-x86_64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-mkosi-base-x86_64-2020-10-30.1
rules:
......
......@@ -14,7 +14,7 @@ ci-fairy-base-image:
.ci-fairy-tag:
variables:
FDO_DISTRIBUTION_TAG: sha256-8dfd966d36784771d67cdfca46bfe23ae7a320cbed3fb5c6f883508bd8d157c0
FDO_DISTRIBUTION_TAG: sha256-f35ed97d970ffc3c1447bf388756ed6d54cc8e662f01718cb71aef2f650acece
# The actual ci-fairy image with ci-fairy installed
# This image uses the sha of the ci-fairy script itself as tag.
......
......@@ -31,6 +31,6 @@
# Variables provided by this template should be considered read-only.
#
.fdo.ci-fairy:
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-8dfd966d36784771d67cdfca46bfe23ae7a320cbed3fb5c6f883508bd8d157c0
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f35ed97d970ffc3c1447bf388756ed6d54cc8e662f01718cb71aef2f650acece
variables:
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-8dfd966d36784771d67cdfca46bfe23ae7a320cbed3fb5c6f883508bd8d157c0
\ No newline at end of file
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f35ed97d970ffc3c1447bf388756ed6d54cc8e662f01718cb71aef2f650acece
\ No newline at end of file
......@@ -9,6 +9,7 @@ import git
import gitdb
import gitlab
from gitlab import Gitlab
import io
import jinja2
import json
import fnmatch
......@@ -855,9 +856,10 @@ def lint(ctx, filename):
@click.option('--config', help='YAML configuration file', required=False)
@click.option('--root', help='YAML node to use as root node', type=str)
@click.option('--output-file', '-o', help='output file to write to (default: stdout)')
@click.option('--verify', is_flag=True, help='Compare the generated file against the existing one and fail if they differ')
@click.argument('template', required=False)
@click.pass_context
def generate_template(ctx, config, root, template, output_file):
def generate_template(ctx, config, root, template, output_file, verify):
'''
Generate a file based on the given Jinja2 template and the YAML
configuration file.
......@@ -872,6 +874,10 @@ def generate_template(ctx, config, root, template, output_file):
Where the --root option is specified, the children of that root node are
the data to be used in the template. For nested root nodes, use
/path/to/node.
Where the --verify option is given, the output file is not overwritten.
Instead, ci-fairy compares the newly generated to the existing output
file and exits with an error if the two differs.
'''
if template is None:
......@@ -898,9 +904,10 @@ def generate_template(ctx, config, root, template, output_file):
sys.exit(1)
if output_file is None:
output_file = sys.stdout
outfile = sys.stdout
output_file = 'stdout'
else:
output_file = open(Path(output_file), 'w')
outfile = open(Path(output_file), 'r' if verify else 'w')
if configfile == '-':
fd = sys.stdin
......@@ -925,7 +932,41 @@ def generate_template(ctx, config, root, template, output_file):
extensions=['jinja2.ext.do'])
env.globals["ci_fairy"] = _env_ci_fairy
template = env.get_template(templatefile.name)
template.stream(data).dump(output_file)
if verify:
import difflib
# both need to be in format [ 'line1\n', 'line2\n', ...]
newfile = [f'{l}\n' for l in template.render(data).split('\n')]
oldfile = outfile.readlines()
diff = difflib.unified_diff(oldfile,
newfile,
tofile=f'{output_file} (generated)',
fromfile=output_file)
red = ''
green = ''
reset = ''
try:
if os.isatty(sys.stdout.fileno()):
red = colored.fg('red')
green = colored.fg('green')
reset = colored.attr('reset')
# pytest replaces sys.stdout and throws an exception on fileno()
except io.UnsupportedOperation:
pass
for l in diff:
if l[0] == '-':
prefix = red
elif l[0] == '+':
prefix = green
else:
prefix = ''
print(f'{prefix}{l}{reset}', end='')
if diff:
sys.exit(1)
else:
template.stream(data).dump(outfile)
@ci_fairy.command()
......
......@@ -534,6 +534,31 @@ def test_template_ci_fairy_hashfiles():
'file7: dbc23bd9d9e6d52829aefdca4da566620337c2136b7942af85b6e21f664f7684'
def test_template_verify():
runner = CliRunner()
with runner.isolated_filesystem():
with open('test.yml', 'w') as fd:
fd.write(DEFAULT_YAML)
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed\n')
# generate the first file
args = ['generate-template', '--config', 'test.yml', '--output-file', 'outfile', 'test.tmpl']
result = runner.invoke(ci_fairy.ci_fairy, args)
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('added line\n')
args.append('--verify')
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
print(result.stdout)
assert '-line to be removed' in result.stdout
assert '+added line' in result.stdout
def test_commits_needs_git_repo(caplog):
runner = CliRunner()
with runner.isolated_filesystem():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment