Commit 16f790f9 authored by Peter Hutterer's avatar Peter Hutterer
Browse files

ci-fairy: fix generate-template --verify



The previous version struggled with trailing linebreaks. Fix that by making
sure we ignore any changes on the final newline (Jinja2 doesn't write those
out anyway).

Let's make sure we read, split and rejoin old and new data in exactly the same
way so we don't have to deal with stray line endings.

And the generator returned by difflib is always True. Unclear why this worked
in my local tests when I wrote the original support...
Signed-off-by: Peter Hutterer's avatarPeter Hutterer <peter.hutterer@who-t.net>
parent 8445ff7a
Pipeline #233294 passed with stages
in 1 minute and 47 seconds
......@@ -130,7 +130,7 @@ ci-fairy images:
.ci-fairy-tag:
variables:
FDO_DISTRIBUTION_TAG: sha256-5237284c60722b1022de6f2cd2ff1dfb9ae17b024df80385efa23f28da18aad3
FDO_DISTRIBUTION_TAG: sha256-7c065dbe56844cd44c7149a82f53e9b9d7ac2b24d61daba53bf7a08615bdca38
.ci-fairy-local-image:
extends:
......@@ -268,7 +268,7 @@ test published images:
script:
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-aarch64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-x86_64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-5237284c60722b1022de6f2cd2ff1dfb9ae17b024df80385efa23f28da18aad3
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-7c065dbe56844cd44c7149a82f53e9b9d7ac2b24d61daba53bf7a08615bdca38
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-base-x86_64-2020-10-30.1
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-mkosi-base-x86_64-2020-10-30.1
rules:
......
......@@ -19,7 +19,7 @@ ci-fairy-base-image:
.ci-fairy-tag:
variables:
FDO_DISTRIBUTION_TAG: sha256-5237284c60722b1022de6f2cd2ff1dfb9ae17b024df80385efa23f28da18aad3
FDO_DISTRIBUTION_TAG: sha256-7c065dbe56844cd44c7149a82f53e9b9d7ac2b24d61daba53bf7a08615bdca38
# The actual ci-fairy image with ci-fairy installed
# This image uses the sha of the ci-fairy script itself as tag.
......
......@@ -31,6 +31,6 @@
# Variables provided by this template should be considered read-only.
#
.fdo.ci-fairy:
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-5237284c60722b1022de6f2cd2ff1dfb9ae17b024df80385efa23f28da18aad3
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-7c065dbe56844cd44c7149a82f53e9b9d7ac2b24d61daba53bf7a08615bdca38
variables:
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-5237284c60722b1022de6f2cd2ff1dfb9ae17b024df80385efa23f28da18aad3
\ No newline at end of file
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-7c065dbe56844cd44c7149a82f53e9b9d7ac2b24d61daba53bf7a08615bdca38
\ No newline at end of file
......@@ -953,9 +953,18 @@ def generate_template(ctx, config, root, template, output_file, verify):
if verify:
import difflib
newfile = template.render(data)
oldfile = outfile.read()
# difflib requires a weird format and we may insert line endings
# converting to that. Jinja2 does not insert trailing
# newlines for the last line, so even where we differ on the last
# newline, we'll assume the files are identical.
#
# both need to be in format [ 'line1\n', 'line2\n', ...]
newfile = [f'{l}\n' for l in template.render(data).split('\n')]
oldfile = outfile.readlines()
newfile = [f'{l}\n' for l in newfile.split('\n')]
oldfile = [f'{l}\n' for l in oldfile.split('\n')]
diff = difflib.unified_diff(oldfile,
newfile,
tofile=f'{output_file} (generated)',
......@@ -973,7 +982,10 @@ def generate_template(ctx, config, root, template, output_file, verify):
except io.UnsupportedOperation:
pass
# diff evaluates to True even where it's empty...
have_diff = False
for l in diff:
have_diff = True
if l[0] == '-':
prefix = red
elif l[0] == '+':
......@@ -982,7 +994,7 @@ def generate_template(ctx, config, root, template, output_file, verify):
prefix = ''
print(f'{prefix}{l}{reset}', end='')
if diff:
if have_diff:
sys.exit(1)
else:
template.stream(data).dump(outfile)
......
......@@ -622,30 +622,70 @@ def test_template_ci_fairy_sha256sum():
f'file4: {sha}'
def test_template_verify():
# Run twice, once with trailing newline in the source file, once without
@pytest.mark.parametrize("last_line",
['line to be removed\n', 'line to be removed'])
def test_template_verify(last_line):
runner = CliRunner()
with runner.isolated_filesystem():
with open('test.yml', 'w') as fd:
fd.write(DEFAULT_YAML)
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed\n')
fd.write(last_line)
# generate the first file
args = ['generate-template', '--config', 'test.yml', '--output-file', 'outfile', 'test.tmpl']
result = runner.invoke(ci_fairy.ci_fairy, args)
# File "output" is generated now
args.append('--verify')
# check we're happy with identical data
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('added line\n')
fd.write(last_line)
result = runner.invoke(ci_fairy.ci_fairy, args)
print(result.stdout)
assert result.exit_code == 0
args.append('--verify')
# add a line, remove a line
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('added line\n')
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
print(result.stdout)
assert '-line to be removed' in result.stdout
assert '+added line' in result.stdout
# changes in the trailing newline on last line is ignored
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed') # without newline
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
# changes in the trailing newline on last line is ignored
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed\n') # with newline
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 0
# but other whitespace is an error
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed ') # extra space
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
# but other whitespace is an error
with open('test.tmpl', 'w') as fd:
fd.write('three is {{numbers.three}}\n')
fd.write('line to be removed\n\n') # extra newline
result = runner.invoke(ci_fairy.ci_fairy, args)
assert result.exit_code == 1
def test_commits_needs_git_repo(caplog):
runner = CliRunner()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment