Commit d57148ef authored by Benjamin Tissoires's avatar Benjamin Tissoires
Browse files

ci_fairy: add support for include/extends in ci-fairy config files

taken from https://github.com/whot/uji/blob/master/uji.py

 also licensed
under MIT
Signed-off-by: Benjamin Tissoires's avatarBenjamin Tissoires <benjamin.tissoires@gmail.com>
parent b20a5355
......@@ -166,7 +166,7 @@ ci-fairy images:
.ci-fairy-tag:
variables:
FDO_DISTRIBUTION_TAG: sha256-c953306ebd442ff7a5a15f7c59d337a58dd435e7efc103329ea279178ef46680
FDO_DISTRIBUTION_TAG: sha256-f5d8b85463265e195c3b1ab703d23b7dd640598d6f0a50389950424f2d210840
.ci-fairy-local-image:
extends:
......@@ -355,7 +355,7 @@ test published images:
stage: test published images
script:
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:buildah-2021-03-18.0
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-c953306ebd442ff7a5a15f7c59d337a58dd435e7efc103329ea279178ef46680
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f5d8b85463265e195c3b1ab703d23b7dd640598d6f0a50389950424f2d210840
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-base-2021-03-18.0
- skopeo inspect docker://quay.io/freedesktop.org/ci-templates:qemu-mkosi-base-2021-03-18.0
rules:
......
......@@ -23,7 +23,7 @@ ci-fairy-base-image:
.ci-fairy-tag:
variables:
FDO_DISTRIBUTION_TAG: sha256-c953306ebd442ff7a5a15f7c59d337a58dd435e7efc103329ea279178ef46680
FDO_DISTRIBUTION_TAG: sha256-f5d8b85463265e195c3b1ab703d23b7dd640598d6f0a50389950424f2d210840
# The actual ci-fairy image with ci-fairy installed
# This image uses the sha of the ci-fairy script itself as tag.
......
......@@ -31,6 +31,6 @@
# Variables provided by this template should be considered read-only.
#
.fdo.ci-fairy:
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-c953306ebd442ff7a5a15f7c59d337a58dd435e7efc103329ea279178ef46680
image: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f5d8b85463265e195c3b1ab703d23b7dd640598d6f0a50389950424f2d210840
variables:
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-c953306ebd442ff7a5a15f7c59d337a58dd435e7efc103329ea279178ef46680
\ No newline at end of file
FDO_DISTRIBUTION_IMAGE: quay.io/freedesktop.org/ci-templates:ci-fairy-sha256-f5d8b85463265e195c3b1ab703d23b7dd640598d6f0a50389950424f2d210840
\ No newline at end of file
......@@ -3,6 +3,7 @@
import boto3
import botocore
import click
import collections
import colored
import functools
import gitdb
......@@ -22,6 +23,7 @@ import time
import urllib.parse
import yaml
from botocore.client import Config
from copy import deepcopy
from pathlib import Path
......@@ -892,6 +894,214 @@ def lint(ctx, filename):
logger.error(e)
class YamlError(Exception):
pass
class ExtendedYaml(collections.UserDict):
'''
A version of YAML that supports extra keywords.
Requirements
============
An extended YAML file must be a dictionary.
Features
========
extends:
--------
Supported in: top-level dictionaries
The ``extends:`` keyword makes the current dictionary inherit all
members of the extended dictionary, according to the following rules::
- where the value is a non-empty dict, the base and new dicts are merged
- where the value is a non-empty list, the base and new list are
concatinated
- where the value is an empty dict or empty list, the new value is the
empty dict/list.
- otherwise, the new value overwrites the base value.
Example::
foo:
bar: [1, 2]
baz:
a: 'a'
b: 'b'
bat: 'foobar'
subfoo:
extends: bar
bar: [3, 4]
baz:
c: 'c'
bat: 'subfoobar'
Results in the effective values for subfoo::
subfoo:
bar: [1, 2, 3, 4]
baz: {a: 'a', b: 'b', c: 'c'}
bat: 'subfoobar'
foo:
bar: 1
include:
--------
Supported in: top-level only
The ``include:`` keyword includes the specified file at the place. The
path to the included file is relative to the source file.
Example::
# content of firstfile
foo:
bar: [1, 2]
#content of secondfile
bar:
baz: [3, 4]
include: firstfile
foobar:
extends: foo
Not that the included file will work with the normal YAML rules,
specifically: where the included file has a key with the same name this
section will be overwritten with the later-defined. The position of the
``include`` statement thus matters a lot.
version:
--------
The YAML file version (optional). Handled as attribute on this object and does not
show up in the dictionary otherwise. This version must be a top-level
entry in the YAML file in the form "version: 1" or whatever integer
number and is exposed as the "version" attribute.
Where other files are included, the version of that file must be
identical to the first version found in any file.
:: attribute: version
An optional attribute with the version number as specified in the
YAML file(s).
'''
def __init__(self, include_path=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.include_path = include_path
def __load(self, stream):
data = io.StringIO()
self.__process_includes(stream, dest=data)
data = yaml.safe_load(data.getvalue())
if not isinstance(data, dict):
raise YamlError('Invalid YAML data format, expected a dictionary')
if data.get('extends'):
raise YamlError('Invalid section name "extends", this is a reserved keyword')
data = self.__process_extends(data)
for k, v in data.items():
self[k] = v
def __process_includes(self, source, dest, level=0):
'''
Handles include: statements. Reads the source line-by-line
and write it to the destination. Where a ``include: filename`` line
is present, the filename is opened and this function is called
recursively for that file.
'''
if level > 10: # 10 levels of includes must be enough.
return ''
for line in source:
# version check - all included files must have the same version
# as the original file
if line.startswith('version:'):
version = int(line[len('version:'):].strip())
try:
if self.version != version:
raise YamlError(f'Cannot include file {source}, version mismatch')
except AttributeError:
self.version = version
continue
if not line.startswith('include: '):
dest.write(line)
continue
# used for test cases only, really. all uji user cases use a
# file anyway, not a string.
if not self.include_path:
raise YamlError('Cannot include from a text stream')
filename = line[len('include:'):].strip()
with open(Path(self.include_path) / filename) as included:
self.__process_includes(included, dest, level + 1)
def __process_extends(self, yaml):
def merge(a, b):
'''
Helper function to a and b together:
- Where a and b are lists, the result is a + b
- Where a and b are dicts, the result is the union of a and b
- Otherwise, the result is b
This performs a deep copy of all lists/dicts so the result does what
you'd expect.
'''
if type(a) != type(b):
raise ValueError()
if isinstance(a, list):
return a + b
elif isinstance(a, dict):
merged = {}
for k, v in a.items():
merged[k] = v
for k, v in b.items():
merged[k] = v
return merged
else:
return b
# yaml is modified in the loop, so we have to list() it
for section, data in list(yaml.items()):
if not isinstance(data, dict):
continue
referenced = data.get('extends')
if not referenced:
continue
if referenced not in yaml or referenced == section:
raise YamlError(f'Invalid section for "extends: {referenced}"')
# We need deep copies to avoid references to lists within dicts,
# etc.
combined = deepcopy(yaml[referenced])
data = deepcopy(data)
for item, value in data.items():
if item == 'extends':
continue
try:
base = combined[item]
except KeyError:
# base doesn't have this key, so we can just
# write it out
combined[item] = value
else:
try:
combined[item] = merge(base, value)
except ValueError:
raise YamlError(f'Mismatched types for {item} in {section} vs {referenced}')
yaml[section] = combined
return yaml
@classmethod
def load_from_file(cls, filename):
from pathlib import Path
path = Path(filename)
if not path.is_file():
raise YamlError(f'"{filename}" is not a file')
with open(path) as f:
yml = ExtendedYaml(include_path=Path(filename).parent)
yml.__load(f)
return yml
@classmethod
def load_from_stream(cls, stream):
yml = ExtendedYaml()
yml.__load(io.StringIO(stream))
return yml
class ci_fairyFileSystemLoader(jinja2.FileSystemLoader):
def get_source(self, environment, template):
source, filename, uptodate = super().get_source(environment, template)
......@@ -958,11 +1168,10 @@ def generate_template(ctx, config, root, template, output_file, verify):
data = {}
for cfile, rootnode in itertools.zip_longest(config, root):
if cfile == '-':
fd = sys.stdin
cdata = ExtendedYaml.load_from_stream(sys.stdin)
else:
fd = open(cfile)
cdata = ExtendedYaml.load_from_file(cfile)
cdata = yaml.safe_load(fd)
if rootnode:
try:
if rootnode.startswith('/'):
......
......@@ -7,8 +7,10 @@ import git
import json
import pytest
from pathlib import Path
from textwrap import dedent
import ci_fairy
from ci_fairy import ExtendedYaml, YamlError
GITLAB_TEST_URL = 'https://test.gitlab.url'
GITLAB_TEST_PROJECT_ID = '11'
......@@ -764,6 +766,181 @@ def test_template_verify(last_line):
assert result.exit_code == 1
def test_template_extendedyaml_load():
yml = ExtendedYaml.load_from_stream('foo: bar')
assert yml is not None
def test_template_extendedyaml_load_invalid():
with pytest.raises(YamlError):
ExtendedYaml.load_from_stream('1')
with pytest.raises(YamlError):
ExtendedYaml.load_from_stream('[1, 2]')
with pytest.raises(YamlError):
ExtendedYaml.load_from_stream('[1, 2]')
with pytest.raises(YamlError):
ExtendedYaml.load_from_stream('foo')
def test_template_extendedyaml_key_access():
yml = ExtendedYaml.load_from_stream('foo: bar\nbaz: bat')
assert yml['foo'] == 'bar'
assert yml['baz'] == 'bat'
yml = ExtendedYaml.load_from_stream('foo: [1, 2]\nbaz: bat')
assert yml['foo'] == [1, 2]
assert yml['baz'] == 'bat'
yml = ExtendedYaml.load_from_stream('foo: [1, 2]\nbaz: 3')
assert yml['foo'] == [1, 2]
assert yml['baz'] == 3
def test_template_extendedyaml_iteration():
data = dedent('''\
foo: bar
baz: bat
''')
yml = ExtendedYaml.load_from_stream(data)
assert [k for k in yml] == ['foo', 'baz']
assert [(k, v) for (k, v) in yml.items()] == [('foo', 'bar'), ('baz', 'bat')]
def test_template_extendedyaml_extends():
data = dedent('''\
foo:
one: two
bar:
extends: foo
''')
yml = ExtendedYaml.load_from_stream(data)
assert yml['foo']['one'] == 'two'
assert yml['bar']['one'] == 'two'
data = dedent('''\
foo:
one: two
bar:
extends: foo
one: three
''')
yml = ExtendedYaml.load_from_stream(data)
assert yml['foo']['one'] == 'two'
assert yml['bar']['one'] == 'three'
data = dedent('''\
foo:
one: [1, 2]
bar:
extends: foo
one: [3, 4]
''')
yml = ExtendedYaml.load_from_stream(data)
assert yml['foo']['one'] == [1, 2]
assert yml['bar']['one'] == [1, 2, 3, 4]
def test_template_extendedyaml_extends_invalid():
data = dedent('''\
foo:
one: [1, 2]
extends: bar
''')
with pytest.raises(YamlError) as e:
ExtendedYaml.load_from_stream(data)
assert 'Invalid section name' in e.message
data = dedent('''\
foo:
one: [1, 2]
bar:
extends: foobar
''')
with pytest.raises(YamlError) as e:
ExtendedYaml.load_from_stream(data)
assert 'Invalid section' in e.message
data = dedent('''\
foo:
one: [1, 2]
bar:
extends: bar
one: str
''')
with pytest.raises(YamlError) as e:
ExtendedYaml.load_from_stream(data)
assert 'Mismatched' in e.message
def test_template_extendedyaml_include():
runner = CliRunner()
with runner.isolated_filesystem():
with open('to-include.yaml', 'w') as fd:
fd.write('version: 1\n')
fd.write('\n')
fd.write('one:\n')
fd.write(' two: three\n')
fd.write(' four: five\n')
fd.write(' six: [7, 8]\n')
with open('include-test.yaml', 'w') as fd:
fd.write('foo:\n')
fd.write(' bar: baz\n')
fd.write('\n')
fd.write('include: to-include.yaml\n')
yml = ExtendedYaml.load_from_file(Path('include-test.yaml'))
assert yml['foo']['bar'] == 'baz'
assert yml['one']['two'] == 'three'
def test_template_extendedyaml_infinite_include():
runner = CliRunner()
with runner.isolated_filesystem():
with open('infinite-include.yaml', 'w') as fd:
fd.write('foo:\n')
fd.write(' bar: infinite\n')
fd.write('\n')
fd.write('include: infinite-include.yaml\n')
yml = ExtendedYaml.load_from_file('infinite-include.yaml')
assert yml['foo']['bar'] == 'infinite'
def test_template_extendedyaml_version():
data = dedent('''\
foo:
one: [1, 2]
''')
yml = ExtendedYaml.load_from_stream(data)
with pytest.raises(AttributeError):
yml.version
data = dedent('''\
version: 1
foo:
one: [1, 2]
''')
yml = ExtendedYaml.load_from_stream(data)
assert yml.version == 1
runner = CliRunner()
with runner.isolated_filesystem():
with pytest.raises(YamlError) as e:
with open('to-include.yaml', 'w') as fd:
fd.write('version: 1\n')
fd.write('\n')
fd.write('one:\n')
fd.write(' two: three\n')
fd.write(' four: five\n')
fd.write(' six: [7, 8]\n')
with open('wrong-version.yaml', 'w') as fd:
fd.write('version: 2\n')
fd.write('include: to-include.yaml\n')
ExtendedYaml.load_from_file(Path('wrong-version.yaml'))
assert 'version mismatch' in e.message
def test_commits_needs_git_repo(caplog):
runner = CliRunner()
with runner.isolated_filesystem():
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment