Commit a6ecc580 authored by Peter Hutterer's avatar Peter Hutterer
Browse files

Use ci-fairy to generate our templates

Calling ci-fairy from python looks a bit clumsy but since we need access to
the globals.yml for some of the variables, it's still better to do that than
use bash.
Signed-off-by: Peter Hutterer's avatarPeter Hutterer <>
parent 84ba0d08
......@@ -22,7 +22,8 @@ stages:
sanity check:
extends: .pip_install
- pip3 install --user jinja2 PyYAML
- pip3 install --user PyYAML
- pip3 install --user .
- python3 ./src/
- git diff --exit-code && exit 0 || true
#!/bin/env python3
import jinja2
import yaml
import re
import sys
import tempfile
from pathlib import Path
if __name__ == '__main__':
config_data = {}
env = jinja2.Environment(loader=jinja2.FileSystemLoader('./src'),
# load the various definitions in the provided YAML files
src_folder = Path('src')
with open(src_folder / 'globals.yml') as fd:
globs = yaml.load(fd, Loader=yaml.Loader)
sys.path.insert(0, './tools')
import ci_fairy # noqa
# load the example so we can verify for wrong keys in the distribution
# files
with open(src_folder / 'distribution.example') as fd:
root = next(iter(yaml.load(fd, Loader=yaml.Loader).values()))
allowed_keys = tuple(root.keys())
special_files = ('globals.yml')
for distrib in [x for x in src_folder.iterdir()
if ('.yml') and not in special_files)]:
with open(distrib) as fd:
data = yaml.load(fd, Loader=yaml.Loader)
def generate_template(configs, dest, template):
Invoke ci-fairy generate-template with the given arguments.
# for each root element in each yaml file
for config in data.values():
for key in config:
assert key in allowed_keys, f'Invalid key: {key}'
.. param:: configs
An array of tuples of type (path, rootnode).
# add missing keys from the globals
for key, value in globs.items():
assert key not in config
config[key] = value
config_args = []
for config, root in configs:
config_args += ['--config', f'{config}']
if root is not None:
config_args += ['--root', f'{root}']
# store the generated values in the base config dictionary
args = ['generate-template'] + config_args + ['--output-file', f'{dest}', f'{template}']
ci_fairy.main(args, standalone_mode=False)
out_folder = Path('templates')
if __name__ == '__main__':
src_folder = Path('src')
ci_folder = Path('.gitlab-ci')
# and render each distribution in the templates source directory
for distrib, config in sorted(config_data.items()):
# load our distribution template
template = env.get_template('template.tmpl')
# and our ci template
template_ci = env.get_template('template-ci.tmpl')
dest = out_folder / f'{distrib}.yml'
dest_ci = ci_folder / f'{distrib}-ci.yml'
print(f'generating {dest}')
with open(dest, 'w') as out_stream:
print(f'generating {dest_ci}')
with open(dest_ci, 'w') as out_stream:
# generate the bootstrap file
template_bootstrap = env.get_template('bootstrap.tmpl')
dest = '.gitlab-ci/bootstrap-ci.yml'
print(f'generating {dest}')
with open(dest, 'w') as out_stream:
globals_file = src_folder / 'globals.yml'
for distrib in [x for x in src_folder.iterdir()
if ('.yml') and x != globals_file)]:
name =[:-4] # drop .yml
# generate the distribution's template file,
# e.g. /templates/alpine.yml
template = src_folder / 'template.tmpl'
dest = Path('templates') / f'{name}.yml'
config = [(f'{globals_file}', ''), (f'{distrib}', f'{name}')]
generate_template(configs=config, dest=dest, template=template)
# generate our CI file, e.g. /.gitlab-ci/alpine-ci.yml
template = src_folder / 'template-ci.tmpl'
dest = ci_folder / f'{name}-ci.yml'
generate_template(configs=config, dest=dest, template=template)
# Generate the bootstrap-ci file
dest = ci_folder / 'bootstrap-ci.yml'
template = src_folder / 'bootstrap.tmpl'
config = [(f'{globals_file}', None)]
generate_template(configs=config, dest=dest, template=template)
# We've generated all the templates, search for any
# full image reference $registry/$path:$imagespec anywhere in your
# templates and add that to the remote_images so we can test those
# during the CI pipeline
with open(globals_file) as fd:
config = yaml.load(fd, Loader=yaml.Loader)
registry, path = config['ci_templates_registry'], config['ci_templates_registry_path']
pattern = re.compile(f'^[^#]+({registry}{path}:[\\w:.-]+)')
remote_images = []
......@@ -86,16 +71,21 @@ if __name__ == '__main__':
if matches:
# finally, regenerate the .gitlab-ci.yml
template_general_ci = env.get_template('gitlab-ci.tmpl')
distribs = [d for d in sorted(config_data.keys()) if d not in ('defaults', 'globals')]
dest = '.gitlab-ci.yml'
config = {'distribs': distribs,
'remote_images': sorted(set(remote_images))}
print(f'generating {dest}')
with open(dest, 'w') as out_stream:
# write out a temporary config file with the remote images and the
# distribution names. Use that to generate our .gitlab-ci.yml file
with tempfile.NamedTemporaryFile() as new_config:
s = '", "'.join(sorted(set(remote_images)))
new_config.write(f'remote_images: ["{s}"]\n'.encode('utf8'))
distributions = [[:-4] for d in src_folder.iterdir()
if ('.yml') and d != globals_file)]
s = '", "'.join(sorted(set(distributions)))
new_config.write(f'distribs: ["{s}"]\n'.encode('utf8'))
# now generate the .gitlab-ci.yml file
dest = Path('.gitlab-ci.yml')
template = src_folder / 'gitlab-ci.tmpl'
config = [(f'{globals_file}', None),
(f'{}', None)]
generate_template(configs=config, dest=dest, template=template)
......@@ -22,7 +22,8 @@ stages:
sanity check:
extends: .pip_install
- pip3 install --user jinja2 PyYAML
- pip3 install --user PyYAML
- pip3 install --user .
- python3 ./src/
- git diff --exit-code && exit 0 || true
......@@ -1296,8 +1296,8 @@ def wait_for_pipeline(ctx, project, sha, latest, interval):
def main():
def main(*args, **kwargs):
ci_fairy(*args, **kwargs)
if __name__ == '__main__':
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment