Commit 576ddac5 authored by Nirbheek Chauhan's avatar Nirbheek Chauhan 🐜
Browse files

Rewrite MSVC/MinGW toolchain config for UWP cross compilation

Unlike any other platform, on Windows we build recipes with both
MSVC and MinGW. All Meson recipes can be built with MSVC, but the
Autotools recipes always use MinGW.

This broke the fundamental assumption that `self.config.config_env`
contains all the information needed by the build system to find the
toolchain that we want to use.

At the time, we hacked around it by unsetting MinGW toolchain env vars
inside ``, and storing msvc toolchain env vars in
`self.config.msvc_toolchain_env`. We would then set those only when
building recipes that can use MSVC.

This completely breaks down when you want to cross-compile on Windows,
because we need some env vars to stay in the env, and others to only
be in the cross file.

To fix this, a bunch of things had to change:
1. Use actual objects for environment variable values: `EnvValue` and
   other classes that inherit from it. These allow easier
   merging/overriding of env var values.
2. Separate out env vars according to their stated purpose:
   a) Configuration for build tools (WINEPREFIX, WINDEBUG, etc)
   b) Configuration for toolchain (LIBRARY_PATH, INCLUDE, LIB, etc)
   c) Configuration for build system (CC, CFLAGS, ac_cv_*, etc)
3. Always use a native-file or a cross-file for selecting the
   toolchain when using Meson.
4. Rename meson_cross_properties to meson_properties, since we use
   those even when not cross-compiling.

Future TODO: port all env var usage in and config/*.config
to the new EnvValue* objects

Part-of: <gstreamer/cerbero!480>
parent 0b991206
......@@ -24,10 +24,12 @@ import shlex
import subprocess
import asyncio
from pathlib import Path
from itertools import chain
from cerbero.enums import Platform, Architecture, Distro, LibraryType
from cerbero.errors import FatalError
from cerbero.utils import shell, to_unixpath, add_system_libs
from cerbero.utils import EnvValue, EnvValueSingle, EnvValueArg, EnvValueCmd, EnvValuePath
from cerbero.utils import messages as m
......@@ -89,7 +91,10 @@ class EnvVarOp:
if self.var in env:
del env[self.var]
env[self.var] = self.sep.join(self.vals)
if len(self.vals) == 1:
env[self.var] = self.vals[0]
env[self.var] = self.sep.join(self.vals)
def append(self, env):
if self.var not in env:
......@@ -184,46 +189,37 @@ class ModifyEnvBase:
def setup_toolchain_env_ops(self):
if self.config.qt5_pkgconfigdir:
self.append_env('PKG_CONFIG_LIBDIR', self.config.qt5_pkgconfigdir, sep=os.pathsep)
if self.config.platform != Platform.WINDOWS:
if self.config.target_platform != Platform.WINDOWS:
if self.using_msvc():
toolchain_env = self.config.msvc_toolchain_env
if isinstance(self, Meson):
if self.using_msvc():
toolchain_env = self.config.msvc_env_for_toolchain.items()
toolchain_env = self.config.mingw_env_for_toolchain.items()
toolchain_env = self.config.mingw_toolchain_env
if self.using_msvc():
toolchain_env = chain(self.config.msvc_env_for_toolchain.items(),
toolchain_env = chain(self.config.mingw_env_for_toolchain.items(),
# Set the toolchain environment
for var, (val, sep) in toolchain_env.items():
for var, val in toolchain_env:
# We prepend PATH and replace the rest
if var == 'PATH':
self.prepend_env(var, val, sep=sep)
self.prepend_env(var, val.get(), sep=val.sep)
self.set_env(var, val, sep=sep)
self.set_env(var, val.get(), sep=val.sep)
def unset_toolchain_env(self):
# These toolchain env vars set by us are for GCC, so unset them if
# we're building with MSVC (or cross-compiling with Meson)
for var in ('CC', 'CXX', 'OBJC', 'OBJCXX', 'AR', 'WINDRES', 'STRIP',
if var in self.env:
# Env vars that are edited by the recipe will be restored by
# @modify_environment when we return from the build step but
# other env vars won't be, so add those.
self.set_env (var, None, when='now-with-restore')
if self.using_msvc():
# Restore msvc toolchain env which should be preserved
for key, (val, sep) in self.config.msvc_toolchain_env.items():
if var == key:
self.set_env(var, val, sep=sep, when='now')
# Re-add *FLAGS that weren't set by the toolchain config, but instead
# were set in the recipe or other places via @modify_environment
if self.using_msvc():
for each in self._new_env:
if var == each.var:
self.set_env(var, None, when='now-with-restore')
def check_reentrancy(self):
if self._old_env:
......@@ -254,7 +250,7 @@ class ModifyEnvBase:
# Store old env
for var in self._env_vars:
self._save_env_var (var)
# Modify env
for env_op in self._new_env:
......@@ -446,9 +442,6 @@ class MakefilesBase (Build, ModifyEnvBase):
if self.requires_non_src_build:
self.config_sh = os.path.join('../', self.config_sh)
if self.using_msvc():
substs = {
'config-sh': self.config_sh,
'prefix': self.config.prefix,
......@@ -482,8 +475,6 @@ class MakefilesBase (Build, ModifyEnvBase):
async def compile(self):
if self.using_msvc():
await shell.async_call(self.make, self.make_dir, logfile=self.logfile, env=self.env)
......@@ -677,8 +668,7 @@ class CMake (MakefilesBase):
self.make += ['VERBOSE=1']
await MakefilesBase.configure(self)
system = '{system}'
......@@ -692,14 +682,10 @@ endian = '{endian}'
c = {CC}
cpp = {CXX}
objc = {OBJC}
objcpp = {OBJCXX}
ar = {AR}
pkgconfig = 'pkg-config'
pkgconfig = {PKG_CONFIG}
......@@ -715,7 +701,6 @@ class Meson (Build, ModifyEnvBase) :
meson_sh = None
meson_options = None
meson_cross_properties = None
meson_backend = 'ninja'
# All meson recipes are MSVC-compatible, except if the code itself isn't
can_msvc = True
......@@ -729,10 +714,6 @@ class Meson (Build, ModifyEnvBase) :
cross_props = copy.deepcopy(self.config.meson_cross_properties)
cross_props.update(self.config.meson_cross_properties or {})
self.meson_cross_properties = cross_props
# Find Meson
if not self.meson_sh:
# meson installs `meson.exe` on windows and `meson` on other
......@@ -795,7 +776,7 @@ class Meson (Build, ModifyEnvBase) :
value = getattr(self.config.variants, variant_name) if variant_name else False
self.meson_options[opt_name] = self._get_option_value(opt_type, value)
def _get_cpu_family(self):
def _get_target_cpu_family(self):
if Architecture.is_arm(self.config.target_arch):
if Architecture.is_arm32(self.config.target_arch):
return 'arm'
......@@ -808,105 +789,150 @@ class Meson (Build, ModifyEnvBase) :
moc_name ='qmake', 'moc')
return str(qmake.parent / moc_name)
def _write_meson_cross_file(self):
# Take cross toolchain from _old_env because we removed them from the
# env so meson doesn't detect them as the native toolchain.
# Same for *FLAGS below.
if self.using_msvc():
cc = ['cl']
cxx = ['cl']
ar = ['lib']
cc = self.env['CC'].split()
cxx = self.env['CXX'].split()
ar = self.env['AR'].split()
strip = self.env.get('STRIP', '').split()
windres = self.env.get('WINDRES', '').split()
# We do not use cmake dependency files, speed up the build by disabling it
cross_binaries = {'cmake': ['false']}
if 'STRIP' in self.env:
cross_binaries['strip'] = self.env['STRIP'].split()
if 'WINDRES' in self.env:
cross_binaries['windres'] = self.env['WINDRES'].split()
if 'OBJC' in self.env:
cross_binaries['objc'] = self.env['OBJC'].split()
if 'OBJCXX' in self.env:
cross_binaries['objcpp'] = self.env['OBJCXX'].split()
if self.config.qt5_qmake_path:
cross_binaries['qmake'] = [self.config.qt5_qmake_path]
cross_binaries['moc'] = [self._get_moc_path(self.config.qt5_qmake_path)]
# *FLAGS are only passed to the native compiler, so while
# cross-compiling we need to pass these through the cross file.
c_args = shlex.split(self.env.get('CFLAGS', ''))
cpp_args = shlex.split(self.env.get('CXXFLAGS', ''))
objc_args = shlex.split(self.env.get('OBJCFLAGS', ''))
objcpp_args = shlex.split(self.env.get('OBJCXXFLAGS', ''))
# Link args
c_link_args = shlex.split(self.env.get('LDFLAGS', ''))
cpp_link_args = c_link_args
if 'OBJLDFLAGS' in self.env:
objc_link_args = shlex.split(self.env['OBJLDFLAGS'])
def _get_meson_target_file_contents(self):
Get the toolchain configuration for the target machine. This will
either go into a cross file or a native file depending on whether we're
cross-compiling or not.
def merge_env(old_env, new_env):
ret_env = {}
# Set/merge new values
for k, new_v in new_env.items():
new_v = EnvValue.from_key(k, new_v)
if k not in old_env:
ret_env[k] = new_v
old_v = old_env[k]
assert(isinstance(old_v, EnvValue))
if isinstance(old_v, (EnvValueSingle, EnvValueCmd)) or (new_v == old_v):
ret_env[k] = new_v
elif isinstance(old_v, (EnvValuePath, EnvValueArg)):
ret_env[k] = new_v + old_v
raise FatalError("Don't know how to combine the environment "
"variable '%s' with values '%s' and '%s'" % (k, new_v, old_v))
# Set remaining old values
for k in old_env.keys():
if k not in new_env:
ret_env[k] = old_env[k]
return ret_env
# Extract toolchain config for the build system from the appropriate
# config env dict. Start with `self.env`, since it contains toolchain
# config set by the recipe and when building for target platforms other
# than Windows, it also contains build tools and the env for the
# toolchain set by config/*.config.
# On Windows, the toolchain config is `self.config.msvc_env_for_build_system`
# or `self.config.mingw_env_for_build_system` depending on which toolchain
# this recipe will use.
if self.config.target_platform == Platform.WINDOWS:
if self.using_msvc():
build_env = dict(self.config.msvc_env_for_build_system)
build_env = dict(self.config.mingw_env_for_build_system)
objc_link_args = c_link_args
objcpp_link_args = objc_link_args
build_env = {}
# Override/merge toolchain env with recipe env and return a new dict
# with values as EnvValue objects
build_env = merge_env(build_env, self.env)
cc = build_env.pop('CC')
cxx = build_env.pop('CXX')
objc = build_env.pop('OBJC', [])
objcxx = build_env.pop('OBJCXX', [])
ar = build_env.pop('AR')
# We currently don't set the pre-processor or the linker when building with meson
build_env.pop('CPP', None)
build_env.pop('LD', None)
# Operate on a copy of the recipe properties to avoid accumulating args
# from all archs when doing universal builds
cross_properties = copy.deepcopy(self.meson_cross_properties)
for args in ('c_args', 'cpp_args', 'objc_args', 'objcpp_args',
'c_link_args', 'cpp_link_args', 'objc_link_args',
if args in cross_properties:
cross_properties[args] += locals()[args]
props = {}
build_env.pop('CPP', None) # Meson does not read this
build_env.pop('CPPFLAGS', None) # Meson does not read this, and it's duplicated in *FLAGS
props['c_args'] = build_env.pop('CFLAGS', [])
props['cpp_args'] = build_env.pop('CXXFLAGS', [])
props['objc_args'] = build_env.pop('OBJCFLAGS', [])
props['objcpp_args'] = build_env.pop('OBJCXXFLAGS', [])
# Link args
props['c_link_args'] = build_env.pop('LDFLAGS', [])
props['cpp_link_args'] = props['c_link_args']
props['objc_link_args'] = build_env.pop('OBJLDFLAGS', props['c_link_args'])
props['objcpp_link_args'] = props['objc_link_args']
for key, value in self.config.meson_properties.items():
if key not in props:
props[key] = value
cross_properties[args] = locals()[args]
props[key] += value
# We do not use cmake dependency files, speed up the build by disabling it
binaries = {'cmake': ['false']}
# Get qmake and moc paths
if self.config.qt5_qmake_path:
binaries['qmake'] = [self.config.qt5_qmake_path]
binaries['moc'] = [self._get_moc_path(self.config.qt5_qmake_path)]
# Try to detect build tools in the remaining env vars
for name, tool in build_env.items():
if tool and shutil.which(tool[0]):
binaries[name.lower()] = tool
extra_properties = ''
for k, v in cross_properties.items():
for k, v in props.items():
extra_properties += '{} = {}\n'.format(k, str(v))
extra_binaries = ''
for k, v in cross_binaries.items():
for k, v in binaries.items():
extra_binaries += '{} = {}\n'.format(k, str(v))
# Create a cross-info file that tells Meson and GCC how to cross-compile
# this project
cross_file = os.path.join(self.meson_dir, 'meson-cross-file.txt')
contents = MESON_CROSS_FILE_TPL.format(
contents = MESON_FILE_TPL.format(
# Assume all ARM sub-archs are in little endian mode
# Assume all supported target archs are little endian
with open(cross_file, 'w') as f:
return cross_file
return contents
def _write_meson_native_file(self):
def _get_meson_dummy_file_contents(self):
Get a toolchain configuration that points to `false` for everything.
This forces Meson to not detect a build-machine (native) compiler when
# Tell meson to not use a native compiler for anything
false = ['false']
# We do not use cmake dependency files, speed up the build by disabling it
native_binaries = {'cmake': ['false']}
if self.config.qt5_qmake_path:
native_binaries['qmake'] = [self.config.qt5_qmake_path]
native_binaries['moc'] = [self._get_moc_path(self.config.qt5_qmake_path)]
extra_binaries = ''
for k, v in native_binaries.items():
extra_binaries += '{} = {}\n'.format(k, str(v))
extra_binaries = 'cmake = {}'.format(str(false))
contents = MESON_FILE_TPL.format(
return contents
native_file = os.path.join(self.meson_dir, 'meson-native-file.txt')
contents = MESON_NATIVE_FILE_TPL.format(extra_binaries=extra_binaries)
with open(native_file, 'w') as f:
def _write_meson_file(self, contents, fname):
fpath = os.path.join(self.meson_dir, fname)
with open(fpath, 'w') as f:
return native_file
return fpath
async def configure(self):
......@@ -954,20 +980,17 @@ class Meson (Build, ModifyEnvBase) :
# Don't enable bitcode by passing flags manually, use the option
if self.config.ios_platform == 'iPhoneOS':
self.meson_options.update({'b_bitcode': 'true'})
# Get platform config in the form of a meson native/cross file
contents = self._get_meson_target_file_contents()
# If cross-compiling, write contents to the cross file and get contents for
# a native file that will cause all native compiler detection to fail.
# Else, write contents to a native file.
if self.config.cross_compiling():
meson_cmd += ['--cross-file', self._write_meson_cross_file()]
meson_cmd += ['--native-file', self._write_meson_native_file()]
if self.config.cross_compiling() or self.using_msvc():
# We export the cross toolchain with env vars, but Meson picks the
# native toolchain from these, so unset them.
# NOTE: This means we require a native compiler on the build
# machine when cross-compiling, which in practice is not a problem
# Also, on Windows these toolchain env vars set by us are for GCC,
# so unset them if we're building with MSVC
meson_cmd += ['--cross-file', self._write_meson_file(contents, 'meson-cross-file.txt')]
contents = self._get_meson_dummy_file_contents()
meson_cmd += ['--native-file', self._write_meson_file(contents, 'meson-native-file.txt')]
if 'default_library' in self.meson_options:
raise RuntimeError('Do not set `default_library` in self.meson_options, use self.library_type instead')
......@@ -975,6 +998,15 @@ class Meson (Build, ModifyEnvBase) :
for (key, value) in self.meson_options.items():
meson_cmd += ['-D%s=%s' % (key, str(value))]
# We export the target toolchain with env vars, but that confuses Meson
# when cross-compiling (it will pick the env vars for the build
# machine). We always set this using the cross file or native file as
# applicable, so always unset these.
# FIXME: We need an argument for meson that tells it to not pick up the
# toolchain from the env.
await shell.async_call(meson_cmd, self.meson_dir, logfile=self.logfile, env=self.env)
......@@ -45,8 +45,8 @@ def find_shlib_regex(config, libname, prefix, libdir, ext, regex):
return matches
def get_implib_dllname(config, path):
if config.msvc_toolchain_env and path.endswith('.lib'):
lib_exe = shutil.which('lib', path=config.msvc_toolchain_env['PATH'][0])
if config.msvc_env_for_toolchain and path.endswith('.lib'):
lib_exe = shutil.which('lib', path=config.msvc_env_for_toolchain['PATH'][0])
if not lib_exe:
raise FatalError('lib.exe not found, check cerbero configuration')
......@@ -28,7 +28,7 @@ from cerbero import enums
from cerbero.errors import FatalError, ConfigurationError
from cerbero.utils import _, system_info, validate_packager, shell
from cerbero.utils import to_unixpath, to_winepath, parse_file, detect_qt5
from cerbero.utils import EnvVar
from cerbero.utils import EnvVar, EnvValue
from cerbero.utils import messages as m
from cerbero.ide.vs.env import get_vs_year_version
......@@ -163,10 +163,11 @@ class Config (object):
'target_arch_flags', 'sysroot', 'isysroot',
'extra_lib_path', 'cached_sources', 'tools_prefix',
'ios_min_version', 'toolchain_path', 'mingw_perl_prefix',
'msvc_version', 'msvc_toolchain_env', 'mingw_toolchain_env',
'meson_cross_properties', 'manifest', 'extra_properties',
'qt5_qmake_path', 'qt5_pkgconfigdir', 'for_shell',
'package_tarball_compression', 'extra_mirrors',
'msvc_env_for_toolchain', 'mingw_env_for_toolchain',
'msvc_env_for_build_system', 'mingw_env_for_build_system',
'msvc_version', 'meson_properties', 'manifest',
'extra_properties', 'qt5_qmake_path', 'qt5_pkgconfigdir',
'for_shell', 'package_tarball_compression', 'extra_mirrors',
'extra_bootstrap_packages', 'moltenvk_prefix',
'vs_install_path', 'vs_install_version']
......@@ -328,9 +329,9 @@ class Config (object):
ret_env = {}
for k in new_env.keys():
new_v = new_env[k]
if isinstance(new_v, list):
# Toolchain env is in a different format
new_v = new_v[0]
# Must not accidentally use this with EnvValue objects
if isinstance(new_v, EnvValue):
raise AssertionError('{!r}: {!r}'.format(k, new_v))
if k not in old_env or k in override_env:
ret_env[k] = new_v
......@@ -475,15 +476,6 @@ class Config (object):
env['C_INCLUDE_PATH'] = includedir
env['CPLUS_INCLUDE_PATH'] = includedir
# On Windows, we have a toolchain env that we need to set, but only
# when running as a shell
if self.platform == Platform.WINDOWS and self.for_shell:
if self.can_use_msvc():
toolchain_env = self.msvc_toolchain_env
toolchain_env = self.mingw_toolchain_env
env = self._merge_env(env, toolchain_env)
# merge the config env with this new env
# LDFLAGS and PATH were already merged above
new_env = self._merge_env(self.config_env, env, override_env=('LDFLAGS', 'PATH'))
......@@ -540,7 +532,7 @@ class Config (object):
self.set_property('extra_build_tools', [])
self.set_property('distro_packages_install', True)
self.set_property('interactive', m.console_is_interactive())
self.set_property('meson_cross_properties', {})
self.set_property('meson_properties', {})
self.set_property('manifest', None)
self.set_property('extra_properties', {})
self.set_property('extra_mirrors', [])
......@@ -626,7 +618,7 @@ class Config (object):
return self.target_distro_version >= distro_version
def _parse(self, filename, reset=True):
config = {'os': os, '__file__': filename, 'env' : self.config_env,
config = {'os': os, '__file__': filename, 'env': self.config_env,
'cross': self.cross_compiling()}
if not reset:
for prop in self._properties:
......@@ -17,8 +17,9 @@
# Boston, MA 02111-1307, USA.
import os
import shutil
import sys
import shlex
import shutil
import sysconfig
......@@ -673,10 +674,90 @@ class EnvVar:
def is_arg(var):
return var in ('CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'LDFLAGS',
def is_cmd(var):
return var in ('AR', 'AS', 'CC', 'CPP', 'CXX', 'DLLTOOL', 'GENDEF',
class EnvValue(list):
Env var value (list of strings) with an associated separator
def __init__(self, sep, *values):
self.sep = sep
def get(self):
return str.join(self.sep, self)
def from_key(key, value):
if EnvVar.is_path(key):
return EnvValuePath(value)
if EnvVar.is_arg(key):
return EnvValueArg(value)
if EnvVar.is_cmd(key):
return EnvValueCmd(value)
return EnvValueSingle(value)
class EnvValueSingle(EnvValue):
Env var with a single value
def __init__(self, *values):
if len(values) == 1:
if not isinstance(values[0], list):
values = ([values[0]],)
elif len(values[0]) > 1:
raise ValueError('EnvValue can only have a single value, not multiple')
super().__init__(None, *values)
def __iadd__(self, new):
if len(self) != 0:
raise ValueError('In-place add not allowed for EnvValue {!r}'.format(self))
return super().__iadd__(new)
def get(self):
return self[0]
class EnvValueArg(EnvValue):
Env var containing a list of quoted arguments separated by space
def __init__(self, *values):
if len(values) == 1 and not isinstance(values[0], list):
values = (shlex.split(values[0]),)
super().__init__(' ', *values)
def get(self):
return ' '.join([shlex.quote(x) for x in self])