Commit 7462b065 authored by Nirbheek Chauhan's avatar Nirbheek Chauhan 🐜
Browse files

cerbero: Rework environment modification in recipes

Environment variable modification in a recipe used to be done with:

  self.append_env, self.prepend_env, or self.new_env

All of these were dictionaries of {string:string} mappings, which
means that if a recipe wanted to, say, append to `CFLAGS` from
multiple places within the recipe (f.ex., `glib.recipe`), you had to
carefully juggle `=` and `+=` in recipes, which was error-prone
(f.ex., `gstreamer-1.0.recipe` `variants.nodebug` was broken).

Now that we also conditionally use `self.append_env['CFLAGS']` in
`cerbero/build/build.py` for bitcode support with make-based build
systems, it's impossible to get this right in recipes. This was
causing the cross-ios-universal builds to fail on recipes that
directly set `self.append_env['CFLAGS'] = 'foo'` such as pixman.

The dictionaries have now been replaced with the following functions:

  self.append_env(varname, value1, value2, ..., sep=separator)
  self.prepend_env(varname, value1, value2, ..., sep=separa...
parent 461803fe
......@@ -91,106 +91,117 @@ def modify_environment(func):
''' Decorator to modify the build environment '''
def call(*args):
self = args[0]
prepend_env = self.prepend_env
append_env = self.append_env
new_env = self.new_env.copy()
if self.use_system_libs and self.config.allow_system_libs:
self._add_system_libs(new_env)
old_env = self._modify_env(prepend_env, append_env, new_env)
self._add_system_libs()
self._modify_env()
res = func(*args)
self._restore_env(old_env)
self._restore_env()
return res
call.__name__ = func.__name__
return call
class EnvVarOp:
'''
An operation to be done on the values of a particular env var
'''
def __init__(self, op, var, vals, sep):
if op == 'append':
op = self.append
elif op == 'prepend':
op = self.prepend
elif op == 'set':
op = self.set
else:
raise ValueError('Unknown op: {!r}'.format(op))
self.execute = op
self.var = var
self.vals = vals
self.sep = sep
def set(self):
if not self.vals:
# An empty array means unset the env var
if self.var in os.environ:
del os.environ[self.var]
else:
os.environ[self.var] = self.sep.join(self.vals)
def append(self):
if self.var not in os.environ:
os.environ[self.var] = self.sep.join(self.vals)
else:
os.environ[self.var] += self.sep + self.sep.join(self.vals)
def prepend(self):
if self.var not in os.environ:
os.environ[self.var] = self.sep.join(self.vals)
else:
old = os.environ[self.var]
os.environ[self.var] = self.sep.join(self.vals) + self.sep + old
class ModifyEnvBase:
'''
Base class for build systems that require extra env variables
Base class for build systems and recipes that require extra env variables
'''
append_env = None
prepend_env = None
new_env = None
use_system_libs = False
def __init__(self):
if self.append_env is None:
self.append_env = {}
if self.prepend_env is None:
self.prepend_env = {}
if self.new_env is None:
self.new_env = {}
self._old_env = None
def _modify_env(self, prepend_env, append_env, new_env):
# An array of #EnvVarOp operations that will be performed sequentially
# on the env when @modify_environment is called.
self._new_env = []
# Set of env vars that will be modified
self._env_vars = set()
# Old environment to restore
self._old_env = {}
def append_env(self, var, *vals, sep=' '):
self._env_vars.add(var)
self._new_env.append(EnvVarOp('append', var, vals, sep))
def prepend_env(self, var, *vals, sep=' '):
self._env_vars.add(var)
self._new_env.append(EnvVarOp('prepend', var, vals, sep))
def set_env(self, var, *vals, sep=' '):
self._env_vars.add(var)
self._new_env.append(EnvVarOp('set', var, vals, sep))
def _modify_env(self):
'''
Modifies the build environment appending the values in
append_env or replacing the values in new_env
Modifies the build environment by inserting env vars from new_env
'''
if self._old_env is not None:
return None
self._old_env = {}
for var in chain(prepend_env.keys(), append_env.keys(), new_env.keys()):
assert(not self._old_env)
# Store old env
for var in self._env_vars:
if var in os.environ:
self._old_env[var] = os.environ[var]
# Modify env
for env_op in self._new_env:
env_op.execute()
for var, (val, sep) in self._iter_env(prepend_env):
if var not in os.environ or not os.environ[var]:
os.environ[var] = val
else:
os.environ[var] = '{}{}{}'.format(val, sep, os.environ[var])
for var, (val, sep) in self._iter_env(append_env):
if var not in os.environ or not os.environ[var]:
os.environ[var] = val
else:
os.environ[var] = '{}{}{}'.format(os.environ[var], sep, val)
for var, val in new_env.items():
if val is None:
if var in os.environ:
del os.environ[var]
else:
os.environ[var] = val
return self._old_env
@staticmethod
def _iter_env(env):
for var, val in env.items():
if isinstance(val, str):
yield var, (val, ' ')
continue
elif isinstance(val, list):
if len(val) == 1:
yield var, (val[0], ' ')
continue
elif len(val) == 2:
yield var, (val[0], val[1])
continue
raise AssertionError('Invalid value for env: {!r}'.format(val))
def _restore_env(self, old_env):
def _restore_env(self):
''' Restores the old environment '''
if old_env is None:
return
for var, val in old_env.items():
for var, val in self._old_env.items():
if val is None:
if var in os.environ:
del os.environ[var]
else:
os.environ[var] = val
self._old_env = None
self._old_env.clear()
def _add_system_libs(self, new_env):
def _add_system_libs(self):
'''
Add /usr/lib/pkgconfig to PKG_CONFIG_PATH so the system's .pc file
can be found.
'''
new_env = {}
add_system_libs(self.config, new_env)
for var, val in new_env.items():
self.set_env(var, val)
class MakefilesBase (Build, ModifyEnvBase):
......@@ -223,30 +234,23 @@ class MakefilesBase (Build, ModifyEnvBase):
self.make += ' -j%d' % self.config.num_of_cpus
# Make sure user's env doesn't mess up with our build.
self.new_env['MAKEFLAGS'] = None
self.set_env('MAKEFLAGS')
# Disable site config, which is set on openSUSE
self.new_env['CONFIG_SITE'] = None
self.set_env('CONFIG_SITE')
# Only add this for non-meson recipes, and only for iPhoneOS
if self.config.ios_platform == 'iPhoneOS':
bitcode_cflags = ' -fembed-bitcode '
# FIXME: Can't pass -bitcode_bundle to Makefile projects because we
bitcode_cflags = ['-fembed-bitcode']
# NOTE: Can't pass -bitcode_bundle to Makefile projects because we
# can't control what options they pass while linking dylibs
bitcode_ldflags = bitcode_cflags #+ '-Wl,-bitcode_bundle '
# FIXME: Use real objects here instead of strings to clean up this ugliness
if 'CFLAGS' in self.append_env:
self.append_env['CFLAGS'] += bitcode_cflags
else:
self.append_env['CFLAGS'] = bitcode_cflags
if 'CCASFLAGS' in self.append_env:
self.append_env['CCASFLAGS'] += bitcode_cflags
else:
self.append_env['CCASFLAGS'] = bitcode_cflags
bitcode_ldflags = bitcode_cflags #+ ['-Wl,-bitcode_bundle']
self.append_env('CFLAGS', *bitcode_cflags)
self.append_env('CXXFLAGS', *bitcode_cflags)
self.append_env('OBCCFLAGS', *bitcode_cflags)
self.append_env('OBJCXXFLAGS', *bitcode_cflags)
self.append_env('CCASFLAGS', *bitcode_cflags)
# Autotools only adds LDFLAGS when doing compiler checks,
# so add -fembed-bitcode again
if 'LDFLAGS' in self.append_env:
self.append_env['LDFLAGS'] += bitcode_ldflags
else:
self.append_env['LDFLAGS'] = bitcode_ldflags
self.append_env('LDFLAGS', *bitcode_ldflags)
@modify_environment
def configure(self):
......@@ -352,7 +356,7 @@ class Autotools (MakefilesBase):
if self.use_system_libs and self.config.allow_system_libs:
use_configure_cache = False
if self.new_env or self.append_env or self.prepend_env:
if self._new_env:
use_configure_cache = False
if use_configure_cache and self.can_use_configure_cache:
......@@ -465,7 +469,8 @@ class Meson (Build, ModifyEnvBase) :
if self.using_msvc():
# Set the MSVC toolchain environment
self.prepend_env.update(self.config.msvc_toolchain_env)
for var, (val, sep) in self.config.msvc_toolchain_env.items():
self.prepend_env(var, val, sep=sep)
# Find Meson
if not self.meson_sh:
......@@ -603,12 +608,9 @@ class Meson (Build, ModifyEnvBase) :
# set in the recipe or other places via @modify_environment
if self.using_msvc():
for var in ('CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'LDFLAGS'):
if var in self.append_env or var in self.prepend_env:
os.environ[var] = ''
if var in self.prepend_env:
os.environ[var] = self.prepend_env[var]
if var in self.append_env:
os.environ[var] += self.append_env[var]
for env_op in self._new_env:
if env_op.var == var:
env_op.execute()
if 'default_library' in self.meson_options:
raise RuntimeError('Do not set `default_library` in self.meson_options, use self.meson_default_library instead')
......
......@@ -96,6 +96,7 @@ if platform == Platform.WINDOWS:
from cerbero.ide.vs.env import vcvarsall, get_msvc_env, append_path
# Contains only the env vars that MSVC needs, including any existing vars
# that were appended/prepended and have new values
# FIXME: Use EnvVarOp class from cerbero/build/build.py
msvc_toolchain_env = {}
for key, value in get_msvc_env(vcvarsall, arch, target_arch).items():
if key in ('PATH', 'PATHEXT', 'INCLUDE', 'LIB'):
......
......@@ -19,5 +19,4 @@ class Recipe(recipe.Recipe):
def prepare(self):
if self.config.target_platform == Platform.ANDROID:
self.configure_options += ' --disable-oss'
self.append_env['CFLAGS'] = " -fPIC "
self.append_env('CFLAGS', '-fPIC')
......@@ -22,4 +22,4 @@ class Recipe(recipe.Recipe):
# default one provided with mingw/msys
if self.config.target_platform == Platform.WINDOWS:
self.deps.remove('m4')
self.append_env['M4'] = '/bin/m4'
self.set_env('M4', '/bin/m4')
......@@ -23,7 +23,7 @@ class Recipe(recipe.Recipe):
def prepare(self):
if self.config.target_platform == Platform.WINDOWS:
self.configure_options += ' --enable-threads=win32'
self.append_env['LDFLAGS'] = '-liconv'
self.append_env('LDFLAGS', '-liconv')
def post_install(self):
if self.config.platform == Platform.WINDOWS:
......
......@@ -26,7 +26,7 @@ class Recipe(recipe.Recipe):
if self.config.target_platform != Platform.LINUX:
# Disable valgrind code on non-Linux, in the best case it just
# gives us compiler errors :)
self.append_env['CFLAGS'] = ' -DNVALGRIND=1 '
self.append_env('CFLAGS', '-DNVALGRIND=1')
self.meson_options.update({'xattr': 'false'})
# macOS provides libiconv as a separate library
if self.config.target_platform == Platform.DARWIN:
......@@ -40,4 +40,4 @@ class Recipe(recipe.Recipe):
if self.config.target_platform == Platform.WINDOWS:
# Want secure versions of stdlib functions. Glib already defines
# _WIN32_WINNT, so undefine it on the cmdline to avoid warnings
self.append_env['CFLAGS'] = ' -DMINGW_HAS_SECURE_API=1 -U_WIN32_WINNT '
self.append_env('CFLAGS', '-DMINGW_HAS_SECURE_API=1', '-U_WIN32_WINNT')
......@@ -11,4 +11,4 @@ class Recipe(recipe.Recipe):
def prepare(self):
if needs_xcode8_sdk_workaround(self.config):
self.append_env['ac_cv_func_mkostemp'] = 'no'
self.set_env('ac_cv_func_mkostemp', 'no')
......@@ -14,4 +14,4 @@ class Recipe(recipe.Recipe):
def prepare(self):
self.configure_options += " --disable-gcc-warnings"
self.append_env['CFLAGS'] = " -Wno-error=cast-align"
self.append_env('CFLAGS', '-Wno-error=cast-align')
......@@ -13,4 +13,4 @@ class Recipe(recipe.Recipe):
files_share = ['share/aclocal/pkg.m4']
def prepare(self):
self.append_env['CFLAGS'] = " -Wno-error=format-nonliteral "
self.append_env('CFLAGS', '-Wno-error=format-nonliteral')
......@@ -17,5 +17,5 @@ class Recipe(recipe.Recipe):
# because the headers and C library define the symbol and xz throws
# an error at runtime. Forcibly tell configure that clock_gettime is
# not available and use the fallback.
self.append_env['ac_cv_search_clock_gettime'] = 'no'
self.append_env['ac_cv_func_clock_gettime'] = 'no'
self.set_env('ac_cv_search_clock_gettime', 'no')
self.set_env('ac_cv_func_clock_gettime', 'no')
......@@ -12,4 +12,4 @@ class Recipe(recipe.Recipe):
def prepare(self):
if self.config.target_platform == Platform.WINDOWS:
self.append_env['LDFLAGS'] = '-lintl'
self.append_env('LDFLAGS', '-lintl')
......@@ -41,9 +41,9 @@ class Recipe(recipe.Recipe):
def prepare(self):
# extra flags needed for gcc 4.9
if self.config.target_distro == Distro.ARCH:
self.append_env['CFLAGS'] = " -fno-lto "
self.append_env['CXXFLAGS'] = " -fno-lto "
self.append_env['CPPFLAGS'] = " -fno-lto "
self.append_env('CFLAGS', '-fno-lto')
self.append_env('CXXFLAGS', '-fno-lto')
self.append_env('CPPFLAGS', '-fno-lto')
if self.config.variants.x11:
self.files_devel += ['lib/pkgconfig/cairo-xlib-xrender.pc',
......@@ -59,4 +59,4 @@ class Recipe(recipe.Recipe):
# we fail to compile with -D_FILE_OFFSET_BITS=64
# because we don't use clang and we use a platform < 21 (Lollipop)
# See $NDK_ROOT/sysroot/usr/include/sys/mman.h as one example
self.new_env['ac_cv_func_mmap'] = 'no'
self.set_env('ac_cv_func_mmap', 'no')
......@@ -29,7 +29,7 @@ class Recipe(recipe.Recipe):
def prepare(self):
if needs_xcode8_sdk_workaround(self.config):
self.append_env['ac_cv_func_mkostemp'] = 'no'
self.set_env('ac_cv_func_mkostemp', 'no')
if self.config.target_platform == Platform.WINDOWS:
if self.config.target_arch == Architecture.X86_64:
self.configure_options += ' --with-arch=x86_64 '
......
......@@ -25,4 +25,4 @@ class Recipe(recipe.Recipe):
self.make_install = 'cd %s && make PACKAGE=gettext-tools install' % intl_path
if self.config.target_platform == Platform.WINDOWS:
self.configure_options += ' --enable-threads=win32'
self.append_env['CFLAGS'] = '-Dlocale_charset=intl_locale_charset'
self.append_env('CFLAGS', '-Dlocale_charset=intl_locale_charset')
......@@ -89,23 +89,21 @@ class Recipe(recipe.Recipe):
files_lang = ['glib20']
def _set_gio_flags(self, path1=None, path2=None, use_old_uri_scheme=False):
self.append_env['CFLAGS'] += self._gio_flags(path1, path2, use_old_uri_scheme)
self.append_env('CFLAGS', *self._gio_flags(path1, path2, use_old_uri_scheme))
def _gio_flags(self, path1=None, path2=None, use_old_uri_scheme=False):
flags = ''
flags = []
def escape(path):
return '\\"%s\\"' % path
if path1 is not None:
flags += ' -DGST_SDK_GLIB_GIO_DISTRO_GIO_MODULE_PATH=%s' % escape(path1)
flags.append('-DGST_SDK_GLIB_GIO_DISTRO_GIO_MODULE_PATH=' + escape(path1))
if path2 is not None:
flags += ' -DGST_SDK_GLIB_GIO_DISTRO_GIO_MODULE_PATH2=%s' % escape(path2)
flags.append('-DGST_SDK_GLIB_GIO_DISTRO_GIO_MODULE_PATH2=' + escape(path2))
if use_old_uri_scheme:
flags += ' -DGST_SDK_GLIB_GIO_OLD_URI_SCHEME_HANDLERS=1'
flags.append('-DGST_SDK_GLIB_GIO_OLD_URI_SCHEME_HANDLERS=1')
return flags
def prepare(self):
# We add CFLAGS in various places, so initialize it first
self.append_env['CFLAGS'] = ''
# Glib doesn't support static libraries on Windows yet
if self.config.target_platform == Platform.WINDOWS:
self.meson_default_library = 'shared'
......@@ -113,7 +111,7 @@ class Recipe(recipe.Recipe):
if self.config.target_platform != Platform.LINUX:
# Disable valgrind code on non-Linux, in the best case it just
# gives us compiler errors :)
self.append_env['CFLAGS'] += ' -DNVALGRIND=1 '
self.append_env('CFLAGS', '-DNVALGRIND=1')
self.meson_options.update({'xattr': 'false'})
self.deps.append('proxy-libintl')
......@@ -138,11 +136,11 @@ class Recipe(recipe.Recipe):
# we fail to compile with -D_FILE_OFFSET_BITS=64
# because we don't use clang and we use a platform < 21 (Lollipop)
# See $NDK_ROOT/sysroot/usr/include/sys/mman.h as one example
self.meson_cross_properties['c_args'] = ['-U_FILE_OFFSET_BITS']
self.append_env('CFLAGS', '-U_FILE_OFFSET_BITS')
elif self.config.target_platform == Platform.WINDOWS:
# Want secure versions of stdlib functions. Glib already defines
# _WIN32_WINNT, so undefine it on the cmdline to avoid warnings
self.append_env['CFLAGS'] += ' -DMINGW_HAS_SECURE_API=1 -U_WIN32_WINNT '
self.append_env('CFLAGS', '-DMINGW_HAS_SECURE_API=1', '-U_WIN32_WINNT')
elif self.config.target_platform in [Platform.DARWIN, Platform.IOS]:
self.files_devel.append(os.path.join('lib', 'glib-2.0', 'include', '*', 'glibconfig.h'))
arch = self.config.target_arch
......@@ -152,13 +150,13 @@ class Recipe(recipe.Recipe):
arch = 'arm64'
elif Architecture.is_arm(arch):
arch = 'arm'
extra_flags = ''
extra_flags = []
if self.config.target_platform == Platform.IOS:
# Disable mac OS X specifics
extra_flags = '-DGST_SDK_IOS=1'
extra_flags = ['-DGST_SDK_IOS=1']
# XXX: Why did we disable these for iOS?
#self.configure_options += ' --disable-carbon --disable-modular-tests --disable-cocoa'
self.append_env['CFLAGS'] += extra_flags
self.append_env('CFLAGS', *extra_flags)
elif self.config.target_platform == Platform.LINUX:
path1 = '/usr/lib/gio/modules'
......
......@@ -35,7 +35,7 @@ class Recipe(recipe.Recipe):
self.configure_options = ' --disable-assembly'
elif self.config.target_platform == Platform.ANDROID:
# gmp use CFLAGS to compile and link some programs during configure
self.append_env['CFLAGS'] = os.environ.get('LDFLAGS', '')
self.append_env('CFLAGS', os.environ['LDFLAGS'])
if self.config.target_platform in [Platform.DARWIN, Platform.IOS]:
self.files_devel.append(os.path.join('include', '*', 'gmp.h'))
......
......@@ -38,8 +38,8 @@ class Recipe(recipe.Recipe):
def prepare(self):
if needs_xcode8_sdk_workaround(self.config):
self.append_env['ac_cv_func_clock_gettime'] = 'no'
self.append_env['ac_cv_func_clock_settime'] = 'no'
self.set_env('ac_cv_func_clock_gettime', 'no')
self.set_env('ac_cv_func_clock_settime', 'no')
if self.config.target_platform == Platform.WINDOWS:
self.configure_options += ' --enable-threads=win32'
self.can_use_configure_cache = False
......@@ -49,14 +49,14 @@ class Recipe(recipe.Recipe):
self.configure_options += ' --disable-cxx'
self.files_libs.remove('libgnutlsxx')
# Fix build with NDK 16
self.append_env['ac_cv_header_stdatomic_h'] = 'no'
self.set_env('ac_cv_header_stdatomic_h', 'no')
v = DistroVersion.get_android_api_version(self.config.target_distro_version)
if self.config.target_arch in [Architecture.ARM, Architecture.ARMv7, Architecture.X86] and v < 21:
# FIXME: HACK to make projects compile with NDK 16
# we fail to compile with -D_FILE_OFFSET_BITS=64
# because we don't use clang and we use a platform < 21 (Lollipop)
# See $NDK_ROOT/sysroot/usr/include/sys/mman.h as one example
self.new_env['ac_cv_func_mmap'] = 'no'
self.set_env('ac_cv_func_mmap', 'no')
if self.config.target_platform == Platform.IOS:
if self.config.target_arch == Architecture.ARM64:
self.configure_options += ' --disable-hardware-acceleration'
......
......@@ -55,7 +55,7 @@ class Recipe(recipe.Recipe):
if self.config.target_platform in [Platform.DARWIN, Platform.IOS]:
arch = self.config.target_arch
if arch == Architecture.X86_64:
self.new_env['SSE41_CFLAGS'] = " "
self.set_env('SSE41_CFLAGS', ' ')
def post_install(self):
if self.config.target_platform in [Platform.DARWIN, Platform.IOS]:
......
......@@ -26,10 +26,6 @@ class Recipe(custom.GStreamer):
]
def prepare(self):
# Default AS is $CC, except iOS (set below)
if Architecture.is_arm(self.config.target_arch):
self.new_env = {'AS': os.environ.get('CC', '')}
if self.config.target_platform == Platform.DARWIN:
if self.config.target_arch == Architecture.X86_64:
asflags = ' -arch x86_64 -m64'
......@@ -57,7 +53,7 @@ class Recipe(custom.GStreamer):
elif self.config.target_platform == Platform.IOS:
if Architecture.is_arm(self.config.target_arch):
if 'GAS' in os.environ:
self.new_env = {'AS': os.environ['GAS']}
self.set_env('AS', os.environ['GAS'])
# Some optimisations that were previously silently disabled
# cause warnings now. Ignore them
libavextraconf = " --extra-cflags='-Wno-ignored-optimization-argument' "
......@@ -77,7 +73,7 @@ class Recipe(custom.GStreamer):
self.configure_options += ' --with-libav-extra-configure="{}" '.format(libavextraconf)
if self.config.variants.nodebug:
self.append_env['CFLAGS'] += ' -DGST_LEVEL_MAX=GST_LEVEL_FIXME'
self.append_env('CFLAGS', '-DGST_LEVEL_MAX=GST_LEVEL_FIXME')
super(Recipe, self).prepare()
......
......@@ -16,15 +16,12 @@ class Recipe(custom.GStreamer):
def prepare(self):
if self.config.variants.rpi:
flags = (
' -I=/opt/vc/include'
' -I=/opt/vc/include/IL'
' -I=/opt/vc/include/interface/vcos/pthreads'
' -I=/opt/vc/include/interface/vmcs_host/linux '
)
flags = ['-I/opt/vc/include', '-I/opt/vc/include/IL',
'-I/opt/vc/include/interface/vcos/pthreads',
'-I/opt/vc/include/interface/vmcs_host/linux']
self.configure_options += ' --with-omx-target=rpi '
self.append_env['CFLAGS'] = flags
self.append_env['CPPFLAGS'] = flags
self.append_env['LDFLAGS'] = ' -L=/opt/vc/lib '
self.append_env('CFLAGS', *flags)
self.append_env('CPPFLAGS', *flags)
self.append_env('LDFLAGS', '-L/opt/vc/lib')
else:
self.configure_options += ' --with-omx-target=generic '
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment