build.py 41 KB
Newer Older
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
20
21
22
23
import re
import copy
import shutil
import shlex
24
import subprocess
25
import asyncio
26
from pathlib import Path
27
from itertools import chain
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
28

29
from cerbero.enums import Platform, Architecture, Distro, LibraryType
30
from cerbero.errors import FatalError
31
from cerbero.utils import shell, to_unixpath, add_system_libs
32
from cerbero.utils import EnvValue, EnvValueSingle, EnvValueArg, EnvValueCmd, EnvValuePath
33
from cerbero.utils import messages as m
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
34

35

36
37
38
39
40
41
42
43
def get_optimization_from_config(config):
    if config.variants.optimization:
        if config.target_platform in (Platform.ANDROID, Platform.IOS):
            return 's'
        return '2'
    return '0'


44
def modify_environment(func):
45
46
47
48
49
    '''
    Decorator to modify the build environment

    When called recursively, it only modifies the environment once.
    '''
50
51
    def call(*args):
        self = args[0]
52
        try:
53
            self._modify_env()
54
55
56
57
            res = func(*args)
            return res
        finally:
            self._restore_env()
58

59
    async def async_call(*args):
60
        self = args[0]
61
62
63
        try:
            self._modify_env()
            res = await func(*args)
64
            return res
65
66
        finally:
            self._restore_env()
67

68
69
70
71
72
73
74
75
    if asyncio.iscoroutinefunction(func):
        ret = async_call
    else:
        ret = call

    ret.__name__ = func.__name__
    return ret

76

77
78
79
80
81
class EnvVarOp:
    '''
    An operation to be done on the values of a particular env var
    '''
    def __init__(self, op, var, vals, sep):
82
        self.execute = getattr(self, op)
83
        self.op = op
84
85
86
87
        self.var = var
        self.vals = vals
        self.sep = sep

88
    def set(self, env):
89
90
        if not self.vals:
            # An empty array means unset the env var
91
92
            if self.var in env:
                del env[self.var]
93
        else:
94
95
96
97
            if len(self.vals) == 1:
                env[self.var] = self.vals[0]
            else:
                env[self.var] = self.sep.join(self.vals)
98

99
    def append(self, env):
100
101
102
103
104
105
        # Avoid appending trailing space
        val = self.sep.join(self.vals)
        if not val:
            return
        if self.var not in env or not env[self.var]:
            env[self.var] = val
106
        else:
107
            env[self.var] += self.sep + val
108

109
    def prepend(self, env):
110
111
112
113
114
115
        # Avoid prepending a leading space
        val = self.sep.join(self.vals)
        if not val:
            return
        if self.var not in env or not env[self.var]:
            env[self.var] = val
116
        else:
117
            env[self.var] = val + self.sep + env[self.var]
118

Jorge Zapata's avatar
Jorge Zapata committed
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
    def remove(self, env):
        if self.var not in env:
            return
        # Split values taking in account spaces and quotes if the
        # separator is ' '
        if self.sep == ' ':
            old = shlex.split(env[self.var])
        else:
            old = env[self.var].split(self.sep)
        new = [x for x in old if x not in self.vals]
        if self.sep == ' ':
            env[self.var] = self.sep.join([shlex.quote(sub) for sub in new])
        else:
            env[self.var] = self.sep.join(new)

134
    def __repr__(self):
135
136
137
138
        vals = "None"
        if self.sep:
            vals = self.sep.join(self.vals)
        return "<EnvVarOp " + self.op + " " + self.var + " with " + vals + ">"
139

140

141
142
class ModifyEnvBase:
    '''
143
    Base class for build systems and recipes that require extra env variables
144
145
146
147
148
    '''

    use_system_libs = False

    def __init__(self):
149
150
151
152
153
154
155
        # An array of #EnvVarOp operations that will be performed sequentially
        # on the env when @modify_environment is called.
        self._new_env = []
        # Set of env vars that will be modified
        self._env_vars = set()
        # Old environment to restore
        self._old_env = {}
156
157
158
159
160
161
162

        class ModifyEnvFuncWrapper(object):
            def __init__(this, target, method):
                this.target = target
                this.method = method

            def __call__(this, var, *vals, sep=' ', when='later'):
163
164
                if vals == (None,):
                    vals = None
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
                op = EnvVarOp(this.method, var, vals, sep)
                if when == 'later':
                    this.target.check_reentrancy()
                    this.target._env_vars.add(var)
                    this.target._new_env.append(op)
                elif when == 'now-with-restore':
                    this.target._save_env_var(var)
                    op.execute(this.target.env)
                elif when == 'now':
                    op.execute(this.target.env)
                else:
                    raise RuntimeError('Unknown when value: ' + when)

            def __repr__(this):
                return "<ModifyEnvFuncWrapper " + this.method + " for " + repr(this.target) + "  at " + str(hex(id(this))) + ">"

        for i in ('append', 'prepend', 'set', 'remove'):
            setattr(self, i + '_env', ModifyEnvFuncWrapper(self, i))
183
184
185
186
187
188
189
190

    def setup_buildtype_env_ops(self):
        buildtype_args = '-Wall '
        if self.config.variants.debug:
            buildtype_args += '-g '
        buildtype_args += '-O{} '.format(get_optimization_from_config(self.config))
        for var in ('CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'OBJCFLAGS'):
            self.append_env(var, buildtype_args)
191

192
193
194
    def setup_toolchain_env_ops(self):
        if self.config.qt5_pkgconfigdir:
            self.append_env('PKG_CONFIG_LIBDIR', self.config.qt5_pkgconfigdir, sep=os.pathsep)
195
        if self.config.target_platform != Platform.WINDOWS:
196
            return
197
198
199
200
201
202

        if isinstance(self, Meson):
            if self.using_msvc():
                toolchain_env = self.config.msvc_env_for_toolchain.items()
            else:
                toolchain_env = self.config.mingw_env_for_toolchain.items()
203
        else:
204
205
206
207
208
209
            if self.using_msvc():
                toolchain_env = chain(self.config.msvc_env_for_toolchain.items(),
                                      self.config.msvc_env_for_build_system.items())
            else:
                toolchain_env = chain(self.config.mingw_env_for_toolchain.items(),
                                      self.config.mingw_env_for_build_system.items())
210
        # Set the toolchain environment
211
        for var, val in toolchain_env:
212
213
214
            # PATH and LDFLAGS are already set in self.env by config.py, so we
            # need to prepend those.
            if var in ('PATH', 'LDFLAGS'):
215
                self.prepend_env(var, val.get(), sep=val.sep)
216
            else:
217
                self.set_env(var, val.get(), sep=val.sep)
218
219
220
221
222
223
224
225

    def unset_toolchain_env(self):
        for var in ('CC', 'CXX', 'OBJC', 'OBJCXX', 'AR', 'WINDRES', 'STRIP',
                    'CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'OBJCFLAGS', 'LDFLAGS'):
            if var in self.env:
                # Env vars that are edited by the recipe will be restored by
                # @modify_environment when we return from the build step but
                # other env vars won't be, so add those.
226
                self.set_env(var, None, when='now-with-restore')
227

228
229
230
231
    def check_reentrancy(self):
        if self._old_env:
            raise RuntimeError('Do not modify the env inside @modify_environment, it will have no effect')

232
233
234
235
236
237
238
239
    @modify_environment
    def get_recipe_env(self):
        '''
        Used in oven.py to start a shell prompt with the correct env on recipe
        build failure
        '''
        return self.env.copy()

240
    def _save_env_var(self, var):
241
242
243
244
245
246
247
        # Will only store the first 'save'.
        if var not in self._old_env:
            if var in self.env:
                self._old_env[var] = self.env[var]
            else:
                self._old_env[var] = None

248
    def _modify_env(self):
249
        '''
250
        Modifies the build environment by inserting env vars from new_env
251
        '''
252
253
254
        # Don't modify env again if already did it once for this function call
        if self._old_env:
            return
255
256
        # Store old env
        for var in self._env_vars:
257
            self._save_env_var(var)
258
259
        # Modify env
        for env_op in self._new_env:
260
            env_op.execute(self.env)
261

262
    def _restore_env(self):
263
        ''' Restores the old environment '''
264
        for var, val in self._old_env.items():
265
            if val is None:
266
267
                if var in self.env:
                    del self.env[var]
268
            else:
269
                self.env[var] = val
270
        self._old_env.clear()
271

272
    def maybe_add_system_libs(self, step=''):
273
274
275
276
        '''
        Add /usr/lib/pkgconfig to PKG_CONFIG_PATH so the system's .pc file
        can be found.
        '''
277
278
279
280
281
282
283
284
285
        # Note: this is expected to be called with the environment already
        # modified using @{async_,}modify_environment

        # don't add system libs unless explicitly asked for
        if not self.use_system_libs or not self.config.allow_system_libs:
            return;

        # this only works because add_system_libs() does very little
        # this is a possible source of env conflicts
286
        new_env = {}
287
        add_system_libs(self.config, new_env, self.env)
288
289
290
291
292
293
294
295

        if step != 'configure':
            # gobject-introspection gets the paths to internal libraries all
            # wrong if we add system libraries during compile.  We should only
            # need PKG_CONFIG_PATH during configure so just unset it everywhere
            # else we will get linker errors compiling introspection binaries
            if 'PKG_CONFIG_PATH' in new_env:
                del new_env['PKG_CONFIG_PATH']
296
        for var, val in new_env.items():
297
            self.set_env(var, val, when='now-with-restore')
298
299


300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
class Build(object):
    '''
    Base class for build handlers

    @ivar recipe: the parent recipe
    @type recipe: L{cerbero.recipe.Recipe}
    @ivar config: cerbero's configuration
    @type config: L{cerbero.config.Config}
    '''

    library_type = LibraryType.BOTH
    # Whether this recipe's build system can be built with MSVC
    can_msvc = False

    def __init__(self):
        self._properties_keys = []

    @modify_environment
    def get_env(self, var, default=None):
        if var in self.env:
            return self.env[var]
        return default

    def using_msvc(self):
        if not self.can_msvc:
            return False
        if not self.config.variants.visualstudio:
            return False
        return True

330
331
332
    def using_uwp(self):
        if not self.config.variants.uwp:
            return False
333
334
335
336
337
338
        # When the uwp variant is enabled, we must never select recipes that
        # don't have can_msvc = True
        if not self.can_msvc:
            raise RuntimeError("Tried to build a recipe that can't use MSVC when using UWP")
        if not self.config.variants.visualstudio:
            raise RuntimeError("visualstudio variant wasn't set when uwp variant was set")
339
340
        return True

341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
    async def configure(self):
        '''
        Configures the module
        '''
        raise NotImplemented("'configure' must be implemented by subclasses")

    async def compile(self):
        '''
        Compiles the module
        '''
        raise NotImplemented("'make' must be implemented by subclasses")

    async def install(self):
        '''
        Installs the module
        '''
        raise NotImplemented("'install' must be implemented by subclasses")

    def check(self):
        '''
        Runs any checks on the module
        '''
        pass


366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
class CustomBuild(Build, ModifyEnvBase):

    def __init__(self):
        Build.__init__(self)
        ModifyEnvBase.__init__(self)

    async def configure(self):
        pass

    async def compile(self):
        pass

    async def install(self):
        pass


382
class MakefilesBase (Build, ModifyEnvBase):
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
383
    '''
384
    Base class for makefiles build systems like autotools and cmake
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
385
386
    '''

387
388
    config_sh = ''
    configure_tpl = ''
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
389
    configure_options = ''
390
391
    make = None
    make_install = None
Vincent Penquerc'h's avatar
Vincent Penquerc'h committed
392
    make_check = None
393
    make_clean = None
394
    allow_parallel_build = True
395
    srcdir = '.'
396
    requires_non_src_build = False
397
398
    # recipes often use shell constructs
    config_sh_needs_shell = True
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
399

400
401
    def __init__(self):
        Build.__init__(self)
402
        ModifyEnvBase.__init__(self)
403
404

        self.setup_toolchain_env_ops()
405
406
        if not self.using_msvc():
            self.setup_buildtype_env_ops()
407

408
409
410
        if self.requires_non_src_build:
            self.make_dir = os.path.join (self.config_src_dir, "cerbero-build-dir")
        else:
411
412
            self.make_dir = os.path.abspath(os.path.join(self.config_src_dir,
                                                           self.srcdir))
413
414
415
416
417

        self.make = self.make or ['make', 'V=1']
        self.make_install = self.make_install or ['make', 'install']
        self.make_clean = self.make_clean or ['make', 'clean']

418
419
        if self.config.allow_parallel_build and self.allow_parallel_build \
                and self.config.num_of_cpus > 1:
420
            self.make += ['-j%d' % self.config.num_of_cpus]
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
421

422
        # Make sure user's env doesn't mess up with our build.
423
        self.set_env('MAKEFLAGS', when='now')
Georg Lippitsch's avatar
Georg Lippitsch committed
424
        # Disable site config, which is set on openSUSE
425
        self.set_env('CONFIG_SITE', when='now')
426
427
        # Only add this for non-meson recipes, and only for iPhoneOS
        if self.config.ios_platform == 'iPhoneOS':
428
429
            bitcode_cflags = ['-fembed-bitcode']
            # NOTE: Can't pass -bitcode_bundle to Makefile projects because we
430
            # can't control what options they pass while linking dylibs
431
            bitcode_ldflags = bitcode_cflags #+ ['-Wl,-bitcode_bundle']
432
433
434
435
436
437
            self.append_env('ASFLAGS', *bitcode_cflags, when='now')
            self.append_env('CFLAGS', *bitcode_cflags, when='now')
            self.append_env('CXXFLAGS', *bitcode_cflags, when='now')
            self.append_env('OBJCFLAGS', *bitcode_cflags, when='now')
            self.append_env('OBJCXXFLAGS', *bitcode_cflags, when='now')
            self.append_env('CCASFLAGS', *bitcode_cflags, when='now')
438
439
            # Autotools only adds LDFLAGS when doing compiler checks,
            # so add -fembed-bitcode again
440
            self.append_env('LDFLAGS', *bitcode_ldflags, when='now')
Georg Lippitsch's avatar
Georg Lippitsch committed
441

442
    async def configure(self):
443
444
445
446
        '''
        Base configure method

        When called from a method in deriverd class, that method has to be
447
        decorated with modify_environment decorator.
448
        '''
449
450
451
452
453
        if not os.path.exists(self.make_dir):
            os.makedirs(self.make_dir)
        if self.requires_non_src_build:
            self.config_sh = os.path.join('../', self.config_sh)

454
        substs = {
455
            'config-sh': self.config_sh,
456
457
            'prefix': self.config.prefix,
            'libdir': self.config.libdir,
458
459
460
            'host': self.config.host,
            'target': self.config.target,
            'build': self.config.build,
461
            'options': self.configure_options,
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
            'build_dir': self.build_dir,
            'make_dir': self.make_dir,
        }

        # Construct a command list when possible
        if not self.config_sh_needs_shell:
            configure_cmd = []
            for arg in self.configure_tpl.split():
                if arg == '%(options)s':
                    options = self.configure_options
                    if isinstance(options, str):
                        options = options.split()
                    configure_cmd += options
                else:
                    configure_cmd.append(arg % substs)
        else:
            configure_cmd = self.configure_tpl % substs
479

480
481
        self.maybe_add_system_libs(step='configure')

482
483
        await shell.async_call(configure_cmd, self.make_dir,
                               logfile=self.logfile, env=self.env)
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
484

485
    @modify_environment
486
    async def compile(self):
487
        self.maybe_add_system_libs(step='compile')
488
        await shell.async_call(self.make, self.make_dir, logfile=self.logfile, env=self.env)
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
489

490
    @modify_environment
491
    async def install(self):
492
        self.maybe_add_system_libs(step='install')
493
        await shell.async_call(self.make_install, self.make_dir, logfile=self.logfile, env=self.env)
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
494

495
    @modify_environment
496
    def clean(self):
497
        self.maybe_add_system_libs(step='clean')
498
        shell.new_call(self.make_clean, self.make_dir, logfile=self.logfile, env=self.env)
499

500
    @modify_environment
Vincent Penquerc'h's avatar
Vincent Penquerc'h committed
501
502
    def check(self):
        if self.make_check:
503
            self.maybe_add_system_libs(step='check')
504
            shell.new_call(self.make_check, self.build_dir, logfile=self.logfile, env=self.env)
Vincent Penquerc'h's avatar
Vincent Penquerc'h committed
505

Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
506

507
508
509
510
class Makefile (MakefilesBase):
    '''
    Build handler for Makefile project
    '''
511
    @modify_environment
512
513
514
515
    async def configure(self):
        await MakefilesBase.configure(self)


516
517
518
class Autotools (MakefilesBase):
    '''
    Build handler for autotools project
519
520
521
522

    @cvar override_libtool: overrides ltmain.sh to generate a libtool
                            script with the one built by cerbero.
    @type override_libtool: boolean
523
524
    '''

525
526
    autoreconf = False
    autoreconf_sh = 'autoreconf -f -i'
527
528
    config_sh = './configure'
    configure_tpl = "%(config-sh)s --prefix %(prefix)s "\
529
                    "--libdir %(libdir)s"
530
    add_host_build_target = True
531
532
    can_use_configure_cache = True
    supports_cache_variables = True
533
    disable_introspection = False
534
    override_libtool = True
535

536
537
538
539
    def __init__(self):
        MakefilesBase.__init__(self)
        self.make_check = self.make_check or ['make', 'check']

540
    @modify_environment
541
    async def configure(self):
542
543
        # Build with PIC for static linking
        self.configure_tpl += ' --with-pic '
544
        # Only use --disable-maintainer mode for real autotools based projects
545
546
        if os.path.exists(os.path.join(self.config_src_dir, 'configure.in')) or\
                os.path.exists(os.path.join(self.config_src_dir, 'configure.ac')):
547
548
            self.configure_tpl += " --disable-maintainer-mode "
            self.configure_tpl += " --disable-silent-rules "
549
550
            # Never build gtk-doc documentation
            self.configure_tpl += " --disable-gtk-doc "
551

552
553
        if self.config.variants.gi and not self.disable_introspection \
                and self.use_gobject_introspection():
554
555
556
557
            self.configure_tpl += " --enable-introspection "
        else:
            self.configure_tpl += " --disable-introspection "

558
        if self.autoreconf:
559
560
            await shell.async_call(self.autoreconf_sh, self.config_src_dir,
                                   logfile=self.logfile, env=self.env)
561

562
563
564
        # Use our own config.guess and config.sub
        config_datadir = os.path.join(self.config._relative_path('data'), 'autotools')
        cfs = {'config.guess': config_datadir, 'config.sub': config_datadir}
565
        # ensure our libtool modifications are actually picked up by recipes
566
        if self.name != 'libtool' and self.override_libtool:
567
568
569
570
            cfs['ltmain.sh'] = os.path.join(self.config.build_tools_prefix, 'share/libtool/build-aux')
        for cf, srcdir in cfs.items():
            find_cmd = 'find {} -type f -name {}'.format(self.config_src_dir, cf)
            files = await shell.async_call_output(find_cmd, logfile=self.logfile, env=self.env)
571
            files = files.split('\n')
572
573
            files.remove('')
            for f in files:
574
                o = os.path.join(srcdir, cf)
575
                m.log("CERBERO: copying %s to %s" % (o, f), self.logfile)
576
                shutil.copy(o, f)
577

578
579
        if self.config.platform == Platform.WINDOWS and \
                self.supports_cache_variables:
580
581
582
            # On windows, environment variables are upperscase, but we still
            # need to pass things like am_cv_python_platform in lowercase for
            # configure and autogen.sh
583
            for k, v in self.env.items():
584
                if k[2:6] == '_cv_':
585
586
                    self.configure_tpl += ' %s="%s"' % (k, v)

587
588
589
590
591
592
593
        if self.add_host_build_target:
            if self.config.host is not None:
                self.configure_tpl += ' --host=%(host)s'
            if self.config.build is not None:
                self.configure_tpl += ' --build=%(build)s'
            if self.config.target is not None:
                self.configure_tpl += ' --target=%(target)s'
594

595
        use_configure_cache = self.config.use_configure_cache
596
        if self.use_system_libs and self.config.allow_system_libs:
597
            use_configure_cache = False
598

599
        if self._new_env:
600
601
            use_configure_cache = False

602
        if use_configure_cache and self.can_use_configure_cache:
603
            cache = os.path.join(self.config.sources, '.configure.cache')
604
            self.configure_tpl += ' --cache-file=%s' % cache
605

606
607
608
        # Add at the very end to allow recipes to override defaults
        self.configure_tpl += "  %(options)s "

609
        await MakefilesBase.configure(self)
610
611
612
613
614
615
616


class CMake (MakefilesBase):
    '''
    Build handler for cmake projects
    '''

617
    config_sh_needs_shell = False
618
    config_sh = 'cmake'
619
    configure_tpl = '%(config-sh)s -DCMAKE_INSTALL_PREFIX=%(prefix)s ' \
620
621
                    '-H%(make_dir)s ' \
                    '-B%(build_dir)s ' \
622
                    '-DCMAKE_LIBRARY_OUTPUT_PATH=%(libdir)s ' \
623
624
625
                    '-DCMAKE_INSTALL_LIBDIR=lib ' \
                    '-DCMAKE_INSTALL_BINDIR=bin ' \
                    '-DCMAKE_INSTALL_INCLUDEDIR=include ' \
626
                    '%(options)s -DCMAKE_BUILD_TYPE=Release '\
627
                    '-DCMAKE_FIND_ROOT_PATH=$CERBERO_PREFIX '\
628
                    '-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true '
629

630
631
    def __init__(self):
        MakefilesBase.__init__(self)
632
633
        self.build_dir = os.path.join(self.build_dir, '_builddir')

634

635
    @modify_environment
636
    async def configure(self):
637
638
639
640
        cc = self.env.get('CC', 'gcc')
        cxx = self.env.get('CXX', 'g++')
        cflags = self.env.get('CFLAGS', '')
        cxxflags = self.env.get('CXXFLAGS', '')
641
642
        # FIXME: CMake doesn't support passing "ccache $CC"
        if self.config.use_ccache:
643
644
            cc = cc.replace('ccache', '').strip()
            cxx = cxx.replace('ccache', '').strip()
645
646
        cc = cc.split(' ')[0]
        cxx = cxx.split(' ')[0]
647

648
649
        if self.configure_options:
            self.configure_options = self.configure_options.split()
650
651
        else:
            self.configure_options = []
652

653
        if self.config.target_platform == Platform.WINDOWS:
654
            self.configure_options += ['-DCMAKE_SYSTEM_NAME=Windows']
655
        elif self.config.target_platform == Platform.ANDROID:
656
            self.configure_options += ['-DCMAKE_SYSTEM_NAME=Linux']
657
        if self.config.platform == Platform.WINDOWS:
658
            self.configure_options += ['-G', 'Unix Makefiles']
659
660

        # FIXME: Maybe export the sysroot properly instead of doing regexp magic
661
        if self.config.target_platform in [Platform.DARWIN, Platform.IOS]:
662
663
            r = re.compile(r".*-isysroot ([^ ]+) .*")
            sysroot = r.match(cflags).group(1)
664
            self.configure_options += ['-DCMAKE_OSX_SYSROOT=' + sysroot]
665

666
667
668
669
670
671
672
        self.configure_options += [
            '-DCMAKE_C_COMPILER=' + cc,
            '-DCMAKE_CXX_COMPILER=' + cxx,
            '-DCMAKE_C_FLAGS=' + cflags,
            '-DCMAKE_CXX_FLAGS=' + cxxflags,
            '-DLIB_SUFFIX=' + self.config.lib_suffix,
        ]
673

674
675
        cmake_cache = os.path.join(self.make_dir, 'CMakeCache.txt')
        cmake_files = os.path.join(self.make_dir, 'CMakeFiles')
676
677
678
679
        if os.path.exists(cmake_cache):
            os.remove(cmake_cache)
        if os.path.exists(cmake_files):
            shutil.rmtree(cmake_files)
680
        self.make += ['VERBOSE=1']
681
        await MakefilesBase.configure(self)
682
683
        # as build_dir is different from source dir, makefile location will be in build_dir.
        self.make_dir = self.build_dir
684

685
MESON_FILE_TPL = \
686
687
688
'''
[host_machine]
system = '{system}'
689
cpu_family = '{cpu_family}'
690
691
692
693
cpu = '{cpu}'
endian = '{endian}'

[properties]
694
{extra_properties}
695
696
697
698

[binaries]
c = {CC}
cpp = {CXX}
699
700
objc = {OBJC}
objcpp = {OBJCXX}
701
ar = {AR}
702
pkgconfig = {PKG_CONFIG}
703
704
705
{extra_binaries}
'''

706
707
708
709
710
711
712
713
714
class Meson (Build, ModifyEnvBase) :
    '''
    Build handler for meson project
    '''

    make = None
    make_install = None
    make_check = None
    make_clean = None
715

716
    meson_sh = None
717
    meson_options = None
718
    meson_backend = 'ninja'
719
720
    # All meson recipes are MSVC-compatible, except if the code itself isn't
    can_msvc = True
721
722
    # Build files require a build machine compiler when cross-compiling
    meson_needs_build_machine_compiler = False
723
    meson_builddir = "_builddir"
724
725

    def __init__(self):
726
727
        self.meson_options = self.meson_options or {}

728
729
730
        Build.__init__(self)
        ModifyEnvBase.__init__(self)

731
732
        self.setup_toolchain_env_ops()

733
734
        # Find Meson
        if not self.meson_sh:
735
736
            # meson installs `meson.exe` on windows and `meson` on other
            # platforms that read shebangs
737
            self.meson_sh = os.path.join(self.config.build_tools_prefix, 'bin', 'meson')
738
739
740

        # Find ninja
        if not self.make:
741
            self.make = ['ninja', '-v', '-d', 'keeprsp']
742
        if not self.make_install:
743
            self.make_install = self.make + ['install']
744
        if not self.make_check:
745
            self.make_check = self.make + ['test']
746
        if not self.make_clean:
747
            self.make_clean = self.make + ['clean']
748

749
750
751
752
753
754
755
756
    @staticmethod
    def _get_option_value(opt_type, value):
        if opt_type == 'feature':
            return 'enabled' if value else 'disabled'
        if opt_type == 'boolean':
            return 'true' if value else 'false'
        raise AssertionError('Invalid option type {!r}'.format(opt_type))

757
    def _set_option(self, opt_names, variant_name):
758
        '''
759
760
        Parse the meson_options.txt file, figure out whether any of the provided option names exist,
        figure out the type, and enable/disable it as per the cerbero configuration.
761
762
        '''
        # Don't overwrite if it's already set
763
        if opt_names.intersection(self.meson_options):
764
765
766
            return
        # Error out on invalid usage
        if not os.path.isdir(self.build_dir):
767
            raise FatalError('Build directory doesn\'t exist yet?')
768
769
770
771
        # Check if the option exists, and if so, what the type is
        meson_options = os.path.join(self.build_dir, 'meson_options.txt')
        if not os.path.isfile(meson_options):
            return
772
        opt_name = None
773
        opt_type = None
774
        with open(meson_options, 'r', encoding='utf-8') as f:
775
            options = f.read()
776
777
778
779
            # iterate over all option()s individually
            option_regex = "option\s*\(\s*(?:'(?P<name>[^']+)')\s*,\s*(?P<entry>(?P<identifier>[a-zA-Z0-9]+)\s*:\s*(?:(?P<string>'[^']+')|[^'\),\s]+)\s*,?\s*)+\)"
            for match in re.finditer(option_regex, options, re.MULTILINE):
                option = match.group(0)
780
                # find the option(), if it exists
781
                opt_name = match.group('name')
782
                if opt_name in opt_names:
783
784
785
786
787
788
789
790
                    # get the type of the option
                    type_regex = "type\s*:\s*'(?P<type>[^']+)'"
                    ty = re.search (type_regex, option, re.MULTILINE)
                    if ty and ty.group('type') in ('feature', 'boolean'):
                        opt_type = ty.group('type')
                        break
                    else:
                        raise FatalError('Unable to detect type of option {!r}'.format(opt_name))
791
        if opt_name and opt_type:
792
793
            value = getattr(self.config.variants, variant_name) if variant_name else False
            self.meson_options[opt_name] = self._get_option_value(opt_type, value)
794

795
    def _get_target_cpu_family(self):
796
797
798
799
800
801
802
        if Architecture.is_arm(self.config.target_arch):
            if Architecture.is_arm32(self.config.target_arch):
                return 'arm'
            else:
                return 'aarch64'
        return self.config.target_arch

803
804
805
806
807
    def _get_moc_path(self, qmake_path):
        qmake = Path(qmake_path)
        moc_name = qmake.name.replace('qmake', 'moc')
        return str(qmake.parent / moc_name)

808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
    def _get_meson_target_file_contents(self):
        '''
        Get the toolchain configuration for the target machine. This will
        either go into a cross file or a native file depending on whether we're
        cross-compiling or not.
        '''
        def merge_env(old_env, new_env):
            ret_env = {}
            # Set/merge new values
            for k, new_v in new_env.items():
                new_v = EnvValue.from_key(k, new_v)
                if k not in old_env:
                    ret_env[k] = new_v
                    continue
                old_v = old_env[k]
                assert(isinstance(old_v, EnvValue))
                if isinstance(old_v, (EnvValueSingle, EnvValueCmd)) or (new_v == old_v):
                    ret_env[k] = new_v
                elif isinstance(old_v, (EnvValuePath, EnvValueArg)):
                    ret_env[k] = new_v + old_v
                else:
                    raise FatalError("Don't know how to combine the environment "
                        "variable '%s' with values '%s' and '%s'" % (k, new_v, old_v))
            # Set remaining old values
            for k in old_env.keys():
                if k not in new_env:
                    ret_env[k] = old_env[k]
            return ret_env

        # Extract toolchain config for the build system from the appropriate
        # config env dict. Start with `self.env`, since it contains toolchain
        # config set by the recipe and when building for target platforms other
        # than Windows, it also contains build tools and the env for the
        # toolchain set by config/*.config.
        #
        # On Windows, the toolchain config is `self.config.msvc_env_for_build_system`
        # or `self.config.mingw_env_for_build_system` depending on which toolchain
        # this recipe will use.
        if self.config.target_platform == Platform.WINDOWS:
            if self.using_msvc():
                build_env = dict(self.config.msvc_env_for_build_system)
            else:
                build_env = dict(self.config.mingw_env_for_build_system)
851
        else:
852
853
854
855
856
857
858
            build_env = {}
        # Override/merge toolchain env with recipe env and return a new dict
        # with values as EnvValue objects
        build_env = merge_env(build_env, self.env)

        cc = build_env.pop('CC')
        cxx = build_env.pop('CXX')
859
860
        objc = build_env.pop('OBJC', ['false'])
        objcxx = build_env.pop('OBJCXX', ['false'])
861
862
863
864
        ar = build_env.pop('AR')
        # We currently don't set the pre-processor or the linker when building with meson
        build_env.pop('CPP', None)
        build_env.pop('LD', None)
865

866
867
        # Operate on a copy of the recipe properties to avoid accumulating args
        # from all archs when doing universal builds
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
        props = {}
        build_env.pop('CPP', None) # Meson does not read this
        build_env.pop('CPPFLAGS', None) # Meson does not read this, and it's duplicated in *FLAGS
        props['c_args'] = build_env.pop('CFLAGS', [])
        props['cpp_args'] = build_env.pop('CXXFLAGS', [])
        props['objc_args'] = build_env.pop('OBJCFLAGS', [])
        props['objcpp_args'] = build_env.pop('OBJCXXFLAGS', [])
        # Link args
        props['c_link_args'] = build_env.pop('LDFLAGS', [])
        props['cpp_link_args'] = props['c_link_args']
        props['objc_link_args'] = build_env.pop('OBJLDFLAGS', props['c_link_args'])
        props['objcpp_link_args'] = props['objc_link_args']
        for key, value in self.config.meson_properties.items():
            if key not in props:
                props[key] = value
883
            else:
884
885
886
887
888
889
890
891
892
893
                props[key] += value

        # We do not use cmake dependency files, speed up the build by disabling it
        binaries = {'cmake': ['false']}
        # Get qmake and moc paths
        if self.config.qt5_qmake_path:
            binaries['qmake'] = [self.config.qt5_qmake_path]
            binaries['moc'] = [self._get_moc_path(self.config.qt5_qmake_path)]

        # Try to detect build tools in the remaining env vars
894
        build_tool_paths = build_env['PATH'].get()
895
        for name, tool in build_env.items():
896
897
898
899
900
901
            # Autoconf env vars, incorrectly detected as a build tool because of 'yes'
            if name.startswith('ac_cv'):
                continue
            # Files are always executable on Windows
            if name in ('HISTFILE', 'GST_REGISTRY_1_0'):
                continue
902
            if tool and shutil.which(tool[0], path=build_tool_paths):
903
                binaries[name.lower()] = tool
904

905
        extra_properties = ''
906
        for k, v in props.items():
907
908
            extra_properties += '{} = {}\n'.format(k, str(v))

909
        extra_binaries = ''
910
        for k, v in binaries.items():
911
912
            extra_binaries += '{} = {}\n'.format(k, str(v))

913
        contents = MESON_FILE_TPL.format(
914
915
                system=self.config.target_platform,
                cpu=self.config.target_arch,
916
917
                cpu_family=self._get_target_cpu_family(),
                # Assume all supported target archs are little endian
918
919
920
                endian='little',
                CC=cc,
                CXX=cxx,
921
922
                OBJC=objc,
                OBJCXX=objcxx,
923
                AR=ar,
924
                PKG_CONFIG="'pkg-config'",
925
                extra_binaries=extra_binaries,
926
                extra_properties=extra_properties)
927
        return contents
928

929
930
931
932
933
934
935
936
937
938
939
    def _get_meson_native_file_contents(self):
        '''
        Get a toolchain configuration that points to the build machine's
        toolchain. On Windows, this is the MinGW toolchain that we ship. On
        Linux and macOS, this is the system-wide compiler.
        '''
        false = ['false']
        if self.config.platform == Platform.WINDOWS:
            cc = self.config.mingw_env_for_build_system['CC']
            cxx = self.config.mingw_env_for_build_system['CXX']
            ar = self.config.mingw_env_for_build_system['AR']
940
941
            objc = false
            objcxx = false
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
        elif self.config.platform == Platform.DARWIN:
            cc = ['clang']
            cxx = ['clang++']
            ar = ['ar']
            objc = cc
            objcxx = cxx
        else:
            cc = ['cc']
            cxx = ['c++']
            ar = ['ar']
            objc = false
            objcxx = false
        # We do not use cmake dependency files, speed up the build by disabling it
        extra_binaries = 'cmake = {}'.format(str(false))
        contents = MESON_FILE_TPL.format(
                system=self.config.platform,
                cpu=self.config.arch,
                cpu_family=self.config.arch,
                endian='little',
                CC=cc,
                CXX=cxx,
                OBJC=objc,
                OBJCXX=objcxx,
                AR=ar,
                PKG_CONFIG=false,
                extra_binaries=extra_binaries,
                extra_properties='')
        return contents

971
972
973
974
975
976
977
978
    def _get_meson_dummy_file_contents(self):
        '''
        Get a toolchain configuration that points to `false` for everything.
        This forces Meson to not detect a build-machine (native) compiler when
        cross-compiling.
        '''
        # Tell meson to not use a native compiler for anything
        false = ['false']
979
        # We do not use cmake dependency files, speed up the build by disabling it
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
        extra_binaries = 'cmake = {}'.format(str(false))
        contents = MESON_FILE_TPL.format(
                system=self.config.platform,
                cpu=self.config.arch,
                cpu_family=self.config.arch,
                endian='little',
                CC=false,
                CXX=false,
                OBJC=false,
                OBJCXX=false,
                AR=false,
                PKG_CONFIG=false,
                extra_binaries=extra_binaries,
                extra_properties='')
        return contents
995

996
997
    def _write_meson_file(self, contents, fname):
        fpath = os.path.join(self.meson_dir, fname)
998
        with open(fpath, 'w', encoding='utf-8') as f:
999
            f.write(contents)
1000
        return fpath
1001

1002
    @modify_environment
1003
    async def configure(self):
1004
        # self.build_dir is different on each call to configure() when doing universal builds
1005
        self.meson_dir = os.path.join(self.build_dir, self.meson_builddir)
1006
1007
1008
1009
1010
1011
1012
1013
        if os.path.exists(self.meson_dir):
            # Only remove if it's not empty
            if os.listdir(self.meson_dir):
                shutil.rmtree(self.meson_dir)
                os.makedirs(self.meson_dir)
        else:
            os.makedirs(self.meson_dir)

1014
1015
        # Explicitly enable/disable introspection, same as Autotools
        self._set_option({'introspection', 'gir'}, 'gi')
1016
1017
        # Control python support using the variant
        self._set_option({'python'}, 'python')
1018
1019
1020
1021
        # Always disable gtk-doc, same as Autotools
        self._set_option({'gtk_doc'}, None)
        # Automatically disable examples
        self._set_option({'examples'}, None)
1022

1023
1024
1025
        # NOTE: self.tagged_for_release is set in recipes/custom.py
        is_gstreamer_recipe = hasattr(self, 'tagged_for_release')
        # Enable -Werror for gstreamer recipes and when running under CI
1026
        if is_gstreamer_recipe and self.config.variants.werror:
1027
1028
1029
1030
            # Let recipes override the value
            if 'werror' not in self.meson_options:
                self.meson_options['werror'] = 'true'

1031
1032
        debug = 'true' if self.config.variants.debug else 'false'
        opt = get_optimization_from_config(self.config)
1033

1034
1035
1036
        if self.library_type == LibraryType.NONE:
            raise RuntimeException("meson recipes cannot be LibraryType.NONE")

1037
        meson_cmd = [self.meson_sh, '--prefix=' + self.config.prefix,
1038
1039
            '--libdir=lib' + self.config.lib_suffix, '-Ddebug=' + debug,
            '--default-library=' + self.library_type, '-Doptimization=' + opt,
1040
            '--backend=' + self.meson_backend, '--wrap-mode=nodownload']
1041

1042
1043
1044
        if self.using_msvc():
            meson_cmd.append('-Db_vscrt=' + self.config.variants.vscrt)

1045
1046
1047
        # Don't enable bitcode by passing flags manually, use the option
        if self.config.ios_platform == 'iPhoneOS':
            self.meson_options.update({'b_bitcode': 'true'})
1048
1049
1050
1051
1052
1053
1054

        # Get platform config in the form of a meson native/cross file
        contents = self._get_meson_target_file_contents()
        # If cross-compiling, write contents to the cross file and get contents for
        # a native file that will cause all native compiler detection to fail.
        #
        # Else, write contents to a native file.
1055
        if self.config.cross_compiling():
1056
            meson_cmd += ['--cross-file', self