__init__.py 29.4 KB
Newer Older
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

19
20
import os
import sys
21
22
import shlex
import shutil
23
import argparse
24
25
26
27
try:
    import sysconfig
except:
    from distutils import sysconfig
28
29
30
31
try:
    import xml.etree.cElementTree as etree
except ImportError:
    from lxml import etree
32
from distutils.version import StrictVersion
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
33
import gettext
34
import platform as pplatform
35
import re
36
import asyncio
37
from pathlib import Path, PureWindowsPath, PurePath
38
from collections.abc import Iterable
39

40
from cerbero.enums import Platform, Architecture, Distro, DistroVersion
41
from cerbero.errors import FatalError, CommandError
42
from cerbero.utils import messages as m
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
43
44
45
46
47
48
49

_ = gettext.gettext
N_ = lambda x: x


class ArgparseArgument(object):

50
    def __init__(self, *name, **kwargs):
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
51
52
53
        self.name = name
        self.args = kwargs

54
    def add_to_parser(self, parser):
55
        parser.add_argument(*self.name, **self.args)
56
57


58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
class StoreBool(argparse.Action):

    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        super().__init__(option_strings, dest, **kwargs)

    def __call__(self, parser, namespace, value, option_string=None):
        if value == 'yes':
            bvalue = True
        elif value == 'no':
            bvalue = False
        else:
            raise AssertionError
        setattr(namespace, self.dest, bvalue)


73
def user_is_root():
74
75
76
77
        ''' Check if the user running the process is root '''
        return hasattr(os, 'getuid') and os.getuid() == 0


Josep Torra's avatar
Josep Torra committed
78
def determine_num_of_cpus():
79
    ''' Number of virtual or logical CPUs on this system '''
Josep Torra's avatar
Josep Torra committed
80
81
82
83
84

    # Python 2.6+
    try:
        import multiprocessing
        return multiprocessing.cpu_count()
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
85
    except (ImportError, NotImplementedError):
Josep Torra's avatar
Josep Torra committed
86
87
88
        return 1


89
90
91
92
93
94
def to_winpath(path):
    if path.startswith('/'):
        path = '%s:%s' % (path[1], path[2:])
    return path.replace('/', '\\')


95
96
def to_unixpath(path):
    if path[1] == ':':
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
97
        path = '/%s%s' % (path[0], path[2:])
98
99
100
    return path


101
def to_winepath(path):
102
103
    # wine maps the filesystem root '/' to 'z:\'
    return str(PureWindowsPath('z:') / PurePath(path))
104
105


106
107
108
109
def fix_winpath(path):
    return path.replace('\\', '/')


110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def windows_arch():
    """
    Detecting the 'native' architecture of Windows is not a trivial task. We
    cannot trust that the architecture that Python is built for is the 'native'
    one because you can run 32-bit apps on 64-bit Windows using WOW64 and
    people sometimes install 32-bit Python on 64-bit Windows.
    """
    # These env variables are always available. See:
    # https://msdn.microsoft.com/en-us/library/aa384274(VS.85).aspx
    # https://blogs.msdn.microsoft.com/david.wang/2006/03/27/howto-detect-process-bitness/
    arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower()
    if not arch:
        # If this doesn't exist, something is messing with the environment
        try:
            arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
        except KeyError:
            raise FatalError(_('Unable to detect Windows architecture'))
    return arch

129
130
131
132
133
134
135
def system_info():
    '''
    Get the sysem information.
    Return a tuple with the platform type, the architecture and the
    distribution
    '''
    # Get the platform info
136
137
138
    platform = os.environ.get('OS', '').lower()
    if not platform:
        platform = sys.platform
139
140
141
142
143
144
145
146
147
148
149
    if platform.startswith('win'):
        platform = Platform.WINDOWS
    elif platform.startswith('darwin'):
        platform = Platform.DARWIN
    elif platform.startswith('linux'):
        platform = Platform.LINUX
    else:
        raise FatalError(_("Platform %s not supported") % platform)

    # Get the architecture info
    if platform == Platform.WINDOWS:
150
        arch = windows_arch()
151
        if arch in ('x64', 'amd64'):
152
            arch = Architecture.X86_64
153
        elif arch == 'x86':
154
            arch = Architecture.X86
155
156
        else:
            raise FatalError(_("Windows arch %s is not supported") % arch)
157
158
159
160
161
162
    else:
        uname = os.uname()
        arch = uname[4]
        if arch == 'x86_64':
            arch = Architecture.X86_64
        elif arch.endswith('86'):
163
            arch = Architecture.X86
164
165
166
167
        elif arch.startswith('armv7'):
            arch = Architecture.ARMv7
        elif arch.startswith('arm'):
            arch = Architecture.ARM
168
169
        else:
            raise FatalError(_("Architecture %s not supported") % arch)
170
171
172

    # Get the distro info
    if platform == Platform.LINUX:
173
174
175
176
177
178
179
180
181
182
183
        if sys.version_info >= (3, 8, 0):
            try:
                import distro
            except ImportError:
                print('''Python >= 3.8 detected and the 'distro' python package was not found.
Please install the 'python3-distro' or 'python-distro' package from your linux package manager or from pypi using pip.
Terminating.''', file=sys.stderr)
                sys.exit(1)
            d = distro.linux_distribution()
        else:
            d = pplatform.linux_distribution()
184
185
186
187
188
189

        if d[0] == '' and d[1] == '' and d[2] == '':
            if os.path.exists('/etc/arch-release'):
                # FIXME: the python2.7 platform module does not support Arch Linux.
                # Mimic python3.4 platform.linux_distribution() output.
                d = ('arch', 'Arch', 'Linux')
190
191
192
193
            elif os.path.exists('/etc/os-release'):
                with open('/etc/os-release', 'r') as f:
                    if 'ID="amzn"\n' in f.readlines():
                        d = ('RedHat', 'amazon', '')
194
195
196
                    else:
                        f.seek(0, 0)
                        for line in f:
197
198
199
200
201
202
203
                            # skip empty lines and comment lines
                            if line.strip() and not line.lstrip().startswith('#'):
                                k,v = line.rstrip().split("=")
                                if k == 'NAME':
                                    name = v.strip('"')
                                elif k == 'VERSION_ID':
                                    version = v.strip('"')
204
                        d = (name, version, '');
205

206
        if d[0] in ['Ubuntu', 'debian', 'Debian GNU/Linux', 'LinuxMint', 'Linux Mint']:
207
            distro = Distro.DEBIAN
208
209
210
211
212
            distro_version = d[2].lower()
            split_str = d[2].split()
            if split_str:
                distro_version = split_str[0].lower()
            if distro_version in ['maverick', 'isadora']:
213
                distro_version = DistroVersion.UBUNTU_MAVERICK
214
            elif distro_version in ['lucid', 'julia']:
215
                distro_version = DistroVersion.UBUNTU_LUCID
216
            elif distro_version in ['natty', 'katya']:
217
                distro_version = DistroVersion.UBUNTU_NATTY
218
            elif distro_version in ['oneiric', 'lisa']:
219
                distro_version = DistroVersion.UBUNTU_ONEIRIC
220
            elif distro_version in ['precise', 'maya']:
221
                distro_version = DistroVersion.UBUNTU_PRECISE
222
            elif distro_version in ['quantal', 'nadia']:
223
                distro_version = DistroVersion.UBUNTU_QUANTAL
224
            elif distro_version in ['raring', 'olivia']:
225
                distro_version = DistroVersion.UBUNTU_RARING
226
            elif distro_version in ['saucy', 'petra']:
Josep Torra's avatar
Josep Torra committed
227
                distro_version = DistroVersion.UBUNTU_SAUCY
228
            elif distro_version in ['trusty', 'qiana', 'rebecca']:
229
                distro_version = DistroVersion.UBUNTU_TRUSTY
230
            elif distro_version in ['utopic']:
231
                distro_version = DistroVersion.UBUNTU_UTOPIC
232
            elif distro_version in ['vivid']:
233
                distro_version = DistroVersion.UBUNTU_VIVID
234
            elif distro_version in ['wily']:
Alistair Buxton's avatar
Alistair Buxton committed
235
                distro_version = DistroVersion.UBUNTU_WILY
236
            elif distro_version in ['xenial', 'sarah', 'serena', 'sonya', 'sylvia']:
Xavier Claessens's avatar
Xavier Claessens committed
237
                distro_version = DistroVersion.UBUNTU_XENIAL
238
            elif distro_version in ['artful']:
Xavier Claessens's avatar
Xavier Claessens committed
239
                distro_version = DistroVersion.UBUNTU_ARTFUL
240
            elif distro_version in ['bionic', 'tara', 'tessa', 'tina', 'tricia']:
241
                distro_version = DistroVersion.UBUNTU_BIONIC
242
            elif distro_version in ['cosmic']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
243
                distro_version = DistroVersion.UBUNTU_COSMIC
244
            elif distro_version in ['disco']:
245
                distro_version = DistroVersion.UBUNTU_DISCO
246
            elif distro_version in ['eoan']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
247
                distro_version = DistroVersion.UBUNTU_EOAN
248
            elif distro_version in ['focal', 'ulyana']:
Seungha Yang's avatar
Seungha Yang committed
249
                distro_version = DistroVersion.UBUNTU_FOCAL
250
            elif d[1].startswith('6.'):
251
                distro_version = DistroVersion.DEBIAN_SQUEEZE
252
            elif d[1].startswith('7.') or d[1].startswith('wheezy'):
253
                distro_version = DistroVersion.DEBIAN_WHEEZY
254
            elif d[1].startswith('8.') or d[1].startswith('jessie'):
Sebastian Dröge's avatar
Sebastian Dröge committed
255
                distro_version = DistroVersion.DEBIAN_JESSIE
256
            elif d[1].startswith('9.') or d[1].startswith('stretch'):
257
                distro_version = DistroVersion.DEBIAN_STRETCH
258
259
            elif d[1].startswith('10.') or d[1].startswith('buster'):
                distro_version = DistroVersion.DEBIAN_BUSTER
260
261
            elif d[1].startswith('11.') or d[1].startswith('bullseye'):
                distro_version = DistroVersion.DEBIAN_BULLSEYE
262
263
            elif d[1] == 'unstable' and d[2] == 'sid':
                distro_version = DistroVersion.DEBIAN_SID
264
265
            else:
                raise FatalError("Distribution '%s' not supported" % str(d))
Olivier Crête's avatar
Olivier Crête committed
266
        elif d[0] in ['RedHat', 'Fedora', 'CentOS', 'Red Hat Enterprise Linux Server', 'CentOS Linux']:
267
            distro = Distro.REDHAT
268
269
270
271
272
273
            if d[1] == '16':
                distro_version = DistroVersion.FEDORA_16
            elif d[1] == '17':
                distro_version = DistroVersion.FEDORA_17
            elif d[1] == '18':
                distro_version = DistroVersion.FEDORA_18
Sebastian Dröge's avatar
Sebastian Dröge committed
274
275
            elif d[1] == '19':
                distro_version = DistroVersion.FEDORA_19
Thibault Saunier's avatar
Thibault Saunier committed
276
277
            elif d[1] == '20':
                distro_version = DistroVersion.FEDORA_20
278
279
            elif d[1] == '21':
                distro_version = DistroVersion.FEDORA_21
280
281
            elif d[1] == '22':
                distro_version = DistroVersion.FEDORA_22
Olivier Crête's avatar
Olivier Crête committed
282
283
            elif d[1] == '23':
                distro_version = DistroVersion.FEDORA_23
284
285
            elif d[1] == '24':
                distro_version = DistroVersion.FEDORA_24
286
287
            elif d[1] == '25':
                distro_version = DistroVersion.FEDORA_25
288
289
            elif d[1] == '26':
                distro_version = DistroVersion.FEDORA_26
Olivier Crête's avatar
Olivier Crête committed
290
291
            elif d[1] == '27':
                distro_version = DistroVersion.FEDORA_27
292
293
            elif d[1] == '28':
                distro_version = DistroVersion.FEDORA_28
Nicolas Dufresne's avatar
Nicolas Dufresne committed
294
295
            elif d[1] == '29':
                distro_version = DistroVersion.FEDORA_29
Olivier Crête's avatar
Olivier Crête committed
296
297
            elif d[1] == '30':
                distro_version = DistroVersion.FEDORA_30
298
299
            elif d[1] == '31':
                distro_version = DistroVersion.FEDORA_31
300
301
            elif d[1] == '32':
                distro_version = DistroVersion.FEDORA_32
302
303
304
305
            elif d[0] == 'Fedora':
                # str(int()) is for ensuring that the fedora version is
                # actually a number
                distro_version = 'fedora_' + str(int(d[1]))
306
307
            elif d[1].startswith('6.'):
                distro_version = DistroVersion.REDHAT_6
Olivier Crête's avatar
Olivier Crête committed
308
309
            elif d[1].startswith('7.'):
                distro_version = DistroVersion.REDHAT_7
310
311
            elif d[1].startswith('8.'):
                distro_version = DistroVersion.REDHAT_8
312
313
            elif d[1] == 'amazon':
                distro_version = DistroVersion.AMAZON_LINUX
314
315
316
            else:
                # FIXME Fill this
                raise FatalError("Distribution '%s' not supported" % str(d))
317
        elif d[0].strip() in ['openSUSE']:
318
            distro = Distro.SUSE
Georg Lippitsch's avatar
Georg Lippitsch committed
319
            if d[1] == '42.2':
Georg Lippitsch's avatar
Georg Lippitsch committed
320
                distro_version = DistroVersion.OPENSUSE_42_2
Georg Lippitsch's avatar
Georg Lippitsch committed
321
322
            elif d[1] == '42.3':
                distro_version = DistroVersion.OPENSUSE_42_3
323
324
            else:
                # FIXME Fill this
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
325
326
                raise FatalError("Distribution OpenSuse '%s' "
                                 "not supported" % str(d))
327
328
329
        elif d[0].strip() in ['openSUSE Tumbleweed']:
            distro = Distro.SUSE
            distro_version = DistroVersion.OPENSUSE_TUMBLEWEED
330
        elif d[0].strip() in ['arch', 'Arch Linux']:
331
332
            distro = Distro.ARCH
            distro_version = DistroVersion.ARCH_ROLLING
333
334
335
        elif d[0].strip() in ['Gentoo Base System']:
            distro = Distro.GENTOO
            distro_version = DistroVersion.GENTOO_VERSION
336
337
338
339
        else:
            raise FatalError("Distribution '%s' not supported" % str(d))
    elif platform == Platform.WINDOWS:
        distro = Distro.WINDOWS
340
        win32_ver = pplatform.win32_ver()[0]
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
341
342
        dmap = {'xp': DistroVersion.WINDOWS_XP,
                'vista': DistroVersion.WINDOWS_VISTA,
Sebastian Dröge's avatar
Sebastian Dröge committed
343
                '7': DistroVersion.WINDOWS_7,
344
                'post2008Server': DistroVersion.WINDOWS_8,
Sebastian Dröge's avatar
Sebastian Dröge committed
345
                '8': DistroVersion.WINDOWS_8,
346
347
                'post2012Server': DistroVersion.WINDOWS_8_1,
                '8.1': DistroVersion.WINDOWS_8_1,
Sebastian Dröge's avatar
Sebastian Dröge committed
348
                '10': DistroVersion.WINDOWS_10}
349
350
        if win32_ver in dmap:
            distro_version = dmap[win32_ver]
351
        else:
352
353
354
            raise FatalError("Windows version '%s' not supported" % win32_ver)
    elif platform == Platform.DARWIN:
        distro = Distro.OS_X
FLUENDO's avatar
FLUENDO committed
355
        ver = pplatform.mac_ver()[0]
356
        if ver.startswith(('11.0', '10.16')):
Nirbheek Chauhan's avatar
Nirbheek Chauhan committed
357
358
            distro_version = DistroVersion.OS_X_BIG_SUR
        elif ver.startswith('10.15'):
359
360
            distro_version = DistroVersion.OS_X_CATALINA
        elif ver.startswith('10.14'):
Justin Kim's avatar
Justin Kim committed
361
362
            distro_version = DistroVersion.OS_X_MOJAVE
        elif ver.startswith('10.13'):
Boris Prohaska's avatar
Boris Prohaska committed
363
364
            distro_version = DistroVersion.OS_X_HIGH_SIERRA
        elif ver.startswith('10.12'):
Pierre Lamot's avatar
Pierre Lamot committed
365
366
            distro_version = DistroVersion.OS_X_SIERRA
        elif ver.startswith('10.11'):
367
368
            distro_version = DistroVersion.OS_X_EL_CAPITAN
        elif ver.startswith('10.10'):
369
370
            distro_version = DistroVersion.OS_X_YOSEMITE
        elif ver.startswith('10.9'):
Josep Torra's avatar
Josep Torra committed
371
372
            distro_version = DistroVersion.OS_X_MAVERICKS
        elif ver.startswith('10.8'):
373
            distro_version = DistroVersion.OS_X_MOUNTAIN_LION
FLUENDO's avatar
FLUENDO committed
374
375
        else:
            raise FatalError("Mac version %s not supported" % ver)
376

Josep Torra's avatar
Josep Torra committed
377
378
379
    num_of_cpus = determine_num_of_cpus()

    return platform, arch, distro, distro_version, num_of_cpus
380
381
382
383


def validate_packager(packager):
    # match packager in the form 'Name <email>'
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
384
    expr = r'(.*\s)*[<]([a-zA-Z0-9+_\-\.]+@'\
385
        '[0-9a-zA-Z][.-0-9a-zA-Z]*.[a-zA-Z]+)[>]$'
386
    return bool(re.match(expr, packager))
387
388


389
def copy_files(origdir, destdir, files, extensions, target_platform, logfile=None):
390
391
392
393
394
    for f in files:
        f = f % extensions
        install_dir = os.path.dirname(os.path.join(destdir, f))
        if not os.path.exists(install_dir):
            os.makedirs(install_dir)
395
396
        if destdir[1] == ':':
            # windows path
397
398
399
400
401
            relprefix = to_unixpath(destdir)[2:]
        else:
            relprefix = destdir[1:]
        orig = os.path.join(origdir, relprefix, f)
        dest = os.path.join(destdir, f)
402
        m.action("copying %s to %s" % (orig, dest), logfile=logfile)
403
404
405
406
        try:
            shutil.copy(orig, dest)
        except IOError:
            m.warning("Could not copy %s to %s" % (orig, dest))
407
408
409
410
411
412


def remove_list_duplicates(seq):
    ''' Remove list duplicates maintaining the order '''
    seen = set()
    seen_add = seen.add
413
    return [x for x in seq if x not in seen and not seen_add(x)]
414
415
416


def parse_file(filename, dict):
417
418
    if '__file__' not in dict:
        dict['__file__'] = filename
419
    try:
420
421
        exec(compile(open(filename).read(), filename, 'exec'), dict)
    except Exception as ex:
422
423
        import traceback
        traceback.print_exc()
424
        raise ex
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443


def escape_path(path):
    path = path.replace('\\', '/')
    path = path.replace('(', '\\\(').replace(')', '\\\)')
    path = path.replace(' ', '\\\\ ')
    return path


def get_wix_prefix():
    if 'WIX' in os.environ:
        wix_prefix = os.path.join(os.environ['WIX'], 'bin')
    else:
        wix_prefix = 'C:/Program Files%s/Windows Installer XML v3.5/bin'
        if not os.path.exists(wix_prefix):
            wix_prefix = wix_prefix % ' (x86)'
    if not os.path.exists(wix_prefix):
        raise FatalError("The required packaging tool 'WiX' was not found")
    return escape_path(to_unixpath(wix_prefix))
444

445
def add_system_libs(config, new_env, old_env=None):
446
447
448
449
450
451
    '''
    Add /usr/lib/pkgconfig to PKG_CONFIG_PATH so the system's .pc file
    can be found.
    '''
    arch = config.target_arch
    libdir = 'lib'
452

453
454
455
456
457
458
459
460
461
    # Only use this when compiling on Linux for Linux and not cross-compiling
    # to some other Linux
    if config.platform != Platform.LINUX:
        return
    if config.target_platform != Platform.LINUX:
        return
    if config.cross_compiling():
        return

462
    if arch == Architecture.X86_64:
Georg Lippitsch's avatar
Georg Lippitsch committed
463
        if config.distro == Distro.REDHAT or config.distro == Distro.SUSE:
464
            libdir = 'lib64'
465

466
467
468
469
    sysroot = '/'
    if config.sysroot:
        sysroot = config.sysroot

470
471
472
    if not old_env:
        old_env = os.environ

473
    search_paths = []
474
475
    if old_env.get('PKG_CONFIG_LIBDIR', None):
       search_paths += [old_env['PKG_CONFIG_LIBDIR']]
476
477
    if old_env.get('PKG_CONFIG_PATH', None):
       search_paths += [old_env['PKG_CONFIG_PATH']]
478
    search_paths += [
479
        os.path.join(sysroot, 'usr', libdir, 'pkgconfig'),
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
        os.path.join(sysroot, 'usr/share/pkgconfig')]

    if config.target_distro == Distro.DEBIAN:
        host = None
        if arch == Architecture.ARM:
            host = 'arm-linux-gnueabi'
        elif arch == Architecture.ARM64:
            host = 'aarch64-linux-gnu'
        elif arch == Architecture.X86:
            host = 'i386-linux-gnu'
        elif Architecture.is_arm(arch):
            host = 'arm-linux-gnueabihf'
        else:
            host = '%s-linux-gnu' % arch

        search_paths.append(os.path.join(sysroot, 'usr/lib/%s/pkgconfig' % host))

497
    new_env['PKG_CONFIG_PATH'] = ':'.join(search_paths)
498

499
500
    search_paths = [os.environ.get('ACLOCAL_PATH', ''),
        os.path.join(sysroot, 'usr/share/aclocal')]
501
    new_env['ACLOCAL_PATH'] = ':'.join(search_paths)
502
503
504
505
506

def needs_xcode8_sdk_workaround(config):
    '''
    Returns whether the XCode 8 clock_gettime, mkostemp, getentropy workaround
    from https://bugzilla.gnome.org/show_bug.cgi?id=772451 is needed
507
508

    These symbols are only available on macOS 10.12+ and iOS 10.0+
509
    '''
510
511
512
513
514
515
    if config.target_platform == Platform.DARWIN:
        if StrictVersion(config.min_osx_sdk_version) < StrictVersion('10.12'):
            return True
    elif config.target_platform == Platform.IOS:
        if StrictVersion(config.ios_min_version) < StrictVersion('10.0'):
            return True
516
    return False
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543

def _qmake_or_pkgdir(qmake):
    qmake_path = Path(qmake)
    if not qmake_path.is_file():
        m.warning('QMAKE={!r} does not exist'.format(str(qmake_path)))
        return (None, None)
    pkgdir = (qmake_path.parent.parent / 'lib/pkgconfig')
    if pkgdir.is_dir():
        return (pkgdir.as_posix(), qmake_path.as_posix())
    return (None, qmake_path.as_posix())

def detect_qt5(platform, arch, is_universal):
    '''
    Returns both the path to the pkgconfig directory and the path to qmake:
    (pkgdir, qmake). If `pkgdir` could not be found, it will be None

    Returns (None, None) if nothing was found.
    '''
    path = None
    qt5_prefix = os.environ.get('QT5_PREFIX', None)
    qmake_path = os.environ.get('QMAKE', None)
    if not qt5_prefix and not qmake_path:
        return (None, None)
    if qt5_prefix and not os.path.isdir(qt5_prefix):
        m.warning('QT5_PREFIX={!r} does not exist'.format(qt5_prefix))
        return (None, None)
    if qmake_path:
544
545
546
547
548
549
550
551
552
        try:
            qt_version = shell.check_output([qmake_path, '-query', 'QT_VERSION']).strip()
            qt_version = [int(v) for v in qt_version.split('.')]
        except CommandError as e:
            m.warning('QMAKE={!r} failed to execute:\n{}'.format(str(qmake_path), str(e)))
            qt_version = [0, 0]
        if len(qt_version) >= 2 and qt_version[:2] < [5, 14] and \
           is_universal and platform == Platform.ANDROID:
            # require QT5_PREFIX before Qt 5.14 with android universal
553
554
            if not qt5_prefix:
                m.warning('Please set QT5_PREFIX if you want to build '
555
                          'the Qt5 plugin for android-universal with Qt < 5.14')
556
557
558
559
560
561
562
                return (None, None)
        else:
            ret = _qmake_or_pkgdir(qmake_path)
            if ret != (None, None) or not qt5_prefix:
                return ret
    # qmake path is invalid, find pkgdir or qmake from qt5 prefix
    if platform == Platform.ANDROID:
563
564
565
566
567
        # Qt => 5.14
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android/bin/qmake'))
        if ret != (None, None):
            return ret
        # Qt < 5.14
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
        if arch == Architecture.ARMv7:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_armv7/bin/qmake'))
        elif arch == Architecture.ARM64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_arm64_v8a/bin/qmake'))
        elif arch == Architecture.X86:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_x86/bin/qmake'))
        elif arch == Architecture.X86_64:
            # Qt binaries do not ship a qmake for android_x86_64
            return (None, None)
    elif platform == Platform.DARWIN:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'clang_64/bin/qmake'))
    elif platform == Platform.IOS:
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'ios/bin/qmake'))
    elif platform == Platform.LINUX:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'gcc_64/bin/qmake'))
    elif platform == Platform.WINDOWS:
        # There are several msvc and mingw toolchains to pick from, and we
        # can't pick it for the user.
        m.warning('You must set QMAKE instead of QT5_PREFIX on Windows')
        return (None, None)
    if ret == (None, None):
        m.warning('Unsupported arch {!r} on platform {!r}'.format(arch, platform))
    return ret
593

594
595
596
597
598
599
600
601
602
603
# asyncio.Semaphore classes set their working event loop internally on
# creation, so we need to ensure the proper loop has already been set by then.
# This is especially important if we create global semaphores that are
# initialized at the very beginning, since on Windows, the default
# SelectorEventLoop is not available.
def CerberoSemaphore(value=1):
    get_event_loop() # this ensures the proper event loop is already created
    return asyncio.Semaphore(value)

def get_event_loop():
604
605
606
607
608
609
610
611
612
613
614
615
616
    try:
        loop = asyncio.get_event_loop()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

    # On Windows the default SelectorEventLoop is not available:
    # https://docs.python.org/3.5/library/asyncio-subprocess.html#windows-event-loop
    if sys.platform == 'win32' and \
       not isinstance(loop, asyncio.ProactorEventLoop):
        loop = asyncio.ProactorEventLoop()
        asyncio.set_event_loop(loop)

617
618
619
620
621
622
    # Avoid spammy BlockingIOError warnings with older python versions
    if sys.platform != 'win32' and \
       sys.version_info < (3, 8, 0):
        asyncio.set_child_watcher(asyncio.FastChildWatcher())
        asyncio.get_child_watcher().attach_loop(loop)

623
624
625
    return loop

def run_until_complete(tasks):
626
627
628
629
630
631
632
633
634
    '''
    Runs one or many tasks, blocking until all of them have finished.
    @param tasks: A single Future or a list of Futures to run
    @type tasks: Future or list of Futures
    @return: the result of the asynchronous task execution (if only
             one task) or a list of all results in case of multiple
             tasks. Result is None if operation is cancelled.
    @rtype: any type or list of any types in case of multiple tasks
    '''
635
636
    loop = get_event_loop()

637
638
    try:
        if isinstance(tasks, Iterable):
639
            result = loop.run_until_complete(asyncio.gather(*tasks))
640
        else:
641
642
            result = loop.run_until_complete(tasks)
        return result
643
    except asyncio.CancelledError:
644
        return None
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659

async def run_tasks(tasks, done_async=None):
    """
    Runs @tasks until completion or until @done_async returns
    """
    class QueueDone(Exception):
        pass

    if done_async:
        async def queue_done():
            # This is how we exit the asyncio.wait once everything is done
            # as otherwise asyncio.wait will wait for our tasks to complete
            await done_async
            raise QueueDone()

660
        task = asyncio.ensure_future (queue_done())
661
662
        tasks.append(task)

663
664
665
666
    async def shutdown(abnormal=True):
        tasks_minus_current = [t for t in tasks]
        [task.cancel() for task in tasks_minus_current]
        ret = await asyncio.gather(*tasks_minus_current, return_exceptions=True)
667
668
        # we want to find any actual exception rather than one
        # that may be returned from task.cancel()
669
        cancelled = None
670
        for e in ret:
671
672
            if isinstance(e, asyncio.CancelledError):
                cancelled = e
673
674
675
676
            if isinstance(e, Exception) \
               and not isinstance(e, asyncio.CancelledError) \
               and not isinstance(e, QueueDone):
                raise e
677
678
679
680
        if abnormal and cancelled:
            # use cancelled as a last resort we would prefer to throw any
            # other exception but only if something abnormal happened
            raise cancelled
681
682
683
684

    try:
        await asyncio.gather(*tasks, return_exceptions=False)
    except asyncio.CancelledError:
685
        raise
686
    except QueueDone:
687
        await shutdown(abnormal=False)
688
689
690
    except Exception:
        await shutdown()
        raise
691
692
693
694
695
696
697
698
699
700
701
702
703
704


class EnvVar:
    @staticmethod
    def is_path(var):
        return var in ('LD_LIBRARY_PATH', 'PATH', 'MANPATH', 'INFOPATH',
                'PKG_CONFIG_PATH', 'PKG_CONFIG_LIBDIR', 'GI_TYPELIB_PATH',
                'XDG_DATA_DIRS', 'XDG_CONFIG_DIRS', 'GST_PLUGIN_PATH',
                'GST_PLUGIN_PATH_1_0', 'PYTHONPATH', 'MONO_PATH', 'LIB',
                'INCLUDE', 'PATHEXT')

    @staticmethod
    def is_arg(var):
        return var in ('CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'LDFLAGS',
705
                'OBJCFLAGS', 'OBJCXXFLAGS', 'OBJLDFLAGS', 'CCASFLAGS')
706
707
708
709
710

    @staticmethod
    def is_cmd(var):
        return var in ('AR', 'AS', 'CC', 'CPP', 'CXX', 'DLLTOOL', 'GENDEF',
                'LD', 'NM', 'OBJC', 'OBJCOPY', 'OBJCXX', 'PERL', 'PYTHON',
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
                'RANLIB', 'RC', 'STRIP', 'WINDRES')


class EnvValue(list):
    '''
    Env var value (list of strings) with an associated separator
    '''

    def __init__(self, sep, *values):
        self.sep = sep
        super().__init__(*values)

    def get(self):
        return str.join(self.sep, self)

    @staticmethod
    def from_key(key, value):
        if EnvVar.is_path(key):
            return EnvValuePath(value)
        if EnvVar.is_arg(key):
            return EnvValueArg(value)
        if EnvVar.is_cmd(key):
            return EnvValueCmd(value)
        return EnvValueSingle(value)


class EnvValueSingle(EnvValue):
    '''
    Env var with a single value
    '''

    def __init__(self, *values):
        if len(values) == 1:
            if not isinstance(values[0], list):
                values = ([values[0]],)
            elif len(values[0]) > 1:
                raise ValueError('EnvValue can only have a single value, not multiple')
        super().__init__(None, *values)

    def __iadd__(self, new):
        if len(self) != 0:
            raise ValueError('In-place add not allowed for EnvValue {!r}'.format(self))
        return super().__iadd__(new)

    def get(self):
        return self[0]


class EnvValueArg(EnvValue):
    '''
    Env var containing a list of quoted arguments separated by space
    '''

    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (shlex.split(values[0]),)
        super().__init__(' ', *values)

    def get(self):
        return ' '.join([shlex.quote(x) for x in self])


class EnvValueCmd(EnvValueArg):
    '''
    Env var containing a command and a list of arguments separated by space
    '''

    def __iadd__(self, new):
        if isinstance(new, EnvValueCmd):
            raise ValueError('In-place add not allowed for EnvValueCmd {!r} and EnvValueCmd {!r}'.format(self, new))
        return super().__iadd__(new)


class EnvValuePath(EnvValue):
    '''
    Env var containing a list of paths separated by os.pathsep, which is `:` or `;`
    '''
    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (values[0].split(os.pathsep),)
        super().__init__(os.pathsep, *values)