__init__.py 29.2 KB
Newer Older
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

19
20
import os
import sys
21
22
import shlex
import shutil
23
import argparse
24
25
26
27
try:
    import sysconfig
except:
    from distutils import sysconfig
28
29
30
31
try:
    import xml.etree.cElementTree as etree
except ImportError:
    from lxml import etree
32
from distutils.version import StrictVersion
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
33
import gettext
34
import platform as pplatform
35
import re
36
import asyncio
37
from pathlib import Path, PureWindowsPath, PurePath
38
from collections.abc import Iterable
39

40
from cerbero.enums import Platform, Architecture, Distro, DistroVersion
41
from cerbero.errors import FatalError, CommandError
42
from cerbero.utils import messages as m
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
43
44
45
46
47
48
49

_ = gettext.gettext
N_ = lambda x: x


class ArgparseArgument(object):

50
    def __init__(self, *name, **kwargs):
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
51
52
53
        self.name = name
        self.args = kwargs

54
    def add_to_parser(self, parser):
55
        parser.add_argument(*self.name, **self.args)
56
57


58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
class StoreBool(argparse.Action):

    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        super().__init__(option_strings, dest, **kwargs)

    def __call__(self, parser, namespace, value, option_string=None):
        if value == 'yes':
            bvalue = True
        elif value == 'no':
            bvalue = False
        else:
            raise AssertionError
        setattr(namespace, self.dest, bvalue)


73
def user_is_root():
74
75
76
77
        ''' Check if the user running the process is root '''
        return hasattr(os, 'getuid') and os.getuid() == 0


Josep Torra's avatar
Josep Torra committed
78
def determine_num_of_cpus():
79
    ''' Number of virtual or logical CPUs on this system '''
Josep Torra's avatar
Josep Torra committed
80
81
82
83
84

    # Python 2.6+
    try:
        import multiprocessing
        return multiprocessing.cpu_count()
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
85
    except (ImportError, NotImplementedError):
Josep Torra's avatar
Josep Torra committed
86
87
88
        return 1


89
90
91
92
93
94
def to_winpath(path):
    if path.startswith('/'):
        path = '%s:%s' % (path[1], path[2:])
    return path.replace('/', '\\')


95
96
def to_unixpath(path):
    if path[1] == ':':
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
97
        path = '/%s%s' % (path[0], path[2:])
98
99
100
    return path


101
def to_winepath(path):
102
103
    # wine maps the filesystem root '/' to 'z:\'
    return str(PureWindowsPath('z:') / PurePath(path))
104
105


106
107
108
109
def fix_winpath(path):
    return path.replace('\\', '/')


110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def windows_arch():
    """
    Detecting the 'native' architecture of Windows is not a trivial task. We
    cannot trust that the architecture that Python is built for is the 'native'
    one because you can run 32-bit apps on 64-bit Windows using WOW64 and
    people sometimes install 32-bit Python on 64-bit Windows.
    """
    # These env variables are always available. See:
    # https://msdn.microsoft.com/en-us/library/aa384274(VS.85).aspx
    # https://blogs.msdn.microsoft.com/david.wang/2006/03/27/howto-detect-process-bitness/
    arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower()
    if not arch:
        # If this doesn't exist, something is messing with the environment
        try:
            arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
        except KeyError:
            raise FatalError(_('Unable to detect Windows architecture'))
    return arch

129
130
131
132
133
134
135
def system_info():
    '''
    Get the sysem information.
    Return a tuple with the platform type, the architecture and the
    distribution
    '''
    # Get the platform info
136
137
138
    platform = os.environ.get('OS', '').lower()
    if not platform:
        platform = sys.platform
139
140
141
142
143
144
145
146
147
148
149
    if platform.startswith('win'):
        platform = Platform.WINDOWS
    elif platform.startswith('darwin'):
        platform = Platform.DARWIN
    elif platform.startswith('linux'):
        platform = Platform.LINUX
    else:
        raise FatalError(_("Platform %s not supported") % platform)

    # Get the architecture info
    if platform == Platform.WINDOWS:
150
        arch = windows_arch()
151
        if arch in ('x64', 'amd64'):
152
            arch = Architecture.X86_64
153
        elif arch == 'x86':
154
            arch = Architecture.X86
155
156
        else:
            raise FatalError(_("Windows arch %s is not supported") % arch)
157
158
159
160
161
162
    else:
        uname = os.uname()
        arch = uname[4]
        if arch == 'x86_64':
            arch = Architecture.X86_64
        elif arch.endswith('86'):
163
            arch = Architecture.X86
164
165
166
167
        elif arch.startswith('armv7'):
            arch = Architecture.ARMv7
        elif arch.startswith('arm'):
            arch = Architecture.ARM
168
169
        else:
            raise FatalError(_("Architecture %s not supported") % arch)
170
171
172

    # Get the distro info
    if platform == Platform.LINUX:
173
174
175
176
177
178
179
180
181
182
183
        if sys.version_info >= (3, 8, 0):
            try:
                import distro
            except ImportError:
                print('''Python >= 3.8 detected and the 'distro' python package was not found.
Please install the 'python3-distro' or 'python-distro' package from your linux package manager or from pypi using pip.
Terminating.''', file=sys.stderr)
                sys.exit(1)
            d = distro.linux_distribution()
        else:
            d = pplatform.linux_distribution()
184
185
186
187
188
189

        if d[0] == '' and d[1] == '' and d[2] == '':
            if os.path.exists('/etc/arch-release'):
                # FIXME: the python2.7 platform module does not support Arch Linux.
                # Mimic python3.4 platform.linux_distribution() output.
                d = ('arch', 'Arch', 'Linux')
190
191
192
193
            elif os.path.exists('/etc/os-release'):
                with open('/etc/os-release', 'r') as f:
                    if 'ID="amzn"\n' in f.readlines():
                        d = ('RedHat', 'amazon', '')
194
195
196
                    else:
                        f.seek(0, 0)
                        for line in f:
197
198
199
200
201
202
203
                            # skip empty lines and comment lines
                            if line.strip() and not line.lstrip().startswith('#'):
                                k,v = line.rstrip().split("=")
                                if k == 'NAME':
                                    name = v.strip('"')
                                elif k == 'VERSION_ID':
                                    version = v.strip('"')
204
                        d = (name, version, '');
205

206
        if d[0] in ['Ubuntu', 'debian', 'Debian GNU/Linux', 'LinuxMint', 'Linux Mint']:
207
            distro = Distro.DEBIAN
208
209
210
211
212
            distro_version = d[2].lower()
            split_str = d[2].split()
            if split_str:
                distro_version = split_str[0].lower()
            if distro_version in ['maverick', 'isadora']:
213
                distro_version = DistroVersion.UBUNTU_MAVERICK
214
            elif distro_version in ['lucid', 'julia']:
215
                distro_version = DistroVersion.UBUNTU_LUCID
216
            elif distro_version in ['natty', 'katya']:
217
                distro_version = DistroVersion.UBUNTU_NATTY
218
            elif distro_version in ['oneiric', 'lisa']:
219
                distro_version = DistroVersion.UBUNTU_ONEIRIC
220
            elif distro_version in ['precise', 'maya']:
221
                distro_version = DistroVersion.UBUNTU_PRECISE
222
            elif distro_version in ['quantal', 'nadia']:
223
                distro_version = DistroVersion.UBUNTU_QUANTAL
224
            elif distro_version in ['raring', 'olivia']:
225
                distro_version = DistroVersion.UBUNTU_RARING
226
            elif distro_version in ['saucy', 'petra']:
Josep Torra's avatar
Josep Torra committed
227
                distro_version = DistroVersion.UBUNTU_SAUCY
228
            elif distro_version in ['trusty', 'qiana', 'rebecca']:
229
                distro_version = DistroVersion.UBUNTU_TRUSTY
230
            elif distro_version in ['utopic']:
231
                distro_version = DistroVersion.UBUNTU_UTOPIC
232
            elif distro_version in ['vivid']:
233
                distro_version = DistroVersion.UBUNTU_VIVID
234
            elif distro_version in ['wily']:
Alistair Buxton's avatar
Alistair Buxton committed
235
                distro_version = DistroVersion.UBUNTU_WILY
236
            elif distro_version in ['xenial', 'sarah', 'serena', 'sonya', 'sylvia']:
Xavier Claessens's avatar
Xavier Claessens committed
237
                distro_version = DistroVersion.UBUNTU_XENIAL
238
            elif distro_version in ['artful']:
Xavier Claessens's avatar
Xavier Claessens committed
239
                distro_version = DistroVersion.UBUNTU_ARTFUL
240
            elif distro_version in ['bionic', 'tara', 'tessa', 'tina', 'tricia']:
241
                distro_version = DistroVersion.UBUNTU_BIONIC
242
            elif distro_version in ['cosmic']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
243
                distro_version = DistroVersion.UBUNTU_COSMIC
244
            elif distro_version in ['disco']:
245
                distro_version = DistroVersion.UBUNTU_DISCO
246
            elif distro_version in ['eoan']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
247
                distro_version = DistroVersion.UBUNTU_EOAN
248
            elif distro_version in ['focal', 'ulyana']:
Seungha Yang's avatar
Seungha Yang committed
249
                distro_version = DistroVersion.UBUNTU_FOCAL
250
            elif d[1].startswith('6.'):
251
                distro_version = DistroVersion.DEBIAN_SQUEEZE
252
            elif d[1].startswith('7.') or d[1].startswith('wheezy'):
253
                distro_version = DistroVersion.DEBIAN_WHEEZY
254
            elif d[1].startswith('8.') or d[1].startswith('jessie'):
Sebastian Dröge's avatar
Sebastian Dröge committed
255
                distro_version = DistroVersion.DEBIAN_JESSIE
256
            elif d[1].startswith('9.') or d[1].startswith('stretch'):
257
                distro_version = DistroVersion.DEBIAN_STRETCH
258
259
            elif d[1].startswith('10.') or d[1].startswith('buster'):
                distro_version = DistroVersion.DEBIAN_BUSTER
260
261
            elif d[1].startswith('11.') or d[1].startswith('bullseye'):
                distro_version = DistroVersion.DEBIAN_BULLSEYE
262
263
            elif d[1] == 'unstable' and d[2] == 'sid':
                distro_version = DistroVersion.DEBIAN_SID
264
265
            else:
                raise FatalError("Distribution '%s' not supported" % str(d))
Olivier Crête's avatar
Olivier Crête committed
266
        elif d[0] in ['RedHat', 'Fedora', 'CentOS', 'Red Hat Enterprise Linux Server', 'CentOS Linux']:
267
            distro = Distro.REDHAT
268
269
270
271
272
273
            if d[1] == '16':
                distro_version = DistroVersion.FEDORA_16
            elif d[1] == '17':
                distro_version = DistroVersion.FEDORA_17
            elif d[1] == '18':
                distro_version = DistroVersion.FEDORA_18
Sebastian Dröge's avatar
Sebastian Dröge committed
274
275
            elif d[1] == '19':
                distro_version = DistroVersion.FEDORA_19
Thibault Saunier's avatar
Thibault Saunier committed
276
277
            elif d[1] == '20':
                distro_version = DistroVersion.FEDORA_20
278
279
            elif d[1] == '21':
                distro_version = DistroVersion.FEDORA_21
280
281
            elif d[1] == '22':
                distro_version = DistroVersion.FEDORA_22
Olivier Crête's avatar
Olivier Crête committed
282
283
            elif d[1] == '23':
                distro_version = DistroVersion.FEDORA_23
284
285
            elif d[1] == '24':
                distro_version = DistroVersion.FEDORA_24
286
287
            elif d[1] == '25':
                distro_version = DistroVersion.FEDORA_25
288
289
            elif d[1] == '26':
                distro_version = DistroVersion.FEDORA_26
Olivier Crête's avatar
Olivier Crête committed
290
291
            elif d[1] == '27':
                distro_version = DistroVersion.FEDORA_27
292
293
            elif d[1] == '28':
                distro_version = DistroVersion.FEDORA_28
Nicolas Dufresne's avatar
Nicolas Dufresne committed
294
295
            elif d[1] == '29':
                distro_version = DistroVersion.FEDORA_29
Olivier Crête's avatar
Olivier Crête committed
296
297
            elif d[1] == '30':
                distro_version = DistroVersion.FEDORA_30
298
299
            elif d[1] == '31':
                distro_version = DistroVersion.FEDORA_31
300
301
            elif d[1] == '32':
                distro_version = DistroVersion.FEDORA_32
302
303
            elif d[1].startswith('6.'):
                distro_version = DistroVersion.REDHAT_6
Olivier Crête's avatar
Olivier Crête committed
304
305
            elif d[1].startswith('7.'):
                distro_version = DistroVersion.REDHAT_7
306
307
            elif d[1].startswith('8.'):
                distro_version = DistroVersion.REDHAT_8
308
309
            elif d[1] == 'amazon':
                distro_version = DistroVersion.AMAZON_LINUX
310
311
312
            else:
                # FIXME Fill this
                raise FatalError("Distribution '%s' not supported" % str(d))
313
        elif d[0].strip() in ['openSUSE']:
314
            distro = Distro.SUSE
Georg Lippitsch's avatar
Georg Lippitsch committed
315
            if d[1] == '42.2':
Georg Lippitsch's avatar
Georg Lippitsch committed
316
                distro_version = DistroVersion.OPENSUSE_42_2
Georg Lippitsch's avatar
Georg Lippitsch committed
317
318
            elif d[1] == '42.3':
                distro_version = DistroVersion.OPENSUSE_42_3
319
320
            else:
                # FIXME Fill this
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
321
322
                raise FatalError("Distribution OpenSuse '%s' "
                                 "not supported" % str(d))
323
324
325
        elif d[0].strip() in ['openSUSE Tumbleweed']:
            distro = Distro.SUSE
            distro_version = DistroVersion.OPENSUSE_TUMBLEWEED
326
        elif d[0].strip() in ['arch', 'Arch Linux']:
327
328
            distro = Distro.ARCH
            distro_version = DistroVersion.ARCH_ROLLING
329
330
331
        elif d[0].strip() in ['Gentoo Base System']:
            distro = Distro.GENTOO
            distro_version = DistroVersion.GENTOO_VERSION
332
333
334
335
        else:
            raise FatalError("Distribution '%s' not supported" % str(d))
    elif platform == Platform.WINDOWS:
        distro = Distro.WINDOWS
336
        win32_ver = pplatform.win32_ver()[0]
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
337
338
        dmap = {'xp': DistroVersion.WINDOWS_XP,
                'vista': DistroVersion.WINDOWS_VISTA,
Sebastian Dröge's avatar
Sebastian Dröge committed
339
                '7': DistroVersion.WINDOWS_7,
340
                'post2008Server': DistroVersion.WINDOWS_8,
Sebastian Dröge's avatar
Sebastian Dröge committed
341
                '8': DistroVersion.WINDOWS_8,
342
343
                'post2012Server': DistroVersion.WINDOWS_8_1,
                '8.1': DistroVersion.WINDOWS_8_1,
Sebastian Dröge's avatar
Sebastian Dröge committed
344
                '10': DistroVersion.WINDOWS_10}
345
346
        if win32_ver in dmap:
            distro_version = dmap[win32_ver]
347
        else:
348
349
350
            raise FatalError("Windows version '%s' not supported" % win32_ver)
    elif platform == Platform.DARWIN:
        distro = Distro.OS_X
FLUENDO's avatar
FLUENDO committed
351
        ver = pplatform.mac_ver()[0]
Nirbheek Chauhan's avatar
Nirbheek Chauhan committed
352
353
354
        if ver.startswith('11.0'):
            distro_version = DistroVersion.OS_X_BIG_SUR
        elif ver.startswith('10.15'):
355
356
            distro_version = DistroVersion.OS_X_CATALINA
        elif ver.startswith('10.14'):
Justin Kim's avatar
Justin Kim committed
357
358
            distro_version = DistroVersion.OS_X_MOJAVE
        elif ver.startswith('10.13'):
Boris Prohaska's avatar
Boris Prohaska committed
359
360
            distro_version = DistroVersion.OS_X_HIGH_SIERRA
        elif ver.startswith('10.12'):
Pierre Lamot's avatar
Pierre Lamot committed
361
362
            distro_version = DistroVersion.OS_X_SIERRA
        elif ver.startswith('10.11'):
363
364
            distro_version = DistroVersion.OS_X_EL_CAPITAN
        elif ver.startswith('10.10'):
365
366
            distro_version = DistroVersion.OS_X_YOSEMITE
        elif ver.startswith('10.9'):
Josep Torra's avatar
Josep Torra committed
367
368
            distro_version = DistroVersion.OS_X_MAVERICKS
        elif ver.startswith('10.8'):
369
            distro_version = DistroVersion.OS_X_MOUNTAIN_LION
FLUENDO's avatar
FLUENDO committed
370
371
        else:
            raise FatalError("Mac version %s not supported" % ver)
372

Josep Torra's avatar
Josep Torra committed
373
374
375
    num_of_cpus = determine_num_of_cpus()

    return platform, arch, distro, distro_version, num_of_cpus
376
377
378
379


def validate_packager(packager):
    # match packager in the form 'Name <email>'
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
380
    expr = r'(.*\s)*[<]([a-zA-Z0-9+_\-\.]+@'\
381
        '[0-9a-zA-Z][.-0-9a-zA-Z]*.[a-zA-Z]+)[>]$'
382
    return bool(re.match(expr, packager))
383
384


385
def copy_files(origdir, destdir, files, extensions, target_platform, logfile=None):
386
387
388
389
390
    for f in files:
        f = f % extensions
        install_dir = os.path.dirname(os.path.join(destdir, f))
        if not os.path.exists(install_dir):
            os.makedirs(install_dir)
391
392
        if destdir[1] == ':':
            # windows path
393
394
395
396
397
            relprefix = to_unixpath(destdir)[2:]
        else:
            relprefix = destdir[1:]
        orig = os.path.join(origdir, relprefix, f)
        dest = os.path.join(destdir, f)
398
        m.action("copying %s to %s" % (orig, dest), logfile=logfile)
399
400
401
402
        try:
            shutil.copy(orig, dest)
        except IOError:
            m.warning("Could not copy %s to %s" % (orig, dest))
403
404
405
406
407
408


def remove_list_duplicates(seq):
    ''' Remove list duplicates maintaining the order '''
    seen = set()
    seen_add = seen.add
409
    return [x for x in seq if x not in seen and not seen_add(x)]
410
411
412


def parse_file(filename, dict):
413
414
    if '__file__' not in dict:
        dict['__file__'] = filename
415
    try:
416
417
        exec(compile(open(filename).read(), filename, 'exec'), dict)
    except Exception as ex:
418
419
        import traceback
        traceback.print_exc()
420
        raise ex
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439


def escape_path(path):
    path = path.replace('\\', '/')
    path = path.replace('(', '\\\(').replace(')', '\\\)')
    path = path.replace(' ', '\\\\ ')
    return path


def get_wix_prefix():
    if 'WIX' in os.environ:
        wix_prefix = os.path.join(os.environ['WIX'], 'bin')
    else:
        wix_prefix = 'C:/Program Files%s/Windows Installer XML v3.5/bin'
        if not os.path.exists(wix_prefix):
            wix_prefix = wix_prefix % ' (x86)'
    if not os.path.exists(wix_prefix):
        raise FatalError("The required packaging tool 'WiX' was not found")
    return escape_path(to_unixpath(wix_prefix))
440

441
def add_system_libs(config, new_env, old_env=None):
442
443
444
445
446
447
    '''
    Add /usr/lib/pkgconfig to PKG_CONFIG_PATH so the system's .pc file
    can be found.
    '''
    arch = config.target_arch
    libdir = 'lib'
448

449
450
451
452
453
454
455
456
457
    # Only use this when compiling on Linux for Linux and not cross-compiling
    # to some other Linux
    if config.platform != Platform.LINUX:
        return
    if config.target_platform != Platform.LINUX:
        return
    if config.cross_compiling():
        return

458
    if arch == Architecture.X86_64:
Georg Lippitsch's avatar
Georg Lippitsch committed
459
        if config.distro == Distro.REDHAT or config.distro == Distro.SUSE:
460
            libdir = 'lib64'
461

462
463
464
465
    sysroot = '/'
    if config.sysroot:
        sysroot = config.sysroot

466
467
468
    if not old_env:
        old_env = os.environ

469
    search_paths = []
470
471
    if old_env.get('PKG_CONFIG_LIBDIR', None):
       search_paths += [old_env['PKG_CONFIG_LIBDIR']]
472
473
    if old_env.get('PKG_CONFIG_PATH', None):
       search_paths += [old_env['PKG_CONFIG_PATH']]
474
    search_paths += [
475
        os.path.join(sysroot, 'usr', libdir, 'pkgconfig'),
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
        os.path.join(sysroot, 'usr/share/pkgconfig')]

    if config.target_distro == Distro.DEBIAN:
        host = None
        if arch == Architecture.ARM:
            host = 'arm-linux-gnueabi'
        elif arch == Architecture.ARM64:
            host = 'aarch64-linux-gnu'
        elif arch == Architecture.X86:
            host = 'i386-linux-gnu'
        elif Architecture.is_arm(arch):
            host = 'arm-linux-gnueabihf'
        else:
            host = '%s-linux-gnu' % arch

        search_paths.append(os.path.join(sysroot, 'usr/lib/%s/pkgconfig' % host))

493
    new_env['PKG_CONFIG_PATH'] = ':'.join(search_paths)
494

495
496
    search_paths = [os.environ.get('ACLOCAL_PATH', ''),
        os.path.join(sysroot, 'usr/share/aclocal')]
497
    new_env['ACLOCAL_PATH'] = ':'.join(search_paths)
498
499
500
501
502

def needs_xcode8_sdk_workaround(config):
    '''
    Returns whether the XCode 8 clock_gettime, mkostemp, getentropy workaround
    from https://bugzilla.gnome.org/show_bug.cgi?id=772451 is needed
503
504

    These symbols are only available on macOS 10.12+ and iOS 10.0+
505
    '''
506
507
508
509
510
511
    if config.target_platform == Platform.DARWIN:
        if StrictVersion(config.min_osx_sdk_version) < StrictVersion('10.12'):
            return True
    elif config.target_platform == Platform.IOS:
        if StrictVersion(config.ios_min_version) < StrictVersion('10.0'):
            return True
512
    return False
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539

def _qmake_or_pkgdir(qmake):
    qmake_path = Path(qmake)
    if not qmake_path.is_file():
        m.warning('QMAKE={!r} does not exist'.format(str(qmake_path)))
        return (None, None)
    pkgdir = (qmake_path.parent.parent / 'lib/pkgconfig')
    if pkgdir.is_dir():
        return (pkgdir.as_posix(), qmake_path.as_posix())
    return (None, qmake_path.as_posix())

def detect_qt5(platform, arch, is_universal):
    '''
    Returns both the path to the pkgconfig directory and the path to qmake:
    (pkgdir, qmake). If `pkgdir` could not be found, it will be None

    Returns (None, None) if nothing was found.
    '''
    path = None
    qt5_prefix = os.environ.get('QT5_PREFIX', None)
    qmake_path = os.environ.get('QMAKE', None)
    if not qt5_prefix and not qmake_path:
        return (None, None)
    if qt5_prefix and not os.path.isdir(qt5_prefix):
        m.warning('QT5_PREFIX={!r} does not exist'.format(qt5_prefix))
        return (None, None)
    if qmake_path:
540
541
542
543
544
545
546
547
548
        try:
            qt_version = shell.check_output([qmake_path, '-query', 'QT_VERSION']).strip()
            qt_version = [int(v) for v in qt_version.split('.')]
        except CommandError as e:
            m.warning('QMAKE={!r} failed to execute:\n{}'.format(str(qmake_path), str(e)))
            qt_version = [0, 0]
        if len(qt_version) >= 2 and qt_version[:2] < [5, 14] and \
           is_universal and platform == Platform.ANDROID:
            # require QT5_PREFIX before Qt 5.14 with android universal
549
550
            if not qt5_prefix:
                m.warning('Please set QT5_PREFIX if you want to build '
551
                          'the Qt5 plugin for android-universal with Qt < 5.14')
552
553
554
555
556
557
558
                return (None, None)
        else:
            ret = _qmake_or_pkgdir(qmake_path)
            if ret != (None, None) or not qt5_prefix:
                return ret
    # qmake path is invalid, find pkgdir or qmake from qt5 prefix
    if platform == Platform.ANDROID:
559
560
561
562
563
        # Qt => 5.14
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android/bin/qmake'))
        if ret != (None, None):
            return ret
        # Qt < 5.14
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
        if arch == Architecture.ARMv7:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_armv7/bin/qmake'))
        elif arch == Architecture.ARM64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_arm64_v8a/bin/qmake'))
        elif arch == Architecture.X86:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_x86/bin/qmake'))
        elif arch == Architecture.X86_64:
            # Qt binaries do not ship a qmake for android_x86_64
            return (None, None)
    elif platform == Platform.DARWIN:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'clang_64/bin/qmake'))
    elif platform == Platform.IOS:
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'ios/bin/qmake'))
    elif platform == Platform.LINUX:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'gcc_64/bin/qmake'))
    elif platform == Platform.WINDOWS:
        # There are several msvc and mingw toolchains to pick from, and we
        # can't pick it for the user.
        m.warning('You must set QMAKE instead of QT5_PREFIX on Windows')
        return (None, None)
    if ret == (None, None):
        m.warning('Unsupported arch {!r} on platform {!r}'.format(arch, platform))
    return ret
589

590
591
592
593
594
595
596
597
598
599
# asyncio.Semaphore classes set their working event loop internally on
# creation, so we need to ensure the proper loop has already been set by then.
# This is especially important if we create global semaphores that are
# initialized at the very beginning, since on Windows, the default
# SelectorEventLoop is not available.
def CerberoSemaphore(value=1):
    get_event_loop() # this ensures the proper event loop is already created
    return asyncio.Semaphore(value)

def get_event_loop():
600
601
602
603
604
605
606
607
608
609
610
611
612
    try:
        loop = asyncio.get_event_loop()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

    # On Windows the default SelectorEventLoop is not available:
    # https://docs.python.org/3.5/library/asyncio-subprocess.html#windows-event-loop
    if sys.platform == 'win32' and \
       not isinstance(loop, asyncio.ProactorEventLoop):
        loop = asyncio.ProactorEventLoop()
        asyncio.set_event_loop(loop)

613
614
615
616
617
618
    # Avoid spammy BlockingIOError warnings with older python versions
    if sys.platform != 'win32' and \
       sys.version_info < (3, 8, 0):
        asyncio.set_child_watcher(asyncio.FastChildWatcher())
        asyncio.get_child_watcher().attach_loop(loop)

619
620
621
    return loop

def run_until_complete(tasks):
622
623
624
625
626
627
628
629
630
    '''
    Runs one or many tasks, blocking until all of them have finished.
    @param tasks: A single Future or a list of Futures to run
    @type tasks: Future or list of Futures
    @return: the result of the asynchronous task execution (if only
             one task) or a list of all results in case of multiple
             tasks. Result is None if operation is cancelled.
    @rtype: any type or list of any types in case of multiple tasks
    '''
631
632
    loop = get_event_loop()

633
634
    try:
        if isinstance(tasks, Iterable):
635
            result = loop.run_until_complete(asyncio.gather(*tasks))
636
        else:
637
638
            result = loop.run_until_complete(tasks)
        return result
639
    except asyncio.CancelledError:
640
        return None
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655

async def run_tasks(tasks, done_async=None):
    """
    Runs @tasks until completion or until @done_async returns
    """
    class QueueDone(Exception):
        pass

    if done_async:
        async def queue_done():
            # This is how we exit the asyncio.wait once everything is done
            # as otherwise asyncio.wait will wait for our tasks to complete
            await done_async
            raise QueueDone()

656
        task = asyncio.ensure_future (queue_done())
657
658
        tasks.append(task)

659
660
661
662
    async def shutdown(abnormal=True):
        tasks_minus_current = [t for t in tasks]
        [task.cancel() for task in tasks_minus_current]
        ret = await asyncio.gather(*tasks_minus_current, return_exceptions=True)
663
664
        # we want to find any actual exception rather than one
        # that may be returned from task.cancel()
665
        cancelled = None
666
        for e in ret:
667
668
            if isinstance(e, asyncio.CancelledError):
                cancelled = e
669
670
671
672
            if isinstance(e, Exception) \
               and not isinstance(e, asyncio.CancelledError) \
               and not isinstance(e, QueueDone):
                raise e
673
674
675
676
        if abnormal and cancelled:
            # use cancelled as a last resort we would prefer to throw any
            # other exception but only if something abnormal happened
            raise cancelled
677
678
679
680

    try:
        await asyncio.gather(*tasks, return_exceptions=False)
    except asyncio.CancelledError:
681
        raise
682
    except QueueDone:
683
        await shutdown(abnormal=False)
684
685
686
    except Exception:
        await shutdown()
        raise
687
688
689
690
691
692
693
694
695
696
697
698
699
700


class EnvVar:
    @staticmethod
    def is_path(var):
        return var in ('LD_LIBRARY_PATH', 'PATH', 'MANPATH', 'INFOPATH',
                'PKG_CONFIG_PATH', 'PKG_CONFIG_LIBDIR', 'GI_TYPELIB_PATH',
                'XDG_DATA_DIRS', 'XDG_CONFIG_DIRS', 'GST_PLUGIN_PATH',
                'GST_PLUGIN_PATH_1_0', 'PYTHONPATH', 'MONO_PATH', 'LIB',
                'INCLUDE', 'PATHEXT')

    @staticmethod
    def is_arg(var):
        return var in ('CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'LDFLAGS',
701
                'OBJCFLAGS', 'OBJCXXFLAGS', 'OBJLDFLAGS', 'CCASFLAGS')
702
703
704
705
706

    @staticmethod
    def is_cmd(var):
        return var in ('AR', 'AS', 'CC', 'CPP', 'CXX', 'DLLTOOL', 'GENDEF',
                'LD', 'NM', 'OBJC', 'OBJCOPY', 'OBJCXX', 'PERL', 'PYTHON',
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
                'RANLIB', 'RC', 'STRIP', 'WINDRES')


class EnvValue(list):
    '''
    Env var value (list of strings) with an associated separator
    '''

    def __init__(self, sep, *values):
        self.sep = sep
        super().__init__(*values)

    def get(self):
        return str.join(self.sep, self)

    @staticmethod
    def from_key(key, value):
        if EnvVar.is_path(key):
            return EnvValuePath(value)
        if EnvVar.is_arg(key):
            return EnvValueArg(value)
        if EnvVar.is_cmd(key):
            return EnvValueCmd(value)
        return EnvValueSingle(value)


class EnvValueSingle(EnvValue):
    '''
    Env var with a single value
    '''

    def __init__(self, *values):
        if len(values) == 1:
            if not isinstance(values[0], list):
                values = ([values[0]],)
            elif len(values[0]) > 1:
                raise ValueError('EnvValue can only have a single value, not multiple')
        super().__init__(None, *values)

    def __iadd__(self, new):
        if len(self) != 0:
            raise ValueError('In-place add not allowed for EnvValue {!r}'.format(self))
        return super().__iadd__(new)

    def get(self):
        return self[0]


class EnvValueArg(EnvValue):
    '''
    Env var containing a list of quoted arguments separated by space
    '''

    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (shlex.split(values[0]),)
        super().__init__(' ', *values)

    def get(self):
        return ' '.join([shlex.quote(x) for x in self])


class EnvValueCmd(EnvValueArg):
    '''
    Env var containing a command and a list of arguments separated by space
    '''

    def __iadd__(self, new):
        if isinstance(new, EnvValueCmd):
            raise ValueError('In-place add not allowed for EnvValueCmd {!r} and EnvValueCmd {!r}'.format(self, new))
        return super().__iadd__(new)


class EnvValuePath(EnvValue):
    '''
    Env var containing a list of paths separated by os.pathsep, which is `:` or `;`
    '''
    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (values[0].split(os.pathsep),)
        super().__init__(os.pathsep, *values)