__init__.py 29.6 KB
Newer Older
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Andoni Morales Alastruey <ylatuya@gmail.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

19
20
import os
import sys
21
22
import shlex
import shutil
23
import argparse
24
25
26
27
try:
    import sysconfig
except:
    from distutils import sysconfig
28
29
30
31
try:
    import xml.etree.cElementTree as etree
except ImportError:
    from lxml import etree
32
from distutils.version import StrictVersion
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
33
import gettext
34
import platform as pplatform
35
import re
36
import asyncio
37
from pathlib import Path, PureWindowsPath, PurePath
38
from collections.abc import Iterable
39

40
from cerbero.enums import Platform, Architecture, Distro, DistroVersion
41
from cerbero.errors import FatalError, CommandError
42
from cerbero.utils import messages as m
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
43
44
45
46
47
48
49

_ = gettext.gettext
N_ = lambda x: x


class ArgparseArgument(object):

50
    def __init__(self, *name, **kwargs):
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
51
52
53
        self.name = name
        self.args = kwargs

54
    def add_to_parser(self, parser):
55
        parser.add_argument(*self.name, **self.args)
56
57


58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
class StoreBool(argparse.Action):

    def __init__(self, option_strings, dest, nargs=None, **kwargs):
        super().__init__(option_strings, dest, **kwargs)

    def __call__(self, parser, namespace, value, option_string=None):
        if value == 'yes':
            bvalue = True
        elif value == 'no':
            bvalue = False
        else:
            raise AssertionError
        setattr(namespace, self.dest, bvalue)


73
def user_is_root():
74
75
76
77
        ''' Check if the user running the process is root '''
        return hasattr(os, 'getuid') and os.getuid() == 0


Josep Torra's avatar
Josep Torra committed
78
def determine_num_of_cpus():
79
    ''' Number of virtual or logical CPUs on this system '''
Josep Torra's avatar
Josep Torra committed
80
81
82
83
84

    # Python 2.6+
    try:
        import multiprocessing
        return multiprocessing.cpu_count()
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
85
    except (ImportError, NotImplementedError):
Josep Torra's avatar
Josep Torra committed
86
87
88
        return 1


89
90
91
92
93
94
def to_winpath(path):
    if path.startswith('/'):
        path = '%s:%s' % (path[1], path[2:])
    return path.replace('/', '\\')


95
96
def to_unixpath(path):
    if path[1] == ':':
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
97
        path = '/%s%s' % (path[0], path[2:])
98
99
100
    return path


101
def to_winepath(path):
102
103
    # wine maps the filesystem root '/' to 'z:\'
    return str(PureWindowsPath('z:') / PurePath(path))
104
105


106
107
108
109
def fix_winpath(path):
    return path.replace('\\', '/')


110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def windows_arch():
    """
    Detecting the 'native' architecture of Windows is not a trivial task. We
    cannot trust that the architecture that Python is built for is the 'native'
    one because you can run 32-bit apps on 64-bit Windows using WOW64 and
    people sometimes install 32-bit Python on 64-bit Windows.
    """
    # These env variables are always available. See:
    # https://msdn.microsoft.com/en-us/library/aa384274(VS.85).aspx
    # https://blogs.msdn.microsoft.com/david.wang/2006/03/27/howto-detect-process-bitness/
    arch = os.environ.get('PROCESSOR_ARCHITEW6432', '').lower()
    if not arch:
        # If this doesn't exist, something is messing with the environment
        try:
            arch = os.environ['PROCESSOR_ARCHITECTURE'].lower()
        except KeyError:
            raise FatalError(_('Unable to detect Windows architecture'))
    return arch

129
130
131
132
133
134
135
def system_info():
    '''
    Get the sysem information.
    Return a tuple with the platform type, the architecture and the
    distribution
    '''
    # Get the platform info
136
137
138
    platform = os.environ.get('OS', '').lower()
    if not platform:
        platform = sys.platform
139
140
141
142
143
144
145
146
147
148
149
    if platform.startswith('win'):
        platform = Platform.WINDOWS
    elif platform.startswith('darwin'):
        platform = Platform.DARWIN
    elif platform.startswith('linux'):
        platform = Platform.LINUX
    else:
        raise FatalError(_("Platform %s not supported") % platform)

    # Get the architecture info
    if platform == Platform.WINDOWS:
150
        arch = windows_arch()
151
        if arch in ('x64', 'amd64'):
152
            arch = Architecture.X86_64
153
        elif arch == 'x86':
154
            arch = Architecture.X86
155
156
        else:
            raise FatalError(_("Windows arch %s is not supported") % arch)
157
158
159
160
161
162
    else:
        uname = os.uname()
        arch = uname[4]
        if arch == 'x86_64':
            arch = Architecture.X86_64
        elif arch.endswith('86'):
163
            arch = Architecture.X86
164
165
166
167
        elif arch.startswith('armv7'):
            arch = Architecture.ARMv7
        elif arch.startswith('arm'):
            arch = Architecture.ARM
168
169
        else:
            raise FatalError(_("Architecture %s not supported") % arch)
170
171
172

    # Get the distro info
    if platform == Platform.LINUX:
173
174
175
176
177
178
179
180
181
182
183
        if sys.version_info >= (3, 8, 0):
            try:
                import distro
            except ImportError:
                print('''Python >= 3.8 detected and the 'distro' python package was not found.
Please install the 'python3-distro' or 'python-distro' package from your linux package manager or from pypi using pip.
Terminating.''', file=sys.stderr)
                sys.exit(1)
            d = distro.linux_distribution()
        else:
            d = pplatform.linux_distribution()
184
185
186
187
188
189

        if d[0] == '' and d[1] == '' and d[2] == '':
            if os.path.exists('/etc/arch-release'):
                # FIXME: the python2.7 platform module does not support Arch Linux.
                # Mimic python3.4 platform.linux_distribution() output.
                d = ('arch', 'Arch', 'Linux')
190
191
192
193
            elif os.path.exists('/etc/os-release'):
                with open('/etc/os-release', 'r') as f:
                    if 'ID="amzn"\n' in f.readlines():
                        d = ('RedHat', 'amazon', '')
194
195
196
                    else:
                        f.seek(0, 0)
                        for line in f:
197
198
199
200
201
202
203
                            # skip empty lines and comment lines
                            if line.strip() and not line.lstrip().startswith('#'):
                                k,v = line.rstrip().split("=")
                                if k == 'NAME':
                                    name = v.strip('"')
                                elif k == 'VERSION_ID':
                                    version = v.strip('"')
204
                        d = (name, version, '');
205

206
        if d[0] in ['Ubuntu', 'debian', 'Debian GNU/Linux', 'LinuxMint', 'Linux Mint']:
207
            distro = Distro.DEBIAN
208
209
210
211
212
            distro_version = d[2].lower()
            split_str = d[2].split()
            if split_str:
                distro_version = split_str[0].lower()
            if distro_version in ['maverick', 'isadora']:
213
                distro_version = DistroVersion.UBUNTU_MAVERICK
214
            elif distro_version in ['lucid', 'julia']:
215
                distro_version = DistroVersion.UBUNTU_LUCID
216
            elif distro_version in ['natty', 'katya']:
217
                distro_version = DistroVersion.UBUNTU_NATTY
218
            elif distro_version in ['oneiric', 'lisa']:
219
                distro_version = DistroVersion.UBUNTU_ONEIRIC
220
            elif distro_version in ['precise', 'maya']:
221
                distro_version = DistroVersion.UBUNTU_PRECISE
222
            elif distro_version in ['quantal', 'nadia']:
223
                distro_version = DistroVersion.UBUNTU_QUANTAL
224
            elif distro_version in ['raring', 'olivia']:
225
                distro_version = DistroVersion.UBUNTU_RARING
226
            elif distro_version in ['saucy', 'petra']:
Josep Torra's avatar
Josep Torra committed
227
                distro_version = DistroVersion.UBUNTU_SAUCY
228
            elif distro_version in ['trusty', 'qiana', 'rebecca']:
229
                distro_version = DistroVersion.UBUNTU_TRUSTY
230
            elif distro_version in ['utopic']:
231
                distro_version = DistroVersion.UBUNTU_UTOPIC
232
            elif distro_version in ['vivid']:
233
                distro_version = DistroVersion.UBUNTU_VIVID
234
            elif distro_version in ['wily']:
Alistair Buxton's avatar
Alistair Buxton committed
235
                distro_version = DistroVersion.UBUNTU_WILY
236
            elif distro_version in ['xenial', 'sarah', 'serena', 'sonya', 'sylvia']:
Xavier Claessens's avatar
Xavier Claessens committed
237
                distro_version = DistroVersion.UBUNTU_XENIAL
238
            elif distro_version in ['artful']:
Xavier Claessens's avatar
Xavier Claessens committed
239
                distro_version = DistroVersion.UBUNTU_ARTFUL
240
            elif distro_version in ['bionic', 'tara', 'tessa', 'tina', 'tricia']:
241
                distro_version = DistroVersion.UBUNTU_BIONIC
242
            elif distro_version in ['cosmic']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
243
                distro_version = DistroVersion.UBUNTU_COSMIC
244
            elif distro_version in ['disco']:
245
                distro_version = DistroVersion.UBUNTU_DISCO
246
            elif distro_version in ['eoan']:
Maxim Paymushkin's avatar
Maxim Paymushkin committed
247
                distro_version = DistroVersion.UBUNTU_EOAN
248
            elif distro_version in ['focal', 'ulyana']:
Seungha Yang's avatar
Seungha Yang committed
249
                distro_version = DistroVersion.UBUNTU_FOCAL
250
            elif d[1].startswith('6.'):
251
                distro_version = DistroVersion.DEBIAN_SQUEEZE
252
            elif d[1].startswith('7.') or d[1].startswith('wheezy'):
253
                distro_version = DistroVersion.DEBIAN_WHEEZY
254
            elif d[1].startswith('8.') or d[1].startswith('jessie'):
Sebastian Dröge's avatar
Sebastian Dröge committed
255
                distro_version = DistroVersion.DEBIAN_JESSIE
256
            elif d[1].startswith('9.') or d[1].startswith('stretch'):
257
                distro_version = DistroVersion.DEBIAN_STRETCH
258
259
            elif d[1].startswith('10.') or d[1].startswith('buster'):
                distro_version = DistroVersion.DEBIAN_BUSTER
260
261
            elif d[1].startswith('11.') or d[1].startswith('bullseye'):
                distro_version = DistroVersion.DEBIAN_BULLSEYE
262
263
            elif d[1] == 'unstable' and d[2] == 'sid':
                distro_version = DistroVersion.DEBIAN_SID
264
265
            else:
                raise FatalError("Distribution '%s' not supported" % str(d))
266
        elif d[0] in ['RedHat', 'Fedora', 'CentOS', 'Red Hat Enterprise Linux Server', 'CentOS Linux', 'Amazon Linux']:
267
            distro = Distro.REDHAT
268
269
270
271
272
273
            if d[1] == '16':
                distro_version = DistroVersion.FEDORA_16
            elif d[1] == '17':
                distro_version = DistroVersion.FEDORA_17
            elif d[1] == '18':
                distro_version = DistroVersion.FEDORA_18
Sebastian Dröge's avatar
Sebastian Dröge committed
274
275
            elif d[1] == '19':
                distro_version = DistroVersion.FEDORA_19
Thibault Saunier's avatar
Thibault Saunier committed
276
277
            elif d[1] == '20':
                distro_version = DistroVersion.FEDORA_20
278
279
            elif d[1] == '21':
                distro_version = DistroVersion.FEDORA_21
280
281
            elif d[1] == '22':
                distro_version = DistroVersion.FEDORA_22
Olivier Crête's avatar
Olivier Crête committed
282
283
            elif d[1] == '23':
                distro_version = DistroVersion.FEDORA_23
284
285
            elif d[1] == '24':
                distro_version = DistroVersion.FEDORA_24
286
287
            elif d[1] == '25':
                distro_version = DistroVersion.FEDORA_25
288
289
            elif d[1] == '26':
                distro_version = DistroVersion.FEDORA_26
Olivier Crête's avatar
Olivier Crête committed
290
291
            elif d[1] == '27':
                distro_version = DistroVersion.FEDORA_27
292
293
            elif d[1] == '28':
                distro_version = DistroVersion.FEDORA_28
Nicolas Dufresne's avatar
Nicolas Dufresne committed
294
295
            elif d[1] == '29':
                distro_version = DistroVersion.FEDORA_29
Olivier Crête's avatar
Olivier Crête committed
296
297
            elif d[1] == '30':
                distro_version = DistroVersion.FEDORA_30
298
299
            elif d[1] == '31':
                distro_version = DistroVersion.FEDORA_31
300
301
            elif d[1] == '32':
                distro_version = DistroVersion.FEDORA_32
302
303
304
305
            elif d[0] == 'Fedora':
                # str(int()) is for ensuring that the fedora version is
                # actually a number
                distro_version = 'fedora_' + str(int(d[1]))
306
307
            elif d[1].startswith('6.'):
                distro_version = DistroVersion.REDHAT_6
Olivier Crête's avatar
Olivier Crête committed
308
309
            elif d[1].startswith('7.'):
                distro_version = DistroVersion.REDHAT_7
310
311
            elif d[1].startswith('8.'):
                distro_version = DistroVersion.REDHAT_8
312
313
            elif d[0] == 'Amazon Linux' and d[1].startswith('2'):
                distro_version = DistroVersion.AMAZON_LINUX_2
314
315
            elif d[1] == 'amazon':
                distro_version = DistroVersion.AMAZON_LINUX
316
317
318
            else:
                # FIXME Fill this
                raise FatalError("Distribution '%s' not supported" % str(d))
319
        elif d[0].strip() in ['openSUSE']:
320
            distro = Distro.SUSE
Georg Lippitsch's avatar
Georg Lippitsch committed
321
            if d[1] == '42.2':
Georg Lippitsch's avatar
Georg Lippitsch committed
322
                distro_version = DistroVersion.OPENSUSE_42_2
Georg Lippitsch's avatar
Georg Lippitsch committed
323
324
            elif d[1] == '42.3':
                distro_version = DistroVersion.OPENSUSE_42_3
325
326
            else:
                # FIXME Fill this
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
327
328
                raise FatalError("Distribution OpenSuse '%s' "
                                 "not supported" % str(d))
329
330
331
        elif d[0].strip() in ['openSUSE Tumbleweed']:
            distro = Distro.SUSE
            distro_version = DistroVersion.OPENSUSE_TUMBLEWEED
332
        elif d[0].strip() in ['arch', 'Arch Linux']:
333
334
            distro = Distro.ARCH
            distro_version = DistroVersion.ARCH_ROLLING
335
336
337
        elif d[0].strip() in ['Gentoo Base System']:
            distro = Distro.GENTOO
            distro_version = DistroVersion.GENTOO_VERSION
338
339
340
341
        else:
            raise FatalError("Distribution '%s' not supported" % str(d))
    elif platform == Platform.WINDOWS:
        distro = Distro.WINDOWS
342
        win32_ver = pplatform.win32_ver()[0]
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
343
344
        dmap = {'xp': DistroVersion.WINDOWS_XP,
                'vista': DistroVersion.WINDOWS_VISTA,
Sebastian Dröge's avatar
Sebastian Dröge committed
345
                '7': DistroVersion.WINDOWS_7,
346
                'post2008Server': DistroVersion.WINDOWS_8,
Sebastian Dröge's avatar
Sebastian Dröge committed
347
                '8': DistroVersion.WINDOWS_8,
348
349
                'post2012Server': DistroVersion.WINDOWS_8_1,
                '8.1': DistroVersion.WINDOWS_8_1,
Sebastian Dröge's avatar
Sebastian Dröge committed
350
                '10': DistroVersion.WINDOWS_10}
351
352
        if win32_ver in dmap:
            distro_version = dmap[win32_ver]
353
        else:
354
355
356
            raise FatalError("Windows version '%s' not supported" % win32_ver)
    elif platform == Platform.DARWIN:
        distro = Distro.OS_X
FLUENDO's avatar
FLUENDO committed
357
        ver = pplatform.mac_ver()[0]
358
        if ver.startswith(('11.', '10.16')):
Nirbheek Chauhan's avatar
Nirbheek Chauhan committed
359
360
            distro_version = DistroVersion.OS_X_BIG_SUR
        elif ver.startswith('10.15'):
361
362
            distro_version = DistroVersion.OS_X_CATALINA
        elif ver.startswith('10.14'):
Justin Kim's avatar
Justin Kim committed
363
364
            distro_version = DistroVersion.OS_X_MOJAVE
        elif ver.startswith('10.13'):
Boris Prohaska's avatar
Boris Prohaska committed
365
366
            distro_version = DistroVersion.OS_X_HIGH_SIERRA
        elif ver.startswith('10.12'):
Pierre Lamot's avatar
Pierre Lamot committed
367
368
            distro_version = DistroVersion.OS_X_SIERRA
        elif ver.startswith('10.11'):
369
370
            distro_version = DistroVersion.OS_X_EL_CAPITAN
        elif ver.startswith('10.10'):
371
372
            distro_version = DistroVersion.OS_X_YOSEMITE
        elif ver.startswith('10.9'):
Josep Torra's avatar
Josep Torra committed
373
374
            distro_version = DistroVersion.OS_X_MAVERICKS
        elif ver.startswith('10.8'):
375
            distro_version = DistroVersion.OS_X_MOUNTAIN_LION
FLUENDO's avatar
FLUENDO committed
376
377
        else:
            raise FatalError("Mac version %s not supported" % ver)
378

Josep Torra's avatar
Josep Torra committed
379
380
381
    num_of_cpus = determine_num_of_cpus()

    return platform, arch, distro, distro_version, num_of_cpus
382
383
384
385


def validate_packager(packager):
    # match packager in the form 'Name <email>'
Andoni Morales Alastruey's avatar
Andoni Morales Alastruey committed
386
    expr = r'(.*\s)*[<]([a-zA-Z0-9+_\-\.]+@'\
387
        '[0-9a-zA-Z][.-0-9a-zA-Z]*.[a-zA-Z]+)[>]$'
388
    return bool(re.match(expr, packager))
389
390


391
def copy_files(origdir, destdir, files, extensions, target_platform, logfile=None):
392
393
394
395
396
    for f in files:
        f = f % extensions
        install_dir = os.path.dirname(os.path.join(destdir, f))
        if not os.path.exists(install_dir):
            os.makedirs(install_dir)
397
398
        if destdir[1] == ':':
            # windows path
399
400
401
402
403
            relprefix = to_unixpath(destdir)[2:]
        else:
            relprefix = destdir[1:]
        orig = os.path.join(origdir, relprefix, f)
        dest = os.path.join(destdir, f)
404
        m.action("copying %s to %s" % (orig, dest), logfile=logfile)
405
406
407
408
        try:
            shutil.copy(orig, dest)
        except IOError:
            m.warning("Could not copy %s to %s" % (orig, dest))
409
410
411
412
413
414


def remove_list_duplicates(seq):
    ''' Remove list duplicates maintaining the order '''
    seen = set()
    seen_add = seen.add
415
    return [x for x in seq if x not in seen and not seen_add(x)]
416
417
418


def parse_file(filename, dict):
419
420
    if '__file__' not in dict:
        dict['__file__'] = filename
421
    try:
422
423
        exec(compile(open(filename).read(), filename, 'exec'), dict)
    except Exception as ex:
424
425
        import traceback
        traceback.print_exc()
426
        raise ex
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445


def escape_path(path):
    path = path.replace('\\', '/')
    path = path.replace('(', '\\\(').replace(')', '\\\)')
    path = path.replace(' ', '\\\\ ')
    return path


def get_wix_prefix():
    if 'WIX' in os.environ:
        wix_prefix = os.path.join(os.environ['WIX'], 'bin')
    else:
        wix_prefix = 'C:/Program Files%s/Windows Installer XML v3.5/bin'
        if not os.path.exists(wix_prefix):
            wix_prefix = wix_prefix % ' (x86)'
    if not os.path.exists(wix_prefix):
        raise FatalError("The required packaging tool 'WiX' was not found")
    return escape_path(to_unixpath(wix_prefix))
446

447
def add_system_libs(config, new_env, old_env=None):
448
449
450
451
452
453
    '''
    Add /usr/lib/pkgconfig to PKG_CONFIG_PATH so the system's .pc file
    can be found.
    '''
    arch = config.target_arch
    libdir = 'lib'
454

455
456
457
458
459
460
461
462
463
    # Only use this when compiling on Linux for Linux and not cross-compiling
    # to some other Linux
    if config.platform != Platform.LINUX:
        return
    if config.target_platform != Platform.LINUX:
        return
    if config.cross_compiling():
        return

464
    if arch == Architecture.X86_64:
Georg Lippitsch's avatar
Georg Lippitsch committed
465
        if config.distro == Distro.REDHAT or config.distro == Distro.SUSE:
466
            libdir = 'lib64'
467

468
469
470
471
    sysroot = '/'
    if config.sysroot:
        sysroot = config.sysroot

472
473
474
    if not old_env:
        old_env = os.environ

475
    search_paths = []
476
477
    if old_env.get('PKG_CONFIG_LIBDIR', None):
       search_paths += [old_env['PKG_CONFIG_LIBDIR']]
478
479
    if old_env.get('PKG_CONFIG_PATH', None):
       search_paths += [old_env['PKG_CONFIG_PATH']]
480
    search_paths += [
481
        os.path.join(sysroot, 'usr', libdir, 'pkgconfig'),
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
        os.path.join(sysroot, 'usr/share/pkgconfig')]

    if config.target_distro == Distro.DEBIAN:
        host = None
        if arch == Architecture.ARM:
            host = 'arm-linux-gnueabi'
        elif arch == Architecture.ARM64:
            host = 'aarch64-linux-gnu'
        elif arch == Architecture.X86:
            host = 'i386-linux-gnu'
        elif Architecture.is_arm(arch):
            host = 'arm-linux-gnueabihf'
        else:
            host = '%s-linux-gnu' % arch

        search_paths.append(os.path.join(sysroot, 'usr/lib/%s/pkgconfig' % host))

499
    new_env['PKG_CONFIG_PATH'] = ':'.join(search_paths)
500

501
502
    search_paths = [os.environ.get('ACLOCAL_PATH', ''),
        os.path.join(sysroot, 'usr/share/aclocal')]
503
    new_env['ACLOCAL_PATH'] = ':'.join(search_paths)
504
505
506
507
508

def needs_xcode8_sdk_workaround(config):
    '''
    Returns whether the XCode 8 clock_gettime, mkostemp, getentropy workaround
    from https://bugzilla.gnome.org/show_bug.cgi?id=772451 is needed
509
510

    These symbols are only available on macOS 10.12+ and iOS 10.0+
511
    '''
512
513
514
515
516
517
    if config.target_platform == Platform.DARWIN:
        if StrictVersion(config.min_osx_sdk_version) < StrictVersion('10.12'):
            return True
    elif config.target_platform == Platform.IOS:
        if StrictVersion(config.ios_min_version) < StrictVersion('10.0'):
            return True
518
    return False
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545

def _qmake_or_pkgdir(qmake):
    qmake_path = Path(qmake)
    if not qmake_path.is_file():
        m.warning('QMAKE={!r} does not exist'.format(str(qmake_path)))
        return (None, None)
    pkgdir = (qmake_path.parent.parent / 'lib/pkgconfig')
    if pkgdir.is_dir():
        return (pkgdir.as_posix(), qmake_path.as_posix())
    return (None, qmake_path.as_posix())

def detect_qt5(platform, arch, is_universal):
    '''
    Returns both the path to the pkgconfig directory and the path to qmake:
    (pkgdir, qmake). If `pkgdir` could not be found, it will be None

    Returns (None, None) if nothing was found.
    '''
    path = None
    qt5_prefix = os.environ.get('QT5_PREFIX', None)
    qmake_path = os.environ.get('QMAKE', None)
    if not qt5_prefix and not qmake_path:
        return (None, None)
    if qt5_prefix and not os.path.isdir(qt5_prefix):
        m.warning('QT5_PREFIX={!r} does not exist'.format(qt5_prefix))
        return (None, None)
    if qmake_path:
546
547
548
549
550
551
552
553
554
        try:
            qt_version = shell.check_output([qmake_path, '-query', 'QT_VERSION']).strip()
            qt_version = [int(v) for v in qt_version.split('.')]
        except CommandError as e:
            m.warning('QMAKE={!r} failed to execute:\n{}'.format(str(qmake_path), str(e)))
            qt_version = [0, 0]
        if len(qt_version) >= 2 and qt_version[:2] < [5, 14] and \
           is_universal and platform == Platform.ANDROID:
            # require QT5_PREFIX before Qt 5.14 with android universal
555
556
            if not qt5_prefix:
                m.warning('Please set QT5_PREFIX if you want to build '
557
                          'the Qt5 plugin for android-universal with Qt < 5.14')
558
559
560
561
562
563
564
                return (None, None)
        else:
            ret = _qmake_or_pkgdir(qmake_path)
            if ret != (None, None) or not qt5_prefix:
                return ret
    # qmake path is invalid, find pkgdir or qmake from qt5 prefix
    if platform == Platform.ANDROID:
565
566
567
568
569
        # Qt => 5.14
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android/bin/qmake'))
        if ret != (None, None):
            return ret
        # Qt < 5.14
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
        if arch == Architecture.ARMv7:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_armv7/bin/qmake'))
        elif arch == Architecture.ARM64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_arm64_v8a/bin/qmake'))
        elif arch == Architecture.X86:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'android_x86/bin/qmake'))
        elif arch == Architecture.X86_64:
            # Qt binaries do not ship a qmake for android_x86_64
            return (None, None)
    elif platform == Platform.DARWIN:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'clang_64/bin/qmake'))
    elif platform == Platform.IOS:
        ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'ios/bin/qmake'))
    elif platform == Platform.LINUX:
        if arch == Architecture.X86_64:
            ret = _qmake_or_pkgdir(os.path.join(qt5_prefix, 'gcc_64/bin/qmake'))
    elif platform == Platform.WINDOWS:
        # There are several msvc and mingw toolchains to pick from, and we
        # can't pick it for the user.
        m.warning('You must set QMAKE instead of QT5_PREFIX on Windows')
        return (None, None)
    if ret == (None, None):
        m.warning('Unsupported arch {!r} on platform {!r}'.format(arch, platform))
    return ret
595

596
597
598
599
600
601
602
603
604
605
# asyncio.Semaphore classes set their working event loop internally on
# creation, so we need to ensure the proper loop has already been set by then.
# This is especially important if we create global semaphores that are
# initialized at the very beginning, since on Windows, the default
# SelectorEventLoop is not available.
def CerberoSemaphore(value=1):
    get_event_loop() # this ensures the proper event loop is already created
    return asyncio.Semaphore(value)

def get_event_loop():
606
607
608
609
610
611
612
613
614
615
616
617
618
    try:
        loop = asyncio.get_event_loop()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

    # On Windows the default SelectorEventLoop is not available:
    # https://docs.python.org/3.5/library/asyncio-subprocess.html#windows-event-loop
    if sys.platform == 'win32' and \
       not isinstance(loop, asyncio.ProactorEventLoop):
        loop = asyncio.ProactorEventLoop()
        asyncio.set_event_loop(loop)

619
620
621
622
623
624
    # Avoid spammy BlockingIOError warnings with older python versions
    if sys.platform != 'win32' and \
       sys.version_info < (3, 8, 0):
        asyncio.set_child_watcher(asyncio.FastChildWatcher())
        asyncio.get_child_watcher().attach_loop(loop)

625
626
627
    return loop

def run_until_complete(tasks):
628
629
630
631
632
633
634
635
636
    '''
    Runs one or many tasks, blocking until all of them have finished.
    @param tasks: A single Future or a list of Futures to run
    @type tasks: Future or list of Futures
    @return: the result of the asynchronous task execution (if only
             one task) or a list of all results in case of multiple
             tasks. Result is None if operation is cancelled.
    @rtype: any type or list of any types in case of multiple tasks
    '''
637
638
    loop = get_event_loop()

639
640
    try:
        if isinstance(tasks, Iterable):
641
            result = loop.run_until_complete(asyncio.gather(*tasks))
642
        else:
643
644
            result = loop.run_until_complete(tasks)
        return result
645
    except asyncio.CancelledError:
646
        return None
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661

async def run_tasks(tasks, done_async=None):
    """
    Runs @tasks until completion or until @done_async returns
    """
    class QueueDone(Exception):
        pass

    if done_async:
        async def queue_done():
            # This is how we exit the asyncio.wait once everything is done
            # as otherwise asyncio.wait will wait for our tasks to complete
            await done_async
            raise QueueDone()

662
        task = asyncio.ensure_future (queue_done())
663
664
        tasks.append(task)

665
666
667
668
    async def shutdown(abnormal=True):
        tasks_minus_current = [t for t in tasks]
        [task.cancel() for task in tasks_minus_current]
        ret = await asyncio.gather(*tasks_minus_current, return_exceptions=True)
669
670
        # we want to find any actual exception rather than one
        # that may be returned from task.cancel()
671
        cancelled = None
672
        for e in ret:
673
674
            if isinstance(e, asyncio.CancelledError):
                cancelled = e
675
676
677
678
            if isinstance(e, Exception) \
               and not isinstance(e, asyncio.CancelledError) \
               and not isinstance(e, QueueDone):
                raise e
679
680
681
682
        if abnormal and cancelled:
            # use cancelled as a last resort we would prefer to throw any
            # other exception but only if something abnormal happened
            raise cancelled
683
684
685
686

    try:
        await asyncio.gather(*tasks, return_exceptions=False)
    except asyncio.CancelledError:
687
        raise
688
    except QueueDone:
689
        await shutdown(abnormal=False)
690
691
692
    except Exception:
        await shutdown()
        raise
693
694
695
696
697
698
699
700
701
702
703
704
705
706


class EnvVar:
    @staticmethod
    def is_path(var):
        return var in ('LD_LIBRARY_PATH', 'PATH', 'MANPATH', 'INFOPATH',
                'PKG_CONFIG_PATH', 'PKG_CONFIG_LIBDIR', 'GI_TYPELIB_PATH',
                'XDG_DATA_DIRS', 'XDG_CONFIG_DIRS', 'GST_PLUGIN_PATH',
                'GST_PLUGIN_PATH_1_0', 'PYTHONPATH', 'MONO_PATH', 'LIB',
                'INCLUDE', 'PATHEXT')

    @staticmethod
    def is_arg(var):
        return var in ('CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'LDFLAGS',
707
                'OBJCFLAGS', 'OBJCXXFLAGS', 'OBJLDFLAGS', 'CCASFLAGS')
708
709
710
711
712

    @staticmethod
    def is_cmd(var):
        return var in ('AR', 'AS', 'CC', 'CPP', 'CXX', 'DLLTOOL', 'GENDEF',
                'LD', 'NM', 'OBJC', 'OBJCOPY', 'OBJCXX', 'PERL', 'PYTHON',
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
                'RANLIB', 'RC', 'STRIP', 'WINDRES')


class EnvValue(list):
    '''
    Env var value (list of strings) with an associated separator
    '''

    def __init__(self, sep, *values):
        self.sep = sep
        super().__init__(*values)

    def get(self):
        return str.join(self.sep, self)

    @staticmethod
    def from_key(key, value):
        if EnvVar.is_path(key):
            return EnvValuePath(value)
        if EnvVar.is_arg(key):
            return EnvValueArg(value)
        if EnvVar.is_cmd(key):
            return EnvValueCmd(value)
        return EnvValueSingle(value)


class EnvValueSingle(EnvValue):
    '''
    Env var with a single value
    '''

    def __init__(self, *values):
        if len(values) == 1:
            if not isinstance(values[0], list):
                values = ([values[0]],)
            elif len(values[0]) > 1:
                raise ValueError('EnvValue can only have a single value, not multiple')
        super().__init__(None, *values)

    def __iadd__(self, new):
        if len(self) != 0:
            raise ValueError('In-place add not allowed for EnvValue {!r}'.format(self))
        return super().__iadd__(new)

    def get(self):
        return self[0]


class EnvValueArg(EnvValue):
    '''
    Env var containing a list of quoted arguments separated by space
    '''

    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (shlex.split(values[0]),)
        super().__init__(' ', *values)

    def get(self):
        return ' '.join([shlex.quote(x) for x in self])


class EnvValueCmd(EnvValueArg):
    '''
    Env var containing a command and a list of arguments separated by space
    '''

    def __iadd__(self, new):
        if isinstance(new, EnvValueCmd):
            raise ValueError('In-place add not allowed for EnvValueCmd {!r} and EnvValueCmd {!r}'.format(self, new))
        return super().__iadd__(new)


class EnvValuePath(EnvValue):
    '''
    Env var containing a list of paths separated by os.pathsep, which is `:` or `;`
    '''
    def __init__(self, *values):
        if len(values) == 1 and not isinstance(values[0], list):
            values = (values[0].split(os.pathsep),)
        super().__init__(os.pathsep, *values)