Commit 3996812e authored by Nirbheek Chauhan's avatar Nirbheek Chauhan 🐜
Browse files

cerbero: Split all bootstrappers into fetch/extract/start

Now when you run bootstrap --offline, the bootstrappers will
not require network access.

All URLs downloaded in the fetch() step now point to locations with
fixed contents and are checksummed.

FIXME: On Windows mingw-get is still run which uses the network

https://bugzilla.gnome.org/show_bug.cgi?id=797316
parent 63e1b6d5
......@@ -34,15 +34,15 @@ class BootstrapTarball(BaseTarball):
class BootstrapperBase (object):
# Dict of URLs to be fetched and their checksums
# List of URLs to be fetched
fetch_urls = None
# Dict of extract steps to be done
# List of extract steps to be performed
extract_steps = None
def __init__(self, config, offline):
self.config = config
self.offline = offline
self.fetch_urls = {}
self.fetch_urls = []
self.extract_steps = []
self.sources = {}
......@@ -50,7 +50,8 @@ class BootstrapperBase (object):
raise NotImplemented("'start' must be implemented by subclasses")
def fetch(self):
for url, checksum in self.fetch_urls.items():
'Fetch bootstrap binaries'
for (url, checksum) in self.fetch_urls:
source = BootstrapTarball(self.config, self.offline, url, checksum,
self.config.local_sources)
self.sources[url] = source
......
......@@ -17,34 +17,46 @@
# Boston, MA 02111-1307, USA.
import os
import shutil
from cerbero.bootstrap import BootstrapperBase
from cerbero.bootstrap.bootstrapper import register_bootstrapper
from cerbero.config import Distro, FatalError
from cerbero.utils import _, shell
NDK_VERSION = 'r16'
NDK_BASE_URL = 'https://dl.google.com/android/repository/android-ndk-%s-%s-%s.zip'
NDK_CHECKSUMS = {
'android-ndk-r16-linux-x86_64.zip': 'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e',
'android-ndk-r16-darwin-x86_64.zip': '589a567c4fc84f5f7219cae56d199b63059033e08512b6d8132739c90ef9483d',
'android-ndk-r16-windows-x86_64.zip': 'fb4ce0bbcb927dc1bb507d3dbb1aef0f243e5184b3dd5d956a450bcf4b0808df',
'android-ndk-r16-windows-x86.zip': 'aaa52b9e8b283be1249376f7373704687a5b13106575f613378e6d44c663f782',
}
class AndroidBootstrapper (BootstrapperBase):
NDK_BASE_URL = 'http://dl.google.com/android/repository/'
NDK_VERSION = 'r16'
NDK_ZIP = 'android-ndk-' + NDK_VERSION + '-%s-%s.zip'
def __init__(self, config, offline):
super().__init__(config, offline)
self.prefix = self.config.toolchain_prefix
url = NDK_BASE_URL % (NDK_VERSION, self.config.platform, self.config.arch)
self.fetch_urls.append((url, NDK_CHECKSUMS[os.path.basename(url)]))
self.extract_steps.append((url, True, self.prefix))
def start(self):
dest = self.config.toolchain_prefix
ndk_zip = self.NDK_ZIP % (self.config.platform, self.config.arch)
zip_file = os.path.join(dest, ndk_zip)
try:
os.makedirs(dest)
except:
pass
shell.download("%s/%s" % (self.NDK_BASE_URL, ndk_zip), zip_file)
if not os.path.exists(os.path.join(dest, "ndk-build")):
try:
shell.call('unzip %s' % ndk_zip, dest)
shell.call('mv android-ndk-%s/* .' % self.NDK_VERSION, dest)
except Exception as ex:
raise FatalError(_("Error installing Android NDK: %s") % (ex))
if not os.path.exists(self.prefix):
os.makedirs(self.prefix)
ndkdir = os.path.join(self.prefix, 'android-ndk-' + NDK_VERSION)
if not os.path.isdir(ndkdir):
return
# Android NDK extracts to android-ndk-$NDK_VERSION, so move its
# contents to self.prefix
for d in os.listdir(ndkdir):
dest = os.path.join(self.prefix, d)
if os.path.exists(dest):
shutil.rmtree(dest)
shutil.move(os.path.join(ndkdir, d), self.prefix)
os.rmdir(ndkdir)
def register_all():
......
......@@ -17,18 +17,23 @@
# Boston, MA 02111-1307, USA.
import os
import tempfile
import shutil
from cerbero.bootstrap import BootstrapperBase
from cerbero.bootstrap.bootstrapper import register_bootstrapper
from cerbero.config import Distro, DistroVersion
from cerbero.config import Distro
from cerbero.utils import shell
CPANM_VERSION = '1.7044'
CPANM_URL_TPL = 'https://raw.githubusercontent.com/miyagawa/cpanminus/{}/cpanm'
CPANM_CHECKSUM = '22b92506243649a73cfb55c5990cedd24cdbb20b15b4530064d2496d94d1642b'
class OSXBootstrapper (BootstrapperBase):
CPANM_URL = 'https://raw.github.com/miyagawa/cpanminus/master/cpanm'
def __init__(self, config, offline):
super().__init__(config, offline)
url = CPANM_URL_TPL.format(CPANM_VERSION)
self.fetch_urls.append((url, CPANM_CHECKSUM))
def start(self):
# skip system package install if not needed
......@@ -37,14 +42,10 @@ class OSXBootstrapper (BootstrapperBase):
self._install_perl_deps()
def _install_perl_deps(self):
# Install cpan-minus, a zero-conf CPAN wrapper
cpanm_installer = tempfile.NamedTemporaryFile()
shell.download(self.CPANM_URL, cpanm_installer.name, overwrite=True)
shell.call('chmod +x %s' % cpanm_installer.name)
cpanm_installer = os.path.join(self.config.local_sources, 'cpanm')
shell.call('chmod +x %s' % cpanm_installer)
# Install XML::Parser, required for intltool
shell.call("sudo %s XML::Parser" % cpanm_installer.name)
cpanm_installer.close()
shell.call("sudo %s XML::Parser" % cpanm_installer)
def register_all():
......
......@@ -30,18 +30,32 @@ from cerbero.utils import messages as m
# Toolchain
GCC_VERSION = '4.7.3'
MINGW_DOWNLOAD_SOURCE = 'http://gstreamer.freedesktop.org/data/cerbero/toolchain/windows'
MINGW_TARBALL_TPL = "mingw-%s-gcc-%s-%s-%s.tar.xz"
MINGW_DOWNLOAD_TPL = 'https://gstreamer.freedesktop.org/data/cerbero/toolchain/windows/mingw-%s-gcc-%s-%s-%s.tar.xz'
MINGW_CHECKSUMS = {
'mingw-w32-gcc-4.7.3-linux-x86.tar.xz': '16a3ad2584f0dc25ec122029143b186c99f362d1be1a77a338431262491004ae',
'mingw-w64-gcc-4.7.3-linux-x86_64.tar.xz': 'e673536cc89a778043789484f691d7e35458a5d72638dc4b0123f92ecf868592',
'mingw-w32-gcc-4.7.3-windows-x86.tar.xz': 'da783488ab3a2b28471c13ece97c643f8e8ec774308fb2a01152b23618f13a33',
'mingw-w32-gcc-4.7.3-windows-x86_64.tar.xz': '820fa7490b3d738b9cf8c360cdd9a7aeb0592510a8ea50486e721b5b92722b08',
}
# MinGW Perl
PERL_VERSION = '5.24.0'
MINGW_PERL_TPL = 'https://sourceforge.net/projects/perl-mingw/files/{0}/perl-{0}-mingw32.zip'
MINGW_PERL_CHECKSUM = '9d4db40959727d43b4ff4b9884f5aebda20292347978036683684c2608c1397b'
# Extra dependencies
MINGWGET_DEPS = ['msys-wget', 'msys-flex', 'msys-bison', 'msys-perl']
GNOME_FTP = 'http://ftp.gnome.org/pub/gnome/binaries/win32/'
WINDOWS_BIN_DEPS = ['intltool/0.40/intltool_0.40.4-1_win32.zip']
# Khronos wglext.h for Windows GL
OPENGL_COMMIT = 'a4b0c7d5d10a8fca5866c6d08a608010843a4b36'
KHRONOS_WGL_TPL = 'https://raw.githubusercontent.com/KhronosGroup/OpenGL-Registry/{}/api/GL/wglext.h'
WGL_CHECKSUM = '8961c809d180e3590fca32053341fe3a83394edcb936f7699f0045feadb16115'
# Extra binary dependencies
GNOME_FTP = 'https://download.gnome.org/binaries/win32/'
WINDOWS_BIN_DEPS = [
('intltool/0.40/intltool_0.40.4-1_win32.zip',
'7180a780cee26c5544c06a73513c735b7c8c107db970b40eb7486ea6c936cb33')]
# MSYS packages needed
MINGWGET_DEPS = ['msys-wget', 'msys-flex', 'msys-bison', 'msys-perl']
class WindowsBootstrapper(BootstrapperBase):
'''
......@@ -60,6 +74,30 @@ class WindowsBootstrapper(BootstrapperBase):
else:
self.version = 'w64'
self.platform = self.config.platform
# Register all network resources this bootstrapper needs. They will all
# be downloaded into self.config.local_sources
#
# MinGW toolchain
url = MINGW_DOWNLOAD_TPL % (self.version, GCC_VERSION, self.platform, self.arch)
self.fetch_urls.append((url, MINGW_CHECKSUMS[os.path.basename(url)]))
self.extract_steps.append((url, True, self.prefix))
# wglext.h
url = KHRONOS_WGL_TPL.format(OPENGL_COMMIT)
self.fetch_urls.append((url, WGL_CHECKSUM))
inst_path = os.path.join(self.prefix, self.config.host, 'include/GL')
self.extract_steps.append((url, False, inst_path))
if self.platform == Platform.WINDOWS:
# MinGW Perl needed by openssl
url = MINGW_PERL_TPL.format(PERL_VERSION)
self.fetch_urls.append((url, MINGW_PERL_CHECKSUM))
self.extract_steps.append((url, True, self.perl_prefix))
# Newer versions of binary deps such as intltool. Must be extracted
# after the MinGW toolchain from above is extracted so that it
# replaces the older files.
for dep, checksum in WINDOWS_BIN_DEPS:
url = GNOME_FTP + dep
self.fetch_urls.append((url, checksum))
self.extract_steps.append((url, True, self.prefix))
def start(self):
if not git.check_line_endings(self.config.platform):
......@@ -67,19 +105,14 @@ class WindowsBootstrapper(BootstrapperBase):
"endings conversion. You can fix it running:\n"
"$git config core.autocrlf false")
self.check_dirs()
self.fix_mingw()
self.fix_non_prefixed_strings()
if self.platform == Platform.WINDOWS:
self.msys_mingw_bindir = Path(shutil.which('mingw-get')).parent
self.fix_openssl_mingw_perl()
self.fix_bin_deps()
# FIXME: This uses the network
self.install_mingwget_deps()
self.install_mingw()
if self.platform == Platform.WINDOWS:
self.remove_mingw_unused()
self.add_non_prefixed_strings()
if self.platform == Platform.WINDOWS:
# After mingw has been installed
self.install_bin_deps()
self.install_gl_headers()
if self.platform == Platform.WINDOWS:
self.install_openssl_mingw_perl()
self.fix_mingw_unused()
def check_dirs(self):
if not os.path.exists(self.perl_prefix):
......@@ -90,14 +123,7 @@ class WindowsBootstrapper(BootstrapperBase):
if not os.path.exists(etc_path):
os.makedirs(etc_path)
def install_mingw(self):
tarball = MINGW_TARBALL_TPL % (self.version, GCC_VERSION,
self.platform, self.arch)
tarfile = os.path.join(self.prefix, tarball)
tarfile = os.path.abspath(tarfile)
shell.download("%s/%s" % (MINGW_DOWNLOAD_SOURCE, tarball), tarfile, check_cert=False)
shell.unpack(tarfile, self.prefix)
def fix_mingw(self):
self.fix_lib_paths()
if self.arch == Architecture.X86:
try:
......@@ -105,17 +131,12 @@ class WindowsBootstrapper(BootstrapperBase):
except Exception:
pass
def install_openssl_mingw_perl(self):
def fix_openssl_mingw_perl(self):
'''
This perl is only used by openssl; we can't use it everywhere else
because it can't find msys tools, and so perl scripts like autom4te
fail to run, f.ex., m4. Lucky for us, openssl doesn't use those.
'''
url = MINGW_PERL_TPL.format(PERL_VERSION)
tarfile = os.path.join(self.perl_prefix, os.path.basename(url))
tarfile = os.path.abspath(tarfile)
shell.download(url, tarfile, check_cert=False)
shell.unpack(tarfile, self.perl_prefix)
# Move perl installation from perl-5.xx.y to perl
perldir = os.path.join(self.perl_prefix, 'perl-' + PERL_VERSION)
for d in os.listdir(perldir):
......@@ -123,36 +144,21 @@ class WindowsBootstrapper(BootstrapperBase):
if os.path.exists(dest):
shutil.rmtree(dest)
shutil.move(os.path.join(perldir, d), self.perl_prefix)
os.rmdir(perldir)
def install_mingwget_deps(self):
for dep in MINGWGET_DEPS:
shell.call('mingw-get install %s' % dep)
def install_gl_headers(self):
m.action("Installing wglext.h")
if self.arch == Architecture.X86:
inst_path = os.path.join(self.prefix, 'i686-w64-mingw32/include/GL/wglext.h')
else:
inst_path = os.path.join(self.prefix, 'x86_64-w64-mingw32/include/GL/wglext.h')
gl_header = 'http://www.opengl.org/registry/api/GL/wglext.h'
shell.download(gl_header, inst_path, False, check_cert=False)
def install_bin_deps(self):
# FIXME: build intltool as part of the build tools bootstrap
for url in WINDOWS_BIN_DEPS:
temp = fix_winpath(tempfile.mkdtemp())
path = os.path.join(temp, 'download.zip')
shell.download(GNOME_FTP + url, path)
shell.unpack(path, self.config.toolchain_prefix)
def fix_bin_deps(self):
# replace /opt/perl/bin/perl in intltool
files = shell.ls_files(['bin/intltool*'], self.config.toolchain_prefix)
files = shell.ls_files(['bin/intltool*'], self.prefix)
for f in files:
shell.replace(os.path.join(self.config.toolchain_prefix, f),
shell.replace(os.path.join(self.prefix, f),
{'/opt/perl/bin/perl': '/bin/perl'})
return
def fix_lib_paths(self):
orig_sysroot = self.find_mingw_sys_root()
orig_sysroot = self._find_mingw_sys_root()
if self.config.platform != Platform.WINDOWS:
new_sysroot = os.path.join(self.prefix, 'mingw', 'lib')
else:
......@@ -164,7 +170,7 @@ class WindowsBootstrapper(BootstrapperBase):
path = os.path.abspath(os.path.join(lib_path, path))
shell.replace(path, {orig_sysroot: new_sysroot})
def find_mingw_sys_root(self):
def _find_mingw_sys_root(self):
if self.config.platform != Platform.WINDOWS:
f = os.path.join(self.prefix, 'mingw', 'lib', 'libstdc++.la')
else:
......@@ -178,18 +184,19 @@ class WindowsBootstrapper(BootstrapperBase):
print("Replacing old libdir : ", libdir)
return libdir.strip()[1:-1]
def remove_mingw_unused(self):
def fix_mingw_unused(self):
msys_mingw_bindir = Path(shutil.which('mingw-get')).parent
# Fixes checks in configure, where cpp -v is called
# to get some include dirs (which doesn't looks like a good idea).
# If we only have the host-prefixed cpp, this problem is gone.
#
# MSYS's link.exe overrides MSVC's link.exe in new shells, so rename it
for tool in ('cpp', 'link'):
if (self.msys_mingw_bindir / (tool + '.exe')).is_file():
os.replace(self.msys_mingw_bindir / (tool + '.exe'),
self.msys_mingw_bindir / (tool + '.exe.bck'))
if (msys_mingw_bindir / (tool + '.exe')).is_file():
os.replace(msys_mingw_bindir / (tool + '.exe'),
msys_mingw_bindir / (tool + '.exe.bck'))
def add_non_prefixed_strings(self):
def fix_non_prefixed_strings(self):
# libtool m4 macros uses non-prefixed 'strings' command. We need to
# create a copy here
if self.config.platform == Platform.WINDOWS:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment