Commit e137093d authored by Nicolas Dufresne's avatar Nicolas Dufresne

ci: Update to use new external storage for cerbero deps

The goal of this change is to reduce egress on the FDO servers.
parent b00c68e6
Pipeline #128676 waiting for manual action with stages
in 17 seconds
......@@ -47,17 +47,14 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
def get_cerbero_last_build_info (namespace : str, branch : str):
base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs"
url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log"
def get_cerbero_last_build_info (branch : str):
# Take the log from slowest build to reduce cache misses, the logs are
# uploaded as soon as they are ready.
url = "https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log"
deps = [{'commit': None}]
try:
# The logs are only available if all jobs have passed so it does not
# matter which distro/arch is picked.
values = { 'job': "cerbero deps fedora x86_64" }
data = urllib.parse.urlencode(values)
req = urllib.request.Request(f"{url}?{data}")
req = urllib.request.Request(url)
resp = urllib.request.urlopen(req);
deps = json.loads(resp.read())
except urllib.error.URLError:
......@@ -76,7 +73,7 @@ def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]
# Special case cerbero to avoid cache misses
if module == 'cerbero':
sha = get_cerbero_last_build_info(namespace, branch)
sha = get_cerbero_last_build_info(branch)
if sha is not None:
return sha, sha
......
......@@ -371,6 +371,7 @@ valgrind ges:
CERBERO_OVERRIDDEN_DIST_DIR: ""
# location where cerbero is cached on the host
CERBERO_HOST_DIR: "/cerbero/"
CERBERO_DEPS: "${CERBERO_HOME}/cerbero-deps.tar.xz"
before_script:
# FIXME Wrong namespace
# Workaround build-tools having hardcoded internal path
......@@ -381,13 +382,10 @@ valgrind ges:
- ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero
- rsync -aH ${CERBERO_HOST_DIR} .
- |
if test -f cerbero-deps.tar.xz ; then
if test -f ${CERBERO_DEPS} ; then
echo "Extracting xz deps tarball"
export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible
time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf cerbero-deps.tar.xz
elif test -f cerbero-deps.tar.gz; then
echo "Extracting gz deps tarball"
time tar -C ${CERBERO_HOME} -xf cerbero-deps.tar.gz
time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf ${CERBERO_DEPS}
fi
- echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc
- echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
......@@ -397,7 +395,7 @@ valgrind ges:
- $CERBERO $CERBERO_ARGS show-config
- $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
- $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
- $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
- test -f ${CERBERO_DEPS} || $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
- test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
&& test -d ${CERBERO_HOME}/dist/${ARCH}
&& mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR}
......@@ -559,15 +557,12 @@ build msys2 :
gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0
gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0
gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice
- $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} --skip-fetch --job-id=${CI_JOB_ID}
- test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
&& mkdir -p ${CERBERO_HOME}/dist/${ARCH}
&& rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH}
- export CERBERO_DEPS="cerbero-deps.tar.xz"
- export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible
- time tar -C ${CERBERO_HOME} --exclude=var/tmp --use-compress-program=xz -cf $CERBERO_DEPS
build-tools build-tools.cache
dist/${ARCH} ${ARCH}.cache
- $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH}
- test "x${CERBERO_PRIVATE_SSH_KEY}" = "x"
|| $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH}
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
expire_in: '10 days'
......@@ -576,7 +571,7 @@ build msys2 :
- "manifest.xml"
- "${CERBERO_HOME}/logs"
- "${CERBERO_HOME}/cerbero-deps.log"
- "cerbero-deps.tar.*"
- "${CERBERO_DEPS}"
#
# Cerbero Linux X86_64 build
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment