Commit e137093d authored by Nicolas Dufresne's avatar Nicolas Dufresne

ci: Update to use new external storage for cerbero deps

The goal of this change is to reduce egress on the FDO servers.
parent b00c68e6
Pipeline #128676 waiting for manual action with stages
in 17 seconds
...@@ -47,17 +47,14 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0' ...@@ -47,17 +47,14 @@ os.environ['GIT_TERMINAL_PROMPT'] = '0'
def git(*args, repository_path='.'): def git(*args, repository_path='.'):
return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode() return subprocess.check_output(["git"] + list(args), cwd=repository_path).decode()
def get_cerbero_last_build_info (namespace : str, branch : str): def get_cerbero_last_build_info (branch : str):
base_url = f"https://gitlab.freedesktop.org/{namespace}/cerbero/-/jobs" # Take the log from slowest build to reduce cache misses, the logs are
url = f"{base_url}/artifacts/{branch}/raw/cerbero-build/cerbero-deps.log" # uploaded as soon as they are ready.
url = "https://artifacts.gstreamer-foundation.net/cerbero-deps/{branch}/cross-ios/universal/cerbero-deps.log"
deps = [{'commit': None}] deps = [{'commit': None}]
try: try:
# The logs are only available if all jobs have passed so it does not req = urllib.request.Request(url)
# matter which distro/arch is picked.
values = { 'job': "cerbero deps fedora x86_64" }
data = urllib.parse.urlencode(values)
req = urllib.request.Request(f"{url}?{data}")
resp = urllib.request.urlopen(req); resp = urllib.request.urlopen(req);
deps = json.loads(resp.read()) deps = json.loads(resp.read())
except urllib.error.URLError: except urllib.error.URLError:
...@@ -76,7 +73,7 @@ def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str] ...@@ -76,7 +73,7 @@ def get_branch_info(module: str, namespace: str, branch: str) -> Tuple[str, str]
# Special case cerbero to avoid cache misses # Special case cerbero to avoid cache misses
if module == 'cerbero': if module == 'cerbero':
sha = get_cerbero_last_build_info(namespace, branch) sha = get_cerbero_last_build_info(branch)
if sha is not None: if sha is not None:
return sha, sha return sha, sha
......
...@@ -371,6 +371,7 @@ valgrind ges: ...@@ -371,6 +371,7 @@ valgrind ges:
CERBERO_OVERRIDDEN_DIST_DIR: "" CERBERO_OVERRIDDEN_DIST_DIR: ""
# location where cerbero is cached on the host # location where cerbero is cached on the host
CERBERO_HOST_DIR: "/cerbero/" CERBERO_HOST_DIR: "/cerbero/"
CERBERO_DEPS: "${CERBERO_HOME}/cerbero-deps.tar.xz"
before_script: before_script:
# FIXME Wrong namespace # FIXME Wrong namespace
# Workaround build-tools having hardcoded internal path # Workaround build-tools having hardcoded internal path
...@@ -381,13 +382,10 @@ valgrind ges: ...@@ -381,13 +382,10 @@ valgrind ges:
- ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero - ln -sf $(pwd) ../../${CI_PROJECT_NAMESPACE}/cerbero
- rsync -aH ${CERBERO_HOST_DIR} . - rsync -aH ${CERBERO_HOST_DIR} .
- | - |
if test -f cerbero-deps.tar.xz ; then if test -f ${CERBERO_DEPS} ; then
echo "Extracting xz deps tarball" echo "Extracting xz deps tarball"
export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible
time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf cerbero-deps.tar.xz time tar -C ${CERBERO_HOME} --use-compress-program=xzcat -xf ${CERBERO_DEPS}
elif test -f cerbero-deps.tar.gz; then
echo "Extracting gz deps tarball"
time tar -C ${CERBERO_HOME} -xf cerbero-deps.tar.gz
fi fi
- echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc - echo "home_dir = \"$(pwd)/${CERBERO_HOME}\"" >> localconf.cbc
- echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc - echo "local_sources = \"$(pwd)/${CERBERO_SOURCES}\"" >> localconf.cbc
...@@ -397,7 +395,7 @@ valgrind ges: ...@@ -397,7 +395,7 @@ valgrind ges:
- $CERBERO $CERBERO_ARGS show-config - $CERBERO $CERBERO_ARGS show-config
- $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only - $CERBERO $CERBERO_ARGS fetch-bootstrap --build-tools-only
- $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0 - $CERBERO $CERBERO_ARGS fetch-package --deps gstreamer-1.0
- $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} - test -f ${CERBERO_DEPS} || $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH}
- test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
&& test -d ${CERBERO_HOME}/dist/${ARCH} && test -d ${CERBERO_HOME}/dist/${ARCH}
&& mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR} && mkdir -p ${CERBERO_OVERRIDDEN_DIST_DIR}
...@@ -559,15 +557,12 @@ build msys2 : ...@@ -559,15 +557,12 @@ build msys2 :
gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0
gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0
gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice gst-libav-1.0 gst-validate gst-editing-services-1.0 libnice
- $CERBERO $CERBERO_ARGS fetch-cache --branch ${GST_UPSTREAM_BRANCH} --skip-fetch --job-id=${CI_JOB_ID}
- test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x" - test "x${CERBERO_OVERRIDDEN_DIST_DIR}" != "x"
&& mkdir -p ${CERBERO_HOME}/dist/${ARCH} && mkdir -p ${CERBERO_HOME}/dist/${ARCH}
&& rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH} && rsync -aH ${CERBERO_OVERRIDDEN_DIST_DIR}/ ${CERBERO_HOME}/dist/${ARCH}
- export CERBERO_DEPS="cerbero-deps.tar.xz" - $CERBERO $CERBERO_ARGS gen-cache --branch ${GST_UPSTREAM_BRANCH}
- export XZ_OPT="--threads=0" # pushing a command through sh variable and into tar correctly seems impossible - test "x${CERBERO_PRIVATE_SSH_KEY}" = "x"
- time tar -C ${CERBERO_HOME} --exclude=var/tmp --use-compress-program=xz -cf $CERBERO_DEPS || $CERBERO $CERBERO_ARGS upload-cache --branch ${GST_UPSTREAM_BRANCH}
build-tools build-tools.cache
dist/${ARCH} ${ARCH}.cache
artifacts: artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}" name: "${CI_JOB_NAME}_${CI_COMMIT_SHA}"
expire_in: '10 days' expire_in: '10 days'
...@@ -576,7 +571,7 @@ build msys2 : ...@@ -576,7 +571,7 @@ build msys2 :
- "manifest.xml" - "manifest.xml"
- "${CERBERO_HOME}/logs" - "${CERBERO_HOME}/logs"
- "${CERBERO_HOME}/cerbero-deps.log" - "${CERBERO_HOME}/cerbero-deps.log"
- "cerbero-deps.tar.*" - "${CERBERO_DEPS}"
# #
# Cerbero Linux X86_64 build # Cerbero Linux X86_64 build
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment