Skip to content
GitLab
Commit 75b57ba6 authored by Filip Pokryvka's avatar Filip Pokryvka 🤸🏻
Browse files

run.centos-ci: run tests on 2 machines in parallel

parent e4d61153
Hide whitespace changes
Inline Side-by-side
run/centos-ci/merge_junit.py 0 → 100644
View file @ 75b57ba6
import sys
import xml.etree.ElementTree as ET
test_count = 0
testsuite = ET.Element('testsuite')
for file_name in sys.argv[1:]:
try:
with open(file_name) as f:
junit = ET.fromstring(f.read())
test_count += int(junit.attrib["tests"])
testsuite.extend(junit.getchildren())
except Exception as e:
sys.stderr.write("error while file: " + file_name + "\n" + str(e))
testsuite.attrib["tests"] = str(test_count)
print(ET.tostring(testsuite, "utf-8").decode("utf-8"))
run/centos-ci/node_runner.py
View file @ 75b57ba6
......@@ -9,6 +9,7 @@ import os
import yaml
import json
def check_build(baseurl):
copr_log = "backend.log.gz"
last = "curl -s %s \
......@@ -26,10 +27,10 @@ def check_build(baseurl):
backend_url = baseurl+"/"+last_dir+"/"+copr_log
try:
logging.debug ("Opening %s" %backend_url)
logging.debug("Opening %s" % backend_url)
back = urllib.request.urlopen(backend_url)
except:
logging.debug ("Trying the last but one as current one probably running")
logging.debug("Trying the last but one as current one probably running")
try:
out = subprocess.check_output(last_but_one, shell=True)
logging.debug(out)
......@@ -38,10 +39,10 @@ def check_build(baseurl):
logging.debug(backend_url)
back = urllib.request.urlopen(backend_url)
except:
logging.debug ("FATAL: Cannot get last but one either.")
logging.debug("FATAL: Cannot get last but one either.")
return False
with gzip.open(back,'r') as f:
with gzip.open(back, 'r') as f:
readfile = f.read().decode('utf-8')
if 'Worker failed build' in readfile:
logging.debug("Copr build failed")
......@@ -50,11 +51,12 @@ def check_build(baseurl):
logging.debug("Copr build succesfull")
return True
def add_epel_crb_repos():
# Add some extra repos
epel_url = "https://dl.fedoraproject.org/pub/epel/"
rpm = "epel-release-latest-8.noarch.rpm"
subprocess.call("dnf -y install %s%s" %(epel_url, rpm), shell=True)
subprocess.call("dnf -y install %s%s" % (epel_url, rpm), shell=True)
# For some reason names can differ, so enable both powertools
subprocess.call("yum install -y 'dnf-command(config-manager)'", shell=True)
subprocess.call("yum config-manager --set-enabled PowerTools", shell=True)
......@@ -62,6 +64,7 @@ def add_epel_crb_repos():
# Enable build deps for NM
subprocess.call("yum -y copr enable nmstate/nm-build-deps", shell=True)
def write_copr(nm_dir):
host = "https://copr-be.cloud.fedoraproject.org"
dirs = "results/networkmanager"
......@@ -75,7 +78,7 @@ def write_copr(nm_dir):
with open("/etc/yum.repos.d/nm-copr.repo", "w") as cfg:
cfg.write('[nm-copr-repo]\n')
cfg.write('name=nm-copr-repo\n')
cfg.write('baseurl=%s\n' %baseurl)
cfg.write('baseurl=%s\n' % baseurl)
cfg.write('enable=1\n')
cfg.write('gpgcheck=0\n')
cfg.write('skip_if_unavailable=0\n')
......@@ -89,13 +92,13 @@ def get_testmapper(testbranch):
#gh_url = "https://raw.githubusercontent.com/"
#gh_dirs = "NetworkManager/NetworkManager-ci/%s" %testbranch
gl_url = "https://gitlab.freedesktop.org"
gl_dirs = "NetworkManager/NetworkManager-ci/-/raw/%s" %testbranch
testmapper_url = '%s/%s/mapper.yaml' %(gl_url, gl_dirs)
gl_dirs = "NetworkManager/NetworkManager-ci/-/raw/%s" % testbranch
testmapper_url = '%s/%s/mapper.yaml' % (gl_url, gl_dirs)
try:
logging.debug ("Opening %s" %testmapper_url)
logging.debug("Opening %s" % testmapper_url)
return urllib.request.urlopen(testmapper_url)
except Exception as e:
logging.debug ("No testmapper: " + str(e))
logging.debug("No testmapper: " + str(e))
return None
......@@ -135,34 +138,36 @@ def generate_junit(results_dir):
root = ET.ElementTree()
testsuite = ET.Element('testsuite', tests=str(len(passed) + len(failed)))
for passed_test in passed:
pt_name = passed_test[passed_test.find("ci_Test")+12:]
pt_name = passed_test[passed_test.find("_Test")+12:]
testcase = ET.Element('testcase', classname="tests", name=pt_name)
system_out = ET.Element('system-out')
system_out.text = "LOG:\n%s/artifact/" %os.environ['BUILD_URL']
system_out.text = system_out.text+ passed_test + ".html"
system_out.text = "LOG:\n%s/artifact/" % os.environ['BUILD_URL']
system_out.text = system_out.text + passed_test + ".html"
testcase.append(system_out)
testsuite.append(testcase)
for failed_test in failed:
ft_name = failed_test[failed_test.find("ci_Test")+12:]
ft_name = failed_test[failed_test.find("_Test")+12:]
testcase = ET.Element('testcase', classname="tests", name=ft_name)
failure = ET.Element('failure')
failure.text = "Error\nLOG:\n%s/artifact/FAIL-%s.html" % (os.environ['BUILD_URL'], failed_test)
failure.text = "Error\nLOG:\n%s/artifact/FAIL-%s.html" % (
os.environ['BUILD_URL'], failed_test)
testcase.append(failure)
testsuite.append(testcase)
root._setroot(testsuite)
junit_path = "%s/junit.xml" %results_dir
junit_path = "%s/junit.xml" % results_dir
root.write(junit_path)
logging.debug("JUNIT Done")
return 0
def zip_journal (results_dir):
cmd = "journalctl -b --no-pager -o short-monotonic --all | bzip2 --best > %s/journal.log.bz2" %results_dir
def zip_journal(results_dir):
cmd = "journalctl -b --no-pager -o short-monotonic --all | bzip2 --best > %s/journal.log.bz2" % results_dir
subprocess.call(cmd, shell=True)
def get_features_from_mapper(branch):
mapper = get_testmapper(branch)
print (mapper)
print(mapper)
if mapper:
content = mapper.read().decode('utf-8')
content_parsed = yaml.load(content)
......@@ -177,28 +182,37 @@ def get_features_from_mapper(branch):
return (features)
return None
def process_raw_features(raw_features, testbranch, gitlab_trigger=None):
def process_raw_features(raw_features, testbranch, machine_id, gitlab_trigger=None):
tests = ""
if 'best' in raw_features:
if gitlab_trigger:
features = get_modified_features_for_testarea (gitlab_trigger)
features = get_modified_features_for_testarea(gitlab_trigger)
if features == None or features == []:
features = ["all"]
features = get_features_from_mapper(testbranch)
else:
features = ["all"]
logging.debug("running best effort execution to shorten time: %s" %features)
features = get_features_from_mapper(testbranch)
logging.debug("running best effort execution to shorten time: %s" % features)
elif not raw_features or 'all' in raw_features:
features = get_features_from_mapper(testbranch)
else:
features = [x.strip() for x in raw_features.split(',')]
if machine_id > 0:
# do +1 here, so the first machine is always nonempty
middle = int((len(features)+1)/2)
if machine_id == 1:
features = features[:middle]
else:
features = features[middle:]
for test in get_test_cases_for_features(features, testbranch):
tests=tests+test+" "
tests = tests+test+" "
return tests.strip()
def install_all_nm_packages ():
def install_all_nm_packages():
# Install all packages and remove them
add_epel_crb_repos()
......@@ -210,14 +224,17 @@ def install_all_nm_packages ():
NetworkManager-bluetooth NetworkManager-libnm-devel \
--skip-broken", shell=True)
def remove_all_nm_packages ():
def remove_all_nm_packages():
logging.debug("Removing NM rpms")
cmd1 = "rpm -ea --nodeps $(rpm -qa | grep NetworkManager)"
subprocess.call(cmd1, shell=True)
logging.debug("Done")
def prepare_log_dir (results_dir):
subprocess.call("mkdir -p %s" %results_dir, shell=True)
def prepare_log_dir(results_dir):
subprocess.call("mkdir -p %s" % results_dir, shell=True)
def prepare_box(nm_refspec, nm_mr, nm_repo=None):
if not nm_mr:
......@@ -225,8 +242,8 @@ def prepare_box(nm_refspec, nm_mr, nm_repo=None):
results_dir = "/tmp/results/"
build_dir = "/root/nm-build"
install_all_nm_packages ()
prepare_log_dir (results_dir)
install_all_nm_packages()
prepare_log_dir(results_dir)
# Prepare copr repo if we know branch
dir = None
......@@ -246,39 +263,42 @@ def prepare_box(nm_refspec, nm_mr, nm_repo=None):
if nm_refspec == "nm-1-26":
dir = "NetworkManager-CI-1.26-git"
remove_all_nm_packages ()
remove_all_nm_packages()
if dir:
logging.debug("prepare %s copr repo" %dir)
if not write_copr (dir):
logging.debug("prepare %s copr repo" % dir)
if not write_copr(dir):
return 1
# Install new rpms
cmd2 = "yum -y install --repo nm-copr-repo NetworkManager*"
return subprocess.call(cmd2, shell=True)
else:
# compile from source
logging.debug("Building from refspec id %s" %nm_refspec)
logging.debug("Building from refspec id %s" % nm_refspec)
if not nm_repo:
cmd0 = "sh run/centos-ci/scripts/./build.sh %s %s" %(nm_refspec, nm_mr)
cmd0 = "sh run/centos-ci/scripts/./build.sh %s %s" % (nm_refspec, nm_mr)
else:
logging.debug("of repo %s" %nm_repo)
cmd0 = "BUILD_REPO=%s sh run/centos-ci/scripts/./build.sh %s %s" %(nm_repo, nm_refspec, nm_mr)
logging.debug("of repo %s" % nm_repo)
cmd0 = "BUILD_REPO=%s sh run/centos-ci/scripts/./build.sh %s %s" % (
nm_repo, nm_refspec, nm_mr)
if subprocess.call(cmd0, shell=True) == 1:
# We want config.log to be archived
cmd1 = "cp %s/NetworkManager/config.log %s" %(build_dir,results_dir)
cmd1 = "cp %s/NetworkManager/config.log %s" % (build_dir, results_dir)
subprocess.call(cmd1, shell=True)
else:
return 0
return 1
def install_workarounds ():
def install_workarounds():
# pass
cmd0 = "yum -y install https://vbenes.fedorapeople.org/NM/libndp_rhbz1933041/libndp-1.7-5.el8.x86_64.rpm https://vbenes.fedorapeople.org/NM/libndp_rhbz1933041/libndp-devel-1.7-5.el8.x86_64.rpm"
subprocess.call(cmd0, shell=True)
cmd1 = "yum -y install crda make"
subprocess.call(cmd1, shell=True)
def set_gitlab (trigger_data, gl_token):
def set_gitlab(trigger_data, gl_token):
with open("/etc/python-gitlab.cfg", "w") as cfg:
cfg.write('[global]\n')
cfg.write('default = gitlab.freedesktop.org\n')
......@@ -286,7 +306,7 @@ def set_gitlab (trigger_data, gl_token):
cfg.write('timeout = 30\n')
cfg.write('[gitlab.freedesktop.org]\n')
cfg.write('url = https://gitlab.freedesktop.org\n')
cfg.write('private_token = %s\n' %gl_token)
cfg.write('private_token = %s\n' % gl_token)
cfg.write("\n")
cfg.close()
......@@ -298,6 +318,7 @@ def set_gitlab (trigger_data, gl_token):
gitlab_trigger = GitlabTrigger(data)
return gitlab_trigger
def get_modified_features_for_testarea(gl_trigger):
default_exlude = ['dcb', 'wifi', 'infiniband', 'wol', 'sriov', 'gsm']
features = []
......@@ -319,12 +340,14 @@ def get_modified_features_for_testarea(gl_trigger):
features.append(m.group(1))
return features
def get_nm_mrid (gl_trigger):
def get_nm_mrid(gl_trigger):
if gl_trigger.repository == 'NetworkManager':
return "mr%s" %(gl_trigger.merge_request_id)
return "mr%s" % (gl_trigger.merge_request_id)
return None
def post_results (gl_trigger):
def post_results(gl_trigger, machine_id=0):
msg = "CentOS Testing Summary\n\n"
if os.path.exists('/tmp/summary.txt'):
with open('/tmp/summary.txt', 'r') as f:
......@@ -334,22 +357,29 @@ def post_results (gl_trigger):
f = lines[1].strip()
s = lines[2].strip()
if p == '0' and f == '0' and s == '0':
msg+= "Something went wrong!\n"
msg += "Something went wrong!\n"
if f != '0':
msg+= "Result: Unstable: Some tests failed!\n"
msg += "Result: Unstable: Some tests failed!\n"
else:
msg+= "Result: STABLE: All tests passing!\n"
msg+="\nPassed: %s, Failed: %s, Skipped: %s\n\n" %(p, f, s)
if gl_trigger.repository == "NetworkManager-ci":
gl_trigger.play_commit_job()
msg += "Result: STABLE: All tests passing!\n"
msg += "\nPassed: %s, Failed: %s, Skipped: %s\n\n" % (p, f, s)
# TODO call later
# if gl_trigger.repository == "NetworkManager-ci":
# try:
# gl_trigger.play_commit_job()
# except:
# pass
else:
msg+= "Aborted!\n"
msg += "Aborted!\n"
with open('/etc/redhat-release') as f:
msg+="Executed on: %s" %(f.read())
msg+="\n\n%s" %os.environ['BUILD_URL']
msg += "Executed on: %s" % (f.read())
if machine_id:
msg += "\nMachine: %d/2" % machine_id
msg += "\n\n%s" % os.environ['BUILD_URL']
gl_trigger.post_commit_comment(msg)
def main ():
def main():
logging.basicConfig(level=logging.DEBUG)
logging.debug("reading params")
parser = argparse.ArgumentParser()
......@@ -360,6 +390,7 @@ def main ():
parser.add_argument('-g', '--gitlab_token')
parser.add_argument('-d', '--trigger_data')
parser.add_argument('-r', '--nm_repo')
parser.add_argument('-m', '--machine')
parser.add_argument('-D', '--do_not_touch_NM', action="store_true", default=False)
args = parser.parse_args()
......@@ -391,6 +422,13 @@ def main ():
nm_repo = args.nm_repo
else:
nm_repo = "https://gitlab.freedesktop.org/NetworkManager/NetworkManager/"
if args.machine:
try:
machine_id = int(args.machine)
except:
machine_id = 0
else:
machine_id = 0
gitlab_trigger = None
nm_mr = None
......@@ -399,24 +437,24 @@ def main ():
gitlab_trigger = set_gitlab(trigger_data, gl_token)
if gitlab_trigger:
tests = process_raw_features (raw_features, test_branch, gitlab_trigger)
nm_mr = get_nm_mrid (gitlab_trigger)
tests = process_raw_features(raw_features, test_branch, machine_id, gitlab_trigger)
nm_mr = get_nm_mrid(gitlab_trigger)
else:
tests = process_raw_features (raw_features, test_branch)
tests = process_raw_features(raw_features, test_branch, machine_id)
if tests == "":
tests = "pass"
logging.debug("no tests to run, running just pass test")
else:
logging.debug("tests to run: %s" %tests)
logging.debug("tests to run: %s" % tests)
install_workarounds ()
install_workarounds()
if nm_refspec:
if prepare_box (nm_refspec, nm_mr, nm_repo) != 0:
if prepare_box(nm_refspec, nm_mr, nm_repo) != 0:
logging.debug("Compilation failed")
exit_code = 2
cmd = "sh run/centos-ci/scripts/./runtest.sh %s" %tests
cmd = "MACHINE_ID=%d sh run/centos-ci/scripts/./runtest.sh %s" % (machine_id, tests)
if gitlab_trigger:
gitlab_trigger.set_pipeline('running')
runtest = subprocess.Popen(cmd, shell=True)
......@@ -429,13 +467,13 @@ def main ():
gitlab_trigger.set_pipeline('failed')
if exit_code == 2:
gitlab_trigger.set_pipeline('canceled')
post_results (gitlab_trigger)
post_results(gitlab_trigger, machine_id)
generate_junit ("/tmp/results")
zip_journal ("/tmp/results")
generate_junit("/tmp/results")
zip_journal("/tmp/results")
logging.debug("All Done. Exit with %s" %exit_code)
sys.exit (exit_code)
logging.debug("All Done. Exit with %s" % exit_code)
sys.exit(exit_code)
if __name__ == "__main__":
......
run/centos-ci/runner.groovy
View file @ 75b57ba6
......@@ -30,8 +30,13 @@ node('cico-workspace') {
}
stage('get cico node') {
node = sh(script: "cico --debug node get -f value -c hostname -c comment --release ${RELEASE}", returnStdout: true).trim().tokenize(' ')
env.node_hostname = "${node[0]}.ci.centos.org"
env.node_ssid = "${node[1]}"
env.node1_hostname = "${node[0]}.ci.centos.org"
env.node1_ssid = "${node[1]}"
}
stage('get cico node') {
node = sh(script: "cico --debug node get -f value -c hostname -c comment --release ${RELEASE}", returnStdout: true).trim().tokenize(' ')
env.node2_hostname = "${node[0]}.ci.centos.org"
env.node2_ssid = "${node[1]}"
}
stage('run tests') {
......@@ -50,49 +55,80 @@ node('cico-workspace') {
if (TRIGGER_DATA) {
run += " -d ${TD}"
}
println("Running install")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname} '${install}'"
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname} '${install2}'"
println("Running clone")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname} '${clone}'"
println("Running install on machine 1")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${install}'"
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${install2}'"
println("Running install on machine 2")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname} '${install}'"
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname} '${install2}'"
println("Running clone on machine 1")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${clone}'"
println("Running clone on machine 2")
sh "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname} '${clone}'"
println("Running tests")
sh """
set +x
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname} '${run}'
echo "Running tests on 2 machines, progress is visible in Workspaces"
{ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${run} -m 1' ; } &> m1.stdout &
{ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname} '${run} -m 2' ; } &> m2.stdout &
wait
cat m1.stdout
cat m2.stdout
"""
}
}
finally {
try {
stage('publish results') {
sh "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname}:/tmp/results/* ."
// Check if we have RESULT so whole pipeline was not canceled
sh "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname}:/tmp/results/* ."
sh 'sleep 10'
sh "mv RESULT.txt RESULT.m1.txt || true"
sh "mv journal.log.bz2 journal.log.m1.bz2 || true"
sh "mv junit.xml junit.m1.xml || true"
sh "scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname}:/tmp/results/* ."
sh 'sleep 10'
if (!fileExists('RESULT.txt')) {
sh "mv RESULT.txt RESULT.m2.txt || true"
sh "mv journal.log.bz2 journal.log.m2.bz2 || true"
sh "mv junit.xml junit.m2.xml || true"
// Check if we have RESULT so whole pipeline was not canceled
if (!fileExists('RESULT.m1.txt') || !fileExists('RESULT.m2.txt')) {
// Compilation failed there is config.log
if (!fileExists('config.log')) {
println("Pipeline canceled! We do have no RESULT.txt or config.log")
cancel = "cd NetworkManager-ci; python3 run/centos-ci/pipeline_cancel.py ${env.BUILD_URL} ${GL_TOKEN} ${TD}"
sh """
set +x
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node_hostname} '${cancel}'
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${cancel}'
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node2_hostname} '${cancel}'
"""
}
}
println("Merge junit.xml")
merge_junit = "cd NetworkManager-ci; python3 run/centos-ci/merge_junit.py /tmp/results/junit.m1.xml /tmp/results/junit.m2.xml"
sh """
set +x
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no junit.m2.xml root@${node1_hostname}:/tmp/results/junit.m2.xml
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} 'mv /tmp/results/junit.xml /tmp/results/junit.m1.xml'
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname} '${merge_junit} > /tmp/results/junit.xml'
scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${node1_hostname}:/tmp/results/junit.xml junit.xml
sleep 10
"""
archiveArtifacts '*.*'
junit 'junit.xml'
}
stage('reserve') {
if (RESERVE != "0s") {
println("You can log in via:")
println("ssh root@${node_hostname}")
println("ssh root@${node1_hostname}")
println("ssh root@${node2_hostname}")
}
sh 'sleep ${RESERVE}'
}
}
finally {
stage('return cico node') {
sh 'cico node done ${node_ssid} > commandResult'
sh 'cico node done ${node1_ssid} > commandResult'
sh 'cico node done ${node2_ssid} > commandResult'
}
}
}
......
run/centos-ci/scripts/runtest.sh
View file @ 75b57ba6
......@@ -34,7 +34,11 @@ for test in $@; do
fi
# Start test itself with timeout
export TEST="NetworkManager-ci_Test$counter"_"$test"
if [ -z "$MACHINE_ID" -o "$MACHINE_ID" = "0" ]; then
export TEST="NetworkManager-ci_Test${counter}_$test"
else
export TEST="NetworkManager-ci-M${MACHINE_ID}_Test${counter}_$test"
fi
cmd=$(sed -n "/- $test:/,/ - /p" mapper.yaml | grep -e "run:" | awk -F: '{print $2}')
if [ "$cmd" == "" ] ; then
cmd="run/./runtest.sh $test"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment