mirror of https://gitee.com/anolis/sysom.git
597 lines
28 KiB
Python
597 lines
28 KiB
Python
# -*- coding: utf-8 -*- #
|
|
"""
|
|
Time 2023/1/13 14:14
|
|
Author: zhangque (ydjohn)
|
|
Email ydzhang@linux.alibaba.com
|
|
File builder.py
|
|
Description: This is the main program of hotfix builder
|
|
"""
|
|
import os
|
|
from os import listdir
|
|
from clogger import logger
|
|
import threading
|
|
import requests
|
|
import json
|
|
import platform
|
|
import shutil
|
|
import re
|
|
import sys
|
|
import subprocess
|
|
import configparser
|
|
import redis
|
|
import time
|
|
from cec_base.consumer import Consumer, dispatch_consumer
|
|
from cec_base.admin import dispatch_admin
|
|
from cec_base.producer import dispatch_producer, Producer
|
|
from conf.settings import YAML_CONFIG
|
|
from sysom_utils import CecTarget, SysomFramework
|
|
|
|
class ServerConnector():
|
|
|
|
def __init__(self, hotfix_msg_topic=None, hotfix_job_received_topic=None):
|
|
self.cec = dispatch_producer(YAML_CONFIG.get_cec_url(CecTarget.PRODUCER))
|
|
#self.cec = SysomFramework.cec_consumer(cec_topic)
|
|
self.cec_msg_topic = hotfix_msg_topic
|
|
self.cec_hotfix_job_received_topic = hotfix_job_received_topic
|
|
|
|
# Here, we design to use some status more than building status
|
|
# in building status , we include : waiting\building\success\failed
|
|
# but more, we can send : sync to server for log and hotfix name sync
|
|
def update_hotfix_building_status(self, id, status):
|
|
self.cec.produce(self.cec_msg_topic, {
|
|
"hotfix_id" : id,
|
|
"status" : status
|
|
})
|
|
|
|
|
|
def sync_rpm_name_cec(self, hotfix_id, rpm_name):
|
|
logger.info("produce rpm_name ")
|
|
self.cec.produce(self.cec_msg_topic, {
|
|
"hotfix_id" : hotfix_id,
|
|
"type": "rpm",
|
|
"sync" : True,
|
|
"rpm_name" : rpm_name
|
|
})
|
|
|
|
def sync_building_log_cec(self, hotfix_id):
|
|
logger.info("sync hotfix build log")
|
|
self.cec.produce(self.cec_msg_topic, {
|
|
"hotfix_id": hotfix_id,
|
|
"type": "log",
|
|
"sync": True
|
|
})
|
|
|
|
|
|
class HotfixBuilder():
|
|
|
|
def __init__(self, con: dict):
|
|
cache = SysomFramework.gcache("deleted_hotfix")
|
|
self.cache = cache
|
|
#######################
|
|
# Server config
|
|
#######################
|
|
self.nfs_dir_home = con.builder.nfs_dir_home
|
|
self.max_retry_time = con.cec.max_retry
|
|
self.sleep_time = con.cec.retry_sleep_time
|
|
self.hotfix_base = con.builder.hotfix_base
|
|
self.builder_hotfix_package_repo = con.builder.package_repo
|
|
self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True)
|
|
self.cec_hotfix_topic = con.cec.hotfix_topic # send
|
|
self.cec_hotfix_job_topic = con.cec.job_topic # listen
|
|
self.local_arch = os.uname().release.split(".")[-1]
|
|
self.connector = ServerConnector(hotfix_msg_topic=con.cec.hotfix_topic)
|
|
self.tmpdir=con.builder.tmpdir
|
|
self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir
|
|
self.hotfix_id = None
|
|
self.fd = None
|
|
self.prepare_env()
|
|
|
|
##################################################################
|
|
# Logging config
|
|
##################################################################
|
|
from clogger import logger
|
|
log_format = "%(asctime)s | %(levelname)s | %(message)s"
|
|
logger.set_format(log_format)
|
|
logger.set_level("INFO")
|
|
|
|
def run(self):
|
|
self.thread_runner.start()
|
|
|
|
def change_building_status(self, status):
|
|
self.connector.update_hotfix_building_status(self.hotfix_id, status)
|
|
|
|
def die(self, msg):
|
|
self.fd.write("%s \n" % msg)
|
|
self.change_building_status("failed")
|
|
|
|
def prepare_env(self):
|
|
|
|
# get the img_list image information and pull them based on machine's kernel arch
|
|
image_config_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json"))
|
|
config_data = json.load(image_config_file)
|
|
machine_kernel = platform.uname().release
|
|
arch = machine_kernel.split(".")[-1]
|
|
for each_version in config_data[arch]:
|
|
image = config_data[arch][each_version]
|
|
os.system("docker pull {}".format(image))
|
|
|
|
if not os.path.exists(self.hotfix_base):
|
|
os.makedirs(self.hotfix_base)
|
|
|
|
if not os.path.exists(self.tmpdir):
|
|
os.makedirs(self.tmpdir)
|
|
|
|
# checkout the log directory
|
|
if not os.path.exists(os.path.join(self.nfs_dir_home, "log")):
|
|
os.makedirs(os.path.join(self.nfs_dir_home, "log"))
|
|
|
|
# checkout the rpm directory
|
|
if not os.path.exists(os.path.join(self.nfs_dir_home, "rpm")):
|
|
os.makedirs(os.path.join(self.nfs_dir_home, "rpm"))
|
|
|
|
# directory to storage the devel-package\debuginfo\config\vmlinux
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "devel_pack")):
|
|
os.makedirs(os.path.join(self.builder_hotfix_package_repo, "devel_pack"))
|
|
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "debuginfo_pack")):
|
|
os.makedirs(os.path.join(self.builder_hotfix_package_repo, "debuginfo_pack"))
|
|
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "kernel_config")):
|
|
os.makedirs(os.path.join(self.builder_hotfix_package_repo, "kernel_config"))
|
|
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "vmlinux")):
|
|
os.makedirs(os.path.join(self.builder_hotfix_package_repo, "vmlinux"))
|
|
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "src_pack")):
|
|
os.makedirs(os.path.join(self.builder_hotfix_package_repo, "src_pack"))
|
|
|
|
# copy build_hotfix.sh to BASE
|
|
if os.path.exists(os.path.join(os.getcwd(), "script", "build_hotfix.sh")):
|
|
shutil.copy(os.path.join(os.getcwd(), "script", "build_hotfix.sh"), self.hotfix_base)
|
|
else:
|
|
logger.error("ERROR: cannot find build_hotfix.sh")
|
|
|
|
if os.path.exists(os.path.join(os.getcwd(), "script", "build_rpm.sh")):
|
|
shutil.copy(os.path.join(os.getcwd(), "script", "build_rpm.sh"), self.hotfix_base)
|
|
else:
|
|
logger.error("ERROR: cannot find build_rpm.sh")
|
|
|
|
def clear_tmpdir(self):
|
|
os.system("rm -rf {} && mkdir {} ".format(self.tmpdir, self.tmpdir))
|
|
os.system("rm -rf {}/*patch".format(self.hotfix_base))
|
|
|
|
def find_build_rpm(self, kernel_version):
|
|
arch = kernel_version.split(".")[-1]
|
|
directory = os.path.join(self.tmpdir, "rpmbuild", "RPMS", arch)
|
|
rpms = []
|
|
for root, dirs, files in os.walk(directory):
|
|
for eachfile in files:
|
|
if re.search(".rpm", eachfile):
|
|
rpms.append(eachfile)
|
|
return rpms
|
|
|
|
def check_config(self, kernel_version):
|
|
kernel_config_directory = os.path.join(self.builder_hotfix_package_repo, "kernel_config")
|
|
if not os.path.exists(kernel_config_directory):
|
|
os.makedirs(kernel_config_directory)
|
|
return None
|
|
config_name = "config-" + kernel_version
|
|
if os.path.exists(os.path.join(kernel_config_directory, config_name)):
|
|
return os.path.join(kernel_config_directory, config_name)
|
|
else:
|
|
return None
|
|
|
|
def check_vmlinux(self, kernel_version):
|
|
vmlinx_directory = os.path.join(self.builder_hotfix_package_repo, "vmlinux")
|
|
if not os.path.exists(vmlinx_directory):
|
|
os.makedirs(vmlinx_directory)
|
|
return None
|
|
vmlinux_name = "vmlinux-" + kernel_version
|
|
if os.path.exists(os.path.join(vmlinx_directory, vmlinux_name)):
|
|
return os.path.join(vmlinx_directory, vmlinux_name)
|
|
else:
|
|
return None
|
|
|
|
def check_devel_package(self, devel_link, kernel_version):
|
|
devel_package_directory = os.path.join(self.builder_hotfix_package_repo, "devel_pack")
|
|
kernel_config_directory = os.path.join(self.builder_hotfix_package_repo, "kernel_config")
|
|
|
|
if not os.path.exists(devel_package_directory):
|
|
os.makedirs(devel_package_directory)
|
|
|
|
devel_package = devel_link.split("/")[-1]
|
|
if os.path.exists(os.path.join(devel_package_directory, devel_package)):
|
|
# release devel package and storage config file
|
|
os.system("cp {} /tmp/a.rpm".format(os.path.join(devel_package_directory, devel_package)))
|
|
os.system("cd /tmp && rpm2cpio a.rpm | cpio -dim".format(kernel_version))
|
|
os.system("cd /tmp && cp ./usr/src/kernels/{}/.config {}/config-{} && rm -rf ./usr && rm -rf a.rpm".format(kernel_version, kernel_config_directory, kernel_version))
|
|
else:
|
|
# download devel package,copy devel-package and config file to its own directory
|
|
self.fd.write("Downloading devel package from {}...\n".format(devel_link))
|
|
os.system("wget -P {} {}".format(devel_package_directory, devel_link))
|
|
os.system("cp {} /tmp/a.rpm".format(os.path.join(devel_package_directory, devel_package)))
|
|
os.system("cd /tmp && rpm2cpio a.rpm | cpio -dim".format(kernel_version))
|
|
os.system("cd /tmp && cp ./usr/src/kernels/{}/.config {}/config-{} && rm -rf ./usr && rm -rf a.rpm".format(kernel_version, kernel_config_directory, kernel_version))
|
|
config_name = "config-" + kernel_version
|
|
config_name = os.path.join(kernel_config_directory, config_name)
|
|
return config_name
|
|
|
|
def check_debuginfo_package(self, debuginfo_link, kernel_version):
|
|
debuginfo_package_directory = os.path.join(self.builder_hotfix_package_repo, "debuginfo_pack")
|
|
vmlinux_directory = os.path.join(self.builder_hotfix_package_repo, "vmlinux")
|
|
|
|
if not os.path.exists(debuginfo_package_directory):
|
|
os.makedirs(debuginfo_package_directory)
|
|
|
|
debuginfo_package = debuginfo_link.split("/")[-1]
|
|
if os.path.exists(os.path.join(debuginfo_package_directory, debuginfo_package)):
|
|
os.system("cp {} /tmp/b.rpm".format(os.path.join(debuginfo_package_directory, debuginfo_package)))
|
|
os.system("cd /tmp && rpm2cpio b.rpm | cpio -dim".format(kernel_version))
|
|
os.system("cd /tmp && cp ./usr/lib/debug/lib/modules/{}/vmlinux {}/vmlinux-{} && rm -rf ./usr && rm -rf b.rpm".format(kernel_version, vmlinux_directory, kernel_version))
|
|
else:
|
|
# download debuginfo package,copy debuginfo-package and vmlinx to its own directory
|
|
self.fd.write("Downloading debuginfo package from {}...\n".format(debuginfo_link))
|
|
os.system("wget -P {} {}".format(debuginfo_package_directory, debuginfo_link))
|
|
os.system("cp {} /tmp/b.rpm".format(os.path.join(debuginfo_package_directory, debuginfo_package)))
|
|
os.system("cd /tmp && rpm2cpio /tmp/b.rpm | cpio -dim >> /dev/null".format(os.path.join(debuginfo_package_directory, debuginfo_package), kernel_version))
|
|
os.system("cd /tmp && cp ./usr/lib/debug/lib/modules/{}/vmlinux {}/vmlinux-{} && rm -rf ./usr && rm -rf b.rpm".format(kernel_version, vmlinux_directory, kernel_version))
|
|
vmlinux_name = "vmlinux-" + kernel_version
|
|
vmlinux_name = os.path.join(vmlinux_directory, vmlinux_name)
|
|
return vmlinux_name
|
|
|
|
def check_kernel_source(self, src_repo, source_code_repo, kernel_version, is_src_package):
|
|
if not is_src_package:
|
|
if not os.path.exists(os.path.join(self.hotfix_base, "kernel_repos", source_code_repo)):
|
|
os.system("cd {} && git clone {}".format(os.path.join(self.hotfix_base, "kernel_repos"), src_repo)) # if the kernel source is not exist, clone the repo
|
|
else:
|
|
os.system("cd {} && git fetch && git pull".format(os.path.join(self.hotfix_base, "kernel_repos", source_code_repo))) # source_code_repo: cloud-kernel
|
|
else:
|
|
# use src package , source_code_repo is download link, download src.rpm
|
|
kernel_no_arch = kernel_version.rsplit(".", 1)[0]
|
|
src_name = "%s.src.rpm" % kernel_no_arch
|
|
logger.info("Using src.rpm of : %s " % src_name)
|
|
if not os.path.exists(os.path.join(self.builder_hotfix_package_repo, "src_pack", src_name)):
|
|
os.system("cd {} && wget {} -O {}".format(os.path.join(self.builder_hotfix_package_repo, "src_pack"), source_code_repo, src_name))
|
|
|
|
def get_building_image(self, kernel_version):
|
|
arch = kernel_version.split(".")[-1]
|
|
image_list_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json"))
|
|
images = json.load(image_list_file)
|
|
return images[arch]['anolis']
|
|
|
|
def extract_description_from_patch(self, local_patch):
|
|
description = None
|
|
with open(local_patch, "r") as patch:
|
|
for line in patch:
|
|
if re.search("Subject", line):
|
|
description=line.split("Subject: [PATCH] ")[1].replace("\n","")
|
|
break
|
|
if description:
|
|
return description
|
|
return "None"
|
|
|
|
"""
|
|
when using .src.rpm, input kernel_version, before this function, the src.rpm should be downloaded
|
|
and stored in the src_pack repo
|
|
this function is to convert rpm to source rpm into direcotry link git repo
|
|
"""
|
|
def get_source_code_from_rpm(self, kernel_version):
|
|
kernel_no_arch = kernel_version.rsplit(".", 1)[0]
|
|
src_rpm_name = kernel_no_arch + ".src.rpm"
|
|
src_pack_dir = os.path.join(self.builder_hotfix_package_repo, "src_pack")
|
|
if not os.path.exists(os.path.join(src_pack_dir, src_rpm_name)):
|
|
return None
|
|
os.system("rm -rf {} && mkdir {} ".format(self.src_rpm_tmpdir, self.src_rpm_tmpdir, self.src_rpm_tmpdir))
|
|
shutil.copy(os.path.join(src_pack_dir, src_rpm_name), self.src_rpm_tmpdir)
|
|
os.system("pushd {} && rpm2cpio {} | cpio -dmi >> /dev/null".format(self.src_rpm_tmpdir, src_rpm_name))
|
|
# walk the directory, find out the linux-{}.tar.xz file, release it, and the source code name is linux-{}
|
|
src = None
|
|
|
|
# find the .pem file if exist
|
|
for each_file in listdir(self.src_rpm_tmpdir):
|
|
if re.search(".pem", each_file):
|
|
pem_file = os.path.join(self.src_rpm_tmpdir, each_file)
|
|
break
|
|
|
|
for each_file in listdir(self.src_rpm_tmpdir):
|
|
if re.search(".tar.xz", each_file):
|
|
src = each_file
|
|
break
|
|
if src is None:
|
|
return None
|
|
|
|
os.system("pushd {} && tar -xvf {} >> /dev/null".format(self.src_rpm_tmpdir, src))
|
|
src = src.replace(".tar.xz", "")
|
|
|
|
# remove the source directory to hotfix tmpdir
|
|
os.system("cp -a {} {}".format(os.path.join(self.src_rpm_tmpdir, src), self.tmpdir))
|
|
|
|
# copy the .pem file to certs directory of kernel
|
|
if pem_file is not None:
|
|
cert_directory = os.path.join(self.tmpdir, src, "certs")
|
|
shutil.copy(pem_file, cert_directory)
|
|
|
|
return os.path.join(self.tmpdir, src)
|
|
|
|
"""
|
|
build the supported kernel like : anolis
|
|
"""
|
|
def build_supported_kernel(self, parameters):
|
|
# get the hotfix building parametes
|
|
hotfix_id = parameters['hotfix_id']
|
|
kernel_version = parameters['kernel_version']
|
|
hotfix_name = parameters['hotfix_name']
|
|
# find the patch_path in builder local
|
|
patch_path = parameters['patch_path'].split("/")[-1]
|
|
patch_path = os.path.join(self.nfs_dir_home, "patch", patch_path)
|
|
log_file = parameters['log_file']
|
|
git_repo = parameters['git_repo']
|
|
is_src_package = False
|
|
source_code_repo = git_repo.split("/")[-1].rstrip(".git") # findout the kernel repo name
|
|
log = ""
|
|
output = ""
|
|
log_file_path = os.path.join(self.nfs_dir_home, "log", log_file)
|
|
self.fd = open(log_file_path, "a")
|
|
|
|
self.fd.write("=========================================================\n")
|
|
self.fd.write("Created Hotfix Building Task ... \n")
|
|
self.fd.write("Kernel Version: %s\n" % kernel_version)
|
|
self.fd.write("Patch file: %s\n" % patch_path)
|
|
self.fd.write("Hotfix name : %s\n" % hotfix_name)
|
|
self.fd.write("=========================================================\n")
|
|
|
|
self.check_kernel_source(git_repo, source_code_repo, kernel_version, is_src_package)
|
|
|
|
image = self.get_building_image(kernel_version)
|
|
if image is None:
|
|
raise Exception("No specify building image ... ")
|
|
return None
|
|
else:
|
|
self.fd.write("Using Building Image : %s \n" % image)
|
|
|
|
# move the patch to base
|
|
try:
|
|
local_patch = os.path.join(self.hotfix_base, parameters['patch_path'].split("/")[-1])
|
|
logger.info("the local patch is : %s " % local_patch)
|
|
shutil.copy(patch_path, local_patch)
|
|
except Exception as e:
|
|
self.fd.write(str(e))
|
|
logger.error(str(e))
|
|
|
|
description = self.extract_description_from_patch(local_patch)
|
|
# run the build hotfix script
|
|
cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -r {} -t NULL ".format(
|
|
self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image,
|
|
self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, source_code_repo
|
|
)
|
|
self.fd.write(cmd+"\n")
|
|
self.fd.close()
|
|
logger.info(cmd)
|
|
|
|
cmd += " 2>&1 >> %s" % log_file_path
|
|
|
|
p=subprocess.Popen(cmd, shell=True)
|
|
return_code=p.wait()
|
|
logger.info("The return code is %d" % return_code)
|
|
|
|
rpm_names = self.find_build_rpm(kernel_version)
|
|
logger.info(rpm_names)
|
|
# if rpm is more than one, upload it one by one
|
|
for each_rpm in rpm_names:
|
|
logger.info("==> sync rpm : %s" % each_rpm)
|
|
self.connector.sync_rpm_name_cec(hotfix_id, each_rpm)
|
|
|
|
self.connector.sync_building_log_cec(self.hotfix_id)
|
|
|
|
# check the last output
|
|
if return_code == 0:
|
|
self.change_building_status("success")
|
|
else:
|
|
os.system("echo \"BUILD FAILED\" >> %s" % log_file_path)
|
|
self.change_building_status("failed")
|
|
|
|
"""
|
|
build the customize kernel from user defined
|
|
"""
|
|
def build_customize_kernel(self, parameters):
|
|
# get the hotfix building parameters
|
|
hotfix_id = parameters['hotfix_id']
|
|
kernel_version = parameters['kernel_version']
|
|
hotfix_name = parameters['hotfix_name']
|
|
devel_link = parameters['devel_link']
|
|
debuginfo_link = parameters['debuginfo_link']
|
|
src_repo = parameters['src_repo'] # if git, git_repo; if src, src download repo
|
|
src_origin = parameters['src_origin']# if git, git tag/branch; if src, src download link
|
|
# find the patch_path in builder local
|
|
patch_path = parameters['patch_path'].split("/")[-1]
|
|
patch_path = os.path.join(self.nfs_dir_home, "patch", patch_path)
|
|
log_file = parameters['log_file']
|
|
image = parameters['image']
|
|
is_src_package = parameters['is_src_package']
|
|
|
|
log = ""
|
|
output = ""
|
|
log_file_path = os.path.join(self.nfs_dir_home, "log", log_file)
|
|
self.fd = open(log_file_path, "a")
|
|
|
|
self.fd.write("=========================================================\n")
|
|
self.fd.write("Created Hotfix Building Task ... \n")
|
|
self.fd.write("Kernel Version: %s\n" % kernel_version)
|
|
self.fd.write("Patch file: %s\n" % patch_path)
|
|
self.fd.write("Hotfix name : %s\n" % hotfix_name)
|
|
self.fd.write("Using Building Image : %s \n" % image)
|
|
self.fd.write("=========================================================\n")
|
|
|
|
if not is_src_package:
|
|
self.fd.write("Using the source repo...\n")
|
|
source_code_repo = src_repo.split("/")[-1].rstrip(".git") # findout the kernel repo name: cloud-kernel
|
|
else:
|
|
self.fd.write("Using the src.rpm...\n")
|
|
source_code_repo = src_origin # if use .src.rpm, source code is in src_pack directory, not a directory
|
|
src_rpm_name = src_origin.split("/")[-1] # Here, src_origin is the link to src.rpm
|
|
|
|
if len(image) == 0:
|
|
image = self.get_building_image(kernel_version)
|
|
if image is None:
|
|
raise Exception("No specify building image ... ")
|
|
return None
|
|
else:
|
|
self.fd.write("Using Building Image : %s \n" % image)
|
|
|
|
self.fd.write("Checking the source code...\n")
|
|
self.check_kernel_source(src_repo, source_code_repo, kernel_version, is_src_package)
|
|
|
|
if is_src_package:
|
|
source_code_path = self.get_source_code_from_rpm(kernel_version)
|
|
if source_code_path is None:
|
|
logger.error("Get Source Code from Rpm Failed...")
|
|
self.die("Get Source Code from Rpm Failed...\n")
|
|
raise Exception('Get Source Code from .src.rpm failed...')
|
|
return None
|
|
|
|
self.fd.write("Checking the devel package for config file...\n")
|
|
kernel_config = self.check_config(kernel_version)
|
|
if kernel_config is None:
|
|
self.fd.write("kernel_config is not found...Now check the buffer of kernel-devel...\n")
|
|
kernel_config = self.check_devel_package(devel_link, kernel_version)
|
|
if not os.path.exists(kernel_config):
|
|
self.die("Get the kernel config file failed...\n")
|
|
return None
|
|
else:
|
|
self.fd.write("Succeed in getting the kernel config from kernel-devel package...\n")
|
|
|
|
self.fd.write("Checking the vmlinux...\n")
|
|
vmlinux = self.check_vmlinux(kernel_version)
|
|
if vmlinux is None:
|
|
self.fd.write("vmlinux not found...Now checking the buffer of kernel-debuginfo...\n")
|
|
vmlinux = self.check_debuginfo_package(debuginfo_link, kernel_version)
|
|
if not os.path.exists(vmlinux):
|
|
self.die("Get the kernel vmlinux failed...\n")
|
|
return None
|
|
else:
|
|
self.fd.write("Succeed in getting vmlinux from debuginfo...\n")
|
|
|
|
|
|
# move the patch to base
|
|
try:
|
|
local_patch = os.path.join(self.hotfix_base, parameters['patch_path'].split("/")[-1])
|
|
shutil.copy(patch_path, local_patch)
|
|
except Exception as e:
|
|
self.die("Error when copying the source patch, may be this patch is not exist?")
|
|
return None
|
|
|
|
description = self.extract_description_from_patch(local_patch)
|
|
|
|
# run the build hotfix script,
|
|
if is_src_package:
|
|
# use src.rpm
|
|
cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} 2>&1 1 >> {} ".format(
|
|
self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image,
|
|
self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_path, log_file_path
|
|
)
|
|
else:
|
|
# use git branch for source management
|
|
git_branch = src_origin
|
|
cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} -t {} 2>&1 1 >> {}".format(
|
|
self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image,
|
|
self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_repo, git_branch, log_file_path
|
|
)
|
|
|
|
self.fd.write(cmd + "\n")
|
|
self.fd.close()
|
|
logger.info(cmd)
|
|
|
|
p=subprocess.Popen(cmd, shell=True)
|
|
return_code=p.wait()
|
|
logger.info("The return code is %d" % return_code)
|
|
|
|
rpm_names = self.find_build_rpm(kernel_version)
|
|
|
|
# if rpm is more than one, upload it one by one
|
|
for each_rpm in rpm_names:
|
|
self.connector.sync_rpm_name_cec(hotfix_id, each_rpm)
|
|
|
|
self.connector.sync_building_log_cec(self.hotfix_id)
|
|
# check the last output
|
|
if return_code == 0:
|
|
self.change_building_status("success")
|
|
else:
|
|
os.system("echo \"BUILD FAILED\" >> %s" % log_file_path)
|
|
self.change_building_status("failed")
|
|
|
|
'''
|
|
Each event is an object, the parameter is inside event.value
|
|
event.value is a dictionary.
|
|
|
|
For supported kernel, we use gitee for source code management.
|
|
However, for customize kernel, user may have different ways for their
|
|
source code management. Therefore, we support git/.src.rpm for kernel source
|
|
|
|
'''
|
|
def build(self):
|
|
consumer_id = Consumer.generate_consumer_id()
|
|
|
|
if self.local_arch == "x86_64":
|
|
consumer = SysomFramework.cec_consumer(self.cec_hotfix_job_topic,
|
|
consumer_id=consumer_id,
|
|
group_id="hotfix_job_group_x86")
|
|
else:
|
|
consumer = SysomFramework.cec_consumer(self.cec_hotfix_job_topic,
|
|
consumer_id=consumer_id,
|
|
group_id="hotfix_job_group_arm")
|
|
retry_time = 0
|
|
while retry_time < self.max_retry_time:
|
|
for event in consumer:
|
|
try:
|
|
parameters = event.value
|
|
log_file = parameters['log_file']
|
|
self.hotfix_id = parameters['hotfix_id']
|
|
log_file_path = os.path.join(self.nfs_dir_home, "log", log_file)
|
|
is_deleted = self.cache.load(str(self.hotfix_id))
|
|
# if this hotfix_id is not exist in the deleted pool
|
|
if not is_deleted:
|
|
if parameters['arch'] == self.local_arch:
|
|
# this operation aims to clear the previous log if rebuild
|
|
with open(log_file_path, "w") as f:
|
|
f.write("=========================================================\n")
|
|
f.write(" Sysom Hotfix Building System \n")
|
|
f.write("=========================================================\n")
|
|
|
|
self.change_building_status("building")
|
|
|
|
# for each run, update the repo
|
|
cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path)
|
|
with os.popen(cmd) as process:
|
|
output = process.read()
|
|
|
|
customize = parameters['customize']
|
|
# before build one job, clear the tmpdir
|
|
self.clear_tmpdir()
|
|
if not customize:
|
|
self.build_supported_kernel(parameters)
|
|
else:
|
|
self.build_customize_kernel(parameters)
|
|
logger.info(log_file)
|
|
self.fd.close()
|
|
else:
|
|
logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id)
|
|
except Exception as e:
|
|
logger.error(str(e))
|
|
self.change_building_status("failed")
|
|
finally:
|
|
if self.fd is not None:
|
|
self.fd.close()
|
|
consumer.ack(event)
|
|
time.sleep(self.sleep_time)
|
|
retry_time += 1
|
|
|
|
|
|
class HotfixBuilderMain():
|
|
|
|
def __init__(self, config: YAML_CONFIG) -> None:
|
|
self.parameters = config.get_service_config()
|
|
|
|
def start(self):
|
|
hotfix_builder = HotfixBuilder(self.parameters)
|
|
hotfix_builder.run()
|