老版本pungi生成ISO
老版本pungi生成ISO的python文件
import pprint
import os
import shutil
import ConfigParser
import logging
import re
import time
import subprocess
import pwd
import sys
import uploadDir
# import pyjsonrpc
import rpm
import pexpect
def read_rpm_header(rpmfile): #获取rpm的header，rpmfile是rpm包的路径加名字如：/root/wqz/gnutls-3.6.14-7.p01.ky10.x86_64.rpm
ts = rpm.TransactionSet()
ts.setVSFlags(-1)
fd = os.open(rpmfile, os.O_RDONLY)
h = ts.hdrFromFdno(fd)
os.close(fd)
return h # <rpm.hdr object at 0x7fa09958c5d0>
def check_rpm_ok(rpmfile): #检查rpm
fd = None
try:
ts = rpm.TransactionSet()
ts.setVSFlags(-1)
fd = os.open(rpmfile, os.O_RDONLY)
h = ts.hdrFromFdno(fd)
return True
except Exception as e:
return False
finally:
if fd != None:
os.close(fd)
def get_rpm_nvr(rpmfile): #获取rpm的nvr
h = read_rpm_header(rpmfile)
# nvr = ('-').join([h['name'], h['version'], h['release']]) # gnutls-3.6.14-7.p01.ky10
return h['Signature']
def sign(cmd): #签名相关
password = 'NeoKylinq1w2e3r4t5Linuxy6u7i8o9p0Server'
# os.system(cmd)
# # sleep briefly (give user a chance to ctrl-C)
# time.sleep(2)
# # Use expect to give the passphrase
# # LANG=C to have english question 'pass phrase'
os.environ['LC_ALL'] = 'C'
child = pexpect.spawn(cmd)
# Wait for 'pass phrase'
child.expect('phrase:')
if not password:
child.sendline('\r')
else:
child.sendline("%s" % password)
child.expect(pexpect.EOF)
# before (last 100 chars): b'You must set "%_gpg_name" in your macro file\r\n'
# vim ~/.rpmmacros
def do_sign(srpm_dir): #签名相关
for path, dirs, files in os.walk(srpm_dir):
for f in files:
if f.endswith('.rpm'):
file_path = os.path.join(path, f)
cmd = 'rpm --addsign %s' % file_path
print
cmd
sign(cmd)
print
"finished"
def sign_singel_rpm(f): #签名相关
if f.endswith('.rpm'):
cmd = 'rpm --addsign %s' % f
print
cmd
sign(cmd)
def run_command(cmd, errorMessage, logger): #产生子进程，生成的只能够返回
'''Execute cmd'''
sign = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
logger.info(cmd)
while sign.poll() == None:
out = sign.stdout.readline()
if out is None:
continue
else:
result = out.strip().strip('\n')
if result != "":
logger.debug(result)
sign.wait()
if sign.returncode != 0:
if errorMessage != None:
logger.error(errorMessage)
if exit == True:
sys.exit(1)
return False
else:
return True
def get_koji_server(): #获取到koji的server地址
'''
Get default koji server
'''
cff = ConfigParser.RawConfigParser()
cff.read('/etc/koji.conf')
koji_server = cff.get('koji', 'server')
return koji_server
def get_packages_server(): #获取koji的topurl地址
'''
Get default packages download server
'''
cff = ConfigParser.RawConfigParser()
cff.read('/etc/koji.conf')
packages_server = cff.get('koji', 'topurl')
return packages_server
def set_user(name): #用户相关
uid = pwd.getpwnam(name)[2]
gid = pwd.getpwnam(name)[3]
os.setgid(gid)
os.setuid(uid)
def change_user(name): #用户相关
def _change_user(func):
def _change(*args, **kargs):
cur_gid = os.getgid()
cur_uid = os.getuid()
set_user(name)
func(*args, **kargs)
os.setgid(cur_gid)
os.setuid(cur_uid)
return _change
return _change_user
# os.setgid(gid)
def _get_args(): # 获取到args.config #todo 这个地方的配置文件不可少
"""
Parser args, support python >= 2.7
"""
import argparse
parser = argparse.ArgumentParser(description="The tools to release os iso.")
parser.add_argument("-c", "--config", help="Please provide config file") #python python脚本-1.py -c haha 或者python python脚本-1.py --config haha 这个后面跟的就是配置文件名称
args = parser.parse_args()
if args.config:
return args.config
else:
sys.exit(1)
def _get_args_old(): # 获取到args.config
"""
Parser args, support python <2.7
"""
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-c", "--config", dest="config",
help="Please provide config file", metavar="FILE") #python test1.py -c haha 或者python test1.py --config haha
(options, args) = parser.parse_args()
if options.config:
return options.config
else:
sys.exit(1)
class SignCan(): #签名相关
def __init__(self, log_file):
self.log_file = log_file
def get_nvr(self): #通过log_file文件获取到下载失败的包
nvr_list = []
with open(self.log_file) as f:
file_lines = f.readlines()
for line in file_lines:
if "ERROR: can't download" in line:
nvr_list.append('-'.join(line.split('packages/')[-1].split('/data/')[0].split('/')))
return list(set(nvr_list))
def run_command(self, cmd): #创建子进程
p = subprocess.Popen(cmd, shell=True)
p.wait()
def sign(self):
nvrs = self.get_nvr()
if len(nvrs) == 0:
return False
for nvr in nvrs:
cmd = 'su kojiadmin -c "koji write-signed-rpm 7a486d9f %s"' % nvr
print
cmd
self.run_command(cmd)
return True
class ReleaseOS():
def _get_value(self): # 读取配置文件信息
if sys.version_info < (2, 7):
config = ConfigParser.RawConfigParser()
config_file = _get_args_old()
else:
config = ConfigParser.RawConfigParser(allow_no_value=True)
config_file = _get_args()
if os.path.exists(config_file):
config.read(config_file)
else:
print
'Can not found %s file' % config_file
sys.exit(1)
# self.mock_conf_name = config.get('mock','mock_conf_name') #mock config file just name ,not include suffix
self.ftp_ip = config.get('ftp', 'ip')
self.ftp_user = config.get('ftp', 'user')
self.ftp_passwd = config.get('ftp', 'password')
self.ftp_upload_dir = config.get('ftp', 'upload_dir')
self.pungi_user = config.get('mock', 'user')
self.replace_boot = config.getboolean('info', 'replace_boot')
self.replace_comps = config.getboolean('info', 'replace_comps')
self.boot_dir = config.get('info', 'boot_dir')
# self.target_dir = config.get('info','target_dir')
self.product_name = config.get('info', 'product_name')
self.ks_file_path = config.get('info', 'ks_file_path')
# temp vaule, for pylorax and pypungi, add by huachao
self.newpungi_file_path = config.get('info', 'newpungi_file_path')
self.newpungi_init_path = config.get('info', 'newpungi_init_path')
self.newpylorax_init_path = config.get('info', 'newpylorax_init_path')
self.newpylorax_treebuilder_path = config.get('info', 'newpylorax_treebuilder_path')
self.newpylorax_ltmpl_path = config.get('info', 'newpylorax_ltmpl_path')
# end by huachao
self.ks_file_name = os.path.basename(self.ks_file_path)
self.bugzilla_url = config.get('info', 'bugzilla_url')
self.tag = config.get('info', 'tag')
self.target_arch = config.get('info', 'target_arch')
self.type = config.get('info', 'type')
self.build = config.get('info', 'build')
self.build_num = self.build.split('-')[1].split('Build')[1]
if config.has_option('info', 'branch'):
self.branch = config.get('info', 'branch')
self.branch_num = re.findall(r'\d+', self.branch)[0]
else:
self.branch = None
self.pungi_args = config.get('pungi', 'args')
self.product_info = config.get('info', 'product_info')
self.before_repo_package = config.get('info', 'before_repo_package')
self.after_repo_package = config.get('info', 'after_repo_package')
# self.other_package = config.get('info','other_package')
# mash_output_dir = 'ns7.0/os'
self.release = config.get('info', 'release')
self.neokylin_release = config.get('info', 'neokylin_release')
self.comsxml = config.get('info', 'compsxml')
self.download_url = config.get('info', 'download_url')
self.inherit = config.get('mash', 'inherit')
self.output_dir = config.get('info', 'output_dir')
self.release_dir = config.get('info', 'release_dir')
self.notboot_dir = config.get('info', 'not_boot_dir')
self.local_iso_dir = os.path.join(config.get('info', 'local_iso_dir'), self.tag)
self.mashed_output_dir = os.path.join(self.output_dir, 'mashed', self.tag, self.type)
self.target_dir = os.path.join(self.release, self.target_arch, self.type)
self.log_dir = os.path.join(config.get('info', 'log_dir'), self.tag)
self.mock_conf_name = ('-').join([self.tag, self.target_arch])
self.mock_build_root = os.path.join('/var/lib/mock', self.mock_conf_name, 'root')
self.upload_iso_dir = os.path.join(self.local_iso_dir, self.cur_time)
_release_dir = os.path.join(self.release_dir, self.tag, self.type)
for _dir in [self.log_dir, self.local_iso_dir, _release_dir, self.mashed_output_dir]:
if not os.path.exists(_dir):
os.makedirs(_dir)
def __init__(self):
"""
tag : koji tag
type: os or updates
release: release version
inhert: True or False, is not inhert from other tag
"""
self.cur_time = time.strftime('%Y%m%d%H', time.localtime(time.time()))
self.cur_year = time.strftime('%Y', time.localtime(time.time()))
self.cur_day = time.strftime('%Y%m%d', time.localtime(time.time()))
self.cur_mon = time.strftime('%m', time.localtime(time.time()))
self._get_value()
self.log_file = os.path.join(self.log_dir, '%s_%s.log' % (self.build, self.cur_time))
self.logger = logging.getLogger()
handler = logging.FileHandler(self.log_file)
self.logger.addHandler(handler)
self.logger.setLevel(logging.DEBUG)
self.iso_path = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', self.release,
self.target_arch, 'iso')
def create_mash_file(self, version, output_dir, type, tag, xml_path, inherit=True): #通过tag创建mash文件，并设置相关值
'''
Create mash file
version: 7.0
output_subdir:ns7.0/os
tag: ns7.0
'''
output_subdir = os.path.join(tag, type, self.cur_time)
cf = ConfigParser.RawConfigParser()
# name = 'ns%s' % version
name = tag
file_path = os.path.join('/etc/mash', '%s.mash' % name)
if os.path.isfile(file_path):
os.remove(file_path)
cf.read(file_path)
if not cf.has_section(name):
cf.add_section(name)
cf.set(name, 'rpm_path', '%(arch)s/Packages')
cf.set(name, 'repodata_path', '%(arch)s/')
cf.set(name, 'source', 'True')
# cf.set(name, 'source_path', 'source/SRPMS')
cf.set(name, 'output_subdir', output_subdir)
cf.set(name, 'debuginfo', 'False')
cf.set(name, 'multilib', 'True')
cf.set(name, 'multilib_method', 'devel')
cf.set(name, 'multilib_file', 'list_pkgs.%s' % name)
cf.set(name, 'tag', tag)
cf.set(name, 'inherit', inherit)
cf.set(name, 'strict_keys', 'True')
cf.set(name, 'keys', '7a486d9f')
if self.target_arch == 'ppc64le':
cf.set(name, 'arches', self.target_arch)
elif self.target_arch == 'x86_64':
# cf.set(name, 'arches', 'i386 x86_64')
cf.set(name, 'arches', 'x86_64')
elif self.target_arch == 'aarch64':
# cf.set(name, 'arches', 'i386 x86_64')
cf.set(name, 'arches', 'aarch64')
cf.set(name, 'delta', 'True')
cf.set(name, 'max_delta_rpm_size', '800000000')
cf.set(name, 'distro_tags', 'cpe:/o:neokylin:ns:%s Null' % version)
cf.set(name, 'hash_packages', 'False')
cf.write(open(file_path, 'w'))
cmd = 'mash -f %s -o %s %s' % (xml_path, output_dir, tag)
self.logger.info('Start mash ....')
if run_command(cmd, None, self.logger):
self.check_mashed_error()
return os.path.join(output_dir, output_subdir)
return False
def check_mashed_error(self): #检查
sc = SignCan(self.log_file)
if sc.sign() is True:
self.logger.info("mashed rpm include unsiged , and fixed, please try agin.")
sys.exit(0)
def set_mash_default_config(self): #把koji相关的地址写到mash.conf文件中
'''
Set default koji_server
'''
mash_config = '/etc/mash/mash.conf'
cf = ConfigParser.RawConfigParser()
cf.read(mash_config)
koji_server = get_koji_server()
packages_server = get_packages_server()
if cf.has_section('defaults'):
cf.set('defaults', 'buildhost', koji_server)
cf.set('defaults', 'repodir', packages_server)
cf.write(open(mash_config, 'w'))
def create_pungi_mock_file(self, tag, target_arch): #创建/etc/mock/%s.cfg文件，并写入对应的值
"""
koji_ip: koji server ip adderss
base_release : koji tag
target_arch: x86 or x86_64
"""
koji_ip = get_koji_server().split('/')[-2].strip()
root = '-'.join([tag, target_arch])
content = '''
config_opts['basedir'] = '/var/lib/mock'
config_opts['chroot_setup_cmd'] = 'groupinstall build'
config_opts['target_arch'] = '%s'
config_opts['root'] = '%s'
config_opts['plugin_conf']['bind_mount_enable'] = True
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev','/dev/'))
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev/pts','/dev/pts/'))
config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/dev/shm','/dev/shm/'))
config_opts['yum.conf'] = """
[main]
cachedir=/var/cache/yum
debuglevel=1
logfile=/var/log/yum.log
reposdir=/dev/null
retries=20
obsoletes=1
gpgcheck=0
assumeyes=1
# repos
[build]
tag=build
baseurl=http://%s/kojifiles/repos/%s-build/latest/%s/
"""
''' % (target_arch, root, koji_ip, tag, target_arch)
file = '/etc/mock/%s.cfg' % root
with open(file, 'w') as f:
if os.path.exists(file):
shutil.copy(file, '%s.bak' % file)
f.writelines(content)
def sign_tag_key(self, tag): # 签名
cmd = 'sigulsign_unsigned.py -vv --tag=%s --inherit --just-list ns6key' % tag
if run_command(cmd, None, self.logger):
password = 'NeoKylinq1w2e3r4t5Linuxy6u7i8o9p0Server'
cmd = 'sigulsign_unsigned.py -v -v --tag=%s --inherit --password=%s ns6key' % (tag, password)
return run_command(cmd, None, self.logger)
return True
def relink_mash_repo(self, mash_repo_path, release_dir, mashed_output_dir, tag): #生成output_dir链接released_dir/repo_dir
repo_dir = os.path.join(release_dir, tag, self.type, 'repo')
released_dir = os.path.join(release_dir, tag, self.type, self.cur_time)
if not os.path.isdir(mash_repo_path):
raise Exception("Directory %s not exist,Please check!" % mash_repo_path)
shutil.move(mash_repo_path, mashed_output_dir)
self.logger.info('Moved %s to %s.' % (mash_repo_path, mashed_output_dir))
base_dir = os.path.dirname(mashed_output_dir)
sub_dir = [dir for dir in os.listdir(base_dir) if
os.path.isdir(os.path.join(base_dir, dir)) and re.findall(r'\d', dir)]
if len(sub_dir) >= 2:
max_dir = max(sub_dir)
for dir in sub_dir:
if dir != max_dir:
rm_dir = os.path.join(base_dir, dir)
# shutil.rmtree(rm_dir)
self.logger.warning('Delete dir %s .' % rm_dir)
for _dir in [released_dir, repo_dir]:
if os.path.islink(_dir):
os.unlink(_dir)
self.logger.info('Delete link %s .' % _dir)
output_dir = os.path.join(mashed_output_dir, self.cur_time)
os.symlink(output_dir, released_dir)
os.symlink(output_dir, repo_dir)
self.logger.info('Link %s to %s' % (output_dir, released_dir))
self.logger.info('Link %s to %s' % (output_dir, repo_dir))
def init_pungi_mock_env(self, user, mock_conf_name, ks_file): # 生成mock环境，并下载相关软件和文件
cmd1 = 'su - %s -c "mock -r %s --init"' % (user, mock_conf_name)
cmd2 = 'su - %s -c "mock -r %s --install pungi"' % (user, mock_conf_name)
cmd3 = 'su - %s -c "mock -r %s --copyin %s / "' % (user, mock_conf_name, ks_file)
# temp do,
# copy file to /usr/bin/pungi, Affect: del 'ns'
# copy pypungi/__init__.py, Affect:
# copy pylorax/__init__.py, Affect:
cmd4 = 'su - %s -c "mock -r %s --copyin %s /usr/bin/"' % (user, mock_conf_name, self.newpungi_file_path)
cmd5 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pypungi"' % (
user, mock_conf_name, self.newpungi_init_path)
cmd6 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pylorax"' % (
user, mock_conf_name, self.newpylorax_init_path)
cmd7 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pylorax"' % (
user, mock_conf_name, self.newpylorax_ltmpl_path)
build_root = os.path.join('/var/lib/mock', mock_conf_name)
if os.path.exists(build_root):
shutil.rmtree(build_root)
self.logger.info('Delete build_root %s' % build_root)
for cmd in [cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7]:
run_command(cmd, None, self.logger)
# relink_mash_repo('/rpm/rpmbuild/SPECS/20150415','/root/release')
# print max(re.findall(r'\w\d',str))
def init_pungi_mock_env_old(self, user, mock_conf_name, ks_file):
cmd1 = 'su - %s -c "mock -r %s --init"' % (user, mock_conf_name)
cmd2 = 'su - %s -c "mock -r %s --install pungi"' % (user, mock_conf_name)
cmd3 = 'su - %s -c "mock -r %s --copyin %s / "' % (user, mock_conf_name, ks_file)
cmd4 = 'su - %s -c "mock -r %s --copyin %s /usr/bin/"' % (user, mock_conf_name, self.newpungi_file_path)
cmd5 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pypungi"' % (
user, mock_conf_name, self.newpungi_init_path)
cmd6 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pylorax"' % (
user, mock_conf_name, self.newpylorax_init_path)
cmd7 = 'su - %s -c "mock -r %s --copyin %s /usr/lib/python2.7/site-packages/pylorax"' % (
user, mock_conf_name, self.newpylorax_ltmpl_path)
build_root = os.path.join('/var/lib/mock', mock_conf_name)
if os.path.exists(build_root):
shutil.rmtree(build_root)
self.logger.info('Delete build_root %s' % build_root)
for cmd in [cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7]:
run_command(cmd, None, self.logger)
def mock_copy(self, source, target): # 把外面的文件复制到mock环境里
# cmd = 'su - %s -c "mock -r %s --copyin %s %s"' % (self.pungi_user,self.mock_config, source, target)
# cmd = 'su - %s -c "mock -r %s --copyin %s %s"' % (self.pungi_user,self.mock_config, source, target)
cmd = 'mock -r %s --copyin %s %s' % (self.mock_conf_name, source, target)
runcmd = "su - %s -c '%s'" % (self.pungi_user, cmd)
return run_command(runcmd, None, self.logger)
def copy_file_to_mock(self, src_dir, target_dir):
"""
Copy file to mock env.
"""
# if 'PackageSec' in target_dir:
# path_PackageSec = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', target_dir)
# os.mkdir(path_PackageSec)
if os.path.exists(src_dir):
copy_files = os.listdir(src_dir)
if len(copy_files) > 0:
for f in copy_files:
self.mock_copy(os.path.join(src_dir, f), os.path.join(target_dir, f))
# @change_user('pungier')
def do_pungi(self, ks_file_name, name, version): #制作iso并返回路径
"""
for example: /usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks \
--name "ns6.5" --ver adv-build19 --bugurl http://bugzilla.cs2c.com.cn
"""
# Test
# ks_file = 'ns6.5-sc2.2-lic.ks'
# name = 'ns6.5' 'NeoKylin Linux Advanced Server'
# version = 'adv-build19' 'v10'
# bugzilla_url = 'http://bugzilla.cs2c.com.cn'
change_user('pungier')
# if self.release == '7.0':
# pungi_args = '/usr/bin/pungi --force --nosource --nodebuginfo -c /%s ' \
# ' --name \\"%s\\" --ver \\"%s\\" --release \\"%s\\" --bugurl \\"%s\\" --isfinal --nohash --nogreedy --full-archlist'\
# % (ks_file_name,name,version,self.build, self.bugzilla_url)
if self.branch is not None:
pungi_args = '%s -c /%s --name \\"%s\\" --ver \\"%s\\" --release \\"%s\\" --bugurl \\"%s\\" ' % (
self.pungi_args, ks_file_name, name, version, '-'.join([self.build, self.branch]), self.bugzilla_url)
# add debug, /usr/bin/pungi --force --nosource --nodebuginfo --isfinal --nohash --nogreedy --full-archlist -c /ns7.6.ks --name "NeoKylin Linux Advanced Server" --ver "v10" --release "Release-Build02-lic" --bugurl "http://bugzilla.cs2c.com.cn"
else:
pungi_args = '%s -c /%s --name \\"%s\\" --ver \\"%s\\" --release \\"%s\\" --bugurl \\"%s\\" ' % (
self.pungi_args, ks_file_name, name, version, self.build, self.bugzilla_url)
# elif self.release == '6.5':
# pungi_args = '/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /%s'\
# ' --name \\"%s\\" --ver \\"%s\\" --bugurl \\"%s\\"'\
# % (ks_file_name,name,version, self.bugzilla_url)
cmds = []
# Gather package
cmd1 = 'mock -r %s --chroot "' % self.mock_conf_name + pungi_args + ' -G"'
c = "su - %s -c '%s'" % (self.pungi_user, cmd1)
run_command(c, None, self.logger)
# cmds.append(cmd1)
# Copy other pkg to Packages before create repo
self.copy_file_to_mock(self.before_repo_package, os.path.join(self.target_dir, 'Packages'))
# Replace neokylin_release #修改neokylin_release版本号
self.modify_neokylin_release()
# Repace default xml file #如果有comps则是用新的
if self.replace_comps is True:
self.mock_copy(self.comsxml, '/work/%s/\"%s\"' % (
self.target_arch, '-'.join([self.product_name.replace(' ', '-'), self.release, 'comps.xml'])))
#创建repo新版本没有了
# createrepo -d -o os os -g ${COMPS_FILE} 替换
#/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks --name "ns6.5" --ver adv-build19 --bugurl http://bugzilla.cs2c.com.cn
# -G（Gather stage） 从 yum 源将 ks 中列出的包及依赖下载到本地，即包收集阶段
# -C（Createrepo stage）将这些包做成一个本地的 yum 源，即 createrepo
# -B（BuildInstall stage）生成 ISO 中的 image 文件等；F13使用 buildinstall 脚本——属于anaconda包；
# -I （CreateISO stage）将 ISO 根目录（installtree）做成 ISO ，即 mkisofs
# Create repo
cmd2 = 'mock -r %s --chroot "' % self.mock_conf_name + pungi_args + ' -C"'
c = "su - %s -c '%s'" % (self.pungi_user, cmd2)
run_command(c, None, self.logger)
# cmds.append(cmd2)
if self.replace_boot is True:
self.copy_file_to_mock(self.boot_dir, self.target_dir)
# Create BuildInstall
else:
cmd3 = 'mock -r %s --chroot "' % self.mock_conf_name + pungi_args + ' -B"'
cmds.append(cmd3)
for cmd in cmds:
c = "su - %s -c '%s'" % (self.pungi_user, cmd)
run_command(c, None, self.logger)
# Copy bot bootfile ,if notboot_dir is not none
self.copy_file_to_mock(self.notboot_dir, self.target_dir)
# Copy other pkg to Packages
self.copy_file_to_mock(self.after_repo_package, os.path.join(self.target_dir, 'Packages'))
# copy product info
self.write_product_info()
# Delete useless dir
useless_dir = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', self.target_dir,
'.olddata')
self.delete_useless_dir(useless_dir)
# Create ISO
cmd = 'mock -r %s --chroot "' % self.mock_conf_name + pungi_args + ' -I"'
c = "su - %s -c '%s'" % (self.pungi_user, cmd)
run_command(c, None, self.logger)
iso_path = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', self.release,
self.target_arch, 'iso')
return iso_path
def mash_repo(self): # mash文件更新
# if self.sign_tag_key(self.tag):
if True:
self.set_mash_default_config()
mash_repo_path = self.create_mash_file(self.release, self.output_dir, self.type, self.tag, self.comsxml,
self.inherit)
self.logger.info('mash_repo_path value is %s' % mash_repo_path)
self.logger.info('self.release_dir value is %s' % self.release_dir)
self.logger.info('self.mashed_output_dir value is %s' % self.mashed_output_dir)
self.logger.info('self.tag value is %s' % self.tag)
if mash_repo_path:
self.relink_mash_repo(mash_repo_path, self.release_dir, self.mashed_output_dir, self.tag)
return True
else:
self.logger.error('Mash repo error.')
# else:
# raise Exception("Sign rpm failed,please check log file")
def create_iso(self):
self.create_pungi_mock_file(self.tag, self.target_arch) #创建/etc/mock/%s.cfg文件，并写入对应的值
self.init_pungi_mock_env(self.pungi_user, self.mock_conf_name, self.ks_file_path) # 生成mock环境，并下载相关软件和文件
self.do_pungi(self.ks_file_name, self.product_name, self.release) #制作iso并返回路径
self.create_iso_package_list() #记录iso中所需要的package
self.copy_neokylin_release_to_iso_dir() #复制neokylin_release到iso文件中
self.copy_iso_to_dir() #复制iso到upload_iso_dir中
self.copy_log_file() #复制log文件
self.check_iso_size() #检查iso文件大小
self.upload_iso() #上传iso
url = 'http://10.1.110.123/release/Release/os/%s/%s.iso' \
% ('/'.join([self.tag, self.cur_year, self.cur_mon, self.cur_time]),
'-'.join([self.release, self.build, self.target_arch]))
if '10' in self.release:
url = 'http://10.1.110.123/release/Release/os/%s/%s.iso' \
% ('/'.join([self.tag, self.target_arch, self.cur_year, self.cur_mon, self.cur_time]),
'-'.join(['NeoKylin-Server-10-x86', self.build, self.cur_day]))
# self.write_deploy_config(url)
# pprint.pprint(url)
self.logger.info('SUCCESS! ISO download path : %s' % url)
return True
def main(self):
if self.mash_repo():
self.create_iso()
def write_product_info(self): #写入产品信息
product_file = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', '.productinfo')
cur_date = time.strftime('%Y%m%d', time.localtime(time.time()))
if self.branch is not None:
content = self.product_info % (
self.release, self.build_num, cur_date, self.branch, cur_date)
else:
# content = self.product_info % (self.release, re.findall(r'\d+', self.build)[0], cur_date)
content = self.product_info % ('V10', self.build_num, cur_date)
with open(product_file, 'w') as f:
f.writelines(content)
f.write(os.linesep)
self.mock_copy(product_file, self.target_dir)
def copy_iso_to_dir(self): #复制iso到upload_iso_dir中
file_num = len(os.listdir(self.iso_path))
# dest_dir = os.path.join(self.local_iso_dir,self.cur_time)
if os.path.exists(self.iso_path) and file_num > 0:
shutil.copytree(self.iso_path, self.upload_iso_dir)
self.logger.info('copy %s to %s' % (self.iso_path, self.upload_iso_dir))
def copy_neokylin_release_to_iso_dir(self): #复制neokylin_release到iso文件中
"""
Copy neokylin release rpm to iso dir for upload to release
:return:
"""
self.iso_path = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', self.release,
self.target_arch, 'iso')
root_os = os.path.join('/var/lib/mock', '-'.join([self.tag, self.target_arch]), 'root', self.release,
self.target_arch, 'os')
for root, dirs, files in os.walk(root_os):
for f in files:
rpm_path = os.path.join(root, f)
if rpm_path.endswith('.rpm') and f.startswith('neokylin-release'):
shutil.copy2(rpm_path, self.iso_path)
return True
return False
def upload_iso(self): #上传iso
cur_dir = os.getcwd()
os.chdir(self.local_iso_dir)
target_dir = os.path.join(self.ftp_upload_dir, self.target_arch, self.cur_year, self.cur_mon)
uploadDir.upload_folder_to_ftp(self.ftp_ip, self.ftp_user, self.ftp_passwd, target_dir, self.cur_time)
# iso_path = self.download_url + '/%s/%s/%s/%s/%s' % (self.tag, self.target_arch, self.cur_year, self.cur_mon, self.cur_time)
# uploadDir.set_os_deploy(iso_path, self.tag)
# self.logger.info('Upload %s to %s' % (os.path.join(self.local_iso_dir, self.cur_time), target_dir))
# os.chdir(cur_dir)
def modify_neokylin_release(self): #修改neokylin_release信息
# source = '/tmp/neokylin-release-6Server-ns6.5.21.lic.Adv.src.rpm'
# spec_file = '/builddir/build/SPECS/neokylin-release.spec'
# spec_dir = '/builddir/build/SPECS'
# spec_file = None
# for path, dir, files in os.walk(spec_dir):
# for f in files:
# if f.endswith('.spec'):
# spec_file = os.path.join(path, f)
# if spec_file is None:
# raise Exception('Can not found spec file in %s' % spec_dir)
run_script = '-'.join([self.tag, self.build])
target = os.path.basename(self.neokylin_release) #返回neokylin_release路径的最后的名称
tmp_run_script = os.path.join('/tmp', run_script) #拼接路径
self.mock_copy(self.neokylin_release, '/%s' % target)
package_dir = '%s/%s/os/Packages/' % (self.release, self.target_arch)
with open(tmp_run_script, 'w') as f:
cmd1 = 'rpm -ivh %s' % target
cmd2 = 'SPEC_FILE=`find builddir/build/SPECS/|grep .spec`'
if 'nolic' in self.tag:
cmd3 = 'sed -i "s/\%define build_id.*/\%define\ build_id ' + '%s/" $SPEC_FILE' % self.build_num
else:
cmd3 = 'sed -i "s/\%define build_id.*/\%define\ build_id ' + '%s.lic/" $SPEC_FILE' % self.build_num
cmd4 = 'sed -i "s/\%define build_branch.*/\%define\ build_branch ' + '%s/" $SPEC_FILE' % self.branch
cmd5 = 'sed -i "s/\%define build_date.*/\%define build_date ' + '%s/" $SPEC_FILE' % self.cur_day
cmd6 = 'rpmbuild -ba $SPEC_FILE'
cmd7 = 'find %s|grep neokylin-release|xargs rm -rvf' % package_dir
cmd8 = 'cp -rvf `find builddir/build/RPMS/ -name "*neokylin-release*" ! -name "*debuginfo*"` %s' % package_dir
# cmd9 = 'cp -rvf `find builddir/build/SRPMS/ -name "*neokylin-release*" ! -name "*src.rpm*"` %s' % package_dir
cmd9 = 'cp -rvf `find builddir/build/SRPMS/ -name "*neokylin-release*"` /'
for cmd in [cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9]:
f.write(cmd)
f.write(os.linesep)
self.mock_copy(tmp_run_script, '/%s' % run_script)
_cmd = 'mock -r %s --chroot "sh %s" ' % (self.mock_conf_name, run_script)
_cmd = "su - %s -c '%s'" % (self.pungi_user, _cmd)
run_command(_cmd, None, self.logger)
for rpm_path, dirs, files in os.walk(os.path.join(self.mock_build_root, package_dir)):
for f in files:
rpm_file = os.path.join(rpm_path, f) # 把files里面的所有文件都放到rpm_path目录下，dirs是目录
if rpm_file.endswith('.rpm') and 'neokylin-release' in f:
sign_singel_rpm(rpm_file)
self.logger.info("sign rpm %s successfully." % rpm_file)
break
def create_iso_package_list(self): #记录iso中所需要的package
package_dir = ' %s/%s/os' % (self.release, self.target_arch)
iso_dir = '%s/%s/iso' % (self.release, self.target_arch)
run_script = '-'.join([self.product_name.replace(' ', '-'), self.release, self.build, 'package.txt'])
cmd1 = "find %s|grep rpm$|xargs rpm -qp --qf " % (
package_dir) + '"%{N}-%{V}-%{R}.%{ARCH}.rpm\\n"' + "|sort > %s/%s" % (iso_dir, run_script)
tmp_run_script = os.path.join('/tmp', run_script)
with open(tmp_run_script, 'w') as f:
for cmd in [cmd1]:
f.write(cmd)
f.write(os.linesep)
self.mock_copy(tmp_run_script, '/%s' % run_script)
_cmd = 'mock -r %s --chroot "sh %s" ' % (self.mock_conf_name, run_script)
_cmd = "su - %s -c '%s'" % (self.pungi_user, _cmd)
run_command(_cmd, None, self.logger)
def check_iso_size(self): #检查iso文件大小
"""
Check iso size less than 4706074624,
:return:
"""
# iso_dir = '%s/%s/iso' % (self.release, self.target_arch)
iso_dir = self.iso_path
for root, dirs, files in os.walk(iso_dir):
for f in files:
file_name = os.path.join(root, f)
self.logger.info("%s file size is %s." % (file_name, os.path.getsize(file_name)))
if file_name.endswith('.iso') and os.path.getsize(file_name) > 4706074624:
self.logger.error("%s file size too large, can't be record." % file_name)
raise Exception("%s file size too large, can't be record." % file_name)
return False
return True
def copy_log_file(self): #复制log文件
build_iso_log = os.path.join(self.mock_build_root, 'logs/%s.log' % self.target_arch)
shutil.copy(build_iso_log, self.upload_iso_dir)
shutil.copy(self.log_file, self.upload_iso_dir)
def write_deploy_config(self, url): #把iso路径写入到/etc/contiudelivery/deploy.conf 文件中
"""
Write deploy.config
"""
config_file = '/etc/contiudelivery/deploy.conf'
config = ConfigParser.ConfigParser()
with open(config_file, 'r') as cfg:
config.readfp(cfg)
config.set('iso', 'iso_path', url)
config.set('iso', 'http_user', 'cloud')
config.set('iso', 'http_passwd', 'cloud')
with open(config_file, 'w') as cfg:
config.write(cfg)
def delete_useless_dir(self, useless_dir): #删除无用的的目录
if os.path.isdir(useless_dir):
shutil.rmtree(useless_dir)
self.logger.info('Found useless dir %s and deleted' % useless_dir)
class DeployClient(): #和iso制作无关
def __init__(self, project):
config = ConfigParser.ConfigParser()
config.read('/etc/contiudelivery/deploy.conf')
self.http_client = pyjsonrpc.HttpClient(
url='http://%s:%s/jsonrpc' % (config.get('deploy', 'ip'), config.get('deploy', 'port'))
) #生成一个连接、远程连接
self.http_client_deploy = config.get('deploy', 'deploy')
self.config = config
self.project = project
def send_deploy_info(self):
return self.__deploy_common(self.project, self.config.items(self.project))
def __deploy_common(self, component, configs):
params = dict(configs)
params.update({'component': component})
if self.http_client_deploy:
# info = self.http_client.deploy(params)
info = self.http_client.notify("deploy", params) # 向http_client连接发送数据
# logger.info("Send deploy info %s, and return %s" % (params,info))
print
"Send deploy info %s, and return %s" % (params, info)
return info
if __name__ == '__main__':
rs = ReleaseOS()
rs.mash_repo()
result = rs.create_iso()
# if result:
# de = DeployClient('iso')
# de.send_deploy_info()
# rs.create_iso_package_list()
# url = 'http://10.1.110.123/release/Release/os/%s/%s/%s.iso' \
# % (rs.tag, rs.cur_time, '-'.join([rs.tag, rs.build, rs.target_arch]))
# rs.write_deploy_config(url)
# rs.copy_file_to_mock(rs.other_package, os.path.join(rs.target_dir,'Packages'))
# iso_name = sys.argv[3]
# Test copy noboot_dir
# rs.copy_notboot_file(rs.notboot_dir, rs.target_dir)
#
# su - pungier -c 'mock -r ns6.5-sc2.2-x86_64 --chroot "/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks --name \"NeoKylin Linux Advanced Server\" --ver \"6.5\" --bugurl \"http://bugzilla.cs2c.com.cn\" -G"'
# su - pungier -c 'mock -r ns6.5-sc2.2-x86_64 --chroot "/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks --name \"NeoKylin Linux Advanced Server\" --ver \"6.5\" --bugurl \"http://bugzilla.cs2c.com.cn\" -C"'
# su - pungier -c 'mock -r ns6.5-sc2.2-x86_64 --chroot "/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks --name \"NeoKylin Linux Advanced Server\" --ver \"6.5\" --bugurl \"http://bugzilla.cs2c.com.cn\" -B"'
# su - pungier -c 'mock -r ns6.5-sc2.2-x86_64 --chroot "/usr/bin/pungi --force --nosource --nodebuginfo --nosplitmedia -c /ns6.5-sc2.2-lic.ks --name \"NeoKylin Linux Advanced Server\" --ver \"6.5\" --bugurl \"http://bugzilla.cs2c.com.cn\" -I"'
