From 7f7154ef5694b65ef1723ce287bcd8a0ec79c48d Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 5 Jun 2019 19:57:03 -0400 Subject: [PATCH] Correct build failure after mock updrade Problem: Mockchain-parallel requires python2-requests, but it is not found. Mockchain-parallel runs inside docker. the Dockerfile does not explicitly list python2-requests as a requires package. The Dockerfile does list mock, which used to resolve to mock-1.4.15-1.el7.noarch, which pulled in python2-requests. Centos/EPEL upgraded to mock-1.4.16-1.el7.noarch 2019-06-04, and it now pulls in python36-requests, rather than python2-requests. Mockchain-parallel also uses functions from yum and mock. Yum remains at python2, mock is now at python3. There is also one api change in the new mock. Whatever solution should allow mockchain-parallel function in both a new or an old container. The difference being the mock version, and the verion of python required to run mock/mockchain. Supporting the old mock will also allow more time for conversion of old build systems that pre-date the conversion to docker. Solution: 1) For now, ship two versions of mock-parallel. One compatible with the new mock (python 3, new api). The second compatible with the older mock (python 2, old api). The old version can eventually be retired, so not worth trying to break out common functions. 2) Remove the dependency on yum. We only needed one function to split an rpm file name into components (package-name, version, release, architecture, epoch). We'll write our own. 3) mockchain-parallel becomes a wrapper that detects which mock is present, and from that determine which version of mock-parallel-* to use, and which python to use. Change-Id: Ia56dfd39e349096b09351e89ad3d4f024e60e4e4 Depends-On: I7c5d9b52eed49cdce02224603b9bfd17e4426558 Closes-Bug: 1831768 Signed-off-by: Scott Little --- build-tools/build-rpms-parallel | 3 +- build-tools/mockchain-parallel | 1282 ++----------------------- build-tools/mockchain-parallel-1.3.4 | 1206 +++++++++++++++++++++++ build-tools/mockchain-parallel-1.4.16 | 1213 +++++++++++++++++++++++ build-tools/stxRpmUtils.py | 41 + 5 files changed, 2552 insertions(+), 1193 deletions(-) create mode 100755 build-tools/mockchain-parallel-1.3.4 create mode 100755 build-tools/mockchain-parallel-1.4.16 create mode 100644 build-tools/stxRpmUtils.py diff --git a/build-tools/build-rpms-parallel b/build-tools/build-rpms-parallel index a0c277a7..a99a4c7e 100755 --- a/build-tools/build-rpms-parallel +++ b/build-tools/build-rpms-parallel @@ -616,7 +616,7 @@ kill_descendents () local relevant_recursive_children="$ME" local relevant_recursive_promote_children="mock" - local relevant_other_children="mockchain-parallel" + local relevant_other_children="mockchain-parallel mockchain-parallel-1.3.4 mockchain-parallel-1.4.16" local recursive_promote_children=$(for relevant_child in $relevant_recursive_promote_children; do pgrep -P $kill_pid $relevant_child; done) local recursive_children=$(for relevant_child in $relevant_recursive_children; do pgrep -P $kill_pid $relevant_child; done) @@ -2282,6 +2282,7 @@ echo "CMD_OPTIONS=$CMD_OPTIONS" echo "MAX_WORKERS=$MAX_WORKERS" echo "MOCKCHAIN_RESOURCE_ALLOCATION=$MOCKCHAIN_RESOURCE_ALLOCATION" + CMD="$CMD_PREFIX mockchain-parallel -r $BUILD_CFG -l $BUILD_BASE --recurse --workers=$MAX_WORKERS --worker-resources=$MOCKCHAIN_RESOURCE_ALLOCATION --basedir=$MY_WORKSPACE --log=$MOCKCHAIN_LOG --tmp_prefix=$USER --addrepo=$LOCAL_URL --addrepo=$LOCAL_SRC_URL $CMD_OPTIONS -m --rebuild $SRPMS_LIST" echo "" echo "$CMD -m --define='_tis_dist .tis' -m --define='platform_release $PLATFORM_RELEASE'" diff --git a/build-tools/mockchain-parallel b/build-tools/mockchain-parallel index 194b631c..7e37e5b1 100755 --- a/build-tools/mockchain-parallel +++ b/build-tools/mockchain-parallel @@ -1,1207 +1,105 @@ -#!/usr/bin/python -tt -# -*- coding: utf-8 -*- -# vim: noai:ts=4:sw=4:expandtab - -# by skvidal@fedoraproject.org -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. +#!/bin/bash # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. + # -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -# copyright 2012 Red Hat, Inc. - -# SUMMARY -# mockchain -# take a mock config and a series of srpms -# rebuild them one at a time -# adding each to a local repo -# so they are available as build deps to next pkg being built -from __future__ import print_function - -import cgi -# pylint: disable=deprecated-module -import optparse -import os -import re -import shutil -import subprocess -import sys -import tempfile -import time -import multiprocessing -import signal -import psutil - -import requests -# pylint: disable=import-error -from six.moves.urllib_parse import urlsplit - -import mockbuild.util - -from rpmUtils.miscutils import splitFilename - - -# all of the variables below are substituted by the build system -__VERSION__="1.3.4" -SYSCONFDIR="/etc" -PYTHONDIR="/usr/lib/python2.7/site-packages" -PKGPYTHONDIR="/usr/lib/python2.7/site-packages/mockbuild" -MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") -# end build system subs - -mockconfig_path = '/etc/mock' - -def rpmName(path): - filename = os.path.basename(path) - (n, v, r, e, a) = splitFilename(filename) - return n - -def createrepo(path): - global max_workers - if os.path.exists(path + '/repodata/repomd.xml'): - comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] - else: - comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] - cmd = subprocess.Popen( - comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = cmd.communicate() - return out, err - - -g_opts = optparse.Values() - -def parse_args(args): - parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') - parser.add_option( - '-r', '--root', default=None, dest='chroot', - metavar="CONFIG", - help="chroot config name/base to use in the mock build") - parser.add_option( - '-l', '--localrepo', default=None, - help="local path for the local repo, defaults to making its own") - parser.add_option( - '-c', '--continue', default=False, action='store_true', - dest='cont', - help="if a pkg fails to build, continue to the next one") - parser.add_option( - '-a', '--addrepo', default=[], action='append', - dest='repos', - help="add these repo baseurls to the chroot's yum config") - parser.add_option( - '--recurse', default=False, action='store_true', - help="if more than one pkg and it fails to build, try to build the rest and come back to it") - parser.add_option( - '--log', default=None, dest='logfile', - help="log to the file named by this option, defaults to not logging") - parser.add_option( - '--workers', default=1, dest='max_workers', - help="number of parallel build jobs") - parser.add_option( - '--worker-resources', default="", dest='worker_resources', - help="colon seperated list, how much mem in gb for each workers temfs") - parser.add_option( - '--basedir', default='/var/lib/mock', dest='basedir', - help="path to workspace") - parser.add_option( - '--tmp_prefix', default=None, dest='tmp_prefix', - help="tmp dir prefix - will default to username-pid if not specified") - parser.add_option( - '-m', '--mock-option', default=[], action='append', - dest='mock_option', - help="option to pass directly to mock") - parser.add_option( - '--mark-slow-name', default=[], action='append', - dest='slow_pkg_names_raw', - help="package name that is known to build slowly") - parser.add_option( - '--mark-slow-path', default=[], action='append', - dest='slow_pkgs_raw', - help="package path that is known to build slowly") - parser.add_option( - '--mark-big-name', default=[], action='append', - dest='big_pkg_names_raw', - help="package name that is known to require a lot of disk space to build") - parser.add_option( - '--mark-big-path', default=[], action='append', - dest='big_pkgs_raw', - help="package path that is known to require a lot of disk space to build") - parser.add_option( - '--srpm-dependency-file', default=None, - dest='srpm_dependency_file', - help="path to srpm dependency file") - parser.add_option( - '--rpm-dependency-file', default=None, - dest='rpm_dependency_file', - help="path to rpm dependency file") - parser.add_option( - '--rpm-to-srpm-map-file', default=None, - dest='rpm_to_srpm_map_file', - help="path to rpm to srpm map file") - - opts, args = parser.parse_args(args) - if opts.recurse: - opts.cont = True - - if not opts.chroot: - print("You must provide an argument to -r for the mock chroot") - sys.exit(1) - - if len(sys.argv) < 3: - print("You must specify at least 1 package to build") - sys.exit(1) - - return opts, args - - -REPOS_ID = [] - -slow_pkg_names={} -slow_pkgs={} -big_pkg_names={} -big_pkgs={} - -def generate_repo_id(baseurl): - """ generate repository id for yum.conf out of baseurl """ - repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') - repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) - suffix = '' - i = 1 - while repoid + suffix in REPOS_ID: - suffix = str(i) - i += 1 - repoid = repoid + suffix - REPOS_ID.append(repoid) - return repoid - - -def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): - # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) - - try: - with open(infile) as f: - code = compile(f.read(), infile, 'exec') - # pylint: disable=exec-used - exec(code) - - config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) - config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) - # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) - # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) - if tmpfs_size_gb > 0: - config_opts['plugin_conf']['tmpfs_enable'] = True - config_opts['plugin_conf']['tmpfs_opts'] = {} - config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 - config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb - config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' - config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True - # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) - # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) - - with open(destfile, 'w') as br_dest: - for k, v in list(config_opts.items()): - br_dest.write("config_opts[%r] = %r\n" % (k, v)) - - try: - log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) - if not os.path.isdir(config_opts['cache_topdir']): - os.makedirs(config_opts['cache_topdir'], exist_ok=True) - except (IOError, OSError): - return False, "Could not create dir: %s" % config_opts['cache_topdir'] - - cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) - try: - log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) - if not os.path.isdir(cache_dir): - os.makedirs(cache_dir) - except (IOError, OSError): - return False, "Could not create dir: %s" % cache_dir - - return True, '' - except (IOError, OSError): - return False, "Could not write mock config to %s" % destfile - - return True, '' - -def set_basedir(infile, destfile, basedir, opts): - log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) - try: - with open(infile) as f: - code = compile(f.read(), infile, 'exec') - # pylint: disable=exec-used - exec(code) - - config_opts['basedir'] = basedir - config_opts['resultdir'] = '{0}/result'.format(basedir) - config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) - config_opts['root'] = 'mock/b0' - config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) - - with open(destfile, 'w') as br_dest: - for k, v in list(config_opts.items()): - br_dest.write("config_opts[%r] = %r\n" % (k, v)) - return True, '' - except (IOError, OSError): - return False, "Could not write mock config to %s" % destfile - - return True, '' - -def add_local_repo(infile, destfile, baseurl, repoid=None): - """take a mock chroot config and add a repo to it's yum.conf - infile = mock chroot config file - destfile = where to save out the result - baseurl = baseurl of repo you wish to add""" - global config_opts - - try: - with open(infile) as f: - code = compile(f.read(), infile, 'exec') - # pylint: disable=exec-used - exec(code) - if not repoid: - repoid = generate_repo_id(baseurl) - else: - REPOS_ID.append(repoid) - localyumrepo = """ -[%s] -name=%s -baseurl=%s -enabled=1 -skip_if_unavailable=1 -metadata_expire=0 -cost=1 -best=1 -""" % (repoid, baseurl, baseurl) - - config_opts['yum.conf'] += localyumrepo - with open(destfile, 'w') as br_dest: - for k, v in list(config_opts.items()): - br_dest.write("config_opts[%r] = %r\n" % (k, v)) - return True, '' - except (IOError, OSError): - return False, "Could not write mock config to %s" % destfile - - return True, '' - - -def do_build(opts, cfg, pkg): - - # returns 0, cmd, out, err = failure - # returns 1, cmd, out, err = success - # returns 2, None, None, None = already built - - signal.signal(signal.SIGTERM, child_signal_handler) - signal.signal(signal.SIGINT, child_signal_handler) - signal.signal(signal.SIGHUP, child_signal_handler) - signal.signal(signal.SIGABRT, child_signal_handler) - s_pkg = os.path.basename(pkg) - pdn = s_pkg.replace('.src.rpm', '') - resdir = '%s/%s' % (opts.local_repo_dir, pdn) - resdir = os.path.normpath(resdir) - if not os.path.exists(resdir): - os.makedirs(resdir) - - success_file = resdir + '/success' - fail_file = resdir + '/fail' - - if os.path.exists(success_file): - # return 2, None, None, None - sys.exit(2) - - # clean it up if we're starting over :) - if os.path.exists(fail_file): - os.unlink(fail_file) - - if opts.uniqueext == '': - mockcmd = ['/usr/bin/mock', - '--configdir', opts.config_path, - '--resultdir', resdir, - '-r', cfg, ] - else: - mockcmd = ['/usr/bin/mock', - '--configdir', opts.config_path, - '--resultdir', resdir, - '--uniqueext', opts.uniqueext, - '-r', cfg, ] - # heuristic here, if user pass for mock "-d foo", but we must be care to leave - # "-d'foo bar'" or "--define='foo bar'" as is - compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') - compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') - for option in opts.mock_option: - r_match = compiled_re_1.match(option) - if r_match: - mockcmd.extend([r_match.group(1), r_match.group(2)]) - else: - r_match = compiled_re_2.match(option) - if r_match: - mockcmd.extend([r_match.group(1), r_match.group(2)]) - else: - mockcmd.append(option) - - print('building %s' % s_pkg) - mockcmd.append(pkg) - # print("mockcmd: %s" % str(mockcmd)) - cmd = subprocess.Popen( - mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = cmd.communicate() - if cmd.returncode == 0: - with open(success_file, 'w') as f: - f.write('done\n') - ret = 1 - else: - if (isinstance(err, bytes)): - err = err.decode("utf-8") - sys.stderr.write(err) - with open(fail_file, 'w') as f: - f.write('undone\n') - ret = 0 - - # return ret, cmd, out, err - sys.exit(ret) - - -def log(lf, msg): - if lf: - now = time.time() - try: - with open(lf, 'a') as f: - f.write(str(now) + ':' + msg + '\n') - except (IOError, OSError) as e: - print('Could not write to logfile %s - %s' % (lf, str(e))) - print(msg) - - -config_opts = {} - -worker_data = [] -workers = 0 -max_workers = 1 - -build_env = [] - -failed = [] -built_pkgs = [] - -local_repo_dir = "" - -pkg_to_name={} -name_to_pkg={} -srpm_dependencies_direct={} -rpm_dependencies_direct={} -rpm_to_srpm_map={} -no_dep_list = [ "bash", "kernel" , "kernel-rt" ] - - -def init_build_env(slots, opts, config_opts_in): - global build_env - - orig_chroot_name=config_opts_in['chroot_name'] - orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) - # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) - for i in range(0,slots): - new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) - new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) - tmpfs_size_gb = 0 - if opts.worker_resources == "": - if i > 0: - tmpfs_size_gb = 2 * (1 + slots - i) - else: - resource_array=opts.worker_resources.split(':') - if i < len(resource_array): - tmpfs_size_gb=int(resource_array[i]) - else: - log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) - sys.exit(1) - if i == 0 and tmpfs_size_gb != 0: - log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) - sys.exit(1) - build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) - - res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) - if not res: - log(opts.logfile, "Error: Could not write out local config: %s" % msg) - sys.exit(1) - - -idle_build_env_last_awarded = 0 -def get_idle_build_env(slots): - global build_env - global idle_build_env_last_awarded - visited = 0 - - if slots < 1: - return -1 - - i = idle_build_env_last_awarded - 1 - if i < 0 or i >= slots: - i = slots - 1 - - while visited < slots: - if build_env[i]['state'] == 'Idle': - build_env[i]['state'] = 'Busy' - idle_build_env_last_awarded = i - return i - visited = visited + 1 - i = i - 1 - if i < 0: - i = slots - 1 - return -1 - -def release_build_env(idx): - global build_env - - build_env[idx]['state'] = 'Idle' - -def get_best_rc(a, b): - print("get_best_rc: a=%s" % str(a)) - print("get_best_rc: b=%s" % str(b)) - if (b == {}) and (a != {}): - return a - if (a == {}) and (b != {}): - return b - - if (b['build_name'] is None) and (not a['build_name'] is None): - return a - if (a['build_name'] is None) and (not b['build_name'] is None): - return b - - if a['unbuilt_deps'] < b['unbuilt_deps']: - return a - if b['unbuilt_deps'] < a['unbuilt_deps']: - return b - - if a['depth'] < b['depth']: - return a - if b['depth'] < a['depth']: - return b - - print("get_best_rc: uncertain %s vs %s" % (a,b)) - return a - -unbuilt_dep_list_print=False -def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): - global srpm_dependencies_direct - global rpm_dependencies_direct - global rpm_to_srpm_map - global no_dep_list - global unbuilt_dep_list_print - - first_iteration=False - unbuilt = [] - if name in no_dep_list: - return unbuilt - - if checked is None: - first_iteration=True - checked=[] - - # Count unbuild dependencies - if first_iteration: - dependencies_direct=srpm_dependencies_direct - else: - dependencies_direct=rpm_dependencies_direct - - if name in dependencies_direct: - for rdep in dependencies_direct[name]: - sdep='???' - if rdep in rpm_to_srpm_map: - sdep = rpm_to_srpm_map[rdep] - if rdep != name and sdep != name and not rdep in checked: - if (not first_iteration) and (sdep in no_dep_list): - continue - checked.append(rdep) - if sdep in unbuilt_pkg_names: - if not sdep in unbuilt: - unbuilt.append(sdep) - if depth > 0: - child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) - for sub_sdep in child_unbuilt: - if sub_sdep != name: - if not sub_sdep in unbuilt: - unbuilt.append(sub_sdep) - - return unbuilt - -def can_build_at_idx(build_idx, name, opts): - global pkg_to_name - global name_to_pkg - global big_pkgs - global big_pkg_names - global slow_pkgs - global slow_pkg_names - global build_env - - fs_size_gb = 0 - size_gb = 0 - speed = 0 - pkg = name_to_pkg[name] - if name in big_pkg_names: - size_gb=big_pkg_names[name] - if pkg in big_pkgs: - size_gb=big_pkgs[pkg] - if name in slow_pkg_names: - speed=slow_pkg_names[name] - if pkg in slow_pkgs: - speed=slow_pkgs[pkg] - fs_size_gb = build_env[build_idx]['fs_size_gb'] - return fs_size_gb == 0 or fs_size_gb >= size_gb - -def schedule(build_idx, pkgs, opts): - global worker_data - global pkg_to_name - global name_to_pkg - global big_pkgs - global big_pkg_names - global slow_pkgs - global slow_pkg_names - - unbuilt_pkg_names=[] - building_pkg_names=[] - unprioritized_pkg_names=[] - - for pkg in pkgs: - name = pkg_to_name[pkg] - unbuilt_pkg_names.append(name) - unprioritized_pkg_names.append(name) - - prioritized_pkg_names=[] - - for wd in worker_data: - pkg = wd['pkg'] - if not pkg is None: - name = pkg_to_name[pkg] - building_pkg_names.append(name) - - # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) - if len(big_pkg_names) or len(big_pkgs): - next_unprioritized_pkg_names = unprioritized_pkg_names[:] - for name in unprioritized_pkg_names: - pkg = name_to_pkg[name] - if name in big_pkg_names or pkg in big_pkgs: - prioritized_pkg_names.append(name) - next_unprioritized_pkg_names.remove(name) - unprioritized_pkg_names = next_unprioritized_pkg_names[:] - - if len(slow_pkg_names) or len(slow_pkgs): - next_unprioritized_pkg_names = unprioritized_pkg_names[:] - for name in unprioritized_pkg_names: - pkg = name_to_pkg[name] - if name in slow_pkg_names or pkg in slow_pkgs: - if can_build_at_idx(build_idx, name, opts): - prioritized_pkg_names.append(name) - next_unprioritized_pkg_names.remove(name) - unprioritized_pkg_names = next_unprioritized_pkg_names[:] - - for name in unprioritized_pkg_names: - if can_build_at_idx(build_idx, name, opts): - prioritized_pkg_names.append(name) - - name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) - if not name_out is None: - pkg_out = name_to_pkg[name_out] - else: - pkg_out = None - # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) - # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) - return pkg_out - - -def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): - global pkg_to_name - global name_to_pkg - global no_dep_list - - max_depth = 3 - - if len(pkg_names) == 0: - return None - - unbuilt_deps={} - building_deps={} - for depth in range(max_depth,-1,-1): - unbuilt_deps[depth]={} - building_deps[depth]={} - - for depth in range(max_depth,-1,-1): - checked=[] - reordered_pkg_names = pkg_names[:] - # for name in reordered_pkg_names: - while len(reordered_pkg_names): - name = reordered_pkg_names.pop(0) - if name in checked: - continue - - # log(opts.logfile, "checked.append(%s)" % name) - checked.append(name) - - pkg = name_to_pkg[name] - # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) - if not name in unbuilt_deps[depth]: - unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) - if not name in building_deps[depth]: - building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) - # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) - # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) - if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: - if can_build_at_idx(build_idx, name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) - return name - else: - # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) - continue - - if not name in unbuilt_deps[0]: - unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) - if not name in building_deps[0]: - building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) - # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) - # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) - if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): - if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: - if can_build_at_idx(build_idx, name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) - return name - else: - # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) - continue - - loop = False - for dep_name in unbuilt_deps[depth][name]: - if name == dep_name: - continue - - # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) - if dep_name in checked: - continue - - # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) - if not dep_name in unbuilt_deps[depth]: - unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) - if not dep_name in building_deps[depth]: - building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) - # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) - # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) - if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: - if can_build_at_idx(build_idx, dep_name, opts): - log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) - return dep_name - - if not dep_name in unbuilt_deps[0]: - unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) - if not dep_name in building_deps[0]: - building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) - # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) - # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) - if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): - if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: - if can_build_at_idx(build_idx, dep_name, opts): - log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) - return dep_name - - if name in unbuilt_deps[0][dep_name]: - loop = True - # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) - - if loop and len(building_deps[depth][name]) == 0: - log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) - return name - - for dep_name in unbuilt_deps[depth][name]: - if dep_name in reordered_pkg_names: - # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) - reordered_pkg_names.remove(dep_name) - reordered_pkg_names.insert(0,dep_name) - - # log(opts.logfile, "schedule2: Nothing buildable at this time") - return None - - - - -def read_deps(opts): - read_srpm_deps(opts) - read_rpm_deps(opts) - read_map_deps(opts) - -def read_srpm_deps(opts): - global srpm_dependencies_direct - - if opts.srpm_dependency_file == None: +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# The following tries to choose the best mockchain-parallel-* implementation +# to use, based on the version of /usr/bin/mockchain +# +# We want to use a compatable API, and to use the same python version. +# + +interpreter_path () { + local path=${1} + if [ ! -f ${path} ]; then + return 1 + fi + readlink -f $(head -n 1 ${path} | sed 's/^#!//' | awk '{ print $1 }' ) +} + +get__version__ () { + local path=${1} + if [ ! -f ${path} ]; then + return 1 + fi + grep __VERSION__= ${path} | cut -d '=' -f 2 | sed 's/"//g' +} + +VC_LESS_THAN=0 +VC_EQUAL=1 +VC_GREATER_THAN=2 +ver_comp () { + local v1=${1} + local v2=${2} + local v_greater="" + + if [ "${v1}" == "${v2}" ]; then + echo $VC_EQUAL return + fi - if not os.path.exists(opts.srpm_dependency_file): - log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) - sys.exit(1) - - with open(opts.srpm_dependency_file) as f: - lines = f.readlines() - for line in lines: - (name,deps) = line.rstrip().split(';') - srpm_dependencies_direct[name]=deps.split(',') - -def read_rpm_deps(opts): - global rpm_dependencies_direct - - if opts.rpm_dependency_file == None: + v_greater=$((echo ${v1}; echo ${v2}) | sort -rV | head -n 1) + if [ "${v1}" == "${v_greater}" ]; then + echo $VC_GREATER_THAN return + fi - if not os.path.exists(opts.rpm_dependency_file): - log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) - sys.exit(1) + echo $VC_LESS_THAN +} - with open(opts.rpm_dependency_file) as f: - lines = f.readlines() - for line in lines: - (name,deps) = line.rstrip().split(';') - rpm_dependencies_direct[name]=deps.split(',') +MOCKCHAIN_PATH="/usr/bin/mockchain" +MOCKCHAIN_PARALLEL_PATH_ROOT="${MY_REPO}/build-tools/mockchain-parallel" +DEFAULT_MOCKCHAIN_PARALLEL_PATH="${MOCKCHAIN_PARALLEL_PATH_ROOT}-1.3.4" -def read_map_deps(opts): - global rpm_to_srpm_map +MOCKCHAIN_INTERPRETER_PATH=$(interpreter_path ${MOCKCHAIN_PATH}) +MOCKCHAIN_VER=$(get__version__ ${MOCKCHAIN_PATH}) +if [ -z "${MOCKCHAIN_VER}" ]; then + echo "Error: Failed to determine version of '${MOCKCHAIN_PATH}'" + exit 1 +fi - if opts.rpm_to_srpm_map_file == None: - return +BEST_VER="" +BEST_MOCKCHAIN_PARALLEL_PATH="" - if not os.path.exists(opts.rpm_to_srpm_map_file): - log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) - sys.exit(1) - - with open(opts.rpm_to_srpm_map_file) as f: - lines = f.readlines() - for line in lines: - (rpm,srpm) = line.rstrip().split(';') - rpm_to_srpm_map[rpm]=srpm - - -def reaper(opts): - global built_pkgs - global failed - global worker_data - global workers - - reaped = 0 - need_createrepo = False - last_reaped = -1 - while reaped > last_reaped: - last_reaped = reaped - for wd in worker_data: - p = wd['proc'] - ret = p.exitcode - if ret is not None: - pkg = wd['pkg'] - b = int(wd['build_index']) - p.join() - worker_data.remove(wd) - workers = workers - 1 - reaped = reaped + 1 - release_build_env(b) - - log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) - - if ret == 0: - failed.append(pkg) - log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) - if opts.recurse and not stop_signal: - log(opts.logfile, "Will try to build again (if some other package will succeed).") - else: - log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) - elif ret == 1: - log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) - built_pkgs.append(pkg) - need_createrepo = True - elif ret == 2: - log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) - - if need_createrepo: - # createrepo with the new pkgs - err = createrepo(opts.local_repo_dir)[1] - if err.strip(): - log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) - log(opts.logfile, "Err: %s" % err) - - return reaped - -stop_signal = False - -def on_terminate(proc): - print("process {} terminated with exit code {}".format(proc, proc.returncode)) - -def kill_proc_and_descentents(parent, need_stop=False, verbose=False): - global g_opts - - if need_stop: - if verbose: - log(g_opts.logfile, "Stop %d" % parent.pid) - - try: - parent.send_signal(signal.SIGSTOP) - except: - # perhaps mock still running as root, give it a sec to drop pivledges and try again - time.sleep(1) - parent.send_signal(signal.SIGSTOP) - - children = parent.children(recursive=False) - - for p in children: - kill_proc_and_descentents(p, need_stop=True, verbose=verbose) - - if verbose: - log(g_opts.logfile, "Terminate %d" % parent.pid) - - # parent.send_signal(signal.SIGTERM) - try: - parent.terminate() - except: - # perhaps mock still running as root, give it a sec to drop pivledges and try again - time.sleep(1) - parent.terminate() - - if need_stop: - if verbose: - log(g_opts.logfile, "Continue %d" % parent.pid) - - parent.send_signal(signal.SIGCONT) - - -def child_signal_handler(signum, frame): - global g_opts - my_pid = os.getpid() - # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) - p = psutil.Process(my_pid) - kill_proc_and_descentents(p) - try: - sys.exit(0) - except SystemExit as e: - os._exit(0) - -def signal_handler(signum, frame): - global g_opts - global stop_signal - global workers - global worker_data - stop_signal = True - - # Signal processes to complete - log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) - for wd in worker_data: - p = wd['proc'] - ret = p.exitcode - if ret is None: - # log(g_opts.logfile, "terminate child %d" % p.pid) - p.terminate() - else: - log(g_opts.logfile, "child return code was %d" % ret) - - # Wait for remaining processes to complete - log(g_opts.logfile, "===== wait for signaled jobs to complete =====") - while len(worker_data) > 0: - log(g_opts.logfile, " remaining workers: %d" % workers) - reaped = reaper(g_opts) - if reaped == 0: - time.sleep(0.1) - - try: - sys.exit(1) - except SystemExit as e: - os._exit(1) - -def main(args): - opts, args = parse_args(args) - # take mock config + list of pkgs - - global g_opts - global stop_signal - global build_env - global worker_data - global workers - global max_workers - - global slow_pkg_names - global slow_pkgs - global big_pkg_names - global big_pkgs - max_workers = int(opts.max_workers) - - global failed - global built_pkgs - - cfg = opts.chroot - pkgs = args[1:] - - # transform slow/big package options into dictionaries - for line in opts.slow_pkg_names_raw: - speed,name = line.split(":") - if speed != "": - slow_pkg_names[name]=int(speed) - for line in opts.slow_pkgs_raw: - speed,pkg = line.split(":") - if speed != "": - slow_pkgs[pkg]=int(speed) - for line in opts.big_pkg_names_raw: - size_gb,name = line.split(":") - if size_gb != "": - big_pkg_names[name]=int(size_gb) - for line in opts.big_pkgs_raw: - size_gb,pkg = line.split(":") - if size_gb != "": - big_pkgs[pkg]=int(size_gb) - - # Set up a mapping between pkg path and pkg name - global pkg_to_name - global name_to_pkg - for pkg in pkgs: - if not pkg.endswith('.rpm'): - log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) +for MOCKCHAIN_PARALLEL_PATH in $(ls -1 ${MOCKCHAIN_PARALLEL_PATH_ROOT}-*); do + MOCKCHAIN_PARALLEL_VER=$(get__version__ ${MOCKCHAIN_PARALLEL_PATH}) + if [ -z "${MOCKCHAIN_PARALLEL_VER}" ]; then + echo "Warning: Failed to determine version of '${MOCKCHAIN_PARALLEL_PATH}'" + continue + fi + COMP=$(ver_comp "${MOCKCHAIN_VER}" "${MOCKCHAIN_PARALLEL_VER}") + echo $MOCKCHAIN_PARALLEL_PATH $MOCKCHAIN_PARALLEL_VER $COMP + if [ $COMP -eq $VC_EQUAL ]; then + BEST_VER=${MOCKCHAIN_PARALLEL_VER} + BEST_MOCKCHAIN_PARALLEL_PATH=${MOCKCHAIN_PARALLEL_PATH} + break + fi + if [ $COMP -gt $VC_EQUAL ]; then + if [ "${BEST_VER}" == "" ]; then + BEST_VER=${MOCKCHAIN_PARALLEL_VER} + BEST_MOCKCHAIN_PARALLEL_PATH=${MOCKCHAIN_PARALLEL_PATH} continue + fi - try: - name = rpmName(pkg) - except OSError as e: - print("Could not parse rpm %s" % pkg) - sys.exit(1) + COMP=$(ver_comp ${MOCKCHAIN_PARALLEL_VER} ${BEST_VER}) + if [ $COMP -gt $VC_EQUAL ]; then + BEST_VER=${MOCKCHAIN_PARALLEL_VER} + BEST_MOCKCHAIN_PARALLEL_PATH=${MOCKCHAIN_PARALLEL_PATH} + fi + fi +done - pkg_to_name[pkg] = name - name_to_pkg[name] = pkg +MOCKCHAIN_PARALLEL_INTERPRETER_PATH=${BEST_MOCKCHAIN_PARALLEL_INTERPRETER_PATH} +MOCKCHAIN_PARALLEL_PATH=${BEST_MOCKCHAIN_PARALLEL_PATH} - read_deps(opts) +if [ -z "${MOCKCHAIN_PARALLEL_PATH}" ]; then + MOCKCHAIN_PARALLEL_PATH="${DEFAULT_MOCKCHAIN_PARALLEL_PATH}" +fi - global config_opts - config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) - - if not opts.tmp_prefix: - try: - opts.tmp_prefix = os.getlogin() - except OSError as e: - print("Could not find login name for tmp dir prefix add --tmp_prefix") - sys.exit(1) - pid = os.getpid() - opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) - - if opts.basedir != "/var/lib/mock": - opts.uniqueext = '' - - # create a tempdir for our local info - if opts.localrepo: - local_tmp_dir = os.path.abspath(opts.localrepo) - if not os.path.exists(local_tmp_dir): - os.makedirs(local_tmp_dir) - os.chmod(local_tmp_dir, 0o755) - else: - pre = 'mock-chain-%s-' % opts.uniqueext - local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') - os.chmod(local_tmp_dir, 0o755) - - if opts.logfile: - opts.logfile = os.path.join(local_tmp_dir, opts.logfile) - if os.path.exists(opts.logfile): - os.unlink(opts.logfile) - - log(opts.logfile, "starting logfile: %s" % opts.logfile) - - opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') - - if not os.path.exists(opts.local_repo_dir): - os.makedirs(opts.local_repo_dir, mode=0o755) - - local_baseurl = "file://%s" % opts.local_repo_dir - log(opts.logfile, "results dir: %s" % opts.local_repo_dir) - opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') - - if not os.path.exists(opts.config_path): - os.makedirs(opts.config_path, mode=0o755) - - log(opts.logfile, "config dir: %s" % opts.config_path) - - my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) - - # modify with localrepo - res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') - if not res: - log(opts.logfile, "Error: Could not write out local config: %s" % msg) - sys.exit(1) - - for baseurl in opts.repos: - res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) - if not res: - log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) - sys.exit(1) - - res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) - if not res: - log(opts.logfile, "Error: Could not write out local config: %s" % msg) - sys.exit(1) - - # these files needed from the mock.config dir to make mock run - for fn in ['site-defaults.cfg', 'logging.ini']: - pth = mockconfig_path + '/' + fn - shutil.copyfile(pth, opts.config_path + '/' + fn) - - # createrepo on it - err = createrepo(opts.local_repo_dir)[1] - if err.strip(): - log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) - log(opts.logfile, "Err: %s" % err) - sys.exit(1) - - init_build_env(max_workers, opts, config_opts) - - download_dir = tempfile.mkdtemp() - downloaded_pkgs = {} - built_pkgs = [] - try_again = True - to_be_built = pkgs - return_code = 0 - num_of_tries = 0 - - g_opts = opts - signal.signal(signal.SIGTERM, signal_handler) - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGHUP, signal_handler) - signal.signal(signal.SIGABRT, signal_handler) - - while try_again and not stop_signal: - num_of_tries += 1 - failed = [] - - log(opts.logfile, "===== iteration %d start =====" % num_of_tries) - - to_be_built_scheduled = to_be_built[:] - - need_reap = False - while len(to_be_built_scheduled) > 0: - # Free up a worker - while need_reap or workers >= max_workers: - need_reap = False - reaped = reaper(opts) - if reaped == 0: - time.sleep(0.1) - - if workers < max_workers: - workers = workers + 1 - - b = get_idle_build_env(max_workers) - if b < 0: - log(opts.logfile, "Failed to find idle build env for: %s" % pkg) - workers = workers - 1 - need_reap = True - continue - - pkg = schedule(b, to_be_built_scheduled, opts) - if pkg is None: - if workers <= 1: - # Remember we have one build environmnet reserved, so can't test for zero workers - log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) - pkg = to_be_built_scheduled[0] - log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) - else: - release_build_env(b) - workers = workers - 1 - need_reap = True - continue - - to_be_built_scheduled.remove(pkg) - - if not pkg.endswith('.rpm'): - log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) - failed.append(pkg) - release_build_env(b) - need_reap = True - continue - - elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): - url = pkg - try: - log(opts.logfile, 'Fetching %s' % url) - r = requests.get(url) - # pylint: disable=no-member - if r.status_code == requests.codes.ok: - fn = urlsplit(r.url).path.rsplit('/', 1)[1] - if 'content-disposition' in r.headers: - _, params = cgi.parse_header(r.headers['content-disposition']) - if 'filename' in params and params['filename']: - fn = params['filename'] - pkg = download_dir + '/' + fn - with open(pkg, 'wb') as fd: - for chunk in r.iter_content(4096): - fd.write(chunk) - except Exception as e: - log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) - failed.append(url) - release_build_env(b) - need_reap = True - continue - else: - downloaded_pkgs[pkg] = url - - log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) - # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] - p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) - worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) - p.start() - - # Wait for remaining processes to complete - log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) - while workers > 0: - reaped = reaper(opts) - if reaped == 0: - time.sleep(0.1) - log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) - - if failed and opts.recurse: - log(opts.logfile, "failed=%s" % failed) - log(opts.logfile, "to_be_built=%s" % to_be_built) - if len(failed) != len(to_be_built): - to_be_built = failed - try_again = True - log(opts.logfile, 'Some package succeeded, some failed.') - log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) - else: - if max_workers > 1: - max_workers = 1 - to_be_built = failed - try_again = True - log(opts.logfile, 'Some package failed under parallel build.') - log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) - else: - log(opts.logfile, "") - log(opts.logfile, "*** Build Failed ***") - log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) - log(opts.logfile, "*** Build Failed ***") - for pkg in failed: - msg = pkg - if pkg in downloaded_pkgs: - msg = downloaded_pkgs[pkg] - log(opts.logfile, msg) - log(opts.logfile, "") - try_again = False - else: - try_again = False - if failed: - return_code = 2 - - # cleaning up our download dir - shutil.rmtree(download_dir, ignore_errors=True) - - log(opts.logfile, "") - log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) - log(opts.logfile, "") - log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) - if built_pkgs: - if failed: - if len(built_pkgs): - log(opts.logfile, "Some packages successfully built in this order:") - else: - log(opts.logfile, "Packages successfully built in this order:") - for pkg in built_pkgs: - log(opts.logfile, pkg) - return return_code - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) +echo "PYTHONDONTWRITEBYTECODE=true exec ${MOCKCHAIN_INTERPRETER_PATH} ${MOCKCHAIN_PARALLEL_PATH} $@" +PYTHONDONTWRITEBYTECODE=true exec ${MOCKCHAIN_INTERPRETER_PATH} ${MOCKCHAIN_PARALLEL_PATH} "$@" diff --git a/build-tools/mockchain-parallel-1.3.4 b/build-tools/mockchain-parallel-1.3.4 new file mode 100755 index 00000000..ef6d8873 --- /dev/null +++ b/build-tools/mockchain-parallel-1.3.4 @@ -0,0 +1,1206 @@ +#!/usr/bin/python2.7 -tt +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built +from __future__ import print_function + +import cgi +# pylint: disable=deprecated-module +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import multiprocessing +import signal +import psutil + +import requests +# pylint: disable=import-error +from six.moves.urllib_parse import urlsplit + +import mockbuild.util + +from stxRpmUtils import splitRpmFilename + +# all of the variables below are substituted by the build system +__VERSION__="1.3.4" +SYSCONFDIR="/etc" +PYTHONDIR="/usr/lib/python2.7/site-packages" +PKGPYTHONDIR="/usr/lib/python2.7/site-packages/mockbuild" +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs + +mockconfig_path = '/etc/mock' + +def rpmName(path): + filename = os.path.basename(path) + (n, v, r, e, a) = splitRpmFilename(filename) + return n + +def createrepo(path): + global max_workers + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] + else: + comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] + cmd = subprocess.Popen( + comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + + +g_opts = optparse.Values() + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option( + '-r', '--root', default=None, dest='chroot', + metavar="CONFIG", + help="chroot config name/base to use in the mock build") + parser.add_option( + '-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option( + '-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option( + '--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option( + '--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option( + '--workers', default=1, dest='max_workers', + help="number of parallel build jobs") + parser.add_option( + '--worker-resources', default="", dest='worker_resources', + help="colon seperated list, how much mem in gb for each workers temfs") + parser.add_option( + '--basedir', default='/var/lib/mock', dest='basedir', + help="path to workspace") + parser.add_option( + '--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + parser.add_option( + '-m', '--mock-option', default=[], action='append', + dest='mock_option', + help="option to pass directly to mock") + parser.add_option( + '--mark-slow-name', default=[], action='append', + dest='slow_pkg_names_raw', + help="package name that is known to build slowly") + parser.add_option( + '--mark-slow-path', default=[], action='append', + dest='slow_pkgs_raw', + help="package path that is known to build slowly") + parser.add_option( + '--mark-big-name', default=[], action='append', + dest='big_pkg_names_raw', + help="package name that is known to require a lot of disk space to build") + parser.add_option( + '--mark-big-path', default=[], action='append', + dest='big_pkgs_raw', + help="package path that is known to require a lot of disk space to build") + parser.add_option( + '--srpm-dependency-file', default=None, + dest='srpm_dependency_file', + help="path to srpm dependency file") + parser.add_option( + '--rpm-dependency-file', default=None, + dest='rpm_dependency_file', + help="path to rpm dependency file") + parser.add_option( + '--rpm-to-srpm-map-file', default=None, + dest='rpm_to_srpm_map_file', + help="path to rpm to srpm map file") + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print("You must provide an argument to -r for the mock chroot") + sys.exit(1) + + if len(sys.argv) < 3: + print("You must specify at least 1 package to build") + sys.exit(1) + + return opts, args + + +REPOS_ID = [] + +slow_pkg_names={} +slow_pkgs={} +big_pkg_names={} +big_pkgs={} + +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): + # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) + # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) + # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) + if tmpfs_size_gb > 0: + config_opts['plugin_conf']['tmpfs_enable'] = True + config_opts['plugin_conf']['tmpfs_opts'] = {} + config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 + config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb + config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' + config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) + if not os.path.isdir(config_opts['cache_topdir']): + os.makedirs(config_opts['cache_topdir'], exist_ok=True) + except (IOError, OSError): + return False, "Could not create dir: %s" % config_opts['cache_topdir'] + + cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + except (IOError, OSError): + return False, "Could not create dir: %s" % cache_dir + + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def set_basedir(infile, destfile, basedir, opts): + log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['basedir'] = basedir + config_opts['resultdir'] = '{0}/result'.format(basedir) + config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) + config_opts['root'] = 'mock/b0' + config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + global config_opts + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 +cost=1 +best=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + signal.signal(signal.SIGTERM, child_signal_handler) + signal.signal(signal.SIGINT, child_signal_handler) + signal.signal(signal.SIGHUP, child_signal_handler) + signal.signal(signal.SIGABRT, child_signal_handler) + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + # return 2, None, None, None + sys.exit(2) + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + if opts.uniqueext == '': + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '-r', cfg, ] + else: + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '-r', cfg, ] + # heuristic here, if user pass for mock "-d foo", but we must be care to leave + # "-d'foo bar'" or "--define='foo bar'" as is + compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') + compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') + for option in opts.mock_option: + r_match = compiled_re_1.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + r_match = compiled_re_2.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + mockcmd.append(option) + + print('building %s' % s_pkg) + mockcmd.append(pkg) + # print("mockcmd: %s" % str(mockcmd)) + cmd = subprocess.Popen( + mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode == 0: + with open(success_file, 'w') as f: + f.write('done\n') + ret = 1 + else: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + with open(fail_file, 'w') as f: + f.write('undone\n') + ret = 0 + + # return ret, cmd, out, err + sys.exit(ret) + + +def log(lf, msg): + if lf: + now = time.time() + try: + with open(lf, 'a') as f: + f.write(str(now) + ':' + msg + '\n') + except (IOError, OSError) as e: + print('Could not write to logfile %s - %s' % (lf, str(e))) + print(msg) + + +config_opts = {} + +worker_data = [] +workers = 0 +max_workers = 1 + +build_env = [] + +failed = [] +built_pkgs = [] + +local_repo_dir = "" + +pkg_to_name={} +name_to_pkg={} +srpm_dependencies_direct={} +rpm_dependencies_direct={} +rpm_to_srpm_map={} +no_dep_list = [ "bash", "kernel" , "kernel-rt" ] + + +def init_build_env(slots, opts, config_opts_in): + global build_env + + orig_chroot_name=config_opts_in['chroot_name'] + orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) + # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) + for i in range(0,slots): + new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) + new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) + tmpfs_size_gb = 0 + if opts.worker_resources == "": + if i > 0: + tmpfs_size_gb = 2 * (1 + slots - i) + else: + resource_array=opts.worker_resources.split(':') + if i < len(resource_array): + tmpfs_size_gb=int(resource_array[i]) + else: + log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) + sys.exit(1) + if i == 0 and tmpfs_size_gb != 0: + log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) + sys.exit(1) + build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) + + res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + +idle_build_env_last_awarded = 0 +def get_idle_build_env(slots): + global build_env + global idle_build_env_last_awarded + visited = 0 + + if slots < 1: + return -1 + + i = idle_build_env_last_awarded - 1 + if i < 0 or i >= slots: + i = slots - 1 + + while visited < slots: + if build_env[i]['state'] == 'Idle': + build_env[i]['state'] = 'Busy' + idle_build_env_last_awarded = i + return i + visited = visited + 1 + i = i - 1 + if i < 0: + i = slots - 1 + return -1 + +def release_build_env(idx): + global build_env + + build_env[idx]['state'] = 'Idle' + +def get_best_rc(a, b): + print("get_best_rc: a=%s" % str(a)) + print("get_best_rc: b=%s" % str(b)) + if (b == {}) and (a != {}): + return a + if (a == {}) and (b != {}): + return b + + if (b['build_name'] is None) and (not a['build_name'] is None): + return a + if (a['build_name'] is None) and (not b['build_name'] is None): + return b + + if a['unbuilt_deps'] < b['unbuilt_deps']: + return a + if b['unbuilt_deps'] < a['unbuilt_deps']: + return b + + if a['depth'] < b['depth']: + return a + if b['depth'] < a['depth']: + return b + + print("get_best_rc: uncertain %s vs %s" % (a,b)) + return a + +unbuilt_dep_list_print=False +def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): + global srpm_dependencies_direct + global rpm_dependencies_direct + global rpm_to_srpm_map + global no_dep_list + global unbuilt_dep_list_print + + first_iteration=False + unbuilt = [] + if name in no_dep_list: + return unbuilt + + if checked is None: + first_iteration=True + checked=[] + + # Count unbuild dependencies + if first_iteration: + dependencies_direct=srpm_dependencies_direct + else: + dependencies_direct=rpm_dependencies_direct + + if name in dependencies_direct: + for rdep in dependencies_direct[name]: + sdep='???' + if rdep in rpm_to_srpm_map: + sdep = rpm_to_srpm_map[rdep] + if rdep != name and sdep != name and not rdep in checked: + if (not first_iteration) and (sdep in no_dep_list): + continue + checked.append(rdep) + if sdep in unbuilt_pkg_names: + if not sdep in unbuilt: + unbuilt.append(sdep) + if depth > 0: + child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) + for sub_sdep in child_unbuilt: + if sub_sdep != name: + if not sub_sdep in unbuilt: + unbuilt.append(sub_sdep) + + return unbuilt + +def can_build_at_idx(build_idx, name, opts): + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + global build_env + + fs_size_gb = 0 + size_gb = 0 + speed = 0 + pkg = name_to_pkg[name] + if name in big_pkg_names: + size_gb=big_pkg_names[name] + if pkg in big_pkgs: + size_gb=big_pkgs[pkg] + if name in slow_pkg_names: + speed=slow_pkg_names[name] + if pkg in slow_pkgs: + speed=slow_pkgs[pkg] + fs_size_gb = build_env[build_idx]['fs_size_gb'] + return fs_size_gb == 0 or fs_size_gb >= size_gb + +def schedule(build_idx, pkgs, opts): + global worker_data + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + + unbuilt_pkg_names=[] + building_pkg_names=[] + unprioritized_pkg_names=[] + + for pkg in pkgs: + name = pkg_to_name[pkg] + unbuilt_pkg_names.append(name) + unprioritized_pkg_names.append(name) + + prioritized_pkg_names=[] + + for wd in worker_data: + pkg = wd['pkg'] + if not pkg is None: + name = pkg_to_name[pkg] + building_pkg_names.append(name) + + # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) + if len(big_pkg_names) or len(big_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in big_pkg_names or pkg in big_pkgs: + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + if len(slow_pkg_names) or len(slow_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in slow_pkg_names or pkg in slow_pkgs: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + for name in unprioritized_pkg_names: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + + name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) + if not name_out is None: + pkg_out = name_to_pkg[name_out] + else: + pkg_out = None + # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) + # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) + return pkg_out + + +def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): + global pkg_to_name + global name_to_pkg + global no_dep_list + + max_depth = 3 + + if len(pkg_names) == 0: + return None + + unbuilt_deps={} + building_deps={} + for depth in range(max_depth,-1,-1): + unbuilt_deps[depth]={} + building_deps[depth]={} + + for depth in range(max_depth,-1,-1): + checked=[] + reordered_pkg_names = pkg_names[:] + # for name in reordered_pkg_names: + while len(reordered_pkg_names): + name = reordered_pkg_names.pop(0) + if name in checked: + continue + + # log(opts.logfile, "checked.append(%s)" % name) + checked.append(name) + + pkg = name_to_pkg[name] + # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) + if not name in unbuilt_deps[depth]: + unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) + if not name in building_deps[depth]: + building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) + if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + if not name in unbuilt_deps[0]: + unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) + if not name in building_deps[0]: + building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) + if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): + if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + loop = False + for dep_name in unbuilt_deps[depth][name]: + if name == dep_name: + continue + + # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) + if dep_name in checked: + continue + + # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) + if not dep_name in unbuilt_deps[depth]: + unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) + if not dep_name in building_deps[depth]: + building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) + if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if not dep_name in unbuilt_deps[0]: + unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) + if not dep_name in building_deps[0]: + building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) + if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): + if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if name in unbuilt_deps[0][dep_name]: + loop = True + # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) + + if loop and len(building_deps[depth][name]) == 0: + log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) + return name + + for dep_name in unbuilt_deps[depth][name]: + if dep_name in reordered_pkg_names: + # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) + reordered_pkg_names.remove(dep_name) + reordered_pkg_names.insert(0,dep_name) + + # log(opts.logfile, "schedule2: Nothing buildable at this time") + return None + + + + +def read_deps(opts): + read_srpm_deps(opts) + read_rpm_deps(opts) + read_map_deps(opts) + +def read_srpm_deps(opts): + global srpm_dependencies_direct + + if opts.srpm_dependency_file == None: + return + + if not os.path.exists(opts.srpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) + sys.exit(1) + + with open(opts.srpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + srpm_dependencies_direct[name]=deps.split(',') + +def read_rpm_deps(opts): + global rpm_dependencies_direct + + if opts.rpm_dependency_file == None: + return + + if not os.path.exists(opts.rpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) + sys.exit(1) + + with open(opts.rpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + rpm_dependencies_direct[name]=deps.split(',') + +def read_map_deps(opts): + global rpm_to_srpm_map + + if opts.rpm_to_srpm_map_file == None: + return + + if not os.path.exists(opts.rpm_to_srpm_map_file): + log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) + sys.exit(1) + + with open(opts.rpm_to_srpm_map_file) as f: + lines = f.readlines() + for line in lines: + (rpm,srpm) = line.rstrip().split(';') + rpm_to_srpm_map[rpm]=srpm + + +def reaper(opts): + global built_pkgs + global failed + global worker_data + global workers + + reaped = 0 + need_createrepo = False + last_reaped = -1 + while reaped > last_reaped: + last_reaped = reaped + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is not None: + pkg = wd['pkg'] + b = int(wd['build_index']) + p.join() + worker_data.remove(wd) + workers = workers - 1 + reaped = reaped + 1 + release_build_env(b) + + log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) + + if ret == 0: + failed.append(pkg) + log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) + if opts.recurse and not stop_signal: + log(opts.logfile, "Will try to build again (if some other package will succeed).") + else: + log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) + elif ret == 1: + log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) + built_pkgs.append(pkg) + need_createrepo = True + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if need_createrepo: + # createrepo with the new pkgs + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + + return reaped + +stop_signal = False + +def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + +def kill_proc_and_descentents(parent, need_stop=False, verbose=False): + global g_opts + + if need_stop: + if verbose: + log(g_opts.logfile, "Stop %d" % parent.pid) + + try: + parent.send_signal(signal.SIGSTOP) + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.send_signal(signal.SIGSTOP) + + children = parent.children(recursive=False) + + for p in children: + kill_proc_and_descentents(p, need_stop=True, verbose=verbose) + + if verbose: + log(g_opts.logfile, "Terminate %d" % parent.pid) + + # parent.send_signal(signal.SIGTERM) + try: + parent.terminate() + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.terminate() + + if need_stop: + if verbose: + log(g_opts.logfile, "Continue %d" % parent.pid) + + parent.send_signal(signal.SIGCONT) + + +def child_signal_handler(signum, frame): + global g_opts + my_pid = os.getpid() + # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) + p = psutil.Process(my_pid) + kill_proc_and_descentents(p) + try: + sys.exit(0) + except SystemExit as e: + os._exit(0) + +def signal_handler(signum, frame): + global g_opts + global stop_signal + global workers + global worker_data + stop_signal = True + + # Signal processes to complete + log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is None: + # log(g_opts.logfile, "terminate child %d" % p.pid) + p.terminate() + else: + log(g_opts.logfile, "child return code was %d" % ret) + + # Wait for remaining processes to complete + log(g_opts.logfile, "===== wait for signaled jobs to complete =====") + while len(worker_data) > 0: + log(g_opts.logfile, " remaining workers: %d" % workers) + reaped = reaper(g_opts) + if reaped == 0: + time.sleep(0.1) + + try: + sys.exit(1) + except SystemExit as e: + os._exit(1) + +def main(args): + opts, args = parse_args(args) + # take mock config + list of pkgs + + global g_opts + global stop_signal + global build_env + global worker_data + global workers + global max_workers + + global slow_pkg_names + global slow_pkgs + global big_pkg_names + global big_pkgs + max_workers = int(opts.max_workers) + + global failed + global built_pkgs + + cfg = opts.chroot + pkgs = args[1:] + + # transform slow/big package options into dictionaries + for line in opts.slow_pkg_names_raw: + speed,name = line.split(":") + if speed != "": + slow_pkg_names[name]=int(speed) + for line in opts.slow_pkgs_raw: + speed,pkg = line.split(":") + if speed != "": + slow_pkgs[pkg]=int(speed) + for line in opts.big_pkg_names_raw: + size_gb,name = line.split(":") + if size_gb != "": + big_pkg_names[name]=int(size_gb) + for line in opts.big_pkgs_raw: + size_gb,pkg = line.split(":") + if size_gb != "": + big_pkgs[pkg]=int(size_gb) + + # Set up a mapping between pkg path and pkg name + global pkg_to_name + global name_to_pkg + for pkg in pkgs: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + continue + + try: + name = rpmName(pkg) + except OSError as e: + print("Could not parse rpm %s" % pkg) + sys.exit(1) + + pkg_to_name[pkg] = name + name_to_pkg[name] = pkg + + read_deps(opts) + + global config_opts + config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError as e: + print("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + if opts.basedir != "/var/lib/mock": + opts.uniqueext = '' + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + os.chmod(local_tmp_dir, 0o755) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + os.chmod(local_tmp_dir, 0o755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0o755) + + local_baseurl = "file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0o755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) + + # modify with localrepo + res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + # createrepo on it + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + sys.exit(1) + + init_build_env(max_workers, opts, config_opts) + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + return_code = 0 + num_of_tries = 0 + + g_opts = opts + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + signal.signal(signal.SIGABRT, signal_handler) + + while try_again and not stop_signal: + num_of_tries += 1 + failed = [] + + log(opts.logfile, "===== iteration %d start =====" % num_of_tries) + + to_be_built_scheduled = to_be_built[:] + + need_reap = False + while len(to_be_built_scheduled) > 0: + # Free up a worker + while need_reap or workers >= max_workers: + need_reap = False + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + + if workers < max_workers: + workers = workers + 1 + + b = get_idle_build_env(max_workers) + if b < 0: + log(opts.logfile, "Failed to find idle build env for: %s" % pkg) + workers = workers - 1 + need_reap = True + continue + + pkg = schedule(b, to_be_built_scheduled, opts) + if pkg is None: + if workers <= 1: + # Remember we have one build environmnet reserved, so can't test for zero workers + log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) + pkg = to_be_built_scheduled[0] + log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) + else: + release_build_env(b) + workers = workers - 1 + need_reap = True + continue + + to_be_built_scheduled.remove(pkg) + + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + release_build_env(b) + need_reap = True + continue + + elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): + url = pkg + try: + log(opts.logfile, 'Fetching %s' % url) + r = requests.get(url) + # pylint: disable=no-member + if r.status_code == requests.codes.ok: + fn = urlsplit(r.url).path.rsplit('/', 1)[1] + if 'content-disposition' in r.headers: + _, params = cgi.parse_header(r.headers['content-disposition']) + if 'filename' in params and params['filename']: + fn = params['filename'] + pkg = download_dir + '/' + fn + with open(pkg, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + except Exception as e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + release_build_env(b) + need_reap = True + continue + else: + downloaded_pkgs[pkg] = url + + log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) + # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] + p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) + worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) + p.start() + + # Wait for remaining processes to complete + log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) + while workers > 0: + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) + + if failed and opts.recurse: + log(opts.logfile, "failed=%s" % failed) + log(opts.logfile, "to_be_built=%s" % to_be_built) + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package succeeded, some failed.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) + else: + if max_workers > 1: + max_workers = 1 + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package failed under parallel build.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) + else: + log(opts.logfile, "") + log(opts.logfile, "*** Build Failed ***") + log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) + log(opts.logfile, "*** Build Failed ***") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + log(opts.logfile, "") + try_again = False + else: + try_again = False + if failed: + return_code = 2 + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "") + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "") + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log(opts.logfile, "Some packages successfully built in this order:") + else: + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build-tools/mockchain-parallel-1.4.16 b/build-tools/mockchain-parallel-1.4.16 new file mode 100755 index 00000000..ca8ec2b5 --- /dev/null +++ b/build-tools/mockchain-parallel-1.4.16 @@ -0,0 +1,1213 @@ +#!/usr/bin/python3.6 -tt +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built +from __future__ import print_function + +import cgi +# pylint: disable=deprecated-module +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import multiprocessing +import signal +import psutil + +import requests +# pylint: disable=import-error +from six.moves.urllib_parse import urlsplit + +import mockbuild.util + +from stxRpmUtils import splitRpmFilename + + +# all of the variables below are substituted by the build system +__VERSION__="1.4.16" +SYSCONFDIR="/etc" +PYTHONDIR="/usr/lib/python3.6/site-packages" +PKGPYTHONDIR="/usr/lib/python3.6/site-packages/mockbuild" +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs + +mockconfig_path = '/etc/mock' + +def rpmName(path): + filename = os.path.basename(path) + (n, v, r, e, a) = splitRpmFilename(filename) + return n + +def createrepo(path): + global max_workers + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] + else: + comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] + cmd = subprocess.Popen( + comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + + +g_opts = optparse.Values() + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option( + '-r', '--root', default=None, dest='chroot', + metavar="CONFIG", + help="chroot config name/base to use in the mock build") + parser.add_option( + '-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option( + '-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option( + '--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option( + '--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option( + '--workers', default=1, dest='max_workers', + help="number of parallel build jobs") + parser.add_option( + '--worker-resources', default="", dest='worker_resources', + help="colon seperated list, how much mem in gb for each workers temfs") + parser.add_option( + '--basedir', default='/var/lib/mock', dest='basedir', + help="path to workspace") + parser.add_option( + '--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + parser.add_option( + '-m', '--mock-option', default=[], action='append', + dest='mock_option', + help="option to pass directly to mock") + parser.add_option( + '--mark-slow-name', default=[], action='append', + dest='slow_pkg_names_raw', + help="package name that is known to build slowly") + parser.add_option( + '--mark-slow-path', default=[], action='append', + dest='slow_pkgs_raw', + help="package path that is known to build slowly") + parser.add_option( + '--mark-big-name', default=[], action='append', + dest='big_pkg_names_raw', + help="package name that is known to require a lot of disk space to build") + parser.add_option( + '--mark-big-path', default=[], action='append', + dest='big_pkgs_raw', + help="package path that is known to require a lot of disk space to build") + parser.add_option( + '--srpm-dependency-file', default=None, + dest='srpm_dependency_file', + help="path to srpm dependency file") + parser.add_option( + '--rpm-dependency-file', default=None, + dest='rpm_dependency_file', + help="path to rpm dependency file") + parser.add_option( + '--rpm-to-srpm-map-file', default=None, + dest='rpm_to_srpm_map_file', + help="path to rpm to srpm map file") + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print("You must provide an argument to -r for the mock chroot") + sys.exit(1) + + if len(sys.argv) < 3: + print("You must specify at least 1 package to build") + sys.exit(1) + + return opts, args + + +REPOS_ID = [] + +slow_pkg_names={} +slow_pkgs={} +big_pkg_names={} +big_pkgs={} + +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): + # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) + # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) + # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) + if tmpfs_size_gb > 0: + config_opts['plugin_conf']['tmpfs_enable'] = True + config_opts['plugin_conf']['tmpfs_opts'] = {} + config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 + config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb + config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' + config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) + if not os.path.isdir(config_opts['cache_topdir']): + os.makedirs(config_opts['cache_topdir'], exist_ok=True) + except (IOError, OSError): + return False, "Could not create dir: %s" % config_opts['cache_topdir'] + + cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + except (IOError, OSError): + return False, "Could not create dir: %s" % cache_dir + + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def set_basedir(infile, destfile, basedir, opts): + log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['basedir'] = basedir + config_opts['resultdir'] = '{0}/result'.format(basedir) + config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) + config_opts['root'] = 'mock/b0' + config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + # pylint: disable=global-variable-not-assigned + global config_opts + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 +cost=1 +best=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + signal.signal(signal.SIGTERM, child_signal_handler) + signal.signal(signal.SIGINT, child_signal_handler) + signal.signal(signal.SIGHUP, child_signal_handler) + signal.signal(signal.SIGABRT, child_signal_handler) + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + # return 2, None, None, None + sys.exit(2) + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + if opts.uniqueext == '': + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '-r', cfg, ] + else: + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '-r', cfg, ] + # heuristic here, if user pass for mock "-d foo", but we must be care to leave + # "-d'foo bar'" or "--define='foo bar'" as is + compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') + compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') + for option in opts.mock_option: + r_match = compiled_re_1.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + r_match = compiled_re_2.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + mockcmd.append(option) + + print('building %s' % s_pkg) + mockcmd.append(pkg) + # print("mockcmd: %s" % str(mockcmd)) + cmd = subprocess.Popen( + mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode == 0: + with open(success_file, 'w') as f: + f.write('done\n') + ret = 1 + else: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + with open(fail_file, 'w') as f: + f.write('undone\n') + ret = 0 + + # return ret, cmd, out, err + sys.exit(ret) + + +def log(lf, msg): + if lf: + now = time.time() + try: + with open(lf, 'a') as f: + f.write(str(now) + ':' + msg + '\n') + except (IOError, OSError) as e: + print('Could not write to logfile %s - %s' % (lf, str(e))) + print(msg) + + +config_opts = mockbuild.util.TemplatedDictionary() + +worker_data = [] +workers = 0 +max_workers = 1 + +build_env = [] + +failed = [] +built_pkgs = [] + +local_repo_dir = "" + +pkg_to_name={} +name_to_pkg={} +srpm_dependencies_direct={} +rpm_dependencies_direct={} +rpm_to_srpm_map={} +no_dep_list = [ "bash", "kernel" , "kernel-rt" ] + + +def init_build_env(slots, opts, config_opts_in): + global build_env + + orig_chroot_name=config_opts_in['chroot_name'] + orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) + # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) + for i in range(0,slots): + new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) + new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) + tmpfs_size_gb = 0 + if opts.worker_resources == "": + if i > 0: + tmpfs_size_gb = 2 * (1 + slots - i) + else: + resource_array=opts.worker_resources.split(':') + if i < len(resource_array): + tmpfs_size_gb=int(resource_array[i]) + else: + log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) + sys.exit(1) + if i == 0 and tmpfs_size_gb != 0: + log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) + sys.exit(1) + build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) + + res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + +idle_build_env_last_awarded = 0 +def get_idle_build_env(slots): + global build_env + global idle_build_env_last_awarded + visited = 0 + + if slots < 1: + return -1 + + i = idle_build_env_last_awarded - 1 + if i < 0 or i >= slots: + i = slots - 1 + + while visited < slots: + if build_env[i]['state'] == 'Idle': + build_env[i]['state'] = 'Busy' + idle_build_env_last_awarded = i + return i + visited = visited + 1 + i = i - 1 + if i < 0: + i = slots - 1 + return -1 + +def release_build_env(idx): + global build_env + + build_env[idx]['state'] = 'Idle' + +def get_best_rc(a, b): + print("get_best_rc: a=%s" % str(a)) + print("get_best_rc: b=%s" % str(b)) + if (b == {}) and (a != {}): + return a + if (a == {}) and (b != {}): + return b + + if (b['build_name'] is None) and (not a['build_name'] is None): + return a + if (a['build_name'] is None) and (not b['build_name'] is None): + return b + + if a['unbuilt_deps'] < b['unbuilt_deps']: + return a + if b['unbuilt_deps'] < a['unbuilt_deps']: + return b + + if a['depth'] < b['depth']: + return a + if b['depth'] < a['depth']: + return b + + print("get_best_rc: uncertain %s vs %s" % (a,b)) + return a + +unbuilt_dep_list_print=False +def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): + global srpm_dependencies_direct + global rpm_dependencies_direct + global rpm_to_srpm_map + global no_dep_list + global unbuilt_dep_list_print + + first_iteration=False + unbuilt = [] + if name in no_dep_list: + return unbuilt + + if checked is None: + first_iteration=True + checked=[] + + # Count unbuild dependencies + if first_iteration: + dependencies_direct=srpm_dependencies_direct + else: + dependencies_direct=rpm_dependencies_direct + + if name in dependencies_direct: + for rdep in dependencies_direct[name]: + sdep='???' + if rdep in rpm_to_srpm_map: + sdep = rpm_to_srpm_map[rdep] + if rdep != name and sdep != name and not rdep in checked: + if (not first_iteration) and (sdep in no_dep_list): + continue + checked.append(rdep) + if sdep in unbuilt_pkg_names: + if not sdep in unbuilt: + unbuilt.append(sdep) + if depth > 0: + child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) + for sub_sdep in child_unbuilt: + if sub_sdep != name: + if not sub_sdep in unbuilt: + unbuilt.append(sub_sdep) + + return unbuilt + +def can_build_at_idx(build_idx, name, opts): + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + global build_env + + fs_size_gb = 0 + size_gb = 0 + speed = 0 + pkg = name_to_pkg[name] + if name in big_pkg_names: + size_gb=big_pkg_names[name] + if pkg in big_pkgs: + size_gb=big_pkgs[pkg] + if name in slow_pkg_names: + speed=slow_pkg_names[name] + if pkg in slow_pkgs: + speed=slow_pkgs[pkg] + fs_size_gb = build_env[build_idx]['fs_size_gb'] + return fs_size_gb == 0 or fs_size_gb >= size_gb + +def schedule(build_idx, pkgs, opts): + global worker_data + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + + unbuilt_pkg_names=[] + building_pkg_names=[] + unprioritized_pkg_names=[] + + for pkg in pkgs: + name = pkg_to_name[pkg] + unbuilt_pkg_names.append(name) + unprioritized_pkg_names.append(name) + + prioritized_pkg_names=[] + + for wd in worker_data: + pkg = wd['pkg'] + if not pkg is None: + name = pkg_to_name[pkg] + building_pkg_names.append(name) + + # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) + if len(big_pkg_names) or len(big_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in big_pkg_names or pkg in big_pkgs: + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + if len(slow_pkg_names) or len(slow_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in slow_pkg_names or pkg in slow_pkgs: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + for name in unprioritized_pkg_names: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + + name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) + if not name_out is None: + pkg_out = name_to_pkg[name_out] + else: + pkg_out = None + # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) + # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) + return pkg_out + + +def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): + global pkg_to_name + global name_to_pkg + global no_dep_list + + max_depth = 3 + + if len(pkg_names) == 0: + return None + + unbuilt_deps={} + building_deps={} + for depth in range(max_depth,-1,-1): + unbuilt_deps[depth]={} + building_deps[depth]={} + + for depth in range(max_depth,-1,-1): + checked=[] + reordered_pkg_names = pkg_names[:] + # for name in reordered_pkg_names: + while len(reordered_pkg_names): + name = reordered_pkg_names.pop(0) + if name in checked: + continue + + # log(opts.logfile, "checked.append(%s)" % name) + checked.append(name) + + pkg = name_to_pkg[name] + # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) + if not name in unbuilt_deps[depth]: + unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) + if not name in building_deps[depth]: + building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) + if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + if not name in unbuilt_deps[0]: + unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) + if not name in building_deps[0]: + building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) + if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): + if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + loop = False + for dep_name in unbuilt_deps[depth][name]: + if name == dep_name: + continue + + # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) + if dep_name in checked: + continue + + # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) + if not dep_name in unbuilt_deps[depth]: + unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) + if not dep_name in building_deps[depth]: + building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) + if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if not dep_name in unbuilt_deps[0]: + unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) + if not dep_name in building_deps[0]: + building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) + if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): + if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if name in unbuilt_deps[0][dep_name]: + loop = True + # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) + + if loop and len(building_deps[depth][name]) == 0: + log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) + return name + + for dep_name in unbuilt_deps[depth][name]: + if dep_name in reordered_pkg_names: + # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) + reordered_pkg_names.remove(dep_name) + reordered_pkg_names.insert(0,dep_name) + + # log(opts.logfile, "schedule2: Nothing buildable at this time") + return None + + + + +def read_deps(opts): + read_srpm_deps(opts) + read_rpm_deps(opts) + read_map_deps(opts) + +def read_srpm_deps(opts): + global srpm_dependencies_direct + + if opts.srpm_dependency_file == None: + return + + if not os.path.exists(opts.srpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) + sys.exit(1) + + with open(opts.srpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + srpm_dependencies_direct[name]=deps.split(',') + +def read_rpm_deps(opts): + global rpm_dependencies_direct + + if opts.rpm_dependency_file == None: + return + + if not os.path.exists(opts.rpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) + sys.exit(1) + + with open(opts.rpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + rpm_dependencies_direct[name]=deps.split(',') + +def read_map_deps(opts): + global rpm_to_srpm_map + + if opts.rpm_to_srpm_map_file == None: + return + + if not os.path.exists(opts.rpm_to_srpm_map_file): + log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) + sys.exit(1) + + with open(opts.rpm_to_srpm_map_file) as f: + lines = f.readlines() + for line in lines: + (rpm,srpm) = line.rstrip().split(';') + rpm_to_srpm_map[rpm]=srpm + + +def reaper(opts): + global built_pkgs + global failed + global worker_data + global workers + + reaped = 0 + need_createrepo = False + last_reaped = -1 + while reaped > last_reaped: + last_reaped = reaped + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is not None: + pkg = wd['pkg'] + b = int(wd['build_index']) + p.join() + worker_data.remove(wd) + workers = workers - 1 + reaped = reaped + 1 + release_build_env(b) + + log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) + + if ret == 0: + failed.append(pkg) + log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) + if opts.recurse and not stop_signal: + log(opts.logfile, "Will try to build again (if some other package will succeed).") + else: + log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) + if not opts.cont: + sys.exit(1) + elif ret == 1: + log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) + built_pkgs.append(pkg) + need_createrepo = True + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if need_createrepo: + # createrepo with the new pkgs + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + + return reaped + +stop_signal = False + +def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + +def kill_proc_and_descentents(parent, need_stop=False, verbose=False): + global g_opts + + if need_stop: + if verbose: + log(g_opts.logfile, "Stop %d" % parent.pid) + + try: + parent.send_signal(signal.SIGSTOP) + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.send_signal(signal.SIGSTOP) + + children = parent.children(recursive=False) + + for p in children: + kill_proc_and_descentents(p, need_stop=True, verbose=verbose) + + if verbose: + log(g_opts.logfile, "Terminate %d" % parent.pid) + + # parent.send_signal(signal.SIGTERM) + try: + parent.terminate() + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.terminate() + + if need_stop: + if verbose: + log(g_opts.logfile, "Continue %d" % parent.pid) + + parent.send_signal(signal.SIGCONT) + + +def child_signal_handler(signum, frame): + global g_opts + my_pid = os.getpid() + # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) + p = psutil.Process(my_pid) + kill_proc_and_descentents(p) + try: + sys.exit(0) + except SystemExit as e: + os._exit(0) + +def signal_handler(signum, frame): + global g_opts + global stop_signal + global workers + global worker_data + stop_signal = True + + # Signal processes to complete + log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is None: + # log(g_opts.logfile, "terminate child %d" % p.pid) + p.terminate() + else: + log(g_opts.logfile, "child return code was %d" % ret) + + # Wait for remaining processes to complete + log(g_opts.logfile, "===== wait for signaled jobs to complete =====") + while len(worker_data) > 0: + log(g_opts.logfile, " remaining workers: %d" % workers) + reaped = reaper(g_opts) + if reaped == 0: + time.sleep(0.1) + + try: + sys.exit(1) + except SystemExit as e: + os._exit(1) + +def main(args): + opts, args = parse_args(args) + # take mock config + list of pkgs + + global g_opts + global stop_signal + global build_env + global worker_data + global workers + global max_workers + + global slow_pkg_names + global slow_pkgs + global big_pkg_names + global big_pkgs + max_workers = int(opts.max_workers) + + global failed + global built_pkgs + + cfg = opts.chroot + pkgs = args[1:] + + # transform slow/big package options into dictionaries + for line in opts.slow_pkg_names_raw: + speed,name = line.split(":") + if speed != "": + slow_pkg_names[name]=int(speed) + for line in opts.slow_pkgs_raw: + speed,pkg = line.split(":") + if speed != "": + slow_pkgs[pkg]=int(speed) + for line in opts.big_pkg_names_raw: + size_gb,name = line.split(":") + if size_gb != "": + big_pkg_names[name]=int(size_gb) + for line in opts.big_pkgs_raw: + size_gb,pkg = line.split(":") + if size_gb != "": + big_pkgs[pkg]=int(size_gb) + + # Set up a mapping between pkg path and pkg name + global pkg_to_name + global name_to_pkg + for pkg in pkgs: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + continue + + try: + name = rpmName(pkg) + except OSError as e: + print("Could not parse rpm %s" % pkg) + sys.exit(1) + + pkg_to_name[pkg] = name + name_to_pkg[name] = pkg + + read_deps(opts) + + global config_opts + config_opts = mockbuild.util.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError as e: + print("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + if opts.basedir != "/var/lib/mock": + opts.uniqueext = '' + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + os.chmod(local_tmp_dir, 0o755) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + os.chmod(local_tmp_dir, 0o755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0o755) + + local_baseurl = "file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0o755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) + + # modify with localrepo + res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + # createrepo on it + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + # Temporary disable + # https://github.com/rpm-software-management/mock/issues/249 + #sys.exit(1) + + + init_build_env(max_workers, opts, config_opts) + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + return_code = 0 + num_of_tries = 0 + + g_opts = opts + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + signal.signal(signal.SIGABRT, signal_handler) + + while try_again and not stop_signal: + num_of_tries += 1 + failed = [] + + log(opts.logfile, "===== iteration %d start =====" % num_of_tries) + + to_be_built_scheduled = to_be_built[:] + + need_reap = False + while len(to_be_built_scheduled) > 0: + # Free up a worker + while need_reap or workers >= max_workers: + need_reap = False + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + + if workers < max_workers: + workers = workers + 1 + + b = get_idle_build_env(max_workers) + if b < 0: + log(opts.logfile, "Failed to find idle build env for: %s" % pkg) + workers = workers - 1 + need_reap = True + continue + + pkg = schedule(b, to_be_built_scheduled, opts) + if pkg is None: + if workers <= 1: + # Remember we have one build environmnet reserved, so can't test for zero workers + log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) + pkg = to_be_built_scheduled[0] + log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) + else: + release_build_env(b) + workers = workers - 1 + need_reap = True + continue + + to_be_built_scheduled.remove(pkg) + + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + release_build_env(b) + need_reap = True + continue + + elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): + url = pkg + try: + log(opts.logfile, 'Fetching %s' % url) + r = requests.get(url) + # pylint: disable=no-member + if r.status_code == requests.codes.ok: + fn = urlsplit(r.url).path.rsplit('/', 1)[1] + if 'content-disposition' in r.headers: + _, params = cgi.parse_header(r.headers['content-disposition']) + if 'filename' in params and params['filename']: + fn = params['filename'] + pkg = download_dir + '/' + fn + with open(pkg, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + except Exception as e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + release_build_env(b) + need_reap = True + continue + else: + downloaded_pkgs[pkg] = url + + log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) + # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] + p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) + worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) + p.start() + + # Wait for remaining processes to complete + log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) + while workers > 0: + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) + + if failed and opts.recurse: + log(opts.logfile, "failed=%s" % failed) + log(opts.logfile, "to_be_built=%s" % to_be_built) + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package succeeded, some failed.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) + else: + if max_workers > 1: + max_workers = 1 + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package failed under parallel build.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) + else: + log(opts.logfile, "") + log(opts.logfile, "*** Build Failed ***") + log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) + log(opts.logfile, "*** Build Failed ***") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + log(opts.logfile, "") + try_again = False + else: + try_again = False + if failed: + return_code = 2 + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "") + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "") + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log(opts.logfile, "Some packages successfully built in this order:") + else: + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/build-tools/stxRpmUtils.py b/build-tools/stxRpmUtils.py new file mode 100644 index 00000000..f6b1d412 --- /dev/null +++ b/build-tools/stxRpmUtils.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# A place to collect potentially reusable python functions +# + +def splitRpmFilename(filename): + """ + Split an rpm filename into components: + package name, version, release, epoch, architecture + """ + + if filename[-4:] == '.rpm': + filename = filename[:-4] + + idx = filename.rfind('.') + arch = filename[idx+1:] + filename = filename[:idx] + + idx = filename.rfind('-') + rel = filename[idx+1:] + filename = filename[:idx] + + idx = filename.rfind('-') + ver = filename[idx+1:] + filename = filename[:idx] + + idx = filename.find(':') + if idx == -1: + epoch = '' + name = filename + else: + epoch = filename[:idx] + name = filename[idx+1:] + + return name, ver, rel, epoch, arch +