diff --git a/build-tools/mockchain-parallel-2.7 b/build-tools/mockchain-parallel-2.7 new file mode 100755 index 00000000..b9f4bd3f --- /dev/null +++ b/build-tools/mockchain-parallel-2.7 @@ -0,0 +1,1221 @@ +#!/usr/bin/python3 -tt +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +# by skvidal@fedoraproject.org +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# copyright 2012 Red Hat, Inc. + +# SUMMARY +# mockchain +# take a mock config and a series of srpms +# rebuild them one at a time +# adding each to a local repo +# so they are available as build deps to next pkg being built +from __future__ import print_function + +import cgi +# pylint: disable=deprecated-module +import optparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import multiprocessing +import signal +import psutil + +import requests +# pylint: disable=import-error +from six.moves.urllib_parse import urlsplit + +import mockbuild.config as mock_config + +from stxRpmUtils import splitRpmFilename + +# all of the variables below are substituted by the build system +__VERSION__="2.7" +SYSCONFDIR="/etc" +PYTHONDIR="/usr/lib/python3.6/site-packages" +PKGPYTHONDIR="/usr/lib/python3.6/site-packages/mockbuild" +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs + +mockconfig_path = '/etc/mock' + +def rpmName(path): + filename = os.path.basename(path) + (n, v, r, e, a) = splitRpmFilename(filename) + return n + +def createrepo(path): + global max_workers + if os.path.exists(path + '/repodata/repomd.xml'): + comm = ['/usr/bin/createrepo_c', '--update', '--retain-old-md', "%d" % max_workers, "--workers", "%d" % max_workers, path] + else: + comm = ['/usr/bin/createrepo_c', "--workers", "%d" % max_workers, path] + cmd = subprocess.Popen( + comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + return out, err + + +g_opts = optparse.Values() + +def parse_args(args): + parser = optparse.OptionParser('\nmockchain -r mockcfg pkg1 [pkg2] [pkg3]') + parser.add_option( + '-r', '--root', default=None, dest='chroot', + metavar="CONFIG", + help="chroot config name/base to use in the mock build") + parser.add_option( + '-l', '--localrepo', default=None, + help="local path for the local repo, defaults to making its own") + parser.add_option( + '-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', + dest='repos', + help="add these repo baseurls to the chroot's yum config") + parser.add_option( + '--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option( + '--log', default=None, dest='logfile', + help="log to the file named by this option, defaults to not logging") + parser.add_option( + '--workers', default=1, dest='max_workers', + help="number of parallel build jobs") + parser.add_option( + '--worker-resources', default="", dest='worker_resources', + help="colon seperated list, how much mem in gb for each workers temfs") + parser.add_option( + '--basedir', default='/var/lib/mock', dest='basedir', + help="path to workspace") + parser.add_option( + '--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + parser.add_option( + '-m', '--mock-option', default=[], action='append', + dest='mock_option', + help="option to pass directly to mock") + parser.add_option( + '--mark-slow-name', default=[], action='append', + dest='slow_pkg_names_raw', + help="package name that is known to build slowly") + parser.add_option( + '--mark-slow-path', default=[], action='append', + dest='slow_pkgs_raw', + help="package path that is known to build slowly") + parser.add_option( + '--mark-big-name', default=[], action='append', + dest='big_pkg_names_raw', + help="package name that is known to require a lot of disk space to build") + parser.add_option( + '--mark-big-path', default=[], action='append', + dest='big_pkgs_raw', + help="package path that is known to require a lot of disk space to build") + parser.add_option( + '--srpm-dependency-file', default=None, + dest='srpm_dependency_file', + help="path to srpm dependency file") + parser.add_option( + '--rpm-dependency-file', default=None, + dest='rpm_dependency_file', + help="path to rpm dependency file") + parser.add_option( + '--rpm-to-srpm-map-file', default=None, + dest='rpm_to_srpm_map_file', + help="path to rpm to srpm map file") + + opts, args = parser.parse_args(args) + if opts.recurse: + opts.cont = True + + if not opts.chroot: + print("You must provide an argument to -r for the mock chroot") + sys.exit(1) + + if len(sys.argv) < 3: + print("You must specify at least 1 package to build") + sys.exit(1) + + return opts, args + + +REPOS_ID = [] + +slow_pkg_names={} +slow_pkgs={} +big_pkg_names={} +big_pkgs={} + +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = "/".join(baseurl.split('//')[1:]).replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +def set_build_idx(infile, destfile, build_idx, tmpfs_size_gb, opts): + # log(opts.logfile, "set_build_idx: infile=%s, destfile=%s, build_idx=%d, tmpfs_size_gb=%d" % (infile, destfile, build_idx, tmpfs_size_gb)) + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['root'] = config_opts['root'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['rootdir'] = config_opts['rootdir'].replace('b0', 'b{0}'.format(build_idx)) + config_opts['cache_topdir'] = config_opts['cache_topdir'].replace('b0', 'b{0}'.format(build_idx)) + # log(opts.logfile, "set_build_idx: root=%s" % config_opts['root']) + # log(opts.logfile, "set_build_idx: cache_topdir=%s" % config_opts['cache_topdir']) + if tmpfs_size_gb > 0: + config_opts['plugin_conf']['tmpfs_enable'] = True + config_opts['plugin_conf']['tmpfs_opts'] = {} + config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 + config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = "%dg" % tmpfs_size_gb + config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' + config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = True + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_enable=%s" % config_opts['plugin_conf']['tmpfs_enable']) + # log(opts.logfile, "set_build_idx: plugin_conf->tmpfs_opts->max_fs_size=%s" % config_opts['plugin_conf']['tmpfs_opts']['max_fs_size']) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % config_opts['cache_topdir']) + if not os.path.isdir(config_opts['cache_topdir']): + os.makedirs(config_opts['cache_topdir'], exist_ok=True) + except (IOError, OSError): + return False, "Could not create dir: %s" % config_opts['cache_topdir'] + + cache_dir = "%s/%s/mock" % (config_opts['basedir'], config_opts['root']) + try: + log(opts.logfile, "set_build_idx: os.makedirs %s" % cache_dir) + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + except (IOError, OSError): + return False, "Could not create dir: %s" % cache_dir + + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def set_basedir(infile, destfile, basedir, opts): + log(opts.logfile, "set_basedir: infile=%s, destfile=%s, basedir=%s" % (infile, destfile, basedir)) + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + + config_opts['basedir'] = basedir + config_opts['resultdir'] = '{0}/result'.format(basedir) + config_opts['backup_base_dir'] = '{0}/backup'.format(basedir) + config_opts['root'] = 'mock/b0' + config_opts['cache_topdir'] = '{0}/cache/b0'.format(basedir) + config_opts['rootdir'] = '{0}/mock/b0/root'.format(basedir) + + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + +def add_local_repo(infile, destfile, baseurl, repoid=None): + """take a mock chroot config and add a repo to it's yum.conf + infile = mock chroot config file + destfile = where to save out the result + baseurl = baseurl of repo you wish to add""" + global config_opts + + try: + with open(infile) as f: + code = compile(f.read(), infile, 'exec') + # pylint: disable=exec-used + exec(code) + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ +[%s] +name=%s +baseurl=%s +enabled=1 +skip_if_unavailable=1 +metadata_expire=0 +cost=1 +best=1 +""" % (repoid, baseurl, baseurl) + + config_opts['yum.conf'] += localyumrepo + with open(destfile, 'w') as br_dest: + for k, v in list(config_opts.items()): + br_dest.write("config_opts[%r] = %r\n" % (k, v)) + return True, '' + except (IOError, OSError): + return False, "Could not write mock config to %s" % destfile + + return True, '' + + +def do_build(opts, cfg, pkg): + + # returns 0, cmd, out, err = failure + # returns 1, cmd, out, err = success + # returns 2, None, None, None = already built + + signal.signal(signal.SIGTERM, child_signal_handler) + signal.signal(signal.SIGINT, child_signal_handler) + signal.signal(signal.SIGHUP, child_signal_handler) + signal.signal(signal.SIGABRT, child_signal_handler) + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resdir = '%s/%s' % (opts.local_repo_dir, pdn) + resdir = os.path.normpath(resdir) + if not os.path.exists(resdir): + os.makedirs(resdir) + + success_file = resdir + '/success' + fail_file = resdir + '/fail' + + if os.path.exists(success_file): + # return 2, None, None, None + sys.exit(2) + + # clean it up if we're starting over :) + if os.path.exists(fail_file): + os.unlink(fail_file) + + if opts.uniqueext == '': + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--root', cfg, ] + else: + mockcmd = ['/usr/bin/mock', + '--configdir', opts.config_path, + '--resultdir', resdir, + '--uniqueext', opts.uniqueext, + '--root', cfg, ] + + # Ensure repo is up-to-date. + # Note: Merely adding --update to mockcmd failed to update + mockcmd_update=mockcmd + mockcmd_update.append('--update') + cmd = subprocess.Popen( + mockcmd_update, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode != 0: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + + # heuristic here, if user pass for mock "-d foo", but we must be care to leave + # "-d'foo bar'" or "--define='foo bar'" as is + compiled_re_1 = re.compile(r'^(-\S)\s+(.+)') + compiled_re_2 = re.compile(r'^(--[^ =])[ =](\.+)') + for option in opts.mock_option: + r_match = compiled_re_1.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + r_match = compiled_re_2.match(option) + if r_match: + mockcmd.extend([r_match.group(1), r_match.group(2)]) + else: + mockcmd.append(option) + + print('building %s' % s_pkg) + mockcmd.append(pkg) + cmd = subprocess.Popen( + mockcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = cmd.communicate() + if cmd.returncode == 0: + with open(success_file, 'w') as f: + f.write('done\n') + ret = 1 + else: + if (isinstance(err, bytes)): + err = err.decode("utf-8") + sys.stderr.write(err) + with open(fail_file, 'w') as f: + f.write('undone\n') + ret = 0 + + # return ret, cmd, out, err + sys.exit(ret) + + +def log(lf, msg): + if lf: + now = time.time() + try: + with open(lf, 'a') as f: + f.write(str(now) + ':' + msg + '\n') + except (IOError, OSError) as e: + print('Could not write to logfile %s - %s' % (lf, str(e))) + print(msg) + + +config_opts = {} + +worker_data = [] +workers = 0 +max_workers = 1 + +build_env = [] + +failed = [] +built_pkgs = [] + +local_repo_dir = "" + +pkg_to_name={} +name_to_pkg={} +srpm_dependencies_direct={} +rpm_dependencies_direct={} +rpm_to_srpm_map={} +no_dep_list = [ "bash", "kernel" , "kernel-rt" ] + + +def init_build_env(slots, opts, config_opts_in): + global build_env + + orig_chroot_name=config_opts_in['chroot_name'] + orig_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(orig_chroot_name)) + # build_env.append({'state': 'Idle', 'cfg': orig_mock_config}) + for i in range(0,slots): + new_chroot_name = "{0}.b{1}".format(orig_chroot_name, i) + new_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(new_chroot_name)) + tmpfs_size_gb = 0 + if opts.worker_resources == "": + if i > 0: + tmpfs_size_gb = 2 * (1 + slots - i) + else: + resource_array=opts.worker_resources.split(':') + if i < len(resource_array): + tmpfs_size_gb=int(resource_array[i]) + else: + log(opts.logfile, "Error: worker-resources argument '%s' does not supply info for all %d workers" % (opts.worker_resources, slots)) + sys.exit(1) + if i == 0 and tmpfs_size_gb != 0: + log(opts.logfile, "Error: worker-resources argument '%s' must pass '0' as first value" % (opts.worker_resources, slots)) + sys.exit(1) + build_env.append({'state': 'Idle', 'cfg': new_mock_config, 'fs_size_gb': tmpfs_size_gb}) + + res, msg = set_build_idx(orig_mock_config, new_mock_config, i, tmpfs_size_gb, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + +idle_build_env_last_awarded = 0 +def get_idle_build_env(slots): + global build_env + global idle_build_env_last_awarded + visited = 0 + + if slots < 1: + return -1 + + i = idle_build_env_last_awarded - 1 + if i < 0 or i >= slots: + i = slots - 1 + + while visited < slots: + if build_env[i]['state'] == 'Idle': + build_env[i]['state'] = 'Busy' + idle_build_env_last_awarded = i + return i + visited = visited + 1 + i = i - 1 + if i < 0: + i = slots - 1 + return -1 + +def release_build_env(idx): + global build_env + + build_env[idx]['state'] = 'Idle' + +def get_best_rc(a, b): + print("get_best_rc: a=%s" % str(a)) + print("get_best_rc: b=%s" % str(b)) + if (b == {}) and (a != {}): + return a + if (a == {}) and (b != {}): + return b + + if (b['build_name'] is None) and (not a['build_name'] is None): + return a + if (a['build_name'] is None) and (not b['build_name'] is None): + return b + + if a['unbuilt_deps'] < b['unbuilt_deps']: + return a + if b['unbuilt_deps'] < a['unbuilt_deps']: + return b + + if a['depth'] < b['depth']: + return a + if b['depth'] < a['depth']: + return b + + print("get_best_rc: uncertain %s vs %s" % (a,b)) + return a + +unbuilt_dep_list_print=False +def unbuilt_dep_list(name, unbuilt_pkg_names, depth, checked=None): + global srpm_dependencies_direct + global rpm_dependencies_direct + global rpm_to_srpm_map + global no_dep_list + global unbuilt_dep_list_print + + first_iteration=False + unbuilt = [] + if name in no_dep_list: + return unbuilt + + if checked is None: + first_iteration=True + checked=[] + + # Count unbuild dependencies + if first_iteration: + dependencies_direct=srpm_dependencies_direct + else: + dependencies_direct=rpm_dependencies_direct + + if name in dependencies_direct: + for rdep in dependencies_direct[name]: + sdep='???' + if rdep in rpm_to_srpm_map: + sdep = rpm_to_srpm_map[rdep] + if rdep != name and sdep != name and not rdep in checked: + if (not first_iteration) and (sdep in no_dep_list): + continue + checked.append(rdep) + if sdep in unbuilt_pkg_names: + if not sdep in unbuilt: + unbuilt.append(sdep) + if depth > 0: + child_unbuilt = unbuilt_dep_list(rdep, unbuilt_pkg_names, depth-1, checked) + for sub_sdep in child_unbuilt: + if sub_sdep != name: + if not sub_sdep in unbuilt: + unbuilt.append(sub_sdep) + + return unbuilt + +def can_build_at_idx(build_idx, name, opts): + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + global build_env + + fs_size_gb = 0 + size_gb = 0 + speed = 0 + pkg = name_to_pkg[name] + if name in big_pkg_names: + size_gb=big_pkg_names[name] + if pkg in big_pkgs: + size_gb=big_pkgs[pkg] + if name in slow_pkg_names: + speed=slow_pkg_names[name] + if pkg in slow_pkgs: + speed=slow_pkgs[pkg] + fs_size_gb = build_env[build_idx]['fs_size_gb'] + return fs_size_gb == 0 or fs_size_gb >= size_gb + +def schedule(build_idx, pkgs, opts): + global worker_data + global pkg_to_name + global name_to_pkg + global big_pkgs + global big_pkg_names + global slow_pkgs + global slow_pkg_names + + unbuilt_pkg_names=[] + building_pkg_names=[] + unprioritized_pkg_names=[] + + for pkg in pkgs: + name = pkg_to_name[pkg] + unbuilt_pkg_names.append(name) + unprioritized_pkg_names.append(name) + + prioritized_pkg_names=[] + + for wd in worker_data: + pkg = wd['pkg'] + if not pkg is None: + name = pkg_to_name[pkg] + building_pkg_names.append(name) + + # log(opts.logfile, "schedule: build_idx=%d start" % build_idx) + if len(big_pkg_names) or len(big_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in big_pkg_names or pkg in big_pkgs: + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + if len(slow_pkg_names) or len(slow_pkgs): + next_unprioritized_pkg_names = unprioritized_pkg_names[:] + for name in unprioritized_pkg_names: + pkg = name_to_pkg[name] + if name in slow_pkg_names or pkg in slow_pkgs: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + next_unprioritized_pkg_names.remove(name) + unprioritized_pkg_names = next_unprioritized_pkg_names[:] + + for name in unprioritized_pkg_names: + if can_build_at_idx(build_idx, name, opts): + prioritized_pkg_names.append(name) + + name_out = schedule2(build_idx, prioritized_pkg_names, unbuilt_pkg_names, building_pkg_names, opts) + if not name_out is None: + pkg_out = name_to_pkg[name_out] + else: + pkg_out = None + # log(opts.logfile, "schedule: failed to translate '%s' to a pkg" % name_out) + # log(opts.logfile, "schedule: build_idx=%d end: out = %s -> %s" % (build_idx, str(name_out), str(pkg_out))) + return pkg_out + + +def schedule2(build_idx, pkg_names, unbuilt_pkg_names, building_pkg_names, opts): + global pkg_to_name + global name_to_pkg + global no_dep_list + + max_depth = 3 + + if len(pkg_names) == 0: + return None + + unbuilt_deps={} + building_deps={} + for depth in range(max_depth,-1,-1): + unbuilt_deps[depth]={} + building_deps[depth]={} + + for depth in range(max_depth,-1,-1): + checked=[] + reordered_pkg_names = pkg_names[:] + # for name in reordered_pkg_names: + while len(reordered_pkg_names): + name = reordered_pkg_names.pop(0) + if name in checked: + continue + + # log(opts.logfile, "checked.append(%s)" % name) + checked.append(name) + + pkg = name_to_pkg[name] + # log(opts.logfile, "schedule2: check '%s', depth %d" % (name, depth)) + if not name in unbuilt_deps[depth]: + unbuilt_deps[depth][name] = unbuilt_dep_list(name, unbuilt_pkg_names, depth) + if not name in building_deps[depth]: + building_deps[depth][name] = unbuilt_dep_list(name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, depth, unbuilt_deps[depth][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, depth, building_deps[depth][name])) + if len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s', searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + if not name in unbuilt_deps[0]: + unbuilt_deps[0][name] = unbuilt_dep_list(name, unbuilt_pkg_names, 0) + if not name in building_deps[0]: + building_deps[0][name] = unbuilt_dep_list(name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: unbuilt deps for pkg=%s, depth=%d: %s" % (name, 0, unbuilt_deps[0][name])) + # log(opts.logfile, "schedule2: building deps for pkg=%s, depth=%d: %s" % (name, 0, building_deps[0][name])) + if (len(building_deps[depth][name]) == 0 and len(unbuilt_deps[depth][name]) == 1 and unbuilt_deps[depth][name][0] in no_dep_list) or (len(unbuilt_deps[depth][name]) == 0 and len(building_deps[depth][name]) == 1 and building_deps[depth][name][0] in no_dep_list): + if len(unbuilt_deps[0][name]) == 0 and len(building_deps[0][name]) == 0: + if can_build_at_idx(build_idx, name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, searching at depth %d" % (name, depth)) + return name + else: + # log(opts.logfile, "schedule2: Can't build '%s' on 'b%d'" % (name, build_idx)) + continue + + loop = False + for dep_name in unbuilt_deps[depth][name]: + if name == dep_name: + continue + + # log(opts.logfile, "name=%s depends on dep_name=%s, depth=%d" % (name, dep_name, depth)) + if dep_name in checked: + continue + + # log(opts.logfile, "schedule2: check '%s' indirect" % dep_name) + if not dep_name in unbuilt_deps[depth]: + unbuilt_deps[depth][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, depth) + if not dep_name in building_deps[depth]: + building_deps[depth][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, depth) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, unbuilt_deps[depth][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, depth, building_deps[depth][dep_name])) + if len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: deps: no unbuilt deps for '%s', working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if not dep_name in unbuilt_deps[0]: + unbuilt_deps[0][dep_name] = unbuilt_dep_list(dep_name, unbuilt_pkg_names, 0) + if not dep_name in building_deps[0]: + building_deps[0][dep_name] = unbuilt_dep_list(dep_name, building_pkg_names, 0) + # log(opts.logfile, "schedule2: deps: unbuilt deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, unbuilt_deps[0][dep_name])) + # log(opts.logfile, "schedule2: deps: building deps for %s -> %s, depth=%d: %s" % (name, dep_name, 0, building_deps[0][dep_name])) + if (len(building_deps[depth][dep_name]) == 0 and len(unbuilt_deps[depth][dep_name]) == 1 and unbuilt_deps[depth][dep_name][0] in no_dep_list) or (len(unbuilt_deps[depth][dep_name]) == 0 and len(building_deps[depth][dep_name]) == 1 and building_deps[depth][dep_name][0] in no_dep_list): + if len(unbuilt_deps[0][dep_name]) == 0 and len(building_deps[0][dep_name]) == 0: + if can_build_at_idx(build_idx, dep_name, opts): + log(opts.logfile, "schedule2: no unbuilt deps for '%s' except for indirect kernel dep, working towards '%s', searching at depth %d" % (dep_name, name, depth)) + return dep_name + + if name in unbuilt_deps[0][dep_name]: + loop = True + # log(opts.logfile, "schedule2: loop detected: %s <-> %s" % (name, dep_name)) + + if loop and len(building_deps[depth][name]) == 0: + log(opts.logfile, "schedule2: loop detected, try to build '%s'" % name) + return name + + for dep_name in unbuilt_deps[depth][name]: + if dep_name in reordered_pkg_names: + # log(opts.logfile, "schedule2: promote %s to work toward %s" % (dep_name, name)) + reordered_pkg_names.remove(dep_name) + reordered_pkg_names.insert(0,dep_name) + + # log(opts.logfile, "schedule2: Nothing buildable at this time") + return None + + +def read_deps(opts): + read_srpm_deps(opts) + read_rpm_deps(opts) + read_map_deps(opts) + +def read_srpm_deps(opts): + global srpm_dependencies_direct + + if opts.srpm_dependency_file == None: + return + + if not os.path.exists(opts.srpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.srpm_dependency_file) + sys.exit(1) + + with open(opts.srpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + srpm_dependencies_direct[name]=deps.split(',') + +def read_rpm_deps(opts): + global rpm_dependencies_direct + + if opts.rpm_dependency_file == None: + return + + if not os.path.exists(opts.rpm_dependency_file): + log(opts.logfile, "File not found: %s" % opts.rpm_dependency_file) + sys.exit(1) + + with open(opts.rpm_dependency_file) as f: + lines = f.readlines() + for line in lines: + (name,deps) = line.rstrip().split(';') + rpm_dependencies_direct[name]=deps.split(',') + +def read_map_deps(opts): + global rpm_to_srpm_map + + if opts.rpm_to_srpm_map_file == None: + return + + if not os.path.exists(opts.rpm_to_srpm_map_file): + log(opts.logfile, "File not found: %s" % opts.rpm_to_srpm_map_file) + sys.exit(1) + + with open(opts.rpm_to_srpm_map_file) as f: + lines = f.readlines() + for line in lines: + (rpm,srpm) = line.rstrip().split(';') + rpm_to_srpm_map[rpm]=srpm + + +def reaper(opts): + global built_pkgs + global failed + global worker_data + global workers + + reaped = 0 + need_createrepo = False + last_reaped = -1 + while reaped > last_reaped: + last_reaped = reaped + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is not None: + pkg = wd['pkg'] + b = int(wd['build_index']) + p.join() + worker_data.remove(wd) + workers = workers - 1 + reaped = reaped + 1 + release_build_env(b) + + log(opts.logfile, "End build on 'b%d': %s" % (b, pkg)) + + if ret == 0: + failed.append(pkg) + log(opts.logfile, "Error building %s on 'b%d'." % (os.path.basename(pkg), b)) + if opts.recurse and not stop_signal: + log(opts.logfile, "Will try to build again (if some other package will succeed).") + else: + log(opts.logfile, "See logs/results in %s" % opts.local_repo_dir) + elif ret == 1: + log(opts.logfile, "Success building %s on 'b%d'" % (os.path.basename(pkg), b)) + built_pkgs.append(pkg) + need_createrepo = True + elif ret == 2: + log(opts.logfile, "Skipping already built pkg %s" % os.path.basename(pkg)) + + if need_createrepo: + # createrepo with the new pkgs + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + + return reaped + +stop_signal = False + +def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + +def kill_proc_and_descentents(parent, need_stop=False, verbose=False): + global g_opts + + if need_stop: + if verbose: + log(g_opts.logfile, "Stop %d" % parent.pid) + + try: + parent.send_signal(signal.SIGSTOP) + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.send_signal(signal.SIGSTOP) + + try: + children = parent.children(recursive=False) + except: + children = [] + + for p in children: + kill_proc_and_descentents(p, need_stop=True, verbose=verbose) + + if verbose: + log(g_opts.logfile, "Terminate %d" % parent.pid) + + # parent.send_signal(signal.SIGTERM) + try: + parent.terminate() + except: + # perhaps mock still running as root, give it a sec to drop pivledges and try again + time.sleep(1) + parent.terminate() + + if need_stop: + if verbose: + log(g_opts.logfile, "Continue %d" % parent.pid) + + parent.send_signal(signal.SIGCONT) + + +def child_signal_handler(signum, frame): + global g_opts + my_pid = os.getpid() + # log(g_opts.logfile, "--------- child %d recieved signal %d" % (my_pid, signum)) + p = psutil.Process(my_pid) + kill_proc_and_descentents(p) + try: + sys.exit(0) + except SystemExit as e: + os._exit(0) + +def signal_handler(signum, frame): + global g_opts + global stop_signal + global workers + global worker_data + stop_signal = True + + # Signal processes to complete + log(g_opts.logfile, "recieved signal %d, Terminating children" % signum) + for wd in worker_data: + p = wd['proc'] + ret = p.exitcode + if ret is None: + # log(g_opts.logfile, "terminate child %d" % p.pid) + p.terminate() + else: + log(g_opts.logfile, "child return code was %d" % ret) + + # Wait for remaining processes to complete + log(g_opts.logfile, "===== wait for signaled jobs to complete =====") + while len(worker_data) > 0: + log(g_opts.logfile, " remaining workers: %d" % workers) + reaped = reaper(g_opts) + if reaped == 0: + time.sleep(0.1) + + try: + sys.exit(1) + except SystemExit as e: + os._exit(1) + +def main(args): + opts, args = parse_args(args) + # take mock config + list of pkgs + + global g_opts + global stop_signal + global build_env + global worker_data + global workers + global max_workers + + global slow_pkg_names + global slow_pkgs + global big_pkg_names + global big_pkgs + max_workers = int(opts.max_workers) + + global failed + global built_pkgs + + cfg = opts.chroot + pkgs = args[1:] + + # transform slow/big package options into dictionaries + for line in opts.slow_pkg_names_raw: + speed,name = line.split(":") + if speed != "": + slow_pkg_names[name]=int(speed) + for line in opts.slow_pkgs_raw: + speed,pkg = line.split(":") + if speed != "": + slow_pkgs[pkg]=int(speed) + for line in opts.big_pkg_names_raw: + size_gb,name = line.split(":") + if size_gb != "": + big_pkg_names[name]=int(size_gb) + for line in opts.big_pkgs_raw: + size_gb,pkg = line.split(":") + if size_gb != "": + big_pkgs[pkg]=int(size_gb) + + # Set up a mapping between pkg path and pkg name + global pkg_to_name + global name_to_pkg + for pkg in pkgs: + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + continue + + try: + name = rpmName(pkg) + except OSError as e: + print("Could not parse rpm %s" % pkg) + sys.exit(1) + + pkg_to_name[pkg] = name + name_to_pkg[name] = pkg + + read_deps(opts) + + global config_opts + config_opts = mock_config.load_config(mockconfig_path, cfg, None, __VERSION__, PKGPYTHONDIR) + + if not opts.tmp_prefix: + try: + opts.tmp_prefix = os.getlogin() + except OSError as e: + print("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + opts.uniqueext = '%s-%s' % (opts.tmp_prefix, pid) + + if opts.basedir != "/var/lib/mock": + opts.uniqueext = '' + + # create a tempdir for our local info + if opts.localrepo: + local_tmp_dir = os.path.abspath(opts.localrepo) + if not os.path.exists(local_tmp_dir): + os.makedirs(local_tmp_dir) + os.chmod(local_tmp_dir, 0o755) + else: + pre = 'mock-chain-%s-' % opts.uniqueext + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + os.chmod(local_tmp_dir, 0o755) + + if opts.logfile: + opts.logfile = os.path.join(local_tmp_dir, opts.logfile) + if os.path.exists(opts.logfile): + os.unlink(opts.logfile) + + log(opts.logfile, "starting logfile: %s" % opts.logfile) + + opts.local_repo_dir = os.path.normpath(local_tmp_dir + '/results/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.local_repo_dir): + os.makedirs(opts.local_repo_dir, mode=0o755) + + local_baseurl = "file://%s" % opts.local_repo_dir + log(opts.logfile, "results dir: %s" % opts.local_repo_dir) + opts.config_path = os.path.normpath(local_tmp_dir + '/configs/' + config_opts['chroot_name'] + '/') + + if not os.path.exists(opts.config_path): + os.makedirs(opts.config_path, mode=0o755) + + log(opts.logfile, "config dir: %s" % opts.config_path) + + my_mock_config = os.path.join(opts.config_path, "{0}.cfg".format(config_opts['chroot_name'])) + + # modify with localrepo + res, msg = add_local_repo(config_opts['config_file'], my_mock_config, local_baseurl, 'local_build_repo') + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + for baseurl in opts.repos: + res, msg = add_local_repo(my_mock_config, my_mock_config, baseurl) + if not res: + log(opts.logfile, "Error: Could not add: %s to yum config in mock chroot: %s" % (baseurl, msg)) + sys.exit(1) + + res, msg = set_basedir(my_mock_config, my_mock_config, opts.basedir, opts) + if not res: + log(opts.logfile, "Error: Could not write out local config: %s" % msg) + sys.exit(1) + + # these files needed from the mock.config dir to make mock run + for fn in ['site-defaults.cfg', 'logging.ini']: + pth = mockconfig_path + '/' + fn + shutil.copyfile(pth, opts.config_path + '/' + fn) + + # createrepo on it + err = createrepo(opts.local_repo_dir)[1] + if err.strip(): + log(opts.logfile, "Error making local repo: %s" % opts.local_repo_dir) + log(opts.logfile, "Err: %s" % err) + sys.exit(1) + + init_build_env(max_workers, opts, config_opts) + + download_dir = tempfile.mkdtemp() + downloaded_pkgs = {} + built_pkgs = [] + try_again = True + to_be_built = pkgs + return_code = 0 + num_of_tries = 0 + + g_opts = opts + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGHUP, signal_handler) + signal.signal(signal.SIGABRT, signal_handler) + + while try_again and not stop_signal: + num_of_tries += 1 + failed = [] + + log(opts.logfile, "===== iteration %d start =====" % num_of_tries) + + to_be_built_scheduled = to_be_built[:] + + need_reap = False + while len(to_be_built_scheduled) > 0: + # Free up a worker + while need_reap or workers >= max_workers: + need_reap = False + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + + if workers < max_workers: + workers = workers + 1 + + b = get_idle_build_env(max_workers) + if b < 0: + log(opts.logfile, "Failed to find idle build env for: %s" % pkg) + workers = workers - 1 + need_reap = True + continue + + pkg = schedule(b, to_be_built_scheduled, opts) + if pkg is None: + if workers <= 1: + # Remember we have one build environmnet reserved, so can't test for zero workers + log(opts.logfile, "failed to schedule from: %s" % to_be_built_scheduled) + pkg = to_be_built_scheduled[0] + log(opts.logfile, "All workers idle, forcing build of pkg=%s" % pkg) + else: + release_build_env(b) + workers = workers - 1 + need_reap = True + continue + + to_be_built_scheduled.remove(pkg) + + if not pkg.endswith('.rpm'): + log(opts.logfile, "%s doesn't appear to be an rpm - skipping" % pkg) + failed.append(pkg) + release_build_env(b) + need_reap = True + continue + + elif pkg.startswith('http://') or pkg.startswith('https://') or pkg.startswith('ftp://'): + url = pkg + try: + log(opts.logfile, 'Fetching %s' % url) + r = requests.get(url) + # pylint: disable=no-member + if r.status_code == requests.codes.ok: + fn = urlsplit(r.url).path.rsplit('/', 1)[1] + if 'content-disposition' in r.headers: + _, params = cgi.parse_header(r.headers['content-disposition']) + if 'filename' in params and params['filename']: + fn = params['filename'] + pkg = download_dir + '/' + fn + with open(pkg, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) + except Exception as e: + log(opts.logfile, 'Error Downloading %s: %s' % (url, str(e))) + failed.append(url) + release_build_env(b) + need_reap = True + continue + else: + downloaded_pkgs[pkg] = url + + log(opts.logfile, "Start build on 'b%d': %s" % (b, pkg)) + # ret = do_build(opts, config_opts['chroot_name'], pkg)[0] + p = multiprocessing.Process(target=do_build, args=(opts, build_env[b]['cfg'], pkg)) + worker_data.append({'proc': p, 'pkg': pkg, 'build_index': int(b)}) + p.start() + + # Wait for remaining processes to complete + log(opts.logfile, "===== wait for last jobs in iteration %d to complete =====" % num_of_tries) + while workers > 0: + reaped = reaper(opts) + if reaped == 0: + time.sleep(0.1) + log(opts.logfile, "===== iteration %d complete =====" % num_of_tries) + + if failed and opts.recurse: + log(opts.logfile, "failed=%s" % failed) + log(opts.logfile, "to_be_built=%s" % to_be_built) + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package succeeded, some failed.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs, because --recurse is set.' % len(failed)) + else: + if max_workers > 1: + max_workers = 1 + to_be_built = failed + try_again = True + log(opts.logfile, 'Some package failed under parallel build.') + log(opts.logfile, 'Trying to rebuild %s failed pkgs with single thread, because --recurse is set.' % len(failed)) + else: + log(opts.logfile, "") + log(opts.logfile, "*** Build Failed ***") + log(opts.logfile, "Tried %s times - following pkgs could not be successfully built:" % num_of_tries) + log(opts.logfile, "*** Build Failed ***") + for pkg in failed: + msg = pkg + if pkg in downloaded_pkgs: + msg = downloaded_pkgs[pkg] + log(opts.logfile, msg) + log(opts.logfile, "") + try_again = False + else: + try_again = False + if failed: + return_code = 2 + + # cleaning up our download dir + shutil.rmtree(download_dir, ignore_errors=True) + + log(opts.logfile, "") + log(opts.logfile, "Results out to: %s" % opts.local_repo_dir) + log(opts.logfile, "") + log(opts.logfile, "Pkgs built: %s" % len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log(opts.logfile, "Some packages successfully built in this order:") + else: + log(opts.logfile, "Packages successfully built in this order:") + for pkg in built_pkgs: + log(opts.logfile, pkg) + return return_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv))