From f6755e179a030f03d0e1b37e505099923b47a48b Mon Sep 17 00:00:00 2001 From: Scott Little Date: Wed, 4 Mar 2020 12:24:47 -0500 Subject: [PATCH] Enable build of docker images, charts and wheels from a layered build 1) Add config for the container layer. 2) Add support for the new config file 'required_layer_wheel_inc.cfg', Which will specify the default urls for lower layer wheels.inc files. These files must be downloaded by download_mirror.sh, and linked by generate-cgcs-centos-repo.sh. 3) Add options to download_mirror.sh and generate-cgcs-centos-repo.sh allowing overrides or additions to the set of url's for lower layer wheels.inc 4) Changes to allow build scripts running within a docker container to launch a sister (not daughter) container. This will allow build-wheels to run within a container. Story: 2006166 Task: 38980 Depends-On: https://review.opendev.org/711773 Depends-On: https://review.opendev.org/717755 Change-Id: I95c71a0327dc9d919ed835d3661a6f1ec7ce30c7 Signed-off-by: Scott Little --- Dockerfile | 1 + .../centos/containers/required_layer_pkgs.cfg | 5 ++ .../containers/required_layer_wheel_inc.cfg | 4 ++ .../StarlingX_cengn_compiler_layer.repo | 12 +++++ .../StarlingX_cengn_distro_layer.repo | 18 +++++++ .../StarlingX_cengn_flock_layer.repo | 8 ++++ .../centos/flock/required_layer_wheel_inc.cfg | 2 + centos-mirror-tools/download_mirror.sh | 29 ++++++++++- tb.sh | 1 + toCOPY/generate-cgcs-centos-repo.sh | 42 +++++++++++++++- toCOPY/lst_utils.sh | 48 ++++++++++++++++++- 11 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg create mode 100644 centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg create mode 100644 centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo create mode 100644 centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo create mode 100644 centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo create mode 100644 centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg diff --git a/Dockerfile b/Dockerfile index e099cfd0..50ac9c02 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ RUN groupadd -g 751 cgts && \ cpanminus \ createrepo \ deltarpm \ + docker-client \ expat-devel \ flex \ isomd5sum \ diff --git a/centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg b/centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg new file mode 100644 index 00000000..7979f444 --- /dev/null +++ b/centos-mirror-tools/config/centos/containers/required_layer_pkgs.cfg @@ -0,0 +1,5 @@ +compiler,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/rpm.lst +distro,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/rpm.lst +distro,rt,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/rpm.lst +distro,installer,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/rpm.lst +flock,std,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/rpm.lst diff --git a/centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg b/centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg new file mode 100644 index 00000000..1d435945 --- /dev/null +++ b/centos-mirror-tools/config/centos/containers/required_layer_wheel_inc.cfg @@ -0,0 +1,4 @@ +distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc +distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc +flock,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_stable_wheels.inc +flock,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/centos_dev_wheels.inc \ No newline at end of file diff --git a/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo new file mode 100644 index 00000000..22bd747a --- /dev/null +++ b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_compiler_layer.repo @@ -0,0 +1,12 @@ +##### +# StarlingX compiler layer at CENGN + +[Starlingx-cengn_compiler_std_layer] +name=Starlingx-cengn_compiler_std_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/std/ +enabled=1 + +[Starlingx-cengn_compiler_installer_layer] +name=Starlingx-cengn_compiler_installer_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/compiler/latest_build/outputs/RPMS/installer/ +enabled=1 diff --git a/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo new file mode 100644 index 00000000..84bcb31e --- /dev/null +++ b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_distro_layer.repo @@ -0,0 +1,18 @@ +##### +# StarlingX distro layer at CENGN + +[Starlingx-cengn_distro_std_layer] +name=Starlingx-cengn_distro_std_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/std/ +enabled=1 + +[Starlingx-cengn_distro_rt_layer] +name=Starlingx-cengn_distro_rt_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/rt/ +enabled=1 + +[Starlingx-cengn_distro_installer_layer] +name=Starlingx-cengn_distro_installer_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/RPMS/installer/ +enabled=1 + diff --git a/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo new file mode 100644 index 00000000..de7bec46 --- /dev/null +++ b/centos-mirror-tools/config/centos/containers/yum.repos.d/StarlingX_cengn_flock_layer.repo @@ -0,0 +1,8 @@ +##### +# StarlingX flock layer at CENGN + +[Starlingx-cengn_flock_std_layer] +name=Starlingx-cengn_flock_std_layer +baseurl=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/flock/latest_build/outputs/RPMS/std/ +enabled=1 + diff --git a/centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg b/centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg new file mode 100644 index 00000000..c3b4b4f3 --- /dev/null +++ b/centos-mirror-tools/config/centos/flock/required_layer_wheel_inc.cfg @@ -0,0 +1,2 @@ +distro,stable,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_stable_wheels.inc +distro,dev,http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/distro/latest_build/outputs/centos_dev_wheels.inc diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index 8f185bd9..f11ac246 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -43,6 +43,11 @@ usage() { echo " layer's build type. Normally the url(s) is read from" echo " ///required_layer_pkgs.cfg." echo " This option can be used more than once." + echo " -W ,,:" + echo " Override the url for the wheels.inc list of a lower" + echo " layer's build type. Normally the url(s) is read from" + echo " ///required_layer_wheel_inc.cfg." + echo " This option can be used more than once." echo } @@ -105,6 +110,7 @@ rpms_from_centos_3rd_parties_template="rpms_centos3rdparties.lst" rpms_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_pkg_lists rpms_from_layer_repos_dir=${DL_MIRROR_OUTPUT_DIR}/layer_repos image_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_image_inc +wheels_inc_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_wheels_inc build_info_from_layer_build_dir=${DL_MIRROR_OUTPUT_DIR}/layer_build_info tarball_downloads_template="tarball-dl.lst" other_downloads_template="other_downloads.lst" @@ -151,7 +157,7 @@ multiple_dl_flag_check () { # Parse out optional arguments -while getopts "c:Cd:ghI:sl:L:nSuU" o; do +while getopts "c:Cd:ghI:sl:L:nSuUW:" o; do case "${o}" in c) # Pass -c ("use alternate yum.conf") to rpm downloader @@ -173,6 +179,9 @@ while getopts "c:Cd:ghI:sl:L:nSuU" o; do I) set_layer_image_inc_urls "${OPTARG}" ;; + W) + set_layer_wheels_inc_urls "${OPTARG}" + ;; l) # layer set_and_validate_layer "${OPTARG}" @@ -310,6 +319,22 @@ for key in "${!layer_image_inc_urls[@]}"; do fi done +\rm -rf ${wheels_inc_from_layer_build_dir} +mkdir -p ${wheels_inc_from_layer_build_dir} + +for key in "${!layer_wheels_inc_urls[@]}"; do + lower_layer="${key%,*}" + stream="${key#*,}" + url="${layer_wheels_inc_urls[${key}]}" + name_from_url=$(url_to_file_name $url) + list="${wheels_inc_from_layer_build_dir}/${name_from_url}" + curl --silent --fail ${url} > ${list} || + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download from url: ${url}" + exit 1 + fi +done + \rm -rf ${build_info_from_layer_build_dir} mkdir -p ${build_info_from_layer_build_dir} @@ -326,7 +351,7 @@ for key in "${!layer_image_inc_urls[@]}"; do url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' ) name_from_url=$(url_to_file_name $url) dest="${build_info_from_layer_build_dir}/${name_from_url}" - curl --silent --fail ${url} > ${dest} + curl --silent --fail ${url} > ${dest} || if [ $? -ne 0 ]; then echo "ERROR: Failed to download from url: ${url}" exit 1 diff --git a/tb.sh b/tb.sh index 1bb73148..ee4a10ae 100755 --- a/tb.sh +++ b/tb.sh @@ -49,6 +49,7 @@ function run_container { docker run -it --rm \ --name ${TC_CONTAINER_NAME} \ --detach \ + -v /var/run/docker.sock:/var/run/docker.sock \ -v ${LOCALDISK}:/${GUEST_LOCALDISK} \ -v ${HOST_MIRROR_DIR}:/import/mirrors:ro \ -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ diff --git a/toCOPY/generate-cgcs-centos-repo.sh b/toCOPY/generate-cgcs-centos-repo.sh index 934c163e..f3dbf3c2 100755 --- a/toCOPY/generate-cgcs-centos-repo.sh +++ b/toCOPY/generate-cgcs-centos-repo.sh @@ -48,6 +48,11 @@ usage () { echo " layer's build type. Normally the url(s) is read from" echo " ///required_layer_pkgs.cfg." echo " This option can be used more than once." + echo " --layer-wheels-inc-url=,,:" + echo " Override the url for the image include file of a lower" + echo " layer's build type. Normally the url(s) is read from" + echo " ///required_layer_wheel_inc.cfg" + echo " This option can be used more than once." echo " --layer-dir=: Look in provided dir for packages to link to." echo " This option can be used more than once." echo @@ -72,7 +77,7 @@ if [ -z "$MY_REPO" ]; then fi -TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@") +TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-cgcs-centos-repo' -- "$@") if [ $? -ne 0 ]; then echo "getopt error" usage @@ -86,6 +91,7 @@ while true ; do --mirror-dir) mirror_dir=$2 ; shift 2 ;; --layer-dir) layer_dirs+=" ${2/,/ }" ; shift 2 ;; --layer-inc-url) set_layer_image_inc_urls "${2}" ; shift 2 ;; + --layer-wheels-inc-url) set_layer_wheels_inc_urls "${2}" ; shift 2 ;; --layer-pkg-url) set_layer_pkg_urls "${2}" ; shift 2 ;; --config-dir) set_and_validate_config_dir "${2}"; shift 2 ;; --distro) set_and_validate_distro "${2}"; shift 2 ;; @@ -114,6 +120,8 @@ echo "layer_pkg_urls=${layer_pkg_urls[@]}" echo echo "layer_image_inc_urls=${layer_image_inc_urls[@]}" echo +echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}" +echo dest_dir=$MY_REPO/cgcs-centos-repo timestamp="$(date +%F_%H%M)" @@ -126,6 +134,7 @@ TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX) mkdir -p $TMP_LST_DIR lst_file_dir="$TMP_LST_DIR" inc_file_dir="${dest_dir}/layer_image_inc" +wheels_file_dir="${dest_dir}/layer_wheels_inc" build_info_file_dir="${dest_dir}/layer_build_info" rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst" @@ -208,6 +217,37 @@ for key in "${!layer_image_inc_urls[@]}"; do fi done +# +# Dowload wheels inc files from layer_wheels_inc_urls +# +\rm -rf ${wheels_file_dir} +mkdir -p ${wheels_file_dir} +for key in "${!layer_wheels_inc_urls[@]}"; do + lower_layer="${key%,*}" + stream="${key#*,}" + url="${layer_wheels_inc_urls[${key}]}" + name_from_url=$(url_to_file_name "${url}") + + ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}" + + list="${ideal_name}" + + for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do + cp $f ${wheels_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'" + fi + done + + if [ ! -f ${wheels_file_dir}/${list} ]; then + curl --silent --fail ${url} > ${wheels_file_dir}/${list} + if [ $? -ne 0 ]; then + echo "ERROR: Failed to download from url '${url}'" + exit 1 + fi + fi +done + # # Dowload build info files # diff --git a/toCOPY/lst_utils.sh b/toCOPY/lst_utils.sh index dd183042..295fdc92 100644 --- a/toCOPY/lst_utils.sh +++ b/toCOPY/lst_utils.sh @@ -7,6 +7,7 @@ rpms_from_layer_build_template="rpm.lst" image_inc_from_layer_build_template="image.inc" dev_image_inc_from_layer_build_template="image-dev.inc" +wheels_inc_from_layer_build_template="wheels.inc" config_dir=${MY_REPO}/../stx-tools/centos-mirror-tools/config distro="centos" @@ -18,6 +19,9 @@ declare -A layer_pkg_urls # Store urls for image include files of the various layers in an associative array declare -A layer_image_inc_urls +# Store urls for wheel include files of the various layers in an associative array +declare -A layer_wheels_inc_urls + url_to_file_name () { echo "${1}" | sed 's#[:/ ]#-#g' } @@ -74,13 +78,28 @@ set_layer_image_inc_urls () { fi local layer_and_inc_type="${option%,*}" - local layer="${layer_and_build_type%,*}" + local layer="${layer_and_inc_type%,*}" local inc_type="${layer_and_inc_type#*,}" local layer_image_inc_url="${option##*,}" layer_image_inc_urls["${layer_and_inc_type}"]="${layer_image_inc_url}" } +set_layer_wheels_inc_urls () { + local option="${1}" + + if [ "${option}" == "" ]; then + return + fi + + local layer_and_stream="${option%,*}" + local layer="${layer_and_stream%,*}" + local stream="${layer_and_stream#*,}" + local layer_wheels_inc_url="${option##*,}" + + layer_wheels_inc_urls["${layer_and_stream}"]="${layer_wheels_inc_url}" +} + set_layer_pkg_urls () { local option="${1}" @@ -120,6 +139,30 @@ read_layer_image_inc_urls () { done < "${cfg}" } +read_layer_wheels_inc_urls () { + local layer="${1}" + local cfg="${config_dir}/${distro}/${layer}/required_layer_wheel_inc.cfg" + local line="" + local key + + if [ ! -f "${cfg}" ]; then + return 0; + fi + + # Clear all pre-existing entries + for key in "${!layer_wheels_inc_urls[@]}"; do + unset layer_wheels_inc_urls[${key}] + done + + while read line; do + line=$(echo "${line}" | sed 's/^[ \t]*//;s/[ \t]*$//' | grep '^[^#]') + if [ "${line}" == "" ]; then + continue + fi + set_layer_wheels_inc_urls "${line}" + done < "${cfg}" +} + read_layer_pkg_urls () { local layer="${1}" local cfg="${config_dir}/${distro}/${layer}/required_layer_pkgs.cfg" @@ -172,6 +215,7 @@ set_and_validate_distro () { if [ -d ${config_dir}/${distro}/${layer} ]; then read_layer_pkg_urls ${layer} read_layer_image_inc_urls ${layer} + read_layer_wheels_inc_urls ${layer} else echo "Warning: layer ${layer} not defined for distro '${distro}', please provide a valid layer via '-l '" fi @@ -191,6 +235,7 @@ set_and_validate_layer () { read_layer_pkg_urls ${layer} read_layer_image_inc_urls ${layer} + read_layer_wheels_inc_urls ${layer} } # Pick up value of the config_dir from environment if set @@ -205,3 +250,4 @@ fi read_layer_pkg_urls ${layer} read_layer_image_inc_urls ${layer} +read_layer_wheels_inc_urls ${layer}