diff --git a/centos-mirror-tools/dl_rpms.sh b/centos-mirror-tools/dl_rpms.sh index 767761a6..ac51ea6e 100755 --- a/centos-mirror-tools/dl_rpms.sh +++ b/centos-mirror-tools/dl_rpms.sh @@ -1,19 +1,60 @@ #!/bin/bash -e +# +# SPDX-License-Identifier: Apache-2.0 +# # download RPMs/SRPMs from different sources. # this script was originated by Brian Avery, and later updated by Yong Hu +usage() { + echo "$0 [-n] [-c ] " + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations" + echo " -c: Use an alternate yum.conf rather than the system file" + echo " rpm_list: a list of RPM files to be downloaded." + echo " match_level: value could be L1, L2 or L3:" + echo " L1: use name, major version and minor version:" + echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" + echo " L2: use name and major version:" + echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" + echo " L3: use name:" + echo " using vim to search vim-7.4.160-2.el7.src.rpm" + echo " from_where: where to download the RPMs: 'centos'from CentOS Repos," + echo " otherwise from 3rd-party websets" + echo "" +} + +# By default, we use "sudo" and we don't use a local yum.conf. These can +# be overridden via flags. +SUDOCMD="sudo -E" +YUMCONFOPT="" + +# Parse option flags +while getopts "c:nh" o; do + case "${o}" in + n) + # No-sudo + SUDOCMD="" + ;; + c) + # Use an alternate yum.conf + YUMCONFOPT="-c $OPTARG" + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + if [ $# -lt 3 ]; then - echo "$0 " - echo "rpm_list: a list of RPM files to be downloaded." - echo "match_level: value could be L1, L2 or L3:" - echo " L1: use name, major version and minor version:" - echo " vim-7.4.160-2.el7 to search vim-7.4.160-2.el7.src.rpm" - echo " L2: use name and major version:" - echo " using vim-7.4.160 to search vim-7.4.160-2.el7.src.rpm" - echo " L3: use name:" - echo " using vim to search vim-7.4.160-2.el7.src.rpm" - echo "from_where: where to download the RPMs: 'centos'from CentOS Repos," - echo "otherwise from 3rd-party websets" + usage exit -1 fi @@ -83,11 +124,11 @@ download () { fi echo " ------ using $SFILE to search $rpm_name ------" if [ "$_type" == "src" ];then - download_cmd="sudo -E yumdownloader -q -C --source $SFILE" - download_url_cmd="sudo -E yumdownloader --urls -q -C --source $SFILE" + download_cmd="${SUDOCMD} yumdownloader -q ${YUMCONFOPT} -C --source $SFILE" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q ${YUMCONFOPT}-C --source $SFILE" else - download_cmd="sudo -E yumdownloader -q -C $SFILE --archlist=noarch,x86_64" - download_url_cmd="sudo -E yumdownloader --urls -q -C $SFILE --archlist=noarch,x86_64" + download_cmd="${SUDOCMD} yumdownloader -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" + download_url_cmd="${SUDOCMD} yumdownloader --urls -q -C ${YUMCONFOPT} $SFILE --archlist=noarch,x86_64" fi else rpm_name=`echo $ff | cut -d"#" -f1-1` @@ -95,13 +136,13 @@ download () { download_cmd="wget $rpm_url" SFILE=$rpm_name fi - echo "--> run: $download_cmd" + echo "--> run: $download_cmd" if [ "$_type" == "src" ]; then if [ ! -e $MDIR_SRC/$rpm_name ]; then echo "Looking for $rpm_name" if $download_cmd ; then # Success! Record download URL. - # Use 'sort --unique' because sometimes + # Use 'sort --unique' because sometimes # yumdownloader reports the url twice $download_url_cmd | sort --unique >> $URL_SRPMS @@ -144,7 +185,7 @@ download () { } # prime the cache -sudo -E yum makecache +${SUDOCMD} yum ${YUMCONFOPT} makecache #go to download *.noarch.rpm files noarch_rpms=`echo "$(cat $rpms_list | grep '.noarch.rpm')"` diff --git a/centos-mirror-tools/dl_tarball.sh b/centos-mirror-tools/dl_tarball.sh index fe090ad9..c49c7ccd 100755 --- a/centos-mirror-tools/dl_tarball.sh +++ b/centos-mirror-tools/dl_tarball.sh @@ -93,6 +93,10 @@ for line in $(cat $tarball_file); do if [[ "$line" =~ ^'!' ]]; then tarball_name="${tarball_name//!/}" + if [ -e "$output_tarball/$tarball_name" ]; then + echo "Already have $tarball_name" + continue + fi echo $tarball_name pushd $output_tarball if [ "$tarball_name" = "integrity-kmod-e6aef069.tar.gz" ]; then diff --git a/centos-mirror-tools/download_mirror.sh b/centos-mirror-tools/download_mirror.sh index f21d4018..3da64d60 100755 --- a/centos-mirror-tools/download_mirror.sh +++ b/centos-mirror-tools/download_mirror.sh @@ -1,4 +1,65 @@ #!/bin/bash -e +# +# SPDX-License-Identifier: Apache-2.0 +# + +usage() { + echo "$0 [-n] [-c ] [-g]" + echo "" + echo "Options:" + echo " -n: Do not use sudo when performing operations (option passed on to" + echo " subscripts when appropriate)" + echo " -c: Use an alternate yum.conf rather than the system file (option passed" + echo " on to subscripts when appropriate)" + echo " -g: do not change group IDs of downloaded artifacts" + echo "" +} + +rpm_downloader="./dl_rpms.sh" +tarball_downloader="./dl_tarball.sh" +other_downloader="./dl_other_from_centos_repo.sh" + +# track optional arguments +change_group_ids=1 +use_system_yum_conf=1 +rpm_downloader_extra_args="" +tarball_downloader_extra_args="" + +# lst files to use as input +rpms_from_3rd_parties="./rpms_from_3rd_parties.lst" +rpms_from_centos_repo="./rpms_from_centos_repo.lst" +rpms_from_centos_3rd_parties="./rpms_from_centos_3rd_parties.lst" +other_downloads="./other_downloads.lst" + +# Parse out optional -c or -n arguments +while getopts "c:ngh" o; do + case "${o}" in + n) + # Pass -n ("no-sudo") to rpm downloader + rpm_downloader_extra_args="${rpm_downloader_extra_args} -n" + ;; + c) + # Pass -c ("use alternate yum.conf") to rpm downloader + use_system_yum_conf=0 + rpm_downloader_extra_args="${rpm_downloader_extra_args} -c ${OPTARG}" + ;; + g) + # Do not attempt to change group IDs on downloaded packages + change_group_ids=0 + ;; + h) + # Help + usage + exit 0 + ;; + *) + usage + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + echo "--------------------------------------------------------------" echo "WARNING: this script HAS TO access internet (http/https/ftp)," @@ -16,56 +77,51 @@ need_file(){ } # Check extistence of prerequisites files -need_file dl_rpms.sh dl_other_from_centos_repo.sh dl_tarball.sh -need_file rpms_from_3rd_parties.lst -need_file rpms_from_centos_3rd_parties.lst -need_file rpms_from_centos_repo.lst -need_file other_downloads.lst +need_file ${rpm_downloader} ${other_downloader} ${tarball_downloader} +need_file ${rpms_from_3rd_parties} +need_file ${rpms_from_centos_3rd_parties} +need_file ${rpms_from_centos_repo} +need_file ${other_downloads} need_file tarball-dl.lst mvn-artifacts.lst #download RPMs/SRPMs from 3rd_party websites (not CentOS repos) by "wget" echo "step #1: start downloading RPMs/SRPMs from 3rd-party websites..." -# Restore StarlingX_3rd repos from backup -REPO_SOURCE_DIR=/localdisk/yum.repos.d -REPO_DIR=/etc/yum.repos.d -if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then - \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ +if [ ${use_system_yum_conf} -eq 0 ]; then + # Restore StarlingX_3rd repos from backup + REPO_SOURCE_DIR=/localdisk/yum.repos.d + REPO_DIR=/etc/yum.repos.d + if [ -d $REPO_SOURCE_DIR ] && [ -d $REPO_DIR ]; then + \cp -f $REPO_SOURCE_DIR/*.repo $REPO_DIR/ + fi fi -rpm_downloader="./dl_rpms.sh" -$rpm_downloader ./rpms_from_3rd_parties.lst L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_3rd_parties} L1 3rd | tee ./logs/log_download_rpms_from_3rd_party.txt retcode=${PIPESTATUS[0]} if [ $retcode -ne 0 ];then echo "ERROR: something wrong with downloading, please check the log!!" fi # download RPMs/SRPMs from 3rd_party repos by "yumdownloader" -$rpm_downloader ./rpms_from_centos_3rd_parties.lst L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt - -# deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. -\rm -f $REPO_DIR/StarlingX_3rd*.repo +$rpm_downloader ${rpm_downloader_extra_args} ${rpms_from_centos_3rd_parties} L1 3rd-centos | tee ./logs/log_download_rpms_from_centos_3rd_parties_L1.txt +if [ ${use_system_yum_conf} -eq 1 ]; then + # deleting the StarlingX_3rd to avoid pull centos packages from the 3rd Repo. + \rm -f $REPO_DIR/StarlingX_3rd*.repo +fi echo "step #2: start 1st round of downloading RPMs and SRPMs with L1 match criteria..." #download RPMs/SRPMs from CentOS repos by "yumdownloader" $rpm_downloader ./rpms_from_centos_repo.lst L1 centos | tee ./logs/log_download_rpms_from_centos_L1.txt + retcode=${PIPESTATUS[0]} -if [ $retcode -eq 0 ]; then +if [ $retcode -ne 0 ]; then echo "finish 1st round of RPM downloading successfully!" if [ -e "./output/centos_rpms_missing_L1.txt" ]; then missing_num=`wc -l ./output/centos_rpms_missing_L1.txt | cut -d " " -f1-1` if [ "$missing_num" != "0" ];then echo "ERROR: -------RPMs missing $missing_num in yumdownloader with L1 match ---------------" fi - #echo "start 2nd round of downloading Binary RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_rpms_missing_L1.txt L2 centos | tee ./logs/log_download_rpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of RPM downloading successfully!" - # if [ -e "./output/rpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing RPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi if [ -e "./output/centos_srpms_missing_L1.txt" ]; then @@ -73,14 +129,6 @@ if [ $retcode -eq 0 ]; then if [ "$missing_num" != "0" ];then echo "ERROR: --------- SRPMs missing $missing_num in yumdownloader with L1 match ---------------" fi - #echo "start 2nd round of downloading Source RPMs with L2 match criteria..." - #$rpm_downloader ./output/centos_srpms_missing_L1.txt L2 centos | tee ./logs/log_download_srpms_from_centos_L2.txt - #if [ $? == 0 ]; then - # echo "finish 2nd round of SRPM downloading successfully!" - # if [ -e "./srpms_missing_L2.txt" ]; then - # echo "WARNING: we're still missing SRPMs listed in ./rpms_missing_L2.txt !" - # fi - #fi fi else echo "finish 1st round with failures!" @@ -93,9 +141,10 @@ find ./output -type f -name "*.rpm" | xargs rpm -K | grep -i "MISSING KEYS" > ./ find ./output -name "*.i686.rpm" | tee ./output/all_i686.txt find ./output -name "*.i686.rpm" | xargs rm -f -line1=`wc -l rpms_from_3rd_parties.lst | cut -d " " -f1-1` -line2=`wc -l rpms_from_centos_repo.lst | cut -d " " -f1-1` -line3=`wc -l rpms_from_centos_3rd_parties.lst | cut -d " " -f1-1` +line1=`wc -l ${rpms_from_3rd_parties} | cut -d " " -f1-1` +line2=`wc -l ${rpms_from_centos_repo} | cut -d " " -f1-1` +line3=`wc -l ${rpms_from_centos_3rd_parties} | cut -d " " -f1-1` + let total_line=$line1+$line2+$line3 echo "We expect to download $total_line RPMs." num_of_downloaded_rpms=`find ./output -type f -name "*.rpm" | wc -l | cut -d" " -f1-1` @@ -104,14 +153,15 @@ if [ "$total_line" != "$num_of_downloaded_rpms" ]; then echo "WARNING: Not the same number of RPMs in output as RPMs expected to be downloaded, need to check outputs and logs." fi -# change "./output" and sub-folders to 751 (cgcs) group -chown 751:751 -R ./output +if [ $change_group_ids -eq 1 ]; then + # change "./output" and sub-folders to 751 (cgcs) group + chown 751:751 -R ./output +fi echo "step #3: start downloading other files ..." -other_downloader="./dl_other_from_centos_repo.sh" -$other_downloader ./other_downloads.lst ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt +${other_downloader} ${other_downloads} ./output/stx-r1/CentOS/pike/Binary/ | tee ./logs/log_download_other_files_centos.txt retcode=${PIPESTATUS[0]} if [ $retcode -eq 0 ];then echo "step #3: done successfully" @@ -120,7 +170,7 @@ fi # StarlingX requires a group of source code pakages, in this section # they will be downloaded. echo "step #4: start downloading tarball compressed files" -./dl_tarball.sh +${tarball_downloader} ${tarball_downloader_extra_args} echo "IMPORTANT: The following 3 files are just bootstrap versions. Based" echo "on them, the workable images for StarlingX could be generated by" diff --git a/toCOPY/generate-cgcs-centos-repo.sh b/toCOPY/generate-cgcs-centos-repo.sh index 0bd9cf74..a8d89d99 100755 --- a/toCOPY/generate-cgcs-centos-repo.sh +++ b/toCOPY/generate-cgcs-centos-repo.sh @@ -27,37 +27,66 @@ comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml mock_cfg_dest_file=$MY_REPO/cgcs-centos-repo/mock.cfg.proto comps_xml_dest_file=$MY_REPO/cgcs-centos-repo/Binary/comps.xml -if [[ ( ! -d $mirror_dir/Binary ) || ( ! -d $mirror_dir/Source ) ]]; then - echo "The mirror $mirror_dir doesn't has the Binary and Source" +lst_file_dir="$MY_REPO_ROOT_DIR/stx-tools/centos-mirror-tools" +rpm_lst_files="rpms_from_3rd_parties.lst rpms_from_centos_3rd_parties.lst rpms_from_centos_repo.lst" +other_lst_file="other_downloads.lst" +missing_rpms_file=missing.txt + +rm -f ${missing_rpms_file} + +# Strip trailing / from mirror_dir if it was specified... +mirror_dir=$(echo ${mirror_dir} | sed "s%/$%%") + +if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then + echo "The mirror ${mirror_dir} doesn't has the Binary and Source" echo "folders. Please provide a valid mirror" exit -1 fi -if [ ! -d "$dest_dir" ]; then - mkdir -p "$dest_dir" +if [ ! -d "${dest_dir}" ]; then + mkdir -p "${dest_dir}" fi for t in "Binary" "Source" ; do - target_dir=$dest_dir/$t + target_dir=${dest_dir}/$t if [ ! -d "$target_dir" ]; then mkdir -p "$target_dir" else mv -f "$target_dir" "$target_dir-backup-$timestamp" mkdir -p "$target_dir" fi +done - pushd "$mirror_dir/$t"|| exit 1 - find . -type d -exec mkdir -p "${target_dir}"/{} \; - all_files=$(find . -type f -name "*") - popd || exit 1 +unsuccessful_file=0 +for lst_file in ${rpm_lst_files} ; do + grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do + if [ -z "${rpmname}" ]; then + continue + fi + mirror_file=$(find ${mirror_dir} -name ${rpmname}) + if [ -z "${mirror_file}" ]; then + echo "Error -- could not find requested ${rpmname} in ${mirror_dir}" + echo ${rpmname} >> ${missing_rpms_file} + unsuccessful_file=1 + continue + fi - - for ff in $all_files; do + # Great, we found the file! Let's strip the mirror_dir prefix from it... + ff=$(echo ${mirror_file} | sed "s%^${mirror_dir}/%%") f_name=$(basename "$ff") sub_dir=$(dirname "$ff") - ln -sf "$mirror_dir/$t/$ff" "$target_dir/$sub_dir" - echo "Creating symlink for $target_dir/$sub_dir/$f_name" - echo "------------------------------" + + # Make sure we have a subdir (so we don't symlink the first file as + # the subdir name) + mkdir -p ${dest_dir}/${sub_dir} + + # Link it! + echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}" + ln -sf "${mirror_dir}/$ff" "${dest_dir}/${sub_dir}" + if [ $? -ne 0 ]; then + echo "Failed ${mirror_file}: ln -sf \"${mirror_dir}/$ff\" \"${dest_dir}/${sub_dir}\"" + unsuccessful_file=1 + fi done done @@ -75,13 +104,30 @@ fi echo "Copying mock.cfg.proto and comps.xml files." if [ -f "$mock_cfg_dest_file" ]; then - cp "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" + \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp" fi cp "$mock_cfg_file" "$mock_cfg_dest_file" if [ -f "$comps_xml_dest_file" ]; then - cp "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" + \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp" fi cp "$comps_xml_file" "$comps_xml_dest_file" -echo "Done" +# Populate the contents from other list files +cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do + if [ "${targettype}" == "folder" ]; then + echo "Creating folder ${item}" + mkdir -p $MY_REPO/cgcs-centos-repo/Binary/${item} + fi + + if [ "${targettype}" == "file" ]; then + mkdir -p $MY_REPO/cgcs-centos-repo/Binary/$(dirname ${item}) + echo "Creating symlink for $MY_REPO/cgcs-centos-repo/Binary/${item}" + ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/cgcs-centos-repo/Binary/${item} + fi +done + +echo "Done creating repo directory" +if [ ${unsuccessful_file} -ne 0 ]; then + echo "WARNING: Some targets could not be found. Your repo may be incomplete." +fi diff --git a/toCOPY/populate_downloads.sh b/toCOPY/populate_downloads.sh new file mode 100755 index 00000000..1df08e10 --- /dev/null +++ b/toCOPY/populate_downloads.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# + +usage () { + echo "$0 " +} + +if [ $# -ne 1 ]; then + usage + exit -1 +fi + +if [ -z "$MY_REPO" ]; then + echo "\$MY_REPO is not set. Ensure you are running this script" + echo "from the container and \$MY_REPO points to the root of" + echo "your folder tree." + exit -1 +fi + +mirror_dir=$1 +tarball_lst=${MY_REPO_ROOT_DIR}/stx-tools/centos-mirror-tools/tarball-dl.lst +downloads_dir=${MY_REPO}/stx/downloads +extra_downloads="mlnx-ofa_kernel-4.3-OFED.4.3.3.0.2.1.gcf60532.src.rpm libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm rdma-core-43mlnx1-1.43302.src.rpm" + +mkdir -p ${MY_REPO}/stx/downloads + +grep -v "^#" ${tarball_lst} | while read x; do + if [ -z "$x" ]; then + continue + fi + + # Get first element of item & strip leading ! if appropriate + tarball_file=$(echo $x | sed "s/#.*//" | sed "s/^!//") + + # put the file in downloads + source_file=$(find ${mirror_dir}/downloads -name "${tarball_file}") + if [ -z ${source_file} ]; then + echo "Could not find ${tarball_file}" + else + rel_path=$(echo ${source_file} | sed "s%^${mirror_dir}/downloads/%%") + rel_dir_name=$(dirname ${rel_path}) + if [ ! -e ${downloads_dir}/${rel_dir_name}/${tarball_file} ]; then + mkdir -p ${downloads_dir}/${rel_dir_name} + ln -sf ${source_file} ${downloads_dir}/${rel_dir_name}/ + fi + fi +done + +for x in ${extra_downloads}; do + ln -sf ${mirror_dir}/downloads/$x ${downloads_dir} +done