Merge "StarlingX: Tar Compressed files repository"
This commit is contained in:
commit
f240ab8b52
|
@ -105,6 +105,11 @@ if [ $? == 0 ];then
|
||||||
echo "step #3: done successfully"
|
echo "step #3: done successfully"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# StarlingX requires a group of source code pakages, in this section
|
||||||
|
# they will be downloaded.
|
||||||
|
echo "step #4: start downloading tarball compressed files"
|
||||||
|
./tarball-dl.sh
|
||||||
|
|
||||||
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
|
echo "IMPORTANT: The following 3 files are just bootstrap versions. Based"
|
||||||
echo "on them, the workable images for StarlingX could be generated by"
|
echo "on them, the workable images for StarlingX could be generated by"
|
||||||
echo "running \"update-pxe-network-installer\" command after \"build-iso\""
|
echo "running \"update-pxe-network-installer\" command after \"build-iso\""
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,44 @@
|
||||||
|
dpdk-17.05.2.tar.gz#dpdk-stable-17.05.2#https://fast.dpdk.org/rel/dpdk-17.05.2.tar.gz
|
||||||
|
dpkg_1.18.24.tar.xz#dpkg-1.18.24#http://http.debian.net/debian/pool/main/d/dpkg/dpkg_1.18.24.tar.xz
|
||||||
|
drbd-8.4.3.tar.gz#drbd-8.4.3#http://www.linbit.com/downloads/drbd/8.4/archive/drbd-8.4.3.tar.gz
|
||||||
|
drbd-8.4.7-1.tar.gz#drbd-8.4.7-1#http://www.linbit.com/downloads/drbd/8.4/drbd-8.4.7-1.tar.gz
|
||||||
|
dtc-1.4.4.tar.gz#dtc-1.4.4#https://www.kernel.org/pub/software/utils/dtc/dtc-1.4.4.tar.gz
|
||||||
|
e1000e-3.3.6.tar.gz#e1000e-3.3.6#https://sourceforge.net/projects/e1000/files/e1000e%20stable/3.3.6/e1000e-3.3.6.tar.gz
|
||||||
|
expect-lite_4.9.0.tar.gz#expect-lite.proj#https://sourceforge.net/projects/expect-lite/files/expect-lite/expect-lite_4.9.0/expect-lite_4.9.0.tar.gz
|
||||||
|
gnulib-ffc927e.tar.gz#gnulib-ffc927e#http://git.savannah.gnu.org/cgit/gnulib.git/snapshot/gnulib-ffc927eef29016a5219cd969daad8928af6a1f4d.tar.gz
|
||||||
|
i40e-2.4.3.tar.gz#i40e-2.4.3#https://sourceforge.net/projects/e1000/files/i40e%20stable/2.4.3/i40e-2.4.3.tar.gz/download
|
||||||
|
i40evf-3.4.2.tar.gz#i40evf-3.4.2#https://sourceforge.net/projects/e1000/files/i40evf%20stable/3.4.2/i40evf-3.4.2.tar.gz/download
|
||||||
|
ibsh-0.3e.tar.gz#cgcs-users-1.0#https://sourceforge.net/projects/ibsh/files/ibsh/ibsh-0.3e/ibsh-0.3e.tar.gz/download
|
||||||
|
!integrity-kmod-668a8270.tar.gz#integrity#http://git.infradead.org/users/jjs/linux-tpmdd.git/snapshot/fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
ixgbe-5.2.3.tar.gz#ixgbe-5.2.3#https://sourceforge.net/projects/e1000/files/ixgbe%20stable/5.2.3/ixgbe-5.2.3.tar.gz/download
|
||||||
|
ixgbevf-4.2.1.tar.gz#ixgbevf-4.2.1#https://sourceforge.net/projects/e1000/files/ixgbevf%20stable/4.2.1/ixgbevf-4.2.1.tar.gz/download
|
||||||
|
keycodemapdb-8370ba8.tar.gz#keycodemapdb#https://github.com/CendioOssman/keycodemapdb/tarball/8370ba868ef76f816842ba934a2e57d43c882a00
|
||||||
|
kvm-unit-tests.git-4ea7633.tar.bz2#kvm-unit-tests#https://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git/snapshot/kvm-unit-tests-20171020.tar.gz
|
||||||
|
ldapscripts-2.0.8.tgz#ldapscripts-2.0.8#https://sourceforge.net/projects/ldapscripts/files/ldapscripts/ldapscripts-2.0.8/ldapscripts-2.0.8.tgz/download
|
||||||
|
libtpms-0.6.0-4f0d59d.tar.gz#libtpms-0.6.0#https://github.com/stefanberger/libtpms/tarball/c421ca0f4d00c0caceeda8d62c1efb2b7e47db04
|
||||||
|
lldpd-0.9.0.tar.gz#lldpd-0.9.0#https://media.luffy.cx/files/lldpd/lldpd-0.9.0.tar.gz
|
||||||
|
!mariadb-10.1.28.tar.gz#mariadb-10.1.28#https://github.com/MariaDB/server/archive/mariadb-10.1.28.tar.gz
|
||||||
|
!MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64.tgz#MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64#http://www.mellanox.com/downloads/ofed/MLNX_OFED-4.2-1.2.0.0/MLNX_OFED_LINUX-4.2-1.2.0.0-rhel7.4-x86_64.tgz
|
||||||
|
!MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64.tgz#MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64#http://www.mellanox.com/downloads/ofed/MLNX_OFED-4.3-1.0.1.0/MLNX_OFED_LINUX-4.3-1.0.1.0-rhel7.4-x86_64.tgz
|
||||||
|
!mvn.repo.tgz#mvn#https://repo.maven.apache.org/maven2
|
||||||
|
openvswitch-2.8.1.tar.gz#openvswitch-2.8.1#http://openvswitch.org/releases/openvswitch-2.8.1.tar.gz
|
||||||
|
python-cephclient-v0.1.0.5.tar.gz#python-cephclient-0.1.0.5#https://github.com/dmsimard/python-cephclient/archive/v0.1.0.5.tar.gz
|
||||||
|
python-setuptools-v38.5.1.tar.gz#setuptools-38.5.1#https://github.com/pypa/setuptools/archive/v38.5.1.tar.gz
|
||||||
|
python-smartpm-1.4.1.tar.gz#smart-1.4.1#http://launchpad.net/smart/trunk/1.4.1/+download/smart-1.4.1.tar.bz2
|
||||||
|
!qat1.7.upstream.l.1.0.3-42.tar.gz#quickassist#https://01.org/sites/default/files/downloads/intelr-quickassist-technology/qat1.7.upstream.l.1.0.3-42.tar.gz
|
||||||
|
requests-toolbelt-0.5.1.tar.gz#requests-toolbelt-0.5.1#https://github.com/requests/toolbelt/archive/0.5.1.tar.gz
|
||||||
|
rpm-4.14.0.tar.bz2#rpm-4.14.0#https://ftp.osuosl.org/pub/rpm/releases/rpm-4.14.x/rpm-4.14.0.tar.bz2
|
||||||
|
swtpm-0.1.0-253eac5.tar.gz#swtpm-0.1.0#https://github.com/stefanberger/swtpm/tarball/1303be7d03294fb02204cb8242f366cbf0da076d
|
||||||
|
!tpm-kmod-668a8270.tar.gz#tpm#http://git.infradead.org/users/jjs/linux-tpmdd.git/snapshot/fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
!tss2-930.tar.gz#tss2-930#https://sourceforge.net/code-snapshots/git/i/ib/ibmtpm20tss/tss.git/ibmtpm20tss-tss-52539cb81c811c973b26ed23fafd28a700b7cc78.zip
|
||||||
|
puppet-boolean-22b726dd78b0a60a224cc7054aebbf28e9306f62.tar.gz#puppet-boolean#https://github.com/voxpupuli/puppet-boolean/tarball/22b726dd78b0a60a224cc7054aebbf28e9306f62
|
||||||
|
puppet-dnsmasq-cff07e90890662972c97684a2baee964f68ff3ed.tar.gz#packstack/puppet/modules/dnsmasq/#https://github.com/procore/puppet-dnsmasq/tarball/cff07e90890662972c97684a2baee964f68ff3ed
|
||||||
|
puppet-filemapper-9b53310278e76827bbe12a36cc6470d77071abb2.tar.gz#packstack/puppet/modules/filemapper#https://github.com/voxpupuli/puppet-filemapper/tarball/9b53310278e76827bbe12a36cc6470d77071abb2
|
||||||
|
puppetlabs-create_resources-4639819a7f3a4fa9310d2ba583c63e467df7e2c3.tar.gz#packstack/puppet/modules/create_resources#https://github.com/puppetlabs/puppetlabs-create_resources/tarball/4639819a7f3a4fa9310d2ba583c63e467df7e2c3
|
||||||
|
puppetlabs-drbd-496b3ba9cd74a2d12636f9e90a718739a5451169.tar.gz#puppetlabs-drbd#https://github.com/voxpupuli/puppet-drbd/tarball/496b3ba9cd74a2d12636f9e90a718739a5451169
|
||||||
|
puppetlabs-lvm-d0283da637ae24550fb4ba109a48ef8d5d8c8b84.tar.gz#packstack/puppet/modules/lvm#https://github.com/puppetlabs/puppetlabs-lvm/tarball/d0283da637ae24550fb4ba109a48ef8d5d8c8b84
|
||||||
|
puppetlabs-postgresql-d022a56b28b2174456fc0f6adc51a4b54493afad.tar.gz#puppetlabs-postgresql#https://github.com/puppetlabs/puppetlabs-postgresql/tarball/d022a56b28b2174456fc0f6adc51a4b54493afad
|
||||||
|
puppet-ldap-480f13af6d17d1d3fcf0dc7b4bd04b49fa4099e1.tar.gz#puppet-ldap-master#https://github.com/torian/puppet-ldap/tarball/480f13af6d17d1d3fcf0dc7b4bd04b49fa4099e1
|
||||||
|
puppet-network-7deacd5fdc22c0543455878a8d1872f2f5417c1d.tar.gz#packstack/puppet/modules/network#https://github.com/voxpupuli/puppet-network/tarball/7deacd5fdc22c0543455878a8d1872f2f5417c1d
|
||||||
|
puppet-nslcd-b8c19b1ada89865f2e50758e054583798ad8011a.tar.gz#packstack/puppet/modules/nslcd#https://github.com/jlyheden/puppet-nslcd/tarball/b8c19b1ada89865f2e50758e054583798ad8011a
|
||||||
|
puppi-c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81.tar.gz#puppi-master#https://github.com/example42/puppi/tarball/c1c47f4edfd761d1bbde32a75da0c3fa7cc93a81
|
|
@ -0,0 +1,195 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
# The build of StarlingX relies, besides RPM Binaries and Sources, in this
|
||||||
|
# repository which is a collection of packages in the form of Tar Compressed
|
||||||
|
# files and 3 RPMs obtained from a Tar Compressed file. This script and a text
|
||||||
|
# file containing a list of packages enable their download and the creation
|
||||||
|
# of the repository based in common and specific requirements dictated
|
||||||
|
# by the StarlingX building system recipes.
|
||||||
|
|
||||||
|
# input files:
|
||||||
|
# The files tarball-dl.lst and mvn-artifacts.lst contain the list of packages
|
||||||
|
# and artifacts for building this sub-mirror.
|
||||||
|
script_path="$(dirname $(readlink -f $0))"
|
||||||
|
tarball_file="$script_path/tarball-dl.lst"
|
||||||
|
mvn_artf_file="$script_path/mvn-artifacts.lst"
|
||||||
|
|
||||||
|
if [ ! -e $tarball_file -o ! -e $mvn_artf_file ];then
|
||||||
|
echo "$download_list does not exist, please have a check!"
|
||||||
|
exit -1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The 2 categories we can divide the list of packages in the output directory:
|
||||||
|
# - General hosted under "downloads" output directory.
|
||||||
|
# - Puppet hosted under "downloads/puppet" output directory.
|
||||||
|
# to be populated under $MY_REPO/addons/wr-cgcs/layers/cgcs/downloads/puppet
|
||||||
|
|
||||||
|
output_main="output"
|
||||||
|
output_path=$output_main/stx-r1/CentOS/pike
|
||||||
|
output_tarball=$output_path/downloads
|
||||||
|
output_puppet=$output_tarball/puppet
|
||||||
|
|
||||||
|
mkdir -p $output_tarball
|
||||||
|
mkdir -p $output_puppet
|
||||||
|
|
||||||
|
# Download function using wget command
|
||||||
|
|
||||||
|
download_package() {
|
||||||
|
wget --spider $1
|
||||||
|
if [[ $? != 0 ]]; then
|
||||||
|
echo "$1 is broken"
|
||||||
|
else
|
||||||
|
wget -t 5 --wait=15 $1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# This script will iterate over the tarball.lst text file and execute specific
|
||||||
|
# tasks based on the name of the package:
|
||||||
|
|
||||||
|
for line in $(cat $tarball_file); do
|
||||||
|
|
||||||
|
# A line from the text file starting with "#" character is ignored
|
||||||
|
|
||||||
|
if [[ "$line" =~ ^'#' ]]; then
|
||||||
|
echo "Skip $line"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# The text file contains 3 columns separated by a character "#"
|
||||||
|
# - Column 1, name of package including extensions as it is referenced
|
||||||
|
# by the build system recipe, character "!" at the beginning of the name package
|
||||||
|
# denotes special handling is required tarball_name=`echo $line | cut -d"#" -f1-1`
|
||||||
|
# - Column 2, name of the directory path after it is decompressed as it is
|
||||||
|
# referenced in the build system recipe.
|
||||||
|
# - Column 3, the URL for the package download
|
||||||
|
|
||||||
|
tarball_name=$(echo $line | cut -d"#" -f1-1)
|
||||||
|
directory_name=$(echo $line | cut -d"#" -f2-2)
|
||||||
|
tarball_url=$(echo $line | cut -d"#" -f3-3)
|
||||||
|
|
||||||
|
# We have 6 packages from the text file starting with the character "!":
|
||||||
|
# they require special handling besides the common process: remove directory,
|
||||||
|
# remove text from some files, clone a git repository, etc.
|
||||||
|
|
||||||
|
if [[ "$line" =~ ^'!' ]]; then
|
||||||
|
tarball_name="${tarball_name//!/}"
|
||||||
|
echo $tarball_name
|
||||||
|
pushd $output_tarball
|
||||||
|
if [ "$tarball_name" = "integrity-kmod-668a8270.tar.gz" ]; then
|
||||||
|
download_package $tarball_url
|
||||||
|
tar xf fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
mv linux-tpmdd-fd5c786/security/integrity/ $directory_name
|
||||||
|
tar czvf $tarball_name $directory_name
|
||||||
|
rm -rf linux-tpmdd-fd5c786
|
||||||
|
rm fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
elif [ "$tarball_name" = "mariadb-10.1.28.tar.gz" ]; then
|
||||||
|
download_package $tarball_url
|
||||||
|
mkdir $directory_name
|
||||||
|
tar xf $tarball_name --strip-components 1 -C $directory_name
|
||||||
|
rm $tarball_name
|
||||||
|
pushd $directory_name
|
||||||
|
rm -rf storage/tokudb
|
||||||
|
rm ./man/tokuft_logdump.1 ./man/tokuftdump.1
|
||||||
|
sed -e s/tokuft_logdump.1//g -i man/CMakeLists.txt
|
||||||
|
sed -e s/tokuftdump.1//g -i man/CMakeLists.txt
|
||||||
|
popd
|
||||||
|
tar czvf $tarball_name $directory_name
|
||||||
|
rm -rf $directory_name
|
||||||
|
# The mvn.repo.tgz tarball will be created downloading a serie of
|
||||||
|
# of maven artifacts described in mvn-artifacts file.
|
||||||
|
elif [ "$tarball_name" = "mvn.repo.tgz" ]; then
|
||||||
|
mkdir -p "$directory_name"
|
||||||
|
if [ ! -f "$mvn_artf_file" ]; then
|
||||||
|
echo "$mvn_artf_file no found" 1>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
while read -r artf; do
|
||||||
|
echo "download: $(basename $artf)"
|
||||||
|
wget "$tarball_url/$artf" -P "$directory_name/$(dirname $artf)"
|
||||||
|
done < "$mvn_artf_file"
|
||||||
|
|
||||||
|
# Create tarball
|
||||||
|
tar -zcvf "$tarball_name" -C "$directory_name"/ .
|
||||||
|
rm -rf "$directory_name"
|
||||||
|
mv "$tarball_name" "$download_path"
|
||||||
|
elif [[ "$tarball_name" =~ ^'MLNX_OFED_LINUX' ]]; then
|
||||||
|
pkg_version=$(echo "$tarball_name" | cut -d "-" -f2-3)
|
||||||
|
srpm_path="MLNX_OFED_SRC-${pkg_version}/SRPMS/"
|
||||||
|
download_package "$tarball_url"
|
||||||
|
tar -xf "$tarball_name"
|
||||||
|
tar -xf "$directory_name/src/MLNX_OFED_SRC-${pkg_version}.tgz"
|
||||||
|
# This section of code gets specific SRPMs versions according
|
||||||
|
# to the OFED tarbal version,
|
||||||
|
if [ "$pkg_version" = "4.2-1.2.0.0" ]; then
|
||||||
|
cp "$srpm_path/libibverbs-41mlnx1-OFED.4.2.1.0.6.42120.src.rpm" .
|
||||||
|
elif [ "$pkg_version" = "4.3-1.0.1.0" ]; then
|
||||||
|
cp "$srpm_path/mlnx-ofa_kernel-4.3-OFED.4.3.1.0.1.1.g8509e41.src.rpm" .
|
||||||
|
cp "$srpm_path/rdma-core-43mlnx1-1.43101.src.rpm" .
|
||||||
|
else
|
||||||
|
echo "$pkg_version : unknown version"
|
||||||
|
fi
|
||||||
|
rm -f "$tarball_name"
|
||||||
|
rm -rf "MLNX_OFED_SRC-${pkg_version}"
|
||||||
|
rm -rf "$directory_name"
|
||||||
|
elif [ "$tarball_name" = "qat1.7.upstream.l.1.0.3-42.tar.gz" ]; then
|
||||||
|
download_package $tarball_url
|
||||||
|
elif [ "$tarball_name" = "tpm-kmod-668a8270.tar.gz" ]; then
|
||||||
|
download_package $tarball_url
|
||||||
|
tar xf fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
mv linux-tpmdd-fd5c786/drivers/char/tpm $directory_name
|
||||||
|
tar czvf $tarball_name $directory_name
|
||||||
|
rm -rf linux-tpmdd-fd5c786
|
||||||
|
rm -rf $directory_name
|
||||||
|
rm fd5c78694f3f1c875e293de7a641ba8a3d60d00d.tar.gz
|
||||||
|
elif [ "$tarball_name" = "tss2-930.tar.gz" ]; then
|
||||||
|
git clone https://git.code.sf.net/p/ibmtpm20tss/tss ibmtpm20tss-tss
|
||||||
|
pushd ibmtpm20tss-tss
|
||||||
|
git checkout v930
|
||||||
|
rm -rf .git
|
||||||
|
popd
|
||||||
|
mv ibmtpm20tss-tss $directory_name
|
||||||
|
tar czvf $tarball_name $directory_name
|
||||||
|
rm -rf $directory_name
|
||||||
|
fi
|
||||||
|
popd
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# - For the General category and the Puppet category:
|
||||||
|
# - Packages have a common process: download, decompressed,
|
||||||
|
# change the directory path and compressed.
|
||||||
|
|
||||||
|
if [[ "$line" =~ ^pupp* ]]; then
|
||||||
|
download_path=$output_puppet/$tarball_name
|
||||||
|
download_directory=$output_puppet
|
||||||
|
else
|
||||||
|
download_path=$output_tarball/$tarball_name
|
||||||
|
download_directory=$output_tarball
|
||||||
|
fi
|
||||||
|
|
||||||
|
download_cmd="wget -t 5 --wait=15 $tarball_url -O $download_path"
|
||||||
|
|
||||||
|
if [ ! -e $download_path ]; then
|
||||||
|
if $download_cmd ; then
|
||||||
|
echo "Ok: $download_path"
|
||||||
|
pushd $download_directory
|
||||||
|
directory_name_original=$(tar -tf $tarball_name | head -1 | cut -f1 -d"/")
|
||||||
|
if [ "$directory_name" != "$directory_name_original" ]; then
|
||||||
|
mkdir -p $directory_name
|
||||||
|
tar xf $tarball_name --strip-components 1 -C $directory_name
|
||||||
|
tar -czf $tarball_name $directory_name
|
||||||
|
rm -r $directory_name
|
||||||
|
fi
|
||||||
|
popd
|
||||||
|
else
|
||||||
|
echo "Error: Failed to download $tarball_url"
|
||||||
|
fi
|
||||||
|
|
||||||
|
else
|
||||||
|
echo "Already have $download_path"
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
|
|
||||||
|
# End of file
|
||||||
|
|
Loading…
Reference in New Issue