ceph: build script with submodule archive support

Boost and Ceph submodules are stored as archives in SRPM.
They are unpacked in %prep stage so they can be picked up
by build process.

Ceph submodules are added as Source: dependencies with
a path prefix matching the subfolder where the corresponding
archive will be unpacked.

When adding a new submodule please follow this procedure:

1. make sure sudmodule.tar.gz is available in

   DOWNLOADS_DIR="$CGCS_BASE/downloads"

2. add corresponding source line in ceph.spec prefixed by
   the path in the source tree where the archive should
   be unpacked:

   Source29: src/path/to/submodule/submodule.tar.gz

3. add a new line to unpack the submodule:

   unpack_submodule "%{SOURCE29}" "%(dirname %{SOURCEURL29})"

Story: 2003605
Task: 28856

Depends-On: Ic5a03fe903c5119e6f01bd888093360e7e663bbb
Change-Id: Ic9c4aed8dbab5d3e141cf9c1b2b1892731b14779
Co-Authored-By: Scott Little <scott.little@windriver.com>
Co-Authored-By: Tingjie Chen <tingjie.chen@intel.com>
Co-Authored-By: Daniel Badea <daniel.badea@windriver.com>
Signed-off-by: Dehao Shang <dehao.shang@intel.com>
Signed-off-by: Changcheng Liu <changcheng.liu@intel.com>
Signed-off-by: Daniel Badea <daniel.badea@windriver.com>
This commit is contained in:
Dehao Shang 2019-01-31 13:37:38 +08:00 committed by Daniel Badea
parent 7d8f8d6e3f
commit 27426985a3
4 changed files with 385 additions and 30 deletions

301
ceph/ceph/centos/build_srpm Executable file
View File

@ -0,0 +1,301 @@
#!/bin/bash
# Copyright (c) 2019 Intel Corporation.
#
# SPDX-License-Identifier: Apache-2.0
# set -x
source "$SRC_BASE/build-tools/spec-utils"
source "$SRC_BASE/build-tools/srpm-utils"
CUR_DIR=`pwd`
BUILD_DIR="$RPMBUILD_BASE"
echo "ceph customization build_srpm : check environment variable DATA "
if [ "x$DATA" == "x" ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): Environment variable 'DATA' not defined."
exit 1
fi
echo "ceph customization build_srpm : check environment variable DATA = $DATA"
echo "ceph customization build_srpm : check build_srpm.data by executing srpm_source_build_data $DATA "
srpm_source_build_data $DATA
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): Failed to source build data from $DATA"
exit 1
fi
echo "ceph customization build_srpm : get ceph version from ceph.spec."
if [ "x$VERSION" == "x" ]; then
for SPEC in `find $SPECS_BASE -name '*.spec' | sort -V`; do
SPEC_PATH="$SPEC"
VERSION_DERIVED=`spec_evaluate '%{version}' "$SPEC_PATH" 2>> /dev/null`
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): '%{version}' not found in '$PKG_BASE/$SPEC_PATH'"
VERSION_DERIVED=""
fi
if [ "x$VERSION_DERIVED" != "x" ]; then
if [ "x$VERSION" == "x" ]; then
VERSION=$VERSION_DERIVED
else
if [ "x$SRC_DIR" != "x" ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): multiple spec files found, can't set VERSION automatically"
exit 1
fi
fi
fi
done
if [ "x$VERSION" != "x" ]; then
echo "ceph customization build_srpm : Derived VERSION=$VERSION"
else
echo "ERROR: ceph customization build_srpm (${LINENO}): Failed to derive a good VERSION from SPEC file, and none provided."
exit 1
fi
fi
echo "ceph customization build_srpm : get ceph version from ceph.spec : VERSION = $VERSION"
echo "ceph customization build_srpm : get ceph tar_name from ceph.spec."
if [ "x$TAR_NAME" == "x" ]; then
for SPEC in `find $SPECS_BASE -name '*.spec' | sort -V`; do
SPEC_PATH="$SPEC"
SERVICE=`spec_find_global service "$SPEC_PATH" 2>> /dev/null`
if [ $? -eq 0 ]; then
if [ "x$TAR_NAME" == "x" ]; then
TAR_NAME=$SERVICE
else
if [ "x$SRC_DIR" != "x" ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): multiple spec files found, can't set TAR_NAME automatically"
exit 1
fi
fi
else
NAME=`spec_find_tag Name "$SPEC_PATH" 2>> /dev/null`
if [ $? -eq 0 ]; then
if [ "x$TAR_NAME" == "x" ]; then
TAR_NAME=$NAME
else
if [ "x$SRC_DIR" != "x" ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): multiple spec files found, can't set TAR_NAME automatically"
exit 1
fi
fi
else
echo "WARNING: ceph customization build_srpm (${LINENO}): 'Name' not found in '$SPEC_PATH'"
NAME=""
fi
fi
done
if [ "x$TAR_NAME" != "x" ]; then
echo "ceph customization build_srpm : Derived TAR_NAME=$TAR_NAME"
else
echo "ERROR: ceph customization build_srpm (${LINENO}): Failed to derive a good TAR_NAME from SPEC file, and none provided."
exit 1
fi
fi
echo "ceph customization build_srpm : get ceph tar name from ceph.spec : TAR_NAME=$TAR_NAME"
if [ "x$TAR" == "x" ]; then
TAR="$TAR_NAME-$VERSION.tar.gz"
fi
SOURCE_PATH="$BUILD_DIR/SOURCES"
TAR_PATH="$SOURCE_PATH/$TAR"
STAGING=""
echo "ceph customization build_srpm : check copy_list_to_tar and exclude_list_from_tar."
if [ "x$COPY_LIST_TO_TAR" != "x" ] || [ "x$EXCLUDE_LIST_FROM_TAR" != "x" ]; then
STAGING="$BUILD_DIR/staging"
mkdir -p $STAGING
echo "ceph customization build_srpm : check copy_list_to_tar and exclude_list_from_tar....yes, so create new folder : $BUILD_DIR/staging"
else
echo "ceph customization build_srpm : check copy_list_to_tar and exclude_list_from_tar....ignore it."
fi
mkdir -p "$BUILD_DIR/SRPMS"
if [ ! -d "$BUILD_DIR/SRPMS" ]; then
echo "ERROR: ceph customization build_srpm : failed to found $BUILD_DIR/SRPMS"
exit 1
fi
mkdir -p "$SOURCE_PATH"
if [ ! -d "$SOURCE_PATH" ]; then
echo "ERROR: ceph customization build_srpm : failed to found $SOURCE_PATH"
exit 1
fi
if [ "x$SRC_DIR" == "x" -a "x$COPY_LIST" == "x" -a "$ALLOW_EMPTY_RPM" != "true" ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): '$PWD/$DATA' failed to provide at least one of 'SRC_DIR' or 'COPY_LIST'"
exit 1
fi
echo "ceph customization build_srpm : check ceph git tree directory : $SRC_DIR"
if [ "x$SRC_DIR" != "x" ]; then
if [ ! -d "$SRC_DIR" ]; then
echo "ERROR: ceph customization build_srpm : check ceph git tree directory not found: '$SRC_DIR'"
exit 1
fi
fi
pushd $SRC_DIR
echo "ceph customization build_srpm : create src/.git_version file"
(git rev-parse HEAD ; echo "v${VERSION}") 2> /dev/null > src/.git_version
popd
echo "ceph customization build_srpm : check COPY_LIST "
if [ "x$COPY_LIST" != "x" ]; then
echo "ceph customization build_srpm : check COPY_LIST.....yes"
echo "COPY_LIST: $COPY_LIST"
for p in $COPY_LIST; do
# echo "COPY_LIST: $p"
\cp -L -u -r -v $p $SOURCE_PATH
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): COPY_LIST: file not found: '$p'"
exit 1
fi
done
else
echo "ceph customization build_srpm : check COPY_LIST.....igonre it"
fi
echo "ceph customization build_srpm : check staging folder "
if [ "x$STAGING" != "x" ]; then
\cp -L -u -r -v $SRC_DIR $STAGING
echo "COPY_LIST_TO_TAR: $COPY_LIST_TO_TAR"
for p in $COPY_LIST_TO_TAR; do
# echo "COPY_LIST_TO_TAR: $p"
\cp -L -u -r -v $p $STAGING/$SRC_DIR
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): COPY_LIST_TO_TAR: file not found: '$p'"
exit 1
fi
done
echo "EXCLUDE_LIST_FROM_TAR: $EXCLUDE_LIST_FROM_TAR"
for p in $EXCLUDE_LIST_FROM_TAR; do
# echo "EXCLUDE_LIST_FROM_TAR: $p"
echo "rm -rf $STAGING/$SRC_DIR/$p"
\rm -rf $STAGING/$SRC_DIR/$p
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): EXCLUDE_LIST_FROM_TAR: could not remove file: '$p'"
exit 1
fi
done
else
echo "ceph customization build_srpm : check staging folder...don't exist"
fi
TRANSFORM=`echo "$SRC_DIR" | sed 's/^\./\\./' | sed 's:^/::'`
if [ "x$STAGING" != "x" ]; then
pushd $STAGING
fi
echo "ceph customization build_srpm : check whether need ceph tar file"
TAR_NEEDED=0
if [ "x$SRC_DIR" != "x" ]; then
if [ -e $TAR_PATH ]; then
n=`find . -cnewer $TAR_PATH -and ! -path './.git*' \
-and ! -path './build/*' \
-and ! -path './.pc/*' \
-and ! -path './patches/*' \
-and ! -path "./$DISTRO/*" \
-and ! -path './pbr-*.egg/*' \
| wc -l`
if [ $n -gt 0 ]; then
echo "ceph customization build_srpm : check whether need ceph tar file...needed"
TAR_NEEDED=1
fi
else
echo "ceph customization build_srpm : check whether need ceph tar file...needed"
TAR_NEEDED=1
fi
fi
if [ $TAR_NEEDED -gt 0 ]; then
echo "ceph customization build_srpm : create ceph tar file: $TAR_PATH"
tar czf $TAR_PATH $SRC_DIR --transform "s,^$TRANSFORM,$TAR_NAME-$VERSION,"
if [ $? -ne 0 ]; then
if [ "x$STAGING" != "x" ]; then
popd
fi
echo "ERROR: ceph customization build_srpm (${LINENO}): failed to create tar file, cmd: tar czf $TAR_PATH $SRC_DIR --transform \"s,^$TRANSFORM,$TAR_NAME-$VERSION,\""
exit 1
fi
echo "ceph customization build_srpm : create tar file: $TAR_PATH...created"
else
echo "ceph customization build_srpm : ceph tar file not needed."
fi
if [ "x$STAGING" != "x" ]; then
popd
fi
echo "ceph customization build_srpm : check $BUILD_DIR/SPECS folder."
if [ ! -d $BUILD_DIR/SPECS ]; then
echo "ceph customization build_srpm : '$BUILD_DIR/SPECS' does not exist"
exit 1
fi
SPEC_PATH="$BUILD_DIR/SPECS/ceph.spec"
echo "ceph customization build_srpm : check $SPEC_PATH"
if [[ ! -f "$SPEC_PATH" ]]; then
echo "ERROR: ceph customization build_srpm : No spec files found in spec directory '$BUILD_DIR/SPECS'"
exit 1
fi
BOOST_TAR=$(sed -n 's/^Source.*:\s*\(boost_.*\.tar\.bz2\)/\1/p' "$SPEC_PATH")
echo "ceph customization build_srpm : copy boost library ${DOWNLOADS_DIR}/${BOOST_TAR}to $SOURCE_PATH"
cp "${DOWNLOADS_DIR}/${BOOST_TAR}" "$SOURCE_PATH"
for submodule in $(sed -n 's/^Source.*:\s*\(.*\.tar\.gz\)/\1/p' "$SPEC_PATH"); do
echo "ceph customization build_srpm : copy submodule ${DOWNLOADS_DIR}/$(basename ${submodule}) $SOURCE_PATH"
cp "${DOWNLOADS_DIR}/$(basename ${submodule})" "$SOURCE_PATH"
done
RELEASE=`spec_find_tag Release "$SPEC_PATH" 2>> /dev/null`
if [ $? -ne 0 ]; then
echo "ERROR: ceph customiztion build_srpm (${LINENO}): 'Release' not found in '$SPEC_PATH'"
fi
NAME=`spec_find_tag Name "$SPEC_PATH" 2>> /dev/null`
if [ $? -ne 0 ]; then
echo "ERROR: ceph customization build_srpm (${LINENO}): 'Name' not found in '$SPEC_PATH'"
fi
SRPM="$NAME-$VERSION-$RELEASE.src.rpm"
SRPM_PATH="$BUILD_DIR/SRPMS/$SRPM"
spec_validate_tis_release $SPEC_PATH
if [ $? -ne 0 ]; then
echo "ceph customization build_srpm : Validation of $SPEC_PATH failed"
exit 1
fi
BUILD_NEEDED=0
if [ -e $SRPM_PATH ]; then
n=`find . -cnewer $SRPM_PATH | wc -l`
if [ $n -gt 0 ]; then
BUILD_NEEDED=1
fi
else
BUILD_NEEDED=1
fi
if [ $BUILD_NEEDED -gt 0 ]; then
echo "ceph customization build_srpm : ceph spec file : $SPEC_PATH"
echo "ceph customization build_srpm : SRPM build directory : $BUILD_DIR"
echo "ceph customization build_srpm : TIS_PATCH_VER : $TIS_PATCH_VER"
sed -i -e "1 i%define _tis_build_type $BUILD_TYPE" $SPEC_PATH
sed -i -e "1 i%define tis_patch_ver $TIS_PATCH_VER" $SPEC_PATH
ls -la $SOURCE_PATH
echo "ceph customization build_srpm : start to build ceph SRPM "
rpmbuild -bs $SPEC_PATH --define="%_topdir $BUILD_DIR" --undefine=dist --define="_tis_dist .tis"
else
echo "SRPM build not needed"
fi

View File

@ -1,6 +1,7 @@
SRC_DIR="$CGCS_BASE/git/ceph"
DOWNLOADS_DIR="$CGCS_BASE/downloads"
COPY_LIST="files/* $DISTRO/patches/*"
TIS_BASE_SRCREV=3f07f7ff1a5c7bfa8d0de12c966594d5fb7cf4ec
TIS_BASE_SRCREV=02899bfda814146b021136e9d8e80eba494e1126
TIS_PATCH_VER=GITREVCOUNT+1
BUILD_IS_BIG=40
BUILD_IS_SLOW=26

View File

@ -270,17 +270,35 @@ Group: System/Filesystems
URL: http://ceph.com/
Source0: %{?_remote_tarball_prefix}ceph-13.2.2.tar.gz
Source1: ceph.sh
Source2: mgr-restful-plugin
Source3: ceph.conf.pmon
Source4: ceph-init-wrapper.sh
Source5: ceph.conf
Source6: ceph-manage-journal.py
Source7: ceph.service
Source8: mgr-restful-plugin.service
Source9: stx_git_version
Source10: ceph-preshutdown.sh
Source11: starlingx-docker-override.conf
Source1: boost_1_67_0.tar.bz2
Source2: ceph-object-corpus/ceph-object-corpus-e32bf8ca3dc6151ebe7f205ba187815bc18e1cef.tar.gz
Source3: src/civetweb/civetweb-ff2881e2cd5869a71ca91083bad5d12cccd22136.tar.gz
Source4: src/erasure-code/jerasure/jerasure/jerasure-96c76b89d661c163f65a014b8042c9354ccf7f31.tar.gz
Source5: src/erasure-code/jerasure/gf-complete/gf-complete-7e61b44404f0ed410c83cfd3947a52e88ae044e1.tar.gz
Source6: src/rocksdb/rocksdb-f4a857da0b720691effc524469f6db895ad00d8e.tar.gz
Source7: ceph-erasure-code-corpus/ceph-erasure-code-corpus-2d7d78b9cc52e8a9529d8cc2d2954c7d375d5dd7.tar.gz
Source8: src/spdk/spdk-f474ce6930f0a44360e1cc4ecd606d2348481c4c.tar.gz
Source9: src/xxHash/xxHash-1f40c6511fa8dd9d2e337ca8c9bc18b3e87663c9.tar.gz
Source10: src/isa-l/isa-l-7e1a337433a340bc0974ed0f04301bdaca374af6.tar.gz
Source11: src/lua/lua-1fce39c6397056db645718b8f5821571d97869a4.tar.gz
Source12: src/blkin/blkin-f24ceec055ea236a093988237a9821d145f5f7c8.tar.gz
Source13: src/rapidjson/rapidjson-f54b0e47a08782a6131cc3d60f94d038fa6e0a51.tar.gz
Source14: src/googletest/googletest-fdb850479284e2aae047b87df6beae84236d0135.tar.gz
Source15: src/crypto/isa-l/isa-l_crypto/isa-l_crypto-603529a4e06ac8a1662c13d6b31f122e21830352.tar.gz
Source16: src/zstd/zstd-f4340f46b2387bc8de7d5320c0b83bb1499933ad.tar.gz
Source17: src/spdk/dpdk/dpdk-6ece49ad5a26f5e2f5c4af6c06c30376c0ddc387.tar.gz
Source18: src/rapidjson/thirdparty/gtest/googletest-0a439623f75c029912728d80cb7f1b8b48739ca4.tar.gz
Source19: ceph.sh
Source20: mgr-restful-plugin
Source21: ceph.conf.pmon
Source22: ceph-init-wrapper.sh
Source23: ceph.conf
Source24: ceph-manage-journal.py
Source25: ceph.service
Source26: mgr-restful-plugin.service
Source27: ceph-preshutdown.sh
Source28: starlingx-docker-override.conf
%if 0%{?suse_version}
# _insert_obs_source_lines_here
@ -1033,10 +1051,47 @@ python-rbd, python-rgw or python-cephfs instead.
#################################################################################
%prep
%autosetup -p1 -n ceph-13.2.2
# StarlingX :Copy the .git_version file needed by the build
# This commit SHA is from the upstream src rpm which is the base of this repo branch
# TODO: Add a commit hook to update to our latest commit SHA
cp %{SOURCE9} %{_builddir}/%{name}-%{version}/src/.git_version
mkdir -p src/boost
BOOST_VERSION=$(basename "%{SOURCE1}" | sed 's/boost_\(.*\)\.tar\.bz2/\1/')
tar xjf "%{SOURCE1}" -C src/boost \
--exclude="$BOOST_VERSION/libs/*/doc" \
--exclude="$BOOST_VERSION/libs/*/example" \
--exclude="$BOOST_VERSION/libs/*/examples" \
--exclude="$BOOST_VERSION/libs/*/meta" \
--exclude="$BOOST_VERSION/libs/*/test" \
--exclude="$BOOST_VERSION/tools/boostbook" \
--exclude="$BOOST_VERSION/tools/quickbook" \
--exclude="$BOOST_VERSION/tools/auto_index" \
--exclude='doc' \
--exclude='more' \
--exclude='status' \
--strip-components 1
unpack_submodule() {
mkdir -p "$2" && tar xzf "$1" -C "$2" --strip-components 1
}
unpack_submodule "%{SOURCE2}" "%(dirname %{SOURCEURL2})"
unpack_submodule "%{SOURCE3}" "%(dirname %{SOURCEURL3})"
unpack_submodule "%{SOURCE4}" "%(dirname %{SOURCEURL4})"
unpack_submodule "%{SOURCE5}" "%(dirname %{SOURCEURL5})"
unpack_submodule "%{SOURCE6}" "%(dirname %{SOURCEURL6})"
unpack_submodule "%{SOURCE7}" "%(dirname %{SOURCEURL7})"
unpack_submodule "%{SOURCE8}" "%(dirname %{SOURCEURL8})"
unpack_submodule "%{SOURCE9}" "%(dirname %{SOURCEURL9})"
unpack_submodule "%{SOURCE10}" "%(dirname %{SOURCEURL10})"
unpack_submodule "%{SOURCE11}" "%(dirname %{SOURCEURL11})"
unpack_submodule "%{SOURCE12}" "%(dirname %{SOURCEURL12})"
unpack_submodule "%{SOURCE13}" "%(dirname %{SOURCEURL13})"
unpack_submodule "%{SOURCE14}" "%(dirname %{SOURCEURL14})"
unpack_submodule "%{SOURCE15}" "%(dirname %{SOURCEURL15})"
unpack_submodule "%{SOURCE16}" "%(dirname %{SOURCEURL16})"
unpack_submodule "%{SOURCE17}" "%(dirname %{SOURCEURL17})"
unpack_submodule "%{SOURCE18}" "%(dirname %{SOURCEURL18})"
sed -e "s/@VERSION@/%{version}/g" \
-e "s/@RPM_RELEASE@/%{release}/g" \
-e "s/@TARBALL_BASENAME@/ceph-%{version}/g" \
-i alpine/APKBUILD.in
mv alpine/APKBUILD.in alpine/APKBUILD
%build
@ -1234,18 +1289,18 @@ mkdir -p %{buildroot}%{_initrddir}
mkdir -p %{buildroot}%{_sysconfdir}/ceph
mkdir -p %{buildroot}%{_unitdir}
install -m 750 %{SOURCE1} %{buildroot}%{_sysconfdir}/services.d/controller/
install -m 750 %{SOURCE1} %{buildroot}%{_sysconfdir}/services.d/storage/
install -m 750 %{SOURCE1} %{buildroot}%{_sysconfdir}/services.d/worker/
install -m 750 %{SOURCE2} %{buildroot}%{_initrddir}/
install -m 750 %{SOURCE3} %{buildroot}%{_sysconfdir}/ceph/
install -m 750 %{SOURCE4} %{buildroot}%{_initrddir}/ceph-init-wrapper
install -m 640 %{SOURCE5} %{buildroot}%{_sysconfdir}/ceph/
install -m 700 %{SOURCE6} %{buildroot}%{_sbindir}/ceph-manage-journal
install -m 644 %{SOURCE7} %{buildroot}%{_unitdir}/ceph.service
install -m 644 %{SOURCE8} %{buildroot}%{_unitdir}/mgr-restful-plugin.service
install -m 700 %{SOURCE10} %{buildroot}%{_sbindir}/ceph-preshutdown.sh
install -D -m 644 %{SOURCE11} %{buildroot}%{_unitdir}/docker.service.d/starlingx-docker-override.conf
install -m 750 %{SOURCE19} %{buildroot}%{_sysconfdir}/services.d/controller/
install -m 750 %{SOURCE19} %{buildroot}%{_sysconfdir}/services.d/storage/
install -m 750 %{SOURCE19} %{buildroot}%{_sysconfdir}/services.d/worker/
install -m 750 %{SOURCE20} %{buildroot}%{_initrddir}/
install -m 750 %{SOURCE21} %{buildroot}%{_sysconfdir}/ceph/
install -m 750 %{SOURCE22} %{buildroot}%{_initrddir}/ceph-init-wrapper
install -m 640 %{SOURCE23} %{buildroot}%{_sysconfdir}/ceph/
install -m 700 %{SOURCE24} %{buildroot}%{_sbindir}/ceph-manage-journal
install -m 644 %{SOURCE25} %{buildroot}%{_unitdir}/ceph.service
install -m 644 %{SOURCE26} %{buildroot}%{_unitdir}/mgr-restful-plugin.service
install -m 700 %{SOURCE27} %{buildroot}%{_sbindir}/ceph-preshutdown.sh
install -D -m 644 %{SOURCE28} %{buildroot}%{_unitdir}/docker.service.d/starlingx-docker-override.conf
install -m 750 src/init-radosgw %{buildroot}/%{_initrddir}/ceph-radosgw
sed -i '/### END INIT INFO/a SYSTEMCTL_SKIP_REDIRECT=1' %{buildroot}/%{_initrddir}/ceph-radosgw

View File

@ -1,2 +0,0 @@
02899bfda814146b021136e9d8e80eba494e1126
v13.2.2