diff --git a/build-tools/build-docker-images/README b/build-tools/build-docker-images/README index 4acacbf6..37ff9d6d 100644 --- a/build-tools/build-docker-images/README +++ b/build-tools/build-docker-images/README @@ -1,34 +1,51 @@ ## Example commands for building StarlingX images PRIVATE_REGISTRY_USERID=myuser -PRIVATE_REGISTRY=xxx.xxx.xxx.xxx +PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001 +VERSION=2018.11.13 +OS=centos +OS_VERSION=7.5.1804 +OPENSTACK_RELEASE=pike +HOST_PORT=8088 ## Step 1: Build stx-centos time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \ - --version 2018.11.13 \ - --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \ + --os ${OS} \ + --os-version ${OS_VERSION} \ + --version ${VERSION} \ + --user ${PRIVATE_REGISTRY_USERID} \ + --registry ${PRIVATE_REGISTRY} \ --push \ - --repo stx-local-build,http://${HOSTNAME}:8088/localdisk/loadbuild/jenkins/StarlingX_Upstream_build/2018-11-13_20-18-00/std/rpmbuild/RPMS \ - --repo stx-mirror-distro,http://${HOSTNAME}:8088/localdisk/designer/jenkins/StarlingX_upstream/cgcs-root/cgcs-centos-repo/Binary \ + --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \ + --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \ --clean ## Step 2: Build wheels (output as tarball) time $MY_REPO/build-tools/build-wheels/build-wheel-tarball.sh \ - --os centos \ - --release pike + --os ${OS} \ + --os-version ${OS_VERSION} \ + --release ${OPENSTACK_RELEASE} ## Step 3: Build images time $MY_REPO/build-tools/build-docker-images/build-stx-images.sh \ - --os centos \ - --base ${PRIVATE_REGISTRY}:9001/${PRIVATE_REGISTRY_USERID}/stx-centos:2018.11.13 \ - --wheels http://${HOSTNAME}:8088/$MY_WORKSPACE/std/build-wheels-centos-pike/stx-centos-pike-wheels.tar \ - --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \ + --os ${OS} \ + --version ${VERSION} \ + --release ${OPENSTACK_RELEASE} \ + --base ${PRIVATE_REGISTRY}/${PRIVATE_REGISTRY_USERID}/stx-${OS}:${VERSION} \ + --wheels http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/stx-${OS}-${OPENSTACK_RELEASE}-wheels.tar \ + --user ${PRIVATE_REGISTRY_USERID} \ + --registry ${PRIVATE_REGISTRY} \ --push --latest \ --clean +## Note: Verify that lighttpd is not bound to "localhost" +vi /etc/lighttpd/lighttpd.conf +# server.bind = "localhost" +systemctl restart lighttpd + ## Note: You may need to add an iptables rule to allow the docker ## containers to access the http server on your host. For example: -iptables -I INPUT 6 -i docker0 -p tcp --dport 8088 -m state --state NEW,ESTABLISHED -j ACCEPT +iptables -I INPUT 6 -i docker0 -p tcp --dport ${HOST_PORT} -m state --state NEW,ESTABLISHED -j ACCEPT diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 5d0f815d..6ba1cb1b 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -26,6 +26,9 @@ declare -a REPO_LIST REPO_OPTS= LOCAL=no CLEAN=no +TAG_LATEST=no +LATEST_TAG=latest +HOST=${HOSTNAME} function usage { cat >&2 <&2 exit 1 @@ -191,6 +209,7 @@ BASE_IMAGE_PRESENT=$? # Build the image IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION} +IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG} docker build \ --build-arg RELEASE=${OS_VERSION} \ @@ -209,6 +228,16 @@ if [ "${PUSH}" = "yes" ]; then echo "Failed running docker push command" >&2 exit 1 fi + + if [ "$TAG_LATEST" = "yes" ]; then + docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST} + echo "Pushing image: ${IMAGE_NAME_LATEST}" + docker push ${IMAGE_NAME_LATEST} + if [ $? -ne 0 ]; then + echo "Failed running docker push command on latest" >&2 + exit 1 + fi + fi fi if [ "${CLEAN}" = "yes" ]; then diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 345b040b..309794eb 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -21,13 +21,15 @@ SUPPORTED_OS_ARGS=('centos') OS=centos OPENSTACK_RELEASE=pike IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp -RELEASE=dev +PREFIX=dev +LATEST_PREFIX="" PUSH=no DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= WHEELS= CLEAN=no +TAG_LATEST=no declare -a ONLY declare -a SKIP @@ -45,6 +47,9 @@ Options: --push: Push to docker repo --user: Docker repo userid --registry: Docker registry + --prefix: Prefix on the image tag (default: dev) + --latest: Add a 'latest' tag when pushing + --latest-prefix: Alternative prefix on the latest image tag --clean: Remove image(s) from local registry --only : Only build the specified image(s). Multiple images can be specified with a comma-separated list, or with @@ -344,7 +349,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,latest,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -379,6 +384,14 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --prefix) + PREFIX=$2 + shift 2 + ;; + --latest-prefix) + LATEST_PREFIX=$2 + shift 2 + ;; --push) PUSH=yes shift @@ -445,9 +458,24 @@ if [ -z "${BASE}" ]; then exit 1 fi -IMAGE_TAG_BUILD="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-build" -IMAGE_TAG="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-${IMAGE_VERSION}" -IMAGE_TAG_LATEST="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-latest" +IMAGE_TAG="${OS}-${OPENSTACK_RELEASE}" +IMAGE_TAG_LATEST="${IMAGE_TAG}-latest" + +if [ -n "${LATEST_PREFIX}" ]; then + IMAGE_TAG_LATEST="${LATEST_PREFIX}-${IMAGE_TAG_LATEST}" +elif [ -n "${PREFIX}" ]; then + IMAGE_TAG_LATEST="${PREFIX}-${IMAGE_TAG_LATEST}" +fi + +if [ -n "${PREFIX}" ]; then + IMAGE_TAG="${PREFIX}-${IMAGE_TAG}" +fi + +IMAGE_TAG_BUILD="${IMAGE_TAG}-build" + +if [ -n "${IMAGE_VERSION}" ]; then + IMAGE_TAG="${IMAGE_TAG}-${IMAGE_VERSION}" +fi WORKDIR=${MY_WORKSPACE}/std/build-images mkdir -p ${WORKDIR} diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index f97dc993..5fb9f892 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -36,7 +36,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,keep-image,keep-container,release: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -83,8 +83,13 @@ while true; do done BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base + BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}" +# BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME. +# The following will substitute caps with lower case. +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}" + DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 4fa8b7ea..39dd3071 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -17,6 +17,7 @@ fi SUPPORTED_OS_ARGS=('centos') OS=centos +OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PUSH=no @@ -46,6 +47,7 @@ $(basename $0) Options: --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --os-version: Specify OS version --release: Openstack release (default: pike) --push: Push to docker repo --user: Docker repo userid @@ -54,7 +56,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -73,6 +75,10 @@ while true; do OS=$2 shift 2 ;; + --os-version) + OS_VERSION=$2 + shift 2 + ;; --push) PUSH=yes shift @@ -119,7 +125,7 @@ if [ ${VALID_OS} -ne 0 ]; then fi # Build the base wheels and retrieve the StarlingX wheels -${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE} +${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 diff --git a/build-tools/build-wheels/docker/pike-wheels.cfg b/build-tools/build-wheels/docker/pike-wheels.cfg index 1b672a81..11df8e91 100644 --- a/build-tools/build-wheels/docker/pike-wheels.cfg +++ b/build-tools/build-wheels/docker/pike-wheels.cfg @@ -54,6 +54,7 @@ itsdangerous-0.24-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d jaeger_client-3.10.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ee/1d/66cfa8ef438980e523c7ca3ed98c1b865b4e18df4167f32133ae3053c86c/jaeger-client-3.10.0.tar.gz|jaeger-client-3.10.0 jdcal-1.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/9b/fa/40beb2aa43a13f740dd5be367a10a03270043787833409c61b79e69f1dfd/jdcal-1.3.tar.gz|jdcal-1.3 jsonpath_rw-1.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/71/7c/45001b1f19af8c4478489fbae4fc657b21c4c669d7a5a036a86882581d85/jsonpath-rw-1.4.0.tar.gz|jsonpath-rw-1.4.0 +keyring-17.0.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/c8/4d/e1f19eebda5cd58030ecc1f61321f71661cfdb9d911d0b1053ab0ca7351c/keyring-17.0.0-py2.py3-none-any.whl krest-1.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fb/d2/9dbbd3a76f2385041720a0eb51ddab676e688fa8bee8a1489470839616cf/krest-1.3.1.tar.gz|krest-1.3.1 #libvirt_python-4.4.0-cp27-none-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2b/8d/1160cf34dc3d296896eb5c8f4944439ea368b87d2d2431f58d08d6bdf374/libvirt-python-4.4.0.tar.gz|libvirt-python-4.4.0|fix_setup logutils-0.3.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/b2/b57450889bf73da26027f8b995fd5fbfab258ec24ef967e4c1892f7cb121/logutils-0.3.5.tar.gz|logutils-0.3.5|fix_setup @@ -90,7 +91,7 @@ osc_placement-1.3.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages os_net_config-7.3.8-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/f5/f0/17c140a29b5100db11cc186095aa48a90b3ab659844992cfbb5224f75704/os_net_config-7.3.8-py2-none-any.whl osprofiler-1.11.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/80/46/395e951f6c5797f8a320e5df7a8217118b20ccf47441ae29c97b377ffd6c/osprofiler-1.11.0-py2-none-any.whl os_service_types-1.0.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/44/b3/8d03e393fea829f654f737e740059b5f528ecbb991efba1859fa7133ae5e/os_service_types-1.0.0-py2-none-any.whl -os_vif-1.7.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/74/66/c2727e4bb7302029d2406065233c3069176edef803be721c3ccf44a0f9ed/os_vif-1.7.0-py2-none-any.whl +os_vif-1.9.1-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/95/ee/3fe8ccb644b20f73e7c6bbd78d78ceecc30b9bc23e7a96b8eee54a0a9d5e/os_vif-1.9.1-py2-none-any.whl ovs-2.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/8a/14/c0cf1101406f24de2b72fd239ae638c489674d856c8d6dda41da2daa55ba/ovs-2.7.0.tar.gz|ovs-2.7.0 pathlib-1.0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz|pathlib-1.0.1|fix_setup pifpaf-1.9.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/38/4d/0e594192743aa7184051119951431e15e989cc222428d5a4856c528f57db/pifpaf-1.9.2.tar.gz|pifpaf-1.9.2 diff --git a/build-tools/modify-build-cfg b/build-tools/modify-build-cfg index f09069f4..94dc7f8f 100755 --- a/build-tools/modify-build-cfg +++ b/build-tools/modify-build-cfg @@ -92,8 +92,13 @@ if [ -f /usr/lib64/nosync/nosync.so ]; then echo "config_opts['nosync'] = True" >> $FILE fi -grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum'" $FILE || \ - echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum'" >> $FILE +NETWORK_PKGS="" +if [ "containers" == "$BUILD_TYPE" ]; then + NETWORK_PKGS="bind-utils" +fi + +grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" $FILE || \ + echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" >> $FILE # Special case for containers. # rpmbuild_networking is required for invoking helm commands within mock @@ -101,6 +106,10 @@ grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip if [ "containers" == "$BUILD_TYPE" ]; then grep -q "config_opts\['rpmbuild_networking'\] = True" $FILE || \ echo "config_opts['rpmbuild_networking'] = True" >> $FILE + + grep -q "config_opts\['use_host_resolv'\] = True" $FILE || \ + echo "config_opts['use_host_resolv'] = True" >> $FILE + sed -i "/^\[local-std\]/,/^\[/ s/enabled=0/enabled=1/" $FILE fi