From cf0135e886e52e73b69ad8775baa0674ab2c5a35 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Mon, 10 Dec 2018 10:34:53 -0500 Subject: [PATCH 1/3] Add greater flexibility in docker image naming The existing user interface for docker image and wheel creation is not flexible enough to name images is described in the spec, and to push those images to the starlingx organization at docker.hub This update adds new arguements that will allow creation of docker images with compliant names, and will allow them to be pushed to https://hub.docker.com/r/starlingx. I will also change build-base-wheels.sh to allow the 'os-version' to be pushed through to build-wheels-tarball.sh. Story: 2003907 Task: 28370 Change-Id: I4378db63d5ff906ffd636034351e580afa5e3472 Signed-off-by: Scott Little --- build-tools/build-docker-images/README | 41 +++++++++++++------ .../build-docker-images/build-stx-base.sh | 35 ++++++++++++++-- .../build-docker-images/build-stx-images.sh | 38 ++++++++++++++--- build-tools/build-wheels/build-base-wheels.sh | 7 +++- .../build-wheels/build-wheel-tarball.sh | 10 ++++- 5 files changed, 108 insertions(+), 23 deletions(-) diff --git a/build-tools/build-docker-images/README b/build-tools/build-docker-images/README index 4acacbf6..37ff9d6d 100644 --- a/build-tools/build-docker-images/README +++ b/build-tools/build-docker-images/README @@ -1,34 +1,51 @@ ## Example commands for building StarlingX images PRIVATE_REGISTRY_USERID=myuser -PRIVATE_REGISTRY=xxx.xxx.xxx.xxx +PRIVATE_REGISTRY=xxx.xxx.xxx.xxx:9001 +VERSION=2018.11.13 +OS=centos +OS_VERSION=7.5.1804 +OPENSTACK_RELEASE=pike +HOST_PORT=8088 ## Step 1: Build stx-centos time $MY_REPO/build-tools/build-docker-images/build-stx-base.sh \ - --version 2018.11.13 \ - --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \ + --os ${OS} \ + --os-version ${OS_VERSION} \ + --version ${VERSION} \ + --user ${PRIVATE_REGISTRY_USERID} \ + --registry ${PRIVATE_REGISTRY} \ --push \ - --repo stx-local-build,http://${HOSTNAME}:8088/localdisk/loadbuild/jenkins/StarlingX_Upstream_build/2018-11-13_20-18-00/std/rpmbuild/RPMS \ - --repo stx-mirror-distro,http://${HOSTNAME}:8088/localdisk/designer/jenkins/StarlingX_upstream/cgcs-root/cgcs-centos-repo/Binary \ + --repo stx-local-build,http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/rpmbuild/RPMS \ + --repo stx-mirror-distro,http://${HOSTNAME}:${HOST_PORT}/${MY_REPO}/cgcs-root/cgcs-${OS}-repo/Binary \ --clean ## Step 2: Build wheels (output as tarball) time $MY_REPO/build-tools/build-wheels/build-wheel-tarball.sh \ - --os centos \ - --release pike + --os ${OS} \ + --os-version ${OS_VERSION} \ + --release ${OPENSTACK_RELEASE} ## Step 3: Build images time $MY_REPO/build-tools/build-docker-images/build-stx-images.sh \ - --os centos \ - --base ${PRIVATE_REGISTRY}:9001/${PRIVATE_REGISTRY_USERID}/stx-centos:2018.11.13 \ - --wheels http://${HOSTNAME}:8088/$MY_WORKSPACE/std/build-wheels-centos-pike/stx-centos-pike-wheels.tar \ - --user ${PRIVATE_REGISTRY_USERID} --registry ${PRIVATE_REGISTRY}:9001 \ + --os ${OS} \ + --version ${VERSION} \ + --release ${OPENSTACK_RELEASE} \ + --base ${PRIVATE_REGISTRY}/${PRIVATE_REGISTRY_USERID}/stx-${OS}:${VERSION} \ + --wheels http://${HOSTNAME}:${HOST_PORT}/${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/stx-${OS}-${OPENSTACK_RELEASE}-wheels.tar \ + --user ${PRIVATE_REGISTRY_USERID} \ + --registry ${PRIVATE_REGISTRY} \ --push --latest \ --clean +## Note: Verify that lighttpd is not bound to "localhost" +vi /etc/lighttpd/lighttpd.conf +# server.bind = "localhost" +systemctl restart lighttpd + ## Note: You may need to add an iptables rule to allow the docker ## containers to access the http server on your host. For example: -iptables -I INPUT 6 -i docker0 -p tcp --dport 8088 -m state --state NEW,ESTABLISHED -j ACCEPT +iptables -I INPUT 6 -i docker0 -p tcp --dport ${HOST_PORT} -m state --state NEW,ESTABLISHED -j ACCEPT diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index 5d0f815d..6ba1cb1b 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -26,6 +26,9 @@ declare -a REPO_LIST REPO_OPTS= LOCAL=no CLEAN=no +TAG_LATEST=no +LATEST_TAG=latest +HOST=${HOSTNAME} function usage { cat >&2 <&2 exit 1 @@ -191,6 +209,7 @@ BASE_IMAGE_PRESENT=$? # Build the image IMAGE_NAME=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${IMAGE_VERSION} +IMAGE_NAME_LATEST=${DOCKER_REGISTRY}${DOCKER_USER}/stx-${OS}:${LATEST_TAG} docker build \ --build-arg RELEASE=${OS_VERSION} \ @@ -209,6 +228,16 @@ if [ "${PUSH}" = "yes" ]; then echo "Failed running docker push command" >&2 exit 1 fi + + if [ "$TAG_LATEST" = "yes" ]; then + docker tag ${IMAGE_NAME} ${IMAGE_NAME_LATEST} + echo "Pushing image: ${IMAGE_NAME_LATEST}" + docker push ${IMAGE_NAME_LATEST} + if [ $? -ne 0 ]; then + echo "Failed running docker push command on latest" >&2 + exit 1 + fi + fi fi if [ "${CLEAN}" = "yes" ]; then diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index 345b040b..309794eb 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -21,13 +21,15 @@ SUPPORTED_OS_ARGS=('centos') OS=centos OPENSTACK_RELEASE=pike IMAGE_VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp -RELEASE=dev +PREFIX=dev +LATEST_PREFIX="" PUSH=no DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= WHEELS= CLEAN=no +TAG_LATEST=no declare -a ONLY declare -a SKIP @@ -45,6 +47,9 @@ Options: --push: Push to docker repo --user: Docker repo userid --registry: Docker registry + --prefix: Prefix on the image tag (default: dev) + --latest: Add a 'latest' tag when pushing + --latest-prefix: Alternative prefix on the latest image tag --clean: Remove image(s) from local registry --only : Only build the specified image(s). Multiple images can be specified with a comma-separated list, or with @@ -344,7 +349,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,latest,clean -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,push,user:,registry:,release:,base:,wheels:,only:,skip:,prefix:,latest,latest-prefix:,clean -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -379,6 +384,14 @@ while true; do OPENSTACK_RELEASE=$2 shift 2 ;; + --prefix) + PREFIX=$2 + shift 2 + ;; + --latest-prefix) + LATEST_PREFIX=$2 + shift 2 + ;; --push) PUSH=yes shift @@ -445,9 +458,24 @@ if [ -z "${BASE}" ]; then exit 1 fi -IMAGE_TAG_BUILD="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-build" -IMAGE_TAG="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-${IMAGE_VERSION}" -IMAGE_TAG_LATEST="${RELEASE}-${OS}-${OPENSTACK_RELEASE}-latest" +IMAGE_TAG="${OS}-${OPENSTACK_RELEASE}" +IMAGE_TAG_LATEST="${IMAGE_TAG}-latest" + +if [ -n "${LATEST_PREFIX}" ]; then + IMAGE_TAG_LATEST="${LATEST_PREFIX}-${IMAGE_TAG_LATEST}" +elif [ -n "${PREFIX}" ]; then + IMAGE_TAG_LATEST="${PREFIX}-${IMAGE_TAG_LATEST}" +fi + +if [ -n "${PREFIX}" ]; then + IMAGE_TAG="${PREFIX}-${IMAGE_TAG}" +fi + +IMAGE_TAG_BUILD="${IMAGE_TAG}-build" + +if [ -n "${IMAGE_VERSION}" ]; then + IMAGE_TAG="${IMAGE_TAG}-${IMAGE_VERSION}" +fi WORKDIR=${MY_WORKSPACE}/std/build-images mkdir -p ${WORKDIR} diff --git a/build-tools/build-wheels/build-base-wheels.sh b/build-tools/build-wheels/build-base-wheels.sh index f97dc993..5fb9f892 100755 --- a/build-tools/build-wheels/build-base-wheels.sh +++ b/build-tools/build-wheels/build-base-wheels.sh @@ -36,7 +36,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,keep-image,keep-container,release: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,keep-image,keep-container,release: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -83,8 +83,13 @@ while true; do done BUILD_OUTPUT_PATH=${MY_WORKSPACE}/std/build-wheels-${OS}-${OPENSTACK_RELEASE}/base + BUILD_IMAGE_NAME="${USER}-$(basename ${MY_WORKSPACE})-wheelbuilder:${OS}-${OPENSTACK_RELEASE}" +# BUILD_IMAGE_NAME can't have caps if it's passed to docker build -t $BUILD_IMAGE_NAME. +# The following will substitute caps with lower case. +BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME,,}" + DOCKER_FILE=${DOCKER_PATH}/${OS}-dockerfile WHEELS_CFG=${DOCKER_PATH}/${OPENSTACK_RELEASE}-wheels.cfg diff --git a/build-tools/build-wheels/build-wheel-tarball.sh b/build-tools/build-wheels/build-wheel-tarball.sh index 4fa8b7ea..39dd3071 100755 --- a/build-tools/build-wheels/build-wheel-tarball.sh +++ b/build-tools/build-wheels/build-wheel-tarball.sh @@ -17,6 +17,7 @@ fi SUPPORTED_OS_ARGS=('centos') OS=centos +OS_VERSION=7.5.1804 OPENSTACK_RELEASE=pike VERSION=$(date --utc '+%Y.%m.%d.%H.%M') # Default version, using timestamp PUSH=no @@ -46,6 +47,7 @@ $(basename $0) Options: --os: Specify base OS (valid options: ${SUPPORTED_OS_ARGS[@]}) + --os-version: Specify OS version --release: Openstack release (default: pike) --push: Push to docker repo --user: Docker repo userid @@ -54,7 +56,7 @@ Options: EOF } -OPTS=$(getopt -o h -l help,os:,push,clean,user:,release:,version: -- "$@") +OPTS=$(getopt -o h -l help,os:,os-version:,push,clean,user:,release:,version: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -73,6 +75,10 @@ while true; do OS=$2 shift 2 ;; + --os-version) + OS_VERSION=$2 + shift 2 + ;; --push) PUSH=yes shift @@ -119,7 +125,7 @@ if [ ${VALID_OS} -ne 0 ]; then fi # Build the base wheels and retrieve the StarlingX wheels -${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --release ${OPENSTACK_RELEASE} +${MY_SCRIPT_DIR}/build-base-wheels.sh --os ${OS} --os-version ${OS_VERSION} --release ${OPENSTACK_RELEASE} if [ $? -ne 0 ]; then echo "Failure running build-base-wheels.sh" >&2 exit 1 From f1bfa53e87bf847eda924db2030d6e12c2eec687 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Tue, 11 Dec 2018 13:29:02 -0500 Subject: [PATCH 2/3] Fix 'build-pkgs --container' when building in docker build-pkgs --container fails when run within docker due to DNS lookup failure. Need to add pkg bind-utils to mock when building in the 'container' context, and to add an /etc/resolv.conf via 'config_opts['use_host_resolv'] = True' Story: 2003907 Task: 28370 Change-Id: I5f9ec16d29545ba74457a060323e07967e085556 Signed-off-by: Scott Little --- build-tools/modify-build-cfg | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/build-tools/modify-build-cfg b/build-tools/modify-build-cfg index f09069f4..94dc7f8f 100755 --- a/build-tools/modify-build-cfg +++ b/build-tools/modify-build-cfg @@ -92,8 +92,13 @@ if [ -f /usr/lib64/nosync/nosync.so ]; then echo "config_opts['nosync'] = True" >> $FILE fi -grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum'" $FILE || \ - echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum'" >> $FILE +NETWORK_PKGS="" +if [ "containers" == "$BUILD_TYPE" ]; then + NETWORK_PKGS="bind-utils" +fi + +grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" $FILE || \ + echo "config_opts['chroot_setup_cmd'] = 'install @buildsys-build pigz lbzip2 yum $NETWORK_PKGS'" >> $FILE # Special case for containers. # rpmbuild_networking is required for invoking helm commands within mock @@ -101,6 +106,10 @@ grep -q "config_opts\['chroot_setup_cmd'\] = 'install @buildsys-build pigz lbzip if [ "containers" == "$BUILD_TYPE" ]; then grep -q "config_opts\['rpmbuild_networking'\] = True" $FILE || \ echo "config_opts['rpmbuild_networking'] = True" >> $FILE + + grep -q "config_opts\['use_host_resolv'\] = True" $FILE || \ + echo "config_opts['use_host_resolv'] = True" >> $FILE + sed -i "/^\[local-std\]/,/^\[/ s/enabled=0/enabled=1/" $FILE fi From 550c28895fe2b6a50497f758ee94a06f80ad5bbc Mon Sep 17 00:00:00 2001 From: Don Penney Date: Wed, 19 Dec 2018 08:59:02 -0600 Subject: [PATCH 3/3] Update keyring and os-vif in pike wheels tarball StarlingX requires a higher version of the keyring and os-vif python modules than is specified in the stable/pike upper-constraints.txt file. This update specifies a newer version for these modules in the pike-wheels.cfg file to override the constraints for the docker image build. Story: 2004520 Task: 28585 Change-Id: I9bdaaad2575f8832f19f4fdac26dae76fd83c442 Signed-off-by: Don Penney --- build-tools/build-wheels/docker/pike-wheels.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-tools/build-wheels/docker/pike-wheels.cfg b/build-tools/build-wheels/docker/pike-wheels.cfg index 1b672a81..11df8e91 100644 --- a/build-tools/build-wheels/docker/pike-wheels.cfg +++ b/build-tools/build-wheels/docker/pike-wheels.cfg @@ -54,6 +54,7 @@ itsdangerous-0.24-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/d jaeger_client-3.10.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ee/1d/66cfa8ef438980e523c7ca3ed98c1b865b4e18df4167f32133ae3053c86c/jaeger-client-3.10.0.tar.gz|jaeger-client-3.10.0 jdcal-1.3-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/9b/fa/40beb2aa43a13f740dd5be367a10a03270043787833409c61b79e69f1dfd/jdcal-1.3.tar.gz|jdcal-1.3 jsonpath_rw-1.4.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/71/7c/45001b1f19af8c4478489fbae4fc657b21c4c669d7a5a036a86882581d85/jsonpath-rw-1.4.0.tar.gz|jsonpath-rw-1.4.0 +keyring-17.0.0-py2.py3-none-any.whl|pypi|https://files.pythonhosted.org/packages/c8/4d/e1f19eebda5cd58030ecc1f61321f71661cfdb9d911d0b1053ab0ca7351c/keyring-17.0.0-py2.py3-none-any.whl krest-1.3.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/fb/d2/9dbbd3a76f2385041720a0eb51ddab676e688fa8bee8a1489470839616cf/krest-1.3.1.tar.gz|krest-1.3.1 #libvirt_python-4.4.0-cp27-none-linux_x86_64.whl|tar|https://files.pythonhosted.org/packages/2b/8d/1160cf34dc3d296896eb5c8f4944439ea368b87d2d2431f58d08d6bdf374/libvirt-python-4.4.0.tar.gz|libvirt-python-4.4.0|fix_setup logutils-0.3.5-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/49/b2/b57450889bf73da26027f8b995fd5fbfab258ec24ef967e4c1892f7cb121/logutils-0.3.5.tar.gz|logutils-0.3.5|fix_setup @@ -90,7 +91,7 @@ osc_placement-1.3.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages os_net_config-7.3.8-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/f5/f0/17c140a29b5100db11cc186095aa48a90b3ab659844992cfbb5224f75704/os_net_config-7.3.8-py2-none-any.whl osprofiler-1.11.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/80/46/395e951f6c5797f8a320e5df7a8217118b20ccf47441ae29c97b377ffd6c/osprofiler-1.11.0-py2-none-any.whl os_service_types-1.0.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/44/b3/8d03e393fea829f654f737e740059b5f528ecbb991efba1859fa7133ae5e/os_service_types-1.0.0-py2-none-any.whl -os_vif-1.7.0-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/74/66/c2727e4bb7302029d2406065233c3069176edef803be721c3ccf44a0f9ed/os_vif-1.7.0-py2-none-any.whl +os_vif-1.9.1-py2-none-any.whl|pypi|https://files.pythonhosted.org/packages/95/ee/3fe8ccb644b20f73e7c6bbd78d78ceecc30b9bc23e7a96b8eee54a0a9d5e/os_vif-1.9.1-py2-none-any.whl ovs-2.7.0-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/8a/14/c0cf1101406f24de2b72fd239ae638c489674d856c8d6dda41da2daa55ba/ovs-2.7.0.tar.gz|ovs-2.7.0 pathlib-1.0.1-py2-none-any.whl|tar|https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz|pathlib-1.0.1|fix_setup pifpaf-1.9.2-py2.py3-none-any.whl|tar|https://files.pythonhosted.org/packages/38/4d/0e594192743aa7184051119951431e15e989cc222428d5a4856c528f57db/pifpaf-1.9.2.tar.gz|pifpaf-1.9.2