docker-images: don't embed wheel tar in images
This patch fixes a problem introduced by [1]: docker image script embeds the wheels tarball in the docker image via a COPY command; then deletes it during the build. This makes docker images unnecessarily large. This lets us use local tar files with the docker image(s) being built. Recently [2] a patch was merged that adds a web server to the Debian build environment, allowing us to access local files over HTTP and making the COPY step unnecessary in the Docker file. This patch removes the explicit downloading of the wheels tarball and the COPY directive, while passing the wheel tarball to Loci as a URL, allowing Loci to download it during the build w/o contributing to the FS layer size. This reduces the size of stx-fm-rest-api image by ~220 MB. Other Loci images should experience similar savings. [1] https://review.opendev.org/c/starlingx/root/+/857505 [2] https://review.opendev.org/c/starlingx/tools/+/873010 TESTS ================= * Build a loci image with wheels tarball passed as a URL * Build a loci image with wheels tarball passed as a local file name Story: 2010294 Task: 47343 Change-Id: I6dca6f95b0845e549ad35c9b6e95aa58c86fc774 Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
This commit is contained in:
Davlet Panech
parent
221c512834
commit
4183a924cf
|
|
@ -112,6 +112,12 @@ function is_in {
|
|||
return 1
|
||||
}
|
||||
|
||||
function starts_with {
|
||||
local str="$1"
|
||||
local prefix="$2"
|
||||
[[ "${str#$prefix}" != "$str" ]]
|
||||
}
|
||||
|
||||
function is_empty {
|
||||
test $# -eq 0
|
||||
}
|
||||
|
|
@ -121,20 +127,33 @@ function url_basename {
|
|||
echo "$1" | sed -r -e 's/[?#].*//' -e 's#.*/##'
|
||||
}
|
||||
|
||||
# Usage: download $URL $OUTPUT_FILE
|
||||
function download_file {
|
||||
local url="$1"
|
||||
local out_file="$2"
|
||||
if echo "$url" | grep -E -q -e '^https?:' -e '^ftp:' ; then
|
||||
\rm -f "$out_file.tmp" || return 1
|
||||
with_retries 5 wget -O "$out_file.tmp" "$url" || return 1
|
||||
\mv -f "$out_file.tmp" "$out_file" || exit 1
|
||||
function local_path_to_url {
|
||||
local path="$1"
|
||||
|
||||
local abs_path
|
||||
abs_path="$(readlink -f "$path")" || exit 1
|
||||
|
||||
local repo_root
|
||||
repo_root="$(readlink -e "$MY_REPO_ROOT_DIR")" || exit 1
|
||||
|
||||
local workspace_root
|
||||
workspace_root="$(readlink -e "$MY_WORKSPACE")" || exit 1
|
||||
|
||||
local dflt_port
|
||||
if starts_with "$abs_path" "$repo_root" ; then
|
||||
dflt_port="8089"
|
||||
elif starts_with "$abs_path" "$workspace" ; then
|
||||
dflt_port="8088"
|
||||
else
|
||||
local src_file
|
||||
src_file="$(echo "$url" | sed -e 's#^file:/+##')"
|
||||
\cp -a "$src_file" "$out_file" || return 1
|
||||
echo "ERROR: $path: path must start with \$MY_REPO_ROOT_DIR or \$MY_WORKSPACE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "$BUILDER_FILES_URL" ]] ; then
|
||||
echo "${BUILDER_FILES_URL}${path}"
|
||||
else
|
||||
echo "http://${HOSTNAME}:${dflt_port}${path}"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
#
|
||||
|
|
@ -241,16 +260,6 @@ function patch_loci {
|
|||
\rm -rf "${WORKDIR}/loci/stx-wheels/"* || exit 1
|
||||
}
|
||||
|
||||
function download_loci_wheels {
|
||||
local url="$1"
|
||||
out_file="${WORKDIR}/loci/stx-wheels/$(url_basename "$url")"
|
||||
if [[ ! -f "$out_file" ]] ; then
|
||||
echo "Downloading $url => $out_file" >&2
|
||||
download_file "$url" "$out_file" || return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function update_image_record {
|
||||
# Update the image record file with a new/updated entry
|
||||
local LABEL=$1
|
||||
|
|
@ -473,11 +482,7 @@ function build_image_loci {
|
|||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "$BUILDER_FILES_URL" ]] ; then
|
||||
PROJECT_REPO="$BUILDER_FILES_URL/${CLONE_DIR}"
|
||||
else
|
||||
PROJECT_REPO="http://${HOSTNAME}:8088/${CLONE_DIR}"
|
||||
fi
|
||||
PROJECT_REPO="$(local_path_to_url "${CLONE_DIR}")" || exit 1
|
||||
fi
|
||||
|
||||
local -a BUILD_ARGS=
|
||||
|
|
@ -487,14 +492,10 @@ function build_image_loci {
|
|||
|
||||
if [ "${PYTHON3}" == "no" ] ; then
|
||||
echo "Python2 service ${LABEL}"
|
||||
download_loci_wheels "$WHEELS_PY2" || exit 1
|
||||
#BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
|
||||
BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS_PY2")")
|
||||
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS_PY2})
|
||||
else
|
||||
echo "Python3 service ${LABEL}"
|
||||
download_loci_wheels "$WHEELS" || exit 1
|
||||
#BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
|
||||
BUILD_ARGS+=(--build-arg WHEELS="/opt/loci/stx-wheels/$(url_basename "$WHEELS")")
|
||||
BUILD_ARGS+=(--build-arg WHEELS=${WHEELS})
|
||||
fi
|
||||
|
||||
if [ ! -z "$HTTP_PROXY" ]; then
|
||||
|
|
@ -981,12 +982,16 @@ fi
|
|||
for var in WHEELS WHEELS_PY2 ; do
|
||||
# skip empty vars
|
||||
[[ -n "${!var}" ]] || continue
|
||||
# skip network urls
|
||||
echo "${!var}" | grep -E -q -e '^https?:' -e '^ftp:' && continue
|
||||
# http(s) urls are supported by Loci directly -- skip
|
||||
# See https://github.com/openstack/loci/blob/efccd0a853879ac6af6066eda09792d0d3afe9c0/scripts/fetch_wheels.py#L170
|
||||
echo "${!var}" | grep -E -q -e '^https?:' && continue
|
||||
# remove file:/ prefix if any
|
||||
declare "$var=$(echo "${!var}" | sed -r 's#^file:/+##')"
|
||||
# resolve it to an absolute path
|
||||
declare "$var=$(readlink -f "${!var}")" || exit 1
|
||||
# convert it to a local URL
|
||||
url="$(local_path_to_url "${!var}")" || exit 1
|
||||
declare "$var=$url"
|
||||
done
|
||||
|
||||
# Find the directives files
|
||||
|
|
|
|||
|
|
@ -1,39 +1,27 @@
|
|||
From 644ac10c8877444c540aac36111e59c65c47ce59 Mon Sep 17 00:00:00 2001
|
||||
From 7462c9467cd0a1e98ced03517646a4e00f65ddc3 Mon Sep 17 00:00:00 2001
|
||||
From: Davlet Panech <davlet.panech@windriver.com>
|
||||
Date: Thu, 8 Sep 2022 21:04:55 +0000
|
||||
Subject: [PATCH 1/2] starlingx: wheels from filesystem + pkg repos
|
||||
Subject: [PATCH] starlingx: enable/disable package repos
|
||||
|
||||
- Allow WHEELS to be a local file path, rather than URL
|
||||
- Dockerfile: new parameter DIST_REPOS that allows one to
|
||||
enable/disable package repos when building
|
||||
Dockerfile: new parameter DIST_REPOS that allows one to
|
||||
enable/disable package repos when building
|
||||
|
||||
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
|
||||
---
|
||||
.gitignore | 1 +
|
||||
Dockerfile | 6 +-
|
||||
stx-scripts/cleanup.sh | 6 ++
|
||||
stx-scripts/install.sh | 14 ++++
|
||||
Dockerfile | 5 +-
|
||||
stx-scripts/install.sh | 11 +++
|
||||
stx-scripts/setup-package-repos.sh | 126 +++++++++++++++++++++++++++++
|
||||
stx-wheels/.keep | 0
|
||||
6 files changed, 152 insertions(+), 1 deletion(-)
|
||||
create mode 100644 .gitignore
|
||||
create mode 100755 stx-scripts/cleanup.sh
|
||||
4 files changed, 141 insertions(+), 1 deletion(-)
|
||||
create mode 100755 stx-scripts/install.sh
|
||||
create mode 100755 stx-scripts/setup-package-repos.sh
|
||||
create mode 100644 stx-wheels/.keep
|
||||
|
||||
diff --git a/.gitignore b/.gitignore
|
||||
new file mode 100644
|
||||
index 0000000..4b5032a
|
||||
--- /dev/null
|
||||
+++ b/.gitignore
|
||||
@@ -0,0 +1 @@
|
||||
+stx-wheels/[^.]*
|
||||
diff --git a/Dockerfile b/Dockerfile
|
||||
index 3a026a3..145d284 100644
|
||||
index 3a026a3..3baea6c 100644
|
||||
--- a/Dockerfile
|
||||
+++ b/Dockerfile
|
||||
@@ -32,4 +32,8 @@ ARG SPICE_REF=${SPICE_REF:-spice-html5-0.1.6}
|
||||
@@ -32,4 +32,7 @@ ARG SPICE_REF=${SPICE_REF:-spice-html5-0.1.6}
|
||||
COPY scripts /opt/loci/scripts
|
||||
ADD bindep.txt pydep.txt $EXTRA_BINDEP $EXTRA_PYDEP /opt/loci/
|
||||
|
||||
|
|
@ -41,26 +29,13 @@ index 3a026a3..145d284 100644
|
|||
+#RUN /opt/loci/scripts/install.sh
|
||||
+ARG DIST_REPOS
|
||||
+COPY stx-scripts /opt/loci/stx-scripts
|
||||
+COPY stx-wheels /opt/loci/stx-wheels
|
||||
+RUN /opt/loci/stx-scripts/install.sh
|
||||
diff --git a/stx-scripts/cleanup.sh b/stx-scripts/cleanup.sh
|
||||
new file mode 100755
|
||||
index 0000000..6ac890f
|
||||
--- /dev/null
|
||||
+++ b/stx-scripts/cleanup.sh
|
||||
@@ -0,0 +1,6 @@
|
||||
+#!/bin/bash
|
||||
+
|
||||
+set -ex
|
||||
+
|
||||
+rm -rf /opt/loci/stx-wheels/*
|
||||
+
|
||||
diff --git a/stx-scripts/install.sh b/stx-scripts/install.sh
|
||||
new file mode 100755
|
||||
index 0000000..033bdb9
|
||||
index 0000000..da11b75
|
||||
--- /dev/null
|
||||
+++ b/stx-scripts/install.sh
|
||||
@@ -0,0 +1,14 @@
|
||||
@@ -0,0 +1,11 @@
|
||||
+#!/bin/bash
|
||||
+
|
||||
+set -ex
|
||||
|
|
@ -72,9 +47,6 @@ index 0000000..033bdb9
|
|||
+
|
||||
+# run Loci installer
|
||||
+"$LOCI_DIR/scripts/install.sh" "$@"
|
||||
+
|
||||
+# delete wheel tarball etc
|
||||
+"$LOCI_DIR/stx-scripts/cleanup.sh"
|
||||
diff --git a/stx-scripts/setup-package-repos.sh b/stx-scripts/setup-package-repos.sh
|
||||
new file mode 100755
|
||||
index 0000000..dd43612
|
||||
Loading…
Reference in New Issue