starlingx
/
root
Code Issues Proposed changes

debian: port helm chart script to debian 

* build-helm-charts.sh:
  - auto-detect $OS
  - process DEB files on Debian

* deb-utils: new file with utilities for working with binary DEB files

* tox.ini: run unit tests for deb-utils

TESTS
========================================

Run script on CentOS and make sure the generated tarball's contents are
the same as before the patch.

Run script on Debian and make sure the generated tarball's contents look
reasonable.

Story: 2009897
Task: 45293

Depends-On: https://review.opendev.org/c/starlingx/openstack-armada-app/+/840561
Change-Id: Icbcb0bb7b47f623fac8d0851687423396edb5747
Signed-off-by: Davlet Panech <davlet.panech@windriver.com>
This commit is contained in:
Davlet Panech 2022-05-04 13:57:52 -04:00
parent 0625f0425a
commit 0f3670fe64
5 changed files with 439 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
build-tools/build-helm-charts.sh
View File

@ -10,17 +10,10 @@
#
BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1
source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
exit 1
fi
SUPPORTED_OS_ARGS=('centos')
OS=centos
SUPPORTED_OS_ARGS=('centos' 'debian')
OS=
LABEL=""
APP_NAME="stx-openstack"
APP_VERSION_BASE="helm-charts-release-info.inc"
@ -31,6 +24,10 @@ declare -a PATCH_DEPENDENCIES
declare -a APP_PACKAGES
declare -a CHART_PACKAGE_FILES
VERBOSE=false
CPIO_FLAGS=
TAR_FLAGS=
function usage {
cat >&2 <<EOF
Usage:
@ -43,22 +40,26 @@ Options:
--os:
Specify base OS (eg. centos)
-a, --app:
-a, --app NAME:
Specify the application name
-A,--app-version-file:
-A, --app-version-file FILENAME:
Specify the file containing version information for the helm
charts. By default we will search for a file named
$APP_VERSION_BASE in all git repos.
-B,--app-version:
-B, --app-version VERSION:
Specify application (tarball) version, this overrides any other
version information.
-r, --rpm:
Specify the application rpms
-r, --package PACKAGE_NAME,... :
Top-level package(s) containing the helm chart(s), comma-separated.
Default: ${APP_NAME}-helm
-i, --image-record:
--rpm PACKAGE_NAME,... :
(Deprecated) same as --package
-i, --image-record FILENAME :
Specify the path to image record file(s) or url(s).
Multiple files/urls can be specified with a comma-separated
list, or with multiple --image-record arguments.
@ -66,12 +67,13 @@ Options:
in multiple files, the last image reference has higher
priority.
-l, --label:
-l, --label LABEL:
Specify the label of the application tarball. The label
will be appended to the version string in tarball name.
-p, --patch-dependency:
Specify the patch dependency of the application tarball.
-p, --patch-dependency DEPENDENCY,... :
Specify the patch dependency of the application tarball,
comma-separated
Multiple patches can be specified with a comma-separated
list, or with multiple --patch-dependency arguments.
@ -306,17 +308,29 @@ def merge_yaml(yaml_merged, yaml_new):
yaml_out = collections.OrderedDict()
for yaml_file in yaml_files:
print 'Merging yaml from file: %s' % yaml_file
print('Merging yaml from file: %s' % yaml_file)
for document in yaml.load_all(open(yaml_file), Loader=yaml.RoundTripLoader, preserve_quotes=True, version=(1, 1)):
document_name = (document['schema'], document['metadata']['schema'], document['metadata']['name'])
if document_name in yaml_out:
merge_yaml(yaml_out[document_name], document)
else:
yaml_out[document_name] = document
print 'Writing merged yaml file: %s' % yaml_output
print('Writing merged yaml file: %s' % yaml_output)
yaml.dump_all(yaml_out.values(), open(yaml_output, 'w'), Dumper=yaml.RoundTripDumper, default_flow_style=False)
"
python -c "${yaml_script}" ${@}
local python python_found
for python in ${PYTHON2:-python2} ${PYTHON:-python} ${PYTHON3:-python3} ; do
if $python -c 'import ruamel.yaml' >/dev/null 2>&1 ; then
python_found=true
break
fi
done
if [[ -z "$python_found" ]] ; then
echo "ERROR: can't find python!" >&2
exit 1
fi
$python -c "${yaml_script}" ${@} || exit 1
}
# Find a file named $APP_VERSION_BASE at top-level of each git repo
@ -364,8 +378,20 @@ function find_package_files {
"${centos_repo}/Binary/noarch" \
-type f -name "*.tis.noarch.rpm"
else
echo "ERROR: unsupported OS $OS" >&2
exit 1
# FIXME: can't search 3rd-party binary debs because they are not accessible
# on the filesystem, but only as remote files in apt repos
find "${MY_WORKSPACE}/std" \
-mindepth 2 \
-maxdepth 2 \
"(" \
"(" \
-path "${MY_WORKSPACE}/build-wheels" \
-o -path "${MY_WORKSPACE}/build-images" \
-o -path "${MY_WORKSPACE}/build-helm" \
")" -prune \
")" \
-o \
"(" -type f -name "*.stx.*_all.deb" ")"
fi
}
@ -385,7 +411,14 @@ function find_helm_chart_package_files {
echo "searching for package files" >&2
local package_file package_name
for package_file in $(find_package_files) ; do
package_name=$(rpm_get_name "$package_file") || exit 1
package_name="$(
if [[ "$OS" == "centos" ]] ; then
rpm_get_name "$package_file" || exit 1
else
deb_get_control "$package_file" | deb_get_field "Package"
check_pipe_status
fi
)" || exit 1
if [[ -n "${package_names[$package_name]}" && "${package_names[$package_name]}" != "$package_file" ]] ; then
echo "ERROR: found multiple packages named ${package_name}:" >&2
echo " $package_file" >&2
@ -423,8 +456,13 @@ function find_helm_chart_package_files {
fi
local -a dep_package_names=($(
rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/'
check_pipe_status || exit 1
if [[ "$OS" == "centos" ]] ; then
rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/'
check_pipe_status || exit 1
else
deb_get_control "$package_file" | deb_get_simple_depends
check_pipe_status || exit 1
fi
)) || exit 1
# save top-level package
@ -477,8 +515,16 @@ function extract_chart_from_package {
echo "Failed to extract content of helm package: ${package_file}" >&2
exit 1
fi
;;
debian)
deb_extract_content "$package_file" $([[ "$VERBOSE" == "true" ]] && echo --verbose || true)
if ! check_pipe_status ; then
echo "Failed to extract content of helm package: ${package_file}" >&2
exit 1
fi
;;
*)
echo "Unsupported OS ${OS}" >&2
;;
@ -543,8 +589,18 @@ function get_app_version {
echo "extracting version from $1" >&2
local app_version
app_version="$(
rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g'
check_pipe_status
if [[ "$OS" == "centos" ]] ; then
rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g'
check_pipe_status || exit 1
else
control="$(deb_get_control "$1")" || exit 1
version="$(echo "$control" | deb_get_field "Version" | sed -r -e 's/^[^:]+:+//')"
if [[ -z "$version" ]] ; then
echo "ERROR: failed to determine the version of package $1" >&2
exit 1
fi
echo "${version}"
fi
)" || exit 1
echo "APP_VERSION=$app_version" >&2
echo "$app_version"
@ -582,7 +638,10 @@ while true; do
APP_VERSION="$2"
shift 2
;;
-r | --rpm)
-r | --rpm | --package)
if [[ "$1" == "--rpm" ]] ; then
echo "WARNING: option $1 is deprecated, use --package instead" >&2
fi
APP_PACKAGES+=(${2//,/ })
shift 2
;;
@ -623,6 +682,16 @@ else
TAR_FLAGS=-zcf
fi
# Validate OS
if [ -z "$OS" ] ; then
OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)"
if [[ -z "$OS" ]] ; then
echo "Unable to determine OS, please re-run with \`--os' option" >&2
exit 1
elif [[ "$OS" != "debian" ]] ; then
OS="centos"
fi
fi
VALID_OS=1
for supported_os in ${SUPPORTED_OS_ARGS[@]}; do
if [ "$OS" = "${supported_os}" ]; then
@ -636,6 +705,19 @@ if [ ${VALID_OS} -ne 0 ]; then
exit 1
fi
# Required env vars
if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then
echo "Environment not setup for builds" >&2
exit 1
fi
# include SRPM utils
if [[ "$OS" == "centos" ]] ; then
source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1
else
source $BUILD_HELM_CHARTS_DIR/deb-utils.sh || exit 1
fi
# Commenting out this code that attempts to validate the APP_NAME.
# It makes too many assumptions about the location and naming of apps.
#

100
build-tools/deb-utils.sh Normal file
View File

@ -0,0 +1,100 @@
# bash
# vim: set syn=sh:
__DEB_UTILS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/deb-utils
#
# Usage: __deb_get_section DEB_FILE {control|data}
#
# Uncompress and print the specified section to STDOUT in tar format.
# You should pipe it to "tar" to be useful.
#
function __deb_get_section {
local deb_file="$1"
local section="$2"
# find $section.tar.{gz,bz2,xz}
local section_entry
section_entry="$(
ar t "$deb_file" | \grep "^$section[.]" || true
)" || return 1
if [[ -z "$section_entry" ]] ; then
echo "$deb_file: couldn't find ${section}.*" >&2
return 1
fi
# untar it to stdout
local uncompress
case "${section_entry#${section}.}" in
tar.gz | tgz) uncompress="gunzip" ;;
tar.bz2) uncompress="bunzip2" ;;
tar.xz) uncompress="unxz" ;;
*)
echo "$deb_file: unsupported archive format $section_entry" >&2
return 1
esac
ar p "$1" "$section_entry" | $uncompress
check_pipe_status
}
#
# Usage: deb_get_control DEB_FILE
#
# Print the control file from the specified DEB package
#
function deb_get_control {
__deb_get_section "$1" control | tar -O -x ./control
check_pipe_status
}
#
# Usage: deb_extract_content DEB_FILE [--verbose] [PATHS_IN_ARCHIVE...]
#
# Extract deb package content to current directory
#
function deb_extract_content {
__deb_get_section "$1" data | tar -x
check_pipe_status
}
#
# Usage: deb_get_field KEY...
#
# Read a debian control file from STDIN, find the specified fields
# and print their values on STDOUT. With multiple fields, their values
# will be merged in the output w/no separators.
#
# See: https://www.debian.org/doc/debian-policy/ch-controlfields.html
#
function deb_get_field {
${PYTHON3:-python3} "${__DEB_UTILS_DIR}/deb_get_field.py" "$@"
}
#
# Usage: deb_get_simple_depends
#
# Read debian control file from STDIN, then print its immediate runtime
# dependencies to STDOUT, one per line, stripping any conditions and
# operators, e.g.:
#
# ...
# Depends: aaa, bbb [!amd64], ccc | ddd (>= 1.0)
# ...
#
# will be converted to
#
# aaa
# bbb
# ccc
# ddd
#
function deb_get_simple_depends {
local raw_depends
raw_depends=$(deb_get_field 'Pre-Depends' 'Depends') || return 1
echo $raw_depends \
| tr ',|' '\n' \
| sed -r 's/^\s*([^[:space:](><=[]+).*$/\1/' \
| grep -v -E '^\s*$' \
| sort -u
}

56
build-tools/deb-utils/deb_get_field.py Executable file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env python3
import sys, re
def usage():
print ("""\
Usage: %s KEY...
Read a debian control file from STDIN, print KEY values to STDOUT
""" % sys.argv[0])
if len (sys.argv) > 0 and sys.argv[1] == "--help":
usage()
sys.exit(0)
# regex: "^(?:KEY1|KEY2|...)\s*:\s*(.*?)\s*$"
re_field = re.compile (
"^(?:" +
"|".join (
[ re.escape (key) for key in sys.argv[1:] ]
) +
"):\s*(.*?)\s*$"
)
re_ws = re.compile ("^\s*$")
in_header = True
past_1st_paragraph = False
in_multiline_field = False
for line in sys.stdin:
# skip initial empty lines
if in_header and re_ws.fullmatch (line):
continue
in_header = False
# skip everything past the 1st block
if past_1st_paragraph:
continue
if re_ws.fullmatch (line):
past_1st_paragraph = True
continue
# Key: value
match = re_field.fullmatch (line)
if match:
print (match.group(1))
in_multiline_field = True
continue
# line starts with a space or tab: belongs to the previous field
if in_multiline_field and (line.startswith (" ") or line.startswith ("\t")):
print (line[1:].rstrip())
continue
in_multiline_field = False

159
build-tools/unit-tests/deb-utils.sh Executable file
View File

@ -0,0 +1,159 @@
#!/bin/bash
PROGNAME="$(basename "$0")"
REQUIRED_PROGS="${PYTHON3:-python3}"
for prog in ${REQUIRED_PROGS} tar ar ; do
if ! $prog --version >/dev/null 2>&1 ; then
echo "$PROGNAME: WARNING: can't find \"$prog\", skipping tests" >&2
exit 0
fi
done
source "$(dirname "$0")"/../deb-utils.sh || exit 1
source "$(dirname "$0")"/../utils.sh || exit 1
declare -i FAIL_COUNT=0
# Usage: expect EXPECTED ACTUAL [DEPTH]
function expect {
local expected="$1"
local actual="$2"
if [[ "${actual}" != "${expected}" ]] ; then
let depth="${3:-0}"
echo >&2
echo "${BASH_SOURCE[0]}:${BASH_LINENO[${depth}]}: expectation failed:" >&2
echo " actual: [$actual]" >&2
echo " expected: [$expected]" >&2
echo >&2
return 1
fi
return 0
}
# Usage: echo ACTUAL | expect_stdin EXPECTED
function expect_stdin {
expect "$1" "$(cat)" 1
}
#########################################################
# deb_get_field
#########################################################
#####################
echo "\
Dummy1: dummy1
Key1: value1
Dummy2: dummy2
" | deb_get_field "Key1" \
| expect_stdin "value1" \
|| let ++FAIL_COUNT
#####################
echo "
# 1st para
Dummy1: dummy1
Key1: value1
Dummy2: dummy2
# 2nd para
Dummy3: dummy3
Key1: value1
Dummy4: dummy4
" | deb_get_field "Key1" \
| expect_stdin "value1" \
|| let ++FAIL_COUNT
#####################
echo "
# 1st para
Dummy1: dummy1
Dummy2: dummy2
# 2nd para
Dummy3: dummy3
Key1: value1
Dummy4: dummy4
" | deb_get_field "Key1" \
| expect_stdin "" \
|| let ++FAIL_COUNT
#####################
echo "
Dummy1: dummy1
Key1: value1_line1
value1_line2
value1_line3
Dummy2: dummy2_line1
dummy2_line2
dummy2_line3
" | deb_get_field "Key1" \
| expect_stdin $'value1_line1\nvalue1_line2\nvalue1_line3' \
|| let ++FAIL_COUNT
#####################
echo "
Dummy1: dummy1
Key1: value1_line1
value1_line2
" | deb_get_field "Key1" \
| expect_stdin $'value1_line1\nvalue1_line2' \
|| let ++FAIL_COUNT
#####################
echo "
Dummy1: dummy1
Key1: value1_line1
value1_line2
Dummy2: dummy2
" | deb_get_field "Key1" \
| expect_stdin $'value1_line1\nvalue1_line2' \
|| let ++FAIL_COUNT
#####################
echo $'
Dummy1: dummy1
Key1: value1_line1
\tvalue1_line2
Dummy2: dummy2_line1
\tdummy2_line2
' | deb_get_field "Key1" \
| expect_stdin $'value1_line1\nvalue1_line2' \
|| let ++FAIL_COUNT
#########################################################
# deb_get_simple_depends
#########################################################
echo "
Depends: texinfo (>= 1.0), kernel-headers-2.2.10 [!hurd-i386],
hurd-dev [hurd-i386], gnumach-dev [hurd-i386], yy-foo (>= 1.0) | zz-bar
" | deb_get_simple_depends \
| expect_stdin $'gnumach-dev\nhurd-dev\nkernel-headers-2.2.10\ntexinfo\nyy-foo\nzz-bar' \
|| let ++FAIL_COUNT
if [[ $FAIL_COUNT -gt 0 ]] ; then
echo >&2
echo "ERROR: ${FAIL_COUNT} test(s) failed" >&2
echo >&2
exit 1
fi
echo "$PROGNAME: all tests passed" >&2
exit 0

13
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = linters
envlist = linters, unit-tests
minversion = 2.3
skipsdist = True
@ -57,3 +57,14 @@ commands =
[testenv:venv]
basepython = python3
commands = {posargs}
[testenv:unit-tests]
whitelist_externals = bash
basepython = python3
setenv = PYTHON3=python
commands =
bash -c " \
for f in {toxinidir}/build-tools/unit-tests/* ; do \
$f || exit 1 ; \
done ; \
"