diff --git a/build-tools/build-helm-charts.sh b/build-tools/build-helm-charts.sh index 09f3a6fa..520bdd43 100755 --- a/build-tools/build-helm-charts.sh +++ b/build-tools/build-helm-charts.sh @@ -10,17 +10,10 @@ # BUILD_HELM_CHARTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1 source $BUILD_HELM_CHARTS_DIR/utils.sh || exit 1 -# Required env vars -if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then - echo "Environment not setup for builds" >&2 - exit 1 -fi - -SUPPORTED_OS_ARGS=('centos') -OS=centos +SUPPORTED_OS_ARGS=('centos' 'debian') +OS= LABEL="" APP_NAME="stx-openstack" APP_VERSION_BASE="helm-charts-release-info.inc" @@ -31,6 +24,10 @@ declare -a PATCH_DEPENDENCIES declare -a APP_PACKAGES declare -a CHART_PACKAGE_FILES +VERBOSE=false +CPIO_FLAGS= +TAR_FLAGS= + function usage { cat >&2 </dev/null 2>&1 ; then + python_found=true + break + fi + done + if [[ -z "$python_found" ]] ; then + echo "ERROR: can't find python!" >&2 + exit 1 + fi + + $python -c "${yaml_script}" ${@} || exit 1 } # Find a file named $APP_VERSION_BASE at top-level of each git repo @@ -364,8 +378,20 @@ function find_package_files { "${centos_repo}/Binary/noarch" \ -type f -name "*.tis.noarch.rpm" else - echo "ERROR: unsupported OS $OS" >&2 - exit 1 + # FIXME: can't search 3rd-party binary debs because they are not accessible + # on the filesystem, but only as remote files in apt repos + find "${MY_WORKSPACE}/std" \ + -mindepth 2 \ + -maxdepth 2 \ + "(" \ + "(" \ + -path "${MY_WORKSPACE}/build-wheels" \ + -o -path "${MY_WORKSPACE}/build-images" \ + -o -path "${MY_WORKSPACE}/build-helm" \ + ")" -prune \ + ")" \ + -o \ + "(" -type f -name "*.stx.*_all.deb" ")" fi } @@ -385,7 +411,14 @@ function find_helm_chart_package_files { echo "searching for package files" >&2 local package_file package_name for package_file in $(find_package_files) ; do - package_name=$(rpm_get_name "$package_file") || exit 1 + package_name="$( + if [[ "$OS" == "centos" ]] ; then + rpm_get_name "$package_file" || exit 1 + else + deb_get_control "$package_file" | deb_get_field "Package" + check_pipe_status + fi + )" || exit 1 if [[ -n "${package_names[$package_name]}" && "${package_names[$package_name]}" != "$package_file" ]] ; then echo "ERROR: found multiple packages named ${package_name}:" >&2 echo " $package_file" >&2 @@ -423,8 +456,13 @@ function find_helm_chart_package_files { fi local -a dep_package_names=($( - rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/' - check_pipe_status || exit 1 + if [[ "$OS" == "centos" ]] ; then + rpm -qRp "$package_file" | sed 's/rpmlib([a-zA-Z0-9]*)[[:space:]]\?[><=!]\{0,2\}[[:space:]]\?[0-9.-]*//g' | grep -v '/' + check_pipe_status || exit 1 + else + deb_get_control "$package_file" | deb_get_simple_depends + check_pipe_status || exit 1 + fi )) || exit 1 # save top-level package @@ -477,8 +515,16 @@ function extract_chart_from_package { echo "Failed to extract content of helm package: ${package_file}" >&2 exit 1 fi - ;; + + debian) + deb_extract_content "$package_file" $([[ "$VERBOSE" == "true" ]] && echo --verbose || true) + if ! check_pipe_status ; then + echo "Failed to extract content of helm package: ${package_file}" >&2 + exit 1 + fi + ;; + *) echo "Unsupported OS ${OS}" >&2 ;; @@ -543,8 +589,18 @@ function get_app_version { echo "extracting version from $1" >&2 local app_version app_version="$( - rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g' - check_pipe_status + if [[ "$OS" == "centos" ]] ; then + rpm -q --qf '%{VERSION}-%{RELEASE}' -p "$1" | sed 's![.]tis!!g' + check_pipe_status || exit 1 + else + control="$(deb_get_control "$1")" || exit 1 + version="$(echo "$control" | deb_get_field "Version" | sed -r -e 's/^[^:]+:+//')" + if [[ -z "$version" ]] ; then + echo "ERROR: failed to determine the version of package $1" >&2 + exit 1 + fi + echo "${version}" + fi )" || exit 1 echo "APP_VERSION=$app_version" >&2 echo "$app_version" @@ -582,7 +638,10 @@ while true; do APP_VERSION="$2" shift 2 ;; - -r | --rpm) + -r | --rpm | --package) + if [[ "$1" == "--rpm" ]] ; then + echo "WARNING: option $1 is deprecated, use --package instead" >&2 + fi APP_PACKAGES+=(${2//,/ }) shift 2 ;; @@ -623,6 +682,16 @@ else TAR_FLAGS=-zcf fi +# Validate OS +if [ -z "$OS" ] ; then + OS="$(ID= && source /etc/os-release 2>/dev/null && echo $ID || true)" + if [[ -z "$OS" ]] ; then + echo "Unable to determine OS, please re-run with \`--os' option" >&2 + exit 1 + elif [[ "$OS" != "debian" ]] ; then + OS="centos" + fi +fi VALID_OS=1 for supported_os in ${SUPPORTED_OS_ARGS[@]}; do if [ "$OS" = "${supported_os}" ]; then @@ -636,6 +705,19 @@ if [ ${VALID_OS} -ne 0 ]; then exit 1 fi +# Required env vars +if [ -z "${MY_WORKSPACE}" -o -z "${MY_REPO}" ]; then + echo "Environment not setup for builds" >&2 + exit 1 +fi + +# include SRPM utils +if [[ "$OS" == "centos" ]] ; then + source $BUILD_HELM_CHARTS_DIR/srpm-utils || exit 1 +else + source $BUILD_HELM_CHARTS_DIR/deb-utils.sh || exit 1 +fi + # Commenting out this code that attempts to validate the APP_NAME. # It makes too many assumptions about the location and naming of apps. # diff --git a/build-tools/deb-utils.sh b/build-tools/deb-utils.sh new file mode 100644 index 00000000..a8687b10 --- /dev/null +++ b/build-tools/deb-utils.sh @@ -0,0 +1,100 @@ +# bash +# vim: set syn=sh: + +__DEB_UTILS_DIR=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/deb-utils + +# +# Usage: __deb_get_section DEB_FILE {control|data} +# +# Uncompress and print the specified section to STDOUT in tar format. +# You should pipe it to "tar" to be useful. +# +function __deb_get_section { + local deb_file="$1" + local section="$2" + + # find $section.tar.{gz,bz2,xz} + local section_entry + section_entry="$( + ar t "$deb_file" | \grep "^$section[.]" || true + )" || return 1 + if [[ -z "$section_entry" ]] ; then + echo "$deb_file: couldn't find ${section}.*" >&2 + return 1 + fi + + # untar it to stdout + local uncompress + case "${section_entry#${section}.}" in + tar.gz | tgz) uncompress="gunzip" ;; + tar.bz2) uncompress="bunzip2" ;; + tar.xz) uncompress="unxz" ;; + *) + echo "$deb_file: unsupported archive format $section_entry" >&2 + return 1 + esac + ar p "$1" "$section_entry" | $uncompress + check_pipe_status +} + +# +# Usage: deb_get_control DEB_FILE +# +# Print the control file from the specified DEB package +# +function deb_get_control { + __deb_get_section "$1" control | tar -O -x ./control + check_pipe_status +} + +# +# Usage: deb_extract_content DEB_FILE [--verbose] [PATHS_IN_ARCHIVE...] +# +# Extract deb package content to current directory +# +function deb_extract_content { + __deb_get_section "$1" data | tar -x + check_pipe_status +} + +# +# Usage: deb_get_field KEY... +# +# Read a debian control file from STDIN, find the specified fields +# and print their values on STDOUT. With multiple fields, their values +# will be merged in the output w/no separators. +# +# See: https://www.debian.org/doc/debian-policy/ch-controlfields.html +# +function deb_get_field { + ${PYTHON3:-python3} "${__DEB_UTILS_DIR}/deb_get_field.py" "$@" +} + +# +# Usage: deb_get_simple_depends +# +# Read debian control file from STDIN, then print its immediate runtime +# dependencies to STDOUT, one per line, stripping any conditions and +# operators, e.g.: +# +# ... +# Depends: aaa, bbb [!amd64], ccc | ddd (>= 1.0) +# ... +# +# will be converted to +# +# aaa +# bbb +# ccc +# ddd +# +function deb_get_simple_depends { + local raw_depends + raw_depends=$(deb_get_field 'Pre-Depends' 'Depends') || return 1 + echo $raw_depends \ + | tr ',|' '\n' \ + | sed -r 's/^\s*([^[:space:](><=[]+).*$/\1/' \ + | grep -v -E '^\s*$' \ + | sort -u +} + diff --git a/build-tools/deb-utils/deb_get_field.py b/build-tools/deb-utils/deb_get_field.py new file mode 100755 index 00000000..230510ed --- /dev/null +++ b/build-tools/deb-utils/deb_get_field.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import sys, re + +def usage(): + print ("""\ +Usage: %s KEY... +Read a debian control file from STDIN, print KEY values to STDOUT +""" % sys.argv[0]) + +if len (sys.argv) > 0 and sys.argv[1] == "--help": + usage() + sys.exit(0) + +# regex: "^(?:KEY1|KEY2|...)\s*:\s*(.*?)\s*$" +re_field = re.compile ( + "^(?:" + + "|".join ( + [ re.escape (key) for key in sys.argv[1:] ] + ) + + "):\s*(.*?)\s*$" +) + +re_ws = re.compile ("^\s*$") + +in_header = True +past_1st_paragraph = False +in_multiline_field = False + +for line in sys.stdin: + + # skip initial empty lines + if in_header and re_ws.fullmatch (line): + continue + in_header = False + + # skip everything past the 1st block + if past_1st_paragraph: + continue + if re_ws.fullmatch (line): + past_1st_paragraph = True + continue + + # Key: value + match = re_field.fullmatch (line) + if match: + print (match.group(1)) + in_multiline_field = True + continue + + # line starts with a space or tab: belongs to the previous field + if in_multiline_field and (line.startswith (" ") or line.startswith ("\t")): + print (line[1:].rstrip()) + continue + + in_multiline_field = False diff --git a/build-tools/unit-tests/deb-utils.sh b/build-tools/unit-tests/deb-utils.sh new file mode 100755 index 00000000..6be5c2ab --- /dev/null +++ b/build-tools/unit-tests/deb-utils.sh @@ -0,0 +1,159 @@ +#!/bin/bash + +PROGNAME="$(basename "$0")" + +REQUIRED_PROGS="${PYTHON3:-python3}" + +for prog in ${REQUIRED_PROGS} tar ar ; do + if ! $prog --version >/dev/null 2>&1 ; then + echo "$PROGNAME: WARNING: can't find \"$prog\", skipping tests" >&2 + exit 0 + fi +done + +source "$(dirname "$0")"/../deb-utils.sh || exit 1 +source "$(dirname "$0")"/../utils.sh || exit 1 + +declare -i FAIL_COUNT=0 + +# Usage: expect EXPECTED ACTUAL [DEPTH] +function expect { + local expected="$1" + local actual="$2" + if [[ "${actual}" != "${expected}" ]] ; then + let depth="${3:-0}" + echo >&2 + echo "${BASH_SOURCE[0]}:${BASH_LINENO[${depth}]}: expectation failed:" >&2 + echo " actual: [$actual]" >&2 + echo " expected: [$expected]" >&2 + echo >&2 + return 1 + fi + return 0 +} + +# Usage: echo ACTUAL | expect_stdin EXPECTED +function expect_stdin { + expect "$1" "$(cat)" 1 +} + +######################################################### +# deb_get_field +######################################################### + +##################### +echo "\ +Dummy1: dummy1 +Key1: value1 +Dummy2: dummy2 +" | deb_get_field "Key1" \ + | expect_stdin "value1" \ +|| let ++FAIL_COUNT + +##################### +echo " + +# 1st para +Dummy1: dummy1 +Key1: value1 +Dummy2: dummy2 + +# 2nd para +Dummy3: dummy3 +Key1: value1 +Dummy4: dummy4 + +" | deb_get_field "Key1" \ + | expect_stdin "value1" \ +|| let ++FAIL_COUNT + +##################### +echo " + +# 1st para +Dummy1: dummy1 +Dummy2: dummy2 + +# 2nd para +Dummy3: dummy3 +Key1: value1 +Dummy4: dummy4 + +" | deb_get_field "Key1" \ + | expect_stdin "" \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 + value1_line3 +Dummy2: dummy2_line1 + dummy2_line2 + dummy2_line3 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2\nvalue1_line3' \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + +##################### +echo " + +Dummy1: dummy1 +Key1: value1_line1 + value1_line2 +Dummy2: dummy2 + +" | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + + +##################### +echo $' + +Dummy1: dummy1 +Key1: value1_line1 +\tvalue1_line2 +Dummy2: dummy2_line1 +\tdummy2_line2 + +' | deb_get_field "Key1" \ + | expect_stdin $'value1_line1\nvalue1_line2' \ +|| let ++FAIL_COUNT + + +######################################################### +# deb_get_simple_depends +######################################################### + +echo " +Depends: texinfo (>= 1.0), kernel-headers-2.2.10 [!hurd-i386], + hurd-dev [hurd-i386], gnumach-dev [hurd-i386], yy-foo (>= 1.0) | zz-bar +" | deb_get_simple_depends \ + | expect_stdin $'gnumach-dev\nhurd-dev\nkernel-headers-2.2.10\ntexinfo\nyy-foo\nzz-bar' \ +|| let ++FAIL_COUNT + + +if [[ $FAIL_COUNT -gt 0 ]] ; then + echo >&2 + echo "ERROR: ${FAIL_COUNT} test(s) failed" >&2 + echo >&2 + exit 1 +fi +echo "$PROGNAME: all tests passed" >&2 +exit 0 + diff --git a/tox.ini b/tox.ini index 18757c2b..0e42de60 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = linters +envlist = linters, unit-tests minversion = 2.3 skipsdist = True @@ -57,3 +57,14 @@ commands = [testenv:venv] basepython = python3 commands = {posargs} + +[testenv:unit-tests] +whitelist_externals = bash +basepython = python3 +setenv = PYTHON3=python +commands = + bash -c " \ + for f in {toxinidir}/build-tools/unit-tests/* ; do \ + $f || exit 1 ; \ + done ; \ + "