From f457bd15b9ce0be512ec96abcb9b858b0bab2ea8 Mon Sep 17 00:00:00 2001 From: Scott Little Date: Fri, 16 Apr 2021 01:39:17 -0400 Subject: [PATCH] Improved branching tools create_branches_and_tags.sh: - Update the .gitreview files in branched git repos. - When updating a manifest, add the ability to update and use the default revision field. - Create two levels of manifest lockdown, soft and hard. Soft lockdown only sets sha revisions on unbranched projects that lack a revision, or set the revision to master. Hard lockdown applies to all unbranched projects. push_branches_tags.sh: - opendev no longer accepts 'git push' for the delivery of new branches with updates. Instead we must now use separate commands to deliver the tag, the branch, and any updates. Closes-Bug: 1924762 Signed-off-by: Scott Little Change-Id: I6d669ddc80cc9b3cb9e72d65a64589dbccf43ae3 --- .../branching/create_branches_and_tags.sh | 76 ++++- build-tools/branching/push_branches_tags.sh | 78 ++++- build-tools/repo-utils.sh | 130 ++++++++- build-tools/url_utils.sh | 267 ++++++++++++++++++ 4 files changed, 515 insertions(+), 36 deletions(-) create mode 100755 build-tools/url_utils.sh diff --git a/build-tools/branching/create_branches_and_tags.sh b/build-tools/branching/create_branches_and_tags.sh index d87e0987..27c820c4 100755 --- a/build-tools/branching/create_branches_and_tags.sh +++ b/build-tools/branching/create_branches_and_tags.sh @@ -24,7 +24,7 @@ CREATE_BRANCHES_AND_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" source "${CREATE_BRANCHES_AND_TAGS_SH_DIR}/../git-repo-utils.sh" usage () { - echo "create_branches_and_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --manifest [ --lock-down ]]" + echo "create_branches_and_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --gitreview-default ] [ --manifest [ --lock-down | --soft-lock-down ] [ --default-revision ]]" echo "" echo "Create a branch and a tag in all listed projects, and all" echo "projects hosted by all listed remotes. Lists are comma separated." @@ -33,13 +33,19 @@ usage () { echo "If the tag is omitted, one is automativally generate by adding the" echo "prefix 'v' to the branch name." echo "" - echo "If a manifest is requested, it will recieve the name '.xml' and" - echo "it will specify the branch as the revision for all tagged projects." + echo "If a manifest is requested, the current manifest is modified." + echo "to specify the new branch for all select remotes and projects." echo "If lockdown is requested, all other projects get the current" echo "HEAD's sha set as the revision." + echo "If default-revision is selected, then the manifest default revision" + wcho "will be set." + echo "" + echo "If a gitreview-default is selected, then all branched projects" + echo "with a .gitreview file will have a defaultbranch entry added" + echo "or updated." } -TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,help -n 'create_branches_and_tags.sh' -- "$@") +TEMP=$(getopt -o h --long remotes:,projects:,branch:,tag:,manifest,lock-down,hard-lock-down,soft-lock-down,default-revision,gitreview-default,help -n 'create_branches_and_tags.sh' -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -49,6 +55,8 @@ eval set -- "$TEMP" HELP=0 MANIFEST=0 LOCK_DOWN=0 +GITREVIEW_DEFAULT=0 +SET_DEFAULT_REVISION=0 remotes="" projects="" branch="" @@ -59,15 +67,19 @@ repo_root_dir="" while true ; do case "$1" in - -h|--help) HELP=1 ; shift ;; - --remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;; - --projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;; - --branch) branch=$2; shift 2;; - --tag) tag=$2; shift 2;; - --manifest) MANIFEST=1 ; shift ;; - --lock-down) LOCK_DOWN=1 ; shift ;; - --) shift ; break ;; - *) usage; exit 1 ;; + -h|--help) HELP=1 ; shift ;; + --remotes) remotes+=$(echo "$2 " | tr ',' ' '); shift 2;; + --projects) projects+=$(echo "$2 " | tr ',' ' '); shift 2;; + --branch) branch=$2; shift 2;; + --tag) tag=$2; shift 2;; + --manifest) MANIFEST=1 ; shift ;; + --lock-down) LOCK_DOWN=2 ; shift ;; + --hard-lock-down) LOCK_DOWN=2 ; shift ;; + --soft-lock-down) LOCK_DOWN=1 ; shift ;; + --default-revision) SET_DEFAULT_REVISION=1 ; shift ;; + --gitreview-default) GITREVIEW_DEFAULT=1 ; shift ;; + --) shift ; break ;; + *) usage; exit 1 ;; esac done @@ -88,6 +100,37 @@ if [ $? -ne 0 ]; then exit 1 fi +update_gitreview () { + local DIR=$1 + ( + cd $DIR || exit 1 + if [ $GITREVIEW_DEFAULT -eq 1 ] && [ -f .gitreview ]; then + if ! grep -q "^defaultbranch=$branch$" .gitreview; then + echo "Updating defaultbranch in ${DIR}/.gitreview" + if grep -q defaultbranch= .gitreview; then + sed "s#\(defaultbranch=\).*#\1$branch#" -i .gitreview + else + echo "defaultbranch=$branch" >> .gitreview + fi + + git add .gitreview + if [ $? != 0 ] ; then + echo_stderr "ERROR: failed to add .gitreview in ${DIR}" + exit 1 + fi + + git commit -s -m "Update .gitreview for $branch" + if [ $? != 0 ] ; then + echo_stderr "ERROR: failed to commit .gitreview in ${DIR}" + exit 1 + fi + else + echo "defaultbranch in ${DIR}/.gitreview already set" + fi + fi + ) +} + if [ $MANIFEST -eq 1 ]; then manifest=$(repo_manifest $repo_root_dir) if [ $? -ne 0 ]; then @@ -205,6 +248,7 @@ for subgit in $SUBGITS; do git checkout $branch fi + # check if destination tag already exists tag_check=$(git tag -l $tag) if [ -z "$tag_check" ]; then echo "Creating tag $tag in ${subgit}" @@ -216,6 +260,8 @@ for subgit in $SUBGITS; do else echo "Tag '$tag' already exists in ${subgit}" fi + + update_gitreview ${subgit} || exit 1 ) || exit 1 done ) || exit 1 @@ -276,8 +322,10 @@ if [ $MANIFEST -eq 1 ]; then exit 1 fi + update_gitreview ${manifest_dir} || exit 1 + echo "Creating manifest ${new_manifest_name}" - manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} $projects || exit 1 + manifest_set_revision "${manifest}" "${new_manifest}" "$branch" ${LOCK_DOWN} ${SET_DEFAULT_REVISION} $projects || exit 1 echo "Move manifest ${new_manifest_name}, overwriting ${manifest_name}" \cp -f "${manifest}" "${manifest}.save" diff --git a/build-tools/branching/push_branches_tags.sh b/build-tools/branching/push_branches_tags.sh index 1f94cc42..baa73bb6 100755 --- a/build-tools/branching/push_branches_tags.sh +++ b/build-tools/branching/push_branches_tags.sh @@ -17,6 +17,7 @@ PUSH_BRANCHES_TAGS_SH_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )" source "${PUSH_BRANCHES_TAGS_SH_DIR}/../git-repo-utils.sh" +source "${PUSH_BRANCHES_TAGS_SH_DIR}/../url_utils.sh" usage () { echo "push_branches_tags.sh --branch= [--tag=] [ --remotes= ] [ --projects= ] [ --manifest ]" @@ -151,6 +152,8 @@ for subgit in $SUBGITS; do ( cd $subgit + git fetch --all + branch_check=$(git branch -a --list $branch) if [ -z "$branch_check" ]; then echo_stderr "ERROR: Expected branch '$branch' to exist in ${subgit}" @@ -169,21 +172,49 @@ for subgit in $SUBGITS; do exit 1 fi - if [ "${review_method}" == "gerrit" ]; then - remote=$(git_repo_review_remote) - else - remote=$(git_repo_remote) + remote=$(git_remote) + if [ "${remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}" + exit 1 fi - if [ "${remote}" == "" ]; then - echo_stderr "ERROR: Failed to determine remote in ${subgit}" + if [ "${review_method}" == "gerrit" ]; then + review_remote=$(git_repo_review_remote) + else + review_remote=${remote} + fi + + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_remote in ${subgit}" exit 1 fi + branch_check=$(git branch -a --list $remote/$branch) + if [ "${branch_check}" != "" ]; then + echo "Branch $branch already exists in ${subgit}" + exit 0 + fi + echo "Pushing branch $branch in ${subgit}" if [ "${review_method}" == "gerrit" ]; then - echo "git push --tags ${remote} ${branch}" - git push --tags ${remote} ${branch} + url=$(git_repo_review_url) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_url in ${subgit}" + exit 1 + fi + + host=$(url_server "${url}") + port=$(url_port "${url}") + path=$(url_path "${url}") + if [ "${host}" == "review.opendev.org" ]; then + git push ${review_remote} ${tag} && \ + ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ + git review --topic="${branch}" + else + echo "git push --tags ${remote} ${branch}" + git push --tags ${remote} ${branch} + fi else echo "git push --tags --set-upstream ${remote} ${branch}" git push --tags --set-upstream ${remote} ${branch} @@ -232,23 +263,44 @@ if [ $MANIFEST -eq 1 ]; then exit 1 fi - - remote=$(git_review_remote) + remote=$(git_remote) if [ "${remote}" == "" ]; then echo_stderr "ERROR: Failed to determine remote in ${manifest_dir}" exit 1 fi + review_remote=$(git_review_remote) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_remote in ${manifest_dir}" + exit 1 + fi + echo "Pushing branch $branch in ${manifest_dir}" if [ "${review_method}" == "gerrit" ]; then # Is a reviewless push possible as part of creating a new branch in gerrit? - git push --tags ${remote} ${branch} + url=$(git_review_url) + if [ "${review_remote}" == "" ]; then + echo_stderr "ERROR: Failed to determine review_url in ${subgit}" + exit 1 + fi + + host=$(url_server "${url}") + port=$(url_port "${url}") + path=$(url_path "${url}") + if [ "${host}" == "review.opendev.org" ]; then + git push ${review_remote} ${tag} && \ + ssh -p ${port} ${host} gerrit create-branch ${path} ${branch} ${tag} && \ + git config --local --replace-all "branch.${branch}.merge" refs/heads/${branch} && \ + git review --yes --topic="${branch}" + else + git push --tags ${review_remote} ${branch} + fi else - git push --tags --set-upstream ${remote} ${branch} + git push --tags --set-upstream ${review_remote} ${branch} fi if [ $? != 0 ] ; then - echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${remote}' in ${manifest_dir}" + echo_stderr "ERROR: Failed to push tag '${tag}' to remote '${review_remote}' in ${manifest_dir}" exit 1 fi ) || exit 1 diff --git a/build-tools/repo-utils.sh b/build-tools/repo-utils.sh index 9514b0e9..49f3d8eb 100644 --- a/build-tools/repo-utils.sh +++ b/build-tools/repo-utils.sh @@ -140,6 +140,50 @@ repo_is_project () { } +# +# manifest_get_revision_of_project +# +# Extract the revision of a project within the manifest. +# The default revision is supplied in the absence +# of an explicit project revision. +# +# manifest = Path to manifest. +# project-name = name of project. +# +manifest_get_revision_of_project () { + local manifest="${1}" + local project="${2}" + + local default_revision="" + local revision="" + + default_revision=$(manifest_get_default_revision "${manifest}") + revision=$(grep ' +# +# Extract the default revision of the manifest, if any. +# +# manifest = Path to manifest. +# +manifest_get_default_revision () { + local manifest="${1}" + + grep ' # @@ -149,8 +193,10 @@ repo_is_project () { # revision = A branch, tag ,or sha. Branch and SHA can be used # directly, but repo requires that a tag be in the form # "refs/tags/". -# lock_down = 0 or 1. If 1, set a revision on all other non-listed +# lock_down = 0,1 or 2. If 2, set a revision on all other non-listed # projects to equal the SHA of the current git head. +# If 1, similar to 2, but only if the project doesn't have +# some other form of revision specified. # project-list = A space seperated list of projects. Listed projects # will have their revision set to the provided revision # value. @@ -160,9 +206,11 @@ manifest_set_revision () { local new_manifest="${2}" local revision="${3}" local lock_down="${4}" - shift 4 + local set_default="${5}" + shift 5 local projects="${@}" + local old_default_revision="" local repo_root_dir="" local line="" local FOUND=0 @@ -192,11 +240,32 @@ manifest_set_revision () { return 1 fi + old_default_revision=$(manifest_get_default_revision "${old_manifest}") + if [ ${set_default} -eq 1 ] && [ "${old_default_revision}" == "" ]; then + # We only know how to alter an existing default revision, not set a + # new one, so continue without setting a default. + set_default=0 + fi + while IFS= read -r line; do echo "${line}" | grep -q '&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + if echo "$URL" | grep -q '[:][/][/]' ;then + echo "$URL" | sed 's#^\(.*\)://.*$#\1#' + else + echo "http" + fi + return 0 +} + +url_login () { + local URL="$1" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$URL" | sed 's#^.*://\([^/]*\)/.*$#\1#' + return 0 +} + +url_user () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed -e '/@/! s#.*## ; s#\([^@]*\)@.*#\1#' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_port () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed -e '/:/! s#.*## ; s#[^:]*:\([^:]*\)#\1#' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_server () { + local URL="$1" + local LOGIN + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + url_login "$URL" | sed 's#^.*@## ; s#:.*$##' + if [ ${PIPESTATUS[0]} -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_login failed" + return 1 + fi + + return 0 +} + +url_path () { + local URL="$1" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$URL" | sed 's#^.*://[^/]*/\(.*\)$#\1#' + return 0 +} + +# +# url_path_to_fs_path: +# +# Convert url format path to file system format. +# e.g. replace %20 with ' '. +# +# Note: Does NOT test the output path to ensure there are +# no illegal file system characters. +# +url_path_to_fs_path () { + local INPUT_PATH="$1" + local TEMP + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + # Deviate from URI spec by not substituding '+' with ' '. + # It would alias '%20' and we need unique mappings. + # TEMP="${INPUT_PATH//+/ }" + + TEMP="$INPUT_PATH" + printf '%b' "${TEMP//%/\\x}" + return 0 +} + +# +# fs_path_to_url_path: +# +# Convert file system format path to url format. +# e.g. replace ' ' with %20. +# +fs_path_to_url_path () { + local INPUT_PATH="$1" + local LENGTH + local POS + local CHAR + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + LENGTH="${#INPUT_PATH}" + for (( POS = 0; POS < LENGTH; POS++ )); do + CHAR="${1:POS:1}" + case $CHAR in + [/a-zA-Z0-9.~_-]) + # Reference https://metacpan.org/pod/URI::Escape + printf "$CHAR" + ;; + *) + printf '%%%02X' "'$CHAR" + ;; + esac + done + + return 0 +} + +# +# normalize_path: +# +# 1) replace // with / +# 2) replace /./ with / +# 3) Remove trailing / +# 4) Remove leading ./ +# + +normalize_path () { + local INPUT_PATH="$1" + + if [ "$INPUT_PATH" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + echo "$INPUT_PATH" | sed 's#[/]\+#/#g ; s#[/][.][/]#/#g ; s#/$## ; s#^[.]/##' + return 0 +} + + +# +# repo_url_to_sub_path: +# +repo_url_to_sub_path () { + local URL="$1" + local FAMILY="" + local SERVER="" + local URL_PATH="" + local FS_PATH="" + + if [ "$URL" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + # set FAMILY from URL + echo $URL | grep -q 'centos[.]org' && FAMILY=centos + echo $URL | grep -q 'fedoraproject[.]org[/]pub[/]epel' && FAMILY=epel + + SERVER=$(url_server "$URL") + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_server '$URL'" + return 1 + fi + + URL_PATH="$(url_path "$URL")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_path '$URL'" + return 1 + fi + + FS_PATH="$(url_path_to_fs_path "$URL_PATH")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): url_path_to_fs_path '$URL_PATH'" + return 1 + fi + + FS_PATH="$(normalize_path "$FS_PATH")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path '$FS_PATH'" + return 1 + fi + + normalize_path "./$FAMILY/$SERVER/$FS_PATH" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): normalize_path './$FAMILY/$SERVER/$FS_PATH'" + return 1 + fi + + return 0 +} + +CENGN_PROTOCOL="http" +CENGN_HOST="mirror.starlingx.cengn.ca" +CENGN_PORT="80" +CENGN_URL_ROOT="mirror" + +url_to_stx_mirror_url () { + local URL="$1" + local DISTRO="$2" + local URL_PATH="" + local FS_PATH="" + + if [ "$URL" == "" ] || [ "$DISTRO" == "" ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): empty argument" + return 1 + fi + + FS_PATH="$(repo_url_to_sub_path "$URL")" + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): repo_url_to_sub_path '$URL'" + return 1 + fi + + URL_PATH=$(fs_path_to_url_path "$FS_PATH") + if [ $? -ne 0 ]; then + >&2 echo "Error: $FUNCNAME (${LINENO}): fs_path_to_url_path '$FS_PATH'" + return 1 + fi + + echo "$CENGN_PROTOCOL://$CENGN_HOST:$CENGN_PORT/$CENGN_URL_ROOT/$DISTRO/$URL_PATH" + return 0 +}