From 2684e9e3fd77b6cd13d2531131482587c7189a8c Mon Sep 17 00:00:00 2001 From: Scott Little Date: Tue, 12 Sep 2023 16:01:10 -0400 Subject: [PATCH] cengn reference removal - debian mirror.starlingx.cengn.ca no longer exists. CENGN is kindly forwarding requests to the new location mirror.starlingx.windriver.com for now, but that will only last a few months. We need to replace all the references with the new URL. I will also remove as many 'cengn' references as possible, replacing them with 'stx_mirror' TESTS ===== Run "stx config --upgrade" with various combinations of the old, new & missing values. Run "./stx-init-env --rebuild" and verify stx-localrc is updated as well. Partial-Bug: 2033555 Signed-off-by: Scott Little Change-Id: I18491fb5aa0087029d51bb35c236613c1dc092bc --- release/README.rst | 2 +- stx.conf.sample | 8 ++-- stx/dockerfiles/stx-lat-tool.Dockerfile | 2 +- stx/lib/stx/stx_configparser.py | 43 ++++++++++++++----- stx/lib/stx/stx_control.py | 8 ++-- .../stx-builder/configmap/localrc.sample | 6 +-- .../stx-builder/configmap/stx-sources.list | 8 ++-- stx/toCOPY/builder/finishSetup.sh | 8 ++-- stx/toCOPY/pkgbuilder/debbuilder.conf | 4 +- stx/toCOPY/pkgbuilder/debbuilder.py | 14 +++++- 10 files changed, 68 insertions(+), 35 deletions(-) diff --git a/release/README.rst b/release/README.rst index 490c5df1..1587857f 100644 --- a/release/README.rst +++ b/release/README.rst @@ -25,7 +25,7 @@ the lastest green build context at the time: release/build-context.py \ --remote starlingx \ - --context http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \ + --context https://mirror.starlingx.windriver.com/mirror/starlingx/master/centos/latest_green_build/outputs/CONTEXT.sh \ https://opendev.org/starlingx/manifest/raw/branch/master/default.xml | \ release/branch-repo.sh -b r/stx.2.0 -t v2.0.0.rc0 diff --git a/stx.conf.sample b/stx.conf.sample index 25be9430..6bb724ff 100644 --- a/stx.conf.sample +++ b/stx.conf.sample @@ -15,9 +15,9 @@ debian_version = 11.3 # We will append debian_snapshot_timestamp to each of them when creating # apt.sources lists #debian_snapshot_base = http://snapshot.debian.org/archive/debian -debian_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian +debian_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian #debian_security_snapshot_base = http://snapshot.debian.org/archive/debian-security -debian_security_snapshot_base = http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security +debian_security_snapshot_base = https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security debian_snapshot_timestamp = 20220331T000000Z [builder] @@ -34,8 +34,8 @@ mode = local [repomgr] type = aptly -cengnurl = http://mirror.starlingx.cengn.ca:80/mirror -cengnstrategy = cengn_first +stx_mirror_url = https://mirror.starlingx.windriver.com:80/mirror +stx_mirror_strategy = stx_mirror_first sourceslist = deblist = dsclist = diff --git a/stx/dockerfiles/stx-lat-tool.Dockerfile b/stx/dockerfiles/stx-lat-tool.Dockerfile index 5bd25848..9b4bada0 100644 --- a/stx/dockerfiles/stx-lat-tool.Dockerfile +++ b/stx/dockerfiles/stx-lat-tool.Dockerfile @@ -16,7 +16,7 @@ FROM debian:bullseye MAINTAINER Chen Qi -ARG LAT_BINARY_RESOURCE_PATH=http://mirror.starlingx.cengn.ca/mirror/lat-sdk/lat-sdk-20230510 +ARG LAT_BINARY_RESOURCE_PATH=https://mirror.starlingx.windriver.com/mirror/lat-sdk/lat-sdk-20230510 # Update certificates RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates && update-ca-certificates diff --git a/stx/lib/stx/stx_configparser.py b/stx/lib/stx/stx_configparser.py index b57571dd..d9176b9e 100644 --- a/stx/lib/stx/stx_configparser.py +++ b/stx/lib/stx/stx_configparser.py @@ -214,33 +214,54 @@ class STXConfigParser: # delete old key self.__delete_key('project', 'debian_security_snapshot') - # Change debian_snapshot_* to CENGN - if not self.__upgrade_id_exists('debian_snapshot_cengn'): - debian_snapshot_cengn_upgraded = False + # Change debian_snapshot_* to STX_MIRROR + if not self.__upgrade_id_exists('debian_snapshot_stx_mirror'): + debian_snapshot_stx_mirror_upgraded = False # debian_snapshot_base old_value = 'http://snapshot.debian.org/archive/debian' - new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian' + old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian' + new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian' current_value = self.__get('project', 'debian_snapshot_base') - if current_value == old_value: + if current_value == old_value or current_value == old_value2: self.__upgrade_nonempty_key('project', 'debian_snapshot_base', new_value) - debian_snapshot_cengn_upgraded = True + debian_snapshot_stx_mirror_upgraded = True # debian_security_snapshot_base old_value = 'http://snapshot.debian.org/archive/debian-security' - new_value = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security' + old_value2 = 'http://mirror.starlingx.cengn.ca/mirror/debian/debian/snapshot.debian.org/archive/debian-security' + new_value = 'https://mirror.starlingx.windriver.com/mirror/debian/debian/snapshot.debian.org/archive/debian-security' current_value = self.__get('project', 'debian_security_snapshot_base') if current_value == old_value: self.__upgrade_nonempty_key('project', 'debian_security_snapshot_base', new_value) - debian_snapshot_cengn_upgraded = True + debian_snapshot_stx_mirror_upgraded = True - if debian_snapshot_cengn_upgraded: - self.__save_upgrade_id('debian_snapshot_cengn') + if debian_snapshot_stx_mirror_upgraded: + self.__save_upgrade_id('debian_snapshot_stx_mirror') need_restart = True + # Convert cengnurl => stx_mirror_url + if self.cf.has_option('repomgr', 'cengnurl'): + oldstring = self.cf.get('repomgr', 'cengnurl') + newstring = re.sub('http://mirror.starlingx.cengn.ca', + r'https://mirror.starlingx.windriver.com', + oldstring) + self.__upgrade_nonempty_key('repomgr', 'stx_mirror_url', newstring) + # delete old key + self.__delete_key('repomgr', 'cengnurl') + need_restart = True + + # Convert cengnstrategy => stx_mirror_strategy + if self.cf.has_option('repomgr', 'cengnstrategy'): + oldstring = self.cf.get('repomgr', 'cengnstrategy') + newstring = re.sub('cengn', 'stx_mirror', oldstring) + self.__upgrade_nonempty_key('repomgr', 'stx_mirror_strategy', newstring) + # delete old key + self.__delete_key('repomgr', 'cengnstrategy') + # Save changes self.syncConfigFile() if need_restart: - logger.warning("One or more configuration keeys have been upgraded") + logger.warning("One or more configuration keys have been upgraded") logger.warning("These changes will take effect after you restart your builder containers") diff --git a/stx/lib/stx/stx_control.py b/stx/lib/stx/stx_control.py index a5b9cb26..a88d82db 100644 --- a/stx/lib/stx/stx_control.py +++ b/stx/lib/stx/stx_control.py @@ -87,8 +87,8 @@ class HandleControlTask: proxyport = self.config.get('project', 'proxyport') buildbranch = self.config.get('project', 'buildbranch') manifest = self.config.get('project', 'manifest') - cengnurl = self.config.get('repomgr', 'cengnurl') - cengnstrategy = self.config.get('repomgr', 'cengnstrategy') + stx_mirror_url = self.config.get('repomgr', 'stx_mirror_url') + stx_mirror_strategy = self.config.get('repomgr', 'stx_mirror_strategy') sourceslist = self.config.get('repomgr', 'sourceslist') deblist = self.config.get('repomgr', 'deblist') dsclist = self.config.get('repomgr', 'dsclist') @@ -171,8 +171,8 @@ stx-pkgbuilder/configmap/') line = line.replace("@BUILDBRANCH@", buildbranch) line = line.replace("@MANIFEST@", manifest) line = line.replace("@HOSTUSERNAME@", hostusername) - line = line.replace("@CENGNURL@", cengnurl) - line = line.replace("@CENGNSTRATEGY@", cengnstrategy) + line = line.replace("@STX_MIRROR_URL@", stx_mirror_url) + line = line.replace("@STX_MIRROR_STRATEGY@", stx_mirror_strategy) line = line.replace("@OSTREE_OSNAME@", ostree_osname) line = line.replace("@DEBIAN_SNAPSHOT@", debian_snapshot) line = line.replace("@DEBIAN_SECURITY_SNAPSHOT@", debian_security_snapshot) diff --git a/stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample b/stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample index 9ab94b69..e95610cc 100644 --- a/stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample +++ b/stx/stx-build-tools-chart/stx-builder/configmap/localrc.sample @@ -36,10 +36,10 @@ export MANIFEST=@MANIFEST@ export HOSTUSERNAME=@HOSTUSERNAME@ -export CENGNURL=@CENGNURL@ +export STX_MIRROR_URL=@STX_MIRROR_URL@ -# CENGN_STRATEGY value: [ cengn|cengn_first|upstream|upstream_first ] -export CENGN_STRATEGY=@CENGNSTRATEGY@ +# STX_MIRROR_STRATEGY value: [ stx_mirror|stx_mirror_first|upstream|upstream_first ] +export STX_MIRROR_STRATEGY=@STX_MIRROR_STRATEGY@ if [[ x"@fetch@" == x"true" ]];then export SOURCESLIST=/usr/local/bin/stx/@SOURCESLIST@ diff --git a/stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list b/stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list index 8bd90ef2..b365b314 100644 --- a/stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list +++ b/stx/stx-build-tools-chart/stx-builder/configmap/stx-sources.list @@ -1,4 +1,4 @@ -deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main -deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main -deb [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main -deb-src [trusted=yes] http://mirror.starlingx.cengn.ca/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main +deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main +deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ bullseye main +deb [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main +deb-src [trusted=yes] https://mirror.starlingx.windriver.com/mirror/debian/debian/ftp.ca.debian.org/debian/ buster main diff --git a/stx/toCOPY/builder/finishSetup.sh b/stx/toCOPY/builder/finishSetup.sh index 2de5ff97..7127f432 100755 --- a/stx/toCOPY/builder/finishSetup.sh +++ b/stx/toCOPY/builder/finishSetup.sh @@ -4,7 +4,7 @@ REPOMGR=aptly if [ "$REPOMGR" == "aptly" ]; then - CENGN_MIRROR="${CENGNURL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main" + STX_MIRROR="${STX_MIRROR_URL}/debian/debian/deb.debian.org/debian/${DEBIAN_DISTRIBUTION}-${DEBIAN_VERSION} ${DEBIAN_DISTRIBUTION} main" REPO_SNAPSHOT="[check-valid-until=no] ${DEBIAN_SNAPSHOT} ${DEBIAN_DISTRIBUTION} main" REPO_BIN="deb [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-binary ${DEBIAN_DISTRIBUTION} main" REPO_SRC="deb-src [trusted=yes] ${REPOMGR_DEPLOY_URL}deb-local-source ${DEBIAN_DISTRIBUTION} main" @@ -21,10 +21,10 @@ if [ "$REPOMGR" == "aptly" ]; then sed -i "1i\\deb ${REPO_SNAPSHOT}" /etc/apt/sources.list sed -i "1i\\deb-src ${REPO_SNAPSHOT}" /etc/apt/sources.list fi - ret=`grep ${CENGNURL} /etc/apt/sources.list` + ret=`grep ${STX_MIRROR_URL} /etc/apt/sources.list` if [ "x$ret" == "x" ]; then - sed -i "1i\\deb ${CENGN_MIRROR}" /etc/apt/sources.list - sed -i "1i\\deb-src ${CENGN_MIRROR}" /etc/apt/sources.list + sed -i "1i\\deb ${STX_MIRROR}" /etc/apt/sources.list + sed -i "1i\\deb-src ${STX_MIRROR}" /etc/apt/sources.list fi fi diff --git a/stx/toCOPY/pkgbuilder/debbuilder.conf b/stx/toCOPY/pkgbuilder/debbuilder.conf index 26d2f6c1..2dcfe6f3 100644 --- a/stx/toCOPY/pkgbuilder/debbuilder.conf +++ b/stx/toCOPY/pkgbuilder/debbuilder.conf @@ -36,8 +36,8 @@ $external_commands = { 'cp /etc/apt/sources.list tmp.list', 'cat tmp.list', 'sed -i "1 i\deb [trusted=yes] http://stx-stx-repomgr:80/deb-local-binary @DEBIAN_DISTRIBUTION@ main" tmp.list', - 'echo "deb @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', - 'echo "deb-src @CENGNURL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', + 'echo "deb @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', + 'echo "deb-src @STX_MIRROR_URL@/debian/debian/deb.debian.org/debian/@DEBIAN_DISTRIBUTION@-@DEBIAN_VERSION@ @DEBIAN_DISTRIBUTION@ main" >> tmp.list', 'awk \'!a[$0]++\' tmp.list > new.list && mv -f new.list /etc/apt/sources.list', 'cat /etc/apt/sources.list', 'echo "Package: *" > /etc/apt/preferences.d/local_repos', diff --git a/stx/toCOPY/pkgbuilder/debbuilder.py b/stx/toCOPY/pkgbuilder/debbuilder.py index 793f5d16..eb4f188b 100644 --- a/stx/toCOPY/pkgbuilder/debbuilder.py +++ b/stx/toCOPY/pkgbuilder/debbuilder.py @@ -23,7 +23,7 @@ STORE_ROOT = '/localdisk/pkgbuilder' BUILD_ENGINE = 'sbuild' STX_LOCALRC = '/usr/local/bin/stx/stx-localrc' SBUILD_CONF = '/etc/sbuild/sbuild.conf' -ENVIRON_VARS = ['OSTREE_OSNAME', 'CENGNURL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION'] +ENVIRON_VARS = ['OSTREE_OSNAME', 'STX_MIRROR_URL', 'DEBIAN_DISTRIBUTION', 'DEBIAN_VERSION'] REPO_BUILD = 'deb-local-build' @@ -83,6 +83,18 @@ class Debbuilder: return self.logger.debug("%s does exist", STX_LOCALRC) + # backward compatability with CENGNURL + self.logger.debug("Fixing CENGNURL references in stx-localrc") + cmd = "sed -i 's#CENGNURL#STX_MIRROR_URL#' %s" % (STX_LOCALRC) + self.logger.debug('The substitution command is %s', cmd) + try: + outs = subprocess.check_output(cmd, shell=True).decode() + except Exception as e: + self.logger.error(str(e)) + self.logger.error("Failed to substitute %s in %s", "CENGNURL", STX_LOCALRC) + return + # end of backward compatibility + for var in ENVIRON_VARS: self.logger.debug("Fetching %s from stx-localrc", var) cmd = "grep '^export *%s=.*' %s | cut -d \\= -f 2" % (var, STX_LOCALRC)