starlingx
/
root
Code Issues Proposed changes

downloader: add support for arm64 

* Check the host arch.
* Dwnload arch specific binaries for arm64.
* Support arch specific pkg list files, and the
  arch specific one (with suffix _arm64) will be used
  if it exists.
e.g.
debian_pkg_dirs
debian_pkg_dirs_arm64
debian_pkg_dirs_rt
debian_pkg_dirs_rt_arm64
debian_pkg_dirs_installer
debian_pkg_dirs_installer_arm64

base-bullseye.lst
base-bullseye_arm64.lst

os-std.lst
os-std_arm64.lst

Test Plan:
PASS: update packages list for arm64 in repos:
      - starlingx/tools
      - starlingx/kernel
      - starlingx/integ
      - starlingx/virt
      - starlingx/utilities
PASS: downloader -s -b on x86-64 host
PASS: downloader -s -b on arm64 host
PASS: build-pkgs on x86-64 host
PASS: build-image on x86-64 host
PASS: build-pkgs on arm64 host
PASS: build-image on arm64 host
PASS: Deploy AIO-SX on x86-64 target
PASS: Deploy AIO-SX on arm64 target

Story: 2010739
Task: 48013

Change-Id: I5fadf737ee75fc631d57d6bf689d54638548bffe
Signed-off-by: Jackie Huang <jackie.huang@windriver.com>
This commit is contained in:
Jackie Huang 2023-07-26 13:06:48 +08:00
parent ca94cee2e8
commit cce2ca38f8
2 changed files with 34 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
build-tools/stx/debdownloader
View File

@ -109,6 +109,7 @@ if __name__ == "__main__":
# strip epoch
major_ver = version.split(":")[-1]
pname_x86 = ''.join([name, '_', major_ver, '_amd64.deb'])
pname_arm64 = ''.join([name, '_', major_ver, '_arm64.deb'])
pname_all = ''.join([name, '_', major_ver, '_all.deb'])
if repomgr.search_pkg(REPO_BIN, name, version):
logger.info(''.join([name, '-', version,
@ -118,6 +119,10 @@ if __name__ == "__main__":
logger.info(''.join([name, '-', version,
' already downloaded, skip download']))
debs_need_upload.append(pname_x86)
elif debs_downloaded and pname_arm64 in debs_downloaded:
logger.info(''.join([name, '-', version,
' already downloaded, skip download']))
debs_need_upload.append(pname_arm64)
else:
if debs_downloaded and pname_all in debs_downloaded:
logger.info(''.join([name, '-', version,

37
build-tools/stx/downloader
View File

@ -27,8 +27,14 @@ import shutil
import signal
import sys
import utils
import platform
DEFAULT_ARCH = 'amd64'
STX_ARCH = 'amd64'
host_arch = platform.machine()
if host_arch == 'aarch64':
STX_ARCH = "arm64"
DEB_DISTRO = 'bullseye'
REPO_BIN = 'deb-local-binary'
mirror_root = os.environ.get('STX_MIRROR')
stx_src_mirror = os.path.join(mirror_root, 'sources')
@ -115,7 +121,11 @@ def get_all_stx_pkgs():
for r in files:
# Find all types of package dirs?
if r in types_pkg_dirs:
pkgs_file = os.path.join(root, r)
r_arch = r + "_" + STX_ARCH
if os.path.isfile(r_arch):
pkgs_file = os.path.join(root, r_arch)
else:
pkgs_file = os.path.join(root, r)
pkgs.update(get_pkgs_from_list(root, pkgs_file))
return pkgs
@ -153,24 +163,34 @@ def get_all_binary_list(distro=STX_DEFAULT_DISTRO, layers=None, build_types=None
logger.warning(' '.join([build_type, 'is not a valid build_type for distro', distro, 'of layer', layer]))
continue
pattern=''.join(['os-',build_type,'.lst'])
pattern = ''.join(['os-', build_type, '.lst'])
pattern_arch = ''.join(['os-', build_type, '_', STX_ARCH, '.lst'])
for root, dirs, files in os.walk(search_dir):
for f in fnmatch.filter(files, pattern):
layer_binaries[layer].append(os.path.join(root, f))
f_arch = os.path.join(root, pattern_arch)
if os.path.isfile(f_arch):
layer_binaries[layer].append(os.path.join(root, f_arch))
else:
layer_binaries[layer].append(os.path.join(root, f))
logger.info(
f"Binary lists for layer `{layer}`: "
f"{layer_binaries[layer]}"
)
search_dir = os.path.join(stx_config, 'common')
pattern='base-*.lst'
pattern = ''.join(['base-', DEB_DISTRO, '.lst'])
pattern_arch = ''.join(['base-', DEB_DISTRO, '_', STX_ARCH, '.lst'])
if "common" not in layer_binaries:
layer_binaries["common"] = []
for root, dirs, files in os.walk(search_dir):
for f in fnmatch.filter(files, pattern):
layer_binaries["common"].append(os.path.join(root, f))
f_arch = os.path.join(root, pattern_arch)
if os.path.isfile(f_arch):
layer_binaries["common"].append(os.path.join(root, f_arch))
else:
layer_binaries["common"].append(os.path.join(root, f))
logger.info(
f"Binary lists for layer `common`: "
@ -727,13 +747,14 @@ if __name__ == "__main__":
args.download_source = True
if args.download_binary:
all_binary_lists = get_all_binary_list(distro=distro, layers=layers, build_types=build_types)
binary_dl = DebDownloader(DEFAULT_ARCH, stx_bin_mirror, clean_mirror, all_binary_lists)
binary_dl = DebDownloader(STX_ARCH, stx_bin_mirror, clean_mirror, all_binary_lists)
if not binary_dl.create_binary_repo():
sys.exit(1)
if args.download_source:
source_dl = SrcDownloader(DEFAULT_ARCH, stx_src_mirror, clean_mirror)
source_dl = SrcDownloader(STX_ARCH, stx_src_mirror, clean_mirror)
dl_register_signal_handler()
if binary_dl: