From 8e82287b4aa6d8dd74ef198ff8fe64583c529152 Mon Sep 17 00:00:00 2001 From: Charles Short Date: Tue, 22 Jun 2021 09:07:59 -0400 Subject: [PATCH] Address python3 string issues with subprocess This patch updates our Popen and check_output calls to enable newlines for calls that we parse or consume the output for. Without universal_newlines=True, the output is treated as bytes under python3 which leads to issues later where we are using it as strings. See https://docs.python.org/3/glossary.html#term-universal-newlines Story: 2006796 Task: 42667 Signed-off-by: Charles Short Change-Id: I2086628948ca2dfb742de01923b56949ed30dfab (cherry picked from commit 185808f082d2b18cdfe672c768d7d2343ad2f016) --- .../sysinv/sysinv/scripts/manage-partitions | 3 ++ sysinv/sysinv/sysinv/scripts/query_pci_id | 3 +- sysinv/sysinv/sysinv/sysinv/agent/disk.py | 13 ++++--- .../sysinv/agent/lldp/drivers/lldpd/driver.py | 9 +++-- .../sysinv/agent/lldp/drivers/ovs/driver.py | 2 +- sysinv/sysinv/sysinv/sysinv/agent/lvg.py | 8 ++-- sysinv/sysinv/sysinv/sysinv/agent/manager.py | 3 +- .../sysinv/sysinv/sysinv/agent/partition.py | 3 +- sysinv/sysinv/sysinv/sysinv/agent/pci.py | 9 +++-- sysinv/sysinv/sysinv/sysinv/agent/pv.py | 6 ++- .../sysinv/sysinv/api/controllers/v1/utils.py | 3 +- sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py | 4 +- sysinv/sysinv/sysinv/sysinv/common/utils.py | 29 +++++++++++---- .../sysinv/sysinv/conductor/kube_app.py | 7 +++- .../sysinv/sysinv/sysinv/conductor/manager.py | 37 ++++++++++--------- .../versions/050_consolidated_r4.py | 3 +- .../sysinv/sysinv/fpga_agent/manager.py | 7 +++- .../sysinv/fpga_agent/reset_n3000_fpgas.py | 3 +- sysinv/sysinv/sysinv/sysinv/helm/utils.py | 18 ++++++--- .../sysinv/sysinv/sysinv/puppet/kubernetes.py | 2 +- 20 files changed, 112 insertions(+), 60 deletions(-) diff --git a/sysinv/sysinv/sysinv/scripts/manage-partitions b/sysinv/sysinv/sysinv/scripts/manage-partitions index 635a32559d..a73d49a9cd 100755 --- a/sysinv/sysinv/sysinv/scripts/manage-partitions +++ b/sysinv/sysinv/sysinv/scripts/manage-partitions @@ -72,6 +72,7 @@ def _command(arguments, **kwargs): arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, **kwargs) out, err = process.communicate() return out, err, process.returncode @@ -87,6 +88,7 @@ def _command_pipe(arguments1, arguments2=None, **kwargs): arguments1, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, **kwargs) if arguments2: @@ -94,6 +96,7 @@ def _command_pipe(arguments1, arguments2=None, **kwargs): arguments2, stdin=process.stdout, stdout=subprocess.PIPE, + universal_newlines=True, shell=False) process.stdout.close() process = process2 diff --git a/sysinv/sysinv/sysinv/scripts/query_pci_id b/sysinv/sysinv/sysinv/scripts/query_pci_id index a86d6c5bc0..6e629a3318 100755 --- a/sysinv/sysinv/sysinv/scripts/query_pci_id +++ b/sysinv/sysinv/sysinv/scripts/query_pci_id @@ -31,7 +31,8 @@ def main(): cmd = 'python /usr/share/openvswitch/scripts/dpdk-pmdinfo.py ' \ '-r /usr/sbin/ovs-vswitchd' - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, + universal_newlines=True) out, err = p.communicate() result = out.split('\n') diff --git a/sysinv/sysinv/sysinv/sysinv/agent/disk.py b/sysinv/sysinv/sysinv/sysinv/agent/disk.py index da3a90bd5b..5ed981900a 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/disk.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/disk.py @@ -102,7 +102,7 @@ class DiskOperator(object): pvs_command = '{} {}'.format('pvs | grep -w ', device_node) pvs_process = subprocess.Popen(pvs_command, stdout=subprocess.PIPE, - shell=True) + shell=True, universal_newlines=True) pvs_output = pvs_process.stdout.read() if pvs_output: @@ -119,12 +119,14 @@ class DiskOperator(object): # Get the sector size. sector_size_bytes_process = subprocess.Popen( - sector_size_bytes_cmd, stdout=subprocess.PIPE, shell=True) + sector_size_bytes_cmd, stdout=subprocess.PIPE, shell=True, + universal_newlines=True) sector_size_bytes = sector_size_bytes_process.stdout.read().rstrip() # Get the free space. avail_space_sectors_process = subprocess.Popen( - avail_space_sectors_cmd, stdout=subprocess.PIPE, shell=True) + avail_space_sectors_cmd, stdout=subprocess.PIPE, shell=True, + universal_newlines=True) avail_space_sectors_output = avail_space_sectors_process.stdout.read() avail_space_sectors = re.findall('\d+', avail_space_sectors_output)[0].rstrip() @@ -295,7 +297,7 @@ class DiskOperator(object): hdparm_process = subprocess.Popen( hdparm_command, stdout=subprocess.PIPE, - shell=True) + shell=True, universal_newlines=True) hdparm_output = hdparm_process.communicate()[0] if hdparm_process.returncode == 0: second_half = hdparm_output.split(':')[1] @@ -307,7 +309,8 @@ class DiskOperator(object): lsblk_process = subprocess.Popen( lsblk_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) lsblk_output = lsblk_process.communicate()[0] if lsblk_process.returncode == 0: model_num = lsblk_output.strip() diff --git a/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/lldpd/driver.py b/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/lldpd/driver.py index 6c9033d26b..daa750531a 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/lldpd/driver.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/lldpd/driver.py @@ -42,6 +42,7 @@ class SysinvLldpdAgentDriver(base.SysinvLldpDriverBase): json_obj = json p = subprocess.Popen(["lldpcli", "-f", "json", "show", "configuration"], + universal_newlines=True, stdout=subprocess.PIPE) data = json_obj.loads(p.communicate()[0]) @@ -257,7 +258,8 @@ class SysinvLldpdAgentDriver(base.SysinvLldpDriverBase): lldp_agents = [] p = subprocess.Popen(["lldpcli", "-f", "json", "show", "interface", - "detail"], stdout=subprocess.PIPE) + "detail"], stdout=subprocess.PIPE, + universal_newlines=True) data = json_obj.loads(p.communicate()[0]) lldp = data['lldp'][0] @@ -282,7 +284,8 @@ class SysinvLldpdAgentDriver(base.SysinvLldpDriverBase): json_obj = json lldp_neighbours = [] p = subprocess.Popen(["lldpcli", "-f", "json", "show", "neighbor", - "detail"], stdout=subprocess.PIPE) + "detail"], stdout=subprocess.PIPE, + universal_newlines=True) data = json_obj.loads(p.communicate()[0]) lldp = data['lldp'][0] @@ -303,7 +306,7 @@ class SysinvLldpdAgentDriver(base.SysinvLldpDriverBase): def lldp_update_systemname(self, systemname): p = subprocess.Popen(["lldpcli", "-f", "json", "show", "chassis"], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, universal_newlines=True) data = json.loads(p.communicate()[0]) local_chassis = data['local-chassis'][0] diff --git a/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/ovs/driver.py b/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/ovs/driver.py index e6f974dd93..9d3c78e3dd 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/ovs/driver.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/lldp/drivers/ovs/driver.py @@ -24,7 +24,7 @@ class SysinvOVSAgentDriver(lldpd_driver.SysinvLldpdAgentDriver): def run_cmd(self, cmd): p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, universal_newlines=True) p.wait() output, error = p.communicate() if p.returncode != 0: diff --git a/sysinv/sysinv/sysinv/sysinv/agent/lvg.py b/sysinv/sysinv/sysinv/sysinv/agent/lvg.py index 222949dde7..c2e6185ec3 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/lvg.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/lvg.py @@ -40,7 +40,7 @@ class LVGOperator(object): global_filer = 'devices/global_filter=["a|' + \ cinder_device + '|","r|.*|"]' command = command + ['--config', global_filer] - output = subprocess.check_output(command) # pylint: disable=not-callable + output = subprocess.check_output(command, universal_newlines=True) # pylint: disable=not-callable except Exception as e: self.handle_exception("Could not retrieve vgdisplay " "information: %s" % e) @@ -135,7 +135,8 @@ class LVGOperator(object): try: vgdisplay_process = subprocess.Popen(vgdisplay_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) vgdisplay_output = vgdisplay_process.stdout.read() except Exception as e: self.handle_exception("Could not retrieve vgdisplay " @@ -151,7 +152,8 @@ class LVGOperator(object): try: vgdisplay_process = subprocess.Popen(vgdisplay_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) vgdisplay_output = vgdisplay_output + vgdisplay_process.stdout.read() except Exception as e: self.handle_exception("Could not retrieve vgdisplay " diff --git a/sysinv/sysinv/sysinv/sysinv/agent/manager.py b/sysinv/sysinv/sysinv/sysinv/agent/manager.py index 1f180a8b3f..ae0878a66d 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/manager.py @@ -297,7 +297,8 @@ class AgentManager(service.PeriodicService): active_device = 'ttyS0' try: cmd = 'cat /sys/class/tty/console/active | grep ttyS' - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, + universal_newlines=True) output = proc.stdout.read().strip() proc.communicate()[0] if proc.returncode != 0: diff --git a/sysinv/sysinv/sysinv/sysinv/agent/partition.py b/sysinv/sysinv/sysinv/sysinv/agent/partition.py index ead12f5249..fc390a95ae 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/partition.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/partition.py @@ -49,7 +49,8 @@ class PartitionOperator(object): try: sgdisk_process = subprocess.Popen(sgdisk_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) except Exception as e: self.handle_exception("Could not retrieve partition information: " "%s" % e) diff --git a/sysinv/sysinv/sysinv/sysinv/agent/pci.py b/sysinv/sysinv/sysinv/sysinv/agent/pci.py index 2a6f370238..4e4e85e466 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/pci.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/pci.py @@ -273,7 +273,8 @@ class PCIOperator(object): def get_lspci_output_by_addr(self, pciaddr): with open(os.devnull, "w") as fnull: output = subprocess.check_output( # pylint: disable=not-callable - ['lspci', '-vmmks', pciaddr], stderr=fnull) + ['lspci', '-vmmks', pciaddr], stderr=fnull, + universal_newlines=True) return output def get_pci_sriov_vf_driver_name(self, pciaddr, sriov_vfs_pci_address): @@ -342,7 +343,8 @@ class PCIOperator(object): if vendor and device: option = "-d " + vendor + ":" + device cmd.append(option) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + universal_newlines=True) pci_devices = [] for line in p.stdout: @@ -371,7 +373,8 @@ class PCIOperator(object): def inics_get(self): - p = subprocess.Popen(["lspci", "-Dmnn"], stdout=subprocess.PIPE) + p = subprocess.Popen(["lspci", "-Dmnn"], stdout=subprocess.PIPE, + universal_newlines=True) pci_inics = [] for line in p.stdout: diff --git a/sysinv/sysinv/sysinv/sysinv/agent/pv.py b/sysinv/sysinv/sysinv/sysinv/agent/pv.py index 3dbac95aa6..5ca36ccab3 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/pv.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/pv.py @@ -65,7 +65,8 @@ class PVOperator(object): try: pvdisplay_process = subprocess.Popen(pvdisplay_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlnes=True) pvdisplay_output = pvdisplay_process.stdout.read() except Exception as e: self.handle_exception("Could not retrieve pvdisplay " @@ -82,7 +83,8 @@ class PVOperator(object): try: pvdisplay_process = subprocess.Popen(pvdisplay_process, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) pvdisplay_output = pvdisplay_output + pvdisplay_process.stdout.read() except Exception as e: self.handle_exception("Could not retrieve vgdisplay " diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py index f4b5e54046..f6a9c1f2c4 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/utils.py @@ -437,7 +437,8 @@ def is_drbd_fs_resizing(fs_name=None): def is_drbd_fs_syncing(): - output = subprocess.check_output("drbd-overview", stderr=subprocess.STDOUT) # pylint: disable=not-callable + output = subprocess.check_output( # pylint: disable=not-callable + "drbd-overview", stderr=subprocess.STDOUT, universal_newlines=True) LOG.info("is_drbd_fs_syncing returned '%s'" % output) if "sync\'ed" in output: return True diff --git a/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py index bee92e3858..1610e97422 100644 --- a/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/cert_mon/utils.py @@ -121,7 +121,9 @@ def verify_adminep_cert_chain(): cmd = ['openssl', 'verify', '-CAfile', constants.DC_ROOT_CA_CERT_PATH, '-untrusted', ca_tmpfile.name, adminep_tmpfile.name] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) stdout, stderr = proc.communicate() proc.wait() if 0 == proc.returncode: diff --git a/sysinv/sysinv/sysinv/sysinv/common/utils.py b/sysinv/sysinv/sysinv/sysinv/common/utils.py index 0188d5c7e8..02cbdfc4b9 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/common/utils.py @@ -122,6 +122,7 @@ class memoized(object): does not provide weak references; thus would prevent the instance from being garbage collected. ''' + def __init__(self, func): self.func = func self.cache = {} @@ -228,6 +229,13 @@ def execute(*cmd, **kwargs): obj.stdin.close() # pylint: disable=E1101 _returncode = obj.returncode # pylint: disable=E1101 LOG.debug(_('Result was %s') % _returncode) + if result is not None and six.PY3: + (stdout, stderr) = result + # Decode from the locale using using the surrogateescape error + # handler (decoding cannot fail) + stdout = os.fsdecode(stdout) + stderr = os.fsdecode(stderr) + result = (stdout, stderr) if not ignore_exit_code and _returncode not in check_exit_code: (stdout, stderr) = result raise exception.ProcessExecutionError( @@ -926,7 +934,7 @@ def is_virtual(): Determines if the system is virtualized or not ''' subp = subprocess.Popen(['facter', 'is_virtual'], - stdout=subprocess.PIPE) + stdout=subprocess.PIPE, universal_newlines=True) if subp.wait(): raise Exception("Failed to read virtualization status from facter") output = subp.stdout.readlines() @@ -989,7 +997,7 @@ def get_minimum_platform_reserved_memory(dbapi, ihost, numa_node): constants.DISTRIBUTED_CLOUD_CONTROLLER_MEMORY_RESERVED_MIB // numa_node_count elif host_has_function(ihost, constants.WORKER): # Engineer 1G per numa node for disk IO RSS overhead - reserved += constants.DISK_IO_RESIDENT_SET_SIZE_MIB + reserved += constants.DISK_IO_RESIDENT_SET_SIZE_MIB elif ihost['personality'] == constants.CONTROLLER: # Standard controller reserved += constants.STANDARD_CONTROLLER_MEMORY_RESERVED_MIB // numa_node_count @@ -1475,7 +1483,8 @@ def disk_is_gpt(device_node): """ parted_command = '{} {} {}'.format('parted -s', device_node, 'print') parted_process = subprocess.Popen( - parted_command, stdout=subprocess.PIPE, shell=True) + parted_command, stdout=subprocess.PIPE, shell=True, + universal_newlines=True) parted_output = parted_process.stdout.read() if re.search('Partition Table: gpt', parted_output): return True @@ -1698,7 +1707,8 @@ def get_current_fs_size(fs_name): with open(os.devnull, "w") as fnull: try: - lvdisplay_output = subprocess.check_output(args, stderr=fnull) # pylint: disable=not-callable + lvdisplay_output = subprocess.check_output( # pylint: disable=not-callable + args, stderr=fnull, universal_newlines=True) except subprocess.CalledProcessError: raise Exception("Failed to get filesystem %s size" % fs_name) @@ -1718,7 +1728,7 @@ def get_cgts_vg_free_space(): vg_free_str = subprocess.check_output( # pylint: disable=not-callable ['vgdisplay', '-C', '--noheadings', '--nosuffix', '-o', 'vg_free', '--units', 'g', 'cgts-vg'], - close_fds=True).rstrip() + close_fds=True, universal_newlines=True).rstrip() cgts_vg_free = int(float(vg_free_str)) except subprocess.CalledProcessError: LOG.error("Command vgdisplay failed") @@ -2947,7 +2957,8 @@ def get_admin_ep_cert(dc_role): def verify_ca_crt(crt): cmd = ['openssl', 'verify'] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) proc.stdin.write(crt) stdout, stderr = proc.communicate() if 0 == proc.returncode: @@ -2965,7 +2976,8 @@ def verify_intermediate_ca_cert(ca_crt, tls_crt): tmpfile.flush() cmd = ['openssl', 'verify', '-CAfile', tmpfile.name] proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) stdout, stderr = proc.communicate(input=tls_crt) proc.wait() @@ -3000,7 +3012,8 @@ def get_root_ca_cert(): def run_playbook(playbook_command): exec_env = os.environ.copy() exec_env["ANSIBLE_LOG_PATH"] = "/dev/null" - proc = subprocess.Popen(playbook_command, stdout=subprocess.PIPE, env=exec_env) + proc = subprocess.Popen(playbook_command, stdout=subprocess.PIPE, + env=exec_env, universal_newlines=True) out, _ = proc.communicate() LOG.info("ansible-playbook: %s." % out) return proc.returncode diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py index 3c281a95a3..c4b591d688 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py @@ -774,7 +774,8 @@ class AppOperator(object): if app.system_app and '/charts/helm-toolkit' in r: continue try: - output = subprocess.check_output(['helm', 'lint', r]) # pylint: disable=not-callable + output = subprocess.check_output( # pylint: disable=not-callable + ['helm', 'lint', r], universal_newlines=True) if "linted, 0 chart(s) failed" in output: LOG.info("Helm chart %s validated" % os.path.basename(r)) else: @@ -1394,7 +1395,9 @@ class AppOperator(object): p1 = subprocess.Popen(['grep', pattern, logfile], stdout=subprocess.PIPE) p2 = subprocess.Popen(['awk', print_chart], stdin=p1.stdout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) p1.stdout.close() result, err = p2.communicate() if result: diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 9321edd4ee..98bbace0ff 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -4370,7 +4370,7 @@ class ConductorManager(service.PeriodicService): # Save the physical PV associated with cinder volumes for use later if ipv['lvm_vg_name'] == constants.LVG_CINDER_VOLUMES: - cinder_pv_id = ipv['id'] + cinder_pv_id = ipv['id'] # Some of the PVs may have been updated, so get them again. ipvs = self.dbapi.ipv_get_by_ihost(ihost_uuid) @@ -6970,7 +6970,6 @@ class ConductorManager(service.PeriodicService): reinstall_required=False, reboot_required=True, filesystem_list=None): - """Update the storage configuration""" host_uuid_list = [] if update_storage: @@ -7025,7 +7024,6 @@ class ConductorManager(service.PeriodicService): def update_host_filesystem_config(self, context, host=None, filesystem_list=None): - """Update the filesystem configuration for a host""" config_uuid = self._config_update_hosts(context, @@ -9163,7 +9161,8 @@ class ConductorManager(service.PeriodicService): connected = False output = subprocess.check_output("drbd-overview", # pylint: disable=not-callable - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) output = [_f for _f in output.split('\n') if _f] for row in output: @@ -9177,7 +9176,8 @@ class ConductorManager(service.PeriodicService): def _drbd_fs_sync(self): output = subprocess.check_output("drbd-overview", # pylint: disable=not-callable - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) output = [_f for _f in output.split('\n') if _f] fs = [] @@ -9207,7 +9207,8 @@ class ConductorManager(service.PeriodicService): def _drbd_fs_updated(self, context): drbd_dict = subprocess.check_output("drbd-overview", # pylint: disable=not-callable - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) drbd_dict = [_f for _f in drbd_dict.split('\n') if _f] drbd_patch_size = 0 @@ -9279,7 +9280,8 @@ class ConductorManager(service.PeriodicService): """ cmd = "dumpe2fs -h /dev/{}".format(drbd_dev) dumpfs_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True) + stderr=subprocess.PIPE, shell=True, + universal_newlines=True) dumpfs_out, dumpfs_err = dumpfs_proc.communicate() total_size = 0 retcode = dumpfs_proc.returncode @@ -9311,7 +9313,8 @@ class ConductorManager(service.PeriodicService): """ cmd = "blockdev --getpbsz /dev/{}".format(drbd_dev) blockdev_proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=True) + stderr=subprocess.PIPE, shell=True, + universal_newlines=True) blockdev_out, blockdev_err = blockdev_proc.communicate() total_size = 0 retcode = blockdev_proc.returncode @@ -9993,7 +9996,6 @@ class ConductorManager(service.PeriodicService): context, config_uuid, config_dict): - """Apply the file on all hosts affected by supplied personalities. :param context: request context. @@ -10037,7 +10039,6 @@ class ConductorManager(service.PeriodicService): config_uuid, config_dict, force=False): - """Apply manifests on all hosts affected by the supplied personalities. If host_uuids is set in config_dict, only update hiera data and apply manifests for these hosts. @@ -11254,7 +11255,8 @@ class ConductorManager(service.PeriodicService): try: lvdisplay_process = subprocess.Popen(lvdisplay_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) except Exception as e: LOG.error("Could not retrieve lvdisplay information: %s" % e) return lvdisplay_dict @@ -11274,7 +11276,7 @@ class ConductorManager(service.PeriodicService): try: pvs_process = subprocess.Popen(pvs_command, stdout=subprocess.PIPE, - shell=True) + shell=True, universal_newlines=True) except Exception as e: LOG.error("Could not retrieve pvs information: %s" % e) return pvs_dict @@ -11299,7 +11301,8 @@ class ConductorManager(service.PeriodicService): try: get_size_process = subprocess.Popen(get_size_command, stdout=subprocess.PIPE, - shell=True) + shell=True, + universal_newlines=True) except Exception as e: LOG.error("Could not retrieve device information: %s" % e) return partition_size @@ -11693,7 +11696,8 @@ class ConductorManager(service.PeriodicService): raise exception.SysinvException(_( "ERROR: Failed to install license to redundant storage.")) - hostname = subprocess.check_output(["hostname"]).rstrip() # pylint: disable=not-callable + hostname = subprocess.check_output( # pylint: disable=not-callable + ["hostname"], universal_newlines=True).rstrip() validHostnames = [constants.CONTROLLER_0_HOSTNAME, constants.CONTROLLER_1_HOSTNAME] if hostname == 'localhost': @@ -12500,7 +12504,6 @@ class ConductorManager(service.PeriodicService): return overrides def app_has_system_plugins(self, context, app_name): - """Determine if the application has system plugin support. :returns: True if the application has system plugins and can generate @@ -12954,7 +12957,7 @@ class ConductorManager(service.PeriodicService): raise exception.ApplicationLifecycleNotificationException(app_name, str(ex)) def _do_post_action(self, context, operation, success, - remove_revert_operations=None): # noqa 0102 + remove_revert_operations=None): # noqa 0102 hook_info = self._make_backup_hook_info(operation, success) try: @@ -12963,7 +12966,7 @@ class ConductorManager(service.PeriodicService): # if we notified all apps successfully of this POST action, then we need to # remove any 'revert' actions from its associated PRE action: for op in remove_revert_operations if remove_revert_operations is not None else []: - self._backup_actions_log[op] = OrderedDict() + self._backup_actions_log[op] = OrderedDict() except Exception as ex: app_name = app.name if app is not None else None raise exception.ApplicationLifecycleNotificationException(app_name, str(ex)) diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/050_consolidated_r4.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/050_consolidated_r4.py index 4d94003d64..b8eb1525d2 100755 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/050_consolidated_r4.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/050_consolidated_r4.py @@ -68,7 +68,8 @@ def _update_ceph_mon_device_path(ceph_mon_table): process = subprocess.Popen( command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + universal_newlines=True) out, err = process.communicate() device_path = out.rstrip() diff --git a/sysinv/sysinv/sysinv/sysinv/fpga_agent/manager.py b/sysinv/sysinv/sysinv/sysinv/fpga_agent/manager.py index c42f186cce..054490713c 100644 --- a/sysinv/sysinv/sysinv/sysinv/fpga_agent/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/fpga_agent/manager.py @@ -294,7 +294,8 @@ def get_n3000_devices(): constants.N3000_DEVICE] try: - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) # pylint: disable=not-callable + output = subprocess.check_output( # pylint: disable=not-callable + cmd, stderr=subprocess.STDOUT, universal_newlines=True) except subprocess.CalledProcessError as exc: msg = ("Failed to get pci devices with vendor %s and device %s, " "return code is %d, command output: %s." % @@ -378,7 +379,9 @@ def watchdog_action(action): cmd = ["systemctl", action, "hostw"] # Issue the command to stop/start the watchdog - subprocess.check_output(cmd, stderr=subprocess.STDOUT) # pylint: disable=not-callable + subprocess.check_output( # pylint: disable=not-callable + cmd, stderr=subprocess.STDOUT, + universal_newlines=True) except subprocess.CalledProcessError as exc: msg = ("Failed to %s hostw service, " "return code is %d, command output: %s." % diff --git a/sysinv/sysinv/sysinv/sysinv/fpga_agent/reset_n3000_fpgas.py b/sysinv/sysinv/sysinv/sysinv/fpga_agent/reset_n3000_fpgas.py index 56385562d2..f0451eb8a3 100644 --- a/sysinv/sysinv/sysinv/sysinv/fpga_agent/reset_n3000_fpgas.py +++ b/sysinv/sysinv/sysinv/sysinv/fpga_agent/reset_n3000_fpgas.py @@ -35,7 +35,8 @@ def n3000_img_accessible(): cmd = 'docker image list "%s" --format "{{.Repository}}:{{.Tag}}"' % \ constants.OPAE_IMG items = subprocess.check_output(shlex.split(cmd), # pylint: disable=not-callable - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + universal_newlines=True) for line in items.splitlines(): if line == constants.OPAE_IMG: LOG.info('%s image found' % constants.OPAE_IMG) diff --git a/sysinv/sysinv/sysinv/sysinv/helm/utils.py b/sysinv/sysinv/sysinv/sysinv/helm/utils.py index f77e7da742..5dbcd1c71a 100644 --- a/sysinv/sysinv/sysinv/sysinv/helm/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/helm/utils.py @@ -61,7 +61,8 @@ def retrieve_helm_v3_releases(): helm_list = subprocess.Popen( ['helm', '--kubeconfig', kubernetes.KUBERNETES_ADMIN_CONF, 'list', '--all-namespaces', '--output', 'yaml'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(20, kill_process_and_descendants, [helm_list]) try: @@ -108,7 +109,8 @@ def retrieve_helm_v2_releases(): ['helmv2-cli', '--', 'helm', 'list', '--output', 'yaml', '--tiller-connection-timeout', '5'], - env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(20, kill_process_and_descendants, [helm_list]) try: @@ -184,7 +186,8 @@ def delete_helm_release(release): helm_cmd = subprocess.Popen( ['helmv2-cli', '--', 'helm', 'delete', release, '--tiller-connection-timeout', '5'], - env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(20, kill_process_and_descendants, [helm_cmd]) try: @@ -218,7 +221,8 @@ def _retry_on_HelmTillerFailure(ex): helm_reset = subprocess.Popen( ['helmv2-cli', '--', 'helm', 'reset', '--force'], - env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(20, kill_process_and_descendants, [helm_reset]) try: @@ -248,7 +252,8 @@ def get_openstack_pending_install_charts(): ['helmv2-cli', '--', 'helm', 'list', '--namespace', 'openstack', '--pending', '--tiller-connection-timeout', '5'], - env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(20, kill_process_and_descendants, [helm_list]) try: @@ -295,7 +300,8 @@ def install_helm_chart_with_dry_run(args=None): cmd.append(tmpdir) helm_install = subprocess.Popen( - cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) timer = threading.Timer(10, kill_process_and_descendants, [helm_install]) timer.start() diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index 4b18838d6e..2ee2e76cb1 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -274,7 +274,7 @@ class KubernetesPuppet(base.BasePuppet): cmd = ['kubeadm', KUBECONFIG, 'token', 'create', '--print-join-command', '--description', 'Bootstrap token for %s' % host.hostname] - join_cmd = subprocess.check_output(cmd) # pylint: disable=not-callable + join_cmd = subprocess.check_output(cmd, universal_newlines=True) # pylint: disable=not-callable join_cmd_additions += \ " --cri-socket /var/run/containerd/containerd.sock" join_cmd = join_cmd.strip() + join_cmd_additions