diff --git a/pm-qos-mgr/centos/build_srpm.data b/pm-qos-mgr/centos/build_srpm.data deleted file mode 100644 index 3f8ebcf32e..0000000000 --- a/pm-qos-mgr/centos/build_srpm.data +++ /dev/null @@ -1,2 +0,0 @@ -SRC_DIR="src" -TIS_PATCH_VER=1 diff --git a/pm-qos-mgr/centos/pm-qos-mgr.spec b/pm-qos-mgr/centos/pm-qos-mgr.spec deleted file mode 100644 index 99c3c489b8..0000000000 --- a/pm-qos-mgr/centos/pm-qos-mgr.spec +++ /dev/null @@ -1,61 +0,0 @@ -%define debug_package %{nil} -%global pypi_name pm_qos_mgr - -Name: pm-qos-mgr -Version: 1.0 -Release: %{tis_patch_ver}%{?_tis_dist} -Summary: PM QoS CPU wakeup latency manager for kubelet cpu-manager -License: Apache-2.0 -Group: base -URL: unknown -Source0: %{name}-%{version}.tar.gz - -BuildRequires: git -BuildRequires: python-pbr >= 2.0.0 -BuildRequires: python-setuptools -BuildRequires: python2-pip -BuildRequires: systemd-devel - -Requires: python-pbr >= 2.0.0 -Requires: python-inotify -Requires: systemd - -%description -A daemon that monitors kubelet cpu-manager static cpu assignments -and modifies PM QoS CPU wakeup latency. - -%define pythonroot %{_libdir}/python2.7/site-packages - -%prep -%autosetup -n %{name}-%{version} -S git - -# Remove bundled egg-info -rm -rf *.egg-info - -%build -export PBR_VERSION=%{version} -%{__python} setup.py build - -%install -export PBR_VERSION=%{version} -%{__python} setup.py install --root=%{buildroot} \ - --install-lib=%{pythonroot} \ - --prefix=%{_prefix} \ - --install-data=%{_datadir} \ - --single-version-externally-managed - -install -p -D -m 664 pm-qos-mgr.service %{buildroot}%{_unitdir}/pm-qos-mgr.service - -%post -systemctl enable pm-qos-mgr.service - -%clean -rm -rf $RPM_BUILD_ROOT - -%files -%defattr(-,root,root,-) -%doc LICENSE -%{_bindir}/* -%{pythonroot}/%{pypi_name}/* -%{pythonroot}/%{pypi_name}-%{version}*.egg-info -%{_unitdir}/* diff --git a/pm-qos-mgr/src/LICENSE b/pm-qos-mgr/src/LICENSE deleted file mode 100644 index d645695673..0000000000 --- a/pm-qos-mgr/src/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/pm-qos-mgr/src/pm-qos-mgr.service b/pm-qos-mgr/src/pm-qos-mgr.service deleted file mode 100644 index aabb667d53..0000000000 --- a/pm-qos-mgr/src/pm-qos-mgr.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=PM QoS CPU wakeup latency manager for kubelet cpu-manager -After=syslog.target network-online.target remote-fs.target sw-patch.service -Before=kubelet.service - -[Service] -Type=simple -ExecStart=/usr/bin/pm-qos-mgr - -[Install] -WantedBy=multi-user.target diff --git a/pm-qos-mgr/src/pm_qos_mgr/__init__.py b/pm-qos-mgr/src/pm_qos_mgr/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py b/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py deleted file mode 100644 index 02ab3265d6..0000000000 --- a/pm-qos-mgr/src/pm_qos_mgr/pm_qos_mgr.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Purpose: -# This manager watches for changes in /var/lib/kubelet/cpu_manager_state file. -# This sets appropriate PM QoS resume latency constraints for CPUs -# when kubelet cpu-manager is configured with 'static' policy. -# -# This parses the cpu_manager_state file, deduces the cpu-manager policy, -# and CPU lists of Guaranteed pods versus the remaining Default CPUs. -# Guaranteed pods with exclusive CPUs get "low" cpu wakeup latency policy. -# Default CPUs get "high" cpu wakeup latency policy. - -import itertools as it -import json -import logging -import logging.handlers -import os -import pyinotify -import subprocess -import sys - -# Global variables -statefile = '/var/lib/kubelet/cpu_manager_state' -pm_script = '/usr/bin/set-cpu-wakeup-latency.sh' - -LOG = logging.getLogger(__name__) - -def configure_logging(logger, level=logging.DEBUG): - """ Configure logger streams and format. """ - LOG.setLevel(level) - - syslog_facility = logging.handlers.SysLogHandler.LOG_DAEMON - ch = logging.handlers.SysLogHandler(address='/dev/log', - facility=syslog_facility) - ch.setLevel(level) - formatter = logging.Formatter('%(module)s[%(process)d]: %(message)s') - ch.setFormatter(formatter) - LOG.addHandler(ch) - - -def format_range_set(items): - """ Generate pretty-printed value of ranges, such as 3-6,12-17. """ - ranges = [] - for k, iterable in it.groupby(enumerate(sorted(items)), - lambda x: x[1] - x[0]): - rng = list(iterable) - if len(rng) == 1: - s = str(rng[0][1]) - else: - s = "%s-%s" % (rng[0][1], rng[-1][1]) - ranges.append(s) - return ','.join(ranges) - - -def range_to_list(csv_range=None): - """ Convert a string of comma separate ranges into an expanded list - of integers. e.g., '1-3,8-9,15' is converted to [1,2,3,8,9,15]. - """ - if not csv_range: - return [] - ranges = [(lambda L: range(L[0], L[-1] + 1))(map(int, r.split('-'))) - for r in csv_range.split(',')] - return [y for x in ranges for y in x] - - -class ProcessTransientFile(pyinotify.ProcessEvent): - def __init__(self, *args, **kw): - self.policy = None - self.cpusets = {'default': set(), - 'guaranteed': set()} - self.update_pm_qos_cpu_latency() - - def update_pm_qos_cpu_latency(self, event=None): - if self.policy is not None and self.policy != 'static': - return - if event is not None: - LOG.debug('%s, %s', event.pathname, event.maskname) - - # Read JSON formatted state file dictionary - state = {} - try: - with open(statefile, 'r') as f: - state = json.load(f) - except Exception as e: - LOG.error('Could not load: %s, error: %s.', statefile, e) - return - - self.policy = str(state['policyName']) - if self.policy != 'static': - return - - # Determine default cpuset - if 'defaultCpuSet' not in state: - LOG.error('Missing defaultCpuSet.', statefile) - return - default_cpuranges = str(state['defaultCpuSet']) - default_cpuset = set(range_to_list(csv_range=default_cpuranges)) - - # Determine guaranteed cpuset - guaranteed_cpuset = set() - if 'entries' in state: - for pod, cpus in state['entries'].items(): - cpulist = range_to_list(csv_range=cpus) - guaranteed_cpuset.update(cpulist) - guaranteed_cpuranges = format_range_set(guaranteed_cpuset) - - # Update PM QoS resume latency if the set of cpus have changed - if default_cpuset != self.cpusets['default']: - self.cpusets['default'] = default_cpuset.copy() - if default_cpuset: - pm_policy = 'high' - LOG.info('Set PM policy: %s, CPUs: %s', - pm_policy, default_cpuranges) - command = [pm_script, pm_policy, default_cpuranges] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - output, errors = proc.communicate() - if errors: - LOG.error('Problem with command: %s, error: %s', - command, errors) - - if guaranteed_cpuset != self.cpusets['guaranteed']: - self.cpusets['guaranteed'] = guaranteed_cpuset.copy() - if guaranteed_cpuset: - pm_policy = 'low' - LOG.info('Set PM policy: %s, CPUs: %s', - pm_policy, guaranteed_cpuranges) - command = [pm_script, pm_policy, guaranteed_cpuranges] - proc = subprocess.Popen(command, stdout=subprocess.PIPE) - output, errors = proc.communicate() - if errors: - LOG.error('Problem with command: %s, error: %s', - command, errors) - - def process_IN_MOVED_TO(self, event): - """ Handler for watched IN_MOVED_TO events. - - kubelet cpu-manager overwrites state-file by moving a temp file, - so this is the expected handler. - """ - self.update_pm_qos_cpu_latency(event) - - -def main(): - """ A shell command for pm-qos-daemon. """ - configure_logging(LOG, level=logging.INFO) - if os.geteuid() != 0: - LOG.error('Require sudo/root.') - sys.exit(1) - - LOG.info('Watching: %s', statefile) - watch_manager = pyinotify.WatchManager() - notifier = pyinotify.Notifier(watch_manager) - flags = pyinotify.IN_MOVED_TO - watch_manager.watch_transient_file(statefile, flags, ProcessTransientFile) - - try: - notifier.loop() - except pyinotify.NotifierError as err: - LOG.error('Problem with notifier.loop(), error: %s', err) - -if __name__ == "__main__": - main() diff --git a/pm-qos-mgr/src/setup.cfg b/pm-qos-mgr/src/setup.cfg deleted file mode 100644 index 63d503e4e1..0000000000 --- a/pm-qos-mgr/src/setup.cfg +++ /dev/null @@ -1,38 +0,0 @@ -[metadata] -license_files = LICENSE -name = pm_qos_mgr -summary = PM QoS CPU wakeup latency manager for kubelet cpu-manager -author = StarlingX -author-email = starlingx-discuss@lists.starlingx.io -home-page = http://www.starlingx.io/ -classifier = - Environment :: OpenStack - Intended Audience :: Information Technology - Intended Audience :: System Administrators - License :: OSI Approved :: Apache Software License - Operating System :: POSIX :: Linux - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.5 - -[global] -setup-hooks = - pbr.hooks.setup_hook - -[files] -packages = - pm_qos_mgr - -[entry_points] -console_scripts = - pm-qos-mgr = pm_qos_mgr.pm_qos_mgr:main - -[wheel] -universal = 1 - -[egg_info] -tag_build = -tag_date = 0 -tag_svn_revision = 0 diff --git a/pm-qos-mgr/src/setup.py b/pm-qos-mgr/src/setup.py deleted file mode 100644 index 6ca01beeaa..0000000000 --- a/pm-qos-mgr/src/setup.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -import setuptools - -setuptools.setup( - setup_requires=['pbr>=2.0.0'], - pbr=True) - diff --git a/worker-utils/.gitignore b/worker-utils/.gitignore deleted file mode 100644 index 06c1a0e807..0000000000 --- a/worker-utils/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -!.distro -.distro/centos7/rpmbuild/RPMS -.distro/centos7/rpmbuild/SRPMS -.distro/centos7/rpmbuild/BUILD -.distro/centos7/rpmbuild/BUILDROOT -.distro/centos7/rpmbuild/SOURCES/worker-utils*tar.gz diff --git a/worker-utils/centos/build_srpm.data b/worker-utils/centos/build_srpm.data deleted file mode 100644 index a356789ac5..0000000000 --- a/worker-utils/centos/build_srpm.data +++ /dev/null @@ -1,3 +0,0 @@ -SRC_DIR="worker-utils" -COPY_LIST="$SRC_DIR/LICENSE" -TIS_PATCH_VER=4 diff --git a/worker-utils/centos/worker-utils.spec b/worker-utils/centos/worker-utils.spec deleted file mode 100644 index 1607d9051b..0000000000 --- a/worker-utils/centos/worker-utils.spec +++ /dev/null @@ -1,58 +0,0 @@ -Summary: Initial worker node resource reservation and misc. utilities -Name: worker-utils -Version: 1.0 -Release: %{tis_patch_ver}%{?_tis_dist} -License: Apache-2.0 -Group: base -Packager: Wind River -URL: unknown -Source0: %{name}-%{version}.tar.gz -Source1: LICENSE - -BuildRequires: systemd-devel -Requires: systemd -Requires: python -Requires: /bin/systemctl -Requires: perl - -%description -Initial worker node resource reservation and misc. utilities - -%define local_bindir /usr/bin/ -%define local_etc_initd /etc/init.d/ -%define local_etc_platform /etc/platform/ -%define local_etc_goenabledd /etc/goenabled.d/ - -%define debug_package %{nil} - -%prep -%setup - -%build -make - -%install -make install BINDIR=%{buildroot}%{local_bindir} \ - INITDDIR=%{buildroot}%{local_etc_initd} \ - GOENABLEDDIR=%{buildroot}%{local_etc_goenabledd} \ - PLATFORMCONFDIR=%{buildroot}%{local_etc_platform} \ - SYSTEMDDIR=%{buildroot}%{_unitdir} - -%post -/bin/systemctl enable affine-platform.sh.service >/dev/null 2>&1 -/bin/systemctl enable affine-tasks.service >/dev/null 2>&1 - -%clean -rm -rf $RPM_BUILD_ROOT - -%files - -%defattr(-,root,root,-) - -%{local_bindir}/* -%{local_etc_initd}/* -%{local_etc_goenabledd}/* -%config(noreplace) %{local_etc_platform}/worker_reserved.conf - -%{_unitdir}/affine-platform.sh.service -%{_unitdir}/affine-tasks.service diff --git a/worker-utils/worker-utils/LICENSE b/worker-utils/worker-utils/LICENSE deleted file mode 100644 index d645695673..0000000000 --- a/worker-utils/worker-utils/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/worker-utils/worker-utils/Makefile b/worker-utils/worker-utils/Makefile deleted file mode 100644 index 160d6b50c2..0000000000 --- a/worker-utils/worker-utils/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# -# SPDX-License-Identifier: Apache-2.0 -# - -BINDIR ?= /usr/bin -INITDDIR ?= /etc/init.d/ -GOENABLEDDIR ?= /etc/goenabled.d/ -PLATFORMCONFDIR ?= /etc/platform -SYSTEMDDIR ?= /usr/lib/systemd/system/ - -all: - python -m compileall topology.py - -install: - install -d -m 755 $(BINDIR) - install -d -m 755 $(INITDDIR) - install -d -m 755 $(GOENABLEDDIR) - install -d -m 755 $(PLATFORMCONFDIR) - install -d -m 755 $(SYSTEMDDIR) - install -p -D -m 755 affine-platform.sh $(INITDDIR)/affine-platform.sh - install -p -D -m 755 affine-tasks.sh $(INITDDIR)/affine-tasks.sh - install -p -D -m 755 cpumap_functions.sh $(INITDDIR)/cpumap_functions.sh - install -p -D -m 755 task_affinity_functions.sh $(INITDDIR)/task_affinity_functions.sh - install -p -D -m 755 ps-sched.sh $(BINDIR)/ps-sched.sh - install -p -D -m 755 topology.py $(BINDIR)/topology.py - install -p -D -m 755 topology.pyc $(BINDIR)/topology.pyc - install -p -D -m 755 affine-interrupts.sh $(BINDIR)/affine-interrupts.sh - install -p -D -m 755 set-cpu-wakeup-latency.sh $(BINDIR)/set-cpu-wakeup-latency.sh - install -p -D -m 755 topology $(BINDIR)/topology - install -p -D -m 644 worker_reserved.conf $(PLATFORMCONFDIR)/worker_reserved.conf - install -p -D -m 755 worker-goenabled.sh $(GOENABLEDDIR)/worker-goenabled.sh - install -p -D -m 664 affine-platform.sh.service $(SYSTEMDDIR)/affine-platform.sh.service - install -p -D -m 664 affine-tasks.service $(SYSTEMDDIR)/affine-tasks.service diff --git a/worker-utils/worker-utils/affine-interrupts.sh b/worker-utils/worker-utils/affine-interrupts.sh deleted file mode 100644 index 6a7c228e24..0000000000 --- a/worker-utils/worker-utils/affine-interrupts.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -################################################################################ -# Copyright (c) 2015-2016 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -# Purpose: -# Affine the interface IRQ to specified cpulist. -# -# Usage: /usr/bin/affine-interrupts.sh interface cpulist -# -# Define minimal path -PATH=/bin:/usr/bin:/usr/local/bin - -# logger setup -WHOAMI=`basename $0` -LOG_FACILITY=user -LOG_PRIORITY=info -TMPLOG=/tmp/${WHOAMI}.log - -# LOG() - generates log and puts in temporary file -function LOG { - logger -t "${0##*/}[$$]" -p ${LOG_FACILITY}.${LOG_PRIORITY} "$@" - echo "${0##*/}[$$]" "$@" >> ${TMPLOG} -} -function INFO { - MSG="INFO" - LOG "${MSG} $@" -} -function ERROR { - MSG="ERROR" - LOG "${MSG} $@" -} - -if [ "$#" -ne 2 ]; then - ERROR "Interface name and cpulist are required" - exit 1 -fi - -interface=$1 -cpulist=$2 - -# Find PCI device matching interface, keep last matching device name -dev=$(find /sys/devices -name "${interface}" | \ - perl -ne 'print $1 if /([[:xdigit:]]{4}:[[:xdigit:]]{2}:[[:xdigit:]]{2}\.[[:xdigit:]])\/[[:alpha:]]/;') - -# Obtain all IRQs for this device -irq=$(cat /sys/bus/pci/devices/${dev}/irq 2>/dev/null) -msi_irqs=$(ls /sys/bus/pci/devices/${dev}/msi_irqs 2>/dev/null | xargs) - -INFO $LINENO "affine ${interface} (dev:${dev} irq:${irq} msi_irqs:${msi_irqs}) with cpus (${cpulist})" - -for i in $(echo "${irq} ${msi_irqs}"); do echo $i; done | \ - xargs --no-run-if-empty -i{} \ - /bin/bash -c "[[ -e /proc/irq/{} ]] && echo ${cpulist} > /proc/irq/{}/smp_affinity_list" 2>/dev/null - -exit 0 diff --git a/worker-utils/worker-utils/affine-platform.sh b/worker-utils/worker-utils/affine-platform.sh deleted file mode 100755 index 1d0d847c22..0000000000 --- a/worker-utils/worker-utils/affine-platform.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/bin/bash -################################################################################ -# Copyright (c) 2013 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -### BEGIN INIT INFO -# Provides: affine-platform -# Required-Start: -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Affine platform -### END INIT INFO - -# Define minimal path -PATH=/bin:/usr/bin:/usr/local/bin - -LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} -CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} -TASK_AFFINITY_FUNCTIONS=${TASK_AFFINITY_FUNCTIONS:-"/etc/init.d/task_affinity_functions.sh"} -source /etc/init.d/functions -[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} -[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} -[[ -e ${TASK_AFFINITY_FUNCTIONS} ]] && source ${TASK_AFFINITY_FUNCTIONS} -linkname=$(readlink -n -f $0) -scriptname=$(basename $linkname) - -# Enable debug logs -LOG_DEBUG=1 - -. /etc/platform/platform.conf - -################################################################################ -# Affine all running tasks to the CPULIST provided in the first parameter. -################################################################################ -function affine_tasks { - local CPULIST=$1 - local PIDLIST - local RET=0 - - # Get number of logical cpus - N_CPUS=$(cat /proc/cpuinfo 2>/dev/null | \ - awk '/^[pP]rocessor/ { n +=1 } END { print (n>0) ? n : 1}') - - # Calculate platform cores cpumap - PLATFORM_COREMASK=$(cpulist_to_cpumap ${CPULIST} ${N_CPUS}) - - # Set default IRQ affinity - echo ${PLATFORM_COREMASK} > /proc/irq/default_smp_affinity - - # Affine all PCI/MSI interrupts to platform cores; this overrides - # irqaffinity boot arg, since that does not handle IRQs for PCI devices - # on numa nodes that do not intersect with platform cores. - PCIDEVS=/sys/bus/pci/devices - declare -a irqs=() - irqs+=($(cat ${PCIDEVS}/*/irq 2>/dev/null | xargs)) - irqs+=($(ls ${PCIDEVS}/*/msi_irqs 2>/dev/null | grep -E '^[0-9]+$' | xargs)) - # flatten list of irqs, removing duplicates - irqs=($(echo ${irqs[@]} | tr ' ' '\n' | sort -nu)) - log_debug "Affining all PCI/MSI irqs(${irqs[@]}) with cpus (${CPULIST})" - for i in ${irqs[@]}; do - /bin/bash -c "[[ -e /proc/irq/${i} ]] && echo ${CPULIST} > /proc/irq/${i}/smp_affinity_list" 2>/dev/null - done - if [[ "$subfunction" == *"worker,lowlatency" ]]; then - # Affine work queues to platform cores - echo ${PLATFORM_COREMASK} > /sys/devices/virtual/workqueue/cpumask - echo ${PLATFORM_COREMASK} > /sys/bus/workqueue/devices/writeback/cpumask - - # On low latency compute reassign the per cpu threads rcuc, ksoftirq, - # ktimersoftd to FIFO along with the specified priority - PIDLIST=$( ps -e -p 2 |grep rcuc | awk '{ print $1; }') - for PID in ${PIDLIST[@]}; do - chrt -p -f 4 ${PID} 2>/dev/null - done - - PIDLIST=$( ps -e -p 2 |grep ksoftirq | awk '{ print $1; }') - for PID in ${PIDLIST[@]}; do - chrt -p -f 2 ${PID} 2>/dev/null - done - - PIDLIST=$( ps -e -p 2 |grep ktimersoftd | awk '{ print $1; }') - for PID in ${PIDLIST[@]}; do - chrt -p -f 3 ${PID} 2>/dev/null - done - fi - - return 0 -} - -################################################################################ -# Start Action -################################################################################ -function start { - local RET=0 - - echo -n "Starting ${scriptname}: " - - ## Check whether we are root (need root for taskset) - if [ $UID -ne 0 ]; then - log_error "require root or sudo" - RET=1 - return ${RET} - fi - - ## Define platform cpulist to be thread siblings of core 0 - PLATFORM_CPULIST=$(get_platform_cpu_list) - - # Affine all tasks to platform cpulist - affine_tasks ${PLATFORM_CPULIST} - RET=$? - if [ ${RET} -ne 0 ]; then - log_error "Failed to affine tasks ${PLATFORM_CPULIST}, rc=${RET}" - return ${RET} - fi - - print_status ${RET} - return ${RET} -} - -################################################################################ -# Stop Action - don't do anything -################################################################################ -function stop { - local RET=0 - echo -n "Stopping ${scriptname}: " - print_status ${RET} - return ${RET} -} - -################################################################################ -# Restart Action -################################################################################ -function restart { - stop - start -} - -################################################################################ -# Main Entry -# -################################################################################ -case "$1" in -start) - start - ;; -stop) - stop - ;; -restart|reload) - restart - ;; -status) - echo -n "OK" - ;; -*) - echo $"Usage: $0 {start|stop|restart|reload|status}" - exit 1 -esac - -exit $? diff --git a/worker-utils/worker-utils/affine-platform.sh.service b/worker-utils/worker-utils/affine-platform.sh.service deleted file mode 100644 index f124182bcc..0000000000 --- a/worker-utils/worker-utils/affine-platform.sh.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=Titanium Cloud Affine Platform -After=syslog.service network.service dbus.service sw-patch.service -Before=workerconfig.service - -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/etc/init.d/affine-platform.sh start -ExecStop=/etc/init.d/affine-platform.sh stop -ExecReload=/etc/init.d/affine-platform.sh restart - -[Install] -WantedBy=multi-user.target diff --git a/worker-utils/worker-utils/affine-tasks.service b/worker-utils/worker-utils/affine-tasks.service deleted file mode 100644 index f2e65655b6..0000000000 --- a/worker-utils/worker-utils/affine-tasks.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=StarlingX Affine Tasks -After=syslog.service network.service dbus.service sw-patch.service affine-platform.sh.service -Before=workerconfig.service - -[Service] -Type=simple -ExecStart=/etc/init.d/affine-tasks.sh start - -[Install] -WantedBy=multi-user.target diff --git a/worker-utils/worker-utils/affine-tasks.sh b/worker-utils/worker-utils/affine-tasks.sh deleted file mode 100644 index ddf566e53f..0000000000 --- a/worker-utils/worker-utils/affine-tasks.sh +++ /dev/null @@ -1,536 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2019 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -# -# chkconfig: 2345 80 80 -# - -### BEGIN INIT INFO -# Provides: affine-tasks -# Required-Start: -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: reaffine tasks on AIO -# Description: This script will dynamically reaffine tasks -# and k8s-infra cgroup cpuset on AIO nodes only. This accomodates -# CPU intensive phases of work. Tasks are initially allowed to float -# across all cores. Once system is at steady-state, this will ensure -# that K8S pods are constrained to platform cores and do not run on -# cores with VMs/containers. -### END INIT INFO -# -# Background: -# There is significant parallel CPU intensive activity: -# - during stx-application apply before critical openstack pods are running, -# e.g., to download docker images, and start all pods. -# - during init and pod recovery after reboot or DOR. -# -# This enables use of all cpus during CPU intensive phase, otherwise the -# startup processing time is considerably longer and we easily hit timeout. -# -# This script waits forever for sufficient platform readiness criteria -# (e.g., system critical pods are recovered, nova-compute is running, -# cinder-volume is running, openstack pods are running), and we have waited -# a short stabilization period before reaffining to the platform cpus. -# -# NOTE: child cgroup cpuset and nodeset must be a subset of the parent -# cgroup's attributes. This requires traversing the tree hierachy in -# specific order when dynamically modifying these attributes. -# -################################################################################ -# Define minimal path -PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin - -CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} -[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} - -# Bring in platform definitions -. /etc/platform/platform.conf - -# Environment for kubectl -export KUBECONFIG=/etc/kubernetes/admin.conf - -# Global parameters -CGDIR_K8S=/sys/fs/cgroup/cpuset/k8s-infra -INIT_INTERVAL_SECONDS=10 -CHECK_INTERVAL_SECONDS=30 -PRINT_INTERVAL_SECONDS=300 -STABILIZATION_SECONDS=150 - -# Define pidfile -LNAME=$(readlink -n -f $0) -NAME=$(basename $LNAME) -PIDFILE=/var/run/${NAME}.pid - -# Define number of logical cpus -LOGICAL_CPUS=$(getconf _NPROCESSORS_ONLN) - -# Define the memory nodeset and cpuset that span all online cpus and nodes -ONLINE_NODES=$(/bin/cat /sys/devices/system/node/online) -ONLINE_CPUS=$(/bin/cat /sys/devices/system/cpu/online) -ONLINE_MASK=$(cpulist_to_cpumap ${ONLINE_CPUS} ${LOGICAL_CPUS} | \ - awk '{print tolower($0)}') - -ISOL_CPUS=$(/bin/cat /sys/devices/system/cpu/isolated) -if [ ! -z "${ISOL_CPUS}" ]; then - ISOL_CPUMAP=$(cpulist_to_cpumap ${ISOL_CPUS} ${LOGICAL_CPUS}) - NONISOL_CPUMAP=$(invert_cpumap ${ISOL_CPUMAP} ${LOGICAL_CPUS}) - NONISOL_CPUS=$(cpumap_to_cpulist ${NONISOL_CPUMAP} ${LOGICAL_CPUS}) - NONISOL_MASK=$(cpulist_to_cpumap ${NONISOL_CPUS} ${LOGICAL_CPUS} | \ - awk '{print tolower($0)}') -else - ISOL_CPUMAP='0' - NONISOL_CPUS=${ONLINE_CPUS} - NONISOL_MASK=${ONLINE_MASK} -fi - -# Define platform memory nodeset and cpuset -PLATFORM_NODES=$(cat /sys/devices/system/node/online) -PLATFORM_CPUS=$(platform_expanded_cpu_list) - -# Global variables -NOT_READY_REASON="" -STABLE=0 - -# Log info message to /var/log/daemon.log -function LOG { - logger -p daemon.info -t "${NAME}($$): " "$@" -} - -# Log error message to /var/log/daemon.log -function ERROR { - logger -s -p daemon.error -t "${NAME}($$): " "$@" -} - -# Update cgroup k8s-infra cpuset and nodeset to span all non-isolated cpus. -function update_cgroup_cpuset_k8s_infra_all { - # Set all cgroup cpuset and nodeset in tree hierarchy order. - # This will always work, no matter the previous cpuset state. - find ${CGDIR_K8S} -type d | \ - while read d; do - /bin/echo ${ONLINE_NODES} > ${d}/cpuset.mems 2>/dev/null - /bin/echo ${NONISOL_CPUS} > ${d}/cpuset.cpus 2>/dev/null - done - LOG "Update ${CGDIR_K8S}," \ - "ONLINE_NODES=${ONLINE_NODES}, NONISOL_CPUS=${NONISOL_CPUS}" -} - -# Update cgroup k8s-infra to span platform cpuset and nodeset. -function update_cgroup_cpuset_k8s_infra_platform { - # Clear any existing cpuset settings. This ensures that the - # subsequent shrink to platform cpuset will always work. - update_cgroup_cpuset_k8s_infra_all - - # Set all cgroup cpuset and nodeset in depth-first order. - # NOTE: this only works if we are shrinking the cpuset. - find ${CGDIR_K8S} -depth -type d | \ - while read d; do - /bin/echo ${PLATFORM_NODES} > ${d}/cpuset.mems 2>/dev/null - /bin/echo ${PLATFORM_CPUS} > ${d}/cpuset.cpus 2>/dev/null - done - LOG "Update ${CGDIR_K8S}," \ - "PLATFORM_NODES=${PLATFORM_NODES}, PLATFORM_CPUS=${PLATFORM_CPUS}" -} - -# Check criteria for K8s platform ready on this node. -# i.e., k8s-infra is configured, kubelet is running -function is_k8s_platform_ready { - local PASS=0 - local FAIL=1 - - # Global variable - NOT_READY_REASON="" - - # Check that cgroup cpuset k8s-infra has been configured - if [ ! -e ${CGDIR_K8S} ]; then - NOT_READY_REASON="k8s-infra not configured" - return ${FAIL} - fi - - # Check that kubelet is running and stable - if systemctl is-active kubelet --quiet; then - PID=$(systemctl show kubelet.service -p MainPID | \ - awk -vFS='=' '{print $2}') - if [ ${PID} -eq 0 ]; then - NOT_READY_REASON="kubelet not running" - return ${FAIL} - fi - up=$(ps -p ${PID} -o etimes= 2>/dev/null | awk '{print $1}') - if ! { [ -n "${up}" -a ${up} -ge 30 ]; } - then - NOT_READY_REASON="kubelet not yet stable" - return ${FAIL} - fi - else - NOT_READY_REASON="kubelet not running" - return ${FAIL} - fi - - LOG "kubelet is ready" - return ${PASS} -} - -# Determine whether this node has 'static' cpu manager policy. -# NOTE: This check assumes that kubelet is already running locally. -function is_static_cpu_manager_policy { - local PASS=0 - local FAIL=1 - - state=$(cat /var/lib/kubelet/cpu_manager_state 2>/dev/null) - if [[ $state =~ \"policyName\":.?\"static\" ]]; then - return ${PASS} - else - return ${FAIL} - fi -} - -# Check criteria for K8s platform steady-state ready on this node. -# i.e., kube-system pods have recovered, kube application apply -# has completed, nova-compute is running, cinder-volume is running. -# NOTE: This function depends on kubectl commands, so is only -# usable on controllers. -function is_k8s_platform_steady_state_ready { - local PASS=0 - local FAIL=1 - local this_node=${HOSTNAME} - - # Global variable - NOT_READY_REASON="" - - # Check that kube-system pods have recovered on this node - npods=$(kubectl get pods --namespace kube-system --no-headers \ - --field-selector spec.nodeName=${this_node} 2>/dev/null | \ - awk ' -BEGIN { n=0; } -!/Completed|Running/ { n+=1 } -END { printf "%d\n", n; } -') - if [ ${npods} -gt 0 ]; then - NOT_READY_REASON="${npods} kube-system pods not recovered" - STABLE=0 - return ${FAIL} - fi - - # Wait for a few critical openstack pods to be running if this is - # an openstack-compute-node. This is not an exhaustive list. - # Make sure that all openstack pods on this node are running. - labels=$(kubectl get node ${this_node} \ - --no-headers --show-labels 2>/dev/null | awk '{print $NF}') - if [[ $labels =~ openstack-compute-node=enabled ]]; then - # nova-compute is one of the last charts to recover after reboot - PODS=( $(kubectl get pods --namespace openstack --no-headers \ - --selector application=nova,component=compute \ - --field-selector \ - spec.nodeName=${this_node},status.phase=Running 2>/dev/null) ) - if [ ${#PODS[@]} -eq 0 ]; then - NOT_READY_REASON="nova-compute pod not running" - STABLE=0 - return ${FAIL} - fi - - # cinder-volume is one of the last charts to recover after reboot - PODS=( $(kubectl get pods --namespace openstack --no-headers \ - --selector application=cinder,component=volume \ - --field-selector \ - spec.nodeName=${this_node},status.phase=Running 2>/dev/null) ) - if [ ${#PODS[@]} -eq 0 ]; then - NOT_READY_REASON="cinder-volume pod not running" - STABLE=0 - return ${FAIL} - fi - - # Check that all openstack pods on this node have recovered - npods=$(kubectl get pods --namespace openstack --no-headers \ - --field-selector spec.nodeName=${this_node} 2>/dev/null | \ - awk ' -BEGIN { n=0; } -!/Completed|Running/ { n+=1 } -END { printf "%d\n", n; } -') - if [ ${npods} -gt 0 ]; then - NOT_READY_REASON="${npods} openstack pods not recovered" - STABLE=0 - return ${FAIL} - fi - fi - - # Evaluate elapsed time since check criteria pass - if [ ${STABLE} -eq 0 ]; then - STABLE=${SECONDS} - fi - dt=$(( ${SECONDS} - ${STABLE} )) - if [ ${dt} -lt ${STABILIZATION_SECONDS} ]; then - NOT_READY_REASON="stabilization wait" - return ${FAIL} - fi - - LOG "K8S is ready" - return ${PASS} -} - -# Get number of DRBD resources started. -# Returns 0 if DRBD not ready. -function number_drbd_resources_started { - local started - - # Number of started DRBD resources - started=$(cat /proc/drbd 2>/dev/null | \ - awk '/cs:/ { n+=1; } END {printf "%d\n", n}') - echo "${started}" -} - -# Check criteria for all drbd resources started. -# i.e., see running DRBD worker threads for each configured resource. -function all_drbd_resources_started { - local PASS=0 - local FAIL=1 - local -i started=0 - local -i resources=0 - - # Global variable - NOT_READY_REASON="" - - # Number of started DRBD resources - started=$(number_drbd_resources_started) - if [ ${started} -eq 0 ]; then - NOT_READY_REASON="no drbd resources started" - return ${FAIL} - fi - - # Number of expected DRBD resources - resources=$(drbdadm sh-resources | \ - awk -vFS='[[:space:]]' 'END {print NF}') - if [ ${started} -ne ${resources} ]; then - NOT_READY_REASON="${started} of ${resources} drbd resources started" - return ${FAIL} - fi - - return ${PASS} -} - -function affine_drbd_tasks { - local CPUS=$1 - local pidlist - - LOG "Affine drbd tasks, CPUS=${CPUS}" - - # Affine drbd_r_* threads to all cores. The DRBD receiver threads are - # particularly CPU intensive. Leave the other DRBD threads alone. - pidlist=$(pgrep drbd_r_) - for pid in ${pidlist[@]}; do - taskset --pid --cpu-list ${CPUS} ${pid} > /dev/null 2>&1 - done -} - -# Return list of reaffineable pids. This includes all processes, but excludes -# kernel threads, vSwitch, and anything in K8S or qemu/kvm. -function reaffineable_pids { - local pids_excl - local pidlist - - pids_excl=$(ps -eL -o pid=,comm= | \ - awk -vORS=',' '/eal-intr-thread|kthreadd/ {print $1}' | \ - sed 's/,$/\n/') - pidlist=$(ps --ppid ${pids_excl} -p ${pids_excl} --deselect \ - -o pid=,cgroup= | \ - awk '!/k8s-infra|machine.slice/ {print $1; }') - echo "${pidlist[@]}" -} - -function affine_tasks_to_all_cores { - local pidlist - local count=0 - - LOG "Affine all tasks, CPUS: ${NONISOL_CPUS};" \ - "online=${ONLINE_CPUS} (0x${ONLINE_MASK})," \ - "isol=${ISOL_CPUS}, nonisol=${NONISOL_CPUS} (0x${NONISOL_MASK})" - - pidlist=( $(reaffineable_pids) ) - for pid in ${pidlist[@]}; do - count=$((${count} + 1)) - taskset --all-tasks --pid --cpu-list \ - ${NONISOL_CPUS} ${pid} > /dev/null 2>&1 - done - - LOG "Affined ${count} processes to all cores." -} - -function affine_tasks_to_platform_cores { - local pidlist - local count=0 - - LOG "Affine all tasks, PLATFORM_CPUS=${PLATFORM_CPUS}" - - pidlist=( $(reaffineable_pids) ) - for pid in ${pidlist[@]}; do - pid_mask=$(taskset -p $pid 2> /dev/null | awk '{print $6}') - if [ "${pid_mask}" == "${NONISOL_MASK}" ]; then - count=$((${count} + 1)) - taskset --all-tasks --pid --cpu-list \ - ${PLATFORM_CPUS} ${pid} > /dev/null 2>&1 - fi - done - - # Reaffine vSwitch tasks that span multiple cpus to platform cpus - pidlist=$(ps -eL -o pid=,comm= | awk '/eal-intr-thread/ {print $1}') - for pid in ${pidlist[@]}; do - count=$((${count} + 1)) - grep Cpus_allowed_list /proc/${pid}/task/*/status 2>/dev/null | \ - sed 's#/# #g' | awk '/,|-/ {print $4}' | \ - xargs --no-run-if-empty -i{} \ - taskset --pid --cpu-list ${PLATFORM_CPUS} {} > /dev/null 2>&1 - done - - # Reaffine drbd_r_* threads to platform cpus - affine_drbd_tasks ${PLATFORM_CPUS} - - LOG "Affined ${count} processes to platform cores." -} - -function start { - # Ensure this only runs on AIO - if ! { [[ "$nodetype" = "controller" ]] && [[ $subfunction = *worker* ]]; } - then - LOG "Not AIO, nothing to do." - return - fi - - # Abort if another instantiation is already running - if [ -e ${PIDFILE} ]; then - PID=$(cat ${PIDFILE}) - if [ -n "${PID}" -a -e /proc/${PID} ]; then - ERROR "Aborting, ${PID} already running: ${PIDFILE}." - exit 1 - else - OUT=$(rm -v -f ${PIDFILE}) - LOG "${OUT}" - fi - fi - - LOG "Starting." - - # Create pidfile to indicate the script is running - echo $$ > ${PIDFILE} - - # Affine all tasks to float on all cores - affine_tasks_to_all_cores - - # Wait for kubelet to be running - t0=${SECONDS} - until is_k8s_platform_ready; do - dt=$(( ${SECONDS} - ${t0} )) - if [ ${dt} -ge ${PRINT_INTERVAL_SECONDS} ]; then - t0=${SECONDS} - LOG "Recovery wait, elapsed ${SECONDS} seconds." \ - "Reason: ${NOT_READY_REASON}" - fi - sleep ${INIT_INTERVAL_SECONDS} - done - - # Update K8S cpuset so that pods float on all cpus - # NOTE: dynamic cpuset changes incompatible with static policy - if ! is_static_cpu_manager_policy; then - update_cgroup_cpuset_k8s_infra_all - fi - - # Wait for all DRBD resources to have started. Affine DRBD tasks - # to float on all cores as we find them. - until all_drbd_resources_started; do - started=$(number_drbd_resources_started) - if [ ${started} -gt 0 ]; then - affine_drbd_tasks ${NONISOL_CPUS} - fi - dt=$(( ${SECONDS} - ${t0} )) - if [ ${dt} -ge ${PRINT_INTERVAL_SECONDS} ]; then - t0=${SECONDS} - LOG "Recovery wait, elapsed ${SECONDS} seconds." \ - "Reason: ${NOT_READY_REASON}" - fi - sleep ${INIT_INTERVAL_SECONDS} - done - affine_drbd_tasks ${NONISOL_CPUS} - - # Wait until K8s pods have recovered and nova-compute is running - t0=${SECONDS} - until is_k8s_platform_steady_state_ready; do - dt=$(( ${SECONDS} - ${t0} )) - if [ ${dt} -ge ${PRINT_INTERVAL_SECONDS} ]; then - t0=${SECONDS} - LOG "Recovery wait, elapsed ${SECONDS} seconds." \ - "Reason: ${NOT_READY_REASON}" - fi - sleep ${CHECK_INTERVAL_SECONDS} - done - - # Update K8S cpuset to platform cores - if ! is_static_cpu_manager_policy; then - update_cgroup_cpuset_k8s_infra_platform - fi - - # Affine all floating tasks back to platform cores - affine_tasks_to_platform_cores - - # Remove pidfile after successful completion - rm -f ${PIDFILE} - - LOG "Complete." -} - -function stop { - LOG "Stopping." - - # Forcibly stop any running instantiation - if [ -e ${PIDFILE} ]; then - PID=$(cat ${PIDFILE}) - if [ -n "${PID}" -a -e /proc/${PID} ]; then - LOG "Stopping ${PID}: ${PIDFILE}." - kill -9 ${PID} - timeout 20 tail --pid=${PID} -f /dev/null - fi - OUT=$(rm -v -f ${PIDFILE}) - LOG "${OUT}" - fi -} - -function status { - : -} - -function reset { - : -} - -if [ ${UID} -ne 0 ]; then - ERROR "Need sudo/root permission." - exit 1 -fi - -case "$1" in - start) - start - ;; - stop) - stop - ;; - restart|force-reload|reload) - stop - start - ;; - status) - status - ;; - reset) - reset - ;; - *) - echo "Usage: $0 {start|stop|force-reload|restart|reload|status|reset}" - exit 1 - ;; -esac - -exit 0 diff --git a/worker-utils/worker-utils/cpumap_functions.sh b/worker-utils/worker-utils/cpumap_functions.sh deleted file mode 100644 index ab961f45f0..0000000000 --- a/worker-utils/worker-utils/cpumap_functions.sh +++ /dev/null @@ -1,403 +0,0 @@ -#!/bin/bash -################################################################################ -# Copyright (c) 2013-2018 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -### BEGIN INIT INFO -# Provides: cpumap_functions -# Required-Start: -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: cpumap_functions -### END INIT INFO - -source /etc/platform/platform.conf - -################################################################################ -# Utility function to expand a sequence of numbers (e.g., 0-7,16-23) -################################################################################ -function expand_sequence { - SEQUENCE=(${1//,/ }) - DELIMITER=${2:-","} - - LIST= - for entry in ${SEQUENCE[@]}; do - range=(${entry/-/ }) - a=${range[0]} - b=${range[1]:-${range[0]}} - - for i in $(seq $a $b); do - LIST="${LIST}${DELIMITER}${i}" - done - done - echo ${LIST:1} -} - -################################################################################ -# Append a string to comma separated list string -################################################################################ -function append_list { - local PUSH=${1-} - local LIST=${2-} - if [ -z "${LIST}" ]; then - LIST=${PUSH} - else - LIST="${LIST},${PUSH}" - fi - echo ${LIST} - return 0 -} - -################################################################################ -# Condense a sequence of numbers to a list of ranges (e.g, 7-12,15-16) -################################################################################ -function condense_sequence { - local arr=( $(printf '%s\n' "$@" | sort -n) ) - local first - local last - local cpulist="" - for ((i=0; i < ${#arr[@]}; i++)); do - num=${arr[$i]} - if [[ -z $first ]]; then - first=$num - last=$num - continue - fi - if [[ num -ne $((last + 1)) ]]; then - if [[ first -eq last ]]; then - cpulist=$(append_list ${first} ${cpulist}) - else - cpulist=$(append_list "${first}-${last}" ${cpulist}) - fi - first=$num - last=$num - else - : $((last++)) - fi - done - if [[ first -eq last ]]; then - cpulist=$(append_list ${first} ${cpulist}) - else - cpulist=$(append_list "${first}-${last}" ${cpulist}) - fi - echo "$cpulist" -} - -################################################################################ -# Converts a CPULIST (e.g., 0-7,16-23) to a CPUMAP (e.g., 0x00FF00FF). The -# CPU map is returned as a string representation of a large hexidecimal -# number but without the leading "0x" characters. -# -################################################################################ -function cpulist_to_cpumap { - local CPULIST=$1 - local NR_CPUS=$2 - local CPUMAP=0 - local CPUID=0 - if [ -z "${NR_CPUS}" ] || [ ${NR_CPUS} -eq 0 ]; then - echo 0 - return 0 - fi - for CPUID in $(expand_sequence $CPULIST " "); do - if [ "${CPUID}" -lt "${NR_CPUS}" ]; then - CPUMAP=$(echo "${CPUMAP} + (2^${CPUID})" | bc -l) - fi - done - - echo "obase=16;ibase=10;${CPUMAP}" | bc -l - return 0 -} - -################################################################################ -# Converts a CPUMAP (e.g., 0x00FF00FF) to a CPULIST (e.g., 0-7,16-23). The -# CPUMAP is expected in hexidecimal (base=10) form without the leading "0x" -# characters. -# -################################################################################ -function cpumap_to_cpulist { - local CPUMAP - CPUMAP=$(echo "obase=10;ibase=16;$1" | bc -l) - local NR_CPUS=$2 - local list=() - local cpulist="" - for((i=0; i < NR_CPUS; i++)) - do - ## Since 'bc' does not support any bitwise operators this expression: - ## if (CPUMAP & (1 << CPUID)) - ## has to be rewritten like this: - ## if (CPUMAP % (2**(CPUID+1)) > ((2**(CPUID)) - 1)) - ## - ISSET=$(echo "scale=0; (${CPUMAP} % 2^(${i}+1)) > (2^${i})-1" | bc -l) - if [ "${ISSET}" -ne 0 ]; then - list+=($i) - fi - done - cpulist=$(condense_sequence ${list[@]} ) - echo "$cpulist" - return 0 -} - -################################################################################ -# Bitwise NOT of a hexidecimal representation of a CPULIST. The value is -# returned as a hexidecimal value but without the leading "0x" characters -# -################################################################################ -function invert_cpumap { - local CPUMAP - CPUMAP=$(echo "obase=10;ibase=16;$1" | bc -l) - local NR_CPUS=$2 - local INVERSE_CPUMAP=0 - - for CPUID in $(seq 0 $((NR_CPUS - 1))); do - ## See comment in previous function - ISSET=$(echo "scale=0; (${CPUMAP} % 2^(${CPUID}+1)) > (2^${CPUID})-1" | bc -l) - if [ "${ISSET}" -eq 1 ]; then - continue - fi - - INVERSE_CPUMAP=$(echo "${INVERSE_CPUMAP} + (2^${CPUID})" | bc -l) - done - - echo "obase=16;ibase=10;${INVERSE_CPUMAP}" | bc -l - return 0 -} - -################################################################################ -# Builds the complement representation of a CPULIST -# -################################################################################ -function invert_cpulist { - local CPULIST=$1 - local NR_CPUS=$2 - local CPUMAP - CPUMAP=$(cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) - cpumap_to_cpulist $(invert_cpumap ${CPUMAP} ${NR_CPUS}) ${NR_CPUS} - return 0 -} - -################################################################################ -# in_list() - check whether item is contained in list -# param: item -# param: list (i.e. 0-3,8-11) -# returns: 0 - item is contained in list; -# 1 - item is not contained in list -# -################################################################################ -function in_list { - local item="${1-}" - local list="${2-}" - - # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} - local exp_list - exp_list=$(echo ${list} | \ - sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') - - local e - for e in $(eval echo ${exp_list}); do - [[ "$e" == "$item" ]] && return 0 - done - return 1 -} - -################################################################################ -# any_in_list() - check if any item of sublist is contained in list -# param: sublist -# param: list -# returns: 0 - an item of sublist is contained in list; -# 1 - no sublist items contained in list -# -################################################################################ -function any_in_list { - local sublist="$1" - local list="$2" - local e - local exp_list - - # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} - exp_list=$(echo ${list} | \ - sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') - declare -A a_list - for e in $(eval echo ${exp_list}); do - a_list[$e]=1 - done - - # expand list format 0-3,8-11 to a full sequence {0..3} {8..11} - exp_list=$(echo ${sublist} | \ - sed -e 's#,# #g' -e 's#\([0-9]*\)-\([0-9]*\)#{\1\.\.\2}#g') - declare -A a_sublist - for e in $(eval echo ${exp_list}); do - a_sublist[$e]=1 - done - - # Check if any element of sublist is in list - for e in "${!a_sublist[@]}"; do - if [[ "${a_list[$e]}" == 1 ]]; then - return 0 # matches - fi - done - return 1 # no match -} - -################################################################################ -# Return list of CPUs reserved for platform -################################################################################ -function get_platform_cpu_list { - ## Define platform cpulist based on engineering a number of cores and - ## whether this is a combo or not, and include SMT siblings. - if [[ $subfunction = *worker* ]]; then - RESERVE_CONF="/etc/platform/worker_reserved.conf" - [[ -e ${RESERVE_CONF} ]] && source ${RESERVE_CONF} - if [ -n "$PLATFORM_CPU_LIST" ];then - echo "$PLATFORM_CPU_LIST" - return 0 - fi - fi - - local PLATFORM_SOCKET=0 - local PLATFORM_START=0 - local PLATFORM_CORES=1 - if [ "$nodetype" = "controller" ]; then - PLATFORM_CORES=$(($PLATFORM_CORES+1)) - fi - local PLATFORM_CPULIST - PLATFORM_CPULIST=$(topology_to_cpulist ${PLATFORM_SOCKET} ${PLATFORM_START} ${PLATFORM_CORES}) - echo ${PLATFORM_CPULIST} -} - -################################################################################ -# Return number of CPUs reserved for platform -################################################################################ -function get_platform_cpus { - local PLATFORM_CPULIST - PLATFORM_CPULIST=($(platform_expanded_cpu_list | \ - perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \ - sed 's/,/ /g')) - echo ${#PLATFORM_CPULIST[@]} -} - -################################################################################ -# Return list of CPUs reserved for vswitch -################################################################################ -function get_vswitch_cpu_list { - ## Define default avp cpulist based on engineered number of platform cores, - ## engineered avp cores, and include SMT siblings. - if [[ $subfunction = *worker* ]]; then - VSWITCH_CONF="/etc/vswitch/vswitch.conf" - [[ -e ${VSWITCH_CONF} ]] && source ${VSWITCH_CONF} - if [ -n "$VSWITCH_CPU_LIST" ];then - echo "$VSWITCH_CPU_LIST" - return 0 - fi - fi - - local N_CORES_IN_PKG - N_CORES_IN_PKG=$(cat /proc/cpuinfo 2>/dev/null | \ - awk '/^cpu cores/ {n = $4} END { print (n>0) ? n : 1 }') - # engineer platform cores - local PLATFORM_CORES=1 - if [ "$nodetype" = "controller" ]; then - PLATFORM_CORES=$(($PLATFORM_CORES+1)) - fi - - # engineer AVP cores - local AVP_SOCKET=0 - local AVP_START=${PLATFORM_CORES} - local AVP_CORES=1 - if [ ${N_CORES_IN_PKG} -gt 4 ]; then - AVP_CORES=$(($AVP_CORES+1)) - fi - local AVP_CPULIST - AVP_CPULIST=$(topology_to_cpulist ${AVP_SOCKET} ${AVP_START} ${AVP_CORES}) - echo ${AVP_CPULIST} -} - -################################################################################ -# vswitch_expanded_cpu_list() - compute the vswitch cpu list, including it's siblings -################################################################################ -function vswitch_expanded_cpu_list { - list=$(get_vswitch_cpu_list) - - # Expand vswitch cpulist - vswitch_cpulist=$(expand_sequence ${list} " ") - - cpulist="" - for e in $vswitch_cpulist; do - # claim hyperthread siblings if SMT enabled - SIBLINGS_CPULIST=$(cat /sys/devices/system/cpu/cpu${e}/topology/thread_siblings_list 2>/dev/null) - siblings_cpulist=$(expand_sequence ${SIBLINGS_CPULIST} " ") - for s in $siblings_cpulist; do - in_list ${s} ${cpulist} - if [ $? -eq 1 ]; then - cpulist=$(append_list ${s} ${cpulist}) - fi - done - done - - echo "$cpulist" - return 0 -} - -################################################################################ -# platform_expanded_cpu_list() - compute the platform cpu list, including it's siblings -################################################################################ -function platform_expanded_cpu_list { - list=$(get_platform_cpu_list) - - # Expand platform cpulist - platform_cpulist=$(expand_sequence ${list} " ") - - cpulist="" - for e in $platform_cpulist; do - # claim hyperthread siblings if SMT enabled - SIBLINGS_CPULIST=$(cat /sys/devices/system/cpu/cpu${e}/topology/thread_siblings_list 2>/dev/null) - siblings_cpulist=$(expand_sequence ${SIBLINGS_CPULIST} " ") - for s in $siblings_cpulist; do - in_list ${s} ${cpulist} - if [ $? -eq 1 ]; then - cpulist=$(append_list ${s} ${cpulist}) - fi - done - done - - echo "$cpulist" - return 0 -} - -################################################################################ -# Return list of CPUs based on cpu topology. Select the socket, starting core -# within the socket, select number of cores, and SMT siblings. -################################################################################ -function topology_to_cpulist { - local SOCKET=$1 - local CORE_START=$2 - local NUM_CORES=$3 - local CPULIST - CPULIST=$(cat /proc/cpuinfo 2>/dev/null | perl -sne \ -'BEGIN { %T = {}; %H = {}; $L = $P = $C = $S = 0; } -{ - if (/processor\s+:\s+(\d+)/) { $L = $1; } - if (/physical id\s+:\s+(\d+)/) { $P = $1; } - if (/core id\s+:\s+(\d+)/) { - $C = $1; - $T{$P}{$C}++; - $S = $T{$P}{$C}; - $H{$P}{$C}{$S} = $L; - } -} -END { - @cores = sort { $a <=> $b } keys $T{$socket}; - @sel_cores = splice @cores, $core_start, $num_cores; - @lcpus = (); - for $C (@sel_cores) { - for $S (sort {$a <=> $b } keys %{ $H{$socket}{$C} }) { - push @lcpus, $H{$socket}{$C}{$S}; - } - } - printf "%s\n", join(",", @lcpus); -}' -- -socket=${SOCKET} -core_start=${CORE_START} -num_cores=${NUM_CORES}) - echo ${CPULIST} -} diff --git a/worker-utils/worker-utils/cpumap_functions_unit_test.sh b/worker-utils/worker-utils/cpumap_functions_unit_test.sh deleted file mode 100644 index 941352df1c..0000000000 --- a/worker-utils/worker-utils/cpumap_functions_unit_test.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash - -# -# Copyright (c) 2015-2016 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -source /etc/init.d/cpumap_functions.sh - -export NR_CPUS_LIST=("4" "8" "16" "32" "64" "128") -if [ ! -z ${1} ]; then - NR_CPUS_LIST=(${1//,/ }) -fi - -function test_cpumap_to_cpulist { - local NR_CPUS=$1 - declare -A CPULISTS - - if [ ${NR_CPUS} -ge 4 ]; then - CPULISTS["0"]="" - CPULISTS["1"]="0" - CPULISTS["2"]="1" - CPULISTS["3"]="0-1" - CPULISTS["5"]="0,2" - CPULISTS["7"]="0-2" - CPULISTS["F"]="0-3" - CPULISTS["9"]="0,3" - fi - if [ ${NR_CPUS} -ge 8 ]; then - CPULISTS["00"]="" - CPULISTS["11"]="0,4" - CPULISTS["FF"]="0-7" - CPULISTS["81"]="0,7" - fi - if [ ${NR_CPUS} -ge 16 ]; then - CPULISTS["0000"]="" - CPULISTS["1111"]="0,4,8,12" - CPULISTS["FFF"]="0-11" - CPULISTS["F0F"]="0-3,8-11" - CPULISTS["F0F0"]="4-7,12-15" - CPULISTS["FFFF"]="0-15" - CPULISTS["FFFE"]="1-15" - CPULISTS["8001"]="0,15" - fi - if [ ${NR_CPUS} -ge 32 ]; then - CPULISTS["00000000"]="" - CPULISTS["11111111"]="0,4,8,12,16,20,24,28" - CPULISTS["0F0F0F0F"]="0-3,8-11,16-19,24-27" - CPULISTS["F0F0F0F0"]="4-7,12-15,20-23,28-31" - CPULISTS["FFFFFFFF"]="0-31" - CPULISTS["FFFFFFFE"]="1-31" - CPULISTS["80000001"]="0,31" - fi - if [ ${NR_CPUS} -ge 64 ]; then - CPULISTS["0000000000000000"]="" - CPULISTS["1111111111111111"]="0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60" - CPULISTS["0F0F0F0F0F0F0F0F"]="0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59" - CPULISTS["F0F0F0F0F0F0F0F0"]="4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63" - CPULISTS["FFFFFFFFFFFFFFFF"]="0-63" - CPULISTS["FFFFFFFFFFFFFFFE"]="1-63" - CPULISTS["8000000000000001"]="0,63" - fi - if [ ${NR_CPUS} -ge 128 ]; then - CPULISTS["00000000000000000000000000000000"]="" - CPULISTS["11111111111111111111111111111111"]="0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124" - CPULISTS["0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F"]="0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59,64-67,72-75,80-83,88-91,96-99,104-107,112-115,120-123" - CPULISTS["F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0"]="4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63,68-71,76-79,84-87,92-95,100-103,108-111,116-119,124-127" - CPULISTS["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"]="0-127" - CPULISTS["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"]="1-127" - CPULISTS["80000000000000000000000000000001"]="0,127" - fi - - for CPUMAP in ${!CPULISTS[@]}; do - EXPECTED=${CPULISTS[${CPUMAP}]} - CPULIST=$(cpumap_to_cpulist ${CPUMAP} ${NR_CPUS}) - if [ "${CPULIST}" != "${EXPECTED}" ]; then - printf "\n" - echo "error: (cpumap_to_list ${CPUMAP} ${NR_CPUS}) returned \"${CPULIST}\" instead of \"${EXPECTED}\"" - fi - printf "." - done - - printf "\n" -} - -function test_cpulist_to_cpumap { - local NR_CPUS=$1 - declare -A CPUMAPS - - if [ ${NR_CPUS} -ge 4 ]; then - CPUMAPS[" "]="0" - CPUMAPS["0"]="1" - CPUMAPS["1"]="2" - CPUMAPS["0-1"]="3" - CPUMAPS["0,2"]="5" - CPUMAPS["0-2"]="7" - CPUMAPS["0-3"]="F" - CPUMAPS["0,3"]="9" - fi - if [ ${NR_CPUS} -ge 8 ]; then - CPUMAPS["0,4"]="11" - CPUMAPS["0-7"]="FF" - CPUMAPS["0,7"]="81" - fi - if [ ${NR_CPUS} -ge 16 ]; then - CPUMAPS["0,4,8,12"]="1111" - CPUMAPS["0-11"]="FFF" - CPUMAPS["0-3,8-11"]="F0F" - CPUMAPS["4-7,12-15"]="F0F0" - CPUMAPS["0-15"]="FFFF" - CPUMAPS["1-15"]="FFFE" - CPUMAPS["0,15"]="8001" - fi - if [ ${NR_CPUS} -ge 32 ]; then - CPUMAPS["0,4,8,12,16,20,24,28"]="11111111" - CPUMAPS["0-3,8-11,16-19,24-27"]="F0F0F0F" - CPUMAPS["4-7,12-15,20-23,28-31"]="F0F0F0F0" - CPUMAPS["0-31"]="FFFFFFFF" - CPUMAPS["1-31"]="FFFFFFFE" - CPUMAPS["0,31"]="80000001" - fi - if [ ${NR_CPUS} -ge 64 ]; then - CPUMAPS["0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60"]="1111111111111111" - CPUMAPS["0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59"]="F0F0F0F0F0F0F0F" - CPUMAPS["4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63"]="F0F0F0F0F0F0F0F0" - CPUMAPS["0-63"]="FFFFFFFFFFFFFFFF" - CPUMAPS["1-63"]="FFFFFFFFFFFFFFFE" - CPUMAPS["0,63"]="8000000000000001" - fi - if [ ${NR_CPUS} -ge 128 ]; then - CPUMAPS["0,4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124"]="11111111111111111111111111111111" - CPUMAPS["0-3,8-11,16-19,24-27,32-35,40-43,48-51,56-59,64-67,72-75,80-83,88-91,96-99,104-107,112-115,120-123"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F" - CPUMAPS["4-7,12-15,20-23,28-31,36-39,44-47,52-55,60-63,68-71,76-79,84-87,92-95,100-103,108-111,116-119,124-127"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0" - CPUMAPS["0-127"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" - CPUMAPS["1-127"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" - CPUMAPS["0,127"]="80000000000000000000000000000001" - fi - - for CPULIST in ${!CPUMAPS[@]}; do - EXPECTED=${CPUMAPS[${CPULIST}]} - CPUMAP=$(cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) - if [ "${CPUMAP}" != "${EXPECTED}" ]; then - printf "\n" - echo "error: (cpulist_to_cpumap ${CPULIST} ${NR_CPUS}) returned \"${CPUMAP}\" instead of \"${EXPECTED}\"" - fi - printf "." - done - - printf "\n" -} - -function test_invert_cpumap { - local NR_CPUS=$1 - declare -A INVERSES - - if [ $((${NR_CPUS} % 4)) -ne 0 ]; then - echo "test_invert_cpumap skipping NR_CPUS=${NR_CPUS}; not a multiple of 4" - return 0 - fi - - if [ ${NR_CPUS} -ge 4 ]; then - INVERSES["0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" - INVERSES["1"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" - INVERSES["2"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD" - INVERSES["3"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFC" - INVERSES["5"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA" - INVERSES["7"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF8" - INVERSES["F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0" - INVERSES["9"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF6" - fi - if [ ${NR_CPUS} -ge 8 ]; then - INVERSES["11"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEE" - INVERSES["FF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00" - INVERSES["F0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F" - INVERSES["81"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7E" - fi - if [ ${NR_CPUS} -ge 16 ]; then - INVERSES["1111"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFEEEE" - INVERSES["FFF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF000" - INVERSES["F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0" - INVERSES["F0F0"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0F" - INVERSES["0F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFFF0F0" - INVERSES["FFFF"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0000" - INVERSES["FFFE"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF0001" - INVERSES["8001"]="FFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFE" - fi - if [ ${NR_CPUS} -ge 32 ]; then - INVERSES["11111111"]="FFFFFFFFFFFFFFFFFFFFFFFFEEEEEEEE" - INVERSES["0F0F0F0F"]="FFFFFFFFFFFFFFFFFFFFFFFFF0F0F0F0" - INVERSES["F0F0F0F0"]="FFFFFFFFFFFFFFFFFFFFFFFF0F0F0F0F" - INVERSES["FFFFFFFF"]="FFFFFFFFFFFFFFFFFFFFFFFF00000000" - INVERSES["FFFFFFFE"]="FFFFFFFFFFFFFFFFFFFFFFFF00000001" - INVERSES["80000001"]="FFFFFFFFFFFFFFFFFFFFFFFF7FFFFFFE" - fi - if [ ${NR_CPUS} -ge 64 ]; then - INVERSES["1111111111111111"]="FFFFFFFFFFFFFFFFEEEEEEEEEEEEEEEE" - INVERSES["0F0F0F0F0F0F0F0F"]="FFFFFFFFFFFFFFFFF0F0F0F0F0F0F0F0" - INVERSES["F0F0F0F0F0F0F0F0"]="FFFFFFFFFFFFFFFF0F0F0F0F0F0F0F0F" - INVERSES["FFFFFFFFFFFFFFFF"]="FFFFFFFFFFFFFFFF0000000000000000" - INVERSES["FFFFFFFFFFFFFFFE"]="FFFFFFFFFFFFFFFF0000000000000001" - INVERSES["8000000000000001"]="FFFFFFFFFFFFFFFF7FFFFFFFFFFFFFFE" - fi - if [ ${NR_CPUS} -ge 128 ]; then - INVERSES["11111111111111111111111111111111"]="EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE" - INVERSES["0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F"]="F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0" - INVERSES["F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0"]="0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F" - INVERSES["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"]="00000000000000000000000000000000" - INVERSES["FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"]="00000000000000000000000000000001" - INVERSES["80000000000000000000000000000001"]="7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE" - fi - - for CPUMAP in ${!INVERSES[@]}; do - EXPECTED=${INVERSES[${CPUMAP}]} - if [ ${NR_CPUS} -lt 128 ]; then - EXPECTED=$(echo ${EXPECTED} | cut --complement -c1-$((32-((${NR_CPUS}+3)/4)))) - fi - EXPECTED=$(echo ${EXPECTED} | sed -e "s/^0*//") - if [ -z ${EXPECTED} ]; then - EXPECTED="0" - fi - INVERSE=$(invert_cpumap ${CPUMAP} ${NR_CPUS}) - if [ "${INVERSE}" != "${EXPECTED}" ]; then - printf "\n" - echo "error: (invert_cpumap ${CPUMAP} ${NR_CPUS}) returned \"${INVERSE}\" instead of \"${EXPECTED}\"" - fi - printf "." - done - - printf "\n" -} - -for NR_CPUS in ${NR_CPUS_LIST[@]}; do - echo "NR_CPUS=${NR_CPUS}" - test_cpumap_to_cpulist ${NR_CPUS} - test_cpulist_to_cpumap ${NR_CPUS} - test_invert_cpumap ${NR_CPUS} - echo "" -done - -exit 0 diff --git a/worker-utils/worker-utils/ps-sched.sh b/worker-utils/worker-utils/ps-sched.sh deleted file mode 100755 index 0ce78b6938..0000000000 --- a/worker-utils/worker-utils/ps-sched.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -################################################################################ -# Copyright (c) 2013 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -# ps-sched.sh -- gives detailed task listing with scheduling attributes -# -- this is cpu and scheduling intensive version (shell/taskset based) -# (note: does not print fields 'group' or 'timeslice') - -printf "%6s %6s %6s %1c %2s %4s %6s %4s %-24s %2s %-16s %s\n" "PID" "TID" "PPID" "S" "PO" "NICE" "RTPRIO" "PR" "AFFINITY" "P" "COMM" "COMMAND" -ps -eL -o pid=,lwp=,ppid=,state=,class=,nice=,rtprio=,priority=,psr=,comm=,command= | \ - while read pid tid ppid state policy nice rtprio priority psr comm command; do - bitmask=$(taskset -p $tid 2>/dev/null) - aff=${bitmask##*: } - if [ -z "${aff}" ]; then - aff="0x0" - else - aff="0x${aff}" - fi - printf "%6d %6d %6d %1c %2s %4s %6s %4d %-24s %2d %-16s %s\n" $pid $tid $ppid $state $policy $nice $rtprio $priority $aff $psr $comm "$command" - done - -exit 0 diff --git a/worker-utils/worker-utils/set-cpu-wakeup-latency.sh b/worker-utils/worker-utils/set-cpu-wakeup-latency.sh deleted file mode 100644 index 0efa13ec06..0000000000 --- a/worker-utils/worker-utils/set-cpu-wakeup-latency.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# -# Copyright (c) 2017 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Purpose: set PM QoS resume latency constraints for CPUs. -# Usage: /usr/bin/set-cpu-wakeup-latency.sh policy cpulist -# policy may be either "low" or "high" to set appropriate latency. -# "low" means HALT (C1) is the deepest C-state we allow the CPU to enter. -# "high" means we allow the CPU to sleep as deeply as possible. -# cpulist is for specifying a numerical list of processors. -# It may contain multiple items, separated by comma, and ranges. -# For example, 0,5,7,9-11. - -# Define minimal path -PATH=/bin:/usr/bin:/usr/local/bin - -LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} -CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} -[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} -[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} - -if [ $UID -ne 0 ]; then - log_error "$0 requires root or sudo privileges" - exit 1 -fi - -if [ "$#" -ne 2 ]; then - log_error "$0 requires policy and cpulist parameters" - exit 1 -fi - -POLICY=$1 -CPU_LIST=$2 -NUMBER_OF_CPUS=$(getconf _NPROCESSORS_CONF 2>/dev/null) -STATUS=1 - -for CPU_NUM in $(expand_sequence "$CPU_LIST" " "); do - # Check that we are not setting PM QoS policy for non-existing CPU - if [ "$CPU_NUM" -lt "0" ] || [ "$CPU_NUM" -ge "$NUMBER_OF_CPUS" ]; then - log_error "CPU number ${CPU_NUM} is invalid, available CPUs are 0-${NUMBER_OF_CPUS-1}" - exit 1 - fi - - # Obtain CPU wakeup latencies for all C-states available starting from operating state to deepest sleep - declare -a LIMITS=() - LIMITS+=($(cat /sys/devices/system/cpu/cpu${CPU_NUM}/cpuidle/state*/latency 2>/dev/null | xargs | sort)) - if [ ${#LIMITS[@]} -eq 0 ]; then - log_debug "Failed to get PM QoS latency limits for CPU ${CPU_NUM}" - fi - - # Select appropriate CPU wakeup latency based on "low" or "high" policy - case "${POLICY}" in - "low") - # Get first sleep state for "low" policy - if [ ${#LIMITS[@]} -eq 0 ]; then - LATENCY=1 - else - LATENCY=${LIMITS[1]} - fi - ;; - "high") - # Get deepest sleep state for "high" policy - if [ ${#LIMITS[@]} -eq 0 ]; then - LATENCY=1000 - else - LATENCY=${LIMITS[${#LIMITS[@]}-1]} - fi - ;; - *) - log_error "Policy is invalid, can be either low or high" - exit 1 - esac - - # Set the latency for paricular CPU - echo ${LATENCY} > /sys/devices/system/cpu/cpu${CPU_NUM}/power/pm_qos_resume_latency_us 2>/dev/null - RET_VAL=$? - if [ ${RET_VAL} -ne 0 ]; then - log_error "Failed to set PM QoS latency for CPU ${CPU_NUM}, rc=${RET_VAL}" - continue - else - log_debug "Succesfully set PM QoS latency for CPU ${CPU_NUM}, rc=${RET_VAL}" - STATUS=0 - fi -done - -exit ${STATUS} diff --git a/worker-utils/worker-utils/task_affinity_functions.sh b/worker-utils/worker-utils/task_affinity_functions.sh deleted file mode 100755 index 4b184310d5..0000000000 --- a/worker-utils/worker-utils/task_affinity_functions.sh +++ /dev/null @@ -1,233 +0,0 @@ -#!/bin/bash -################################################################################ -# Copyright (c) 2017 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -### BEGIN INIT INFO -# Provides: task_affinity_functions -# Required-Start: -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: task_affinity_functions -### END INIT INFO - -# Define minimal path -PATH=/bin:/usr/bin:/usr/local/bin - -. /etc/platform/platform.conf -LOG_FUNCTIONS=${LOG_FUNCTIONS:-"/etc/init.d/log_functions.sh"} -CPUMAP_FUNCTIONS=${CPUMAP_FUNCTIONS:-"/etc/init.d/cpumap_functions.sh"} -[[ -e ${LOG_FUNCTIONS} ]] && source ${LOG_FUNCTIONS} -[[ -e ${CPUMAP_FUNCTIONS} ]] && source ${CPUMAP_FUNCTIONS} - -# Enable debug logs and tag them -LOG_DEBUG=1 -TAG="TASKAFFINITY:" - -TASK_AFFINING_INCOMPLETE="/etc/platform/.task_affining_incomplete" -N_CPUS=$(getconf _NPROCESSORS_ONLN) -FULLSET_CPUS="0-"$((N_CPUS-1)) -FULLSET_MASK=$(cpulist_to_cpumap ${FULLSET_CPUS} ${N_CPUS}) -PLATFORM_CPUS=$(platform_expanded_cpu_list) -PLATFORM_CPULIST=$(platform_expanded_cpu_list| \ - perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \ - sed 's/,/ /g') -VSWITCH_CPULIST=$(get_vswitch_cpu_list| \ - perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg'| \ - sed 's/,/ /g') -if [[ $vswitch_type =~ none ]]; then - VSWITCH_CPULIST="" -fi - -IDLE_MARK=95.0 -KERNEL=$(uname -a) - -################################################################################ -# Check if a given core is one of the platform cores -################################################################################ -function is_platform_core { - local core=$1 - for CPU in ${PLATFORM_CPULIST}; do - if [ $core -eq $CPU ]; then - return 1 - fi - done - return 0 -} - -################################################################################ -# Check if a given core is one of the vswitch cores -################################################################################ -function is_vswitch_core { - local core=$1 - for CPU in ${VSWITCH_CPULIST}; do - if [ $core -eq $CPU ]; then - return 1 - fi - done - return 0 -} - -# Return list of reaffineable pids. This includes all processes, but excludes -# kernel threads, vSwitch, and anything in K8S or qemu/kvm. -function reaffineable_pids { - local pids_excl - local pidlist - - pids_excl=$(ps -eL -o pid=,comm= | \ - awk -vORS=',' '/eal-intr-thread|kthreadd/ {print $1}' | \ - sed 's/,$/\n/') - pidlist=$(ps --ppid ${pids_excl} -p ${pids_excl} --deselect \ - -o pid=,cgroup= | \ - awk '!/k8s-infra|machine.slice/ {print $1; }') - echo "${pidlist[@]}" -} - -################################################################################ -# The following function can be called by any platform service that needs to -# temporarily make use of idle VM cores to run a short-duration, service -# critical and cpu intensive operation in AIO. For instance, sm can levearage -# the idle cores to speed up swact activity. -# -# At the end of the operation, regarless of the result, the service must be -# calling function affine_tasks_to_platform_cores to re-affine platform tasks -# back to their assigned core(s). -# -# Kernel, vswitch and VM related tasks are untouched. -################################################################################ -function affine_tasks_to_idle_cores { - local cpulist - local cpuocc_list - local vswitch_pid - local pidlist - local idle_cpulist - local platform_cpus - local rc=0 - local cpu=0 - - if [ -f ${TASK_AFFINING_INCOMPLETE} ]; then - read cpulist < ${TASK_AFFINING_INCOMPLETE} - log_debug "${TAG} Tasks have already been affined to CPU ($cpulist)." - return 0 - fi - - if [[ "${KERNEL}" == *" RT "* ]]; then - return 0 - fi - - # Compile a list of cpus with idle percentage greater than 95% in the last - # 5 seconds. - cpuocc_list=($(sar -P ALL 1 5|grep Average|awk '{if(NR>2)print $8}')) - - for idle_value in ${cpuocc_list[@]}; do - is_vswitch_core $cpu - if [ $? -eq 1 ]; then - cpu=$(($cpu+1)) - continue - fi - - is_platform_core $cpu - if [ $? -eq 1 ]; then - # Platform core is added to the idle list by default - idle_cpulist=$idle_cpulist$cpu"," - else - # Non platform core is added to the idle list if it is more - # than 95% idle - if [[ $(echo "$idle_value > ${IDLE_MARK}"|bc) -eq 1 ]]; then - idle_cpulist=$idle_cpulist$cpu"," - fi - fi - cpu=$(($cpu+1)) - done - - idle_cpulist=$(echo $idle_cpulist|sed 's/.$//') - - log_debug "${TAG} Affining all tasks to idle CPU ($idle_cpulist)" - pidlist=( $(reaffineable_pids) ) - for pid in ${pidlist[@]}; do - taskset --all-tasks --pid --cpu-list \ - ${idle_cpulist} ${pid} > /dev/null 2>&1 - done - - # Save the cpu list to the temp file which will be read and removed when - # tasks are reaffined to the platform cores later on. - echo $idle_cpulist > ${TASK_AFFINING_INCOMPLETE} - return $rc -} - -################################################################################ -# The following function is called by sm at the end of swact sequence -# to re-affine management tasks back to the platform cores. -################################################################################ -function affine_tasks_to_platform_cores { - local cpulist - local pidlist - local rc=0 - local count=0 - - if [ ! -f ${TASK_AFFINING_INCOMPLETE} ]; then - dbg_str="${TAG} Either tasks have never been affined to all/idle" - dbg_str="${TAG} cores or they have already been reaffined to" - dbg_str="${TAG} platform cores." - log_debug "$dbg_str" - return 0 - fi - - read cpulist < ${TASK_AFFINING_INCOMPLETE} - - log_debug "${TAG} Reaffining tasks to platform cores (${PLATFORM_CPUS})..." - pidlist=( $(reaffineable_pids) ) - for pid in ${pidlist[@]}; do - taskset --all-tasks --pid --cpu-list \ - ${PLATFORM_CPUS} ${pid} > /dev/null 2>&1 - done - - # Reaffine vSwitch tasks that span multiple cpus to platform cpus - pidlist=$(ps -eL -o pid=,comm= | awk '/eal-intr-thread/ {print $1}') - for pid in ${pidlist[@]}; do - grep Cpus_allowed_list /proc/${pid}/task/*/status 2>/dev/null | \ - sed 's#/# #g' | awk '/,|-/ {print $4}' | \ - xargs --no-run-if-empty -i{} \ - taskset --pid --cpu-list ${PLATFORM_CPUS} {} > /dev/null 2>&1 - done - - rm -rf ${TASK_AFFINING_INCOMPLETE} - return $rc -} - -################################################################################ -# The following function can be leveraged by cron tasks -################################################################################ -function get_most_idle_core { - local cpuocc_list - local cpu=0 - local most_idle_value=${IDLE_MARK} - local most_idle_cpu=0 - - if [[ "${KERNEL}" == *" RT "* ]]; then - echo $cpu - return - fi - - cpuocc_list=($(sar -P ALL 1 5|grep Average|awk '{if(NR>2)print $8}')) - - for idle_value in ${cpuocc_list[@]}; do - is_vswitch_core $cpu - if [ $? -eq 1 ]; then - cpu=$(($cpu+1)) - continue - fi - - if [ $(echo "$idle_value > $most_idle_value"|bc) -eq 1 ]; then - most_idle_value=$idle_value - most_idle_cpu=$cpu - fi - cpu=$(($cpu+1)) - done - - echo $most_idle_cpu -} diff --git a/worker-utils/worker-utils/topology b/worker-utils/worker-utils/topology deleted file mode 100644 index 9f6e26fc25..0000000000 --- a/worker-utils/worker-utils/topology +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2013-2014 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# - -python /usr/bin/topology.pyc diff --git a/worker-utils/worker-utils/topology.py b/worker-utils/worker-utils/topology.py deleted file mode 100755 index 08ed4c52a5..0000000000 --- a/worker-utils/worker-utils/topology.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -################################################################################ -# Copyright (c) 2013 Wind River Systems, Inc. -# -# SPDX-License-Identifier: Apache-2.0 -# -################################################################################ -# -# topology.py -- gives a summary of logical cpu enumeration, -# sockets, cores per package, threads per core, -# total memory, and numa nodes - -from __future__ import print_function -import os -import sys -import re - -class Topology(object): - """ Build up topology information. - (i.e. logical cpu topology, NUMA nodes, memory) - """ - - def __init__(self): - self.num_cpus = 0 - self.num_nodes = 0 - self.num_sockets = 0 - self.num_cores_per_pkg = 0 - self.num_threads_per_core = 0 - - self.topology = {} - self.topology_idx = {} - self.total_memory_MiB = 0 - self.total_memory_nodes_MiB = [] - - self._get_cpu_topology() - self._get_total_memory_MiB() - self._get_total_memory_nodes_MiB() - - def _get_cpu_topology(self): - '''Enumerate logical cpu topology based on parsing /proc/cpuinfo - as function of socket_id, core_id, and thread_id. This updates - topology and reverse index topology_idx mapping. - - :param self - :updates self.num_cpus - number of logical cpus - :updates self.num_nodes - number of sockets; maps to number of numa nodes - :updates self.topology[socket_id][core_id][thread_id] = cpu - :updates self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} - :returns None - ''' - - self.num_cpus = 0 - self.num_nodes = 0 - self.num_sockets = 0 - self.num_cores = 0 - self.num_threads = 0 - self.topology = {} - self.topology_idx = {} - - Thread_cnt = {} - cpu = socket_id = core_id = thread_id = -1 - re_processor = re.compile(r'^[Pp]rocessor\s+:\s+(\d+)') - re_socket = re.compile(r'^physical id\s+:\s+(\d+)') - re_core = re.compile(r'^core id\s+:\s+(\d+)') - - with open('/proc/cpuinfo', 'r') as infile: - for line in infile: - - match = re_processor.search(line) - if match: - cpu = int(match.group(1)) - socket_id = -1; core_id = -1; thread_id = -1 - self.num_cpus += 1 - continue - - match = re_socket.search(line) - if match: - socket_id = int(match.group(1)) - continue - - match = re_core.search(line) - if match: - core_id = int(match.group(1)) - - if socket_id not in Thread_cnt: - Thread_cnt[socket_id] = {} - if core_id not in Thread_cnt[socket_id]: - Thread_cnt[socket_id][core_id] = 0 - else: - Thread_cnt[socket_id][core_id] += 1 - thread_id = Thread_cnt[socket_id][core_id] - - if socket_id not in self.topology: - self.topology[socket_id] = {} - if core_id not in self.topology[socket_id]: - self.topology[socket_id][core_id] = {} - - self.topology[socket_id][core_id][thread_id] = cpu - self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} - continue - self.num_nodes = len(self.topology.keys()) - - # In the case topology not detected, hard-code structures - if self.num_nodes == 0: - n_sockets, n_cores, n_threads = (1, self.num_cpus, 1) - self.topology = {} - for socket_id in range(n_sockets): - self.topology[socket_id] = {} - for core_id in range(n_cores): - self.topology[socket_id][core_id] = {} - for thread_id in range(n_threads): - self.topology[socket_id][core_id][thread_id] = 0 - # Define Thread-Socket-Core order for logical cpu enumeration - self.topology_idx = {} - cpu = 0 - for thread_id in range(n_threads): - for socket_id in range(n_sockets): - for core_id in range(n_cores): - self.topology[socket_id][core_id][thread_id] = cpu - self.topology_idx[cpu] = {'s': socket_id, 'c': core_id, 't': thread_id} - cpu += 1 - self.num_nodes = len(self.topology.keys()) - - self.num_sockets = len(self.topology.keys()) - self.num_cores_per_pkg = len(self.topology[0].keys()) - self.num_threads_per_core = len(self.topology[0][0].keys()) - - return None - - def _get_total_memory_MiB(self): - """Get the total memory for VMs (MiB). - - :updates: total memory for VMs (MiB) - - """ - - self.total_memory_MiB = 0 - - # Total memory - try: - m = open('/proc/meminfo').read().split() - idx_Total = m.index('MemTotal:') + 1 - self.total_memory_MiB = int(m[idx_Total]) / 1024 - except IOError: - # silently ignore IO errors (eg. file missing) - pass - return None - - def _get_total_memory_nodes_MiB(self): - """Get the total memory per numa node for VMs (MiB). - - :updates: total memory per numa node for VMs (MiB) - - """ - - self.total_memory_nodes_MiB = [] - - # Memory of each numa node (MiB) - for node in range(self.num_nodes): - Total_MiB = 0 - - meminfo = "/sys/devices/system/node/node%d/meminfo" % node - try: - m = open(meminfo).read().split() - idx_Total = m.index('MemTotal:') + 1 - Total_MiB = int(m[idx_Total]) / 1024 - except IOError: - # silently ignore IO errors (eg. file missing) - pass - - self.total_memory_nodes_MiB.append(Total_MiB) - return None - - def _print_cpu_topology(self): - '''Print logical cpu topology enumeration as function of: - socket_id, core_id, and thread_id. - - :param self - :returns None - ''' - - cpu_list = self.topology_idx.keys() - cpu_list.sort() - total_memory_GiB = self.total_memory_MiB/1024.0 - - print('TOPOLOGY:') - print('%16s : %5d' % ('logical cpus', self.num_cpus)) - print('%16s : %5d' % ('sockets', self.num_sockets)) - print('%16s : %5d' % ('cores_per_pkg', self.num_cores_per_pkg)) - print('%16s : %5d' % ('threads_per_core', self.num_threads_per_core)) - print('%16s : %5d' % ('numa_nodes', self.num_nodes)) - print('%16s : %5.2f %s' % ('total_memory', total_memory_GiB, 'GiB')) - print('%16s :' % ('memory_per_node'), end=' ') - for node in range(self.num_nodes): - node_memory_GiB = self.total_memory_nodes_MiB[node]/1024.0 - print('%5.2f' % (node_memory_GiB), end=' ') - print('%s' % ('GiB')) - print('') - - print('LOGICAL CPU TOPOLOGY:') - print("%9s :" % 'cpu_id', end=' ') - for cpu in cpu_list: - print("%3d" % cpu, end=' ') - print('') - print("%9s :" % 'socket_id', end=' ') - for cpu in cpu_list: - socket_id = self.topology_idx[cpu]['s'] - print("%3d" % socket_id, end=' ') - print('') - print("%9s :" % 'core_id', end=' ') - for cpu in cpu_list: - core_id = self.topology_idx[cpu]['c'] - print("%3d" % core_id, end=' ') - print('') - print("%9s :" % 'thread_id', end=' ') - for cpu in cpu_list: - thread_id = self.topology_idx[cpu]['t'] - print("%3d" % thread_id, end=' ') - print('') - print('') - - print('CORE TOPOLOGY:') - print("%6s %9s %7s %9s %s" % ('cpu_id', 'socket_id', 'core_id', 'thread_id', 'affinity')) - for cpu in cpu_list: - affinity = 1<::. -# -# Example: To reserve 1500MB and 1 core on NUMA node0, and 1500MB and 1 core -# on NUMA node1, the variable must be specified as follows. -# WORKER_BASE_MEMORY=("node0:1500MB:1" "node1:1500MB:1") -# -################################################################################ -WORKER_BASE_RESERVED=("node0:8000MB:1" "node1:2000MB:0" "node2:2000MB:0" "node3:2000MB:0") - -################################################################################ -# -# List of HugeTLB memory descriptors to configure. Each array element -# consists of a 3-tuple descriptor formatted as: ::. -# The NUMA node specified must exist and the HugeTLB pagesize must be a valid -# value such as 2048kB or 1048576kB. -# -# For example, to request 256 x 2MB HugeTLB pages on NUMA node0 and node1 the -# variable must be specified as follows. -# WORKER_VSWITCH_MEMORY=("node0:2048kB:256" "node1:2048kB:256") -# -################################################################################ -WORKER_VSWITCH_MEMORY=("node0:1048576kB:1" "node1:1048576kB:1" "node2:1048576kB:1" "node3:1048576kB:1") - -################################################################################ -# -# List of VSWITCH physical cores reserved for VSWITCH applications. -# -# Example: To reserve 2 cores on NUMA node0, and 2 cores on NUMA node1, the -# variable must be specified as follows. -# WORKER_VSWITCH_CORES=("node0:2" "node1:2") -# -################################################################################ -WORKER_VSWITCH_CORES=("node0:2" "node1:0" "node2:0" "node3:0")