From 07bc72dfcf69dbf49af26c08ff2c9adf4963d827 Mon Sep 17 00:00:00 2001 From: Heitor Matsui Date: Fri, 1 Sep 2023 17:55:58 -0300 Subject: [PATCH] Create runtime_config table This commit creates a runtime_config table, to allow storing runtime config entries per host so that a follow-up commit will add the capability to retry runtime config requests to hosts that are taking too long to report back to conductor the runtime manifest application success. The table added on this commit also comes with a time-based pruning function, since it has potential to become large depending the size and number of nodes of the deployment, so that this function can be used by the follow-up commit to keep only the entries inside a determined and relevant timeframe. Test Plan: PASS: fresh install/bootstrap/unlock single and multi-node deployments PASS: DX upgrade stx-6 to stx-8 PASS: SX upgrade stx-6 to stx-8 Partial-Bug: 2037090 Change-Id: I40084fe34e8827262dc0453298626ac7394e6bb5 Signed-off-by: Heitor Matsui --- .../sysinv/sysinv/sysinv/common/constants.py | 7 ++ sysinv/sysinv/sysinv/sysinv/db/api.py | 45 ++++++++++++ .../sysinv/sysinv/sysinv/db/sqlalchemy/api.py | 72 +++++++++++++++++++ .../versions/132_runtime_config.py | 50 +++++++++++++ .../sysinv/sysinv/db/sqlalchemy/models.py | 21 ++++++ .../sysinv/sysinv/sysinv/objects/__init__.py | 3 + .../sysinv/sysinv/objects/runtime_config.py | 27 +++++++ 7 files changed, 225 insertions(+) create mode 100644 sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/132_runtime_config.py create mode 100644 sysinv/sysinv/sysinv/sysinv/objects/runtime_config.py diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index b48a3f137a..15cff2f53d 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -2407,3 +2407,10 @@ CSTATE_PATH = "/sys/devices/system/cpu/cpu0/cpuidle" # Auto-recovery limits for kube upgrade abort AUTO_RECOVERY_COUNT = 3 + +# Puppet Runtime Manifest constants +RUNTIME_CONFIG_APPLY_TIMEOUT_IN_SECS = 600 +RUNTIME_CONFIG_STATE_PENDING = "pending" +RUNTIME_CONFIG_STATE_APPLIED = "applied" +RUNTIME_CONFIG_STATE_FAILED = "failed" +RUNTIME_CONFIG_STATE_RETRIED = "retried" diff --git a/sysinv/sysinv/sysinv/sysinv/db/api.py b/sysinv/sysinv/sysinv/sysinv/db/api.py index 2bb9d28a5e..4823be12b0 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/api.py @@ -5054,3 +5054,48 @@ class Connection(object): :param networktype: network type """ + + @abc.abstractmethod + def runtime_config_create(self, values): + """Create a runtime_config entry + + :param values: runtime_config entry relevant information + """ + @abc.abstractmethod + def runtime_config_destroy(self, id): + """Destroy a runtime_config entry + + :param id: runtime_config id or uuid + """ + + @abc.abstractmethod + def runtime_config_prune(self, older_than): + """Prune records older than a given date + + :param older_than: date to filter entries older than it + """ + + @abc.abstractmethod + def runtime_config_update(self, id, values): + """Update a runtime_config entry + + :param id: runtime_config id or uuid + :param values: dictionary containing fields to be updated + """ + + @abc.abstractmethod + def runtime_config_get(self, id, host_id=None): + """Returns a list of runtime_config entries for a given uuid + + :param id: runtime config id or uuid + :param host_id: host id + """ + + @abc.abstractmethod + def runtime_config_get_all(self, config_uuid=None, state=None, older_than=None): + """Returns a list of runtime_config entries with a given filter + + :param config_uuid: runtime_config uuid + :param state: runtime_config state + :param older_than: date to filter entries older than it + """ diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index f7cbd8bceb..ff77b2b0a5 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -9348,3 +9348,75 @@ class Connection(api.Connection): if count != 1: raise exception.NotFound() return query.one() + + @db_objects.objectify(objects.runtime_config) + def runtime_config_create(self, values): + runtime_config = models.RuntimeConfig() + runtime_config.update(values) + with _session_for_write() as session: + try: + session.add(runtime_config) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error('Entry for runtime_config (config_uuid=%s, host_id=%s) already exists.' % ( + values['config_uuid'], values['forihostid'])) + return runtime_config + + @db_objects.objectify(objects.runtime_config) + def runtime_config_destroy(self, id): + with _session_for_write() as session: + query = model_query(models.RuntimeConfig, session=session) + if utils.is_uuid_like(id): + query = query.filter_by(config_uuid=id) + else: + query = query.filter_by(id=id) + query.delete() + + @db_objects.objectify(objects.runtime_config) + def runtime_config_prune(self, older_than): + with _session_for_write() as session: + query = model_query(models.RuntimeConfig, session=session) + query = query.filter(models.RuntimeConfig.created_at < older_than) + result = query.all() + query.delete() + return result + + @db_objects.objectify(objects.runtime_config) + def runtime_config_update(self, id, values): + with _session_for_write() as session: + query = model_query(models.RuntimeConfig, session=session) + query = query.filter_by(id=id) + count = query.update(values) + if count != 1: + raise exception.NotFound() + return query.one() + + @db_objects.objectify(objects.runtime_config) + def runtime_config_get(self, id, host_id=None): + query = model_query(models.RuntimeConfig) + + if utils.is_uuid_like(id): + query = query.filter_by(config_uuid=id) + if host_id: + query = query.filter_by(forihostid=host_id) + else: + query = query.filter_by(id=id) + + try: + result = query.one() + except NoResultFound: + raise exception.NotFound() + except MultipleResultsFound: + raise exception.MultipleResults() + return result + + @db_objects.objectify(objects.runtime_config) + def runtime_config_get_all(self, config_uuid=None, state=None, older_than=None): + query = model_query(models.RuntimeConfig) + if config_uuid: + query = query.filter_by(config_uuid=config_uuid) + if state: + query = query.filter_by(state=state) + if older_than: + query = query.filter(models.RuntimeConfig.created_at < older_than) + return query.all() diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/132_runtime_config.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/132_runtime_config.py new file mode 100644 index 0000000000..a369c789ec --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/132_runtime_config.py @@ -0,0 +1,50 @@ +######################################################################## +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## + +from migrate.changeset import UniqueConstraint +from sqlalchemy import Integer, String, DateTime +from sqlalchemy import Column, MetaData, Table, ForeignKey + +from sysinv.db.sqlalchemy.models import UUID_LENGTH + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + Table('i_host', meta, autoload=True) + + runtime_config = Table( + 'runtime_config', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('config_uuid', String(UUID_LENGTH), nullable=False), + Column('config_dict', String(767)), + Column('state', String(255)), + Column('forihostid', Integer, + ForeignKey('i_host.id', ondelete='CASCADE'), nullable=False), + Column('reserved_1', String(255)), + UniqueConstraint('config_uuid', 'forihostid', name='u_config_uuid_forihostid'), + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + runtime_config.create() + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + runtime_config = Table('runtime_config', meta, autoload=True) + runtime_config.drop() diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index 2d999e35d3..eee4e5febb 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -2169,3 +2169,24 @@ class KubeCmdVersions(Base): kubelet_version = Column(String(255), nullable=False) UniqueConstraint('kubeadm_version', 'kubelet_version', name='u_kubeadm_version_kubelet_version') + + +class RuntimeConfig(Base): + runtimeConfigStateEnum = Enum( + constants.RUNTIME_CONFIG_STATE_PENDING, + constants.RUNTIME_CONFIG_STATE_APPLIED, + constants.RUNTIME_CONFIG_STATE_FAILED, + constants.RUNTIME_CONFIG_STATE_RETRIED, + name="runtimeConfigStateEnum" + ) + + __tablename__ = 'runtime_config' + + id = Column(Integer, primary_key=True) + config_uuid = Column(String(UUID_LENGTH), nullable=False) + config_dict = Column(String(767), nullable=False) + state = Column(runtimeConfigStateEnum, default=constants.RUNTIME_CONFIG_STATE_PENDING) + forihostid = Column(Integer, ForeignKey('i_host.id', ondelete='CASCADE')) + reserved_1 = Column(String(255)) + UniqueConstraint('config_uuid', 'forihostid', + name='u_config_uuid_forihostid') diff --git a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py index 73c3b56b00..49e8e3c365 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py @@ -104,6 +104,7 @@ from sysinv.objects import host_fs from sysinv.objects import restore from sysinv.objects import kube_rootca_update from sysinv.objects import kube_rootca_host_update +from sysinv.objects import runtime_config # alias objects for RPC compatibility @@ -199,6 +200,7 @@ fpga_device = fpga_device.FPGADevice restore = restore.Restore kube_rootca_host_update = kube_rootca_host_update.KubeRootCAHostUpdate kube_rootca_update = kube_rootca_update.KubeRootCAUpdate +runtime_config = runtime_config.RuntimeConfig __all__ = ("system", "cluster", @@ -284,6 +286,7 @@ __all__ = ("system", "restore", "kube_rootca_host_update", "kube_rootca_update", + "runtime_config", # alias objects for RPC compatibility "ihost", "ilvg") diff --git a/sysinv/sysinv/sysinv/sysinv/objects/runtime_config.py b/sysinv/sysinv/sysinv/sysinv/objects/runtime_config.py new file mode 100644 index 0000000000..f7a1d97eca --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/objects/runtime_config.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from sysinv.db import api as db_api +from sysinv.objects import base +from sysinv.objects import utils + + +class RuntimeConfig(base.SysinvObject): + + dbapi = db_api.get_instance() + + fields = { + 'id': int, + 'config_uuid': utils.str_or_none, + 'config_dict': utils.str_or_none, + 'state': utils.str_or_none, + 'forihostid': utils.int_or_none, + 'reserved_1': utils.str_or_none, + }