From ab469de093d1429c6f5281d21be275b51004b9f2 Mon Sep 17 00:00:00 2001 From: Igor Soares Date: Mon, 6 Nov 2023 19:50:50 -0300 Subject: [PATCH] Create kube_app_bundle table This commit creates a new table called kube_app_bundle. This table will be used to store metadata extracted from StarlingX application bundles. Database API methods were created to allow bulk inserts to the table, checking whether it is empty, retrieving entries by application name and pruning all data. A follow-up commit will enable the Application Framework to populate and retrieve data from the table. Test plan: PASS: build-pkgs -a && build-image PASS: AIO-SX fresh install Check if the kube_app_bundle table was created as expected PASS: AIO-DX fresh install Check if the kube_app_bundle table was created as expected PASS: upgrade from stx-8 Check if the kube_app_bundle table was created as expected Story: 2010929 Task: 49097 Change-Id: Ifd10f9e5e4a2d26c42d2b83084e073c7834cd75a Signed-off-by: Igor Soares --- .../sysinv/sysinv/sysinv/common/constants.py | 7 +- .../sysinv/sysinv/sysinv/common/exception.py | 13 +++- sysinv/sysinv/sysinv/sysinv/db/api.py | 69 ++++++++++++++++++- .../sysinv/sysinv/sysinv/db/sqlalchemy/api.py | 64 +++++++++++++++++ .../versions/134_kube_app_bundle.py | 53 ++++++++++++++ .../sysinv/sysinv/db/sqlalchemy/models.py | 27 ++++++++ .../sysinv/sysinv/sysinv/objects/__init__.py | 5 +- .../sysinv/sysinv/objects/kube_app_bundle.py | 30 ++++++++ 8 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/134_kube_app_bundle.py create mode 100644 sysinv/sysinv/sysinv/sysinv/objects/kube_app_bundle.py diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index 8284552023..895888c5b5 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -1966,6 +1966,7 @@ APP_METADATA_ORDERED_APPS = 'ordered_apps' APP_METADATA_UPGRADES = 'upgrades' APP_METADATA_UPDATE_FAILURE_SKIP_RECOVERY = 'update_failure_no_rollback' APP_METADATA_AUTO_UPDATE = 'auto_update' +APP_METADATA_AUTO_UPDATE_DEFAULT_VALUE = True APP_METADATA_FAILED_VERSIONS = 'failed_versions' APP_METADATA_FROM_VERSIONS = 'from_versions' APP_METADATA_SUPPORTED_K8S_VERSION = 'supported_k8s_version' @@ -1975,7 +1976,11 @@ APP_METADATA_MAXIMUM = 'maximum' APP_METADATA_K8S_UPGRADES = 'k8s_upgrades' APP_METADATA_K8S_AUTO_UPDATE_DEFAULT_VALUE = True APP_METADATA_TIMING = 'timing' -APP_METADATA_TIMING_DEFAULT_VALUE = 'post' +APP_METADATA_TIMING_PRE = 'pre' +APP_METADATA_TIMING_POST = 'post' +APP_METADATA_TIMING_DEFAULT_VALUE = APP_METADATA_TIMING_POST +APP_METADATA_NAME = 'app_name' +APP_METADATA_VERSION = 'app_version' APP_EVALUATE_REAPPLY_TYPE_HOST_ADD = 'host-add' APP_EVALUATE_REAPPLY_TYPE_HOST_DELETE = 'host-delete' diff --git a/sysinv/sysinv/sysinv/sysinv/common/exception.py b/sysinv/sysinv/sysinv/sysinv/common/exception.py index 2d082326a6..098592ff90 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/exception.py +++ b/sysinv/sysinv/sysinv/sysinv/common/exception.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright (c) 2013-2022 Wind River Systems, Inc. +# Copyright (c) 2013-2023 Wind River Systems, Inc. # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. @@ -1571,6 +1571,17 @@ class IncompatibleKubeVersion(SysinvException): message = _("The application %(name)s (%(version)s) is incompatible with the current " "Kubernetes version %(kube_version)s.") + +class KubeAppBundleAlreadyExists(Conflict): + message = _("A Kubernetes application bundle with name %(name)s and " + "version %(version)s or with file path %(file_path)s already exists.") + + +class KubeAppBundleAlreadyExistsBulk(Conflict): + message = _("A Kubernetes application bundle with column(s) '%(columns)s' and value(s) " + "'%(values)s' already exists.") + + # # Kubernetes related exceptions # diff --git a/sysinv/sysinv/sysinv/sysinv/db/api.py b/sysinv/sysinv/sysinv/sysinv/db/api.py index 4823be12b0..59f0b3d3e3 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/api.py @@ -16,7 +16,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2021 Wind River Systems, Inc. +# Copyright (c) 2013-2023 Wind River Systems, Inc. # @@ -5099,3 +5099,70 @@ class Connection(object): :param state: runtime_config state :param older_than: date to filter entries older than it """ + + @abc.abstractmethod + def kube_app_bundle_create(self, values): + """Create a kube_app_bundle entry + + :param values: A dictionary with the respective fields and values to be added to the db. + """ + + @abc.abstractmethod + def kube_app_bundle_create_all(self, values_list): + """Create kube_app_bundle entries + + :param values_list: a list containing the dictionaries with the respective + fields and values to be added to the db. + """ + + @abc.abstractmethod + def kube_app_bundle_is_empty(self): + """Check if kube_app_bundle table is empty""" + + @abc.abstractmethod + def kube_app_bundle_get_all(self, + name=None, + limit=None, + marker=None, + sort_key=None, + sort_dir=None): + """Return a list of all kube_app_bundle entries or a list based on a + given filter. + + :param name: Application name. + :param limit: Maximum number of entries to return. + :param marker: The last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: Direction in which results should be sorted. + (asc, desc) + :returns: A list of kube_app_bundle entries with the given name. + """ + + @abc.abstractmethod + def kube_app_bundle_get_by_name(self, + name, + limit=None, + marker=None, + sort_key=None, + sort_dir=None): + """Get kube_app_bundle entries that match a given name + + :param name: Application name. + :param limit: Maximum number of entries to return. + :param marker: The last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: Direction in which results should be sorted. + (asc, desc) + :returns: A list of kube_app_bundle entries with the given name. + """ + + @abc.abstractmethod + def kube_app_bundle_destroy_all(self, file_path=None): + """Delete all records from kube_app_bundle or delete based on a + given filter""" + + @abc.abstractmethod + def kube_app_bundle_destroy_by_file_path(self, file_path): + """Delete records from kube_app_bundle that match a file path""" diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py index edcee4cf56..ffa41d3f26 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/api.py @@ -27,6 +27,7 @@ from oslo_db import exception as db_exc from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import utils as db_utils +from sqlalchemy import insert from sqlalchemy import inspect from sqlalchemy import or_ @@ -9420,3 +9421,66 @@ class Connection(api.Connection): if older_than: query = query.filter(models.RuntimeConfig.created_at < older_than) return query.all() + + @db_objects.objectify(objects.kube_app_bundle) + def kube_app_bundle_create(self, values): + kube_app_bundle = models.KubeAppBundle() + kube_app_bundle.update(values) + with _session_for_write() as session: + try: + session.add(kube_app_bundle) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.KubeAppBundleAlreadyExists( + name=values['name'], + version=values['version'], + file_path=values['file_path']) + return kube_app_bundle + + def kube_app_bundle_create_all(self, values_list): + try: + with _session_for_write() as session: + session.execute( + insert(models.KubeAppBundle), + values_list, + ) + session.flush() + except db_exc.DBDuplicateEntry as e: + columns = ', '.join(e.columns) + raise exception.KubeAppBundleAlreadyExistsBulk(columns=columns, values=e.value) + + def kube_app_bundle_is_empty(self): + result = model_query(models.KubeAppBundle).first() + + return result is None + + @db_objects.objectify(objects.kube_app_bundle) + def kube_app_bundle_get_all(self, name=None, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.KubeAppBundle) + if name: + query = query.filter_by(name=name) + + return _paginate_query(models.KubeAppBundle, limit, marker, + sort_key, sort_dir, query) + + @db_objects.objectify(objects.kube_app_bundle) + def kube_app_bundle_get_by_name(self, name, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + return self.kube_app_bundle_get_all(name, limit, marker, + sort_key, sort_dir) + + def kube_app_bundle_destroy_all(self, file_path=None): + with _session_for_write() as session: + query = model_query(models.KubeAppBundle, session=session) + + if file_path: + query = query.filter_by(file_path=file_path) + + query.delete() + + def kube_app_bundle_destroy_by_file_path(self, file_path): + self.kube_app_bundle_destroy_all(file_path) diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/134_kube_app_bundle.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/134_kube_app_bundle.py new file mode 100644 index 0000000000..e841334e30 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/134_kube_app_bundle.py @@ -0,0 +1,53 @@ +######################################################################## +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## + +from migrate.changeset import UniqueConstraint +from sqlalchemy import Integer, String, DateTime, Boolean, Text +from sqlalchemy import Column, MetaData, Table, ForeignKey +from sysinv.db.sqlalchemy.models import KubeAppBundle + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + kube_app_bundle = Table( + 'kube_app_bundle', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('name', String(255), nullable=False), + Column('version', String(255), nullable=False), + Column('file_path', String(255), nullable=False), + Column('auto_update', Boolean, nullable=False), + Column('k8s_auto_update', Boolean, nullable=False), + Column('k8s_timing', KubeAppBundle.KubeAppBundleTimingEnum, nullable=False), + Column('k8s_minimum_version', String(16), nullable=False), + Column('k8s_maximum_version', String(16), nullable=True), + Column('reserved', Text, nullable=True), + UniqueConstraint('name', 'version', name='u_bundle_name_version'), + UniqueConstraint('file_path', name='u_bundle_file_path'), + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + kube_app_bundle.create() + + # Create KubeApp FK to KubeAppBundle + kube_app = Table('kube_app', meta, autoload=True) + kube_app.create_column(Column('app_bundle_id', Integer, + ForeignKey('kube_app_bundle.id', + ondelete='SET NULL'))) + + +def downgrade(migrate_engine): + raise NotImplementedError('SysInv database downgrade is unsupported.') diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index 3e262fa10a..11b0e7efd8 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -2071,6 +2071,8 @@ class KubeApp(Base): recovery_attempts = Column(Integer, nullable=False, default=0) mode = Column(String(255), nullable=True) app_metadata = Column(JSONEncodedDict) + app_bundle_id = Column(Integer, ForeignKey('kube_app_bundle.id', + ondelete='SET NULL')) UniqueConstraint('name', 'app_version', name='u_app_name_version') @@ -2192,3 +2194,28 @@ class RuntimeConfig(Base): reserved_1 = Column(String(255)) UniqueConstraint('config_uuid', 'forihostid', name='u_config_uuid_forihostid') + + +class KubeAppBundle(Base): + KubeAppBundleTimingEnum = Enum( + constants.APP_METADATA_TIMING_PRE, + constants.APP_METADATA_TIMING_POST, + name="KubeAppBundleTimingEnum" + ) + + __tablename__ = 'kube_app_bundle' + id = Column(Integer, primary_key=True) + name = Column(String(255), nullable=False) + version = Column(String(255), nullable=False) + file_path = Column(String(255), nullable=False) + auto_update = Column(Boolean, nullable=False, + default=constants.APP_METADATA_AUTO_UPDATE_DEFAULT_VALUE) + k8s_auto_update = Column(Boolean, nullable=False, default=True) + k8s_timing = Column(KubeAppBundleTimingEnum, + nullable=False, + default=constants.APP_METADATA_TIMING_DEFAULT_VALUE) + k8s_minimum_version = Column(String(16), nullable=False) + k8s_maximum_version = Column(String(16), nullable=True) + reserved = Column(JSONEncodedDict, nullable=True) + UniqueConstraint('name', 'version', name='u_bundle_name_version') + UniqueConstraint('file_path', name='u_bundle_file_path') diff --git a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py index 49e8e3c365..e266a0600b 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/__init__.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/__init__.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. # -# Copyright (c) 2013-2022 Wind River Systems, Inc. +# Copyright (c) 2013-2023 Wind River Systems, Inc. # from sysinv.objects import address @@ -38,6 +38,7 @@ from sysinv.objects import helm_overrides from sysinv.objects import host from sysinv.objects import host_upgrade from sysinv.objects import kube_app +from sysinv.objects import kube_app_bundle from sysinv.objects import kube_app_releases from sysinv.objects import kube_host_upgrade from sysinv.objects import kube_upgrade @@ -185,6 +186,7 @@ storage_ceph_rook = storage_ceph_rook.StorageCephRook helm_overrides = helm_overrides.HelmOverrides label = label.Label kube_app = kube_app.KubeApp +kube_app_bundle = kube_app_bundle.KubeAppBundle kube_app_releases = kube_app_releases.KubeAppReleases kube_host_upgrade = kube_host_upgrade.KubeHostUpgrade kube_upgrade = kube_upgrade.KubeUpgrade @@ -271,6 +273,7 @@ __all__ = ("system", "storage_ceph_external", "helm_overrides", "kube_app", + "kube_app_bundle", "kube_app_releases", "kube_host_upgrade", "kube_upgrade", diff --git a/sysinv/sysinv/sysinv/sysinv/objects/kube_app_bundle.py b/sysinv/sysinv/sysinv/sysinv/objects/kube_app_bundle.py new file mode 100644 index 0000000000..6314093a72 --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/objects/kube_app_bundle.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from sysinv.db import api as db_api +from sysinv.objects import base +from sysinv.objects import utils + + +class KubeAppBundle(base.SysinvObject): + + dbapi = db_api.get_instance() + + fields = { + 'id': int, + 'name': utils.str_or_none, + 'version': utils.str_or_none, + 'file_path': utils.str_or_none, + 'auto_update': utils.bool_or_none, + 'k8s_auto_update': utils.bool_or_none, + 'k8s_timing': utils.str_or_none, + 'k8s_minimum_version': utils.str_or_none, + 'k8s_maximum_version': utils.str_or_none, + 'reserved': utils.dict_or_none + }