From a7d91e2961ecc114e7936b97d57b9540b71c658c Mon Sep 17 00:00:00 2001 From: Andy Ning Date: Tue, 17 Sep 2019 11:52:02 -0400 Subject: [PATCH] DC remove firewallrules audit from dcorch OAM firewallrules are now managed by Calico GlobalNetworkPolicy configuration via k8s API (not by sysinv anymore). This update removed firewallrules audit from dcorch. Change-Id: I9fab73c016bb4af760c7d78f0db18dcc8bb77057 Closes-Bug: 1844147 Signed-off-by: Andy Ning --- dcorch/api/proxy/apps/controller.py | 8 +- dcorch/api/proxy/common/constants.py | 6 -- dcorch/common/consts.py | 1 - dcorch/drivers/openstack/sysinv_v1.py | 72 ------------- dcorch/engine/sync_services/sysinv.py | 140 +------------------------- 5 files changed, 4 insertions(+), 223 deletions(-) diff --git a/dcorch/api/proxy/apps/controller.py b/dcorch/api/proxy/apps/controller.py index e57fb1521..b5770049d 100644 --- a/dcorch/api/proxy/apps/controller.py +++ b/dcorch/api/proxy/apps/controller.py @@ -394,13 +394,9 @@ class SysinvAPIController(APIController): request_header = self.get_request_header(environ) operation_type = proxy_utils.get_operation_type(environ) resource_type = self._get_resource_type_from_environ(environ) - # Firewall rule and certificate need special processing + # certificate need special processing p_resource_info = 'suppressed' - if resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - resource_info['payload'] = request_body - resource = json.loads(response.body)[resource_type] - resource_id = resource['firewall_sig'] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: + if resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: resource_info['payload'] = request_body resource_info['content_type'] = environ.get('CONTENT_TYPE') resource = json.loads(response.body)[resource_type] diff --git a/dcorch/api/proxy/common/constants.py b/dcorch/api/proxy/common/constants.py index e1373f7c7..1db9da8bb 100755 --- a/dcorch/api/proxy/common/constants.py +++ b/dcorch/api/proxy/common/constants.py @@ -98,10 +98,6 @@ REMOTELOGGING_PATHS = [ '/v1/remotelogging/{uuid}' ] -FIREWALLRULES_PATHS = [ - '/v1/firewallrules/import_firewall_rules' -] - CERTIFICATE_PATHS = [ '/v1/certificate/certificate_install' ] @@ -118,7 +114,6 @@ SYSINV_PATH_MAP = { consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: TRAP_DEST_PATHS, consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: COMMUNITY_STRING_PATHS, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: REMOTELOGGING_PATHS, - consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: FIREWALLRULES_PATHS, consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: CERTIFICATE_PATHS, consts.RESOURCE_TYPE_SYSINV_USER: USER_PATHS, } @@ -334,7 +329,6 @@ ROUTE_METHOD_MAP = { consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST: ['POST', 'DELETE'], consts.RESOURCE_TYPE_SYSINV_SNMP_COMM: ['POST', 'DELETE'], consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: ['PATCH'], - consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: ['POST'], consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: ['POST'], consts.RESOURCE_TYPE_SYSINV_USER: ['PATCH', 'PUT'], }, diff --git a/dcorch/common/consts.py b/dcorch/common/consts.py index 53bb11b43..cf749312e 100644 --- a/dcorch/common/consts.py +++ b/dcorch/common/consts.py @@ -83,7 +83,6 @@ ORCH_REQUEST_ABORTED = "aborted" # SysInv Resources RESOURCE_TYPE_SYSINV_CERTIFICATE = "certificates" RESOURCE_TYPE_SYSINV_DNS = "idns" -RESOURCE_TYPE_SYSINV_FIREWALL_RULES = "firewallrules" RESOURCE_TYPE_SYSINV_NTP = "intp" RESOURCE_TYPE_SYSINV_PTP = "ptp" RESOURCE_TYPE_SYSINV_REMOTE_LOGGING = "remotelogging" diff --git a/dcorch/drivers/openstack/sysinv_v1.py b/dcorch/drivers/openstack/sysinv_v1.py index 01500b6e6..c8c9e4157 100644 --- a/dcorch/drivers/openstack/sysinv_v1.py +++ b/dcorch/drivers/openstack/sysinv_v1.py @@ -11,9 +11,7 @@ # under the License. import hashlib -import os import six -import tsconfig.tsconfig as tsc from cgtsclient import client as cgts_client from cgtsclient.exc import HTTPConflict @@ -470,76 +468,6 @@ class SysinvClient(base.DriverBase): return remotelogging - def get_firewallrules(self): - """Get the firewallrules for this region - - :return: firewallrules - """ - try: - firewallruless = self.client.firewallrules.list() - firewallrules = firewallruless[0] - except Exception as e: - LOG.error("get_firewallrules region={} " - "exception={}".format(self.region_name, e)) - raise exceptions.SyncRequestFailedRetry() - - if not firewallrules: - LOG.info("firewallrules is None for region: {}".format( - self.region_name)) - - else: - LOG.info("get_firewallrules uuid=%s firewall_sig=%s" % - (firewallrules.uuid, firewallrules.firewall_sig)) - - return firewallrules - - def _validate_firewallrules(self, firewall_sig, firewallrules): - firewallrules_sig = hashlib.md5(firewallrules).hexdigest() - - if firewallrules_sig == firewall_sig: - return True - - LOG.info("_validate_firewallrules region={} sig={} mismatch " - "reference firewall_sig={}".format( - self.region_name, firewallrules_sig, firewall_sig)) - return False - - def update_firewallrules(self, - firewall_sig, - firewallrules=None): - """Update the firewallrules for this region - - :param: firewall_sig - :param: firewallrules - :return: ifirewallrules - """ - - if not firewallrules: - # firewallrules not provided, obtain from SystemController - firewall_rules_file = os.path.join( - tsc.CONFIG_PATH, - sysinv_constants.FIREWALL_RULES_FILE) - - with open(firewall_rules_file, 'r') as content_file: - firewallrules = content_file.read() - - LOG.info("update_firewallrules from shared file={}".format( - firewallrules)) - - if not self._validate_firewallrules(firewall_sig, firewallrules): - raise exceptions.SyncRequestFailedRetry() - - try: - ifirewallrules = self.client.firewallrules.import_firewall_rules( - firewallrules) - LOG.info("region={} firewallrules uuid={} firewall_sig={}".format( - self.region_name, ifirewallrules.get('uuid'), firewall_sig)) - except Exception as e: - LOG.error("update_firewallrules exception={}".format(e)) - raise exceptions.SyncRequestFailedRetry() - - return ifirewallrules - def get_certificates(self): """Get the certificates for this region diff --git a/dcorch/engine/sync_services/sysinv.py b/dcorch/engine/sync_services/sysinv.py index d6c8fa345..3c1650107 100644 --- a/dcorch/engine/sync_services/sysinv.py +++ b/dcorch/engine/sync_services/sysinv.py @@ -44,11 +44,9 @@ class SysinvSyncThread(SyncThread): SYSINV_ADD_DELETE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_SNMP_COMM, consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST] - SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES, - consts.RESOURCE_TYPE_SYSINV_CERTIFICATE, + SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_CERTIFICATE, consts.RESOURCE_TYPE_SYSINV_FERNET_REPO] - FIREWALL_SIG_NULL = 'NoCustomFirewallRules' CERTIFICATE_SIG_NULL = 'NoCertificate' RESOURCE_UUID_NULL = 'NoResourceUUID' @@ -66,8 +64,6 @@ class SysinvSyncThread(SyncThread): self.sync_snmp_trapdest, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: self.sync_remotelogging, - consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - self.sync_firewallrules, consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: self.sync_certificate, consts.RESOURCE_TYPE_SYSINV_USER: self.sync_user, @@ -81,7 +77,6 @@ class SysinvSyncThread(SyncThread): self.audit_resources = [ consts.RESOURCE_TYPE_SYSINV_CERTIFICATE, consts.RESOURCE_TYPE_SYSINV_DNS, - consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES, consts.RESOURCE_TYPE_SYSINV_NTP, consts.RESOURCE_TYPE_SYSINV_PTP, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING, @@ -531,78 +526,6 @@ class SysinvSyncThread(SyncThread): iremotelogging.uuid), extra=self.log_extra) - def update_firewallrules(self, firewall_sig, firewallrules=None): - - s_os_client = sdk.OpenStackDriver(self.region_name) - try: - ifirewallrules = s_os_client.sysinv_client.update_firewallrules( - firewall_sig, firewallrules=firewallrules) - return ifirewallrules - except (exceptions.ConnectionRefused, exceptions.NotAuthorized, - exceptions.TimeOut): - LOG.info("update_firewallrules exception Timeout", - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name) - raise exceptions.SyncRequestTimeout - except (AttributeError, TypeError) as e: - LOG.info("update_firewallrules error {} region_name".format(e), - extra=self.log_extra) - s_os_client.delete_region_clients(self.region_name, - clear_token=True) - raise exceptions.SyncRequestFailedRetry - except Exception as e: - LOG.exception(e) - raise exceptions.SyncRequestFailedRetry - - def sync_firewallrules(self, request, rsrc): - # The system is not created with default firewallrules - LOG.info("sync_firewallrules resource_info={}".format( - request.orch_job.resource_info), - extra=self.log_extra) - firewallrules_dict = jsonutils.loads(request.orch_job.resource_info) - payload = firewallrules_dict.get('payload') - # payload is the contents of the POST operation - - if not payload: - LOG.info("sync_firewallrules No payload found in resource_info" - "{}".format(request.orch_job.resource_info), - extra=self.log_extra) - return - - if isinstance(payload, dict): - firewall_sig = payload.get('firewall_sig') - else: - firewall_sig = rsrc.master_id - LOG.info("firewall_sig from master_id={}".format(firewall_sig)) - - ifirewallrules = None - if firewall_sig: - ifirewallrules = self.update_firewallrules(firewall_sig) - else: - firewall_sig = rsrc.master_id - if firewall_sig and firewall_sig != self.FIREWALL_SIG_NULL: - ifirewallrules = self.update_firewallrules( - firewall_sig, - firewallrules=payload) - else: - LOG.info("skipping firewall_sig={}".format(firewall_sig)) - - ifirewallrules_sig = None - try: - ifirewallrules_sig = \ - ifirewallrules.get('firewallrules').get('firewall_sig') - except Exception as e: - LOG.warn("No ifirewallrules={} unknown e={}".format( - ifirewallrules, e)) - - # Ensure subcloud resource is persisted to the DB for later - subcloud_rsrc_id = self.persist_db_subcloud_resource( - rsrc.id, firewall_sig) - - LOG.info("firewallrules {} {} [{}/{}] updated".format(rsrc.id, - subcloud_rsrc_id, ifirewallrules_sig, firewall_sig), - extra=self.log_extra) - def update_certificate(self, signature, certificate=None, data=None): s_os_client = sdk.OpenStackDriver(self.region_name) @@ -869,8 +792,6 @@ class SysinvSyncThread(SyncThread): return self.get_snmp_trapdest_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: return [self.get_remotelogging_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - return [self.get_firewallrules_resource(os_client)] elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: return self.get_certificates_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: @@ -896,8 +817,6 @@ class SysinvSyncThread(SyncThread): return self.get_snmp_trapdest_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: return [self.get_remotelogging_resource(os_client)] - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - return [self.get_firewallrules_resource(os_client)] elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: return self.get_certificates_resources(os_client) elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: @@ -1036,27 +955,6 @@ class SysinvSyncThread(SyncThread): LOG.exception(e) return None - def get_firewallrules_resource(self, os_client): - try: - ifirewallrules = os_client.sysinv_client.get_firewallrules() - return ifirewallrules - except (keystone_exceptions.connection.ConnectTimeout, - keystone_exceptions.ConnectFailure) as e: - LOG.info("get_firewallrules: subcloud {} is not reachable [{}]" - .format(self.subcloud_engine.subcloud.region_name, - str(e)), extra=self.log_extra) - # None will force skip of audit - os_client.delete_region_clients(self.region_name) - return None - except (AttributeError, TypeError) as e: - LOG.info("get_firewallrules_resource error {}".format(e), - extra=self.log_extra) - os_client.delete_region_clients(self.region_name, clear_token=True) - return None - except Exception as e: - LOG.exception(e) - return None - def get_certificates_resources(self, os_client): try: return os_client.sysinv_client.get_certificates() @@ -1130,21 +1028,6 @@ class SysinvSyncThread(SyncThread): "community".format(resource), extra=self.log_extra) return resource.ip_address - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - if hasattr(resource, 'firewall_sig'): - LOG.info("get_resource_id firewall_sig={}".format( - resource.firewall_sig)) - if resource.firewall_sig is None: - return self.FIREWALL_SIG_NULL # master_id cannot be None - return resource.firewall_sig - elif hasattr(resource, 'master_id'): - LOG.info("get_resource_id master_id firewall_sig={}".format( - resource.master_id)) - if resource.master_id is None: - return self.FIREWALL_SIG_NULL # master_id cannot be None - return resource.master_id - else: - LOG.error("no get_resource_id for firewall") elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: if hasattr(resource, 'signature'): LOG.info("get_resource_id signature={}".format( @@ -1234,19 +1117,6 @@ class SysinvSyncThread(SyncThread): i1.transport == i2.transport and i1.port == i2.port) - def same_firewallrules(self, i1, i2): - LOG.debug("same_firewallrules i1={}, i2={}".format(i1, i2), - extra=self.log_extra) - same = True - if i1.firewall_sig and (i1.firewall_sig != i2.firewall_sig): - if i1.firewall_sig == self.FIREWALL_SIG_NULL: - return True - LOG.info("same_firewallrules differ i1={}, i2={}".format(i1, i2), - extra=self.log_extra) - same = False - - return same - def same_certificate(self, i1, i2): LOG.debug("same_certificate i1={}, i2={}".format(i1, i2), extra=self.log_extra) @@ -1296,8 +1166,6 @@ class SysinvSyncThread(SyncThread): return self.same_snmp_trapdest(m_resource, sc_resource) elif resource_type == consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING: return self.same_remotelogging(m_resource, sc_resource) - elif resource_type == consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES: - return self.same_firewallrules(m_resource, sc_resource) elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE: return self.same_certificate(m_resource, sc_resource) elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER: @@ -1363,10 +1231,7 @@ class SysinvSyncThread(SyncThread): resource_id = self.get_resource_id(resource_type, resource) if finding == AUDIT_RESOURCE_MISSING: # default action is create for a 'missing' resource - if resource_id == self.FIREWALL_SIG_NULL: - LOG.info("No custom firewall resource to sync") - return num_of_audit_jobs - elif resource_id == self.CERTIFICATE_SIG_NULL: + if resource_id == self.CERTIFICATE_SIG_NULL: LOG.info("No certificate resource to sync") return num_of_audit_jobs elif resource_id == self.RESOURCE_UUID_NULL: @@ -1396,7 +1261,6 @@ class SysinvSyncThread(SyncThread): consts.RESOURCE_TYPE_SYSINV_SNMP_COMM, consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST, consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING, - consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES, consts.RESOURCE_TYPE_SYSINV_CERTIFICATE, consts.RESOURCE_TYPE_SYSINV_USER, ]