starlingx
/
config
Code Issues Proposed changes

Merge "PTP Configuration Enhancements"

This commit is contained in:
Zuul 2020-02-11 22:03:08 +00:00 committed by Gerrit Code Review
parent f2adc88da8 f1605d465b
commit f34ceeb0f5
9 changed files with 467 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/service_parameter.py
View File

@ -237,7 +237,8 @@ class ServiceParameterController(rest.RestController):
schema = service_parameter.SERVICE_PARAMETER_SCHEMA[service][section]
parameters = (schema.get(service_parameter.SERVICE_PARAM_MANDATORY, []) +
schema.get(service_parameter.SERVICE_PARAM_OPTIONAL, []))
if name not in parameters:
has_wildcard = (constants.SERVICE_PARAM_NAME_WILDCARD in parameters)
if name not in parameters and not has_wildcard:
msg = _("The parameter name %s is invalid for "
"service %s section %s"
% (name, service, section))

23
sysinv/sysinv/sysinv/sysinv/common/constants.py
View File

@ -931,6 +931,12 @@ SERVICE_TYPE_DOCKER = 'docker'
SERVICE_TYPE_HTTP = 'http'
SERVICE_TYPE_OPENSTACK = 'openstack'
SERVICE_TYPE_KUBERNETES = 'kubernetes'
SERVICE_TYPE_PTP = 'ptp'
# For service parameter sections that include a wildcard, any 'name' field will be
# allowed by the API. The wildcard card name will only be matched if no other matches
# are found first.
SERVICE_PARAM_NAME_WILDCARD = '*wildcard*'
SERVICE_PARAM_SECTION_IDENTITY_CONFIG = 'config'
@ -1037,6 +1043,22 @@ DEFAULT_REGISTRIES_INFO = {
SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES = 'certificates'
SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST = 'apiserver_certsan'
# ptp service parameters
SERVICE_PARAM_SECTION_PTP_GLOBAL = 'global'
SERVICE_PARAM_SECTION_PTP_PHC2SYS = 'phc2sys'
SERVICE_PARAM_NAME_PTP_UPDATE_RATE = 'update-rate'
SERVICE_PARAM_NAME_PTP_SUMMARY_UPDATES = 'summary-updates'
PTP_PHC2SYS_DEFAULTS = {
SERVICE_PARAM_NAME_PTP_UPDATE_RATE: 10,
SERVICE_PARAM_NAME_PTP_SUMMARY_UPDATES: 600
}
PTP_PHC2SYS_OPTIONS_MAP = {
SERVICE_PARAM_NAME_PTP_UPDATE_RATE: 'R',
SERVICE_PARAM_NAME_PTP_SUMMARY_UPDATES: 'u'
}
# default filesystem size to 25 MB
SERVICE_PARAM_RADOSGW_FS_SIZE_MB_DEFAULT = 25
@ -1528,6 +1550,7 @@ CLOCK_SYNCHRONIZATION = [
# PTP transport modes
PTP_TRANSPORT_UDP = 'udp'
PTP_TRANSPORT_L2 = 'l2'
PTP_NETWORK_TRANSPORT_IEEE_802_3 = 'L2'
# Backup & Restore
FIX_INSTALL_UUID_INTERVAL_SECS = 30

29
sysinv/sysinv/sysinv/sysinv/common/service_parameter.py
View File

@ -545,6 +545,25 @@ OPENSTACK_HELM_PARAMETER_RESOURCE = {
'openstack::helm::params::endpoint_domain',
}
PTP_GLOBAL_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_WILDCARD
]
PTP_GLOBAL_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_WILDCARD: _validate_not_empty
}
PTP_PHC2SYS_PARAMETER_OPTIONAL = [
constants.SERVICE_PARAM_NAME_PTP_UPDATE_RATE,
constants.SERVICE_PARAM_NAME_PTP_SUMMARY_UPDATES
]
PTP_PHC2SYS_PARAMETER_VALIDATOR = {
constants.SERVICE_PARAM_NAME_PTP_UPDATE_RATE: _validate_float,
# phc2sys summary-updates accepts a range of 0 to UNIT_MAX (ie 2^32 - 1)
constants.SERVICE_PARAM_NAME_PTP_SUMMARY_UPDATES: lambda name, value: _validate_range(name, value, 0, 2 ** 32 - 1)
}
# Service Parameter Schema
SERVICE_PARAM_MANDATORY = 'mandatory'
SERVICE_PARAM_OPTIONAL = 'optional'
@ -629,6 +648,16 @@ SERVICE_PARAMETER_SCHEMA = {
SERVICE_PARAM_DATA_FORMAT: KUBERNETES_CERTIFICATES_PARAMETER_DATA_FORMAT,
},
},
constants.SERVICE_TYPE_PTP: {
constants.SERVICE_PARAM_SECTION_PTP_GLOBAL: {
SERVICE_PARAM_OPTIONAL: PTP_GLOBAL_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: PTP_GLOBAL_PARAMETER_VALIDATOR
},
constants.SERVICE_PARAM_SECTION_PTP_PHC2SYS: {
SERVICE_PARAM_OPTIONAL: PTP_PHC2SYS_PARAMETER_OPTIONAL,
SERVICE_PARAM_VALIDATOR: PTP_PHC2SYS_PARAMETER_VALIDATOR
},
},
constants.SERVICE_TYPE_HTTP: {
constants.SERVICE_PARAM_SECTION_HTTP_CONFIG: {
SERVICE_PARAM_OPTIONAL: HTTPD_PORT_PARAMETER_OPTIONAL,

12
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -5612,10 +5612,18 @@ class ConductorManager(service.PeriodicService):
def update_ptp_config(self, context):
"""Update the PTP configuration"""
self._update_ptp_host_configs(context)
def _update_ptp_host_configs(self, context):
"""Issue config updates to hosts with ptp clocks"""
personalities = [constants.CONTROLLER,
constants.WORKER,
constants.STORAGE]
self._config_update_hosts(context, personalities)
hosts = self.dbapi.ihost_get_list()
ptp_hosts = [host.uuid for host in hosts if host.clock_synchronization == constants.PTP]
if ptp_hosts:
self._config_update_hosts(context, personalities, host_uuids=ptp_hosts, reboot=True)
def update_system_mode_config(self, context):
"""Update the system mode configuration"""
@ -7320,6 +7328,8 @@ class ConductorManager(service.PeriodicService):
elif service == constants.SERVICE_TYPE_OPENSTACK:
# Do nothing. Does not need to update target config of any hosts
pass
elif service == constants.SERVICE_TYPE_PTP:
self._update_ptp_host_configs(context)
else:
# All other services
personalities = [constants.CONTROLLER]

33
sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/050_consolidated_r4.py
View File

@ -9,7 +9,7 @@ from eventlet.green import subprocess
import json
import tsconfig.tsconfig as tsconfig
from migrate.changeset import UniqueConstraint
from sqlalchemy import Boolean, DateTime, Enum, Integer, String, Text
from sqlalchemy import Boolean, DateTime, Integer, String, Text
from sqlalchemy import Column, ForeignKey, MetaData, Table
from sqlalchemy.dialects import postgresql
@ -101,17 +101,26 @@ def upgrade(migrate_engine):
primary_key=True, nullable=False),
mysql_engine=ENGINE, mysql_charset=CHARSET,
autoload=True)
if migrate_engine.url.get_dialect() is postgresql.dialect:
old_serviceEnum = Enum('identity',
'horizon',
'ceph',
'network',
name='serviceEnum')
service_col = service_parameter.c.service
service_col.alter(Column('service', String(16)))
old_serviceEnum.drop(bind=migrate_engine, checkfirst=False)
service_parameter.drop()
meta.remove(service_parameter)
service_parameter = Table(
'service_parameter',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('deleted_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(36), unique=True),
Column('service', String(16)),
Column('section', String(255)),
Column('name', String(255)),
Column('value', String(255)),
UniqueConstraint('service', 'section', 'name',
name='u_servicesectionname'),
mysql_engine=ENGINE,
mysql_charset=CHARSET,
)
service_parameter.create(migrate_engine, checkfirst=False)
# 049_add_controllerfs_scratch.py
controller_fs = Table('controller_fs', meta, autoload=True)

61
sysinv/sysinv/sysinv/sysinv/puppet/platform.py
View File

@ -431,16 +431,61 @@ class PlatformPuppet(base.BasePuppet):
ptp_enabled = True
else:
ptp_enabled = False
return {'platform::ptp::enabled': ptp_enabled}
ptp_config = {
'tx_timestamp_timeout': '20',
'summary_interval': '6',
'clock_servo': 'linreg',
'delay_mechanism': ptp.mechanism.upper(),
'time_stamping': ptp.mode.lower()
}
if ptp.mode.lower() == 'hardware':
ptp_config.update({'boundary_clock_jbod': '1'})
ptp_service_params = self.dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_PTP, section=constants.SERVICE_PARAM_SECTION_PTP_GLOBAL)
# Merge options specified in service parameters with ptp database values and defaults
for param in ptp_service_params:
ptp_config.update({param.name: param.value})
transport = constants.PTP_TRANSPORT_L2
specified_transport = ptp_config.get('network_transport')
if specified_transport:
# Currently we can only set the network transport globally. Setting the transport flag
# to udp will force puppet to apply the correct UDP family to each interface
if specified_transport != constants.PTP_NETWORK_TRANSPORT_IEEE_802_3:
transport = constants.PTP_TRANSPORT_UDP
else:
ptp_config.update({'network_transport': constants.PTP_NETWORK_TRANSPORT_IEEE_802_3})
transport = ptp.transport
# Generate ptp4l global options
ptp4l_options = []
for key, value in ptp_config.items():
ptp4l_options.append({'name': key, 'value': value})
# Get the options for the phc2sys system
phc2sys_config = constants.PTP_PHC2SYS_DEFAULTS
phc2sys_service_params = self.dbapi.service_parameter_get_all(
service=constants.SERVICE_TYPE_PTP,
section=constants.SERVICE_PARAM_SECTION_PTP_PHC2SYS)
for param in phc2sys_service_params:
phc2sys_config.update({param.name: param.value})
phc2sys_options = ''
for key, value in phc2sys_config.items():
phc2sys_options += '-' + constants.PTP_PHC2SYS_OPTIONS_MAP[key] + ' ' + str(value) + ' '
return {
'platform::ptp::enabled':
ptp_enabled,
'platform::ptp::mode':
ptp.mode,
'platform::ptp::transport':
ptp.transport,
'platform::ptp::mechanism':
ptp.mechanism,
'platform::ptp::enabled': ptp_enabled,
'platform::ptp::transport': transport,
'platform::ptp::ptp4l_options': ptp4l_options,
'platform::ptp::phc2sys_options': phc2sys_options
}
def _get_host_sysctl_config(self, host):

5
sysinv/sysinv/sysinv/sysinv/tests/api/base.py
View File

@ -117,12 +117,13 @@ class FunctionalTest(base.TestCase):
return self.post_json(path, expect_errors=expect_errors,
headers=headers, **newargs)
def patch_dict(self, path, data, expect_errors=False):
def patch_dict(self, path, data, expect_errors=False, headers=None):
params = []
for key, value in data.items():
pathkey = '/' + key
params.append({'op': 'replace', 'path': pathkey, 'value': value})
return self.post_json(path, expect_errors=expect_errors, params=params, method='patch')
return self.post_json(path, expect_errors=expect_errors, params=params,
method='patch', headers=headers)
def delete(self, path, expect_errors=False, headers=None,
extra_environ=None, status=None, path_prefix=PATH_PREFIX):

302
sysinv/sysinv/sysinv/sysinv/tests/api/test_service_parameters.py Normal file
View File

@ -0,0 +1,302 @@
#
# Copyright (c) 2019 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
"""
Tests for the API / service_parameter / methods.
"""
from six.moves import http_client
from oslo_utils import uuidutils
from sysinv.common import constants
from sysinv.tests.api import base
from sysinv.tests.db import base as dbbase
from sysinv.tests.db import utils as dbutils
class ApiServiceParameterTestCaseMixin(object):
# API_HEADERS are a generic header passed to most API calls
API_HEADERS = {'User-Agent': 'sysinv-test',
'Content-Type': 'application/json',
'Accept': 'application/json'}
# API_PREFIX is the prefix for the URL
API_PREFIX = '/service_parameter'
# RESULT_KEY is the python table key for the list of results
RESULT_KEY = 'parameters'
# expected_api_fields are attributes that should be populated by
# an API query
expected_api_fields = ['uuid',
'service',
'section',
'name',
'value',
'resource',
'personality'
]
required_post_fields = [
'service',
'section',
'parameters'
'resource',
'personality'
]
# hidden_api_fields are attributes that should not be populated by
# an API query
hidden_api_fields = []
service_parameter_data = [
{
'service': constants.SERVICE_TYPE_HTTP,
'section': constants.SERVICE_PARAM_SECTION_HTTP_CONFIG,
'name': constants.SERVICE_PARAM_HTTP_PORT_HTTP,
'value': str(constants.SERVICE_PARAM_HTTP_PORT_HTTP_DEFAULT)
},
{
'service': constants.SERVICE_TYPE_HTTP,
'section': constants.SERVICE_PARAM_SECTION_HTTP_CONFIG,
'name': constants.SERVICE_PARAM_HTTP_PORT_HTTPS,
'value': str(constants.SERVICE_PARAM_HTTP_PORT_HTTPS_DEFAULT)
},
{
'service': constants.SERVICE_TYPE_KUBERNETES,
'section': constants.SERVICE_PARAM_SECTION_KUBERNETES_CERTIFICATES,
'name': constants.SERVICE_PARAM_NAME_KUBERNETES_API_SAN_LIST,
'value': 'localurl'
}
]
service_parameter_wildcard = {
'service': constants.SERVICE_TYPE_PTP,
'section': constants.SERVICE_PARAM_SECTION_PTP_GLOBAL,
'name': 'network_transport',
'value': 'L2'
}
def setUp(self):
super(ApiServiceParameterTestCaseMixin, self).setUp()
def get_single_url(self, uuid):
return '%s/%s' % (self.API_PREFIX, uuid)
# These methods have generic names and are overridden here
# Future activity: Redo the subclasses to use mixins
def assert_fields(self, api_object):
# check the uuid is a uuid
assert(uuidutils.is_uuid_like(api_object['uuid']))
# Verify that expected attributes are returned
for field in self.expected_api_fields:
self.assertIn(field, api_object)
# Verify that hidden attributes are not returned
for field in self.hidden_api_fields:
self.assertNotIn(field, api_object)
def _create_db_object(self, parameter_data=None):
if not parameter_data:
parameter_data = self.service_parameter_data[0]
return dbutils.create_test_service_parameter(**parameter_data)
def _create_db_objects(self, data_set=None):
if not data_set:
data_set = self.service_parameter_data
data = []
for parameter_data in data_set:
data.append(self._create_db_object(parameter_data))
return data
def get_one(self, uuid, expect_errors=False, error_message=None):
response = self.get_json(self.get_single_url(uuid), headers=self.API_HEADERS)
self.validate_response(response, expect_errors, error_message, json_response=True)
return response
def get_list(self):
response = self.get_json(self.API_PREFIX, headers=self.API_HEADERS)
return response[self.RESULT_KEY]
def patch(self, uuid, data, expect_errors=False, error_message=None):
response = self.patch_dict(self.get_single_url(uuid),
data=data,
expect_errors=expect_errors,
headers=self.API_HEADERS)
self.validate_response(response, expect_errors, error_message)
if expect_errors:
return response
else:
return response.json
def post(self, data, expect_errors=False, error_message=None):
formatted_data = self.format_data(data)
response = self.post_json(self.API_PREFIX,
params=formatted_data,
expect_errors=expect_errors,
headers=self.API_HEADERS)
self.validate_response(response, expect_errors, error_message)
if expect_errors:
return response
else:
return response.json[self.RESULT_KEY][0]
def validate_response(self, response, expect_errors, error_message, json_response=False):
if expect_errors:
self.assertEqual(http_client.BAD_REQUEST, response.status_int)
self.assertEqual('application/json', response.content_type)
if error_message:
self.assertIn(error_message, response.json['error_message'])
elif not json_response:
self.assertEqual(http_client.OK, response.status_int)
def validate_data(self, input_data, response_data):
self.assert_fields(response_data)
for key, value in input_data.items():
if key in self.expected_api_fields:
self.assertEqual(value, response_data[key])
def format_data(self, data):
formatted_data = dict(data)
formatted_data.update({'parameters': {data['name']: data['value']}})
for field in self.required_post_fields:
if field not in formatted_data:
formatted_data[field] = None
return formatted_data
class ApiServiceParameterPostTestSuiteMixin(ApiServiceParameterTestCaseMixin):
def setUp(self):
super(ApiServiceParameterPostTestSuiteMixin, self).setUp()
def test_create_success(self):
# Test creation of object
post_object = self.service_parameter_data[0]
response = self.post(post_object)
self.validate_data(post_object, response)
def test_create_invalid_service(self):
# Test creation with an invalid service name
post_object = dict(self.service_parameter_data[0])
post_object.update({'service': 'not_valid'})
self.post(post_object, expect_errors=True, error_message="Invalid service name")
def test_create_wildcard_success(self):
# Test creation of a section that allows wildcard parameter names
post_object = self.service_parameter_wildcard
response = self.post(post_object)
self.validate_data(post_object, response)
class ApiServiceParameterDeleteTestSuiteMixin(ApiServiceParameterTestCaseMixin):
""" Tests deletion.
Typically delete APIs return NO CONTENT.
python2 and python3 libraries may return different
content_type (None, or empty json) when NO_CONTENT returned.
"""
def setUp(self):
super(ApiServiceParameterDeleteTestSuiteMixin, self).setUp()
self.delete_object = self._create_db_object()
# Delete an object and ensure it is removed
def test_delete(self):
# Delete the API object
uuid = self.delete_object.uuid
response = self.delete(self.get_single_url(uuid),
headers=self.API_HEADERS)
self.assertEqual(response.status_code, http_client.NO_CONTENT)
# Verify the object is no longer returned
results = self.get_list()
returned_uuids = (result.uuid for result in results)
self.assertNotIn(uuid, returned_uuids)
class ApiServiceParameterListTestSuiteMixin(ApiServiceParameterTestCaseMixin):
""" list operations """
def test_empty_list(self):
results = self.get_list()
self.assertEqual([], results)
def test_single_entry(self):
# create a single object
single_object = self._create_db_object()
uuid = single_object.uuid
response = self.get_json(self.get_single_url(uuid))
self.validate_data(single_object, response)
def test_many_entries_in_list(self):
db_obj_list = self._create_db_objects()
response = self.get_list()
# Verify that the input data is found in the result
response_map = {}
for api_object in response:
response_map[api_object['uuid']] = api_object
for db_oject in db_obj_list:
self.validate_data(db_oject, response_map[db_oject.uuid])
class ApiServiceParameterPatchTestSuiteMixin(ApiServiceParameterTestCaseMixin):
def setUp(self):
super(ApiServiceParameterPatchTestSuiteMixin, self).setUp()
self.patch_object = self._create_db_object()
def test_patch_valid(self):
# Update value of patchable field
new_data = {'value': '8077'}
response = self.patch(self.patch_object.uuid, new_data)
# Verify that the attribute was updated
self.patch_object.update(new_data)
self.validate_data(self.patch_object, response)
def test_patch_invalid_value(self):
# Pass a value that fails a semantic check when patched by the API
new_data = {'value': 'a_string'}
self.patch(self.patch_object.uuid, new_data, expect_errors=True,
error_message="must be an integer value")
def test_patch_wildcard_success(self):
# Test modification of a section that allows wildcard parameter names
wildcard_object = self._create_db_object(self.service_parameter_wildcard)
new_data = {'value': 'UDPv4'}
response = self.patch(wildcard_object.uuid, new_data)
wildcard_object.update(new_data)
self.validate_data(wildcard_object, response)
class PlatformIPv4ControllerApiServiceParameterDeleteTestCase(ApiServiceParameterDeleteTestSuiteMixin,
base.FunctionalTest,
dbbase.ProvisionedControllerHostTestCase):
pass
class PlatformIPv4ControllerApiServiceParameterListTestCase(ApiServiceParameterListTestSuiteMixin,
base.FunctionalTest,
dbbase.ProvisionedControllerHostTestCase):
pass
class PlatformIPv4ControllerApiServiceParameterPostTestCase(ApiServiceParameterPostTestSuiteMixin,
base.FunctionalTest,
dbbase.ProvisionedControllerHostTestCase):
pass
class PlatformIPv4ControllerApiServiceParameterPatchTestCase(ApiServiceParameterPatchTestSuiteMixin,
base.FunctionalTest,
dbbase.ProvisionedControllerHostTestCase):
pass

23
sysinv/sysinv/sysinv/sysinv/tests/db/utils.py
View File

@ -1347,6 +1347,29 @@ def create_test_label(**kw):
return dbapi.label_create(label['host_id'], label)
def get_test_service_parameter(**kw):
service_parameter = {
'section': kw.get('section'),
'service': kw.get('service'),
'name': kw.get('name'),
'value': kw.get('value'),
'resource': kw.get('resource'),
'personality': kw.get('personality'),
}
return service_parameter
def create_test_service_parameter(**kw):
"""Create test service parameter in DB and return a service_parameter object.
Function to be used to create test service parameter objects in the database.
:param kw: kwargs with overriding values for service parameter's attributes.
:returns: Test service parameter DB object.
"""
service_parameter = get_test_service_parameter(**kw)
dbapi = db_api.get_instance()
return dbapi.service_parameter_create(service_parameter)
def create_test_oam(**kw):
dbapi = db_api.get_instance()
return dbapi.iextoam_get_one()