From 4ee20f9e403d9fd25954a63179fa786b838d7d9b Mon Sep 17 00:00:00 2001 From: Jose Perez Carranza Date: Mon, 12 Aug 2019 11:08:41 -0500 Subject: [PATCH] [robot] Add configs and environment setup Enable different configurations and setup requirements for deployment environments as well as pip requirements. - Config : Serie of configuration files to be used on deployment as well as exposed to the framework in order to use the values on the test cases. - Qemu : All artifacts needed to deploy the virtual machines that simulate the nodes on libvirt/qemu - Baremetal : Series of configuration files used to setup a deployment on a baremetal environment - requirements.txt, test-requirements.txt : series of packages needed by the suite to work on a python virtualenv. Depends-On: I796dcaf71089424dd37a050691fd0ee003ad3176 Change-Id: I34dfe1c01b27e5e8ce046de3305dd78c7f9d1383 Signed-off-by: Jose Perez Carranza --- automated-robot-suite/Config/__init__.py | 0 automated-robot-suite/Config/config.ini | 54 ++ automated-robot-suite/Config/config.py | 144 ++++ automated-robot-suite/Config/stx-duplex.yml | 16 + .../Config/stx-multinode.yml | 16 + automated-robot-suite/Config/stx-simplex.yml | 12 + automated-robot-suite/Qemu/README.md | 155 +++++ automated-robot-suite/Qemu/__init__.py | 0 automated-robot-suite/Qemu/compute.xml | 92 +++ .../Qemu/configs/duplex.yaml | 18 + .../configs/multinode_controller_storage.yaml | 29 + .../configs/multinode_dedicated_storage.yaml | 42 ++ .../Qemu/configs/simplex.yaml | 12 + .../Qemu/images/qemu_logo.png | Bin 0 -> 19079 bytes .../Qemu/master_controller.xml | 106 +++ automated-robot-suite/Qemu/nat-network.xml | 7 + automated-robot-suite/Qemu/qemu_setup.py | 638 ++++++++++++++++++ automated-robot-suite/Qemu/qemu_setup.yaml | 32 + .../Qemu/slave_controller.xml | 99 +++ automated-robot-suite/Qemu/storage.xml | 92 +++ .../baremetal/baremetal_setup.yaml | 47 ++ .../baremetal/configs/duplex.yaml | 18 + .../configs/multinode_controller_storage.yaml | 32 + .../configs/multinode_dedicated_storage.yaml | 46 ++ .../baremetal/configs/simplex.yaml | 11 + automated-robot-suite/requirements.txt | 14 + automated-robot-suite/test-requirements.txt | 9 + 27 files changed, 1741 insertions(+) create mode 100644 automated-robot-suite/Config/__init__.py create mode 100644 automated-robot-suite/Config/config.ini create mode 100644 automated-robot-suite/Config/config.py create mode 100644 automated-robot-suite/Config/stx-duplex.yml create mode 100644 automated-robot-suite/Config/stx-multinode.yml create mode 100644 automated-robot-suite/Config/stx-simplex.yml create mode 100644 automated-robot-suite/Qemu/README.md create mode 100644 automated-robot-suite/Qemu/__init__.py create mode 100644 automated-robot-suite/Qemu/compute.xml create mode 100644 automated-robot-suite/Qemu/configs/duplex.yaml create mode 100644 automated-robot-suite/Qemu/configs/multinode_controller_storage.yaml create mode 100644 automated-robot-suite/Qemu/configs/multinode_dedicated_storage.yaml create mode 100644 automated-robot-suite/Qemu/configs/simplex.yaml create mode 100644 automated-robot-suite/Qemu/images/qemu_logo.png create mode 100644 automated-robot-suite/Qemu/master_controller.xml create mode 100644 automated-robot-suite/Qemu/nat-network.xml create mode 100644 automated-robot-suite/Qemu/qemu_setup.py create mode 100644 automated-robot-suite/Qemu/qemu_setup.yaml create mode 100644 automated-robot-suite/Qemu/slave_controller.xml create mode 100644 automated-robot-suite/Qemu/storage.xml create mode 100644 automated-robot-suite/baremetal/baremetal_setup.yaml create mode 100644 automated-robot-suite/baremetal/configs/duplex.yaml create mode 100644 automated-robot-suite/baremetal/configs/multinode_controller_storage.yaml create mode 100644 automated-robot-suite/baremetal/configs/multinode_dedicated_storage.yaml create mode 100644 automated-robot-suite/baremetal/configs/simplex.yaml create mode 100644 automated-robot-suite/requirements.txt create mode 100644 automated-robot-suite/test-requirements.txt diff --git a/automated-robot-suite/Config/__init__.py b/automated-robot-suite/Config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/automated-robot-suite/Config/config.ini b/automated-robot-suite/Config/config.ini new file mode 100644 index 0000000..ffe86a5 --- /dev/null +++ b/automated-robot-suite/Config/config.ini @@ -0,0 +1,54 @@ +[general] +LOG_PATH = /tmp/logs +IP_UNIT_0_ADDRESS = 10.10.10.3 +IP_UNIT_1_ADDRESS = 10.10.10.4 +CIRROS_FILE = cirros-0.4.0-x86_64-disk.qcow2 +CENTOS_FILE = CentOS-7-x86_64-GenericCloud.qcow2 +UBUNTU_FILE = xenial-server-cloudimg-amd64-disk1.qcow2 +WINDOWS_FILE = windows_server_2012_r2.qcow2 +APP_TARBALL = stx-openstack.tgz +STX_ISO_FILE = bootimage.iso +ENV_YAML_FILE = Qemu/qemu_setup.yaml +CONFIGURATION_TYPE = simplex +ENVIRONMENT = virtual +CONFIGURATION_FILE = stx-simplex.yml + +[credentials] +STX_DEPLOY_USER_NAME = sysadmin +STX_DEPLOY_USER_PSWD = St4rlingX* + +[iso_installer] +KERNEL_OPTION = 0 +VIRSH_CMD = virsh -c qemu:///system console controller-0 +VMLINUZ = vmlinuz append rootwait +CONSOLES = console=tty0,115200 console=ttyS0,115200 inst.text +SERIAL = serial +OPTS_1 = inst.stage2=hd:LABEL=oe_iso_boot user_namespace.enable=1 +SYS_TYPE_1 = inst.ks=hd:LABEL=oe_iso_boot:/ks.cfg +SYS_TYPE_2 = inst.ks=hd:LABEL=oe_iso_boot:/smallsystem_ks.cfg +SYS_TYPE_3 = inst.ks=hd:LABEL=oe_iso_boot:/smallsystem_lowlatency_ks.cfg +OPTS_2 = boot_device=sda rootfs_device=sda biosdevname=0 usbcore.autosuspend=-1 inst.gpt +SEC_PROF_1 = security_profile=standard +SEC_PROF_2 = security_profile=extended +INITRD = initrd=initrd.img +BOOT_TIMEOUT = 1200 +CONFIG_CONTROLLER_TIMEOUT = 3600 +CONTROLLER_TMP_IP = 10.10.10.3 +CONTROLLER_TMP_GATEWAY = 10.10.10.1 + +[logical_interface] +OAM = enp2s1 +MGMT = enp2s2 + +[baremetal] +HTTP_SERVER = 192.168.200.3 + +[dashboard] +HORIZON_USERNAME = admin +HORIZON_PASSWORD = St4rlingX* +BROWSER = firefox +PROFILE = nk0f1p65.automation + +[qemu] +XML = /etc/libvirt/qemu/networks/autostart/default.xml +CONFIG_FILE = /etc/libvirt/qemu.conf diff --git a/automated-robot-suite/Config/config.py b/automated-robot-suite/Config/config.py new file mode 100644 index 0000000..a8cb28d --- /dev/null +++ b/automated-robot-suite/Config/config.py @@ -0,0 +1,144 @@ +"""Config file parser and config file generator. + +Configuration values will be read from a file called . +A config file with default values can be generated by running this module as a +script and using the --generate option. + +The config file is a standard INI file that contains [sections] and options. + +Variables that are between %(option)s are interpolated automatically by the +ConfigParser (with the caveat that they have to be in the same section). +Variables that are between ${section:option} will be interpolated with the +value from the appropriate section (similar to the way ConfigParse from +Python3 works). + +Example of a config file: + +E.g. + +[Section1] +key_1 = value 1 +key_2 = value 2 +country = mexico + +[section2] +name = John +lastname = Doe +fullname = %(name)s %(lastname)s +country = ${Section1:country} + +To access config values from other modules follow this approach: + +import Config.config as CONF +full_name = CONF.get('section2', 'fullname') +""" + +from __future__ import print_function + +import argparse +import configparser +import os +import re + + +CONFIG_FILE = os.path.join(os.path.dirname(__file__), 'config.ini') +CONFIG = configparser.SafeConfigParser() + + +def get(section, option, raw=False, variables=None): + """Wrapper function to mimic behavior of ConfigParse (Python3). + + Python3 uses a ConfigParse that is different than the Python2 ConfigParser, + it has some additional functionality. One of the biggest differences is + that with ConfigParse you can interpolate variables from one section into + another section of the config file. This functionality is not included in + the ConfigParse. Since it is very useful, this function wraps the original + get function from ConfigParser so it includes the missing interpolation. + :param section: the section where the option to be gotten is + :param option: the option to look for in the config file + :param raw: all the '%' interpolations are expanded in the return values, + unless the raw argument is true. + :param variables: the option is looked up in variables (if provided) + :return: the specified value from the config file + """ + # ensure file readed is up to date always + CONFIG.read(CONFIG_FILE) + original_option = CONFIG.get(section, option, raw=raw, vars=variables) + # find all matches that have the following pattern: ${something} + matches = re.findall(r'\$\{(.*?)\}', original_option) + new_value = original_option + for match in matches: + # interpolate matches found with ${something} with the referenced + # option from the appropriate section + pattern = '${' + match + '}' + value = match.split(':') + foreign_value = CONFIG.get(value[0], value[1]) + new_value = new_value.replace(pattern, foreign_value) + if original_option != new_value: + CONFIG.set(section, option, new_value) + + # return the interpolated value + return CONFIG.get(section, option, raw=raw, vars=variables) + + +def getint(section, option): + """Wrapper that returns the value as an integer. + + :param section: the section where the option to be gotten is + :param option: the option to look for in the config file + :return: the specified value from the config file + """ + return int(CONFIG.get(section, option)) + + +def getfloat(section, option): + """Wrapper that returns the value as a float. + + :param section: the section where the option to be gotten is + :param option: the option to look for in the config file + :return: the specified value from the config file + """ + return float(CONFIG.get(section, option)) + + +def getboolean(section, option): + """Wrapper that returns the value as a boolean. + + :param section: the section where the option to be gotten is + :param option: the option to look for in the config file + :return: the specified value from the config file + """ + value = CONFIG.get(section, option) + return value.lower() == 'true' + + +def _unload_current_values(): + """Removes current sections from the existing config object""" + for section in CONFIG.sections(): + CONFIG.remove_section(section) + + +def create_config(): + """Creates the config file in the current directory""" + if os.path.isfile(CONFIG_FILE): + os.remove(CONFIG_FILE) + with open(CONFIG_FILE, 'wb') as configfile: + _unload_current_values() + CONFIG.write(configfile) + + +def parse_arguments(): + """Parses arguments from the command line""" + parser = argparse.ArgumentParser( + description='Config file generator') + parser.add_argument( + '--generate', action='store_true', + help='Generates a config file with default values') + return parser.parse_args() + + +if __name__ == '__main__': + ARGUMENTS = parse_arguments() + if ARGUMENTS.generate: + create_config() + print('Config file created at: {name}'.format(name=CONFIG_FILE)) diff --git a/automated-robot-suite/Config/stx-duplex.yml b/automated-robot-suite/Config/stx-duplex.yml new file mode 100644 index 0000000..bf3617d --- /dev/null +++ b/automated-robot-suite/Config/stx-duplex.yml @@ -0,0 +1,16 @@ +system_mode: duplex + +dns_servers: + - 8.8.8.8 + +docker_http_proxy: http://: +docker_https_proxy: http://: + +external_oam_subnet: 10.10.10.0/24 +external_oam_gateway_address: 10.10.10.1 +external_oam_floating_address: 10.10.10.2 +external_oam_node_0_address: 10.10.10.3 +external_oam_node_1_address: 10.10.10.4 + +ansible_become_pass: ANSIBLE_PASS +admin_password: ADMIN_PASS diff --git a/automated-robot-suite/Config/stx-multinode.yml b/automated-robot-suite/Config/stx-multinode.yml new file mode 100644 index 0000000..bf3617d --- /dev/null +++ b/automated-robot-suite/Config/stx-multinode.yml @@ -0,0 +1,16 @@ +system_mode: duplex + +dns_servers: + - 8.8.8.8 + +docker_http_proxy: http://: +docker_https_proxy: http://: + +external_oam_subnet: 10.10.10.0/24 +external_oam_gateway_address: 10.10.10.1 +external_oam_floating_address: 10.10.10.2 +external_oam_node_0_address: 10.10.10.3 +external_oam_node_1_address: 10.10.10.4 + +ansible_become_pass: ANSIBLE_PASS +admin_password: ADMIN_PASS diff --git a/automated-robot-suite/Config/stx-simplex.yml b/automated-robot-suite/Config/stx-simplex.yml new file mode 100644 index 0000000..cca8f24 --- /dev/null +++ b/automated-robot-suite/Config/stx-simplex.yml @@ -0,0 +1,12 @@ +dns_servers: + - 8.8.8.8 + +docker_http_proxy: http://: +docker_https_proxy: http://: + +external_oam_subnet: 10.10.10.0/24 +external_oam_gateway_address: 10.10.10.1 +external_oam_floating_address: 10.10.10.3 + +ansible_become_pass: ANSIBLE_PASS +admin_password: ADMIN_PASS diff --git a/automated-robot-suite/Qemu/README.md b/automated-robot-suite/Qemu/README.md new file mode 100644 index 0000000..d4cc402 --- /dev/null +++ b/automated-robot-suite/Qemu/README.md @@ -0,0 +1,155 @@ +[qemu_logo]: ./images/qemu_logo.png + +![alt text][qemu_logo] + +Table of Contents + +- [qemu_setup.yml](#qemu_setupyml) +- [Description](#description) +- [Examples](#examples) + - [1 controller + 2 compute](#1-controller--2-computes) + - [parameters](#parameters) + - [1 controller + 2 computes & 1 controller + 1 compute](#1-controller--2-computes--1-controller--1-compute) + - [Highlights](#highlights) + - [general_system_configurations section](#general_system_configurations-section) + +# qemu_setup.yml +The purpose of this configurations file is to setup easily QEMU in the host. + +# Description +YAML is a human-readable data serialization format that takes concepts from +programming languages such as C, Perl, and Python, and ideas from XML and the +data format of electronic mail. +It is available for several programming languages. + +# Examples + +## 1 controller + 2 computes +Please consider the following example to setup the following configuration: + +`1 controller + 2 computes` + +```yaml +configuration_0: + controller-0: + controller_0_partition_a: 20 + controller_0_partition_b: 10 + controller_0_memory_size: 5120 + controller_0_system_cores: 2 + controller-0-compute-0: + controller_0_compute_0_partition_a: 20 + controller_0_compute_0_partition_b: 20 + controller_0_compute_0_memory_size: 3072 + controller_0_compute_0_system_cores: 1 + controller-0-compute-1: + controller_0_compute_1_partition_a: 20 + controller_0_compute_1_partition_b: 20 + controller_0_compute_1_memory_size: 3072 + controller_0_compute_1_system_cores: 1 +```` + +### parameters + +**`@param: configuration_0`**: which contains the controller and the computes.
+ +**`@param: controller-0`**: which contains the following:
+- `@param: controller_0_partition_a`: which is the controller 0 partition size + A in GB. +- `@param: controller_0_partition_b`: which is the controller 0 partition size + B size in GB. +- `@param: controller_0_memory_size`: which is the controller 0 memory size + in MB. +- `@param: controller_0_system_cores`: which is the controller 0 system cores + to be assigned. + +**`@param: controller-0-compute-0`**: which contains the following:
+- `@param: controller_0_compute_0_partition_a`: which is the controller's + compute 0 partition size A in GB. +- `@param: controller_0_compute_0_partition_b`: which is the controller's + compute 0 partition size B in GB. +- `@param: controller_0_compute_0_memory_size`: which is the controller's + compute 0 memory size B in MB. +- `@param: controller_0_compute_0_system_cores`: which is the controller's + compute 0 system cores to be assigned. + +**`@param: controller-0-compute-1`**: which contains the following:
+- `@param: controller_0_compute_1_partition_a`: which is the controller's + compute 1 partition size A in GB. +- `@param: controller_0_compute_1_partition_b`: which is the controller's + compute 1 partition size B in GB. +- `@param: controller_0_compute_1_memory_size`: which is the controller's + compute 1 memory size B in MB. +- `@param: controller_0_compute_1_system_cores`: which is the controller's + compute 1 system cores to be assigned. + +## 1 controller + 2 computes & 1 controller + 1 compute +Please consider the following example to setup the following configuration: + +`1 controller + 2 computes & 1 controller + 1 compute` + +```yaml +configuration_0: + controller-0: + controller_0_partition_a: 20 + controller_0_partition_b: 10 + controller_0_memory_size: 5120 + controller_0_system_cores: 2 + controller-0-compute-0: + controller_0_compute_0_partition_a: 20 + controller_0_compute_0_partition_b: 20 + controller_0_compute_0_memory_size: 3072 + controller_0_compute_0_system_cores: 1 + controller-0-compute-1: + controller_0_compute_1_partition_a: 20 + controller_0_compute_1_partition_b: 20 + controller_0_compute_1_memory_size: 3072 + controller_0_compute_1_system_cores: 1 +configuration_1: + controller-1: + controller_1_partition_a: 15 + controller_1_partition_b: 10 + controller_1_memory_size: 5120 + controller_1_system_cores: 2 + controller-1-compute-0: + controller_1_compute_0_partition_a: 20 + controller_1_compute_0_partition_b: 20 + controller_1_compute_0_memory_size: 3072 + controller_1_compute_0_system_cores: 1 +``` + +:notebook: the parameters description are the same that the [section above](#parameters). + +## Highlights +Please consider the following when creating the yaml configuration file: + +- The total sum of the partitions must not exceed of the total free disk space + in the host. +- The total sum of the memory size must not exceed of the total free memory + size in the host. +- The total sum of the system cores must not exceed of the total system cores + subtracting the `os_system_cores` assigned in `general_system_configurations` + section. + + +## general_system_configurations section +The general_system_configurations will be explained below: + +```yaml +general_system_configurations: + os_system_memory: 1024 + disk_space_allocated_to_os: 20 + os_system_cores: 2 + default_mount_point: '/' +``` + +**`@param: general_system_configurations`**: which contains the following: +- `@param: os_system_memory`: which is the system memory reserved for the OS. +- `@param: disk_space_allocated_to_os`: which is the disk space reserved for + the OS. +- `@param: os_system_cores`: which is the system cores reserved for the OS. +- `@param: default_mount_point`: which is the mount point where the space in + disk will be analyzed by the script. + +:notebook: The first 3 params are used for system reservation and they are for +the user's consideration in order to avoid that QEMU takes all the resources +making slow the current system. \ No newline at end of file diff --git a/automated-robot-suite/Qemu/__init__.py b/automated-robot-suite/Qemu/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/automated-robot-suite/Qemu/compute.xml b/automated-robot-suite/Qemu/compute.xml new file mode 100644 index 0000000..e5f6522 --- /dev/null +++ b/automated-robot-suite/Qemu/compute.xml @@ -0,0 +1,92 @@ + + NAME + MEMORY + MEMORY + CORES + + /machine + + + hvm + + + + + + + + Nehalem + + + + + + destroy + restart + destroy + + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +