From 25a3a344ded7d687c6d574380f70174be2ed4d9f Mon Sep 17 00:00:00 2001 From: Yvonne Ding Date: Mon, 9 Mar 2020 13:05:28 -0400 Subject: [PATCH] Submission for starlingx pytest framework Include: - added automated test cases for k8s hugepage, qos, large amount of pods on workers - added openstack check in test_openstack_services - added test yaml file for k8s kugepage, qos, large amount of pods on workers Story: 2007406 Task: 39012 Author: Ayyappa Mantri Co-Authored-By: Yvonne Ding Signed-off-by: Yvonne Ding Change-Id: I5f795beec2218afae74b9cba356d74cd34d11295 --- automated-pytest-suite/consts/stx.py | 1 + .../keywords/system_helper.py | 30 +++++ .../z_containers/test_hugepage_pod.py | 115 ++++++++++++++++++ .../z_containers/test_openstack_services.py | 23 +++- .../functional/z_containers/test_qos_pod.py | 65 ++++++++++ .../z_containers/test_scale_pods.py | 69 +++++++++++ .../testfixtures/pre_checks_and_configs.py | 8 +- .../utils/test_files/hugepages_pod.yaml | 23 ++++ .../utils/test_files/qos_deployment.yaml | 62 ++++++++++ .../utils/test_files/rc_deployment.yaml | 43 +++++++ 10 files changed, 432 insertions(+), 7 deletions(-) create mode 100644 automated-pytest-suite/testcases/functional/z_containers/test_hugepage_pod.py create mode 100644 automated-pytest-suite/testcases/functional/z_containers/test_qos_pod.py create mode 100644 automated-pytest-suite/testcases/functional/z_containers/test_scale_pods.py create mode 100644 automated-pytest-suite/utils/test_files/hugepages_pod.yaml create mode 100644 automated-pytest-suite/utils/test_files/qos_deployment.yaml create mode 100644 automated-pytest-suite/utils/test_files/rc_deployment.yaml diff --git a/automated-pytest-suite/consts/stx.py b/automated-pytest-suite/consts/stx.py index f1fa652..c155396 100755 --- a/automated-pytest-suite/consts/stx.py +++ b/automated-pytest-suite/consts/stx.py @@ -659,6 +659,7 @@ class PodStatus: POD_INIT = 'PodInitializing' INIT = 'Init:0/1' PENDING = 'Pending' + TERMINATING = 'Terminating' class AppStatus: diff --git a/automated-pytest-suite/keywords/system_helper.py b/automated-pytest-suite/keywords/system_helper.py index 3bd86cb..7200e8d 100644 --- a/automated-pytest-suite/keywords/system_helper.py +++ b/automated-pytest-suite/keywords/system_helper.py @@ -183,6 +183,36 @@ def get_computes(administrative=None, operational=None, availability=None, auth_info=auth_info) +def get_hypervisors(administrative=None, operational=None, + availability=None, con_ssh=None, + auth_info=Tenant.get('admin_platform')): + """ + Get nodes that can be used as hypervisor/worker. + e.g., in standard config, it will mean worker nodes. In DX+worker config, it will mean worker + nodes and controller nodes. + + Args: + administrative: + operational: + availability: + con_ssh: + auth_info: + + Returns (list): + + """ + computes = get_computes(administrative=administrative, operational=operational, + availability=availability, con_ssh=con_ssh, + auth_info=auth_info) + if is_aio_system(controller_ssh=con_ssh, + auth_info=auth_info): + computes += get_controllers(administrative=administrative, operational=operational, + availability=availability, + con_ssh=con_ssh, auth_info=auth_info) + + return computes + + def get_hosts(personality=None, administrative=None, operational=None, availability=None, hostname=None, strict=True, exclude=False, con_ssh=None, diff --git a/automated-pytest-suite/testcases/functional/z_containers/test_hugepage_pod.py b/automated-pytest-suite/testcases/functional/z_containers/test_hugepage_pod.py new file mode 100644 index 0000000..17ce5e1 --- /dev/null +++ b/automated-pytest-suite/testcases/functional/z_containers/test_hugepage_pod.py @@ -0,0 +1,115 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import yaml + +from pytest import fixture + +from utils import cli +from utils.tis_log import LOG + +from consts.proj_vars import ProjVar +from consts.auth import HostLinuxUser +from consts.auth import Tenant +from keywords import common +from keywords import host_helper +from keywords import kube_helper +from keywords import system_helper +from testfixtures.recover_hosts import HostsToRecover + + +def modify_yaml(file_dir, file_name, str_to_add, hugepage_value): + """ + Add hugepages value to hugepages_pod.yaml file + Args: + file_dir(str): deployment file directory + file_name(str): deployment file name + str_to_add(str): 2M or 1G hugepage to add + hugepage_value(str): hugepage value to assign to str_to_add + Return(str): + returns the file_dir and filename with modified values + """ + with open("{}/{}".format(file_dir, file_name), 'r') as f: + data = yaml.safe_load(f) + data['spec']['containers'][0]['resources']['limits'][str_to_add] = hugepage_value + newfile = "hugepages_pod_{}.yaml".format(hugepage_value) + with open("{}/{}".format(ProjVar.get_var('LOG_DIR'), newfile), 'w') as f: + yaml.dump(data, f) + return ProjVar.get_var('LOG_DIR'), newfile + + +@fixture(scope="module") +def get_hugepage_pod_file(): + """ + Fixture used to return the hugepage deployment file + + - Get the compute-0 if exist, else standby controller + - Check 2M hugepages configured, elsif check 1G is configured + else lock,configure 2G of 1G hugepages and unlock host + - Call modify_yaml function to modify the yaml + file with the values + - Modified file scps to host to deploy hugepages pod + - Deletes the hugepages pod from the host after the test + + """ + if system_helper.is_aio_duplex(): + hostname = system_helper.get_standby_controller_name() + else: + hostname = system_helper.get_hypervisors()[0] + LOG.fixture_step("Checking hugepage values on {}".format(hostname)) + proc_id = 0 + out = host_helper.get_host_memories( + hostname, ('app_hp_avail_2M', 'app_hp_avail_1G'), proc_id) + if out[proc_id][0] > 0: + hugepage_val = "{}Mi".format(out[proc_id][0]) + hugepage_str = "hugepages-2Mi" + elif out[proc_id][1] > 0: + hugepage_val = "{}Gi".format(out[proc_id][1]) + hugepage_str = "hugepages-1Gi" + else: + hugepage_val = "{}Gi".format(2) + cmd = "{} -1G {}".format(proc_id, 2) + hugepage_str = "hugepages-1Gi" + HostsToRecover.add(hostname) + host_helper.lock_host(hostname) + LOG.fixture_step("Configuring hugepage values {} on {}".format( + hugepage_val, hostname)) + cli.system('host-memory-modify {} {}'.format(hostname, cmd), ssh_client=None, + auth_info=Tenant.get('admin_platform')) + host_helper.unlock_host(hostname) + LOG.fixture_step("{} {} pod will be configured on {} proc id {}".format( + hugepage_str, hugepage_val, hostname, proc_id)) + file_dir, file_name = modify_yaml( + "utils/test_files/", "hugepages_pod.yaml", hugepage_str, hugepage_val) + source_path = "{}/{}".format(file_dir, file_name) + home_dir = HostLinuxUser.get_home() + common.scp_from_localhost_to_active_controller( + source_path, dest_path=home_dir) + yield file_name + LOG.fixture_step("Delete hugepages pod") + kube_helper.delete_resources( + resource_names="hugepages-pod") + + +def test_hugepage_pod(get_hugepage_pod_file): + """ + Verify hugepage pod is deployed and running + Args: + get_hugepage_pod_file: module fixture + + Steps: + - Create hugepage pod with deployment file + - Verifies hugepage pod is deployed and running + + Teardown: + - Deletes the hugepages pod from the host + """ + LOG.tc_step("Create hugepage pod with deployment file") + kube_helper.exec_kube_cmd( + sub_cmd="create -f {}".format(get_hugepage_pod_file)) + LOG.tc_step("Verifies hugepage pod is deployed and running") + kube_helper.wait_for_pods_status( + pod_names="hugepages-pod", namespace="default") diff --git a/automated-pytest-suite/testcases/functional/z_containers/test_openstack_services.py b/automated-pytest-suite/testcases/functional/z_containers/test_openstack_services.py index 74b0532..357b10a 100644 --- a/automated-pytest-suite/testcases/functional/z_containers/test_openstack_services.py +++ b/automated-pytest-suite/testcases/functional/z_containers/test_openstack_services.py @@ -15,12 +15,23 @@ from utils.tis_log import LOG def get_valid_controllers(): - controllers = system_helper.get_controllers( - availability=(HostAvailState.AVAILABLE, HostAvailState.DEGRADED, - HostAvailState.ONLINE)) + controllers = system_helper.get_controllers(availability=(HostAvailState.AVAILABLE, + HostAvailState.DEGRADED, + HostAvailState.ONLINE)) return controllers +@fixture(scope='module', autouse=True) +def check_openstack(stx_openstack_required): + pass + + +@fixture(scope='module') +def check_nodes(): + if kube_helper.get_nodes(status='Ready', field='NAME', exclude=True, fail_ok=True): + skip('Not all nodes are ready. Skip stx-openstack re-apply test.') + + def check_openstack_pods_healthy(host, timeout): with host_helper.ssh_to_host(hostname=host) as con_ssh: kube_helper.wait_for_pods_healthy(namespace='stx-openstack', @@ -63,10 +74,10 @@ def test_openstack_services_healthy(): 'controller-0', 'controller-1' ]) -def test_reapply_stx_openstack_no_change(stx_openstack_required, controller): +def test_reapply_stx_openstack_no_change(stx_openstack_applied_required, check_nodes, controller): """ Args: - stx_openstack_required: + stx_openstack_applied_required: Pre-requisite: - stx-openstack application in applied state @@ -193,7 +204,7 @@ def reset_if_modified(request): @mark.sanity @mark.sx_sanity @mark.cpe_sanity -def test_stx_openstack_helm_override_update_and_reset(reset_if_modified): +def test_stx_openstack_helm_override_update_and_reset(check_nodes, reset_if_modified): """ Test helm override for openstack nova chart and reset Args: diff --git a/automated-pytest-suite/testcases/functional/z_containers/test_qos_pod.py b/automated-pytest-suite/testcases/functional/z_containers/test_qos_pod.py new file mode 100644 index 0000000..5338234 --- /dev/null +++ b/automated-pytest-suite/testcases/functional/z_containers/test_qos_pod.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2020 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import json + +from pytest import fixture +from pytest import mark + +from utils.tis_log import LOG +from keywords import common +from keywords import kube_helper +from consts.auth import HostLinuxUser + + +@fixture(scope='module') +def copy_pod_yamls(): + home_dir = HostLinuxUser.get_home() + filename = "qos_deployment.yaml" + ns = "qos" + LOG.fixture_step("Copying deployment yaml file") + common.scp_from_localhost_to_active_controller( + source_path="utils/test_files/{}".format(filename), dest_path=home_dir) + kube_helper.exec_kube_cmd( + sub_cmd="create -f {}".format(filename)) + yield ns + LOG.fixture_step("Delete all pods in namespace {}".format(ns)) + kube_helper.exec_kube_cmd( + sub_cmd="delete pods --all --namespace={}".format(ns)) + LOG.fixture_step("Delete the namespace") + kube_helper.exec_kube_cmd(sub_cmd="delete namespace {}".format(ns)) + + +@mark.parametrize('expected,pod', [("guaranteed", "qos-pod-1"), + ("burstable", "qos-pod-2"), + ("besteffort", "qos-pod-3"), + ("burstable", "qos-pod-with-two-containers")]) +def test_qos_class(copy_pod_yamls, expected, pod): + """ + Testing the Qos class for pods + Args: + copy_pod_yamls : module fixture + expected : test param + pod : test param + Setup: + - Scp qos pod yaml files(module) + - Create the deployment of namespace and qos pods + Steps: + - Check status of the pod + - Check the qos-class type is as expected + Teardown: + - Delete all pods in the namespace + - Delete the namespace + + """ + ns = copy_pod_yamls + kube_helper.wait_for_pods_status(pod_names=pod, namespace=ns) + _, out = kube_helper.exec_kube_cmd( + sub_cmd="get pod {} --namespace={} --output=json".format(pod, ns)) + out = json.loads(out) + LOG.tc_step("pod qos class is {} and expected is {}".format( + out["status"]["qosClass"], expected)) + assert out["status"]["qosClass"].lower() == expected diff --git a/automated-pytest-suite/testcases/functional/z_containers/test_scale_pods.py b/automated-pytest-suite/testcases/functional/z_containers/test_scale_pods.py new file mode 100644 index 0000000..109f0d8 --- /dev/null +++ b/automated-pytest-suite/testcases/functional/z_containers/test_scale_pods.py @@ -0,0 +1,69 @@ +""" +Xstudio testcase reference http://162.248.221.232:8080/nice/xstudio/xstudio.jsp?id=T_18189 +""" +from pytest import fixture +from pytest import skip + +from utils.tis_log import LOG +from keywords import common +from keywords import kube_helper +from keywords import system_helper + +from consts.auth import HostLinuxUser +from consts.stx import PodStatus + + +@fixture(scope='module') +def get_yaml(): + filename = "rc_deployment.yaml" + ns = "rc" + number_nodes = 98 + relicas = number_nodes*len(system_helper.get_hypervisors()) + source_path = "utils/test_files/{}".format(filename) + home_dir = HostLinuxUser.get_home() + common.scp_from_localhost_to_active_controller( + source_path, dest_path=home_dir) + yield ns, relicas, filename + LOG.fixture_step("Delete the deployment") + kube_helper.exec_kube_cmd( + "delete deployment --namespace={} resource-consumer".format(ns)) + LOG.fixture_step("Check pods are terminating") + kube_helper.wait_for_pods_status( + namespace=ns, status=PodStatus.TERMINATING) + LOG.fixture_step("Wait for all pods are deleted") + kube_helper.wait_for_resources_gone(namespace=ns) + LOG.fixture_step("Delete the service and namespace") + kube_helper.exec_kube_cmd( + "delete service rc-service --namespace={}".format(ns)) + kube_helper.exec_kube_cmd("delete namespace {}".format(ns)) + + +def test_scale_pods(get_yaml): + """ + Testing the deployment of high number of pods + Args: + get_yaml : module fixture + Setup: + - Scp deployment file + Steps: + - Check the deployment of resource-consumer + - Check the pods up + - Scale to 99* number of worker nodes + - Check all the pods are running + Teardown: + - Delete the deployment and service + """ + ns, replicas, filename = get_yaml + LOG.tc_step("Create the deployment") + kube_helper.exec_kube_cmd( + sub_cmd="create -f {}".format(filename)) + LOG.tc_step("Check resource consumer pods are running") + state, _ = kube_helper.wait_for_pods_status(namespace=ns, timeout=180) + if state: + LOG.tc_step( + "Scale the resource consumer app to {}* no of worker nodes".format(replicas)) + kube_helper.exec_kube_cmd( + "scale deployment --namespace={} resource-consumer --replicas={}".format(ns, replicas)) + kube_helper.wait_for_pods_status(namespace=ns, timeout=180) + else: + skip("resource consumer deployment failed") diff --git a/automated-pytest-suite/testfixtures/pre_checks_and_configs.py b/automated-pytest-suite/testfixtures/pre_checks_and_configs.py index 7ec7834..ccbd7bf 100755 --- a/automated-pytest-suite/testfixtures/pre_checks_and_configs.py +++ b/automated-pytest-suite/testfixtures/pre_checks_and_configs.py @@ -18,7 +18,7 @@ from utils.tis_log import LOG @fixture(scope='function') -def stx_openstack_required(request): +def stx_openstack_applied_required(request): app_name = 'stx-openstack' if not container_helper.is_stx_openstack_deployed(applied_only=True): skip('stx-openstack application is not applied') @@ -41,6 +41,12 @@ def stx_openstack_required(request): request.addfinalizer(wait_for_recover) +@fixture(scope='module') +def stx_openstack_required(): + if not container_helper.is_stx_openstack_deployed(): + skip('stx-openstack application is not deployed') + + @fixture(scope='session') def skip_for_one_proc(): hypervisor = host_helper.get_up_hypervisors() diff --git a/automated-pytest-suite/utils/test_files/hugepages_pod.yaml b/automated-pytest-suite/utils/test_files/hugepages_pod.yaml new file mode 100644 index 0000000..7196f51 --- /dev/null +++ b/automated-pytest-suite/utils/test_files/hugepages_pod.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + name: hugepages-pod +spec: + containers: + - name: example + image: fedora:latest + command: + - sleep + - inf + volumeMounts: + - mountPath: /hugepages + name: hugepage + resources: + limits: + memory: 100Mi + requests: + memory: 100Mi + volumes: + - name: hugepage + emptyDir: + medium: HugePages \ No newline at end of file diff --git a/automated-pytest-suite/utils/test_files/qos_deployment.yaml b/automated-pytest-suite/utils/test_files/qos_deployment.yaml new file mode 100644 index 0000000..d9a1311 --- /dev/null +++ b/automated-pytest-suite/utils/test_files/qos_deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: qos +--- +apiVersion: v1 +kind: Pod +metadata: + name: qos-pod-1 + namespace: qos +spec: + containers: + - name: qos-demo-ctr + image: nginx + resources: + limits: + memory: "200Mi" + cpu: "700m" + requests: + memory: "200Mi" + cpu: "700m" +--- +apiVersion: v1 +kind: Pod +metadata: + name: qos-pod-2 + namespace: qos +spec: + containers: + - name: qos-demo-2-ctr + image: nginx + resources: + limits: + memory: "200Mi" + requests: + memory: "100Mi" +--- +apiVersion: v1 +kind: Pod +metadata: + name: qos-pod-3 + namespace: qos +spec: + containers: + - name: qos-demo-3-ctr + image: nginx +--- +apiVersion: v1 +kind: Pod +metadata: + name: qos-pod-with-two-containers + namespace: qos +spec: + containers: + - name: qos-demo-4-ctr-1 + image: nginx + resources: + requests: + memory: "200Mi" + + - name: qos-demo-4-ctr-2 + image: redis diff --git a/automated-pytest-suite/utils/test_files/rc_deployment.yaml b/automated-pytest-suite/utils/test_files/rc_deployment.yaml new file mode 100644 index 0000000..813b4e7 --- /dev/null +++ b/automated-pytest-suite/utils/test_files/rc_deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: rc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: resource-consumer + namespace: rc +spec: + replicas: 1 + selector: + matchLabels: + app: resource-consumer + template: + metadata: + labels: + app: resource-consumer + spec: + containers: + - name: resource-consumer + image: "gcr.io/kubernetes-e2e-test-images/resource-consumer:1.4" + ports: + - containerPort: 80 + resources: + requests: + cpu: 100m + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: rc-service + namespace: rc +spec: + selector: + app: resource-consumer + ports: + - protocol: TCP + port: 80 + targetPort: 9376 + type: LoadBalancer