From 8da6f670572e5e17655514a95a8733d262088090 Mon Sep 17 00:00:00 2001 From: Jose Perez Carranza Date: Mon, 12 Aug 2019 11:23:09 -0500 Subject: [PATCH] [robot] Add documentation of the robot suite Add a documentation and license files for the test suite references as well as some tox setup files in case are needed to apply linters on the suite. Depends-On : Ia1e4e9ce011e1e527bd1109d1260bdc0b9a410b9 Signed-off-by: Jose Perez Carranza Change-Id: If81c702688a0cb3daed5c2cbe3181eb3fc6039d8 --- automated-robot-suite/LICENSE | 2 + automated-robot-suite/LICENSE.apache | 202 +++++++ .../docs/HowToWriteGoodTestCases.rst | 548 ++++++++++++++++++ .../docs/RobotFrameStructure.md | 251 ++++++++ automated-robot-suite/setup.cfg | 7 + automated-robot-suite/tox.ini | 53 ++ 6 files changed, 1063 insertions(+) create mode 100644 automated-robot-suite/LICENSE create mode 100644 automated-robot-suite/LICENSE.apache create mode 100644 automated-robot-suite/docs/HowToWriteGoodTestCases.rst create mode 100644 automated-robot-suite/docs/RobotFrameStructure.md create mode 100644 automated-robot-suite/setup.cfg create mode 100644 automated-robot-suite/tox.ini diff --git a/automated-robot-suite/LICENSE b/automated-robot-suite/LICENSE new file mode 100644 index 0000000..e37a957 --- /dev/null +++ b/automated-robot-suite/LICENSE @@ -0,0 +1,2 @@ +All source code id licensed under Apache 2.0 license unless otherwise stated +on the source file. See LICENSE.apache for further details. diff --git a/automated-robot-suite/LICENSE.apache b/automated-robot-suite/LICENSE.apache new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/automated-robot-suite/LICENSE.apache @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/automated-robot-suite/docs/HowToWriteGoodTestCases.rst b/automated-robot-suite/docs/HowToWriteGoodTestCases.rst new file mode 100644 index 0000000..eca0308 --- /dev/null +++ b/automated-robot-suite/docs/HowToWriteGoodTestCases.rst @@ -0,0 +1,548 @@ +.. default-role:: code + +====================================================================== +How to write good test cases using Robot Framework for stx-test-suite +====================================================================== + +.. contents:: Table of contents: + :local: + :depth: 2 + + +Introduction +============ + +- These are high-level guidelines for writing good test cases using Robot + Framework. It is based on the official `how to`__ file + for Robot Framework. + +__ https://github.com/robotframework/HowToWriteGoodTestCases/blob/master/ + HowToWriteGoodTestCases.rst + + - How to actually interact with the system under test is out of + the scope of this document. + +- Most important guideline is keeping test cases as easy to understand as + possible for people familiar with the domain. + + - This typically also eases maintenance. + +Naming +====== + +Test suite names +---------------- + +- Suite names should be as descriptive as possible. + +- Names are created automatically from file or directory names: + + - Extensions are stripped. + - Underscores are converted to spaces. + - If name is all lower case, words are capitalized. + +- Names can be relatively long, but overly long names are not convenient for + the file system. + +- The name of the top level suite can be overridden from the command line + using the `--name` option if needed. + +Examples: + +- `login_tests.robot` -> `Login Tests` +- `IP_v4_and_v6` -> `IP v4 and v6` + + +Test case names +--------------- + +- Test names should be descriptive like the suite names. + +- If a suite contains many similar tests and is well named, + test names can be shorter. + +- Name is exactly the same as you specified in the test case file without any + conversion. + +For example, if we have tests related to invalid login in a file +`invalid_login.robot`, these would be OK test case names: + +.. code:: robotframework + + *** Test Cases *** + Empty Password + Empty Username + Empty Username And Password + Invalid Username + Invalid Password + Invalid Username And Password + +These names would be somewhat long: + +.. code:: robotframework + + *** Test Cases *** + Login With Empty Password Should Fail + Login With Empty Username Should Fail + Login With Empty Username And Password Should Fail + Login With Invalid Username Should Fail + Login With Invalid Password Should Fail + Login With Invalid Username And Invalid Password Should Fail + + +Keyword names +------------- + +- Keyword names should be descriptive and clear. + +- Should explain what the keyword does, not how it does its task(s). + +- Very different abstraction levels (e.g. `Input Text` or `Administrator + logs into system`). + +- There is no clear guideline on whether a keyword should be fully title cased + or have only the first letter be capitalized. + + - Title casing is often used when the keyword name is short + (e.g. `Input Text`). + - Capitalizing just the first letter typically works better with keywords + that are like sentences (e.g. `Administrator logs into system`). These + type of keywords are often higher level. + +Good: + +.. code:: robotframework + + *** Keywords *** + Login With Valid Credentials + +Bad: + +.. code:: robotframework + + *** Keywords *** + Input Valid Username And Valid Password And Click Login Button + + +Naming setup and teardown +------------------------- + +- Try to use name that describes what is done. + + - Possibly use an existing keyword. + +- More abstract names are acceptable if a setup or teardown contains unrelated + steps. + + - Listing steps in name is duplication and a maintenance problem + (e.g. `Login to system, add user, activate alarms and check balance`). + + - Often better to use something generic (e.g. `Initialize system`). + +- BuiltIn keyword `Run Keywords`__ can work well if keywords implementing lower + level steps already exist. + + - Not reusable so best used when the setup or teardown scenario is + needed only once. + +- Everyone working with these tests should always understand what a setup or + teardown does. + +Good: + +.. code:: robotframework + + *** Settings *** + Suite Setup Initialize System + +Good (if only used once): + +.. code:: robotframework + + *** Settings *** + Suite Setup Run Keywords + ... Login To System AND + ... Add User AND + ... Activate Alarms AND + ... Check Balance + +Bad: + +.. code:: robotframework + + *** Settings *** + Suite Setup Login To System, Add User, Activate Alarms And Check Balance + +__ http://robotframework.org/robotframework/latest/libraries/ + BuiltIn.html#Run%20Keywords + + +Documentation +============= + +Test suite documentation +------------------------ + +- Add overall documentation to test case files is mandatory. + +- Should contain background information, why tests are created, notes about + execution environment, etc. + +- Do not just repeat test suite name. + +- Do not include too much details about test cases. + + - Tests should be clear enough to understand alone. + - Duplicate information is waste and maintenance problem. + +- Documentation can contain links to more information. + +- Consider using `test suite metadata`__ if you need to document information + represented as name-value pairs (e.g. `Version: 1.0` or `OS: Linux`). + +__ http://robotframework.org/robotframework/latest/ + RobotFrameworkUserGuide.html#free-test-suite-metadata + +- Documentation and metadata of the top level suite can be set from the + command line using `--doc` and `--metadata` options, respectively. + +Good: + +.. code:: robotframework + + *** Settings *** + Documentation Tests to verify that account withdrawals succeed and + ... fail correctly depending from users account balance + ... and account type dependent rules. + ... See http://internal.example.com/docs/abs.pdf + Metadata Version 0.1 + +Bad (especially if suite is named well like `account_withdrawal.robot`): + +.. code:: robotframework + + *** Settings *** + Documentation Tests Account Withdrawal. + + +Test case documentation +----------------------- + +- Test should have documentation. Only repeating the name of the test is not + valid. + + - Test case structure should be clear enough without documentation or other + comments. + +- Tags are generally more flexible and provide more functionality (statistics, + include/exclude, etc.) than documentation. Tags for type of configuration + are mandatory. + +.. code:: robotframework + + [Tags] Simplex Duplex MN-Local MN-External + +Good: + +.. code:: robotframework + + *** Test Cases *** + Create Flavors for Cirros Instances + [Tags] Simplex Duplex MN-Local MN-External + [Documentation] Create flavors with or without properties to be used + ... to launch Cirros instances. + ${properties}= Catenate ${flavor_property_1} ${flavor_property_2} + Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk} + ... ${flavor_name_1} + Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk} + ... ${properties} ${flavor_name_2} + +Bad: + +.. code:: robotframework + + *** Test Cases *** + Create Flavors for Cirros Instances + [Documentation] Create flavors giving a ram, vcpus and disk with or + ... without properties o be used to launch Cirros instances on Simplex, + ... Duplex or Multinode configuration for SaTarlingX deployment. + ... All is done calling a keyword called Create Flavor with some + ... parameters ... etc + [Tags] Simplex Duplex MN-Local MN-External + ${properties}= Catenate ${flavor_property_1} ${flavor_property_2} + Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk} + ... ${flavor_name_1} + Create Flavor ${flavor_ram} ${flavor_vcpus} ${flavor_disk} + ... ${properties} ${flavor_name_2} + + +User keyword documentation +-------------------------- + +- Its important document what keyword is doing. + +- Important usage is documenting arguments and return values. + +- Shown in resource file documentation generated with Libdoc__ and editors + such as RIDE__ can show it in keyword completion and elsewhere. + +__ http://robotframework.org/robotframework/#built-in-tools +__ https://github.com/robotframework/RIDE + + +Test suite structure +==================== +- All test should be placed under Test/X directory where X indicates the Domain + where Test Case belongs + +- Tests in a suite should be related to each other. + + - Common setup and/or teardown is often a good indicator. + +- Should not have too many tests (max 10) in one file unless they are + `data-driven tests`_. + +- Tests should be independent. Initialization using setup/teardown. + +- Sometimes dependencies between tests cannot be avoided. + + - For example, it can take too much time to initialize all tests separately. + - Never have long chains of dependent tests. + - Consider verifying the status of the previous test using the built-in + `${PREV TEST STATUS}` variable. + + - We try to avoid the usage of keywords inside of a Test Suite unless they are + very specific to the test cases, if a keyword is designed and is generic, + this jeyword should be placed under Resources/ directory. + +.. code:: robotframework + + *** Keywords *** + Run Command + [Arguments] ${cmd} ${fail_if_error}=False ${timeout}=${TIMEOUT} + [Documentation] Execute a command on controller over ssh connection + ... keeping environment visible to the subsequent keywords.Also allows + ... the keyword to fail if there is an error, by default this + ... keyword will not fail and will return the stderr. + + +Test case structure +=================== + +- Test case should be easy to understand. + +- Test case should be tagged and documented + +- One test case should be testing one thing. + + - *Things* can be small (part of a single feature) or large (end-to-end). + +- Select suitable abstraction level. + + - Use abstraction level consistently (single level of abstraction principle). + - Do not include unnecessary details on the test case level. + +- We are using an strategy of `Workflow tests`_ for our development, please + follow the rules + +- Try to not exceed 79 characters when possible, if it nos possible or the + test case get not esasy to read 99 chars are accepted. + +Workflow tests +-------------- + +- Generally have these phases: + + - Precondition (optional, often in setup) + - Action (do something to the system) + - Verification (validate results, mandatory) + - Cleanup (optional, always in teardown to make sure it is executed) + +- Keywords describe what a test does. + + - Use clear keyword names and suitable abstraction level. + - Should contain enough information to run manually. + - Should never need documentation or commenting to explain what the + test does. + +- Different tests can have different abstraction levels. + + - Tests for a detailed functionality are more precise. + - End-to-end tests can be on very high level. + - One test should use only one abstraction level + +- Different styles: + + - More technical tests for lower level details and integration tests. + - "Executable specifications" act as requirements. + - Use domain language. + - Everyone (including customer and/or product owner) should + always understand. + +- Try to not use complex logic on the test case level. + + - No for loops or if/else constructs as much as possible. + - Use variable assignments with care. + - Test cases should not look like scripts! + +- Max 15 steps, preferably less. + +Example using "normal" keyword-driven style: + +.. code:: robotframework + + *** Test Cases *** + Valid Login + Open Browser To Login Page + Input Username demo + Input Password mode + Submit Credentials + Welcome Page Should Be Open + +User keywords +============= + +- Should be easy to understand. + + - Same rules as with workflow tests. + +- Different abstraction levels. + +- Can contain some programming logic (for loops, if/else). + + - Especially on lower level keywords. + - Complex logic in libraries rather than in user keywords. + +- Try to not exceed 79 characters when possible, if it nos possible or the + keyword becomes not esasy to read 99 chars are accepted. + + +Variables +========= + +- Encapsulate long and/or complicated values. + +- Pass information between keywords. + + +Variable naming +--------------- + +- Clear but not too long names. + +- Use case consistently: + + - Lower case with local variables only available inside a certain scope. + - Upper case with others (global, suite or test level). + - Both space and underscore can be used as a word separator. + +- Recommended to also list variables that are set dynamically in the variable + table. + + - Set typically using BuiltIn keyword `Set Suite Variable`__. + - The initial value should explain where/how the real value is set. + +Example: + +.. code:: robotframework + + *** Settings *** + Suite Setup Set Active User + + *** Variables *** + # Default system address. Override when tested agains other instances. + ${SERVER URL} http://sre-12.example.com/ + ${USER} Actual value set dynamically at suite setup + + *** Keywords *** + Set Active User + ${USER} = Get Current User ${SERVER URL} + Set Suite Variable ${USER} + +__ http://robotframework.org/robotframework/latest/libraries/ + BuiltIn.html#Set%20Suite%20Variable + + +Passing and returning values +---------------------------- + +- Common approach is to return values from keywords, assign them to variables + and then pass them as arguments to other keywords. + + - Clear and easy to follow approach. + - Allows creating independent keywords and facilitates re-use. + - Looks like programming and thus not so good on the test case level. + +- Alternative approach is storing information in a library or using the BuiltIn + `Set Test Variable`__ keyword. + + - Avoid programming style on the test case level as much as possible. + - Can be more complex to follow and make reusing keywords harder. + +__ http://robotframework.org/robotframework/latest/libraries/ + BuiltIn.html#Set%20Test%20Variable + +Good for in suite keywords: + +.. code:: robotframework + + *** Test Cases *** + Withdraw From Account + Withdraw From Account $50 + Withdraw Should Have Succeeded + + *** Keywords *** + Withdraw From Account + [Arguments] ${amount} + ${STATUS} = Withdraw From User Account ${USER} ${amount} + Set Test Variable ${STATUS} + + Withdraw Should Have Succeeded + Should Be Equal ${STATUS} SUCCESS + +Good for generic keywords: + +.. code:: robotframework + + *** Test Cases *** + Withdraw From Account + ${status} = Withdraw From Account $50 + Withdraw Should Have Succeeded ${status} + + *** Keywords *** + Withdraw From Account + [Arguments] ${amount} + ${status} = Withdraw From User Account ${USER} ${amount} + [Return] ${status} + + Withdraw Should Have Succeeded + [Arguments] ${status} + Should Be Equal ${status} SUCCESS + + +Avoid sleeping +============== + +- Sleeping is a very fragile way to synchronize tests. + +- Safety margins cause too long sleeps on average. + +- Instead of sleeps, use keyword that polls has a certain action occurred. + + - Keyword names often starts with `Wait ...`. + - Should have a maximum time to wait. + - Possible to wrap other keywords inside the BuiltIn keyword + `Wait Until Keyword Succeeds`__. + +- Sometimes sleeping is the easiest solution. + + - Always use with care. + - Never use in user keywords that are used often by tests or other keywords. + +- Can be useful in debugging to stop execution. + + - `Dialogs library`__ often works better. + +__ http://robotframework.org/robotframework/latest/libraries/ + BuiltIn.html#Wait%20Until%20Keyword%20Succeeds +__ http://robotframework.org/robotframework/latest/libraries/Dialogs.html diff --git a/automated-robot-suite/docs/RobotFrameStructure.md b/automated-robot-suite/docs/RobotFrameStructure.md new file mode 100644 index 0000000..687301f --- /dev/null +++ b/automated-robot-suite/docs/RobotFrameStructure.md @@ -0,0 +1,251 @@ + +# Edge Compute Framework Structure documentation + +Table of Contents + +- [Introduction](#introduction) +- [Edge Compute Folder Structure](#edge-compute-folder-structure) + - [1. Libraries](#1-libraries) + - [2. Resources](#2-resources) + - [3. Variables](#3-variables) + - [4. Utils](#4-utils) + - [5. Tests](#5-test) + - [5.1 Horizon](#51-horizon) + - [5.2 Keystone](#52-keystone) + - [5.3 Nova](#53-nova) + - [5.4 Glance](#54-glance) + - [5.5 Swift](#55-swift) + - [5.6 Neutron](#56-cinder) + - [5.7 Cinder](#57-neutron) + - [5.8 Heat](#58-heat) + - [5.9 Ceilometer](#59-ceilometer) + - [6. Results](#6-results) + - [7. Latest-result](#7-latest-result) + + +## Introduction +This structure tries to fit Robot Framework User Guide good practices.
+For more detailed information please check the following :link: +[Link](http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#documentation-formatting) + +## Edge Compute Folder Structure + +```text +stx-test-suite/ +|--Libraries +|--Resources + |--MyResources file +|--Variables + |--global.py file +|--Utils +|--Tests + |--Horizon component + |--Tests.robot file + + *** Settings *** + [Documentation] + [Tags] + imports... + Resources + Test Setup + Test Teardown + + *** Test Cases + ... + |--__init__.robot file + + ***Settings*** + imports + Suite Setup + Suite Teardown + + ***keywords*** + ... + |--Keystone + |--Nova + |--Glance + |--Swift + |--Neutron + |--Cinder + |--Heat + |--Ceilometer +|--Results + |--YYYYMMDDHHMMSS +|--Latest-results +``` + +### 1. Libraries +Any test library that is **NOT** one of the standard libraries would be here. + +### 2. Resources +Resource files contain user keywords used in test case files and test suite +initialization files providing a mechanism for sharing them.
+Since the resource file structure is very close to test case files, +it is easy to create them. + +Resource files are imported using the _Resource setting_ in the settings table. +The path to the resource file is given in the cell after the setting name. + +**e.g.**
+TEST CASE TEMPLATE
+```text +*** Settings *** +Resource my_resources.html +Resource ../data/resources.html +Resource ${RESOURCES}/common.tsv + +*** Keywords *** +... +``` + +### 3. Variables +This folder should contain scripts declaring global variables, environmental +variables, or any special variable used in the Edge Computing framework. + +**e.g.**
+ + - global.py +```text +CONTROLLER_IP='10.10.10.10' +CONTROLLER_USERNAME='root' +CONTROLLER_PASSWORD='linux' +``` + + - virtual_machine_variables.py +```text +- MEMDisk1=200GB +- MEMDisk2=100GB +``` + +### 4. Utils +This folder should contain generic functions that perform actions across the +Edge Computing execution. + +**e.g.** + + - common_reports.py +```text +- check_results_dir(...) +- create_output_dir(...) +- link_latest_run(...) +``` + +### 5. Test +Contains the Robot Automated Test scenarios per Edge Computing Component. + +#### 5.1 Horizon +Horizon folder should contain test cases focused on exercise the Horizon GUI +where every tenant or application user can launch up/manage their own instances. + +**COMPONENT FOLDER STRUCTURE** + +#### Tests.robot file +```text +*** Settings *** +[Documentation] tag specifying brief description of the Test case. +[Tags] keywords specifying what is being tested. + +Imports... +# Import Order Template + +# {{stdlib imports in human alphabetical order}} +# {{third-party lib imports in human alphabetical order}} +# {{project imports in human alphabetical order}} + +# Import Order Example +import httplib +import logging +import random + +import eventlet +import webob.exc + +import nova.api.ec2 +from nova.api import manager +from nova.api import openstack + +Resources... +Test Setup +Test Teardown + +*** Test Cases *** +... + +``` + +#### __init__.robot file +The **__ init__ file** is a special test suite initialization file.
+It is a test suite **created from a directory** that can have the same +structure and syntax as test case files, except that **they cannot have test +case tables** and **not** all settings are supported. + +```text +*** Settings *** + Imports ... + Suite Setup + Suite Teardown + +*** Keywords *** +``` + +#### 5.2 Keystone +Keystone folder should contain test cases focused on exercise the authorization +or authentication related to every layer of services in OpenStack.
+ +**e.g.** Tokens, Catalog, Policy, and Assignment Service role. + + +#### 5.3 Nova +Nova folder should contain test cases focused on exercise compute domain. + +**e.g.** When you launch a instance is where all the computing and all the +processing actually happens. + +#### 5.4 Glance +Glance folder should contain test cases focused on exercise services for +launching the instances. Glance has all the disk images contained in them +different versions of OS such as Linux, CentOS, Ubuntu, Windows. + + +#### 5.5 Swift +Swift folder should contain test cases focused on exercise databases as a +Object Storage Services where you can store all kind of files `as objects`. + +**REMARK:** Everything is converted to an object and saved in the Storage File +System. + + +#### 5.6 Cinder +Cinder folder should contain test cases focused on exercise databases as a +`Block storage` services where the database is seen like a pluggable storage +which is very similar to the disk storage system in a computer. + + +#### 5.7 Neutron +Neutron folder should contain test cases focused on exercise the `networking` +service associated with all OpenStack Services. + +#### 5.8 Heat +Heat folder should contain test cases focused on exercise Orchestration. + +#### 5.9 Ceilometer +Ceilometer folder should contain tests cases focused on exercise metering and +billing. It checks availability of the services for how much time is an instance +active or running. + +### 6. Results +Test execution results are keep in folders following the next format when the +execution was ran: +```text +%Y%m%d%H%M%S_ +Where + %Y -- Year + %m -- Month + %d -- Day + %H%M%S -- Hour Minute Second +``` + +### 7. Latest-result +
... + +Hope this can help you good look :+1: \ No newline at end of file diff --git a/automated-robot-suite/setup.cfg b/automated-robot-suite/setup.cfg new file mode 100644 index 0000000..3539cbc --- /dev/null +++ b/automated-robot-suite/setup.cfg @@ -0,0 +1,7 @@ +# configuration for flake8 +[flake8] +ignore = W191 +exclude = .venv,.tox,dist,doc,build,*.egg +import-order-style = pep8 +max-line-length = 79 +max-complexity = 15 diff --git a/automated-robot-suite/tox.ini b/automated-robot-suite/tox.ini new file mode 100644 index 0000000..e4d807a --- /dev/null +++ b/automated-robot-suite/tox.ini @@ -0,0 +1,53 @@ +# tox configuration +[tox] +skipsdist = True +envlist = flake8, py27, py36 + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONPATH={toxinidir} +# passed to 'pip install --prie', that will install the dependencies +# listed in those files +deps = -r{toxinidir}/test-requirements.txt + +# => Linters +# ========== + +# settings specific to the flake8 environment +[testenv:flake8] +basepython = python3 +skip_install = True +# The command to run: +commands = flake8 --statistics --count --hang-closing --max-line-length=79 --show-source --import-order-style=pep8 {posargs} +# we only need flake8 and hacking when linting, + +[testenv:venv] +# let you pass additional arguments when invoking tox +commands = {posargs} + +[testenv:py27] +commands = python -m unittest {posargs:discover -vs .} + +[testenv:coverage] +commands = coverage erase + coverage run --source=stx-test-suite -m unittest {posargs:discover -vs .} + coverage html + coverage report --fail-under=80 + +[testenv:pylint] +commands = pylint {posargs} +deps = -r{toxinidir}/test-requirements.txt +#-r{toxinidir}/requirements.txt + +[flake8] +exclude = .git,__pycache__,old,build,dist +max-complexity = 15 +count = True +statistics = True +hang-closing = True +max-line-length = 79 +show-source = True +import-order-style = pep8 +verbose = 1 +jobs = 2 +show-pep8 = True